package getter import ( "context" "fmt" "net/url" "os" "path/filepath" "strings" "cloud.google.com/go/storage" "google.golang.org/api/iterator" ) // GCSGetter is a Getter implementation that will download a module from // a GCS bucket. type GCSGetter struct { getter } func (g *GCSGetter) ClientMode(u *url.URL) (ClientMode, error) { ctx := g.Context() // Parse URL bucket, object, err := g.parseURL(u) if err != nil { return 0, err } client, err := storage.NewClient(ctx) if err != nil { return 0, err } iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object}) for { obj, err := iter.Next() if err != nil && err != iterator.Done { return 0, err } if err == iterator.Done { break } if strings.HasSuffix(obj.Name, "/") { // A directory matched the prefix search, so this must be a directory return ClientModeDir, nil } else if obj.Name != object { // A file matched the prefix search and doesn't have the same name // as the query, so this must be a directory return ClientModeDir, nil } } // There are no directories or subdirectories, and if a match was returned, // it was exactly equal to the prefix search. So return File mode return ClientModeFile, nil } func (g *GCSGetter) Get(dst string, u *url.URL) error { ctx := g.Context() // Parse URL bucket, object, err := g.parseURL(u) if err != nil { return err } // Remove destination if it already exists _, err = os.Stat(dst) if err != nil && !os.IsNotExist(err) { return err } if err == nil { // Remove the destination if err := os.RemoveAll(dst); err != nil { return err } } // Create all the parent directories if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err } client, err := storage.NewClient(ctx) if err != nil { return err } // Iterate through all matching objects. iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object}) for { obj, err := iter.Next() if err != nil && err != iterator.Done { return err } if err == iterator.Done { break } if !strings.HasSuffix(obj.Name, "/") { // Get the object destination path objDst, err := filepath.Rel(object, obj.Name) if err != nil { return err } objDst = filepath.Join(dst, objDst) // Download the matching object. err = g.getObject(ctx, client, objDst, bucket, obj.Name) if err != nil { return err } } } return nil } func (g *GCSGetter) GetFile(dst string, u *url.URL) error { ctx := g.Context() // Parse URL bucket, object, err := g.parseURL(u) if err != nil { return err } client, err := storage.NewClient(ctx) if err != nil { return err } return g.getObject(ctx, client, dst, bucket, object) } func (g *GCSGetter) getObject(ctx context.Context, client *storage.Client, dst, bucket, object string) error { rc, err := client.Bucket(bucket).Object(object).NewReader(ctx) if err != nil { return err } defer rc.Close() // Create all the parent directories if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err } f, err := os.Create(dst) if err != nil { return err } defer f.Close() _, err = Copy(ctx, f, rc) return err } func (g *GCSGetter) parseURL(u *url.URL) (bucket, path string, err error) { if strings.Contains(u.Host, "googleapis.com") { hostParts := strings.Split(u.Host, ".") if len(hostParts) != 3 { err = fmt.Errorf("URL is not a valid GCS URL") return } pathParts := strings.SplitN(u.Path, "/", 5) if len(pathParts) != 5 { err = fmt.Errorf("URL is not a valid GCS URL") return } bucket = pathParts[3] path = pathParts[4] } return }