1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
package getter
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"strings"
"cloud.google.com/go/storage"
"google.golang.org/api/iterator"
)
// GCSGetter is a Getter implementation that will download a module from
// a GCS bucket.
type GCSGetter struct {
getter
}
func (g *GCSGetter) ClientMode(u *url.URL) (ClientMode, error) {
ctx := g.Context()
// Parse URL
bucket, object, err := g.parseURL(u)
if err != nil {
return 0, err
}
client, err := storage.NewClient(ctx)
if err != nil {
return 0, err
}
iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object})
for {
obj, err := iter.Next()
if err != nil && err != iterator.Done {
return 0, err
}
if err == iterator.Done {
break
}
if strings.HasSuffix(obj.Name, "/") {
// A directory matched the prefix search, so this must be a directory
return ClientModeDir, nil
} else if obj.Name != object {
// A file matched the prefix search and doesn't have the same name
// as the query, so this must be a directory
return ClientModeDir, nil
}
}
// There are no directories or subdirectories, and if a match was returned,
// it was exactly equal to the prefix search. So return File mode
return ClientModeFile, nil
}
func (g *GCSGetter) Get(dst string, u *url.URL) error {
ctx := g.Context()
// Parse URL
bucket, object, err := g.parseURL(u)
if err != nil {
return err
}
// Remove destination if it already exists
_, err = os.Stat(dst)
if err != nil && !os.IsNotExist(err) {
return err
}
if err == nil {
// Remove the destination
if err := os.RemoveAll(dst); err != nil {
return err
}
}
// Create all the parent directories
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
client, err := storage.NewClient(ctx)
if err != nil {
return err
}
// Iterate through all matching objects.
iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object})
for {
obj, err := iter.Next()
if err != nil && err != iterator.Done {
return err
}
if err == iterator.Done {
break
}
if !strings.HasSuffix(obj.Name, "/") {
// Get the object destination path
objDst, err := filepath.Rel(object, obj.Name)
if err != nil {
return err
}
objDst = filepath.Join(dst, objDst)
// Download the matching object.
err = g.getObject(ctx, client, objDst, bucket, obj.Name)
if err != nil {
return err
}
}
}
return nil
}
func (g *GCSGetter) GetFile(dst string, u *url.URL) error {
ctx := g.Context()
// Parse URL
bucket, object, err := g.parseURL(u)
if err != nil {
return err
}
client, err := storage.NewClient(ctx)
if err != nil {
return err
}
return g.getObject(ctx, client, dst, bucket, object)
}
func (g *GCSGetter) getObject(ctx context.Context, client *storage.Client, dst, bucket, object string) error {
rc, err := client.Bucket(bucket).Object(object).NewReader(ctx)
if err != nil {
return err
}
defer rc.Close()
// Create all the parent directories
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
f, err := os.Create(dst)
if err != nil {
return err
}
defer f.Close()
_, err = Copy(ctx, f, rc)
return err
}
func (g *GCSGetter) parseURL(u *url.URL) (bucket, path string, err error) {
if strings.Contains(u.Host, "googleapis.com") {
hostParts := strings.Split(u.Host, ".")
if len(hostParts) != 3 {
err = fmt.Errorf("URL is not a valid GCS URL")
return
}
pathParts := strings.SplitN(u.Path, "/", 5)
if len(pathParts) != 5 {
err = fmt.Errorf("URL is not a valid GCS URL")
return
}
bucket = pathParts[3]
path = pathParts[4]
}
return
}
|
