]>
Commit | Line | Data |
---|---|---|
bae9f6d2 JC |
1 | package getter |
2 | ||
3 | import ( | |
107c1cdb | 4 | "context" |
bae9f6d2 JC |
5 | "encoding/xml" |
6 | "fmt" | |
7 | "io" | |
bae9f6d2 JC |
8 | "net/http" |
9 | "net/url" | |
10 | "os" | |
11 | "path/filepath" | |
107c1cdb | 12 | "strconv" |
bae9f6d2 | 13 | "strings" |
15c0b25d | 14 | |
107c1cdb | 15 | safetemp "github.com/hashicorp/go-safetemp" |
bae9f6d2 JC |
16 | ) |
17 | ||
18 | // HttpGetter is a Getter implementation that will download from an HTTP | |
19 | // endpoint. | |
20 | // | |
21 | // For file downloads, HTTP is used directly. | |
22 | // | |
107c1cdb | 23 | // The protocol for downloading a directory from an HTTP endpoint is as follows: |
bae9f6d2 JC |
24 | // |
25 | // An HTTP GET request is made to the URL with the additional GET parameter | |
26 | // "terraform-get=1". This lets you handle that scenario specially if you | |
27 | // wish. The response must be a 2xx. | |
28 | // | |
29 | // First, a header is looked for "X-Terraform-Get" which should contain | |
30 | // a source URL to download. | |
31 | // | |
32 | // If the header is not present, then a meta tag is searched for named | |
33 | // "terraform-get" and the content should be a source URL. | |
34 | // | |
35 | // The source URL, whether from the header or meta tag, must be a fully | |
36 | // formed URL. The shorthand syntax of "github.com/foo/bar" or relative | |
37 | // paths are not allowed. | |
38 | type HttpGetter struct { | |
107c1cdb ND |
39 | getter |
40 | ||
bae9f6d2 JC |
41 | // Netrc, if true, will lookup and use auth information found |
42 | // in the user's netrc file if available. | |
43 | Netrc bool | |
15c0b25d AP |
44 | |
45 | // Client is the http.Client to use for Get requests. | |
46 | // This defaults to a cleanhttp.DefaultClient if left unset. | |
47 | Client *http.Client | |
107c1cdb ND |
48 | |
49 | // Header contains optional request header fields that should be included | |
50 | // with every HTTP request. Note that the zero value of this field is nil, | |
51 | // and as such it needs to be initialized before use, via something like | |
52 | // make(http.Header). | |
53 | Header http.Header | |
bae9f6d2 JC |
54 | } |
55 | ||
56 | func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) { | |
57 | if strings.HasSuffix(u.Path, "/") { | |
58 | return ClientModeDir, nil | |
59 | } | |
60 | return ClientModeFile, nil | |
61 | } | |
62 | ||
63 | func (g *HttpGetter) Get(dst string, u *url.URL) error { | |
107c1cdb | 64 | ctx := g.Context() |
bae9f6d2 JC |
65 | // Copy the URL so we can modify it |
66 | var newU url.URL = *u | |
67 | u = &newU | |
68 | ||
69 | if g.Netrc { | |
70 | // Add auth from netrc if we can | |
71 | if err := addAuthFromNetrc(u); err != nil { | |
72 | return err | |
73 | } | |
74 | } | |
75 | ||
15c0b25d AP |
76 | if g.Client == nil { |
77 | g.Client = httpClient | |
78 | } | |
79 | ||
bae9f6d2 JC |
80 | // Add terraform-get to the parameter. |
81 | q := u.Query() | |
82 | q.Add("terraform-get", "1") | |
83 | u.RawQuery = q.Encode() | |
84 | ||
85 | // Get the URL | |
107c1cdb ND |
86 | req, err := http.NewRequest("GET", u.String(), nil) |
87 | if err != nil { | |
88 | return err | |
89 | } | |
90 | ||
91 | req.Header = g.Header | |
92 | resp, err := g.Client.Do(req) | |
bae9f6d2 JC |
93 | if err != nil { |
94 | return err | |
95 | } | |
107c1cdb | 96 | |
bae9f6d2 JC |
97 | defer resp.Body.Close() |
98 | if resp.StatusCode < 200 || resp.StatusCode >= 300 { | |
99 | return fmt.Errorf("bad response code: %d", resp.StatusCode) | |
100 | } | |
101 | ||
102 | // Extract the source URL | |
103 | var source string | |
104 | if v := resp.Header.Get("X-Terraform-Get"); v != "" { | |
105 | source = v | |
106 | } else { | |
107 | source, err = g.parseMeta(resp.Body) | |
108 | if err != nil { | |
109 | return err | |
110 | } | |
111 | } | |
112 | if source == "" { | |
113 | return fmt.Errorf("no source URL was returned") | |
114 | } | |
115 | ||
116 | // If there is a subdir component, then we download the root separately | |
117 | // into a temporary directory, then copy over the proper subdir. | |
118 | source, subDir := SourceDirSubdir(source) | |
119 | if subDir == "" { | |
107c1cdb ND |
120 | var opts []ClientOption |
121 | if g.client != nil { | |
122 | opts = g.client.Options | |
123 | } | |
124 | return Get(dst, source, opts...) | |
bae9f6d2 JC |
125 | } |
126 | ||
127 | // We have a subdir, time to jump some hoops | |
107c1cdb | 128 | return g.getSubdir(ctx, dst, source, subDir) |
bae9f6d2 JC |
129 | } |
130 | ||
107c1cdb ND |
131 | func (g *HttpGetter) GetFile(dst string, src *url.URL) error { |
132 | ctx := g.Context() | |
15c0b25d AP |
133 | if g.Netrc { |
134 | // Add auth from netrc if we can | |
107c1cdb | 135 | if err := addAuthFromNetrc(src); err != nil { |
15c0b25d AP |
136 | return err |
137 | } | |
138 | } | |
139 | ||
107c1cdb ND |
140 | // Create all the parent directories if needed |
141 | if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { | |
142 | return err | |
15c0b25d AP |
143 | } |
144 | ||
107c1cdb | 145 | f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, os.FileMode(0666)) |
bae9f6d2 JC |
146 | if err != nil { |
147 | return err | |
148 | } | |
107c1cdb ND |
149 | defer f.Close() |
150 | ||
151 | if g.Client == nil { | |
152 | g.Client = httpClient | |
bae9f6d2 JC |
153 | } |
154 | ||
107c1cdb ND |
155 | var currentFileSize int64 |
156 | ||
157 | // We first make a HEAD request so we can check | |
158 | // if the server supports range queries. If the server/URL doesn't | |
159 | // support HEAD requests, we just fall back to GET. | |
160 | req, err := http.NewRequest("HEAD", src.String(), nil) | |
161 | if err != nil { | |
bae9f6d2 JC |
162 | return err |
163 | } | |
107c1cdb ND |
164 | if g.Header != nil { |
165 | req.Header = g.Header | |
166 | } | |
167 | headResp, err := g.Client.Do(req) | |
168 | if err == nil && headResp != nil { | |
169 | headResp.Body.Close() | |
170 | if headResp.StatusCode == 200 { | |
171 | // If the HEAD request succeeded, then attempt to set the range | |
172 | // query if we can. | |
173 | if headResp.Header.Get("Accept-Ranges") == "bytes" { | |
174 | if fi, err := f.Stat(); err == nil { | |
175 | if _, err = f.Seek(0, os.SEEK_END); err == nil { | |
176 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size())) | |
177 | currentFileSize = fi.Size() | |
178 | totalFileSize, _ := strconv.ParseInt(headResp.Header.Get("Content-Length"), 10, 64) | |
179 | if currentFileSize >= totalFileSize { | |
180 | // file already present | |
181 | return nil | |
182 | } | |
183 | } | |
184 | } | |
185 | } | |
186 | } | |
187 | } | |
188 | req.Method = "GET" | |
bae9f6d2 | 189 | |
107c1cdb | 190 | resp, err := g.Client.Do(req) |
bae9f6d2 JC |
191 | if err != nil { |
192 | return err | |
193 | } | |
107c1cdb ND |
194 | switch resp.StatusCode { |
195 | case http.StatusOK, http.StatusPartialContent: | |
196 | // all good | |
197 | default: | |
198 | resp.Body.Close() | |
199 | return fmt.Errorf("bad response code: %d", resp.StatusCode) | |
200 | } | |
201 | ||
202 | body := resp.Body | |
203 | ||
204 | if g.client != nil && g.client.ProgressListener != nil { | |
205 | // track download | |
206 | fn := filepath.Base(src.EscapedPath()) | |
207 | body = g.client.ProgressListener.TrackProgress(fn, currentFileSize, currentFileSize+resp.ContentLength, resp.Body) | |
208 | } | |
209 | defer body.Close() | |
bae9f6d2 | 210 | |
107c1cdb | 211 | n, err := Copy(ctx, f, body) |
15c0b25d AP |
212 | if err == nil && n < resp.ContentLength { |
213 | err = io.ErrShortWrite | |
214 | } | |
bae9f6d2 JC |
215 | return err |
216 | } | |
217 | ||
218 | // getSubdir downloads the source into the destination, but with | |
219 | // the proper subdir. | |
107c1cdb | 220 | func (g *HttpGetter) getSubdir(ctx context.Context, dst, source, subDir string) error { |
15c0b25d AP |
221 | // Create a temporary directory to store the full source. This has to be |
222 | // a non-existent directory. | |
223 | td, tdcloser, err := safetemp.Dir("", "getter") | |
bae9f6d2 JC |
224 | if err != nil { |
225 | return err | |
226 | } | |
15c0b25d | 227 | defer tdcloser.Close() |
bae9f6d2 | 228 | |
107c1cdb ND |
229 | var opts []ClientOption |
230 | if g.client != nil { | |
231 | opts = g.client.Options | |
232 | } | |
bae9f6d2 | 233 | // Download that into the given directory |
107c1cdb | 234 | if err := Get(td, source, opts...); err != nil { |
bae9f6d2 JC |
235 | return err |
236 | } | |
237 | ||
15c0b25d AP |
238 | // Process any globbing |
239 | sourcePath, err := SubdirGlob(td, subDir) | |
240 | if err != nil { | |
241 | return err | |
242 | } | |
243 | ||
bae9f6d2 | 244 | // Make sure the subdir path actually exists |
bae9f6d2 JC |
245 | if _, err := os.Stat(sourcePath); err != nil { |
246 | return fmt.Errorf( | |
247 | "Error downloading %s: %s", source, err) | |
248 | } | |
249 | ||
250 | // Copy the subdirectory into our actual destination. | |
251 | if err := os.RemoveAll(dst); err != nil { | |
252 | return err | |
253 | } | |
254 | ||
255 | // Make the final destination | |
256 | if err := os.MkdirAll(dst, 0755); err != nil { | |
257 | return err | |
258 | } | |
259 | ||
107c1cdb | 260 | return copyDir(ctx, dst, sourcePath, false) |
bae9f6d2 JC |
261 | } |
262 | ||
263 | // parseMeta looks for the first meta tag in the given reader that | |
264 | // will give us the source URL. | |
265 | func (g *HttpGetter) parseMeta(r io.Reader) (string, error) { | |
266 | d := xml.NewDecoder(r) | |
267 | d.CharsetReader = charsetReader | |
268 | d.Strict = false | |
269 | var err error | |
270 | var t xml.Token | |
271 | for { | |
272 | t, err = d.Token() | |
273 | if err != nil { | |
274 | if err == io.EOF { | |
275 | err = nil | |
276 | } | |
277 | return "", err | |
278 | } | |
279 | if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { | |
280 | return "", nil | |
281 | } | |
282 | if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { | |
283 | return "", nil | |
284 | } | |
285 | e, ok := t.(xml.StartElement) | |
286 | if !ok || !strings.EqualFold(e.Name.Local, "meta") { | |
287 | continue | |
288 | } | |
289 | if attrValue(e.Attr, "name") != "terraform-get" { | |
290 | continue | |
291 | } | |
292 | if f := attrValue(e.Attr, "content"); f != "" { | |
293 | return f, nil | |
294 | } | |
295 | } | |
296 | } | |
297 | ||
298 | // attrValue returns the attribute value for the case-insensitive key | |
299 | // `name', or the empty string if nothing is found. | |
300 | func attrValue(attrs []xml.Attr, name string) string { | |
301 | for _, a := range attrs { | |
302 | if strings.EqualFold(a.Name.Local, name) { | |
303 | return a.Value | |
304 | } | |
305 | } | |
306 | return "" | |
307 | } | |
308 | ||
309 | // charsetReader returns a reader for the given charset. Currently | |
310 | // it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful | |
311 | // error which is printed by go get, so the user can find why the package | |
312 | // wasn't downloaded if the encoding is not supported. Note that, in | |
313 | // order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters | |
314 | // greater than 0x7f are not rejected). | |
315 | func charsetReader(charset string, input io.Reader) (io.Reader, error) { | |
316 | switch strings.ToLower(charset) { | |
317 | case "ascii": | |
318 | return input, nil | |
319 | default: | |
320 | return nil, fmt.Errorf("can't decode XML document using charset %q", charset) | |
321 | } | |
322 | } |