vendor/github.com/hashicorp/go-getter/get_http.go

   1 package getter
   2
   3 import (
   4         "context"
   5         "encoding/xml"
   6         "fmt"
   7         "io"
   8         "net/http"
   9         "net/url"
  10         "os"
  11         "path/filepath"
  12         "strconv"
  13         "strings"
  14
  15         safetemp "github.com/hashicorp/go-safetemp"
  16 )
  17
  18 // HttpGetter is a Getter implementation that will download from an HTTP
  19 // endpoint.
  20 //
  21 // For file downloads, HTTP is used directly.
  22 //
  23 // The protocol for downloading a directory from an HTTP endpoint is as follows:
  24 //
  25 // An HTTP GET request is made to the URL with the additional GET parameter
  26 // "terraform-get=1". This lets you handle that scenario specially if you
  27 // wish. The response must be a 2xx.
  28 //
  29 // First, a header is looked for "X-Terraform-Get" which should contain
  30 // a source URL to download.
  31 //
  32 // If the header is not present, then a meta tag is searched for named
  33 // "terraform-get" and the content should be a source URL.
  34 //
  35 // The source URL, whether from the header or meta tag, must be a fully
  36 // formed URL. The shorthand syntax of "github.com/foo/bar" or relative
  37 // paths are not allowed.
  38 type HttpGetter struct {
  39         getter
  40
  41         // Netrc, if true, will lookup and use auth information found
  42         // in the user's netrc file if available.
  43         Netrc bool
  44
  45         // Client is the http.Client to use for Get requests.
  46         // This defaults to a cleanhttp.DefaultClient if left unset.
  47         Client *http.Client
  48
  49         // Header contains optional request header fields that should be included
  50         // with every HTTP request. Note that the zero value of this field is nil,
  51         // and as such it needs to be initialized before use, via something like
  52         // make(http.Header).
  53         Header http.Header
  54 }
  55
  56 func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) {
  57         if strings.HasSuffix(u.Path, "/") {
  58                 return ClientModeDir, nil
  59         }
  60         return ClientModeFile, nil
  61 }
  62
  63 func (g *HttpGetter) Get(dst string, u *url.URL) error {
  64         ctx := g.Context()
  65         // Copy the URL so we can modify it
  66         var newU url.URL = *u
  67         u = &newU
  68
  69         if g.Netrc {
  70                 // Add auth from netrc if we can
  71                 if err := addAuthFromNetrc(u); err != nil {
  72                         return err
  73                 }
  74         }
  75
  76         if g.Client == nil {
  77                 g.Client = httpClient
  78         }
  79
  80         // Add terraform-get to the parameter.
  81         q := u.Query()
  82         q.Add("terraform-get", "1")
  83         u.RawQuery = q.Encode()
  84
  85         // Get the URL
  86         req, err := http.NewRequest("GET", u.String(), nil)
  87         if err != nil {
  88                 return err
  89         }
  90
  91         req.Header = g.Header
  92         resp, err := g.Client.Do(req)
  93         if err != nil {
  94                 return err
  95         }
  96
  97         defer resp.Body.Close()
  98         if resp.StatusCode < 200 || resp.StatusCode >= 300 {
  99                 return fmt.Errorf("bad response code: %d", resp.StatusCode)
 100         }
 101
 102         // Extract the source URL
 103         var source string
 104         if v := resp.Header.Get("X-Terraform-Get"); v != "" {
 105                 source = v
 106         } else {
 107                 source, err = g.parseMeta(resp.Body)
 108                 if err != nil {
 109                         return err
 110                 }
 111         }
 112         if source == "" {
 113                 return fmt.Errorf("no source URL was returned")
 114         }
 115
 116         // If there is a subdir component, then we download the root separately
 117         // into a temporary directory, then copy over the proper subdir.
 118         source, subDir := SourceDirSubdir(source)
 119         if subDir == "" {
 120                 var opts []ClientOption
 121                 if g.client != nil {
 122                         opts = g.client.Options
 123                 }
 124                 return Get(dst, source, opts...)
 125         }
 126
 127         // We have a subdir, time to jump some hoops
 128         return g.getSubdir(ctx, dst, source, subDir)
 129 }
 130
 131 func (g *HttpGetter) GetFile(dst string, src *url.URL) error {
 132         ctx := g.Context()
 133         if g.Netrc {
 134                 // Add auth from netrc if we can
 135                 if err := addAuthFromNetrc(src); err != nil {
 136                         return err
 137                 }
 138         }
 139
 140         // Create all the parent directories if needed
 141         if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
 142                 return err
 143         }
 144
 145         f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, os.FileMode(0666))
 146         if err != nil {
 147                 return err
 148         }
 149         defer f.Close()
 150
 151         if g.Client == nil {
 152                 g.Client = httpClient
 153         }
 154
 155         var currentFileSize int64
 156
 157         // We first make a HEAD request so we can check
 158         // if the server supports range queries. If the server/URL doesn't
 159         // support HEAD requests, we just fall back to GET.
 160         req, err := http.NewRequest("HEAD", src.String(), nil)
 161         if err != nil {
 162                 return err
 163         }
 164         if g.Header != nil {
 165                 req.Header = g.Header
 166         }
 167         headResp, err := g.Client.Do(req)
 168         if err == nil && headResp != nil {
 169                 headResp.Body.Close()
 170                 if headResp.StatusCode == 200 {
 171                         // If the HEAD request succeeded, then attempt to set the range
 172                         // query if we can.
 173                         if headResp.Header.Get("Accept-Ranges") == "bytes" {
 174                                 if fi, err := f.Stat(); err == nil {
 175                                         if _, err = f.Seek(0, os.SEEK_END); err == nil {
 176                                                 req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size()))
 177                                                 currentFileSize = fi.Size()
 178                                                 totalFileSize, _ := strconv.ParseInt(headResp.Header.Get("Content-Length"), 10, 64)
 179                                                 if currentFileSize >= totalFileSize {
 180                                                         // file already present
 181                                                         return nil
 182                                                 }
 183                                         }
 184                                 }
 185                         }
 186                 }
 187         }
 188         req.Method = "GET"
 189
 190         resp, err := g.Client.Do(req)
 191         if err != nil {
 192                 return err
 193         }
 194         switch resp.StatusCode {
 195         case http.StatusOK, http.StatusPartialContent:
 196                 // all good
 197         default:
 198                 resp.Body.Close()
 199                 return fmt.Errorf("bad response code: %d", resp.StatusCode)
 200         }
 201
 202         body := resp.Body
 203
 204         if g.client != nil && g.client.ProgressListener != nil {
 205                 // track download
 206                 fn := filepath.Base(src.EscapedPath())
 207                 body = g.client.ProgressListener.TrackProgress(fn, currentFileSize, currentFileSize+resp.ContentLength, resp.Body)
 208         }
 209         defer body.Close()
 210
 211         n, err := Copy(ctx, f, body)
 212         if err == nil && n < resp.ContentLength {
 213                 err = io.ErrShortWrite
 214         }
 215         return err
 216 }
 217
 218 // getSubdir downloads the source into the destination, but with
 219 // the proper subdir.
 220 func (g *HttpGetter) getSubdir(ctx context.Context, dst, source, subDir string) error {
 221         // Create a temporary directory to store the full source. This has to be
 222         // a non-existent directory.
 223         td, tdcloser, err := safetemp.Dir("", "getter")
 224         if err != nil {
 225                 return err
 226         }
 227         defer tdcloser.Close()
 228
 229         var opts []ClientOption
 230         if g.client != nil {
 231                 opts = g.client.Options
 232         }
 233         // Download that into the given directory
 234         if err := Get(td, source, opts...); err != nil {
 235                 return err
 236         }
 237
 238         // Process any globbing
 239         sourcePath, err := SubdirGlob(td, subDir)
 240         if err != nil {
 241                 return err
 242         }
 243
 244         // Make sure the subdir path actually exists
 245         if _, err := os.Stat(sourcePath); err != nil {
 246                 return fmt.Errorf(
 247                         "Error downloading %s: %s", source, err)
 248         }
 249
 250         // Copy the subdirectory into our actual destination.
 251         if err := os.RemoveAll(dst); err != nil {
 252                 return err
 253         }
 254
 255         // Make the final destination
 256         if err := os.MkdirAll(dst, 0755); err != nil {
 257                 return err
 258         }
 259
 260         return copyDir(ctx, dst, sourcePath, false)
 261 }
 262
 263 // parseMeta looks for the first meta tag in the given reader that
 264 // will give us the source URL.
 265 func (g *HttpGetter) parseMeta(r io.Reader) (string, error) {
 266         d := xml.NewDecoder(r)
 267         d.CharsetReader = charsetReader
 268         d.Strict = false
 269         var err error
 270         var t xml.Token
 271         for {
 272                 t, err = d.Token()
 273                 if err != nil {
 274                         if err == io.EOF {
 275                                 err = nil
 276                         }
 277                         return "", err
 278                 }
 279                 if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
 280                         return "", nil
 281                 }
 282                 if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
 283                         return "", nil
 284                 }
 285                 e, ok := t.(xml.StartElement)
 286                 if !ok || !strings.EqualFold(e.Name.Local, "meta") {
 287                         continue
 288                 }
 289                 if attrValue(e.Attr, "name") != "terraform-get" {
 290                         continue
 291                 }
 292                 if f := attrValue(e.Attr, "content"); f != "" {
 293                         return f, nil
 294                 }
 295         }
 296 }
 297
 298 // attrValue returns the attribute value for the case-insensitive key
 299 // `name', or the empty string if nothing is found.
 300 func attrValue(attrs []xml.Attr, name string) string {
 301         for _, a := range attrs {
 302                 if strings.EqualFold(a.Name.Local, name) {
 303                         return a.Value
 304                 }
 305         }
 306         return ""
 307 }
 308
 309 // charsetReader returns a reader for the given charset. Currently
 310 // it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
 311 // error which is printed by go get, so the user can find why the package
 312 // wasn't downloaded if the encoding is not supported. Note that, in
 313 // order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters
 314 // greater than 0x7f are not rejected).
 315 func charsetReader(charset string, input io.Reader) (io.Reader, error) {
 316         switch strings.ToLower(charset) {
 317         case "ascii":
 318                 return input, nil
 319         default:
 320                 return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
 321         }
 322 }