11 "github.com/aws/aws-sdk-go/aws"
12 "github.com/aws/aws-sdk-go/aws/credentials"
13 "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
14 "github.com/aws/aws-sdk-go/aws/ec2metadata"
15 "github.com/aws/aws-sdk-go/aws/session"
16 "github.com/aws/aws-sdk-go/service/s3"
19 // S3Getter is a Getter implementation that will download a module from
21 type S3Getter struct{}
23 func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
25 region, bucket, path, _, creds, err := g.parseUrl(u)
30 // Create client config
31 config := g.getAWSConfig(region, creds)
32 sess := session.New(config)
33 client := s3.New(sess)
35 // List the object(s) at the given prefix
36 req := &s3.ListObjectsInput{
37 Bucket: aws.String(bucket),
38 Prefix: aws.String(path),
40 resp, err := client.ListObjects(req)
45 for _, o := range resp.Contents {
46 // Use file mode on exact match.
48 return ClientModeFile, nil
51 // Use dir mode if child keys are found.
52 if strings.HasPrefix(*o.Key, path+"/") {
53 return ClientModeDir, nil
57 // There was no match, so just return file mode. The download is going
58 // to fail but we will let S3 return the proper error later.
59 return ClientModeFile, nil
62 func (g *S3Getter) Get(dst string, u *url.URL) error {
64 region, bucket, path, _, creds, err := g.parseUrl(u)
69 // Remove destination if it already exists
71 if err != nil && !os.IsNotExist(err) {
76 // Remove the destination
77 if err := os.RemoveAll(dst); err != nil {
82 // Create all the parent directories
83 if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
87 config := g.getAWSConfig(region, creds)
88 sess := session.New(config)
89 client := s3.New(sess)
91 // List files in path, keep listing until no more objects are found
95 req := &s3.ListObjectsInput{
96 Bucket: aws.String(bucket),
97 Prefix: aws.String(path),
100 req.Marker = aws.String(lastMarker)
103 resp, err := client.ListObjects(req)
108 hasMore = aws.BoolValue(resp.IsTruncated)
110 // Get each object storing each file relative to the destination path
111 for _, object := range resp.Contents {
112 lastMarker = aws.StringValue(object.Key)
113 objPath := aws.StringValue(object.Key)
115 // If the key ends with a backslash assume it is a directory and ignore
116 if strings.HasSuffix(objPath, "/") {
120 // Get the object destination path
121 objDst, err := filepath.Rel(path, objPath)
125 objDst = filepath.Join(dst, objDst)
127 if err := g.getObject(client, objDst, bucket, objPath, ""); err != nil {
136 func (g *S3Getter) GetFile(dst string, u *url.URL) error {
137 region, bucket, path, version, creds, err := g.parseUrl(u)
142 config := g.getAWSConfig(region, creds)
143 sess := session.New(config)
144 client := s3.New(sess)
145 return g.getObject(client, dst, bucket, path, version)
148 func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error {
149 req := &s3.GetObjectInput{
150 Bucket: aws.String(bucket),
151 Key: aws.String(key),
154 req.VersionId = aws.String(version)
157 resp, err := client.GetObject(req)
162 // Create all the parent directories
163 if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
167 f, err := os.Create(dst)
173 _, err = io.Copy(f, resp.Body)
177 func (g *S3Getter) getAWSConfig(region string, creds *credentials.Credentials) *aws.Config {
178 conf := &aws.Config{}
180 // Grab the metadata URL
181 metadataURL := os.Getenv("AWS_METADATA_URL")
182 if metadataURL == "" {
183 metadataURL = "http://169.254.169.254:80/latest"
186 creds = credentials.NewChainCredentials(
187 []credentials.Provider{
188 &credentials.EnvProvider{},
189 &credentials.SharedCredentialsProvider{Filename: "", Profile: ""},
190 &ec2rolecreds.EC2RoleProvider{
191 Client: ec2metadata.New(session.New(&aws.Config{
192 Endpoint: aws.String(metadataURL),
198 conf.Credentials = creds
200 conf.Region = aws.String(region)
206 func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, creds *credentials.Credentials, err error) {
207 // Expected host style: s3.amazonaws.com. They always have 3 parts,
208 // although the first may differ if we're accessing a specific region.
209 hostParts := strings.Split(u.Host, ".")
210 if len(hostParts) != 3 {
211 err = fmt.Errorf("URL is not a valid S3 URL")
215 // Parse the region out of the first part of the host
216 region = strings.TrimPrefix(strings.TrimPrefix(hostParts[0], "s3-"), "s3")
221 pathParts := strings.SplitN(u.Path, "/", 3)
222 if len(pathParts) != 3 {
223 err = fmt.Errorf("URL is not a valid S3 URL")
227 bucket = pathParts[1]
229 version = u.Query().Get("version")
231 _, hasAwsId := u.Query()["aws_access_key_id"]
232 _, hasAwsSecret := u.Query()["aws_access_key_secret"]
233 _, hasAwsToken := u.Query()["aws_access_token"]
234 if hasAwsId || hasAwsSecret || hasAwsToken {
235 creds = credentials.NewStaticCredentials(
236 u.Query().Get("aws_access_key_id"),
237 u.Query().Get("aws_access_key_secret"),
238 u.Query().Get("aws_access_token"),