1 // Copyright 2014 Google LLC
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
39 "cloud.google.com/go/internal/optional"
40 "cloud.google.com/go/internal/trace"
41 "cloud.google.com/go/internal/version"
42 "google.golang.org/api/googleapi"
43 "google.golang.org/api/option"
44 raw "google.golang.org/api/storage/v1"
45 htransport "google.golang.org/api/transport/http"
49 // ErrBucketNotExist indicates that the bucket does not exist.
50 ErrBucketNotExist = errors.New("storage: bucket doesn't exist")
51 // ErrObjectNotExist indicates that the object does not exist.
52 ErrObjectNotExist = errors.New("storage: object doesn't exist")
55 const userAgent = "gcloud-golang-storage/20151204"
58 // ScopeFullControl grants permissions to manage your
59 // data and permissions in Google Cloud Storage.
60 ScopeFullControl = raw.DevstorageFullControlScope
62 // ScopeReadOnly grants permissions to
63 // view your data in Google Cloud Storage.
64 ScopeReadOnly = raw.DevstorageReadOnlyScope
66 // ScopeReadWrite grants permissions to manage your
67 // data in Google Cloud Storage.
68 ScopeReadWrite = raw.DevstorageReadWriteScope
71 var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo)
73 func setClientHeader(headers http.Header) {
74 headers.Set("x-goog-api-client", xGoogHeader)
77 // Client is a client for interacting with Google Cloud Storage.
79 // Clients should be reused instead of created as needed.
80 // The methods of Client are safe for concurrent use by multiple goroutines.
86 // NewClient creates a new Google Cloud Storage client.
87 // The default scope is ScopeFullControl. To use a different scope, like ScopeReadOnly, use option.WithScopes.
88 func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
89 o := []option.ClientOption{
90 option.WithScopes(ScopeFullControl),
91 option.WithUserAgent(userAgent),
93 opts = append(o, opts...)
94 hc, ep, err := htransport.NewClient(ctx, opts...)
96 return nil, fmt.Errorf("dialing: %v", err)
98 rawService, err := raw.New(hc)
100 return nil, fmt.Errorf("storage client: %v", err)
103 rawService.BasePath = ep
111 // Close closes the Client.
113 // Close need not be called at program exit.
114 func (c *Client) Close() error {
115 // Set fields to nil so that subsequent uses will panic.
121 // SignedURLOptions allows you to restrict the access to the signed URL.
122 type SignedURLOptions struct {
123 // GoogleAccessID represents the authorizer of the signed URL generation.
124 // It is typically the Google service account client email address from
125 // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com".
127 GoogleAccessID string
129 // PrivateKey is the Google service account private key. It is obtainable
130 // from the Google Developers Console.
131 // At https://console.developers.google.com/project/<your-project-id>/apiui/credential,
132 // create a service account client ID or reuse one of your existing service account
133 // credentials. Click on the "Generate new P12 key" to generate and download
134 // a new private key. Once you download the P12 file, use the following command
135 // to convert it into a PEM file.
137 // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes
139 // Provide the contents of the PEM file as a byte slice.
140 // Exactly one of PrivateKey or SignBytes must be non-nil.
143 // SignBytes is a function for implementing custom signing.
144 // If your application is running on Google App Engine, you can use appengine's internal signing function:
145 // ctx := appengine.NewContext(request)
146 // acc, _ := appengine.ServiceAccount(ctx)
147 // url, err := SignedURL("bucket", "object", &SignedURLOptions{
148 // GoogleAccessID: acc,
149 // SignBytes: func(b []byte) ([]byte, error) {
150 // _, signedBytes, err := appengine.SignBytes(ctx, b)
151 // return signedBytes, err
156 // Exactly one of PrivateKey or SignBytes must be non-nil.
157 SignBytes func([]byte) ([]byte, error)
159 // Method is the HTTP method to be used with the signed URL.
160 // Signed URLs can be used with GET, HEAD, PUT, and DELETE requests.
164 // Expires is the expiration time on the signed URL. It must be
165 // a datetime in the future.
169 // ContentType is the content type header the client must provide
170 // to use the generated signed URL.
174 // Headers is a list of extension headers the client must provide
175 // in order to use the generated signed URL.
179 // MD5 is the base64 encoded MD5 checksum of the file.
180 // If provided, the client should provide the exact value on the request
181 // header in order to use the signed URL.
187 canonicalHeaderRegexp = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`)
188 excludedCanonicalHeaders = map[string]bool{
189 "x-goog-encryption-key": true,
190 "x-goog-encryption-key-sha256": true,
194 // sanitizeHeaders applies the specifications for canonical extension headers at
195 // https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers.
196 func sanitizeHeaders(hdrs []string) []string {
197 headerMap := map[string][]string{}
198 for _, hdr := range hdrs {
199 // No leading or trailing whitespaces.
200 sanitizedHeader := strings.TrimSpace(hdr)
202 // Only keep canonical headers, discard any others.
203 headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader)
204 if len(headerMatches) == 0 {
208 header := strings.ToLower(strings.TrimSpace(headerMatches[1]))
209 if excludedCanonicalHeaders[headerMatches[1]] {
210 // Do not keep any deliberately excluded canonical headers when signing.
213 value := strings.TrimSpace(headerMatches[2])
215 // Remove duplicate headers by appending the values of duplicates
216 // in their order of appearance.
217 headerMap[header] = append(headerMap[header], value)
221 var sanitizedHeaders []string
222 for header, values := range headerMap {
223 // There should be no spaces around the colon separating the
224 // header name from the header value or around the values
225 // themselves. The values should be separated by commas.
226 // NOTE: The semantics for headers without a value are not clear.
227 // However from specifications these should be edge-cases
228 // anyway and we should assume that there will be no
229 // canonical headers using empty values. Any such headers
230 // are discarded at the regexp stage above.
231 sanitizedHeaders = append(
233 fmt.Sprintf("%s:%s", header, strings.Join(values, ",")),
236 sort.Strings(sanitizedHeaders)
237 return sanitizedHeaders
240 // SignedURL returns a URL for the specified object. Signed URLs allow
241 // the users access to a restricted resource for a limited time without having a
242 // Google account or signing in. For more information about the signed
243 // URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs.
244 func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) {
246 return "", errors.New("storage: missing required SignedURLOptions")
248 if opts.GoogleAccessID == "" {
249 return "", errors.New("storage: missing required GoogleAccessID")
251 if (opts.PrivateKey == nil) == (opts.SignBytes == nil) {
252 return "", errors.New("storage: exactly one of PrivateKey or SignedBytes must be set")
254 if opts.Method == "" {
255 return "", errors.New("storage: missing required method option")
257 if opts.Expires.IsZero() {
258 return "", errors.New("storage: missing required expires option")
261 md5, err := base64.StdEncoding.DecodeString(opts.MD5)
262 if err != nil || len(md5) != 16 {
263 return "", errors.New("storage: invalid MD5 checksum")
266 opts.Headers = sanitizeHeaders(opts.Headers)
268 signBytes := opts.SignBytes
269 if opts.PrivateKey != nil {
270 key, err := parseKey(opts.PrivateKey)
274 signBytes = func(b []byte) ([]byte, error) {
275 sum := sha256.Sum256(b)
276 return rsa.SignPKCS1v15(
286 Path: fmt.Sprintf("/%s/%s", bucket, name),
289 buf := &bytes.Buffer{}
290 fmt.Fprintf(buf, "%s\n", opts.Method)
291 fmt.Fprintf(buf, "%s\n", opts.MD5)
292 fmt.Fprintf(buf, "%s\n", opts.ContentType)
293 fmt.Fprintf(buf, "%d\n", opts.Expires.Unix())
294 if len(opts.Headers) > 0 {
295 fmt.Fprintf(buf, "%s\n", strings.Join(opts.Headers, "\n"))
297 fmt.Fprintf(buf, "%s", u.String())
299 b, err := signBytes(buf.Bytes())
303 encoded := base64.StdEncoding.EncodeToString(b)
305 u.Host = "storage.googleapis.com"
307 q.Set("GoogleAccessId", opts.GoogleAccessID)
308 q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix()))
309 q.Set("Signature", string(encoded))
310 u.RawQuery = q.Encode()
311 return u.String(), nil
314 // ObjectHandle provides operations on an object in a Google Cloud Storage bucket.
315 // Use BucketHandle.Object to get a handle.
316 type ObjectHandle struct {
321 gen int64 // a negative value indicates latest
323 encryptionKey []byte // AES-256 key
324 userProject string // for requester-pays buckets
325 readCompressed bool // Accept-Encoding: gzip
328 // ACL provides access to the object's access control list.
329 // This controls who can read and write this object.
330 // This call does not perform any network operations.
331 func (o *ObjectHandle) ACL() *ACLHandle {
335 // Generation returns a new ObjectHandle that operates on a specific generation
337 // By default, the handle operates on the latest generation. Not
338 // all operations work when given a specific generation; check the API
339 // endpoints at https://cloud.google.com/storage/docs/json_api/ for details.
340 func (o *ObjectHandle) Generation(gen int64) *ObjectHandle {
346 // If returns a new ObjectHandle that applies a set of preconditions.
347 // Preconditions already set on the ObjectHandle are ignored.
348 // Operations on the new handle will return an error if the preconditions are not
349 // satisfied. See https://cloud.google.com/storage/docs/generations-preconditions
351 func (o *ObjectHandle) If(conds Conditions) *ObjectHandle {
357 // Key returns a new ObjectHandle that uses the supplied encryption
358 // key to encrypt and decrypt the object's contents.
360 // Encryption key must be a 32-byte AES-256 key.
361 // See https://cloud.google.com/storage/docs/encryption for details.
362 func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle {
364 o2.encryptionKey = encryptionKey
368 // Attrs returns meta information about the object.
369 // ErrObjectNotExist will be returned if the object is not found.
370 func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error) {
371 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Attrs")
372 defer func() { trace.EndSpan(ctx, err) }()
374 if err := o.validate(); err != nil {
377 call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx)
378 if err := applyConds("Attrs", o.gen, o.conds, call); err != nil {
381 if o.userProject != "" {
382 call.UserProject(o.userProject)
384 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil {
388 setClientHeader(call.Header())
389 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
390 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
391 return nil, ErrObjectNotExist
396 return newObject(obj), nil
399 // Update updates an object with the provided attributes.
400 // All zero-value attributes are ignored.
401 // ErrObjectNotExist will be returned if the object is not found.
402 func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (oa *ObjectAttrs, err error) {
403 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Update")
404 defer func() { trace.EndSpan(ctx, err) }()
406 if err := o.validate(); err != nil {
409 var attrs ObjectAttrs
410 // Lists of fields to send, and set to null, in the JSON.
411 var forceSendFields, nullFields []string
412 if uattrs.ContentType != nil {
413 attrs.ContentType = optional.ToString(uattrs.ContentType)
414 // For ContentType, sending the empty string is a no-op.
415 // Instead we send a null.
416 if attrs.ContentType == "" {
417 nullFields = append(nullFields, "ContentType")
419 forceSendFields = append(forceSendFields, "ContentType")
422 if uattrs.ContentLanguage != nil {
423 attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage)
424 // For ContentLanguage it's an error to send the empty string.
425 // Instead we send a null.
426 if attrs.ContentLanguage == "" {
427 nullFields = append(nullFields, "ContentLanguage")
429 forceSendFields = append(forceSendFields, "ContentLanguage")
432 if uattrs.ContentEncoding != nil {
433 attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding)
434 forceSendFields = append(forceSendFields, "ContentEncoding")
436 if uattrs.ContentDisposition != nil {
437 attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition)
438 forceSendFields = append(forceSendFields, "ContentDisposition")
440 if uattrs.CacheControl != nil {
441 attrs.CacheControl = optional.ToString(uattrs.CacheControl)
442 forceSendFields = append(forceSendFields, "CacheControl")
444 if uattrs.EventBasedHold != nil {
445 attrs.EventBasedHold = optional.ToBool(uattrs.EventBasedHold)
446 forceSendFields = append(forceSendFields, "EventBasedHold")
448 if uattrs.TemporaryHold != nil {
449 attrs.TemporaryHold = optional.ToBool(uattrs.TemporaryHold)
450 forceSendFields = append(forceSendFields, "TemporaryHold")
452 if uattrs.Metadata != nil {
453 attrs.Metadata = uattrs.Metadata
454 if len(attrs.Metadata) == 0 {
455 // Sending the empty map is a no-op. We send null instead.
456 nullFields = append(nullFields, "Metadata")
458 forceSendFields = append(forceSendFields, "Metadata")
461 if uattrs.ACL != nil {
462 attrs.ACL = uattrs.ACL
463 // It's an error to attempt to delete the ACL, so
464 // we don't append to nullFields here.
465 forceSendFields = append(forceSendFields, "Acl")
467 rawObj := attrs.toRawObject(o.bucket)
468 rawObj.ForceSendFields = forceSendFields
469 rawObj.NullFields = nullFields
470 call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx)
471 if err := applyConds("Update", o.gen, o.conds, call); err != nil {
474 if o.userProject != "" {
475 call.UserProject(o.userProject)
477 if uattrs.PredefinedACL != "" {
478 call.PredefinedAcl(uattrs.PredefinedACL)
480 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil {
484 setClientHeader(call.Header())
485 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
486 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
487 return nil, ErrObjectNotExist
492 return newObject(obj), nil
495 // BucketName returns the name of the bucket.
496 func (o *ObjectHandle) BucketName() string {
500 // ObjectName returns the name of the object.
501 func (o *ObjectHandle) ObjectName() string {
505 // ObjectAttrsToUpdate is used to update the attributes of an object.
506 // Only fields set to non-nil values will be updated.
507 // Set a field to its zero value to delete it.
509 // For example, to change ContentType and delete ContentEncoding and
511 // ObjectAttrsToUpdate{
512 // ContentType: "text/html",
513 // ContentEncoding: "",
514 // Metadata: map[string]string{},
516 type ObjectAttrsToUpdate struct {
517 EventBasedHold optional.Bool
518 TemporaryHold optional.Bool
519 ContentType optional.String
520 ContentLanguage optional.String
521 ContentEncoding optional.String
522 ContentDisposition optional.String
523 CacheControl optional.String
524 Metadata map[string]string // set to map[string]string{} to delete
527 // If not empty, applies a predefined set of access controls. ACL must be nil.
528 // See https://cloud.google.com/storage/docs/json_api/v1/objects/patch.
532 // Delete deletes the single specified object.
533 func (o *ObjectHandle) Delete(ctx context.Context) error {
534 if err := o.validate(); err != nil {
537 call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx)
538 if err := applyConds("Delete", o.gen, o.conds, call); err != nil {
541 if o.userProject != "" {
542 call.UserProject(o.userProject)
544 // Encryption doesn't apply to Delete.
545 setClientHeader(call.Header())
546 err := runWithRetry(ctx, func() error { return call.Do() })
547 switch e := err.(type) {
550 case *googleapi.Error:
551 if e.Code == http.StatusNotFound {
552 return ErrObjectNotExist
558 // ReadCompressed when true causes the read to happen without decompressing.
559 func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle {
561 o2.readCompressed = compressed
565 // NewWriter returns a storage Writer that writes to the GCS object
566 // associated with this ObjectHandle.
568 // A new object will be created unless an object with this name already exists.
569 // Otherwise any previous object with the same name will be replaced.
570 // The object will not be available (and any previous object will remain)
571 // until Close has been called.
573 // Attributes can be set on the object by modifying the returned Writer's
574 // ObjectAttrs field before the first call to Write. If no ContentType
575 // attribute is specified, the content type will be automatically sniffed
576 // using net/http.DetectContentType.
578 // It is the caller's responsibility to call Close when writing is done. To
579 // stop writing without saving the data, cancel the context.
580 func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer {
584 donec: make(chan struct{}),
585 ObjectAttrs: ObjectAttrs{Name: o.object},
586 ChunkSize: googleapi.DefaultUploadChunkSize,
590 func (o *ObjectHandle) validate() error {
592 return errors.New("storage: bucket name is empty")
595 return errors.New("storage: object name is empty")
597 if !utf8.ValidString(o.object) {
598 return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object)
603 // parseKey converts the binary contents of a private key file to an
604 // *rsa.PrivateKey. It detects whether the private key is in a PEM container or
605 // not. If so, it extracts the private key from PEM container before
606 // conversion. It only supports PEM containers with no passphrase.
607 func parseKey(key []byte) (*rsa.PrivateKey, error) {
608 if block, _ := pem.Decode(key); block != nil {
611 parsedKey, err := x509.ParsePKCS8PrivateKey(key)
613 parsedKey, err = x509.ParsePKCS1PrivateKey(key)
618 parsed, ok := parsedKey.(*rsa.PrivateKey)
620 return nil, errors.New("oauth2: private key is invalid")
625 // toRawObject copies the editable attributes from o to the raw library's Object type.
626 func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object {
628 if !o.RetentionExpirationTime.IsZero() {
629 ret = o.RetentionExpirationTime.Format(time.RFC3339)
634 EventBasedHold: o.EventBasedHold,
635 TemporaryHold: o.TemporaryHold,
636 RetentionExpirationTime: ret,
637 ContentType: o.ContentType,
638 ContentEncoding: o.ContentEncoding,
639 ContentLanguage: o.ContentLanguage,
640 CacheControl: o.CacheControl,
641 ContentDisposition: o.ContentDisposition,
642 StorageClass: o.StorageClass,
643 Acl: toRawObjectACL(o.ACL),
644 Metadata: o.Metadata,
648 // ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object.
649 type ObjectAttrs struct {
650 // Bucket is the name of the bucket containing this GCS object.
651 // This field is read-only.
654 // Name is the name of the object within the bucket.
655 // This field is read-only.
658 // ContentType is the MIME type of the object's content.
661 // ContentLanguage is the content language of the object's content.
662 ContentLanguage string
664 // CacheControl is the Cache-Control header to be sent in the response
665 // headers when serving the object data.
668 // EventBasedHold specifies whether an object is under event-based hold. New
669 // objects created in a bucket whose DefaultEventBasedHold is set will
670 // default to that value.
673 // TemporaryHold specifies whether an object is under temporary hold. While
674 // this flag is set to true, the object is protected against deletion and
678 // RetentionExpirationTime is a server-determined value that specifies the
679 // earliest time that the object's retention period expires.
680 // This is a read-only field.
681 RetentionExpirationTime time.Time
683 // ACL is the list of access control rules for the object.
686 // If not empty, applies a predefined set of access controls. It should be set
687 // only when writing, copying or composing an object. When copying or composing,
688 // it acts as the destinationPredefinedAcl parameter.
689 // PredefinedACL is always empty for ObjectAttrs returned from the service.
690 // See https://cloud.google.com/storage/docs/json_api/v1/objects/insert
694 // Owner is the owner of the object. This field is read-only.
696 // If non-zero, it is in the form of "user-<userId>".
699 // Size is the length of the object's content. This field is read-only.
702 // ContentEncoding is the encoding of the object's content.
703 ContentEncoding string
705 // ContentDisposition is the optional Content-Disposition header of the object
706 // sent in the response headers.
707 ContentDisposition string
709 // MD5 is the MD5 hash of the object's content. This field is read-only,
710 // except when used from a Writer. If set on a Writer, the uploaded
711 // data is rejected if its MD5 hash does not match this field.
714 // CRC32C is the CRC32 checksum of the object's content using
715 // the Castagnoli93 polynomial. This field is read-only, except when
716 // used from a Writer. If set on a Writer and Writer.SendCRC32C
717 // is true, the uploaded data is rejected if its CRC32c hash does not
721 // MediaLink is an URL to the object's content. This field is read-only.
724 // Metadata represents user-provided metadata, in key/value pairs.
725 // It can be nil if no metadata is provided.
726 Metadata map[string]string
728 // Generation is the generation number of the object's content.
729 // This field is read-only.
732 // Metageneration is the version of the metadata for this
733 // object at this generation. This field is used for preconditions
734 // and for detecting changes in metadata. A metageneration number
735 // is only meaningful in the context of a particular generation
736 // of a particular object. This field is read-only.
739 // StorageClass is the storage class of the object.
740 // This value defines how objects in the bucket are stored and
741 // determines the SLA and the cost of storage. Typical values are
742 // "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD"
743 // and "DURABLE_REDUCED_AVAILABILITY".
744 // It defaults to "STANDARD", which is equivalent to "MULTI_REGIONAL"
745 // or "REGIONAL" depending on the bucket's location settings.
748 // Created is the time the object was created. This field is read-only.
751 // Deleted is the time the object was deleted.
752 // If not deleted, it is the zero value. This field is read-only.
755 // Updated is the creation or modification time of the object.
756 // For buckets with versioning enabled, changing an object's
757 // metadata does not change this property. This field is read-only.
760 // CustomerKeySHA256 is the base64-encoded SHA-256 hash of the
761 // customer-supplied encryption key for the object. It is empty if there is
762 // no customer-supplied encryption key.
763 // See // https://cloud.google.com/storage/docs/encryption for more about
764 // encryption in Google Cloud Storage.
765 CustomerKeySHA256 string
767 // Cloud KMS key name, in the form
768 // projects/P/locations/L/keyRings/R/cryptoKeys/K, used to encrypt this object,
769 // if the object is encrypted by such a key.
771 // Providing both a KMSKeyName and a customer-supplied encryption key (via
772 // ObjectHandle.Key) will result in an error when writing an object.
775 // Prefix is set only for ObjectAttrs which represent synthetic "directory
776 // entries" when iterating over buckets using Query.Delimiter. See
777 // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be
782 // convertTime converts a time in RFC3339 format to time.Time.
783 // If any error occurs in parsing, the zero-value time.Time is silently returned.
784 func convertTime(t string) time.Time {
787 r, _ = time.Parse(time.RFC3339, t)
792 func newObject(o *raw.Object) *ObjectAttrs {
798 owner = o.Owner.Entity
800 md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash)
801 crc32c, _ := decodeUint32(o.Crc32c)
803 if o.CustomerEncryption != nil {
804 sha256 = o.CustomerEncryption.KeySha256
809 ContentType: o.ContentType,
810 ContentLanguage: o.ContentLanguage,
811 CacheControl: o.CacheControl,
812 EventBasedHold: o.EventBasedHold,
813 TemporaryHold: o.TemporaryHold,
814 RetentionExpirationTime: convertTime(o.RetentionExpirationTime),
815 ACL: toObjectACLRules(o.Acl),
817 ContentEncoding: o.ContentEncoding,
818 ContentDisposition: o.ContentDisposition,
822 MediaLink: o.MediaLink,
823 Metadata: o.Metadata,
824 Generation: o.Generation,
825 Metageneration: o.Metageneration,
826 StorageClass: o.StorageClass,
827 CustomerKeySHA256: sha256,
828 KMSKeyName: o.KmsKeyName,
829 Created: convertTime(o.TimeCreated),
830 Deleted: convertTime(o.TimeDeleted),
831 Updated: convertTime(o.Updated),
835 // Decode a uint32 encoded in Base64 in big-endian byte order.
836 func decodeUint32(b64 string) (uint32, error) {
837 d, err := base64.StdEncoding.DecodeString(b64)
842 return 0, fmt.Errorf("storage: %q does not encode a 32-bit value", d)
844 return uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]), nil
847 // Encode a uint32 as Base64 in big-endian byte order.
848 func encodeUint32(u uint32) string {
849 b := []byte{byte(u >> 24), byte(u >> 16), byte(u >> 8), byte(u)}
850 return base64.StdEncoding.EncodeToString(b)
853 // Query represents a query to filter objects from a bucket.
855 // Delimiter returns results in a directory-like fashion.
856 // Results will contain only objects whose names, aside from the
857 // prefix, do not contain delimiter. Objects whose names,
858 // aside from the prefix, contain delimiter will have their name,
859 // truncated after the delimiter, returned in prefixes.
860 // Duplicate prefixes are omitted.
864 // Prefix is the prefix filter to query objects
865 // whose names begin with this prefix.
869 // Versions indicates whether multiple versions of the same
870 // object will be included in the results.
874 // Conditions constrain methods to act on specific generations of
877 // The zero value is an empty set of constraints. Not all conditions or
878 // combinations of conditions are applicable to all methods.
879 // See https://cloud.google.com/storage/docs/generations-preconditions
880 // for details on how these operate.
881 type Conditions struct {
882 // Generation constraints.
883 // At most one of the following can be set to a non-zero value.
885 // GenerationMatch specifies that the object must have the given generation
886 // for the operation to occur.
887 // If GenerationMatch is zero, it has no effect.
888 // Use DoesNotExist to specify that the object does not exist in the bucket.
889 GenerationMatch int64
891 // GenerationNotMatch specifies that the object must not have the given
892 // generation for the operation to occur.
893 // If GenerationNotMatch is zero, it has no effect.
894 GenerationNotMatch int64
896 // DoesNotExist specifies that the object must not exist in the bucket for
897 // the operation to occur.
898 // If DoesNotExist is false, it has no effect.
901 // Metadata generation constraints.
902 // At most one of the following can be set to a non-zero value.
904 // MetagenerationMatch specifies that the object must have the given
905 // metageneration for the operation to occur.
906 // If MetagenerationMatch is zero, it has no effect.
907 MetagenerationMatch int64
909 // MetagenerationNotMatch specifies that the object must not have the given
910 // metageneration for the operation to occur.
911 // If MetagenerationNotMatch is zero, it has no effect.
912 MetagenerationNotMatch int64
915 func (c *Conditions) validate(method string) error {
916 if *c == (Conditions{}) {
917 return fmt.Errorf("storage: %s: empty conditions", method)
919 if !c.isGenerationValid() {
920 return fmt.Errorf("storage: %s: multiple conditions specified for generation", method)
922 if !c.isMetagenerationValid() {
923 return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method)
928 func (c *Conditions) isGenerationValid() bool {
930 if c.GenerationMatch != 0 {
933 if c.GenerationNotMatch != 0 {
942 func (c *Conditions) isMetagenerationValid() bool {
943 return c.MetagenerationMatch == 0 || c.MetagenerationNotMatch == 0
946 // applyConds modifies the provided call using the conditions in conds.
947 // call is something that quacks like a *raw.WhateverCall.
948 func applyConds(method string, gen int64, conds *Conditions, call interface{}) error {
949 cval := reflect.ValueOf(call)
951 if !setConditionField(cval, "Generation", gen) {
952 return fmt.Errorf("storage: %s: generation not supported", method)
958 if err := conds.validate(method); err != nil {
962 case conds.GenerationMatch != 0:
963 if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) {
964 return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method)
966 case conds.GenerationNotMatch != 0:
967 if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) {
968 return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method)
970 case conds.DoesNotExist:
971 if !setConditionField(cval, "IfGenerationMatch", int64(0)) {
972 return fmt.Errorf("storage: %s: DoesNotExist not supported", method)
976 case conds.MetagenerationMatch != 0:
977 if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
978 return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
980 case conds.MetagenerationNotMatch != 0:
981 if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
982 return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
988 func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall) error {
990 call.SourceGeneration(gen)
995 if err := conds.validate("CopyTo source"); err != nil {
999 case conds.GenerationMatch != 0:
1000 call.IfSourceGenerationMatch(conds.GenerationMatch)
1001 case conds.GenerationNotMatch != 0:
1002 call.IfSourceGenerationNotMatch(conds.GenerationNotMatch)
1003 case conds.DoesNotExist:
1004 call.IfSourceGenerationMatch(0)
1007 case conds.MetagenerationMatch != 0:
1008 call.IfSourceMetagenerationMatch(conds.MetagenerationMatch)
1009 case conds.MetagenerationNotMatch != 0:
1010 call.IfSourceMetagenerationNotMatch(conds.MetagenerationNotMatch)
1015 // setConditionField sets a field on a *raw.WhateverCall.
1016 // We can't use anonymous interfaces because the return type is
1017 // different, since the field setters are builders.
1018 func setConditionField(call reflect.Value, name string, value interface{}) bool {
1019 m := call.MethodByName(name)
1023 m.Call([]reflect.Value{reflect.ValueOf(value)})
1027 // conditionsQuery returns the generation and conditions as a URL query
1028 // string suitable for URL.RawQuery. It assumes that the conditions
1029 // have been validated.
1030 func conditionsQuery(gen int64, conds *Conditions) string {
1031 // URL escapes are elided because integer strings are URL-safe.
1034 appendParam := func(s string, n int64) {
1036 buf = append(buf, '&')
1038 buf = append(buf, s...)
1039 buf = strconv.AppendInt(buf, n, 10)
1043 appendParam("generation=", gen)
1049 case conds.GenerationMatch != 0:
1050 appendParam("ifGenerationMatch=", conds.GenerationMatch)
1051 case conds.GenerationNotMatch != 0:
1052 appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch)
1053 case conds.DoesNotExist:
1054 appendParam("ifGenerationMatch=", 0)
1057 case conds.MetagenerationMatch != 0:
1058 appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch)
1059 case conds.MetagenerationNotMatch != 0:
1060 appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch)
1065 // composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods
1066 // that modifyCall searches for by name.
1067 type composeSourceObj struct {
1068 src *raw.ComposeRequestSourceObjects
1071 func (c composeSourceObj) Generation(gen int64) {
1072 c.src.Generation = gen
1075 func (c composeSourceObj) IfGenerationMatch(gen int64) {
1076 // It's safe to overwrite ObjectPreconditions, since its only field is
1077 // IfGenerationMatch.
1078 c.src.ObjectPreconditions = &raw.ComposeRequestSourceObjectsObjectPreconditions{
1079 IfGenerationMatch: gen,
1083 func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) error {
1087 // TODO(jbd): Ask the API team to return a more user-friendly error
1088 // and avoid doing this check at the client level.
1090 return errors.New("storage: not a 32-byte AES-256 key")
1096 headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256")
1097 headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key))
1098 keyHash := sha256.Sum256(key)
1099 headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:]))
1103 // ServiceAccount fetches the email address of the given project's Google Cloud Storage service account.
1104 func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) {
1105 r := c.raw.Projects.ServiceAccount.Get(projectID)
1106 res, err := r.Context(ctx).Do()
1110 return res.EmailAddress, nil