vendor/google.golang.org/appengine/datastore/prop.go

   1 // Copyright 2011 Google Inc. All rights reserved.
   2 // Use of this source code is governed by the Apache 2.0
   3 // license that can be found in the LICENSE file.
   4
   5 package datastore
   6
   7 import (
   8         "fmt"
   9         "reflect"
  10         "strings"
  11         "sync"
  12         "unicode"
  13 )
  14
  15 // Entities with more than this many indexed properties will not be saved.
  16 const maxIndexedProperties = 20000
  17
  18 // []byte fields more than 1 megabyte long will not be loaded or saved.
  19 const maxBlobLen = 1 << 20
  20
  21 // Property is a name/value pair plus some metadata. A datastore entity's
  22 // contents are loaded and saved as a sequence of Properties. An entity can
  23 // have multiple Properties with the same name, provided that p.Multiple is
  24 // true on all of that entity's Properties with that name.
  25 type Property struct {
  26         // Name is the property name.
  27         Name string
  28         // Value is the property value. The valid types are:
  29         //      - int64
  30         //      - bool
  31         //      - string
  32         //      - float64
  33         //      - ByteString
  34         //      - *Key
  35         //      - time.Time
  36         //      - appengine.BlobKey
  37         //      - appengine.GeoPoint
  38         //      - []byte (up to 1 megabyte in length)
  39         //      - *Entity (representing a nested struct)
  40         // This set is smaller than the set of valid struct field types that the
  41         // datastore can load and save. A Property Value cannot be a slice (apart
  42         // from []byte); use multiple Properties instead. Also, a Value's type
  43         // must be explicitly on the list above; it is not sufficient for the
  44         // underlying type to be on that list. For example, a Value of "type
  45         // myInt64 int64" is invalid. Smaller-width integers and floats are also
  46         // invalid. Again, this is more restrictive than the set of valid struct
  47         // field types.
  48         //
  49         // A Value will have an opaque type when loading entities from an index,
  50         // such as via a projection query. Load entities into a struct instead
  51         // of a PropertyLoadSaver when using a projection query.
  52         //
  53         // A Value may also be the nil interface value; this is equivalent to
  54         // Python's None but not directly representable by a Go struct. Loading
  55         // a nil-valued property into a struct will set that field to the zero
  56         // value.
  57         Value interface{}
  58         // NoIndex is whether the datastore cannot index this property.
  59         NoIndex bool
  60         // Multiple is whether the entity can have multiple properties with
  61         // the same name. Even if a particular instance only has one property with
  62         // a certain name, Multiple should be true if a struct would best represent
  63         // it as a field of type []T instead of type T.
  64         Multiple bool
  65 }
  66
  67 // An Entity is the value type for a nested struct.
  68 // This type is only used for a Property's Value.
  69 type Entity struct {
  70         Key        *Key
  71         Properties []Property
  72 }
  73
  74 // ByteString is a short byte slice (up to 1500 bytes) that can be indexed.
  75 type ByteString []byte
  76
  77 // PropertyLoadSaver can be converted from and to a slice of Properties.
  78 type PropertyLoadSaver interface {
  79         Load([]Property) error
  80         Save() ([]Property, error)
  81 }
  82
  83 // PropertyList converts a []Property to implement PropertyLoadSaver.
  84 type PropertyList []Property
  85
  86 var (
  87         typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem()
  88         typeOfPropertyList      = reflect.TypeOf(PropertyList(nil))
  89 )
  90
  91 // Load loads all of the provided properties into l.
  92 // It does not first reset *l to an empty slice.
  93 func (l *PropertyList) Load(p []Property) error {
  94         *l = append(*l, p...)
  95         return nil
  96 }
  97
  98 // Save saves all of l's properties as a slice or Properties.
  99 func (l *PropertyList) Save() ([]Property, error) {
 100         return *l, nil
 101 }
 102
 103 // validPropertyName returns whether name consists of one or more valid Go
 104 // identifiers joined by ".".
 105 func validPropertyName(name string) bool {
 106         if name == "" {
 107                 return false
 108         }
 109         for _, s := range strings.Split(name, ".") {
 110                 if s == "" {
 111                         return false
 112                 }
 113                 first := true
 114                 for _, c := range s {
 115                         if first {
 116                                 first = false
 117                                 if c != '_' && !unicode.IsLetter(c) {
 118                                         return false
 119                                 }
 120                         } else {
 121                                 if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) {
 122                                         return false
 123                                 }
 124                         }
 125                 }
 126         }
 127         return true
 128 }
 129
 130 // structCodec describes how to convert a struct to and from a sequence of
 131 // properties.
 132 type structCodec struct {
 133         // fields gives the field codec for the structTag with the given name.
 134         fields map[string]fieldCodec
 135         // hasSlice is whether a struct or any of its nested or embedded structs
 136         // has a slice-typed field (other than []byte).
 137         hasSlice bool
 138         // keyField is the index of a *Key field with structTag __key__.
 139         // This field is not relevant for the top level struct, only for
 140         // nested structs.
 141         keyField int
 142         // complete is whether the structCodec is complete. An incomplete
 143         // structCodec may be encountered when walking a recursive struct.
 144         complete bool
 145 }
 146
 147 // fieldCodec is a struct field's index and, if that struct field's type is
 148 // itself a struct, that substruct's structCodec.
 149 type fieldCodec struct {
 150         // path is the index path to the field
 151         path    []int
 152         noIndex bool
 153         // omitEmpty indicates that the field should be omitted on save
 154         // if empty.
 155         omitEmpty bool
 156         // structCodec is the codec fot the struct field at index 'path',
 157         // or nil if the field is not a struct.
 158         structCodec *structCodec
 159 }
 160
 161 // structCodecs collects the structCodecs that have already been calculated.
 162 var (
 163         structCodecsMutex sync.Mutex
 164         structCodecs      = make(map[reflect.Type]*structCodec)
 165 )
 166
 167 // getStructCodec returns the structCodec for the given struct type.
 168 func getStructCodec(t reflect.Type) (*structCodec, error) {
 169         structCodecsMutex.Lock()
 170         defer structCodecsMutex.Unlock()
 171         return getStructCodecLocked(t)
 172 }
 173
 174 // getStructCodecLocked implements getStructCodec. The structCodecsMutex must
 175 // be held when calling this function.
 176 func getStructCodecLocked(t reflect.Type) (ret *structCodec, retErr error) {
 177         c, ok := structCodecs[t]
 178         if ok {
 179                 return c, nil
 180         }
 181         c = &structCodec{
 182                 fields: make(map[string]fieldCodec),
 183                 // We initialize keyField to -1 so that the zero-value is not
 184                 // misinterpreted as index 0.
 185                 keyField: -1,
 186         }
 187
 188         // Add c to the structCodecs map before we are sure it is good. If t is
 189         // a recursive type, it needs to find the incomplete entry for itself in
 190         // the map.
 191         structCodecs[t] = c
 192         defer func() {
 193                 if retErr != nil {
 194                         delete(structCodecs, t)
 195                 }
 196         }()
 197
 198         for i := 0; i < t.NumField(); i++ {
 199                 f := t.Field(i)
 200                 // Skip unexported fields.
 201                 // Note that if f is an anonymous, unexported struct field,
 202                 // we will promote its fields.
 203                 if f.PkgPath != "" && !f.Anonymous {
 204                         continue
 205                 }
 206
 207                 tags := strings.Split(f.Tag.Get("datastore"), ",")
 208                 name := tags[0]
 209                 opts := make(map[string]bool)
 210                 for _, t := range tags[1:] {
 211                         opts[t] = true
 212                 }
 213                 switch {
 214                 case name == "":
 215                         if !f.Anonymous {
 216                                 name = f.Name
 217                         }
 218                 case name == "-":
 219                         continue
 220                 case name == "__key__":
 221                         if f.Type != typeOfKeyPtr {
 222                                 return nil, fmt.Errorf("datastore: __key__ field on struct %v is not a *datastore.Key", t)
 223                         }
 224                         c.keyField = i
 225                 case !validPropertyName(name):
 226                         return nil, fmt.Errorf("datastore: struct tag has invalid property name: %q", name)
 227                 }
 228
 229                 substructType, fIsSlice := reflect.Type(nil), false
 230                 switch f.Type.Kind() {
 231                 case reflect.Struct:
 232                         substructType = f.Type
 233                 case reflect.Slice:
 234                         if f.Type.Elem().Kind() == reflect.Struct {
 235                                 substructType = f.Type.Elem()
 236                         }
 237                         fIsSlice = f.Type != typeOfByteSlice
 238                         c.hasSlice = c.hasSlice || fIsSlice
 239                 }
 240
 241                 var sub *structCodec
 242                 if substructType != nil && substructType != typeOfTime && substructType != typeOfGeoPoint {
 243                         var err error
 244                         sub, err = getStructCodecLocked(substructType)
 245                         if err != nil {
 246                                 return nil, err
 247                         }
 248                         if !sub.complete {
 249                                 return nil, fmt.Errorf("datastore: recursive struct: field %q", f.Name)
 250                         }
 251                         if fIsSlice && sub.hasSlice {
 252                                 return nil, fmt.Errorf(
 253                                         "datastore: flattening nested structs leads to a slice of slices: field %q", f.Name)
 254                         }
 255                         c.hasSlice = c.hasSlice || sub.hasSlice
 256                         // If f is an anonymous struct field, we promote the substruct's fields up to this level
 257                         // in the linked list of struct codecs.
 258                         if f.Anonymous {
 259                                 for subname, subfield := range sub.fields {
 260                                         if name != "" {
 261                                                 subname = name + "." + subname
 262                                         }
 263                                         if _, ok := c.fields[subname]; ok {
 264                                                 return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", subname)
 265                                         }
 266                                         c.fields[subname] = fieldCodec{
 267                                                 path:        append([]int{i}, subfield.path...),
 268                                                 noIndex:     subfield.noIndex || opts["noindex"],
 269                                                 omitEmpty:   subfield.omitEmpty,
 270                                                 structCodec: subfield.structCodec,
 271                                         }
 272                                 }
 273                                 continue
 274                         }
 275                 }
 276
 277                 if _, ok := c.fields[name]; ok {
 278                         return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", name)
 279                 }
 280                 c.fields[name] = fieldCodec{
 281                         path:        []int{i},
 282                         noIndex:     opts["noindex"],
 283                         omitEmpty:   opts["omitempty"],
 284                         structCodec: sub,
 285                 }
 286         }
 287         c.complete = true
 288         return c, nil
 289 }
 290
 291 // structPLS adapts a struct to be a PropertyLoadSaver.
 292 type structPLS struct {
 293         v     reflect.Value
 294         codec *structCodec
 295 }
 296
 297 // newStructPLS returns a structPLS, which implements the
 298 // PropertyLoadSaver interface, for the struct pointer p.
 299 func newStructPLS(p interface{}) (*structPLS, error) {
 300         v := reflect.ValueOf(p)
 301         if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
 302                 return nil, ErrInvalidEntityType
 303         }
 304         v = v.Elem()
 305         codec, err := getStructCodec(v.Type())
 306         if err != nil {
 307                 return nil, err
 308         }
 309         return &structPLS{v, codec}, nil
 310 }
 311
 312 // LoadStruct loads the properties from p to dst.
 313 // dst must be a struct pointer.
 314 func LoadStruct(dst interface{}, p []Property) error {
 315         x, err := newStructPLS(dst)
 316         if err != nil {
 317                 return err
 318         }
 319         return x.Load(p)
 320 }
 321
 322 // SaveStruct returns the properties from src as a slice of Properties.
 323 // src must be a struct pointer.
 324 func SaveStruct(src interface{}) ([]Property, error) {
 325         x, err := newStructPLS(src)
 326         if err != nil {
 327                 return nil, err
 328         }
 329         return x.Save()
 330 }