vendor/github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/archive/changes.go

   1 package archive
   2
   3 import (
   4         "archive/tar"
   5         "bytes"
   6         "fmt"
   7         "io"
   8         "io/ioutil"
   9         "os"
  10         "path/filepath"
  11         "sort"
  12         "strings"
  13         "syscall"
  14         "time"
  15
  16         "github.com/fsouza/go-dockerclient/external/github.com/Sirupsen/logrus"
  17         "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/idtools"
  18         "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/pools"
  19         "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/system"
  20 )
  21
  22 // ChangeType represents the change type.
  23 type ChangeType int
  24
  25 const (
  26         // ChangeModify represents the modify operation.
  27         ChangeModify = iota
  28         // ChangeAdd represents the add operation.
  29         ChangeAdd
  30         // ChangeDelete represents the delete operation.
  31         ChangeDelete
  32 )
  33
  34 func (c ChangeType) String() string {
  35         switch c {
  36         case ChangeModify:
  37                 return "C"
  38         case ChangeAdd:
  39                 return "A"
  40         case ChangeDelete:
  41                 return "D"
  42         }
  43         return ""
  44 }
  45
  46 // Change represents a change, it wraps the change type and path.
  47 // It describes changes of the files in the path respect to the
  48 // parent layers. The change could be modify, add, delete.
  49 // This is used for layer diff.
  50 type Change struct {
  51         Path string
  52         Kind ChangeType
  53 }
  54
  55 func (change *Change) String() string {
  56         return fmt.Sprintf("%s %s", change.Kind, change.Path)
  57 }
  58
  59 // for sort.Sort
  60 type changesByPath []Change
  61
  62 func (c changesByPath) Less(i, j int) bool { return c[i].Path < c[j].Path }
  63 func (c changesByPath) Len() int           { return len(c) }
  64 func (c changesByPath) Swap(i, j int)      { c[j], c[i] = c[i], c[j] }
  65
  66 // Gnu tar and the go tar writer don't have sub-second mtime
  67 // precision, which is problematic when we apply changes via tar
  68 // files, we handle this by comparing for exact times, *or* same
  69 // second count and either a or b having exactly 0 nanoseconds
  70 func sameFsTime(a, b time.Time) bool {
  71         return a == b ||
  72                 (a.Unix() == b.Unix() &&
  73                         (a.Nanosecond() == 0 || b.Nanosecond() == 0))
  74 }
  75
  76 func sameFsTimeSpec(a, b syscall.Timespec) bool {
  77         return a.Sec == b.Sec &&
  78                 (a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
  79 }
  80
  81 // Changes walks the path rw and determines changes for the files in the path,
  82 // with respect to the parent layers
  83 func Changes(layers []string, rw string) ([]Change, error) {
  84         var (
  85                 changes     []Change
  86                 changedDirs = make(map[string]struct{})
  87         )
  88
  89         err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
  90                 if err != nil {
  91                         return err
  92                 }
  93
  94                 // Rebase path
  95                 path, err = filepath.Rel(rw, path)
  96                 if err != nil {
  97                         return err
  98                 }
  99
 100                 // As this runs on the daemon side, file paths are OS specific.
 101                 path = filepath.Join(string(os.PathSeparator), path)
 102
 103                 // Skip root
 104                 if path == string(os.PathSeparator) {
 105                         return nil
 106                 }
 107
 108                 // Skip AUFS metadata
 109                 if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched {
 110                         return err
 111                 }
 112
 113                 change := Change{
 114                         Path: path,
 115                 }
 116
 117                 // Find out what kind of modification happened
 118                 file := filepath.Base(path)
 119                 // If there is a whiteout, then the file was removed
 120                 if strings.HasPrefix(file, WhiteoutPrefix) {
 121                         originalFile := file[len(WhiteoutPrefix):]
 122                         change.Path = filepath.Join(filepath.Dir(path), originalFile)
 123                         change.Kind = ChangeDelete
 124                 } else {
 125                         // Otherwise, the file was added
 126                         change.Kind = ChangeAdd
 127
 128                         // ...Unless it already existed in a top layer, in which case, it's a modification
 129                         for _, layer := range layers {
 130                                 stat, err := os.Stat(filepath.Join(layer, path))
 131                                 if err != nil && !os.IsNotExist(err) {
 132                                         return err
 133                                 }
 134                                 if err == nil {
 135                                         // The file existed in the top layer, so that's a modification
 136
 137                                         // However, if it's a directory, maybe it wasn't actually modified.
 138                                         // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
 139                                         if stat.IsDir() && f.IsDir() {
 140                                                 if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
 141                                                         // Both directories are the same, don't record the change
 142                                                         return nil
 143                                                 }
 144                                         }
 145                                         change.Kind = ChangeModify
 146                                         break
 147                                 }
 148                         }
 149                 }
 150
 151                 // If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
 152                 // This block is here to ensure the change is recorded even if the
 153                 // modify time, mode and size of the parent directory in the rw and ro layers are all equal.
 154                 // Check https://github.com/docker/docker/pull/13590 for details.
 155                 if f.IsDir() {
 156                         changedDirs[path] = struct{}{}
 157                 }
 158                 if change.Kind == ChangeAdd || change.Kind == ChangeDelete {
 159                         parent := filepath.Dir(path)
 160                         if _, ok := changedDirs[parent]; !ok && parent != "/" {
 161                                 changes = append(changes, Change{Path: parent, Kind: ChangeModify})
 162                                 changedDirs[parent] = struct{}{}
 163                         }
 164                 }
 165
 166                 // Record change
 167                 changes = append(changes, change)
 168                 return nil
 169         })
 170         if err != nil && !os.IsNotExist(err) {
 171                 return nil, err
 172         }
 173         return changes, nil
 174 }
 175
 176 // FileInfo describes the information of a file.
 177 type FileInfo struct {
 178         parent     *FileInfo
 179         name       string
 180         stat       *system.StatT
 181         children   map[string]*FileInfo
 182         capability []byte
 183         added      bool
 184 }
 185
 186 // LookUp looks up the file information of a file.
 187 func (info *FileInfo) LookUp(path string) *FileInfo {
 188         // As this runs on the daemon side, file paths are OS specific.
 189         parent := info
 190         if path == string(os.PathSeparator) {
 191                 return info
 192         }
 193
 194         pathElements := strings.Split(path, string(os.PathSeparator))
 195         for _, elem := range pathElements {
 196                 if elem != "" {
 197                         child := parent.children[elem]
 198                         if child == nil {
 199                                 return nil
 200                         }
 201                         parent = child
 202                 }
 203         }
 204         return parent
 205 }
 206
 207 func (info *FileInfo) path() string {
 208         if info.parent == nil {
 209                 // As this runs on the daemon side, file paths are OS specific.
 210                 return string(os.PathSeparator)
 211         }
 212         return filepath.Join(info.parent.path(), info.name)
 213 }
 214
 215 func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
 216
 217         sizeAtEntry := len(*changes)
 218
 219         if oldInfo == nil {
 220                 // add
 221                 change := Change{
 222                         Path: info.path(),
 223                         Kind: ChangeAdd,
 224                 }
 225                 *changes = append(*changes, change)
 226                 info.added = true
 227         }
 228
 229         // We make a copy so we can modify it to detect additions
 230         // also, we only recurse on the old dir if the new info is a directory
 231         // otherwise any previous delete/change is considered recursive
 232         oldChildren := make(map[string]*FileInfo)
 233         if oldInfo != nil && info.isDir() {
 234                 for k, v := range oldInfo.children {
 235                         oldChildren[k] = v
 236                 }
 237         }
 238
 239         for name, newChild := range info.children {
 240                 oldChild, _ := oldChildren[name]
 241                 if oldChild != nil {
 242                         // change?
 243                         oldStat := oldChild.stat
 244                         newStat := newChild.stat
 245                         // Note: We can't compare inode or ctime or blocksize here, because these change
 246                         // when copying a file into a container. However, that is not generally a problem
 247                         // because any content change will change mtime, and any status change should
 248                         // be visible when actually comparing the stat fields. The only time this
 249                         // breaks down is if some code intentionally hides a change by setting
 250                         // back mtime
 251                         if statDifferent(oldStat, newStat) ||
 252                                 bytes.Compare(oldChild.capability, newChild.capability) != 0 {
 253                                 change := Change{
 254                                         Path: newChild.path(),
 255                                         Kind: ChangeModify,
 256                                 }
 257                                 *changes = append(*changes, change)
 258                                 newChild.added = true
 259                         }
 260
 261                         // Remove from copy so we can detect deletions
 262                         delete(oldChildren, name)
 263                 }
 264
 265                 newChild.addChanges(oldChild, changes)
 266         }
 267         for _, oldChild := range oldChildren {
 268                 // delete
 269                 change := Change{
 270                         Path: oldChild.path(),
 271                         Kind: ChangeDelete,
 272                 }
 273                 *changes = append(*changes, change)
 274         }
 275
 276         // If there were changes inside this directory, we need to add it, even if the directory
 277         // itself wasn't changed. This is needed to properly save and restore filesystem permissions.
 278         // As this runs on the daemon side, file paths are OS specific.
 279         if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != string(os.PathSeparator) {
 280                 change := Change{
 281                         Path: info.path(),
 282                         Kind: ChangeModify,
 283                 }
 284                 // Let's insert the directory entry before the recently added entries located inside this dir
 285                 *changes = append(*changes, change) // just to resize the slice, will be overwritten
 286                 copy((*changes)[sizeAtEntry+1:], (*changes)[sizeAtEntry:])
 287                 (*changes)[sizeAtEntry] = change
 288         }
 289
 290 }
 291
 292 // Changes add changes to file information.
 293 func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
 294         var changes []Change
 295
 296         info.addChanges(oldInfo, &changes)
 297
 298         return changes
 299 }
 300
 301 func newRootFileInfo() *FileInfo {
 302         // As this runs on the daemon side, file paths are OS specific.
 303         root := &FileInfo{
 304                 name:     string(os.PathSeparator),
 305                 children: make(map[string]*FileInfo),
 306         }
 307         return root
 308 }
 309
 310 // ChangesDirs compares two directories and generates an array of Change objects describing the changes.
 311 // If oldDir is "", then all files in newDir will be Add-Changes.
 312 func ChangesDirs(newDir, oldDir string) ([]Change, error) {
 313         var (
 314                 oldRoot, newRoot *FileInfo
 315         )
 316         if oldDir == "" {
 317                 emptyDir, err := ioutil.TempDir("", "empty")
 318                 if err != nil {
 319                         return nil, err
 320                 }
 321                 defer os.Remove(emptyDir)
 322                 oldDir = emptyDir
 323         }
 324         oldRoot, newRoot, err := collectFileInfoForChanges(oldDir, newDir)
 325         if err != nil {
 326                 return nil, err
 327         }
 328
 329         return newRoot.Changes(oldRoot), nil
 330 }
 331
 332 // ChangesSize calculates the size in bytes of the provided changes, based on newDir.
 333 func ChangesSize(newDir string, changes []Change) int64 {
 334         var (
 335                 size int64
 336                 sf   = make(map[uint64]struct{})
 337         )
 338         for _, change := range changes {
 339                 if change.Kind == ChangeModify || change.Kind == ChangeAdd {
 340                         file := filepath.Join(newDir, change.Path)
 341                         fileInfo, err := os.Lstat(file)
 342                         if err != nil {
 343                                 logrus.Errorf("Can not stat %q: %s", file, err)
 344                                 continue
 345                         }
 346
 347                         if fileInfo != nil && !fileInfo.IsDir() {
 348                                 if hasHardlinks(fileInfo) {
 349                                         inode := getIno(fileInfo)
 350                                         if _, ok := sf[inode]; !ok {
 351                                                 size += fileInfo.Size()
 352                                                 sf[inode] = struct{}{}
 353                                         }
 354                                 } else {
 355                                         size += fileInfo.Size()
 356                                 }
 357                         }
 358                 }
 359         }
 360         return size
 361 }
 362
 363 // ExportChanges produces an Archive from the provided changes, relative to dir.
 364 func ExportChanges(dir string, changes []Change, uidMaps, gidMaps []idtools.IDMap) (Archive, error) {
 365         reader, writer := io.Pipe()
 366         go func() {
 367                 ta := &tarAppender{
 368                         TarWriter: tar.NewWriter(writer),
 369                         Buffer:    pools.BufioWriter32KPool.Get(nil),
 370                         SeenFiles: make(map[uint64]string),
 371                         UIDMaps:   uidMaps,
 372                         GIDMaps:   gidMaps,
 373                 }
 374                 // this buffer is needed for the duration of this piped stream
 375                 defer pools.BufioWriter32KPool.Put(ta.Buffer)
 376
 377                 sort.Sort(changesByPath(changes))
 378
 379                 // In general we log errors here but ignore them because
 380                 // during e.g. a diff operation the container can continue
 381                 // mutating the filesystem and we can see transient errors
 382                 // from this
 383                 for _, change := range changes {
 384                         if change.Kind == ChangeDelete {
 385                                 whiteOutDir := filepath.Dir(change.Path)
 386                                 whiteOutBase := filepath.Base(change.Path)
 387                                 whiteOut := filepath.Join(whiteOutDir, WhiteoutPrefix+whiteOutBase)
 388                                 timestamp := time.Now()
 389                                 hdr := &tar.Header{
 390                                         Name:       whiteOut[1:],
 391                                         Size:       0,
 392                                         ModTime:    timestamp,
 393                                         AccessTime: timestamp,
 394                                         ChangeTime: timestamp,
 395                                 }
 396                                 if err := ta.TarWriter.WriteHeader(hdr); err != nil {
 397                                         logrus.Debugf("Can't write whiteout header: %s", err)
 398                                 }
 399                         } else {
 400                                 path := filepath.Join(dir, change.Path)
 401                                 if err := ta.addTarFile(path, change.Path[1:]); err != nil {
 402                                         logrus.Debugf("Can't add file %s to tar: %s", path, err)
 403                                 }
 404                         }
 405                 }
 406
 407                 // Make sure to check the error on Close.
 408                 if err := ta.TarWriter.Close(); err != nil {
 409                         logrus.Debugf("Can't close layer: %s", err)
 410                 }
 411                 if err := writer.Close(); err != nil {
 412                         logrus.Debugf("failed close Changes writer: %s", err)
 413                 }
 414         }()
 415         return reader, nil
 416 }