]>
Commit | Line | Data |
---|---|---|
9b12e4fe JC |
1 | package archive |
2 | ||
3 | import ( | |
4 | "archive/tar" | |
5 | "bytes" | |
6 | "fmt" | |
7 | "io" | |
8 | "io/ioutil" | |
9 | "os" | |
10 | "path/filepath" | |
11 | "sort" | |
12 | "strings" | |
13 | "syscall" | |
14 | "time" | |
15 | ||
16 | "github.com/fsouza/go-dockerclient/external/github.com/Sirupsen/logrus" | |
17 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/idtools" | |
18 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/pools" | |
19 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/system" | |
20 | ) | |
21 | ||
22 | // ChangeType represents the change type. | |
23 | type ChangeType int | |
24 | ||
25 | const ( | |
26 | // ChangeModify represents the modify operation. | |
27 | ChangeModify = iota | |
28 | // ChangeAdd represents the add operation. | |
29 | ChangeAdd | |
30 | // ChangeDelete represents the delete operation. | |
31 | ChangeDelete | |
32 | ) | |
33 | ||
34 | func (c ChangeType) String() string { | |
35 | switch c { | |
36 | case ChangeModify: | |
37 | return "C" | |
38 | case ChangeAdd: | |
39 | return "A" | |
40 | case ChangeDelete: | |
41 | return "D" | |
42 | } | |
43 | return "" | |
44 | } | |
45 | ||
46 | // Change represents a change, it wraps the change type and path. | |
47 | // It describes changes of the files in the path respect to the | |
48 | // parent layers. The change could be modify, add, delete. | |
49 | // This is used for layer diff. | |
50 | type Change struct { | |
51 | Path string | |
52 | Kind ChangeType | |
53 | } | |
54 | ||
55 | func (change *Change) String() string { | |
56 | return fmt.Sprintf("%s %s", change.Kind, change.Path) | |
57 | } | |
58 | ||
59 | // for sort.Sort | |
60 | type changesByPath []Change | |
61 | ||
62 | func (c changesByPath) Less(i, j int) bool { return c[i].Path < c[j].Path } | |
63 | func (c changesByPath) Len() int { return len(c) } | |
64 | func (c changesByPath) Swap(i, j int) { c[j], c[i] = c[i], c[j] } | |
65 | ||
66 | // Gnu tar and the go tar writer don't have sub-second mtime | |
67 | // precision, which is problematic when we apply changes via tar | |
68 | // files, we handle this by comparing for exact times, *or* same | |
69 | // second count and either a or b having exactly 0 nanoseconds | |
70 | func sameFsTime(a, b time.Time) bool { | |
71 | return a == b || | |
72 | (a.Unix() == b.Unix() && | |
73 | (a.Nanosecond() == 0 || b.Nanosecond() == 0)) | |
74 | } | |
75 | ||
76 | func sameFsTimeSpec(a, b syscall.Timespec) bool { | |
77 | return a.Sec == b.Sec && | |
78 | (a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0) | |
79 | } | |
80 | ||
81 | // Changes walks the path rw and determines changes for the files in the path, | |
82 | // with respect to the parent layers | |
83 | func Changes(layers []string, rw string) ([]Change, error) { | |
84 | var ( | |
85 | changes []Change | |
86 | changedDirs = make(map[string]struct{}) | |
87 | ) | |
88 | ||
89 | err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error { | |
90 | if err != nil { | |
91 | return err | |
92 | } | |
93 | ||
94 | // Rebase path | |
95 | path, err = filepath.Rel(rw, path) | |
96 | if err != nil { | |
97 | return err | |
98 | } | |
99 | ||
100 | // As this runs on the daemon side, file paths are OS specific. | |
101 | path = filepath.Join(string(os.PathSeparator), path) | |
102 | ||
103 | // Skip root | |
104 | if path == string(os.PathSeparator) { | |
105 | return nil | |
106 | } | |
107 | ||
108 | // Skip AUFS metadata | |
109 | if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched { | |
110 | return err | |
111 | } | |
112 | ||
113 | change := Change{ | |
114 | Path: path, | |
115 | } | |
116 | ||
117 | // Find out what kind of modification happened | |
118 | file := filepath.Base(path) | |
119 | // If there is a whiteout, then the file was removed | |
120 | if strings.HasPrefix(file, WhiteoutPrefix) { | |
121 | originalFile := file[len(WhiteoutPrefix):] | |
122 | change.Path = filepath.Join(filepath.Dir(path), originalFile) | |
123 | change.Kind = ChangeDelete | |
124 | } else { | |
125 | // Otherwise, the file was added | |
126 | change.Kind = ChangeAdd | |
127 | ||
128 | // ...Unless it already existed in a top layer, in which case, it's a modification | |
129 | for _, layer := range layers { | |
130 | stat, err := os.Stat(filepath.Join(layer, path)) | |
131 | if err != nil && !os.IsNotExist(err) { | |
132 | return err | |
133 | } | |
134 | if err == nil { | |
135 | // The file existed in the top layer, so that's a modification | |
136 | ||
137 | // However, if it's a directory, maybe it wasn't actually modified. | |
138 | // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar | |
139 | if stat.IsDir() && f.IsDir() { | |
140 | if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) { | |
141 | // Both directories are the same, don't record the change | |
142 | return nil | |
143 | } | |
144 | } | |
145 | change.Kind = ChangeModify | |
146 | break | |
147 | } | |
148 | } | |
149 | } | |
150 | ||
151 | // If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files. | |
152 | // This block is here to ensure the change is recorded even if the | |
153 | // modify time, mode and size of the parent directory in the rw and ro layers are all equal. | |
154 | // Check https://github.com/docker/docker/pull/13590 for details. | |
155 | if f.IsDir() { | |
156 | changedDirs[path] = struct{}{} | |
157 | } | |
158 | if change.Kind == ChangeAdd || change.Kind == ChangeDelete { | |
159 | parent := filepath.Dir(path) | |
160 | if _, ok := changedDirs[parent]; !ok && parent != "/" { | |
161 | changes = append(changes, Change{Path: parent, Kind: ChangeModify}) | |
162 | changedDirs[parent] = struct{}{} | |
163 | } | |
164 | } | |
165 | ||
166 | // Record change | |
167 | changes = append(changes, change) | |
168 | return nil | |
169 | }) | |
170 | if err != nil && !os.IsNotExist(err) { | |
171 | return nil, err | |
172 | } | |
173 | return changes, nil | |
174 | } | |
175 | ||
176 | // FileInfo describes the information of a file. | |
177 | type FileInfo struct { | |
178 | parent *FileInfo | |
179 | name string | |
180 | stat *system.StatT | |
181 | children map[string]*FileInfo | |
182 | capability []byte | |
183 | added bool | |
184 | } | |
185 | ||
186 | // LookUp looks up the file information of a file. | |
187 | func (info *FileInfo) LookUp(path string) *FileInfo { | |
188 | // As this runs on the daemon side, file paths are OS specific. | |
189 | parent := info | |
190 | if path == string(os.PathSeparator) { | |
191 | return info | |
192 | } | |
193 | ||
194 | pathElements := strings.Split(path, string(os.PathSeparator)) | |
195 | for _, elem := range pathElements { | |
196 | if elem != "" { | |
197 | child := parent.children[elem] | |
198 | if child == nil { | |
199 | return nil | |
200 | } | |
201 | parent = child | |
202 | } | |
203 | } | |
204 | return parent | |
205 | } | |
206 | ||
207 | func (info *FileInfo) path() string { | |
208 | if info.parent == nil { | |
209 | // As this runs on the daemon side, file paths are OS specific. | |
210 | return string(os.PathSeparator) | |
211 | } | |
212 | return filepath.Join(info.parent.path(), info.name) | |
213 | } | |
214 | ||
215 | func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) { | |
216 | ||
217 | sizeAtEntry := len(*changes) | |
218 | ||
219 | if oldInfo == nil { | |
220 | // add | |
221 | change := Change{ | |
222 | Path: info.path(), | |
223 | Kind: ChangeAdd, | |
224 | } | |
225 | *changes = append(*changes, change) | |
226 | info.added = true | |
227 | } | |
228 | ||
229 | // We make a copy so we can modify it to detect additions | |
230 | // also, we only recurse on the old dir if the new info is a directory | |
231 | // otherwise any previous delete/change is considered recursive | |
232 | oldChildren := make(map[string]*FileInfo) | |
233 | if oldInfo != nil && info.isDir() { | |
234 | for k, v := range oldInfo.children { | |
235 | oldChildren[k] = v | |
236 | } | |
237 | } | |
238 | ||
239 | for name, newChild := range info.children { | |
240 | oldChild, _ := oldChildren[name] | |
241 | if oldChild != nil { | |
242 | // change? | |
243 | oldStat := oldChild.stat | |
244 | newStat := newChild.stat | |
245 | // Note: We can't compare inode or ctime or blocksize here, because these change | |
246 | // when copying a file into a container. However, that is not generally a problem | |
247 | // because any content change will change mtime, and any status change should | |
248 | // be visible when actually comparing the stat fields. The only time this | |
249 | // breaks down is if some code intentionally hides a change by setting | |
250 | // back mtime | |
251 | if statDifferent(oldStat, newStat) || | |
252 | bytes.Compare(oldChild.capability, newChild.capability) != 0 { | |
253 | change := Change{ | |
254 | Path: newChild.path(), | |
255 | Kind: ChangeModify, | |
256 | } | |
257 | *changes = append(*changes, change) | |
258 | newChild.added = true | |
259 | } | |
260 | ||
261 | // Remove from copy so we can detect deletions | |
262 | delete(oldChildren, name) | |
263 | } | |
264 | ||
265 | newChild.addChanges(oldChild, changes) | |
266 | } | |
267 | for _, oldChild := range oldChildren { | |
268 | // delete | |
269 | change := Change{ | |
270 | Path: oldChild.path(), | |
271 | Kind: ChangeDelete, | |
272 | } | |
273 | *changes = append(*changes, change) | |
274 | } | |
275 | ||
276 | // If there were changes inside this directory, we need to add it, even if the directory | |
277 | // itself wasn't changed. This is needed to properly save and restore filesystem permissions. | |
278 | // As this runs on the daemon side, file paths are OS specific. | |
279 | if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != string(os.PathSeparator) { | |
280 | change := Change{ | |
281 | Path: info.path(), | |
282 | Kind: ChangeModify, | |
283 | } | |
284 | // Let's insert the directory entry before the recently added entries located inside this dir | |
285 | *changes = append(*changes, change) // just to resize the slice, will be overwritten | |
286 | copy((*changes)[sizeAtEntry+1:], (*changes)[sizeAtEntry:]) | |
287 | (*changes)[sizeAtEntry] = change | |
288 | } | |
289 | ||
290 | } | |
291 | ||
292 | // Changes add changes to file information. | |
293 | func (info *FileInfo) Changes(oldInfo *FileInfo) []Change { | |
294 | var changes []Change | |
295 | ||
296 | info.addChanges(oldInfo, &changes) | |
297 | ||
298 | return changes | |
299 | } | |
300 | ||
301 | func newRootFileInfo() *FileInfo { | |
302 | // As this runs on the daemon side, file paths are OS specific. | |
303 | root := &FileInfo{ | |
304 | name: string(os.PathSeparator), | |
305 | children: make(map[string]*FileInfo), | |
306 | } | |
307 | return root | |
308 | } | |
309 | ||
310 | // ChangesDirs compares two directories and generates an array of Change objects describing the changes. | |
311 | // If oldDir is "", then all files in newDir will be Add-Changes. | |
312 | func ChangesDirs(newDir, oldDir string) ([]Change, error) { | |
313 | var ( | |
314 | oldRoot, newRoot *FileInfo | |
315 | ) | |
316 | if oldDir == "" { | |
317 | emptyDir, err := ioutil.TempDir("", "empty") | |
318 | if err != nil { | |
319 | return nil, err | |
320 | } | |
321 | defer os.Remove(emptyDir) | |
322 | oldDir = emptyDir | |
323 | } | |
324 | oldRoot, newRoot, err := collectFileInfoForChanges(oldDir, newDir) | |
325 | if err != nil { | |
326 | return nil, err | |
327 | } | |
328 | ||
329 | return newRoot.Changes(oldRoot), nil | |
330 | } | |
331 | ||
332 | // ChangesSize calculates the size in bytes of the provided changes, based on newDir. | |
333 | func ChangesSize(newDir string, changes []Change) int64 { | |
334 | var ( | |
335 | size int64 | |
336 | sf = make(map[uint64]struct{}) | |
337 | ) | |
338 | for _, change := range changes { | |
339 | if change.Kind == ChangeModify || change.Kind == ChangeAdd { | |
340 | file := filepath.Join(newDir, change.Path) | |
341 | fileInfo, err := os.Lstat(file) | |
342 | if err != nil { | |
343 | logrus.Errorf("Can not stat %q: %s", file, err) | |
344 | continue | |
345 | } | |
346 | ||
347 | if fileInfo != nil && !fileInfo.IsDir() { | |
348 | if hasHardlinks(fileInfo) { | |
349 | inode := getIno(fileInfo) | |
350 | if _, ok := sf[inode]; !ok { | |
351 | size += fileInfo.Size() | |
352 | sf[inode] = struct{}{} | |
353 | } | |
354 | } else { | |
355 | size += fileInfo.Size() | |
356 | } | |
357 | } | |
358 | } | |
359 | } | |
360 | return size | |
361 | } | |
362 | ||
363 | // ExportChanges produces an Archive from the provided changes, relative to dir. | |
364 | func ExportChanges(dir string, changes []Change, uidMaps, gidMaps []idtools.IDMap) (Archive, error) { | |
365 | reader, writer := io.Pipe() | |
366 | go func() { | |
367 | ta := &tarAppender{ | |
368 | TarWriter: tar.NewWriter(writer), | |
369 | Buffer: pools.BufioWriter32KPool.Get(nil), | |
370 | SeenFiles: make(map[uint64]string), | |
371 | UIDMaps: uidMaps, | |
372 | GIDMaps: gidMaps, | |
373 | } | |
374 | // this buffer is needed for the duration of this piped stream | |
375 | defer pools.BufioWriter32KPool.Put(ta.Buffer) | |
376 | ||
377 | sort.Sort(changesByPath(changes)) | |
378 | ||
379 | // In general we log errors here but ignore them because | |
380 | // during e.g. a diff operation the container can continue | |
381 | // mutating the filesystem and we can see transient errors | |
382 | // from this | |
383 | for _, change := range changes { | |
384 | if change.Kind == ChangeDelete { | |
385 | whiteOutDir := filepath.Dir(change.Path) | |
386 | whiteOutBase := filepath.Base(change.Path) | |
387 | whiteOut := filepath.Join(whiteOutDir, WhiteoutPrefix+whiteOutBase) | |
388 | timestamp := time.Now() | |
389 | hdr := &tar.Header{ | |
390 | Name: whiteOut[1:], | |
391 | Size: 0, | |
392 | ModTime: timestamp, | |
393 | AccessTime: timestamp, | |
394 | ChangeTime: timestamp, | |
395 | } | |
396 | if err := ta.TarWriter.WriteHeader(hdr); err != nil { | |
397 | logrus.Debugf("Can't write whiteout header: %s", err) | |
398 | } | |
399 | } else { | |
400 | path := filepath.Join(dir, change.Path) | |
401 | if err := ta.addTarFile(path, change.Path[1:]); err != nil { | |
402 | logrus.Debugf("Can't add file %s to tar: %s", path, err) | |
403 | } | |
404 | } | |
405 | } | |
406 | ||
407 | // Make sure to check the error on Close. | |
408 | if err := ta.TarWriter.Close(); err != nil { | |
409 | logrus.Debugf("Can't close layer: %s", err) | |
410 | } | |
411 | if err := writer.Close(); err != nil { | |
412 | logrus.Debugf("failed close Changes writer: %s", err) | |
413 | } | |
414 | }() | |
415 | return reader, nil | |
416 | } |