diff options
Diffstat (limited to 'vendor/github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/archive/archive.go')
-rw-r--r-- | vendor/github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/archive/archive.go | 1049 |
1 files changed, 0 insertions, 1049 deletions
diff --git a/vendor/github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/archive/archive.go b/vendor/github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/archive/archive.go deleted file mode 100644 index ce84347..0000000 --- a/vendor/github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/archive/archive.go +++ /dev/null | |||
@@ -1,1049 +0,0 @@ | |||
1 | package archive | ||
2 | |||
3 | import ( | ||
4 | "archive/tar" | ||
5 | "bufio" | ||
6 | "bytes" | ||
7 | "compress/bzip2" | ||
8 | "compress/gzip" | ||
9 | "errors" | ||
10 | "fmt" | ||
11 | "io" | ||
12 | "io/ioutil" | ||
13 | "os" | ||
14 | "os/exec" | ||
15 | "path/filepath" | ||
16 | "runtime" | ||
17 | "strings" | ||
18 | "syscall" | ||
19 | |||
20 | "github.com/fsouza/go-dockerclient/external/github.com/Sirupsen/logrus" | ||
21 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/fileutils" | ||
22 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/idtools" | ||
23 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/ioutils" | ||
24 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/pools" | ||
25 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/promise" | ||
26 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/system" | ||
27 | ) | ||
28 | |||
29 | type ( | ||
30 | // Archive is a type of io.ReadCloser which has two interfaces Read and Closer. | ||
31 | Archive io.ReadCloser | ||
32 | // Reader is a type of io.Reader. | ||
33 | Reader io.Reader | ||
34 | // Compression is the state represents if compressed or not. | ||
35 | Compression int | ||
36 | // TarChownOptions wraps the chown options UID and GID. | ||
37 | TarChownOptions struct { | ||
38 | UID, GID int | ||
39 | } | ||
40 | // TarOptions wraps the tar options. | ||
41 | TarOptions struct { | ||
42 | IncludeFiles []string | ||
43 | ExcludePatterns []string | ||
44 | Compression Compression | ||
45 | NoLchown bool | ||
46 | UIDMaps []idtools.IDMap | ||
47 | GIDMaps []idtools.IDMap | ||
48 | ChownOpts *TarChownOptions | ||
49 | IncludeSourceDir bool | ||
50 | // When unpacking, specifies whether overwriting a directory with a | ||
51 | // non-directory is allowed and vice versa. | ||
52 | NoOverwriteDirNonDir bool | ||
53 | // For each include when creating an archive, the included name will be | ||
54 | // replaced with the matching name from this map. | ||
55 | RebaseNames map[string]string | ||
56 | } | ||
57 | |||
58 | // Archiver allows the reuse of most utility functions of this package | ||
59 | // with a pluggable Untar function. Also, to facilitate the passing of | ||
60 | // specific id mappings for untar, an archiver can be created with maps | ||
61 | // which will then be passed to Untar operations | ||
62 | Archiver struct { | ||
63 | Untar func(io.Reader, string, *TarOptions) error | ||
64 | UIDMaps []idtools.IDMap | ||
65 | GIDMaps []idtools.IDMap | ||
66 | } | ||
67 | |||
68 | // breakoutError is used to differentiate errors related to breaking out | ||
69 | // When testing archive breakout in the unit tests, this error is expected | ||
70 | // in order for the test to pass. | ||
71 | breakoutError error | ||
72 | ) | ||
73 | |||
74 | var ( | ||
75 | // ErrNotImplemented is the error message of function not implemented. | ||
76 | ErrNotImplemented = errors.New("Function not implemented") | ||
77 | defaultArchiver = &Archiver{Untar: Untar, UIDMaps: nil, GIDMaps: nil} | ||
78 | ) | ||
79 | |||
80 | const ( | ||
81 | // HeaderSize is the size in bytes of a tar header | ||
82 | HeaderSize = 512 | ||
83 | ) | ||
84 | |||
85 | const ( | ||
86 | // Uncompressed represents the uncompressed. | ||
87 | Uncompressed Compression = iota | ||
88 | // Bzip2 is bzip2 compression algorithm. | ||
89 | Bzip2 | ||
90 | // Gzip is gzip compression algorithm. | ||
91 | Gzip | ||
92 | // Xz is xz compression algorithm. | ||
93 | Xz | ||
94 | ) | ||
95 | |||
96 | // IsArchive checks for the magic bytes of a tar or any supported compression | ||
97 | // algorithm. | ||
98 | func IsArchive(header []byte) bool { | ||
99 | compression := DetectCompression(header) | ||
100 | if compression != Uncompressed { | ||
101 | return true | ||
102 | } | ||
103 | r := tar.NewReader(bytes.NewBuffer(header)) | ||
104 | _, err := r.Next() | ||
105 | return err == nil | ||
106 | } | ||
107 | |||
108 | // IsArchivePath checks if the (possibly compressed) file at the given path | ||
109 | // starts with a tar file header. | ||
110 | func IsArchivePath(path string) bool { | ||
111 | file, err := os.Open(path) | ||
112 | if err != nil { | ||
113 | return false | ||
114 | } | ||
115 | defer file.Close() | ||
116 | rdr, err := DecompressStream(file) | ||
117 | if err != nil { | ||
118 | return false | ||
119 | } | ||
120 | r := tar.NewReader(rdr) | ||
121 | _, err = r.Next() | ||
122 | return err == nil | ||
123 | } | ||
124 | |||
125 | // DetectCompression detects the compression algorithm of the source. | ||
126 | func DetectCompression(source []byte) Compression { | ||
127 | for compression, m := range map[Compression][]byte{ | ||
128 | Bzip2: {0x42, 0x5A, 0x68}, | ||
129 | Gzip: {0x1F, 0x8B, 0x08}, | ||
130 | Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, | ||
131 | } { | ||
132 | if len(source) < len(m) { | ||
133 | logrus.Debugf("Len too short") | ||
134 | continue | ||
135 | } | ||
136 | if bytes.Compare(m, source[:len(m)]) == 0 { | ||
137 | return compression | ||
138 | } | ||
139 | } | ||
140 | return Uncompressed | ||
141 | } | ||
142 | |||
143 | func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) { | ||
144 | args := []string{"xz", "-d", "-c", "-q"} | ||
145 | |||
146 | return cmdStream(exec.Command(args[0], args[1:]...), archive) | ||
147 | } | ||
148 | |||
149 | // DecompressStream decompress the archive and returns a ReaderCloser with the decompressed archive. | ||
150 | func DecompressStream(archive io.Reader) (io.ReadCloser, error) { | ||
151 | p := pools.BufioReader32KPool | ||
152 | buf := p.Get(archive) | ||
153 | bs, err := buf.Peek(10) | ||
154 | if err != nil && err != io.EOF { | ||
155 | // Note: we'll ignore any io.EOF error because there are some odd | ||
156 | // cases where the layer.tar file will be empty (zero bytes) and | ||
157 | // that results in an io.EOF from the Peek() call. So, in those | ||
158 | // cases we'll just treat it as a non-compressed stream and | ||
159 | // that means just create an empty layer. | ||
160 | // See Issue 18170 | ||
161 | return nil, err | ||
162 | } | ||
163 | |||
164 | compression := DetectCompression(bs) | ||
165 | switch compression { | ||
166 | case Uncompressed: | ||
167 | readBufWrapper := p.NewReadCloserWrapper(buf, buf) | ||
168 | return readBufWrapper, nil | ||
169 | case Gzip: | ||
170 | gzReader, err := gzip.NewReader(buf) | ||
171 | if err != nil { | ||
172 | return nil, err | ||
173 | } | ||
174 | readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) | ||
175 | return readBufWrapper, nil | ||
176 | case Bzip2: | ||
177 | bz2Reader := bzip2.NewReader(buf) | ||
178 | readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) | ||
179 | return readBufWrapper, nil | ||
180 | case Xz: | ||
181 | xzReader, chdone, err := xzDecompress(buf) | ||
182 | if err != nil { | ||
183 | return nil, err | ||
184 | } | ||
185 | readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) | ||
186 | return ioutils.NewReadCloserWrapper(readBufWrapper, func() error { | ||
187 | <-chdone | ||
188 | return readBufWrapper.Close() | ||
189 | }), nil | ||
190 | default: | ||
191 | return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) | ||
192 | } | ||
193 | } | ||
194 | |||
195 | // CompressStream compresses the dest with specified compression algorithm. | ||
196 | func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) { | ||
197 | p := pools.BufioWriter32KPool | ||
198 | buf := p.Get(dest) | ||
199 | switch compression { | ||
200 | case Uncompressed: | ||
201 | writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) | ||
202 | return writeBufWrapper, nil | ||
203 | case Gzip: | ||
204 | gzWriter := gzip.NewWriter(dest) | ||
205 | writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) | ||
206 | return writeBufWrapper, nil | ||
207 | case Bzip2, Xz: | ||
208 | // archive/bzip2 does not support writing, and there is no xz support at all | ||
209 | // However, this is not a problem as docker only currently generates gzipped tars | ||
210 | return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) | ||
211 | default: | ||
212 | return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) | ||
213 | } | ||
214 | } | ||
215 | |||
216 | // Extension returns the extension of a file that uses the specified compression algorithm. | ||
217 | func (compression *Compression) Extension() string { | ||
218 | switch *compression { | ||
219 | case Uncompressed: | ||
220 | return "tar" | ||
221 | case Bzip2: | ||
222 | return "tar.bz2" | ||
223 | case Gzip: | ||
224 | return "tar.gz" | ||
225 | case Xz: | ||
226 | return "tar.xz" | ||
227 | } | ||
228 | return "" | ||
229 | } | ||
230 | |||
231 | type tarAppender struct { | ||
232 | TarWriter *tar.Writer | ||
233 | Buffer *bufio.Writer | ||
234 | |||
235 | // for hardlink mapping | ||
236 | SeenFiles map[uint64]string | ||
237 | UIDMaps []idtools.IDMap | ||
238 | GIDMaps []idtools.IDMap | ||
239 | } | ||
240 | |||
241 | // canonicalTarName provides a platform-independent and consistent posix-style | ||
242 | //path for files and directories to be archived regardless of the platform. | ||
243 | func canonicalTarName(name string, isDir bool) (string, error) { | ||
244 | name, err := CanonicalTarNameForPath(name) | ||
245 | if err != nil { | ||
246 | return "", err | ||
247 | } | ||
248 | |||
249 | // suffix with '/' for directories | ||
250 | if isDir && !strings.HasSuffix(name, "/") { | ||
251 | name += "/" | ||
252 | } | ||
253 | return name, nil | ||
254 | } | ||
255 | |||
256 | func (ta *tarAppender) addTarFile(path, name string) error { | ||
257 | fi, err := os.Lstat(path) | ||
258 | if err != nil { | ||
259 | return err | ||
260 | } | ||
261 | |||
262 | link := "" | ||
263 | if fi.Mode()&os.ModeSymlink != 0 { | ||
264 | if link, err = os.Readlink(path); err != nil { | ||
265 | return err | ||
266 | } | ||
267 | } | ||
268 | |||
269 | hdr, err := tar.FileInfoHeader(fi, link) | ||
270 | if err != nil { | ||
271 | return err | ||
272 | } | ||
273 | hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) | ||
274 | |||
275 | name, err = canonicalTarName(name, fi.IsDir()) | ||
276 | if err != nil { | ||
277 | return fmt.Errorf("tar: cannot canonicalize path: %v", err) | ||
278 | } | ||
279 | hdr.Name = name | ||
280 | |||
281 | inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys()) | ||
282 | if err != nil { | ||
283 | return err | ||
284 | } | ||
285 | |||
286 | // if it's not a directory and has more than 1 link, | ||
287 | // it's hardlinked, so set the type flag accordingly | ||
288 | if !fi.IsDir() && hasHardlinks(fi) { | ||
289 | // a link should have a name that it links too | ||
290 | // and that linked name should be first in the tar archive | ||
291 | if oldpath, ok := ta.SeenFiles[inode]; ok { | ||
292 | hdr.Typeflag = tar.TypeLink | ||
293 | hdr.Linkname = oldpath | ||
294 | hdr.Size = 0 // This Must be here for the writer math to add up! | ||
295 | } else { | ||
296 | ta.SeenFiles[inode] = name | ||
297 | } | ||
298 | } | ||
299 | |||
300 | capability, _ := system.Lgetxattr(path, "security.capability") | ||
301 | if capability != nil { | ||
302 | hdr.Xattrs = make(map[string]string) | ||
303 | hdr.Xattrs["security.capability"] = string(capability) | ||
304 | } | ||
305 | |||
306 | //handle re-mapping container ID mappings back to host ID mappings before | ||
307 | //writing tar headers/files. We skip whiteout files because they were written | ||
308 | //by the kernel and already have proper ownership relative to the host | ||
309 | if !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && (ta.UIDMaps != nil || ta.GIDMaps != nil) { | ||
310 | uid, gid, err := getFileUIDGID(fi.Sys()) | ||
311 | if err != nil { | ||
312 | return err | ||
313 | } | ||
314 | xUID, err := idtools.ToContainer(uid, ta.UIDMaps) | ||
315 | if err != nil { | ||
316 | return err | ||
317 | } | ||
318 | xGID, err := idtools.ToContainer(gid, ta.GIDMaps) | ||
319 | if err != nil { | ||
320 | return err | ||
321 | } | ||
322 | hdr.Uid = xUID | ||
323 | hdr.Gid = xGID | ||
324 | } | ||
325 | |||
326 | if err := ta.TarWriter.WriteHeader(hdr); err != nil { | ||
327 | return err | ||
328 | } | ||
329 | |||
330 | if hdr.Typeflag == tar.TypeReg { | ||
331 | file, err := os.Open(path) | ||
332 | if err != nil { | ||
333 | return err | ||
334 | } | ||
335 | |||
336 | ta.Buffer.Reset(ta.TarWriter) | ||
337 | defer ta.Buffer.Reset(nil) | ||
338 | _, err = io.Copy(ta.Buffer, file) | ||
339 | file.Close() | ||
340 | if err != nil { | ||
341 | return err | ||
342 | } | ||
343 | err = ta.Buffer.Flush() | ||
344 | if err != nil { | ||
345 | return err | ||
346 | } | ||
347 | } | ||
348 | |||
349 | return nil | ||
350 | } | ||
351 | |||
352 | func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *TarChownOptions) error { | ||
353 | // hdr.Mode is in linux format, which we can use for sycalls, | ||
354 | // but for os.Foo() calls we need the mode converted to os.FileMode, | ||
355 | // so use hdrInfo.Mode() (they differ for e.g. setuid bits) | ||
356 | hdrInfo := hdr.FileInfo() | ||
357 | |||
358 | switch hdr.Typeflag { | ||
359 | case tar.TypeDir: | ||
360 | // Create directory unless it exists as a directory already. | ||
361 | // In that case we just want to merge the two | ||
362 | if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { | ||
363 | if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { | ||
364 | return err | ||
365 | } | ||
366 | } | ||
367 | |||
368 | case tar.TypeReg, tar.TypeRegA: | ||
369 | // Source is regular file | ||
370 | file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) | ||
371 | if err != nil { | ||
372 | return err | ||
373 | } | ||
374 | if _, err := io.Copy(file, reader); err != nil { | ||
375 | file.Close() | ||
376 | return err | ||
377 | } | ||
378 | file.Close() | ||
379 | |||
380 | case tar.TypeBlock, tar.TypeChar, tar.TypeFifo: | ||
381 | // Handle this is an OS-specific way | ||
382 | if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { | ||
383 | return err | ||
384 | } | ||
385 | |||
386 | case tar.TypeLink: | ||
387 | targetPath := filepath.Join(extractDir, hdr.Linkname) | ||
388 | // check for hardlink breakout | ||
389 | if !strings.HasPrefix(targetPath, extractDir) { | ||
390 | return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) | ||
391 | } | ||
392 | if err := os.Link(targetPath, path); err != nil { | ||
393 | return err | ||
394 | } | ||
395 | |||
396 | case tar.TypeSymlink: | ||
397 | // path -> hdr.Linkname = targetPath | ||
398 | // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file | ||
399 | targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) | ||
400 | |||
401 | // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because | ||
402 | // that symlink would first have to be created, which would be caught earlier, at this very check: | ||
403 | if !strings.HasPrefix(targetPath, extractDir) { | ||
404 | return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) | ||
405 | } | ||
406 | if err := os.Symlink(hdr.Linkname, path); err != nil { | ||
407 | return err | ||
408 | } | ||
409 | |||
410 | case tar.TypeXGlobalHeader: | ||
411 | logrus.Debugf("PAX Global Extended Headers found and ignored") | ||
412 | return nil | ||
413 | |||
414 | default: | ||
415 | return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag) | ||
416 | } | ||
417 | |||
418 | // Lchown is not supported on Windows. | ||
419 | if Lchown && runtime.GOOS != "windows" { | ||
420 | if chownOpts == nil { | ||
421 | chownOpts = &TarChownOptions{UID: hdr.Uid, GID: hdr.Gid} | ||
422 | } | ||
423 | if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { | ||
424 | return err | ||
425 | } | ||
426 | } | ||
427 | |||
428 | for key, value := range hdr.Xattrs { | ||
429 | if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { | ||
430 | return err | ||
431 | } | ||
432 | } | ||
433 | |||
434 | // There is no LChmod, so ignore mode for symlink. Also, this | ||
435 | // must happen after chown, as that can modify the file mode | ||
436 | if err := handleLChmod(hdr, path, hdrInfo); err != nil { | ||
437 | return err | ||
438 | } | ||
439 | |||
440 | aTime := hdr.AccessTime | ||
441 | if aTime.Before(hdr.ModTime) { | ||
442 | // Last access time should never be before last modified time. | ||
443 | aTime = hdr.ModTime | ||
444 | } | ||
445 | |||
446 | // system.Chtimes doesn't support a NOFOLLOW flag atm | ||
447 | if hdr.Typeflag == tar.TypeLink { | ||
448 | if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { | ||
449 | if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { | ||
450 | return err | ||
451 | } | ||
452 | } | ||
453 | } else if hdr.Typeflag != tar.TypeSymlink { | ||
454 | if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { | ||
455 | return err | ||
456 | } | ||
457 | } else { | ||
458 | ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} | ||
459 | if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { | ||
460 | return err | ||
461 | } | ||
462 | } | ||
463 | return nil | ||
464 | } | ||
465 | |||
466 | // Tar creates an archive from the directory at `path`, and returns it as a | ||
467 | // stream of bytes. | ||
468 | func Tar(path string, compression Compression) (io.ReadCloser, error) { | ||
469 | return TarWithOptions(path, &TarOptions{Compression: compression}) | ||
470 | } | ||
471 | |||
472 | // TarWithOptions creates an archive from the directory at `path`, only including files whose relative | ||
473 | // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. | ||
474 | func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { | ||
475 | |||
476 | // Fix the source path to work with long path names. This is a no-op | ||
477 | // on platforms other than Windows. | ||
478 | srcPath = fixVolumePathPrefix(srcPath) | ||
479 | |||
480 | patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns) | ||
481 | |||
482 | if err != nil { | ||
483 | return nil, err | ||
484 | } | ||
485 | |||
486 | pipeReader, pipeWriter := io.Pipe() | ||
487 | |||
488 | compressWriter, err := CompressStream(pipeWriter, options.Compression) | ||
489 | if err != nil { | ||
490 | return nil, err | ||
491 | } | ||
492 | |||
493 | go func() { | ||
494 | ta := &tarAppender{ | ||
495 | TarWriter: tar.NewWriter(compressWriter), | ||
496 | Buffer: pools.BufioWriter32KPool.Get(nil), | ||
497 | SeenFiles: make(map[uint64]string), | ||
498 | UIDMaps: options.UIDMaps, | ||
499 | GIDMaps: options.GIDMaps, | ||
500 | } | ||
501 | |||
502 | defer func() { | ||
503 | // Make sure to check the error on Close. | ||
504 | if err := ta.TarWriter.Close(); err != nil { | ||
505 | logrus.Debugf("Can't close tar writer: %s", err) | ||
506 | } | ||
507 | if err := compressWriter.Close(); err != nil { | ||
508 | logrus.Debugf("Can't close compress writer: %s", err) | ||
509 | } | ||
510 | if err := pipeWriter.Close(); err != nil { | ||
511 | logrus.Debugf("Can't close pipe writer: %s", err) | ||
512 | } | ||
513 | }() | ||
514 | |||
515 | // this buffer is needed for the duration of this piped stream | ||
516 | defer pools.BufioWriter32KPool.Put(ta.Buffer) | ||
517 | |||
518 | // In general we log errors here but ignore them because | ||
519 | // during e.g. a diff operation the container can continue | ||
520 | // mutating the filesystem and we can see transient errors | ||
521 | // from this | ||
522 | |||
523 | stat, err := os.Lstat(srcPath) | ||
524 | if err != nil { | ||
525 | return | ||
526 | } | ||
527 | |||
528 | if !stat.IsDir() { | ||
529 | // We can't later join a non-dir with any includes because the | ||
530 | // 'walk' will error if "file/." is stat-ed and "file" is not a | ||
531 | // directory. So, we must split the source path and use the | ||
532 | // basename as the include. | ||
533 | if len(options.IncludeFiles) > 0 { | ||
534 | logrus.Warn("Tar: Can't archive a file with includes") | ||
535 | } | ||
536 | |||
537 | dir, base := SplitPathDirEntry(srcPath) | ||
538 | srcPath = dir | ||
539 | options.IncludeFiles = []string{base} | ||
540 | } | ||
541 | |||
542 | if len(options.IncludeFiles) == 0 { | ||
543 | options.IncludeFiles = []string{"."} | ||
544 | } | ||
545 | |||
546 | seen := make(map[string]bool) | ||
547 | |||
548 | for _, include := range options.IncludeFiles { | ||
549 | rebaseName := options.RebaseNames[include] | ||
550 | |||
551 | walkRoot := getWalkRoot(srcPath, include) | ||
552 | filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { | ||
553 | if err != nil { | ||
554 | logrus.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err) | ||
555 | return nil | ||
556 | } | ||
557 | |||
558 | relFilePath, err := filepath.Rel(srcPath, filePath) | ||
559 | if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) { | ||
560 | // Error getting relative path OR we are looking | ||
561 | // at the source directory path. Skip in both situations. | ||
562 | return nil | ||
563 | } | ||
564 | |||
565 | if options.IncludeSourceDir && include == "." && relFilePath != "." { | ||
566 | relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator)) | ||
567 | } | ||
568 | |||
569 | skip := false | ||
570 | |||
571 | // If "include" is an exact match for the current file | ||
572 | // then even if there's an "excludePatterns" pattern that | ||
573 | // matches it, don't skip it. IOW, assume an explicit 'include' | ||
574 | // is asking for that file no matter what - which is true | ||
575 | // for some files, like .dockerignore and Dockerfile (sometimes) | ||
576 | if include != relFilePath { | ||
577 | skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs) | ||
578 | if err != nil { | ||
579 | logrus.Debugf("Error matching %s: %v", relFilePath, err) | ||
580 | return err | ||
581 | } | ||
582 | } | ||
583 | |||
584 | if skip { | ||
585 | if !exceptions && f.IsDir() { | ||
586 | return filepath.SkipDir | ||
587 | } | ||
588 | return nil | ||
589 | } | ||
590 | |||
591 | if seen[relFilePath] { | ||
592 | return nil | ||
593 | } | ||
594 | seen[relFilePath] = true | ||
595 | |||
596 | // Rename the base resource. | ||
597 | if rebaseName != "" { | ||
598 | var replacement string | ||
599 | if rebaseName != string(filepath.Separator) { | ||
600 | // Special case the root directory to replace with an | ||
601 | // empty string instead so that we don't end up with | ||
602 | // double slashes in the paths. | ||
603 | replacement = rebaseName | ||
604 | } | ||
605 | |||
606 | relFilePath = strings.Replace(relFilePath, include, replacement, 1) | ||
607 | } | ||
608 | |||
609 | if err := ta.addTarFile(filePath, relFilePath); err != nil { | ||
610 | logrus.Debugf("Can't add file %s to tar: %s", filePath, err) | ||
611 | } | ||
612 | return nil | ||
613 | }) | ||
614 | } | ||
615 | }() | ||
616 | |||
617 | return pipeReader, nil | ||
618 | } | ||
619 | |||
620 | // Unpack unpacks the decompressedArchive to dest with options. | ||
621 | func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { | ||
622 | tr := tar.NewReader(decompressedArchive) | ||
623 | trBuf := pools.BufioReader32KPool.Get(nil) | ||
624 | defer pools.BufioReader32KPool.Put(trBuf) | ||
625 | |||
626 | var dirs []*tar.Header | ||
627 | remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) | ||
628 | if err != nil { | ||
629 | return err | ||
630 | } | ||
631 | |||
632 | // Iterate through the files in the archive. | ||
633 | loop: | ||
634 | for { | ||
635 | hdr, err := tr.Next() | ||
636 | if err == io.EOF { | ||
637 | // end of tar archive | ||
638 | break | ||
639 | } | ||
640 | if err != nil { | ||
641 | return err | ||
642 | } | ||
643 | |||
644 | // Normalize name, for safety and for a simple is-root check | ||
645 | // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: | ||
646 | // This keeps "..\" as-is, but normalizes "\..\" to "\". | ||
647 | hdr.Name = filepath.Clean(hdr.Name) | ||
648 | |||
649 | for _, exclude := range options.ExcludePatterns { | ||
650 | if strings.HasPrefix(hdr.Name, exclude) { | ||
651 | continue loop | ||
652 | } | ||
653 | } | ||
654 | |||
655 | // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in | ||
656 | // the filepath format for the OS on which the daemon is running. Hence | ||
657 | // the check for a slash-suffix MUST be done in an OS-agnostic way. | ||
658 | if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { | ||
659 | // Not the root directory, ensure that the parent directory exists | ||
660 | parent := filepath.Dir(hdr.Name) | ||
661 | parentPath := filepath.Join(dest, parent) | ||
662 | if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { | ||
663 | err = system.MkdirAll(parentPath, 0777) | ||
664 | if err != nil { | ||
665 | return err | ||
666 | } | ||
667 | } | ||
668 | } | ||
669 | |||
670 | path := filepath.Join(dest, hdr.Name) | ||
671 | rel, err := filepath.Rel(dest, path) | ||
672 | if err != nil { | ||
673 | return err | ||
674 | } | ||
675 | if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { | ||
676 | return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) | ||
677 | } | ||
678 | |||
679 | // If path exits we almost always just want to remove and replace it | ||
680 | // The only exception is when it is a directory *and* the file from | ||
681 | // the layer is also a directory. Then we want to merge them (i.e. | ||
682 | // just apply the metadata from the layer). | ||
683 | if fi, err := os.Lstat(path); err == nil { | ||
684 | if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir { | ||
685 | // If NoOverwriteDirNonDir is true then we cannot replace | ||
686 | // an existing directory with a non-directory from the archive. | ||
687 | return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest) | ||
688 | } | ||
689 | |||
690 | if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir { | ||
691 | // If NoOverwriteDirNonDir is true then we cannot replace | ||
692 | // an existing non-directory with a directory from the archive. | ||
693 | return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest) | ||
694 | } | ||
695 | |||
696 | if fi.IsDir() && hdr.Name == "." { | ||
697 | continue | ||
698 | } | ||
699 | |||
700 | if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { | ||
701 | if err := os.RemoveAll(path); err != nil { | ||
702 | return err | ||
703 | } | ||
704 | } | ||
705 | } | ||
706 | trBuf.Reset(tr) | ||
707 | |||
708 | // if the options contain a uid & gid maps, convert header uid/gid | ||
709 | // entries using the maps such that lchown sets the proper mapped | ||
710 | // uid/gid after writing the file. We only perform this mapping if | ||
711 | // the file isn't already owned by the remapped root UID or GID, as | ||
712 | // that specific uid/gid has no mapping from container -> host, and | ||
713 | // those files already have the proper ownership for inside the | ||
714 | // container. | ||
715 | if hdr.Uid != remappedRootUID { | ||
716 | xUID, err := idtools.ToHost(hdr.Uid, options.UIDMaps) | ||
717 | if err != nil { | ||
718 | return err | ||
719 | } | ||
720 | hdr.Uid = xUID | ||
721 | } | ||
722 | if hdr.Gid != remappedRootGID { | ||
723 | xGID, err := idtools.ToHost(hdr.Gid, options.GIDMaps) | ||
724 | if err != nil { | ||
725 | return err | ||
726 | } | ||
727 | hdr.Gid = xGID | ||
728 | } | ||
729 | |||
730 | if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts); err != nil { | ||
731 | return err | ||
732 | } | ||
733 | |||
734 | // Directory mtimes must be handled at the end to avoid further | ||
735 | // file creation in them to modify the directory mtime | ||
736 | if hdr.Typeflag == tar.TypeDir { | ||
737 | dirs = append(dirs, hdr) | ||
738 | } | ||
739 | } | ||
740 | |||
741 | for _, hdr := range dirs { | ||
742 | path := filepath.Join(dest, hdr.Name) | ||
743 | |||
744 | if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { | ||
745 | return err | ||
746 | } | ||
747 | } | ||
748 | return nil | ||
749 | } | ||
750 | |||
751 | // Untar reads a stream of bytes from `archive`, parses it as a tar archive, | ||
752 | // and unpacks it into the directory at `dest`. | ||
753 | // The archive may be compressed with one of the following algorithms: | ||
754 | // identity (uncompressed), gzip, bzip2, xz. | ||
755 | // FIXME: specify behavior when target path exists vs. doesn't exist. | ||
756 | func Untar(tarArchive io.Reader, dest string, options *TarOptions) error { | ||
757 | return untarHandler(tarArchive, dest, options, true) | ||
758 | } | ||
759 | |||
760 | // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, | ||
761 | // and unpacks it into the directory at `dest`. | ||
762 | // The archive must be an uncompressed stream. | ||
763 | func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error { | ||
764 | return untarHandler(tarArchive, dest, options, false) | ||
765 | } | ||
766 | |||
767 | // Handler for teasing out the automatic decompression | ||
768 | func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error { | ||
769 | if tarArchive == nil { | ||
770 | return fmt.Errorf("Empty archive") | ||
771 | } | ||
772 | dest = filepath.Clean(dest) | ||
773 | if options == nil { | ||
774 | options = &TarOptions{} | ||
775 | } | ||
776 | if options.ExcludePatterns == nil { | ||
777 | options.ExcludePatterns = []string{} | ||
778 | } | ||
779 | |||
780 | r := tarArchive | ||
781 | if decompress { | ||
782 | decompressedArchive, err := DecompressStream(tarArchive) | ||
783 | if err != nil { | ||
784 | return err | ||
785 | } | ||
786 | defer decompressedArchive.Close() | ||
787 | r = decompressedArchive | ||
788 | } | ||
789 | |||
790 | return Unpack(r, dest, options) | ||
791 | } | ||
792 | |||
793 | // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. | ||
794 | // If either Tar or Untar fails, TarUntar aborts and returns the error. | ||
795 | func (archiver *Archiver) TarUntar(src, dst string) error { | ||
796 | logrus.Debugf("TarUntar(%s %s)", src, dst) | ||
797 | archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) | ||
798 | if err != nil { | ||
799 | return err | ||
800 | } | ||
801 | defer archive.Close() | ||
802 | |||
803 | var options *TarOptions | ||
804 | if archiver.UIDMaps != nil || archiver.GIDMaps != nil { | ||
805 | options = &TarOptions{ | ||
806 | UIDMaps: archiver.UIDMaps, | ||
807 | GIDMaps: archiver.GIDMaps, | ||
808 | } | ||
809 | } | ||
810 | return archiver.Untar(archive, dst, options) | ||
811 | } | ||
812 | |||
813 | // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. | ||
814 | // If either Tar or Untar fails, TarUntar aborts and returns the error. | ||
815 | func TarUntar(src, dst string) error { | ||
816 | return defaultArchiver.TarUntar(src, dst) | ||
817 | } | ||
818 | |||
819 | // UntarPath untar a file from path to a destination, src is the source tar file path. | ||
820 | func (archiver *Archiver) UntarPath(src, dst string) error { | ||
821 | archive, err := os.Open(src) | ||
822 | if err != nil { | ||
823 | return err | ||
824 | } | ||
825 | defer archive.Close() | ||
826 | var options *TarOptions | ||
827 | if archiver.UIDMaps != nil || archiver.GIDMaps != nil { | ||
828 | options = &TarOptions{ | ||
829 | UIDMaps: archiver.UIDMaps, | ||
830 | GIDMaps: archiver.GIDMaps, | ||
831 | } | ||
832 | } | ||
833 | return archiver.Untar(archive, dst, options) | ||
834 | } | ||
835 | |||
836 | // UntarPath is a convenience function which looks for an archive | ||
837 | // at filesystem path `src`, and unpacks it at `dst`. | ||
838 | func UntarPath(src, dst string) error { | ||
839 | return defaultArchiver.UntarPath(src, dst) | ||
840 | } | ||
841 | |||
842 | // CopyWithTar creates a tar archive of filesystem path `src`, and | ||
843 | // unpacks it at filesystem path `dst`. | ||
844 | // The archive is streamed directly with fixed buffering and no | ||
845 | // intermediary disk IO. | ||
846 | func (archiver *Archiver) CopyWithTar(src, dst string) error { | ||
847 | srcSt, err := os.Stat(src) | ||
848 | if err != nil { | ||
849 | return err | ||
850 | } | ||
851 | if !srcSt.IsDir() { | ||
852 | return archiver.CopyFileWithTar(src, dst) | ||
853 | } | ||
854 | // Create dst, copy src's content into it | ||
855 | logrus.Debugf("Creating dest directory: %s", dst) | ||
856 | if err := system.MkdirAll(dst, 0755); err != nil { | ||
857 | return err | ||
858 | } | ||
859 | logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) | ||
860 | return archiver.TarUntar(src, dst) | ||
861 | } | ||
862 | |||
863 | // CopyWithTar creates a tar archive of filesystem path `src`, and | ||
864 | // unpacks it at filesystem path `dst`. | ||
865 | // The archive is streamed directly with fixed buffering and no | ||
866 | // intermediary disk IO. | ||
867 | func CopyWithTar(src, dst string) error { | ||
868 | return defaultArchiver.CopyWithTar(src, dst) | ||
869 | } | ||
870 | |||
871 | // CopyFileWithTar emulates the behavior of the 'cp' command-line | ||
872 | // for a single file. It copies a regular file from path `src` to | ||
873 | // path `dst`, and preserves all its metadata. | ||
874 | func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { | ||
875 | logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) | ||
876 | srcSt, err := os.Stat(src) | ||
877 | if err != nil { | ||
878 | return err | ||
879 | } | ||
880 | |||
881 | if srcSt.IsDir() { | ||
882 | return fmt.Errorf("Can't copy a directory") | ||
883 | } | ||
884 | |||
885 | // Clean up the trailing slash. This must be done in an operating | ||
886 | // system specific manner. | ||
887 | if dst[len(dst)-1] == os.PathSeparator { | ||
888 | dst = filepath.Join(dst, filepath.Base(src)) | ||
889 | } | ||
890 | // Create the holding directory if necessary | ||
891 | if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil { | ||
892 | return err | ||
893 | } | ||
894 | |||
895 | r, w := io.Pipe() | ||
896 | errC := promise.Go(func() error { | ||
897 | defer w.Close() | ||
898 | |||
899 | srcF, err := os.Open(src) | ||
900 | if err != nil { | ||
901 | return err | ||
902 | } | ||
903 | defer srcF.Close() | ||
904 | |||
905 | hdr, err := tar.FileInfoHeader(srcSt, "") | ||
906 | if err != nil { | ||
907 | return err | ||
908 | } | ||
909 | hdr.Name = filepath.Base(dst) | ||
910 | hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) | ||
911 | |||
912 | remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps) | ||
913 | if err != nil { | ||
914 | return err | ||
915 | } | ||
916 | |||
917 | // only perform mapping if the file being copied isn't already owned by the | ||
918 | // uid or gid of the remapped root in the container | ||
919 | if remappedRootUID != hdr.Uid { | ||
920 | xUID, err := idtools.ToHost(hdr.Uid, archiver.UIDMaps) | ||
921 | if err != nil { | ||
922 | return err | ||
923 | } | ||
924 | hdr.Uid = xUID | ||
925 | } | ||
926 | if remappedRootGID != hdr.Gid { | ||
927 | xGID, err := idtools.ToHost(hdr.Gid, archiver.GIDMaps) | ||
928 | if err != nil { | ||
929 | return err | ||
930 | } | ||
931 | hdr.Gid = xGID | ||
932 | } | ||
933 | |||
934 | tw := tar.NewWriter(w) | ||
935 | defer tw.Close() | ||
936 | if err := tw.WriteHeader(hdr); err != nil { | ||
937 | return err | ||
938 | } | ||
939 | if _, err := io.Copy(tw, srcF); err != nil { | ||
940 | return err | ||
941 | } | ||
942 | return nil | ||
943 | }) | ||
944 | defer func() { | ||
945 | if er := <-errC; err != nil { | ||
946 | err = er | ||
947 | } | ||
948 | }() | ||
949 | |||
950 | err = archiver.Untar(r, filepath.Dir(dst), nil) | ||
951 | if err != nil { | ||
952 | r.CloseWithError(err) | ||
953 | } | ||
954 | return err | ||
955 | } | ||
956 | |||
957 | // CopyFileWithTar emulates the behavior of the 'cp' command-line | ||
958 | // for a single file. It copies a regular file from path `src` to | ||
959 | // path `dst`, and preserves all its metadata. | ||
960 | // | ||
961 | // Destination handling is in an operating specific manner depending | ||
962 | // where the daemon is running. If `dst` ends with a trailing slash | ||
963 | // the final destination path will be `dst/base(src)` (Linux) or | ||
964 | // `dst\base(src)` (Windows). | ||
965 | func CopyFileWithTar(src, dst string) (err error) { | ||
966 | return defaultArchiver.CopyFileWithTar(src, dst) | ||
967 | } | ||
968 | |||
969 | // cmdStream executes a command, and returns its stdout as a stream. | ||
970 | // If the command fails to run or doesn't complete successfully, an error | ||
971 | // will be returned, including anything written on stderr. | ||
972 | func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) { | ||
973 | chdone := make(chan struct{}) | ||
974 | cmd.Stdin = input | ||
975 | pipeR, pipeW := io.Pipe() | ||
976 | cmd.Stdout = pipeW | ||
977 | var errBuf bytes.Buffer | ||
978 | cmd.Stderr = &errBuf | ||
979 | |||
980 | // Run the command and return the pipe | ||
981 | if err := cmd.Start(); err != nil { | ||
982 | return nil, nil, err | ||
983 | } | ||
984 | |||
985 | // Copy stdout to the returned pipe | ||
986 | go func() { | ||
987 | if err := cmd.Wait(); err != nil { | ||
988 | pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) | ||
989 | } else { | ||
990 | pipeW.Close() | ||
991 | } | ||
992 | close(chdone) | ||
993 | }() | ||
994 | |||
995 | return pipeR, chdone, nil | ||
996 | } | ||
997 | |||
998 | // NewTempArchive reads the content of src into a temporary file, and returns the contents | ||
999 | // of that file as an archive. The archive can only be read once - as soon as reading completes, | ||
1000 | // the file will be deleted. | ||
1001 | func NewTempArchive(src Archive, dir string) (*TempArchive, error) { | ||
1002 | f, err := ioutil.TempFile(dir, "") | ||
1003 | if err != nil { | ||
1004 | return nil, err | ||
1005 | } | ||
1006 | if _, err := io.Copy(f, src); err != nil { | ||
1007 | return nil, err | ||
1008 | } | ||
1009 | if _, err := f.Seek(0, 0); err != nil { | ||
1010 | return nil, err | ||
1011 | } | ||
1012 | st, err := f.Stat() | ||
1013 | if err != nil { | ||
1014 | return nil, err | ||
1015 | } | ||
1016 | size := st.Size() | ||
1017 | return &TempArchive{File: f, Size: size}, nil | ||
1018 | } | ||
1019 | |||
1020 | // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes, | ||
1021 | // the file will be deleted. | ||
1022 | type TempArchive struct { | ||
1023 | *os.File | ||
1024 | Size int64 // Pre-computed from Stat().Size() as a convenience | ||
1025 | read int64 | ||
1026 | closed bool | ||
1027 | } | ||
1028 | |||
1029 | // Close closes the underlying file if it's still open, or does a no-op | ||
1030 | // to allow callers to try to close the TempArchive multiple times safely. | ||
1031 | func (archive *TempArchive) Close() error { | ||
1032 | if archive.closed { | ||
1033 | return nil | ||
1034 | } | ||
1035 | |||
1036 | archive.closed = true | ||
1037 | |||
1038 | return archive.File.Close() | ||
1039 | } | ||
1040 | |||
1041 | func (archive *TempArchive) Read(data []byte) (int, error) { | ||
1042 | n, err := archive.File.Read(data) | ||
1043 | archive.read += int64(n) | ||
1044 | if err != nil || archive.read == archive.Size { | ||
1045 | archive.Close() | ||
1046 | os.Remove(archive.File.Name()) | ||
1047 | } | ||
1048 | return n, err | ||
1049 | } | ||