]>
Commit | Line | Data |
---|---|---|
9b12e4fe JC |
1 | package archive |
2 | ||
3 | import ( | |
4 | "archive/tar" | |
5 | "bufio" | |
6 | "bytes" | |
7 | "compress/bzip2" | |
8 | "compress/gzip" | |
9 | "errors" | |
10 | "fmt" | |
11 | "io" | |
12 | "io/ioutil" | |
13 | "os" | |
14 | "os/exec" | |
15 | "path/filepath" | |
16 | "runtime" | |
17 | "strings" | |
18 | "syscall" | |
19 | ||
20 | "github.com/fsouza/go-dockerclient/external/github.com/Sirupsen/logrus" | |
21 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/fileutils" | |
22 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/idtools" | |
23 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/ioutils" | |
24 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/pools" | |
25 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/promise" | |
26 | "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/pkg/system" | |
27 | ) | |
28 | ||
29 | type ( | |
30 | // Archive is a type of io.ReadCloser which has two interfaces Read and Closer. | |
31 | Archive io.ReadCloser | |
32 | // Reader is a type of io.Reader. | |
33 | Reader io.Reader | |
34 | // Compression is the state represents if compressed or not. | |
35 | Compression int | |
36 | // TarChownOptions wraps the chown options UID and GID. | |
37 | TarChownOptions struct { | |
38 | UID, GID int | |
39 | } | |
40 | // TarOptions wraps the tar options. | |
41 | TarOptions struct { | |
42 | IncludeFiles []string | |
43 | ExcludePatterns []string | |
44 | Compression Compression | |
45 | NoLchown bool | |
46 | UIDMaps []idtools.IDMap | |
47 | GIDMaps []idtools.IDMap | |
48 | ChownOpts *TarChownOptions | |
49 | IncludeSourceDir bool | |
50 | // When unpacking, specifies whether overwriting a directory with a | |
51 | // non-directory is allowed and vice versa. | |
52 | NoOverwriteDirNonDir bool | |
53 | // For each include when creating an archive, the included name will be | |
54 | // replaced with the matching name from this map. | |
55 | RebaseNames map[string]string | |
56 | } | |
57 | ||
58 | // Archiver allows the reuse of most utility functions of this package | |
59 | // with a pluggable Untar function. Also, to facilitate the passing of | |
60 | // specific id mappings for untar, an archiver can be created with maps | |
61 | // which will then be passed to Untar operations | |
62 | Archiver struct { | |
63 | Untar func(io.Reader, string, *TarOptions) error | |
64 | UIDMaps []idtools.IDMap | |
65 | GIDMaps []idtools.IDMap | |
66 | } | |
67 | ||
68 | // breakoutError is used to differentiate errors related to breaking out | |
69 | // When testing archive breakout in the unit tests, this error is expected | |
70 | // in order for the test to pass. | |
71 | breakoutError error | |
72 | ) | |
73 | ||
74 | var ( | |
75 | // ErrNotImplemented is the error message of function not implemented. | |
76 | ErrNotImplemented = errors.New("Function not implemented") | |
77 | defaultArchiver = &Archiver{Untar: Untar, UIDMaps: nil, GIDMaps: nil} | |
78 | ) | |
79 | ||
80 | const ( | |
81 | // HeaderSize is the size in bytes of a tar header | |
82 | HeaderSize = 512 | |
83 | ) | |
84 | ||
85 | const ( | |
86 | // Uncompressed represents the uncompressed. | |
87 | Uncompressed Compression = iota | |
88 | // Bzip2 is bzip2 compression algorithm. | |
89 | Bzip2 | |
90 | // Gzip is gzip compression algorithm. | |
91 | Gzip | |
92 | // Xz is xz compression algorithm. | |
93 | Xz | |
94 | ) | |
95 | ||
96 | // IsArchive checks for the magic bytes of a tar or any supported compression | |
97 | // algorithm. | |
98 | func IsArchive(header []byte) bool { | |
99 | compression := DetectCompression(header) | |
100 | if compression != Uncompressed { | |
101 | return true | |
102 | } | |
103 | r := tar.NewReader(bytes.NewBuffer(header)) | |
104 | _, err := r.Next() | |
105 | return err == nil | |
106 | } | |
107 | ||
108 | // IsArchivePath checks if the (possibly compressed) file at the given path | |
109 | // starts with a tar file header. | |
110 | func IsArchivePath(path string) bool { | |
111 | file, err := os.Open(path) | |
112 | if err != nil { | |
113 | return false | |
114 | } | |
115 | defer file.Close() | |
116 | rdr, err := DecompressStream(file) | |
117 | if err != nil { | |
118 | return false | |
119 | } | |
120 | r := tar.NewReader(rdr) | |
121 | _, err = r.Next() | |
122 | return err == nil | |
123 | } | |
124 | ||
125 | // DetectCompression detects the compression algorithm of the source. | |
126 | func DetectCompression(source []byte) Compression { | |
127 | for compression, m := range map[Compression][]byte{ | |
128 | Bzip2: {0x42, 0x5A, 0x68}, | |
129 | Gzip: {0x1F, 0x8B, 0x08}, | |
130 | Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, | |
131 | } { | |
132 | if len(source) < len(m) { | |
133 | logrus.Debugf("Len too short") | |
134 | continue | |
135 | } | |
136 | if bytes.Compare(m, source[:len(m)]) == 0 { | |
137 | return compression | |
138 | } | |
139 | } | |
140 | return Uncompressed | |
141 | } | |
142 | ||
143 | func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) { | |
144 | args := []string{"xz", "-d", "-c", "-q"} | |
145 | ||
146 | return cmdStream(exec.Command(args[0], args[1:]...), archive) | |
147 | } | |
148 | ||
149 | // DecompressStream decompress the archive and returns a ReaderCloser with the decompressed archive. | |
150 | func DecompressStream(archive io.Reader) (io.ReadCloser, error) { | |
151 | p := pools.BufioReader32KPool | |
152 | buf := p.Get(archive) | |
153 | bs, err := buf.Peek(10) | |
154 | if err != nil && err != io.EOF { | |
155 | // Note: we'll ignore any io.EOF error because there are some odd | |
156 | // cases where the layer.tar file will be empty (zero bytes) and | |
157 | // that results in an io.EOF from the Peek() call. So, in those | |
158 | // cases we'll just treat it as a non-compressed stream and | |
159 | // that means just create an empty layer. | |
160 | // See Issue 18170 | |
161 | return nil, err | |
162 | } | |
163 | ||
164 | compression := DetectCompression(bs) | |
165 | switch compression { | |
166 | case Uncompressed: | |
167 | readBufWrapper := p.NewReadCloserWrapper(buf, buf) | |
168 | return readBufWrapper, nil | |
169 | case Gzip: | |
170 | gzReader, err := gzip.NewReader(buf) | |
171 | if err != nil { | |
172 | return nil, err | |
173 | } | |
174 | readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) | |
175 | return readBufWrapper, nil | |
176 | case Bzip2: | |
177 | bz2Reader := bzip2.NewReader(buf) | |
178 | readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) | |
179 | return readBufWrapper, nil | |
180 | case Xz: | |
181 | xzReader, chdone, err := xzDecompress(buf) | |
182 | if err != nil { | |
183 | return nil, err | |
184 | } | |
185 | readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) | |
186 | return ioutils.NewReadCloserWrapper(readBufWrapper, func() error { | |
187 | <-chdone | |
188 | return readBufWrapper.Close() | |
189 | }), nil | |
190 | default: | |
191 | return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) | |
192 | } | |
193 | } | |
194 | ||
195 | // CompressStream compresses the dest with specified compression algorithm. | |
196 | func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) { | |
197 | p := pools.BufioWriter32KPool | |
198 | buf := p.Get(dest) | |
199 | switch compression { | |
200 | case Uncompressed: | |
201 | writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) | |
202 | return writeBufWrapper, nil | |
203 | case Gzip: | |
204 | gzWriter := gzip.NewWriter(dest) | |
205 | writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) | |
206 | return writeBufWrapper, nil | |
207 | case Bzip2, Xz: | |
208 | // archive/bzip2 does not support writing, and there is no xz support at all | |
209 | // However, this is not a problem as docker only currently generates gzipped tars | |
210 | return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) | |
211 | default: | |
212 | return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) | |
213 | } | |
214 | } | |
215 | ||
216 | // Extension returns the extension of a file that uses the specified compression algorithm. | |
217 | func (compression *Compression) Extension() string { | |
218 | switch *compression { | |
219 | case Uncompressed: | |
220 | return "tar" | |
221 | case Bzip2: | |
222 | return "tar.bz2" | |
223 | case Gzip: | |
224 | return "tar.gz" | |
225 | case Xz: | |
226 | return "tar.xz" | |
227 | } | |
228 | return "" | |
229 | } | |
230 | ||
231 | type tarAppender struct { | |
232 | TarWriter *tar.Writer | |
233 | Buffer *bufio.Writer | |
234 | ||
235 | // for hardlink mapping | |
236 | SeenFiles map[uint64]string | |
237 | UIDMaps []idtools.IDMap | |
238 | GIDMaps []idtools.IDMap | |
239 | } | |
240 | ||
241 | // canonicalTarName provides a platform-independent and consistent posix-style | |
242 | //path for files and directories to be archived regardless of the platform. | |
243 | func canonicalTarName(name string, isDir bool) (string, error) { | |
244 | name, err := CanonicalTarNameForPath(name) | |
245 | if err != nil { | |
246 | return "", err | |
247 | } | |
248 | ||
249 | // suffix with '/' for directories | |
250 | if isDir && !strings.HasSuffix(name, "/") { | |
251 | name += "/" | |
252 | } | |
253 | return name, nil | |
254 | } | |
255 | ||
256 | func (ta *tarAppender) addTarFile(path, name string) error { | |
257 | fi, err := os.Lstat(path) | |
258 | if err != nil { | |
259 | return err | |
260 | } | |
261 | ||
262 | link := "" | |
263 | if fi.Mode()&os.ModeSymlink != 0 { | |
264 | if link, err = os.Readlink(path); err != nil { | |
265 | return err | |
266 | } | |
267 | } | |
268 | ||
269 | hdr, err := tar.FileInfoHeader(fi, link) | |
270 | if err != nil { | |
271 | return err | |
272 | } | |
273 | hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) | |
274 | ||
275 | name, err = canonicalTarName(name, fi.IsDir()) | |
276 | if err != nil { | |
277 | return fmt.Errorf("tar: cannot canonicalize path: %v", err) | |
278 | } | |
279 | hdr.Name = name | |
280 | ||
281 | inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys()) | |
282 | if err != nil { | |
283 | return err | |
284 | } | |
285 | ||
286 | // if it's not a directory and has more than 1 link, | |
287 | // it's hardlinked, so set the type flag accordingly | |
288 | if !fi.IsDir() && hasHardlinks(fi) { | |
289 | // a link should have a name that it links too | |
290 | // and that linked name should be first in the tar archive | |
291 | if oldpath, ok := ta.SeenFiles[inode]; ok { | |
292 | hdr.Typeflag = tar.TypeLink | |
293 | hdr.Linkname = oldpath | |
294 | hdr.Size = 0 // This Must be here for the writer math to add up! | |
295 | } else { | |
296 | ta.SeenFiles[inode] = name | |
297 | } | |
298 | } | |
299 | ||
300 | capability, _ := system.Lgetxattr(path, "security.capability") | |
301 | if capability != nil { | |
302 | hdr.Xattrs = make(map[string]string) | |
303 | hdr.Xattrs["security.capability"] = string(capability) | |
304 | } | |
305 | ||
306 | //handle re-mapping container ID mappings back to host ID mappings before | |
307 | //writing tar headers/files. We skip whiteout files because they were written | |
308 | //by the kernel and already have proper ownership relative to the host | |
309 | if !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && (ta.UIDMaps != nil || ta.GIDMaps != nil) { | |
310 | uid, gid, err := getFileUIDGID(fi.Sys()) | |
311 | if err != nil { | |
312 | return err | |
313 | } | |
314 | xUID, err := idtools.ToContainer(uid, ta.UIDMaps) | |
315 | if err != nil { | |
316 | return err | |
317 | } | |
318 | xGID, err := idtools.ToContainer(gid, ta.GIDMaps) | |
319 | if err != nil { | |
320 | return err | |
321 | } | |
322 | hdr.Uid = xUID | |
323 | hdr.Gid = xGID | |
324 | } | |
325 | ||
326 | if err := ta.TarWriter.WriteHeader(hdr); err != nil { | |
327 | return err | |
328 | } | |
329 | ||
330 | if hdr.Typeflag == tar.TypeReg { | |
331 | file, err := os.Open(path) | |
332 | if err != nil { | |
333 | return err | |
334 | } | |
335 | ||
336 | ta.Buffer.Reset(ta.TarWriter) | |
337 | defer ta.Buffer.Reset(nil) | |
338 | _, err = io.Copy(ta.Buffer, file) | |
339 | file.Close() | |
340 | if err != nil { | |
341 | return err | |
342 | } | |
343 | err = ta.Buffer.Flush() | |
344 | if err != nil { | |
345 | return err | |
346 | } | |
347 | } | |
348 | ||
349 | return nil | |
350 | } | |
351 | ||
352 | func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *TarChownOptions) error { | |
353 | // hdr.Mode is in linux format, which we can use for sycalls, | |
354 | // but for os.Foo() calls we need the mode converted to os.FileMode, | |
355 | // so use hdrInfo.Mode() (they differ for e.g. setuid bits) | |
356 | hdrInfo := hdr.FileInfo() | |
357 | ||
358 | switch hdr.Typeflag { | |
359 | case tar.TypeDir: | |
360 | // Create directory unless it exists as a directory already. | |
361 | // In that case we just want to merge the two | |
362 | if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { | |
363 | if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { | |
364 | return err | |
365 | } | |
366 | } | |
367 | ||
368 | case tar.TypeReg, tar.TypeRegA: | |
369 | // Source is regular file | |
370 | file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) | |
371 | if err != nil { | |
372 | return err | |
373 | } | |
374 | if _, err := io.Copy(file, reader); err != nil { | |
375 | file.Close() | |
376 | return err | |
377 | } | |
378 | file.Close() | |
379 | ||
380 | case tar.TypeBlock, tar.TypeChar, tar.TypeFifo: | |
381 | // Handle this is an OS-specific way | |
382 | if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { | |
383 | return err | |
384 | } | |
385 | ||
386 | case tar.TypeLink: | |
387 | targetPath := filepath.Join(extractDir, hdr.Linkname) | |
388 | // check for hardlink breakout | |
389 | if !strings.HasPrefix(targetPath, extractDir) { | |
390 | return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) | |
391 | } | |
392 | if err := os.Link(targetPath, path); err != nil { | |
393 | return err | |
394 | } | |
395 | ||
396 | case tar.TypeSymlink: | |
397 | // path -> hdr.Linkname = targetPath | |
398 | // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file | |
399 | targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) | |
400 | ||
401 | // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because | |
402 | // that symlink would first have to be created, which would be caught earlier, at this very check: | |
403 | if !strings.HasPrefix(targetPath, extractDir) { | |
404 | return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) | |
405 | } | |
406 | if err := os.Symlink(hdr.Linkname, path); err != nil { | |
407 | return err | |
408 | } | |
409 | ||
410 | case tar.TypeXGlobalHeader: | |
411 | logrus.Debugf("PAX Global Extended Headers found and ignored") | |
412 | return nil | |
413 | ||
414 | default: | |
415 | return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag) | |
416 | } | |
417 | ||
418 | // Lchown is not supported on Windows. | |
419 | if Lchown && runtime.GOOS != "windows" { | |
420 | if chownOpts == nil { | |
421 | chownOpts = &TarChownOptions{UID: hdr.Uid, GID: hdr.Gid} | |
422 | } | |
423 | if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { | |
424 | return err | |
425 | } | |
426 | } | |
427 | ||
428 | for key, value := range hdr.Xattrs { | |
429 | if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { | |
430 | return err | |
431 | } | |
432 | } | |
433 | ||
434 | // There is no LChmod, so ignore mode for symlink. Also, this | |
435 | // must happen after chown, as that can modify the file mode | |
436 | if err := handleLChmod(hdr, path, hdrInfo); err != nil { | |
437 | return err | |
438 | } | |
439 | ||
440 | aTime := hdr.AccessTime | |
441 | if aTime.Before(hdr.ModTime) { | |
442 | // Last access time should never be before last modified time. | |
443 | aTime = hdr.ModTime | |
444 | } | |
445 | ||
446 | // system.Chtimes doesn't support a NOFOLLOW flag atm | |
447 | if hdr.Typeflag == tar.TypeLink { | |
448 | if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { | |
449 | if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { | |
450 | return err | |
451 | } | |
452 | } | |
453 | } else if hdr.Typeflag != tar.TypeSymlink { | |
454 | if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { | |
455 | return err | |
456 | } | |
457 | } else { | |
458 | ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} | |
459 | if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { | |
460 | return err | |
461 | } | |
462 | } | |
463 | return nil | |
464 | } | |
465 | ||
466 | // Tar creates an archive from the directory at `path`, and returns it as a | |
467 | // stream of bytes. | |
468 | func Tar(path string, compression Compression) (io.ReadCloser, error) { | |
469 | return TarWithOptions(path, &TarOptions{Compression: compression}) | |
470 | } | |
471 | ||
472 | // TarWithOptions creates an archive from the directory at `path`, only including files whose relative | |
473 | // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. | |
474 | func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { | |
475 | ||
476 | // Fix the source path to work with long path names. This is a no-op | |
477 | // on platforms other than Windows. | |
478 | srcPath = fixVolumePathPrefix(srcPath) | |
479 | ||
480 | patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns) | |
481 | ||
482 | if err != nil { | |
483 | return nil, err | |
484 | } | |
485 | ||
486 | pipeReader, pipeWriter := io.Pipe() | |
487 | ||
488 | compressWriter, err := CompressStream(pipeWriter, options.Compression) | |
489 | if err != nil { | |
490 | return nil, err | |
491 | } | |
492 | ||
493 | go func() { | |
494 | ta := &tarAppender{ | |
495 | TarWriter: tar.NewWriter(compressWriter), | |
496 | Buffer: pools.BufioWriter32KPool.Get(nil), | |
497 | SeenFiles: make(map[uint64]string), | |
498 | UIDMaps: options.UIDMaps, | |
499 | GIDMaps: options.GIDMaps, | |
500 | } | |
501 | ||
502 | defer func() { | |
503 | // Make sure to check the error on Close. | |
504 | if err := ta.TarWriter.Close(); err != nil { | |
505 | logrus.Debugf("Can't close tar writer: %s", err) | |
506 | } | |
507 | if err := compressWriter.Close(); err != nil { | |
508 | logrus.Debugf("Can't close compress writer: %s", err) | |
509 | } | |
510 | if err := pipeWriter.Close(); err != nil { | |
511 | logrus.Debugf("Can't close pipe writer: %s", err) | |
512 | } | |
513 | }() | |
514 | ||
515 | // this buffer is needed for the duration of this piped stream | |
516 | defer pools.BufioWriter32KPool.Put(ta.Buffer) | |
517 | ||
518 | // In general we log errors here but ignore them because | |
519 | // during e.g. a diff operation the container can continue | |
520 | // mutating the filesystem and we can see transient errors | |
521 | // from this | |
522 | ||
523 | stat, err := os.Lstat(srcPath) | |
524 | if err != nil { | |
525 | return | |
526 | } | |
527 | ||
528 | if !stat.IsDir() { | |
529 | // We can't later join a non-dir with any includes because the | |
530 | // 'walk' will error if "file/." is stat-ed and "file" is not a | |
531 | // directory. So, we must split the source path and use the | |
532 | // basename as the include. | |
533 | if len(options.IncludeFiles) > 0 { | |
534 | logrus.Warn("Tar: Can't archive a file with includes") | |
535 | } | |
536 | ||
537 | dir, base := SplitPathDirEntry(srcPath) | |
538 | srcPath = dir | |
539 | options.IncludeFiles = []string{base} | |
540 | } | |
541 | ||
542 | if len(options.IncludeFiles) == 0 { | |
543 | options.IncludeFiles = []string{"."} | |
544 | } | |
545 | ||
546 | seen := make(map[string]bool) | |
547 | ||
548 | for _, include := range options.IncludeFiles { | |
549 | rebaseName := options.RebaseNames[include] | |
550 | ||
551 | walkRoot := getWalkRoot(srcPath, include) | |
552 | filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { | |
553 | if err != nil { | |
554 | logrus.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err) | |
555 | return nil | |
556 | } | |
557 | ||
558 | relFilePath, err := filepath.Rel(srcPath, filePath) | |
559 | if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) { | |
560 | // Error getting relative path OR we are looking | |
561 | // at the source directory path. Skip in both situations. | |
562 | return nil | |
563 | } | |
564 | ||
565 | if options.IncludeSourceDir && include == "." && relFilePath != "." { | |
566 | relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator)) | |
567 | } | |
568 | ||
569 | skip := false | |
570 | ||
571 | // If "include" is an exact match for the current file | |
572 | // then even if there's an "excludePatterns" pattern that | |
573 | // matches it, don't skip it. IOW, assume an explicit 'include' | |
574 | // is asking for that file no matter what - which is true | |
575 | // for some files, like .dockerignore and Dockerfile (sometimes) | |
576 | if include != relFilePath { | |
577 | skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs) | |
578 | if err != nil { | |
579 | logrus.Debugf("Error matching %s: %v", relFilePath, err) | |
580 | return err | |
581 | } | |
582 | } | |
583 | ||
584 | if skip { | |
585 | if !exceptions && f.IsDir() { | |
586 | return filepath.SkipDir | |
587 | } | |
588 | return nil | |
589 | } | |
590 | ||
591 | if seen[relFilePath] { | |
592 | return nil | |
593 | } | |
594 | seen[relFilePath] = true | |
595 | ||
596 | // Rename the base resource. | |
597 | if rebaseName != "" { | |
598 | var replacement string | |
599 | if rebaseName != string(filepath.Separator) { | |
600 | // Special case the root directory to replace with an | |
601 | // empty string instead so that we don't end up with | |
602 | // double slashes in the paths. | |
603 | replacement = rebaseName | |
604 | } | |
605 | ||
606 | relFilePath = strings.Replace(relFilePath, include, replacement, 1) | |
607 | } | |
608 | ||
609 | if err := ta.addTarFile(filePath, relFilePath); err != nil { | |
610 | logrus.Debugf("Can't add file %s to tar: %s", filePath, err) | |
611 | } | |
612 | return nil | |
613 | }) | |
614 | } | |
615 | }() | |
616 | ||
617 | return pipeReader, nil | |
618 | } | |
619 | ||
620 | // Unpack unpacks the decompressedArchive to dest with options. | |
621 | func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { | |
622 | tr := tar.NewReader(decompressedArchive) | |
623 | trBuf := pools.BufioReader32KPool.Get(nil) | |
624 | defer pools.BufioReader32KPool.Put(trBuf) | |
625 | ||
626 | var dirs []*tar.Header | |
627 | remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) | |
628 | if err != nil { | |
629 | return err | |
630 | } | |
631 | ||
632 | // Iterate through the files in the archive. | |
633 | loop: | |
634 | for { | |
635 | hdr, err := tr.Next() | |
636 | if err == io.EOF { | |
637 | // end of tar archive | |
638 | break | |
639 | } | |
640 | if err != nil { | |
641 | return err | |
642 | } | |
643 | ||
644 | // Normalize name, for safety and for a simple is-root check | |
645 | // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: | |
646 | // This keeps "..\" as-is, but normalizes "\..\" to "\". | |
647 | hdr.Name = filepath.Clean(hdr.Name) | |
648 | ||
649 | for _, exclude := range options.ExcludePatterns { | |
650 | if strings.HasPrefix(hdr.Name, exclude) { | |
651 | continue loop | |
652 | } | |
653 | } | |
654 | ||
655 | // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in | |
656 | // the filepath format for the OS on which the daemon is running. Hence | |
657 | // the check for a slash-suffix MUST be done in an OS-agnostic way. | |
658 | if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { | |
659 | // Not the root directory, ensure that the parent directory exists | |
660 | parent := filepath.Dir(hdr.Name) | |
661 | parentPath := filepath.Join(dest, parent) | |
662 | if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { | |
663 | err = system.MkdirAll(parentPath, 0777) | |
664 | if err != nil { | |
665 | return err | |
666 | } | |
667 | } | |
668 | } | |
669 | ||
670 | path := filepath.Join(dest, hdr.Name) | |
671 | rel, err := filepath.Rel(dest, path) | |
672 | if err != nil { | |
673 | return err | |
674 | } | |
675 | if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { | |
676 | return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) | |
677 | } | |
678 | ||
679 | // If path exits we almost always just want to remove and replace it | |
680 | // The only exception is when it is a directory *and* the file from | |
681 | // the layer is also a directory. Then we want to merge them (i.e. | |
682 | // just apply the metadata from the layer). | |
683 | if fi, err := os.Lstat(path); err == nil { | |
684 | if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir { | |
685 | // If NoOverwriteDirNonDir is true then we cannot replace | |
686 | // an existing directory with a non-directory from the archive. | |
687 | return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest) | |
688 | } | |
689 | ||
690 | if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir { | |
691 | // If NoOverwriteDirNonDir is true then we cannot replace | |
692 | // an existing non-directory with a directory from the archive. | |
693 | return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest) | |
694 | } | |
695 | ||
696 | if fi.IsDir() && hdr.Name == "." { | |
697 | continue | |
698 | } | |
699 | ||
700 | if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { | |
701 | if err := os.RemoveAll(path); err != nil { | |
702 | return err | |
703 | } | |
704 | } | |
705 | } | |
706 | trBuf.Reset(tr) | |
707 | ||
708 | // if the options contain a uid & gid maps, convert header uid/gid | |
709 | // entries using the maps such that lchown sets the proper mapped | |
710 | // uid/gid after writing the file. We only perform this mapping if | |
711 | // the file isn't already owned by the remapped root UID or GID, as | |
712 | // that specific uid/gid has no mapping from container -> host, and | |
713 | // those files already have the proper ownership for inside the | |
714 | // container. | |
715 | if hdr.Uid != remappedRootUID { | |
716 | xUID, err := idtools.ToHost(hdr.Uid, options.UIDMaps) | |
717 | if err != nil { | |
718 | return err | |
719 | } | |
720 | hdr.Uid = xUID | |
721 | } | |
722 | if hdr.Gid != remappedRootGID { | |
723 | xGID, err := idtools.ToHost(hdr.Gid, options.GIDMaps) | |
724 | if err != nil { | |
725 | return err | |
726 | } | |
727 | hdr.Gid = xGID | |
728 | } | |
729 | ||
730 | if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts); err != nil { | |
731 | return err | |
732 | } | |
733 | ||
734 | // Directory mtimes must be handled at the end to avoid further | |
735 | // file creation in them to modify the directory mtime | |
736 | if hdr.Typeflag == tar.TypeDir { | |
737 | dirs = append(dirs, hdr) | |
738 | } | |
739 | } | |
740 | ||
741 | for _, hdr := range dirs { | |
742 | path := filepath.Join(dest, hdr.Name) | |
743 | ||
744 | if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { | |
745 | return err | |
746 | } | |
747 | } | |
748 | return nil | |
749 | } | |
750 | ||
751 | // Untar reads a stream of bytes from `archive`, parses it as a tar archive, | |
752 | // and unpacks it into the directory at `dest`. | |
753 | // The archive may be compressed with one of the following algorithms: | |
754 | // identity (uncompressed), gzip, bzip2, xz. | |
755 | // FIXME: specify behavior when target path exists vs. doesn't exist. | |
756 | func Untar(tarArchive io.Reader, dest string, options *TarOptions) error { | |
757 | return untarHandler(tarArchive, dest, options, true) | |
758 | } | |
759 | ||
760 | // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, | |
761 | // and unpacks it into the directory at `dest`. | |
762 | // The archive must be an uncompressed stream. | |
763 | func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error { | |
764 | return untarHandler(tarArchive, dest, options, false) | |
765 | } | |
766 | ||
767 | // Handler for teasing out the automatic decompression | |
768 | func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error { | |
769 | if tarArchive == nil { | |
770 | return fmt.Errorf("Empty archive") | |
771 | } | |
772 | dest = filepath.Clean(dest) | |
773 | if options == nil { | |
774 | options = &TarOptions{} | |
775 | } | |
776 | if options.ExcludePatterns == nil { | |
777 | options.ExcludePatterns = []string{} | |
778 | } | |
779 | ||
780 | r := tarArchive | |
781 | if decompress { | |
782 | decompressedArchive, err := DecompressStream(tarArchive) | |
783 | if err != nil { | |
784 | return err | |
785 | } | |
786 | defer decompressedArchive.Close() | |
787 | r = decompressedArchive | |
788 | } | |
789 | ||
790 | return Unpack(r, dest, options) | |
791 | } | |
792 | ||
793 | // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. | |
794 | // If either Tar or Untar fails, TarUntar aborts and returns the error. | |
795 | func (archiver *Archiver) TarUntar(src, dst string) error { | |
796 | logrus.Debugf("TarUntar(%s %s)", src, dst) | |
797 | archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) | |
798 | if err != nil { | |
799 | return err | |
800 | } | |
801 | defer archive.Close() | |
802 | ||
803 | var options *TarOptions | |
804 | if archiver.UIDMaps != nil || archiver.GIDMaps != nil { | |
805 | options = &TarOptions{ | |
806 | UIDMaps: archiver.UIDMaps, | |
807 | GIDMaps: archiver.GIDMaps, | |
808 | } | |
809 | } | |
810 | return archiver.Untar(archive, dst, options) | |
811 | } | |
812 | ||
813 | // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. | |
814 | // If either Tar or Untar fails, TarUntar aborts and returns the error. | |
815 | func TarUntar(src, dst string) error { | |
816 | return defaultArchiver.TarUntar(src, dst) | |
817 | } | |
818 | ||
819 | // UntarPath untar a file from path to a destination, src is the source tar file path. | |
820 | func (archiver *Archiver) UntarPath(src, dst string) error { | |
821 | archive, err := os.Open(src) | |
822 | if err != nil { | |
823 | return err | |
824 | } | |
825 | defer archive.Close() | |
826 | var options *TarOptions | |
827 | if archiver.UIDMaps != nil || archiver.GIDMaps != nil { | |
828 | options = &TarOptions{ | |
829 | UIDMaps: archiver.UIDMaps, | |
830 | GIDMaps: archiver.GIDMaps, | |
831 | } | |
832 | } | |
833 | return archiver.Untar(archive, dst, options) | |
834 | } | |
835 | ||
836 | // UntarPath is a convenience function which looks for an archive | |
837 | // at filesystem path `src`, and unpacks it at `dst`. | |
838 | func UntarPath(src, dst string) error { | |
839 | return defaultArchiver.UntarPath(src, dst) | |
840 | } | |
841 | ||
842 | // CopyWithTar creates a tar archive of filesystem path `src`, and | |
843 | // unpacks it at filesystem path `dst`. | |
844 | // The archive is streamed directly with fixed buffering and no | |
845 | // intermediary disk IO. | |
846 | func (archiver *Archiver) CopyWithTar(src, dst string) error { | |
847 | srcSt, err := os.Stat(src) | |
848 | if err != nil { | |
849 | return err | |
850 | } | |
851 | if !srcSt.IsDir() { | |
852 | return archiver.CopyFileWithTar(src, dst) | |
853 | } | |
854 | // Create dst, copy src's content into it | |
855 | logrus.Debugf("Creating dest directory: %s", dst) | |
856 | if err := system.MkdirAll(dst, 0755); err != nil { | |
857 | return err | |
858 | } | |
859 | logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) | |
860 | return archiver.TarUntar(src, dst) | |
861 | } | |
862 | ||
863 | // CopyWithTar creates a tar archive of filesystem path `src`, and | |
864 | // unpacks it at filesystem path `dst`. | |
865 | // The archive is streamed directly with fixed buffering and no | |
866 | // intermediary disk IO. | |
867 | func CopyWithTar(src, dst string) error { | |
868 | return defaultArchiver.CopyWithTar(src, dst) | |
869 | } | |
870 | ||
871 | // CopyFileWithTar emulates the behavior of the 'cp' command-line | |
872 | // for a single file. It copies a regular file from path `src` to | |
873 | // path `dst`, and preserves all its metadata. | |
874 | func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { | |
875 | logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) | |
876 | srcSt, err := os.Stat(src) | |
877 | if err != nil { | |
878 | return err | |
879 | } | |
880 | ||
881 | if srcSt.IsDir() { | |
882 | return fmt.Errorf("Can't copy a directory") | |
883 | } | |
884 | ||
885 | // Clean up the trailing slash. This must be done in an operating | |
886 | // system specific manner. | |
887 | if dst[len(dst)-1] == os.PathSeparator { | |
888 | dst = filepath.Join(dst, filepath.Base(src)) | |
889 | } | |
890 | // Create the holding directory if necessary | |
891 | if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil { | |
892 | return err | |
893 | } | |
894 | ||
895 | r, w := io.Pipe() | |
896 | errC := promise.Go(func() error { | |
897 | defer w.Close() | |
898 | ||
899 | srcF, err := os.Open(src) | |
900 | if err != nil { | |
901 | return err | |
902 | } | |
903 | defer srcF.Close() | |
904 | ||
905 | hdr, err := tar.FileInfoHeader(srcSt, "") | |
906 | if err != nil { | |
907 | return err | |
908 | } | |
909 | hdr.Name = filepath.Base(dst) | |
910 | hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) | |
911 | ||
912 | remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps) | |
913 | if err != nil { | |
914 | return err | |
915 | } | |
916 | ||
917 | // only perform mapping if the file being copied isn't already owned by the | |
918 | // uid or gid of the remapped root in the container | |
919 | if remappedRootUID != hdr.Uid { | |
920 | xUID, err := idtools.ToHost(hdr.Uid, archiver.UIDMaps) | |
921 | if err != nil { | |
922 | return err | |
923 | } | |
924 | hdr.Uid = xUID | |
925 | } | |
926 | if remappedRootGID != hdr.Gid { | |
927 | xGID, err := idtools.ToHost(hdr.Gid, archiver.GIDMaps) | |
928 | if err != nil { | |
929 | return err | |
930 | } | |
931 | hdr.Gid = xGID | |
932 | } | |
933 | ||
934 | tw := tar.NewWriter(w) | |
935 | defer tw.Close() | |
936 | if err := tw.WriteHeader(hdr); err != nil { | |
937 | return err | |
938 | } | |
939 | if _, err := io.Copy(tw, srcF); err != nil { | |
940 | return err | |
941 | } | |
942 | return nil | |
943 | }) | |
944 | defer func() { | |
945 | if er := <-errC; err != nil { | |
946 | err = er | |
947 | } | |
948 | }() | |
949 | ||
950 | err = archiver.Untar(r, filepath.Dir(dst), nil) | |
951 | if err != nil { | |
952 | r.CloseWithError(err) | |
953 | } | |
954 | return err | |
955 | } | |
956 | ||
957 | // CopyFileWithTar emulates the behavior of the 'cp' command-line | |
958 | // for a single file. It copies a regular file from path `src` to | |
959 | // path `dst`, and preserves all its metadata. | |
960 | // | |
961 | // Destination handling is in an operating specific manner depending | |
962 | // where the daemon is running. If `dst` ends with a trailing slash | |
963 | // the final destination path will be `dst/base(src)` (Linux) or | |
964 | // `dst\base(src)` (Windows). | |
965 | func CopyFileWithTar(src, dst string) (err error) { | |
966 | return defaultArchiver.CopyFileWithTar(src, dst) | |
967 | } | |
968 | ||
969 | // cmdStream executes a command, and returns its stdout as a stream. | |
970 | // If the command fails to run or doesn't complete successfully, an error | |
971 | // will be returned, including anything written on stderr. | |
972 | func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) { | |
973 | chdone := make(chan struct{}) | |
974 | cmd.Stdin = input | |
975 | pipeR, pipeW := io.Pipe() | |
976 | cmd.Stdout = pipeW | |
977 | var errBuf bytes.Buffer | |
978 | cmd.Stderr = &errBuf | |
979 | ||
980 | // Run the command and return the pipe | |
981 | if err := cmd.Start(); err != nil { | |
982 | return nil, nil, err | |
983 | } | |
984 | ||
985 | // Copy stdout to the returned pipe | |
986 | go func() { | |
987 | if err := cmd.Wait(); err != nil { | |
988 | pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) | |
989 | } else { | |
990 | pipeW.Close() | |
991 | } | |
992 | close(chdone) | |
993 | }() | |
994 | ||
995 | return pipeR, chdone, nil | |
996 | } | |
997 | ||
998 | // NewTempArchive reads the content of src into a temporary file, and returns the contents | |
999 | // of that file as an archive. The archive can only be read once - as soon as reading completes, | |
1000 | // the file will be deleted. | |
1001 | func NewTempArchive(src Archive, dir string) (*TempArchive, error) { | |
1002 | f, err := ioutil.TempFile(dir, "") | |
1003 | if err != nil { | |
1004 | return nil, err | |
1005 | } | |
1006 | if _, err := io.Copy(f, src); err != nil { | |
1007 | return nil, err | |
1008 | } | |
1009 | if _, err := f.Seek(0, 0); err != nil { | |
1010 | return nil, err | |
1011 | } | |
1012 | st, err := f.Stat() | |
1013 | if err != nil { | |
1014 | return nil, err | |
1015 | } | |
1016 | size := st.Size() | |
1017 | return &TempArchive{File: f, Size: size}, nil | |
1018 | } | |
1019 | ||
1020 | // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes, | |
1021 | // the file will be deleted. | |
1022 | type TempArchive struct { | |
1023 | *os.File | |
1024 | Size int64 // Pre-computed from Stat().Size() as a convenience | |
1025 | read int64 | |
1026 | closed bool | |
1027 | } | |
1028 | ||
1029 | // Close closes the underlying file if it's still open, or does a no-op | |
1030 | // to allow callers to try to close the TempArchive multiple times safely. | |
1031 | func (archive *TempArchive) Close() error { | |
1032 | if archive.closed { | |
1033 | return nil | |
1034 | } | |
1035 | ||
1036 | archive.closed = true | |
1037 | ||
1038 | return archive.File.Close() | |
1039 | } | |
1040 | ||
1041 | func (archive *TempArchive) Read(data []byte) (int, error) { | |
1042 | n, err := archive.File.Read(data) | |
1043 | archive.read += int64(n) | |
1044 | if err != nil || archive.read == archive.Size { | |
1045 | archive.Close() | |
1046 | os.Remove(archive.File.Name()) | |
1047 | } | |
1048 | return n, err | |
1049 | } |