From 8dbb5700d22282fb3d992061d93d139dd58b40d6 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Thu, 5 Dec 2024 12:04:08 +0100 Subject: [PATCH 01/48] invented option to preserve filemode --- README.md | 63 +++++++++++++++++++++++++----------------------- cmd/run.go | 2 ++ config.go | 20 ++++++++++++++- target.go | 29 +++++++++++++++++++--- target_disk.go | 5 ++++ target_memory.go | 16 ++++++++++++ unpack_test.go | 50 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 151 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 253da28b..41cb96bd 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,18 @@ go install github.com/hashicorp/go-extract/cmd/goextract@latest These examples demonstrate how to use [hashicorp/go-extract](https://github.com/hashicorp/go-extract) both as a library and as a command-line utility. +### Library + +The simplest way to use the library is to call the `extract.Unpack` function with the default configuration. This function extracts the contents from an `io.Reader` to the specified destination on the local filesystem. + +```go +// Unpack the archive +if err := extract.Unpack(ctx, dst, archive, config.NewConfig()); err != nil { + // Handle error + log.Fatalf("Failed to unpack archive: %v", err) +} +``` + ### Command-line Utility The `goextract` command-line utility offers all available configuration options via dedicated flags. @@ -57,54 +69,45 @@ Flags: --custom-decompress-file-mode=640 File mode for decompressed files. (respecting umask) -D, --deny-symlinks Deny symlink extraction. --insecure-traverse-symlinks Traverse symlinks to directories during extraction. - --max-files=1000 Maximum files (including folder and symlinks) that are extracted before stop. (disable check: -1) + --max-files=100000 Maximum files (including folder and symlinks) that are extracted before stop. (disable check: -1) --max-extraction-size=1073741824 Maximum extraction size that allowed is (in bytes). (disable check: -1) --max-extraction-time=60 Maximum time that an extraction should take (in seconds). (disable check: -1) --max-input-size=1073741824 Maximum input size that allowed is (in bytes). (disable check: -1) -N, --no-untar-after-decompression Disable combined extraction of tar.gz. -O, --overwrite Overwrite if exist. -P, --pattern=PATTERN,... Extracted objects need to match shell file name pattern. + --preserve-filemode Preserve file mode and overwrite umask. -T, --telemetry Print telemetry data to log after extraction. -t, --type="" Type of archive. (7z, br, bz2, gz, lz4, rar, sz, tar, tgz, xz, zip, zst, zz) -v, --verbose Verbose logging. -V, --version Print release version information. ``` -### Library - -The simplest way to use the library is to call the `extract.Unpack` function with the default configuration. This function extracts the contents from an `io.Reader` to the specified destination on the local filesystem. - -```go -// Unpack the archive -if err := extract.Unpack(ctx, dst, archive, config.NewConfig()); err != nil { - // Handle error - log.Fatalf("Failed to unpack archive: %v", err) -} -``` - ## Configuration When calling the `extract.Unpack(..)` function, we need to provide `config` object that contains all available configuration. ```golang // process cli params - cfg := config.NewConfig( - config.WithContinueOnError(..), - config.WithContinueOnUnsupportedFiles(..), - config.WithCreateDestination(..), - config.WithCustomCreateDirMode(..), - config.WithCustomDecompressFileMode(..), - config.WithDenySymlinkExtraction(..), - config.WithExtractType(..), - config.WithTraverseSymlinks(..), - config.WithLogger(..), - config.WithMaxExtractionSize(..), - config.WithMaxFiles(..), - config.WithMaxInputSize(..), - config.WithNoUntarAfterDecompression(..), - config.WithOverwrite(..), - config.WithPatterns(..), - config.WithTelemetryHook(..), + + cfg := extract.NewConfig( + extract.WithContinueOnError(..), + extract.WithContinueOnUnsupportedFiles(..), + extract.WithCreateDestination(..), + extract.WithCustomCreateDirMode(..), + extract.WithCustomDecompressFileMode(..), + extract.WithDenySymlinkExtraction(..), + extract.WithExtractType(..), + extract.WithInsecureTraverseSymlinks(..), + extract.WithLogger(..), + extract.WithMaxExtractionSize(..), + extract.WithMaxFiles(..), + extract.WithMaxInputSize(..), + extract.WithNoUntarAfterDecompression(..), + extract.WithOverwrite(..), + extract.WithPatterns(..), + extract.WithPreserveFilemode(..), + extract.WithTelemetryHook(..), ) [..] diff --git a/cmd/run.go b/cmd/run.go index cf93b199..928d10e5 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -38,6 +38,7 @@ type CLI struct { NoUntarAfterDecompression bool `short:"N" optional:"" default:"false" help:"Disable combined extraction of tar.gz."` Overwrite bool `short:"O" help:"Overwrite if exist."` Pattern []string `short:"P" optional:"" name:"pattern" help:"Extracted objects need to match shell file name pattern."` + PreserveFilemode bool `help:"Preserve file mode and overwrite umask."` Telemetry bool `short:"T" optional:"" default:"false" help:"Print telemetry data to log after extraction."` Type string `short:"t" optional:"" default:"${default_type}" name:"type" help:"Type of archive. (${valid_types})"` Verbose bool `short:"v" optional:"" help:"Verbose logging."` @@ -97,6 +98,7 @@ func Run(version, commit, date string) { extract.WithNoUntarAfterDecompression(cli.NoUntarAfterDecompression), extract.WithOverwrite(cli.Overwrite), extract.WithPatterns(cli.Pattern...), + extract.WithPreserveFilemode(cli.PreserveFilemode), extract.WithTelemetryHook(telemetryDataToLog), ) diff --git a/config.go b/config.go index fd607bab..7b1a1245 100644 --- a/config.go +++ b/config.go @@ -76,6 +76,9 @@ type Config struct { // patterns is a list of file patterns to match files to extract patterns []string + + // preserveFilemode is a flag to preserve the file mode of the extracted files + preserveFilemode bool } // ContinueOnError returns true if the extraction should continue on error. @@ -201,6 +204,12 @@ func (c *Config) Patterns() []string { return c.patterns } +// PreserveFilemode returns true if the file mode of the extracted files should be preserved and +// not adjusted to the umask. +func (c *Config) PreserveFilemode() bool { + return c.preserveFilemode +} + // SetNoUntarAfterDecompression sets the noUntarAfterDecompression flag. If true, tar.gz files // are not untared after decompression. func (c *Config) SetNoUntarAfterDecompression(b bool) { @@ -231,6 +240,7 @@ const ( defaultMaxInputSize = 1 << (10 * 3) // 1 Gb defaultNoUntarAfterDecompression = false // untar after decompression defaultOverwrite = false // do not overwrite existing files + defaultPreserveFilemode = false // do not preserve file mode defaultTraverseSymlinks = false // do not traverse symlinks ) @@ -252,6 +262,7 @@ func NewConfig(opts ...ConfigOption) *Config { config := &Config{ cacheInMemory: defaultCacheInMemory, continueOnError: defaultContinueOnError, + continueOnUnsupportedFiles: defaultContinueOnUnsupportedFiles, createDestination: defaultCreateDestination, customCreateDirMode: defaultCustomCreateDirMode, customDecompressFileMode: defaultCustomDecompressFileMode, @@ -265,7 +276,7 @@ func NewConfig(opts ...ConfigOption) *Config { telemetryHook: defaultTelemetryHook, traverseSymlinks: defaultTraverseSymlinks, noUntarAfterDecompression: defaultNoUntarAfterDecompression, - continueOnUnsupportedFiles: defaultContinueOnUnsupportedFiles, + preserveFilemode: defaultPreserveFilemode, } // Loop through each option @@ -405,6 +416,13 @@ func WithPatterns(pattern ...string) ConfigOption { } } +// WithPreserveFilemode options pattern function to preserve the file mode of the extracted files. +func WithPreserveFilemode(preserve bool) ConfigOption { + return func(c *Config) { + c.preserveFilemode = preserve + } +} + // WithTelemetryHook options pattern function to set a [telemetry.TelemetryHook], which is called after extraction. func WithTelemetryHook(hook TelemetryHook) ConfigOption { return func(c *Config) { diff --git a/target.go b/target.go index e30f4f5a..7f5f726b 100644 --- a/target.go +++ b/target.go @@ -38,6 +38,9 @@ type Target interface { // Stat see docs for os.Stat. Main purpose is to check if a symlink is pointing to a file or directory. Stat(path string) (fs.FileInfo, error) + + // Chmod see docs for os.Chmod. Main purpose is to set the file mode of a file or directory. + Chmod(name string, mode fs.FileMode) error } // createFile is a wrapper around the CreateFile function @@ -73,7 +76,18 @@ func createFile(t Target, dst string, name string, src io.Reader, mode fs.FileMo return 0, fmt.Errorf("cannot create directory: %w", err) } - return t.CreateFile(filepath.Join(dst, name), src, mode, cfg.Overwrite(), maxSize) + path := filepath.Join(dst, name) + n, err := t.CreateFile(path, src, mode, cfg.Overwrite(), maxSize) + if err != nil { + return n, fmt.Errorf("failed to create file: %w", err) + } + if cfg.PreserveFilemode() { + if err := t.Chmod(path, mode); err != nil { + return n, fmt.Errorf("failed to set file mode: %w", err) + } + } + + return n, nil } // createDir is a wrapper around the CreateDir function @@ -120,7 +134,17 @@ func createDir(t Target, dst string, name string, mode fs.FileMode, cfg *Config) name = filepath.Join(parts...) path := filepath.Join(dst, name) - return t.CreateDir(path, mode) + if err := t.CreateDir(path, mode); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + if cfg.PreserveFilemode() { + if err := t.Chmod(path, mode); err != nil { + return fmt.Errorf("failed to set file mode: %w", err) + } + } + + return nil } // createSymlink is a wrapper around the CreateSymlink function @@ -191,7 +215,6 @@ func createSymlink(t Target, dst string, name string, linkTarget string, cfg *Co // create symlink return t.CreateSymlink(linkTarget, filepath.Join(dst, name), cfg.Overwrite()) - } // securityCheck checks if the targetDirectory contains path traversal diff --git a/target_disk.go b/target_disk.go index 6b9e0618..d72210bb 100644 --- a/target_disk.go +++ b/target_disk.go @@ -108,3 +108,8 @@ func (d *TargetDisk) Lstat(name string) (fs.FileInfo, error) { func (d *TargetDisk) Stat(name string) (os.FileInfo, error) { return os.Stat(name) } + +// Chmod changes the mode of the named file to mode. +func (d *TargetDisk) Chmod(name string, mode fs.FileMode) error { + return os.Chmod(name, mode.Perm()) +} diff --git a/target_memory.go b/target_memory.go index ddc34c28..83be0d9c 100644 --- a/target_memory.go +++ b/target_memory.go @@ -240,6 +240,22 @@ func (m *TargetMemory) Open(path string) (fs.File, error) { return &fileEntry{memoryEntry: me, reader: bytes.NewReader(me.data)}, nil } +// Chmod changes the mode of the file at the given path. If the file does not exist, an error is returned. +func (m *TargetMemory) Chmod(path string, mode fs.FileMode) error { + if !fs.ValidPath(path) { + return &fs.PathError{Op: "Chmod", Path: path, Err: fs.ErrInvalid} + } + me, err := m.resolveEntry(path) + if err != nil { + return &fs.PathError{Op: "Chmod", Path: path, Err: err} + } + me.lock.Lock() + defer me.lock.Unlock() + // inverse & with 0777 to remove the file mode bits and then or with the new mode bits + me.fileInfo.(*memoryFileInfo).mode = (me.fileInfo.(*memoryFileInfo).mode &^ 0777) | mode.Perm() + return nil +} + type dirEntry struct { *memoryEntry memory *TargetMemory diff --git a/unpack_test.go b/unpack_test.go index 348eb895..7c2690d2 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1325,6 +1325,56 @@ func TestHasKnownArchiveExtension(t *testing.T) { } } +func TestUnpackWithPreserveFilemode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows systems") + } + testCases := []struct { + name string + permissions fs.FileMode + cfg *extract.Config + expectEq bool + }{ + { + name: "file with permissions 0777, with applied umask", + permissions: 0777, + cfg: extract.NewConfig(extract.WithPreserveFilemode(false)), + expectEq: false, + }, + { + name: "file with permissions 0777, overwrite umask", + permissions: 0777, + cfg: extract.NewConfig(extract.WithPreserveFilemode(true)), + expectEq: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ( + ctx = context.Background() + src = asIoReader(t, packTar(t, []archiveContent{{Name: "test", Content: []byte("hello world"), Mode: tc.permissions}})) + dst = t.TempDir() + cfg = tc.cfg + ) + if err := extract.Unpack(ctx, dst, src, cfg); err != nil { + t.Fatalf("error unpacking archive: %v", err) + } + var winPerm = fmt.Sprintf("%v", toWindowsFileMode(false, tc.permissions)) + t.Logf("expected equal: %v ; archive file mode %s", tc.expectEq, winPerm) + stat, err := os.Stat(filepath.Join(dst, "test")) + if err != nil { + t.Fatalf("error getting file stats: %v", err) + } + eqPem := stat.Mode().Perm() == tc.permissions + if eqPem != tc.expectEq { + t.Fatalf("expected equal: %v ; archive file mode %s, got %s", tc.expectEq, tc.permissions, stat.Mode().Perm()) + } + }) + } + +} + func compressBrotli(t *testing.T, data []byte) []byte { t.Helper() b := new(bytes.Buffer) From 49b513b34da5f5dfb2b06701df5694c713578e86 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 9 Dec 2024 06:33:24 +0100 Subject: [PATCH 02/48] added support for updating ownership, file permissions and change time --- 7zip.go | 38 +++++++++- README.md | 28 ++++---- archive_walker.go | 18 +++-- cmd/run.go | 4 +- config.go | 21 +++--- extractor.go | 53 +++++++++++++- go.mod | 1 + go.sum | 2 + rar.go | 66 ++++++++++++------ tar.go | 26 +++++++ target.go | 22 +++--- target_disk.go | 22 ++++++ target_disk_others.go | 30 ++++++++ target_disk_unix.go | 34 +++++++++ target_memory.go | 107 +++++++++++++++++++++++----- target_memory_test.go | 158 ++++++++++++++++++++++++++++++++++++++++++ unpack_test.go | 122 +++++++++++++++++++++++++------- zip.go | 26 +++++++ 18 files changed, 663 insertions(+), 115 deletions(-) create mode 100644 target_disk_others.go create mode 100644 target_disk_unix.go diff --git a/7zip.go b/7zip.go index 2d295c6d..ae9cb2bb 100644 --- a/7zip.go +++ b/7zip.go @@ -9,6 +9,7 @@ import ( "io" "io/fs" "os" + "time" "github.com/bodgit/sevenzip" ) @@ -122,9 +123,14 @@ func (z *sevenZipEntry) Mode() os.FileMode { } // Linkname returns the linkname of the 7zip entry -// Remark: 7zip does not support symlinks func (z *sevenZipEntry) Linkname() string { - return "" + if !z.IsSymlink() { + return "" + } + f, _ := z.f.Open() + defer f.Close() + c, _ := io.ReadAll(f) + return string(c) } // IsRegular returns true if the 7zip entry is a regular file @@ -140,7 +146,7 @@ func (z *sevenZipEntry) IsDir() bool { // IsSymlink returns true if the 7zip entry is a symlink // Remark: 7zip does not support symlinks func (z *sevenZipEntry) IsSymlink() bool { - return false + return (z.f.FileInfo().Mode()&os.ModeSymlink != 0) } // Open returns a reader for the 7zip entry @@ -152,3 +158,29 @@ func (z *sevenZipEntry) Open() (io.ReadCloser, error) { func (z *sevenZipEntry) Type() fs.FileMode { return fs.FileMode(z.f.FileInfo().Mode()) } + +// AccessTime returns the access time of the 7zip entry +func (z *sevenZipEntry) AccessTime() time.Time { + return z.f.Accessed +} + +// ModTime returns the modification time of the 7zip entry +func (z *sevenZipEntry) ModTime() time.Time { + return z.f.Modified +} + +// Sys returns the system information of the 7zip entry +func (z *sevenZipEntry) Sys() interface{} { + return z.f.FileInfo().Sys() +} + +// Gid returns the group ID of the 7zip entry +func (z *sevenZipEntry) Gid() int { + return os.Getgid() +} + +// Uid returns the user ID of the 7zip entry +func (z *sevenZipEntry) Uid() int { + // get current uid + return os.Getuid() +} diff --git a/README.md b/README.md index 41cb96bd..e1ea5146 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,6 @@ Flags: When calling the `extract.Unpack(..)` function, we need to provide `config` object that contains all available configuration. ```golang - // process cli params - cfg := extract.NewConfig( extract.WithContinueOnError(..), extract.WithContinueOnUnsupportedFiles(..), @@ -106,7 +104,7 @@ When calling the `extract.Unpack(..)` function, we need to provide `config` obje extract.WithNoUntarAfterDecompression(..), extract.WithOverwrite(..), extract.WithPatterns(..), - extract.WithPreserveFilemode(..), + extract.WithPreserveFileAttributes(..), extract.WithTelemetryHook(..), ) @@ -135,18 +133,18 @@ Here is an example collected telemetry data for the extraction of [`terraform-aw ```json { - "LastExtractionError": "", - "ExtractedDirs": 51, - "ExtractionDuration": 48598584, - "ExtractionErrors": 0, - "ExtractedFiles": 241, - "ExtractionSize": 539085, - "ExtractedSymlinks": 0, - "ExtractedType": "tar+gzip", - "InputSize": 81477, - "PatternMismatches": 0, - "UnsupportedFiles": 0, - "LastUnsupportedFile": "" + "last_extraction_error": "", + "extracted_dirs": 51, + "extraction_duration": 55025584, + "extraction_errors": 0, + "extracted_files": 241, + "extraction_size": 539085, + "extracted_symlinks": 0, + "extracted_type": "tar.gz", + "input_size": 81477, + "pattern_mismatches": 0, + "unsupported_files": 0, + "last_unsupported_file": "" } ``` diff --git a/archive_walker.go b/archive_walker.go index 8948e260..b65de83c 100644 --- a/archive_walker.go +++ b/archive_walker.go @@ -6,6 +6,7 @@ package extract import ( "io" "io/fs" + "time" ) // archiveWalker is an interface that represents a file walker in an archive @@ -16,13 +17,18 @@ type archiveWalker interface { // archiveEntry is an interface that represents a file in an archive type archiveEntry interface { - Mode() fs.FileMode - Type() fs.FileMode - Name() string - Linkname() string - Size() int64 - Open() (io.ReadCloser, error) + AccessTime() time.Time + Gid() int IsRegular() bool IsDir() bool IsSymlink() bool + Linkname() string + Mode() fs.FileMode + ModTime() time.Time + Name() string + Open() (io.ReadCloser, error) + Size() int64 + Sys() interface{} + Type() fs.FileMode + Uid() int } diff --git a/cmd/run.go b/cmd/run.go index 928d10e5..15e56252 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -38,7 +38,7 @@ type CLI struct { NoUntarAfterDecompression bool `short:"N" optional:"" default:"false" help:"Disable combined extraction of tar.gz."` Overwrite bool `short:"O" help:"Overwrite if exist."` Pattern []string `short:"P" optional:"" name:"pattern" help:"Extracted objects need to match shell file name pattern."` - PreserveFilemode bool `help:"Preserve file mode and overwrite umask."` + PreserveFileAttributes bool `short:"p" help:"Preserve file attributes from archive (access and modification time, file permissions and owner/group)."` Telemetry bool `short:"T" optional:"" default:"false" help:"Print telemetry data to log after extraction."` Type string `short:"t" optional:"" default:"${default_type}" name:"type" help:"Type of archive. (${valid_types})"` Verbose bool `short:"v" optional:"" help:"Verbose logging."` @@ -98,7 +98,7 @@ func Run(version, commit, date string) { extract.WithNoUntarAfterDecompression(cli.NoUntarAfterDecompression), extract.WithOverwrite(cli.Overwrite), extract.WithPatterns(cli.Pattern...), - extract.WithPreserveFilemode(cli.PreserveFilemode), + extract.WithPreserveFileAttributes(cli.PreserveFileAttributes), extract.WithTelemetryHook(telemetryDataToLog), ) diff --git a/config.go b/config.go index 7b1a1245..7e170ee9 100644 --- a/config.go +++ b/config.go @@ -77,8 +77,8 @@ type Config struct { // patterns is a list of file patterns to match files to extract patterns []string - // preserveFilemode is a flag to preserve the file mode of the extracted files - preserveFilemode bool + // preserveFileAttributes is a flag to preserve the file attributes of the extracted files + preserveFileAttributes bool } // ContinueOnError returns true if the extraction should continue on error. @@ -204,10 +204,9 @@ func (c *Config) Patterns() []string { return c.patterns } -// PreserveFilemode returns true if the file mode of the extracted files should be preserved and -// not adjusted to the umask. -func (c *Config) PreserveFilemode() bool { - return c.preserveFilemode +// PreserveFileAttributes returns true if the file attributes of the extracted files should be preserved. +func (c *Config) PreserveFileAttributes() bool { + return c.preserveFileAttributes } // SetNoUntarAfterDecompression sets the noUntarAfterDecompression flag. If true, tar.gz files @@ -240,7 +239,7 @@ const ( defaultMaxInputSize = 1 << (10 * 3) // 1 Gb defaultNoUntarAfterDecompression = false // untar after decompression defaultOverwrite = false // do not overwrite existing files - defaultPreserveFilemode = false // do not preserve file mode + defaultPreserveFileAttributes = false // do not preserve file attributes defaultTraverseSymlinks = false // do not traverse symlinks ) @@ -276,7 +275,7 @@ func NewConfig(opts ...ConfigOption) *Config { telemetryHook: defaultTelemetryHook, traverseSymlinks: defaultTraverseSymlinks, noUntarAfterDecompression: defaultNoUntarAfterDecompression, - preserveFilemode: defaultPreserveFilemode, + preserveFileAttributes: defaultPreserveFileAttributes, } // Loop through each option @@ -416,10 +415,10 @@ func WithPatterns(pattern ...string) ConfigOption { } } -// WithPreserveFilemode options pattern function to preserve the file mode of the extracted files. -func WithPreserveFilemode(preserve bool) ConfigOption { +// WithPreserveFileAttributes options pattern function to preserve the file attributes of the extracted files. +func WithPreserveFileAttributes(preserve bool) ConfigOption { return func(c *Config) { - c.preserveFilemode = preserve + c.preserveFileAttributes = preserve } } diff --git a/extractor.go b/extractor.go index c23bcae6..bafa6233 100644 --- a/extractor.go +++ b/extractor.go @@ -270,6 +270,27 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * var fileCounter int64 var extractionSize int64 + // check if attributes should be preserved, but as non-root user + if _, ok := t.(*TargetDisk); ok { + if cfg.PreserveFileAttributes() && os.Geteuid() != 0 { + cfg.Logger().Warn("cannot fully preserve file attributes as non-root user: cannot set file ownership", "uid", os.Geteuid()) + } + } + + // set attributes after all modification are done to ensure that + // the timestamps are set correctly + var extractedEntries []archiveEntry + if cfg.PreserveFileAttributes() { + defer func() { + for _, ae := range extractedEntries { + path := filepath.Join(dst, ae.Name()) + if err := setFileAttributes(t, path, ae); err != nil { + cfg.Logger().Error("failed to set file attributes", "path", path, "error", err) + } + } + }() + } + for { // check if context is canceled if ctx.Err() != nil { @@ -332,10 +353,12 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * // do not end on error continue } + if cfg.PreserveFileAttributes() { + extractedEntries = append(extractedEntries, ae) + } // store telemetry and continue td.ExtractedDirs++ - continue // if it's a file create it case ae.IsRegular(): @@ -373,8 +396,10 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * // store telemetry if fileCreated { td.ExtractedFiles++ + if cfg.PreserveFileAttributes() { + extractedEntries = append(extractedEntries, ae) + } } - continue // its a symlink !! case ae.IsSymlink(): @@ -402,10 +427,12 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * // do not end on error continue } + if cfg.PreserveFileAttributes() { + extractedEntries = append(extractedEntries, ae) + } // store telemetry and continue td.ExtractedSymlinks++ - continue default: @@ -426,6 +453,26 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * } } +// setFileAttributes sets the file attributes for the given path and archive entry. +func setFileAttributes(t Target, path string, ae archiveEntry) error { + if ae.IsSymlink() { // only time attributes are supported for symlinks + if err := t.Lchtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { + return fmt.Errorf("failed to lchtimes symlink: %w", err) + } + return nil + } + if err := t.Chown(path, ae.Uid(), ae.Gid()); err != nil { + return fmt.Errorf("failed to chown file: %w", err) + } + if err := t.Chmod(path, ae.Mode().Perm()); err != nil { + return fmt.Errorf("failed to chmod file: %w", err) + } + if err := t.Chtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { + return fmt.Errorf("failed to chtimes file: %w", err) + } + return nil +} + // readerToReaderAtSeeker converts an io.Reader to an io.ReaderAt and io.Seeker func readerToReaderAtSeeker(c *Config, r io.Reader) (seekerReaderAt, error) { if s, ok := r.(seekerReaderAt); ok { diff --git a/go.mod b/go.mod index 2b44b8ab..0af27182 100644 --- a/go.mod +++ b/go.mod @@ -21,5 +21,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.20.0 // indirect ) diff --git a/go.sum b/go.sum index c77a3b9d..818b389d 100644 --- a/go.sum +++ b/go.sum @@ -192,6 +192,8 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/rar.go b/rar.go index 3937482d..383ef7a8 100644 --- a/rar.go +++ b/rar.go @@ -8,6 +8,7 @@ import ( "io" "io/fs" "os" + "time" "github.com/nwaples/rardecode" ) @@ -82,9 +83,9 @@ func (rw *rarWalker) Next() (archiveEntry, error) { return nil, err } re := &rarEntry{fh, rw.r} - if re.IsSymlink() { // symlink not supported - return nil, unsupportedFile(re.Name()) - } + // if re.IsSymlink() { // symlink not supported + // return nil, unsupportedFile(re.Name()) + // } return re, nil } @@ -95,46 +96,71 @@ type rarEntry struct { } // Name returns the name of the file. -func (re *rarEntry) Name() string { - return re.f.Name +func (r *rarEntry) Name() string { + return r.f.Name } // Size returns the size of the file. -func (re *rarEntry) Size() int64 { - return re.f.UnPackedSize +func (r *rarEntry) Size() int64 { + return r.f.UnPackedSize } // Mode returns the mode of the file. -func (z *rarEntry) Mode() os.FileMode { - return z.f.Mode() +func (r *rarEntry) Mode() os.FileMode { + return r.f.Mode() } // Linkname symlinks are not supported. -func (re *rarEntry) Linkname() string { +func (r *rarEntry) Linkname() string { return "" } // IsRegular returns true if the file is a regular file. -func (re *rarEntry) IsRegular() bool { - return re.f.Mode().IsRegular() +func (r *rarEntry) IsRegular() bool { + return r.f.Mode().IsRegular() } // IsDir returns true if the file is a directory. -func (z *rarEntry) IsDir() bool { - return z.f.IsDir +func (r *rarEntry) IsDir() bool { + return r.f.IsDir } // IsSymlink returns true if the file is a symlink. -func (z *rarEntry) IsSymlink() bool { - return z.f.Mode()&fs.ModeSymlink != 0 +func (r *rarEntry) IsSymlink() bool { + return false } // Type returns the type of the file. -func (z *rarEntry) Type() fs.FileMode { - return z.f.Mode().Type() +func (r *rarEntry) Type() fs.FileMode { + return r.f.Mode().Type() } // Open returns a reader for the file. -func (z *rarEntry) Open() (io.ReadCloser, error) { - return io.NopCloser(z.r), nil +func (r *rarEntry) Open() (io.ReadCloser, error) { + return io.NopCloser(r.r), nil +} + +// AccessTime returns the access time of the file. +func (r *rarEntry) AccessTime() time.Time { + return r.f.AccessTime +} + +// ModTime returns the modification time of the file. +func (r *rarEntry) ModTime() time.Time { + return r.f.ModificationTime +} + +// Sys returns the system information of the file. +func (r *rarEntry) Sys() interface{} { + return r.f +} + +// Gid returns the group ID of the file. +func (r *rarEntry) Gid() int { + return os.Getgid() +} + +// Uid returns the user ID of the file. +func (r *rarEntry) Uid() int { + return os.Getuid() } diff --git a/tar.go b/tar.go index 9f80e5fe..e501ab6e 100644 --- a/tar.go +++ b/tar.go @@ -9,6 +9,7 @@ import ( "io" "io/fs" "os" + "time" ) // fileExtensionTar is the file extension for tar files @@ -118,3 +119,28 @@ func (t *tarEntry) Open() (io.ReadCloser, error) { func (t *tarEntry) Type() fs.FileMode { return fs.FileMode(t.hdr.Typeflag) } + +// AccessTime returns the access time of the entry +func (t *tarEntry) AccessTime() time.Time { + return t.hdr.AccessTime +} + +// ModTime returns the modification time of the entry +func (t *tarEntry) ModTime() time.Time { + return t.hdr.ModTime +} + +// Sys returns the system information of the entry +func (t *tarEntry) Sys() interface{} { + return t.hdr +} + +// Gid returns the group id of the entry +func (t *tarEntry) Gid() int { + return t.hdr.Gid +} + +// Uid returns the user id of the entry +func (t *tarEntry) Uid() int { + return t.hdr.Uid +} diff --git a/target.go b/target.go index 7f5f726b..464253ec 100644 --- a/target.go +++ b/target.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strings" + "time" ) // Target specifies all function that are needed to be implemented to extract contents from an archive @@ -41,6 +42,15 @@ type Target interface { // Chmod see docs for os.Chmod. Main purpose is to set the file mode of a file or directory. Chmod(name string, mode fs.FileMode) error + + // Chtimes see docs for os.Chtimes. Main purpose is to set the file times of a file or directory. + Chtimes(name string, atime, mtime time.Time) error + + // Lchtimes see docs for os.Lchtimes. Main purpose is to set the file times of a file or directory. + Lchtimes(name string, atime, mtime time.Time) error + + // Chown see docs for os.Chown. Main purpose is to set the file owner and group of a file or directory. + Chown(name string, uid, gid int) error } // createFile is a wrapper around the CreateFile function @@ -81,12 +91,6 @@ func createFile(t Target, dst string, name string, src io.Reader, mode fs.FileMo if err != nil { return n, fmt.Errorf("failed to create file: %w", err) } - if cfg.PreserveFilemode() { - if err := t.Chmod(path, mode); err != nil { - return n, fmt.Errorf("failed to set file mode: %w", err) - } - } - return n, nil } @@ -138,12 +142,6 @@ func createDir(t Target, dst string, name string, mode fs.FileMode, cfg *Config) return fmt.Errorf("failed to create directory: %w", err) } - if cfg.PreserveFilemode() { - if err := t.Chmod(path, mode); err != nil { - return fmt.Errorf("failed to set file mode: %w", err) - } - } - return nil } diff --git a/target_disk.go b/target_disk.go index d72210bb..597715a3 100644 --- a/target_disk.go +++ b/target_disk.go @@ -8,6 +8,7 @@ import ( "io" "io/fs" "os" + "time" ) // TargetDisk is the struct type that holds all information for interacting with the filesystem @@ -113,3 +114,24 @@ func (d *TargetDisk) Stat(name string) (os.FileInfo, error) { func (d *TargetDisk) Chmod(name string, mode fs.FileMode) error { return os.Chmod(name, mode.Perm()) } + +// Chtimes changes the access and modification times of the named file. +func (d *TargetDisk) Chtimes(name string, atime, mtime time.Time) error { + return os.Chtimes(name, atime, mtime) +} + +// Chown changes the numeric uid and gid of the named file. +func (d *TargetDisk) Chown(name string, uid, gid int) error { + if os.Geteuid() != 0 { + return nil + } + return os.Lchown(name, uid, gid) +} + +// Lchtimes changes the access and modification times of the named file. +func (d *TargetDisk) Lchtimes(name string, atime, mtime time.Time) error { + if CanMaintainSymlinkTimestamps() { + return Lchtimes(name, atime, mtime) + } + return nil +} diff --git a/target_disk_others.go b/target_disk_others.go new file mode 100644 index 00000000..3d4ed04a --- /dev/null +++ b/target_disk_others.go @@ -0,0 +1,30 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd + +package extract + +import ( + "fmt" + "runtime" + "time" +) + +// Lchtimes modifies the access and modified timestamps on a target path +// This capability is only available on unix as of now. +func Lchtimes(path string, atime, mtime time.Time) error { + return fmt.Errorf("Lchtimes is not supported on this platform (%s)", runtime.GOOS) +} + +// CanMaintainSymlinkTimestamps determines whether is is possible to change +// timestamps on symlinks for the the current platform. For regular files +// and directories, attempts are made to restore permissions and timestamps +// after extraction. But for symbolic links, go's cross-platform +// packages (Chmod and Chtimes) are not capable of changing symlink info +// because those methods follow the symlinks. However, a platform-dependent option +// is provided for unix (see Lchtimes) +func CanMaintainSymlinkTimestamps() bool { + return false +} diff --git a/target_disk_unix.go b/target_disk_unix.go new file mode 100644 index 00000000..b87cd86f --- /dev/null +++ b/target_disk_unix.go @@ -0,0 +1,34 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd +// +build darwin dragonfly freebsd linux netbsd openbsd + +package extract + +import ( + "time" + + "golang.org/x/sys/unix" +) + +// Lchtimes modifies the access and modified timestamps on a target path +// This capability is only available on unix as of now. +func Lchtimes(path string, atime, mtime time.Time) error { + + return unix.Lutimes(path, []unix.Timeval{ + {Sec: atime.Unix(), Usec: int32(atime.Nanosecond() / 1e6 % 1e6)}, + {Sec: mtime.Unix(), Usec: int32(mtime.Nanosecond() / 1e6 % 1e6)}, + }) +} + +// CanMaintainSymlinkTimestamps determines whether is is possible to change +// timestamps on symlinks for the the current platform. For regular files +// and directories, attempts are made to restore permissions and timestamps +// after extraction. But for symbolic links, go's cross-platform +// packages (Chmod and Chtimes) are not capable of changing symlink info +// because those methods follow the symlinks. However, a platform-dependent option +// is provided for unix (see Lchtimes) +func CanMaintainSymlinkTimestamps() bool { + return true +} diff --git a/target_memory.go b/target_memory.go index 83be0d9c..fe2acdf2 100644 --- a/target_memory.go +++ b/target_memory.go @@ -100,7 +100,7 @@ func (m *TargetMemory) createFile(path string, mode fs.FileMode, src io.Reader, // create entry m.files.Store(path, &memoryEntry{ - fileInfo: &memoryFileInfo{name: name, size: n, mode: mode.Perm(), modTime: time.Now()}, + fileInfo: &memoryFileInfo{name: name, size: n, mode: mode.Perm(), accessTime: time.Now(), modTime: time.Now()}, data: buf.Bytes(), lock: sync.RWMutex{}, }) @@ -256,6 +256,57 @@ func (m *TargetMemory) Chmod(path string, mode fs.FileMode) error { return nil } +// Chtime changes the access and modification times of the file at the given path. +// If the file does not exist, an error is returned. +func (m *TargetMemory) Chtimes(path string, atime time.Time, mtime time.Time) error { + if !fs.ValidPath(path) { + return &fs.PathError{Op: "Chtimes", Path: path, Err: fs.ErrInvalid} + } + me, err := m.resolveEntry(path) + if err != nil { + return &fs.PathError{Op: "Chtimes", Path: path, Err: err} + } + me.lock.Lock() + defer me.lock.Unlock() + me.fileInfo.(*memoryFileInfo).accessTime = atime + me.fileInfo.(*memoryFileInfo).modTime = mtime + return nil +} + +// Chown changes the owner and group of the file at the given path. +// If the file does not exist, an error is returned. +func (m *TargetMemory) Chown(path string, uid, gid int) error { + if !fs.ValidPath(path) { + return &fs.PathError{Op: "Chtimes", Path: path, Err: fs.ErrInvalid} + } + me, err := m.resolveEntry(path) + if err != nil { + return &fs.PathError{Op: "Chtimes", Path: path, Err: err} + } + me.lock.Lock() + defer me.lock.Unlock() + me.fileInfo.(*memoryFileInfo).uid = uid + me.fileInfo.(*memoryFileInfo).gid = gid + return nil +} + +// Lchtimes changes the access and modification times of the file at the given path. +// If the file does not exist, an error is returned. +func (m *TargetMemory) Lchtimes(path string, atime time.Time, mtime time.Time) error { + if !fs.ValidPath(path) { + return &fs.PathError{Op: "Lchtimes", Path: path, Err: fs.ErrInvalid} + } + me, err := m.resolveEntry(path) + if err != nil { + return &fs.PathError{Op: "Lchtimes", Path: path, Err: err} + } + me.lock.Lock() + defer me.lock.Unlock() + me.fileInfo.(*memoryFileInfo).accessTime = atime + me.fileInfo.(*memoryFileInfo).modTime = mtime + return nil +} + type dirEntry struct { *memoryEntry memory *TargetMemory @@ -466,11 +517,13 @@ func (m *TargetMemory) Lstat(path string) (fs.FileInfo, error) { } // return file info copy + mfi := me.fileInfo.(*memoryFileInfo) return &memoryFileInfo{ - name: me.fileInfo.Name(), - size: me.fileInfo.Size(), - mode: me.fileInfo.Mode(), - modTime: me.fileInfo.ModTime(), + name: mfi.Name(), + size: mfi.Size(), + mode: mfi.Mode(), + accessTime: mfi.AccessTime(), + modTime: mfi.ModTime(), }, nil } @@ -733,20 +786,28 @@ func (me *memoryEntry) Info() (fs.FileInfo, error) { // memoryFileInfo is a FileInfo implementation for the in-memory filesystem type memoryFileInfo struct { - name string - size int64 - mode fs.FileMode - modTime time.Time + accessTime time.Time + gid int + name string + mode fs.FileMode + modTime time.Time + size int64 + uid int } -// Name implements [io/fs.FileInfo] interface -func (fi *memoryFileInfo) Name() string { - return fi.name +// AccessTime returns the access time of the file +func (fi *memoryFileInfo) AccessTime() time.Time { + return fi.accessTime } -// Size implements [io/fs.FileInfo] interface -func (fi *memoryFileInfo) Size() int64 { - return fi.size +// Gid returns the group id of the file +func (fi *memoryFileInfo) Gid() int { + return fi.gid +} + +// IsDir implements [io/fs.FileInfo] interface +func (fi *memoryFileInfo) IsDir() bool { + return fi.mode.IsDir() } // Mode implements [io/fs.FileInfo] interface @@ -759,12 +820,22 @@ func (fi *memoryFileInfo) ModTime() time.Time { return fi.modTime } -// IsDir implements [io/fs.FileInfo] interface -func (fi *memoryFileInfo) IsDir() bool { - return fi.mode.IsDir() +// Name implements [io/fs.FileInfo] interface +func (fi *memoryFileInfo) Name() string { + return fi.name +} + +// Size implements [io/fs.FileInfo] interface +func (fi *memoryFileInfo) Size() int64 { + return fi.size } // Sys implements [io/fs.FileInfo] interface, but returns always nil func (fi *memoryFileInfo) Sys() any { return nil } + +// Uid returns the user id of the file +func (fi *memoryFileInfo) Uid() int { + return fi.uid +} diff --git a/target_memory_test.go b/target_memory_test.go index c3a7170f..2acaf11e 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -5,11 +5,14 @@ package extract_test import ( "bytes" + "context" "io" "io/fs" + "os" p "path" "testing" "testing/fstest" + "time" extract "github.com/hashicorp/go-extract" ) @@ -710,6 +713,16 @@ func TestCreateFile(t *testing.T) { t.Fatalf("CreateFile() failed: %s", err) } + // create the same file, but fail bc it already exists# + if _, err := tm.CreateFile(testPath, bytes.NewReader([]byte(testContent)), fs.FileMode(testPerm), false, -1); err == nil { + t.Fatalf("CreateFile() failed: expected error, got nil") + } + + // create the same file, but overwrite + if _, err := tm.CreateFile(testPath, bytes.NewReader([]byte(testContent)), fs.FileMode(testPerm), true, -1); err != nil { + t.Fatalf("CreateFile() failed: %s", err) + } + // open the file f, err := tm.Open(testPath) if err != nil { @@ -758,3 +771,148 @@ func TestCreateFile(t *testing.T) { t.Fatalf("CreateFile() failed: expected error, got nil") } } + +// TestCreateSymlink tests the CreateSymlink method +func TestCreateSymlink(t *testing.T) { + // instantiate a new memory + tm := extract.NewTargetMemory() + + // test data + testPath := "test" + testLink := "link" + testContent := "test" + testPerm := 0644 + + // create a file + if _, err := tm.CreateFile(testPath, bytes.NewReader([]byte(testContent)), fs.FileMode(testPerm), false, -1); err != nil { + t.Fatalf("CreateFile() failed: %s", err) + } + + // create a symlink + if err := tm.CreateSymlink(testPath, testLink, false); err != nil { + t.Fatalf("CreateSymlink() failed: %s", err) + } + + // open the symlink + f, err := tm.Open(testLink) + if err != nil { + t.Fatalf("Open() failed: %s", err) + } + + // stat the symlink (which is the link target) + stat, err := f.Stat() + if err != nil { + t.Fatalf("Stat() failed: %s", err) + } + + // check name + if stat.Name() != testPath { + t.Fatalf("Name() returned unexpected value: expected %s, got %s", testLink, stat.Name()) + } + + // check mode + if int(stat.Mode().Perm()&fs.ModePerm) != testPerm { + t.Fatalf("Mode() returned unexpected value: expected %d, got %d", testPerm, stat.Mode().Perm()) + } + + // read the symlink + data, err := io.ReadAll(f) + if err != nil { + t.Fatalf("ReadAll() failed: %s", err) + } + if !bytes.Equal(data, []byte(testContent)) { + t.Fatalf("unexpected file contents: expected %s, got %s", testContent, data) + } + + // close the symlink + if err := f.Close(); err != nil { + t.Fatalf("Close() failed: %s", err) + } + + // overwrite the symlink, but fail + if err := tm.CreateSymlink(testPath, testLink, false); err == nil { + t.Fatalf("CreateSymlink() failed: expected error, got nil") + } + + // overwrite the symlink + if err := tm.CreateSymlink(testPath, testLink, true); err != nil { + t.Fatalf("CreateSymlink() failed: %s", err) + } +} + +func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { + uid, gid := os.Geteuid(), os.Getegid() + baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) + testCases := []struct { + name string + contents []archiveContent + packer func(*testing.T, []archiveContent) []byte + expectError bool + }{ + { + name: "unpack tar with preserve file attributes", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + packer: packTar, + }, + { + name: "unpack zip with preserve file attributes", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + packer: packZip, + }, + { + name: "unpack rar with preserve file attributes", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, + }, + packer: packRar2, + }, + { + name: "unpack z7 with preserve file attributes", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "link", Linktarget: "sub/test", Mode: fs.ModeSymlink | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, + }, + packer: pack7z2, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ( + ctx = context.Background() + m = extract.NewTargetMemory() + src = asIoReader(t, tc.packer(t, tc.contents)) + cfg = extract.NewConfig(extract.WithPreserveFileAttributes(true)) + ) + if err := extract.UnpackTo(ctx, m, "", src, cfg); err != nil { + t.Fatalf("error unpacking archive: %v", err) + } + for _, c := range tc.contents { + path := c.Name + stat, err := m.Lstat(path) + if err != nil { + t.Fatalf("error getting file stats: %v", err) + } + if !(c.Mode&fs.ModeSymlink != 0) { // skip symlink checks + if stat.Mode().Perm() != c.Mode.Perm() { + t.Fatalf("expected file mode %v, got %v, file %s", c.Mode.Perm(), stat.Mode().Perm(), path) + } + } + } + }) + } +} diff --git a/unpack_test.go b/unpack_test.go index 7c2690d2..55e8ea4f 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -20,6 +20,7 @@ import ( "runtime" "strings" "testing" + "time" "github.com/andybalholm/brotli" "github.com/dsnet/compress/bzip2" @@ -604,8 +605,8 @@ func TestUnpackWithConfig(t *testing.T) { }, { Name: "dir/link", - Linktarget: "../test", Mode: fs.ModeSymlink | 0755, + Linktarget: "../test", }, } canceledCtx, cancel := context.WithCancel(context.Background()) @@ -1325,27 +1326,57 @@ func TestHasKnownArchiveExtension(t *testing.T) { } } -func TestUnpackWithPreserveFilemode(t *testing.T) { +func TestUnpackWithPreserveFileAttributes(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("skipping test on windows systems") } + + uid, gid := os.Geteuid(), os.Getegid() + baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) testCases := []struct { name string - permissions fs.FileMode - cfg *extract.Config - expectEq bool + contents []archiveContent + packer func(*testing.T, []archiveContent) []byte + expectError bool }{ { - name: "file with permissions 0777, with applied umask", - permissions: 0777, - cfg: extract.NewConfig(extract.WithPreserveFilemode(false)), - expectEq: false, + name: "tar", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + packer: packTar, }, { - name: "file with permissions 0777, overwrite umask", - permissions: 0777, - cfg: extract.NewConfig(extract.WithPreserveFilemode(true)), - expectEq: true, + name: "zip", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + packer: packZip, + }, + { + name: "rar", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, + }, + packer: packRar2, + }, + { + name: "7z", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, + {Name: "link", Linktarget: "sub/test", Mode: fs.ModeSymlink | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, + }, + packer: pack7z2, }, } @@ -1353,26 +1384,27 @@ func TestUnpackWithPreserveFilemode(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var ( ctx = context.Background() - src = asIoReader(t, packTar(t, []archiveContent{{Name: "test", Content: []byte("hello world"), Mode: tc.permissions}})) dst = t.TempDir() - cfg = tc.cfg + src = asIoReader(t, tc.packer(t, tc.contents)) + cfg = extract.NewConfig(extract.WithPreserveFileAttributes(true)) ) if err := extract.Unpack(ctx, dst, src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) } - var winPerm = fmt.Sprintf("%v", toWindowsFileMode(false, tc.permissions)) - t.Logf("expected equal: %v ; archive file mode %s", tc.expectEq, winPerm) - stat, err := os.Stat(filepath.Join(dst, "test")) - if err != nil { - t.Fatalf("error getting file stats: %v", err) - } - eqPem := stat.Mode().Perm() == tc.permissions - if eqPem != tc.expectEq { - t.Fatalf("expected equal: %v ; archive file mode %s, got %s", tc.expectEq, tc.permissions, stat.Mode().Perm()) + for _, c := range tc.contents { + path := filepath.Join(dst, c.Name) + stat, err := os.Lstat(path) + if err != nil { + t.Fatalf("error getting file stats: %v", err) + } + if !(c.Mode&fs.ModeSymlink != 0) { // skip symlink checks + if stat.Mode().Perm() != c.Mode.Perm() { + t.Fatalf("expected file mode %v, got %v, file %s", c.Mode.Perm(), stat.Mode().Perm(), c.Name) + } + } } }) } - } func compressBrotli(t *testing.T, data []byte) []byte { @@ -1500,6 +1532,10 @@ type archiveContent struct { Content []byte Linktarget string Mode fs.FileMode + AccessTime time.Time + ModTime time.Time + Uid int + Gid int } // packTar creates a tar file with the given content @@ -1534,9 +1570,17 @@ func packTar(t *testing.T, content []archiveContent) []byte { Linkname: c.Linktarget, Typeflag: tFlag, } + header.Uid = c.Uid + header.Gid = c.Gid + header.AccessTime = c.AccessTime + header.ModTime = c.ModTime if tFlag == tar.TypeXGlobalHeader { header.Mode = 0 header.Size = 0 + header.Uid = 0 + header.Gid = 0 + header.AccessTime = time.Time{} + header.ModTime = time.Time{} header.Format = tar.FormatPAX header.PAXRecords = map[string]string{} header.PAXRecords["path"] = c.Name @@ -1565,6 +1609,7 @@ func packZip(t *testing.T, content []archiveContent) []byte { Name: c.Name, } h.SetMode(c.Mode) + h.Modified = c.ModTime f, err := w.CreateHeader(h) if err != nil { t.Fatalf("error creating zip header: %v", err) @@ -1599,6 +1644,20 @@ func pack7z(t *testing.T, _ []archiveContent) []byte { return b } +// pack7z2 creates always the same a 7z archive with following files: +// -rw-r--r-- 1 503 20 27B 6 Dez 14:12 test +// drwxr-xr-x 3 503 20 96B 6 Dez 14:12 sub/ +// -rw-r--r-- 1 503 20 27B 6 Dez 14:12 sub/test +// lrwxr-xr-x 1 503 20 8B 6 Dez 14:12 link@ -> sub/test +func pack7z2(t *testing.T, _ []archiveContent) []byte { + t.Helper() + b, err := hex.DecodeString("377abcaf271c00042d5fc057b50000000000000022000000000000004e8d3aa1e0003d00285d00399d486415d3bb7a709d8c05b9a4f8a601c485ca32a1ba56fbed0277df127ac8b5849a02ef89b000000000813307ae0fd100d43ca090a0775ec540189123d516c0a4234b6046777137a236d0c100afd4540a63bac5dbcdd5f4954e1321f89bc2fee32eda1ffebe24d8ec7f5495f31cb107f418f1a438bedfa190f8d5e9bd34f41831a3e85fb8590ee2d3eb6854856ce91c64623e7b1bec5c6bf403f9b195d06eb0810540f173e9abd2005e6a00001706300109808500070b01000123030101055d001000000c80ae0a01d53cb2d70000") + if err != nil { + t.Fatalf("error decoding 7z data: %v", err) + } + return b +} + // packRar creates always the same a rar archive with following files: // - dir <- directory // - test <- file with content 'hello world' @@ -1613,6 +1672,19 @@ func packRar(t *testing.T, _ []archiveContent) []byte { return b } +// packRar2 creates always the same a rar archive with following files: +// -rw-r--r-- 1 503 20 27B 6 Dez 14:07 test +// drwxr-xr-x 3 503 20 96B 6 Dez 14:08 sub/ +// -rw-r--r-- 1 503 20 27B 6 Dez 14:08 sub/test +func packRar2(t *testing.T, _ []archiveContent) []byte { + t.Helper() + b, err := hex.DecodeString("526172211a0701003392b5e50a010506000501018080003afe2e322202030b9b00049b00a48302032d6c9680000104746573740a03132ff752678a911e136861736869207361797320686920746f2074686520776f726c640a7db74f802602030b9b00049b00a48302032d6c96800001087375622f746573740a031334f752672333f02b6861736869207361797320686920746f2074686520776f726c640a5311ba9e1b02030b000100ed8301800001037375620a031334f752673549ed2b1d77565103050400") + if err != nil { + t.Fatalf("error decoding rar data: %v", err) + } + return b +} + // openFile is a helper function to "open" a file, // but it returns an in-memory reader for example purposes. func openFile(_ string) io.ReadCloser { diff --git a/zip.go b/zip.go index e161be57..cbc578c5 100644 --- a/zip.go +++ b/zip.go @@ -10,6 +10,7 @@ import ( "io" "io/fs" "os" + "time" ) // fileExtensionZip is the file extension for zip files. @@ -154,3 +155,28 @@ func (z *zipEntry) Open() (io.ReadCloser, error) { func (z *zipEntry) Type() fs.FileMode { return z.zf.FileHeader.Mode().Type() } + +// AccessTime returns the access time of the entry +func (z *zipEntry) AccessTime() time.Time { + return z.zf.FileHeader.FileInfo().ModTime() +} + +// ModTime returns the modification time of the entry +func (z *zipEntry) ModTime() time.Time { + return z.zf.FileHeader.FileInfo().ModTime() +} + +// Sys returns the system information of the entry +func (z *zipEntry) Sys() interface{} { + return z.zf.FileHeader +} + +// Gid returns the group ID of the entry +func (z *zipEntry) Gid() int { + return os.Getgid() +} + +// Uid returns the user ID of the entry +func (z *zipEntry) Uid() int { + return os.Getuid() +} From 1605a680d84ab1380c0575abad2932b1c83fc415 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 9 Dec 2024 06:58:41 +0100 Subject: [PATCH 03/48] code clean up --- rar.go | 3 --- target.go | 18 +++++------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/rar.go b/rar.go index 383ef7a8..54d81fcd 100644 --- a/rar.go +++ b/rar.go @@ -83,9 +83,6 @@ func (rw *rarWalker) Next() (archiveEntry, error) { return nil, err } re := &rarEntry{fh, rw.r} - // if re.IsSymlink() { // symlink not supported - // return nil, unsupportedFile(re.Name()) - // } return re, nil } diff --git a/target.go b/target.go index 464253ec..0d048470 100644 --- a/target.go +++ b/target.go @@ -86,12 +86,9 @@ func createFile(t Target, dst string, name string, src io.Reader, mode fs.FileMo return 0, fmt.Errorf("cannot create directory: %w", err) } + // create the file path := filepath.Join(dst, name) - n, err := t.CreateFile(path, src, mode, cfg.Overwrite(), maxSize) - if err != nil { - return n, fmt.Errorf("failed to create file: %w", err) - } - return n, nil + return t.CreateFile(path, src, mode, cfg.Overwrite(), maxSize) } // createDir is a wrapper around the CreateDir function @@ -129,20 +126,15 @@ func createDir(t Target, dst string, name string, mode fs.FileMode, cfg *Config) return nil } + // perform security check to ensure that the path is safe to write to if err := securityCheck(t, dst, name, cfg); err != nil { return fmt.Errorf("security check path failed: %w", err) } // combine the path parts := strings.Split(name, "/") - name = filepath.Join(parts...) - path := filepath.Join(dst, name) - - if err := t.CreateDir(path, mode); err != nil { - return fmt.Errorf("failed to create directory: %w", err) - } - - return nil + path := filepath.Join(dst, filepath.Join(parts...)) + return t.CreateDir(path, mode) } // createSymlink is a wrapper around the CreateSymlink function From 0405d3fe73213bd13e6fd9e5820784f201a60e6b Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 9 Dec 2024 12:32:46 +0100 Subject: [PATCH 04/48] Further adjustments added aditional security check adjusted architecture specifics adjusted system specific implementation create system specific path --- target.go | 5 ++- target_disk_unix.go => target_disk_darwin.go | 4 +-- target_disk_linux32.go | 33 ++++++++++++++++++ target_disk_linux64.go | 35 ++++++++++++++++++++ target_memory.go | 2 +- target_memory_test.go | 17 ++++++++-- 6 files changed, 89 insertions(+), 7 deletions(-) rename target_disk_unix.go => target_disk_darwin.go (89%) create mode 100644 target_disk_linux32.go create mode 100644 target_disk_linux64.go diff --git a/target.go b/target.go index 0d048470..298b6bf3 100644 --- a/target.go +++ b/target.go @@ -86,7 +86,10 @@ func createFile(t Target, dst string, name string, src io.Reader, mode fs.FileMo return 0, fmt.Errorf("cannot create directory: %w", err) } - // create the file + // ensure that if the file exist that it is not a symlink + if err := securityCheck(t, dst, name, cfg); err != nil { + return 0, fmt.Errorf("security check path failed: %w", err) + } path := filepath.Join(dst, name) return t.CreateFile(path, src, mode, cfg.Overwrite(), maxSize) } diff --git a/target_disk_unix.go b/target_disk_darwin.go similarity index 89% rename from target_disk_unix.go rename to target_disk_darwin.go index b87cd86f..ebb1817e 100644 --- a/target_disk_unix.go +++ b/target_disk_darwin.go @@ -1,8 +1,8 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 -//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd -// +build darwin dragonfly freebsd linux netbsd openbsd +//go:build darwin +// +build darwin package extract diff --git a/target_disk_linux32.go b/target_disk_linux32.go new file mode 100644 index 00000000..9593e848 --- /dev/null +++ b/target_disk_linux32.go @@ -0,0 +1,33 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build linux && 386 +// +build linux,386 + +package extract + +import ( + "time" + + "golang.org/x/sys/unix" +) + +// Lchtimes modifies the access and modified timestamps on a target path +// This capability is only available on unix as of now. +func Lchtimes(path string, atime, mtime time.Time) error { + return unix.Lutimes(path, []unix.Timeval{ + {Sec: int32(atime.Unix()), Usec: int32(atime.Nanosecond() / 1e6 % 1e6)}, + {Sec: int32(mtime.Unix()), Usec: int32(mtime.Nanosecond() / 1e6 % 1e6)}}, + ) +} + +// CanMaintainSymlinkTimestamps determines whether is is possible to change +// timestamps on symlinks for the the current platform. For regular files +// and directories, attempts are made to restore permissions and timestamps +// after extraction. But for symbolic links, go's cross-platform +// packages (Chmod and Chtimes) are not capable of changing symlink info +// because those methods follow the symlinks. However, a platform-dependent option +// is provided for unix (see Lchtimes) +func CanMaintainSymlinkTimestamps() bool { + return true +} diff --git a/target_disk_linux64.go b/target_disk_linux64.go new file mode 100644 index 00000000..8341f6af --- /dev/null +++ b/target_disk_linux64.go @@ -0,0 +1,35 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build linux && (amd64 || arm64) +// +build linux +// +build amd64 arm64 + +package extract + +import ( + "time" + + "golang.org/x/sys/unix" +) + +// Lchtimes modifies the access and modified timestamps on a target path +// This capability is only available on unix as of now. +func Lchtimes(path string, atime, mtime time.Time) error { + + return unix.Lutimes(path, []unix.Timeval{ + {Sec: atime.Unix(), Usec: int64(atime.Nanosecond() / 1e6 % 1e6)}, + {Sec: mtime.Unix(), Usec: int64(mtime.Nanosecond() / 1e6 % 1e6)}, + }) +} + +// CanMaintainSymlinkTimestamps determines whether is is possible to change +// timestamps on symlinks for the the current platform. For regular files +// and directories, attempts are made to restore permissions and timestamps +// after extraction. But for symbolic links, go's cross-platform +// packages (Chmod and Chtimes) are not capable of changing symlink info +// because those methods follow the symlinks. However, a platform-dependent option +// is provided for unix (see Lchtimes) +func CanMaintainSymlinkTimestamps() bool { + return true +} diff --git a/target_memory.go b/target_memory.go index fe2acdf2..597b9461 100644 --- a/target_memory.go +++ b/target_memory.go @@ -401,7 +401,7 @@ func (m *TargetMemory) resolveEntry(path string) (*memoryEntry, error) { dir := p.Dir(path) existingEntry, err := m.resolvePath(dir) if errors.Is(err, fs.ErrNotExist) { - return nil, fs.ErrNotExist + return nil, err } if err != nil { return nil, &fs.PathError{Op: "resolveEntry", Path: path, Err: err} diff --git a/target_memory_test.go b/target_memory_test.go index 2acaf11e..7f71f610 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -10,6 +10,8 @@ import ( "io/fs" "os" p "path" + "path/filepath" + "strings" "testing" "testing/fstest" "time" @@ -853,7 +855,7 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { name: "unpack tar with preserve file attributes", contents: []archiveContent{ {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + // {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, }, @@ -863,7 +865,7 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { name: "unpack zip with preserve file attributes", contents: []archiveContent{ {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + // {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, }, @@ -901,8 +903,17 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { if err := extract.UnpackTo(ctx, m, "", src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) } + entries, err := m.Glob("**") + if err != nil { + t.Fatalf("error globbing files: %v", err) + } + for _, e := range entries { + t.Logf("entry: %s", e) + } + for _, c := range tc.contents { - path := c.Name + parts := strings.Split(c.Name, "/") // create system specific path + path := filepath.Join(parts...) stat, err := m.Lstat(path) if err != nil { t.Fatalf("error getting file stats: %v", err) From 576854a67672ada2e0f7e9937fa340d0101d7cc5 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 9 Dec 2024 14:55:40 +0100 Subject: [PATCH 05/48] adjusted cli help output --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1ea5146..520cf0eb 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Flags: -N, --no-untar-after-decompression Disable combined extraction of tar.gz. -O, --overwrite Overwrite if exist. -P, --pattern=PATTERN,... Extracted objects need to match shell file name pattern. - --preserve-filemode Preserve file mode and overwrite umask. + -p, --preserve-file-attributes Preserve file attributes from archive (access and modification time, file permissions and owner/group). -T, --telemetry Print telemetry data to log after extraction. -t, --type="" Type of archive. (7z, br, bz2, gz, lz4, rar, sz, tar, tgz, xz, zip, zst, zz) -v, --verbose Verbose logging. From 2160ffea46427d8a9c79bd78e8a101a92631b312 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 10 Dec 2024 06:15:17 +0100 Subject: [PATCH 06/48] adjusted build tag specification --- target_disk_darwin.go | 1 - target_disk_linux32.go | 1 - target_disk_linux64.go | 2 -- 3 files changed, 4 deletions(-) diff --git a/target_disk_darwin.go b/target_disk_darwin.go index ebb1817e..6f68da59 100644 --- a/target_disk_darwin.go +++ b/target_disk_darwin.go @@ -2,7 +2,6 @@ // SPDX-License-Identifier: MPL-2.0 //go:build darwin -// +build darwin package extract diff --git a/target_disk_linux32.go b/target_disk_linux32.go index 9593e848..7af25337 100644 --- a/target_disk_linux32.go +++ b/target_disk_linux32.go @@ -2,7 +2,6 @@ // SPDX-License-Identifier: MPL-2.0 //go:build linux && 386 -// +build linux,386 package extract diff --git a/target_disk_linux64.go b/target_disk_linux64.go index 8341f6af..5b8deddf 100644 --- a/target_disk_linux64.go +++ b/target_disk_linux64.go @@ -2,8 +2,6 @@ // SPDX-License-Identifier: MPL-2.0 //go:build linux && (amd64 || arm64) -// +build linux -// +build amd64 arm64 package extract From 4c99ad2e2ac810a4665550681f326939582c83bb Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 10 Dec 2024 10:38:08 +0100 Subject: [PATCH 07/48] corrected mod time adjustment --- target_disk_darwin.go | 5 ++-- target_disk_linux32.go | 4 +-- target_disk_linux64.go | 4 +-- target_memory_test.go | 56 ++++++++++++++++++++-------------------- unpack_test.go | 58 +++++++++++++++++++++++++++--------------- 5 files changed, 72 insertions(+), 55 deletions(-) diff --git a/target_disk_darwin.go b/target_disk_darwin.go index 6f68da59..97e74c97 100644 --- a/target_disk_darwin.go +++ b/target_disk_darwin.go @@ -14,10 +14,9 @@ import ( // Lchtimes modifies the access and modified timestamps on a target path // This capability is only available on unix as of now. func Lchtimes(path string, atime, mtime time.Time) error { - return unix.Lutimes(path, []unix.Timeval{ - {Sec: atime.Unix(), Usec: int32(atime.Nanosecond() / 1e6 % 1e6)}, - {Sec: mtime.Unix(), Usec: int32(mtime.Nanosecond() / 1e6 % 1e6)}, + {Sec: atime.Unix(), Usec: int32(atime.Nanosecond() / 1e3 % 1e6)}, + {Sec: mtime.Unix(), Usec: int32(mtime.Nanosecond() / 1e3 % 1e6)}, }) } diff --git a/target_disk_linux32.go b/target_disk_linux32.go index 7af25337..8dac293f 100644 --- a/target_disk_linux32.go +++ b/target_disk_linux32.go @@ -15,8 +15,8 @@ import ( // This capability is only available on unix as of now. func Lchtimes(path string, atime, mtime time.Time) error { return unix.Lutimes(path, []unix.Timeval{ - {Sec: int32(atime.Unix()), Usec: int32(atime.Nanosecond() / 1e6 % 1e6)}, - {Sec: int32(mtime.Unix()), Usec: int32(mtime.Nanosecond() / 1e6 % 1e6)}}, + {Sec: int32(atime.Unix()), Usec: int32(atime.Nanosecond() / 1e3 % 1e6)}, + {Sec: int32(mtime.Unix()), Usec: int32(mtime.Nanosecond() / 1e3 % 1e6)}}, ) } diff --git a/target_disk_linux64.go b/target_disk_linux64.go index 5b8deddf..3b289164 100644 --- a/target_disk_linux64.go +++ b/target_disk_linux64.go @@ -16,8 +16,8 @@ import ( func Lchtimes(path string, atime, mtime time.Time) error { return unix.Lutimes(path, []unix.Timeval{ - {Sec: atime.Unix(), Usec: int64(atime.Nanosecond() / 1e6 % 1e6)}, - {Sec: mtime.Unix(), Usec: int64(mtime.Nanosecond() / 1e6 % 1e6)}, + {Sec: atime.Unix(), Usec: int64(atime.Nanosecond() / 1e3 % 1e6)}, + {Sec: mtime.Unix(), Usec: int64(mtime.Nanosecond() / 1e3 % 1e6)}, }) } diff --git a/target_memory_test.go b/target_memory_test.go index 7f71f610..a620976a 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -8,7 +8,6 @@ import ( "context" "io" "io/fs" - "os" p "path" "path/filepath" "strings" @@ -843,13 +842,14 @@ func TestCreateSymlink(t *testing.T) { } func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { - uid, gid := os.Geteuid(), os.Getegid() + uid, gid := 503, 20 baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) testCases := []struct { - name string - contents []archiveContent - packer func(*testing.T, []archiveContent) []byte - expectError bool + name string + contents []archiveContent + packer func(*testing.T, []archiveContent) []byte + doesNotSupportModTime bool + expectError bool }{ { name: "unpack tar with preserve file attributes", @@ -872,23 +872,15 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { packer: packZip, }, { - name: "unpack rar with preserve file attributes", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, - }, - packer: packRar2, + name: "unpack rar with preserve file attributes", + contents: contentsRar2, + doesNotSupportModTime: true, + packer: packRar2, }, { - name: "unpack z7 with preserve file attributes", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "link", Linktarget: "sub/test", Mode: fs.ModeSymlink | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, - }, - packer: pack7z2, + name: "unpack z7 with preserve file attributes", + contents: contents7z2, + packer: pack7z2, }, } @@ -903,13 +895,13 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { if err := extract.UnpackTo(ctx, m, "", src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) } - entries, err := m.Glob("**") - if err != nil { - t.Fatalf("error globbing files: %v", err) - } - for _, e := range entries { - t.Logf("entry: %s", e) - } + // entries, err := m.Glob("**") + // if err != nil { + // t.Fatalf("error globbing files: %v", err) + // } + // for _, e := range entries { + // t.Logf("entry: %s", e) + // } for _, c := range tc.contents { parts := strings.Split(c.Name, "/") // create system specific path @@ -923,6 +915,14 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { t.Fatalf("expected file mode %v, got %v, file %s", c.Mode.Perm(), stat.Mode().Perm(), path) } } + if tc.doesNotSupportModTime { + continue + } + // calculate the time difference + modTimeDiff := abs(stat.ModTime().Sub(c.ModTime).Nanoseconds()) + if modTimeDiff > int64(time.Microsecond) { + t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), path, modTimeDiff) + } } }) } diff --git a/unpack_test.go b/unpack_test.go index 55e8ea4f..65e488ab 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -15,6 +15,7 @@ import ( "fmt" "io" "io/fs" + "math" "os" "path/filepath" "runtime" @@ -1326,18 +1327,23 @@ func TestHasKnownArchiveExtension(t *testing.T) { } } +func abs(v int64) int64 { + return int64(math.Abs(float64(v))) +} + func TestUnpackWithPreserveFileAttributes(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("skipping test on windows systems") } - uid, gid := os.Geteuid(), os.Getegid() + uid, gid := 503, 20 baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) testCases := []struct { - name string - contents []archiveContent - packer func(*testing.T, []archiveContent) []byte - expectError bool + name string + contents []archiveContent + packer func(*testing.T, []archiveContent) []byte + doesNotSupportModTime bool + expectError bool }{ { name: "tar", @@ -1360,23 +1366,15 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { packer: packZip, }, { - name: "rar", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, - }, - packer: packRar2, + name: "rar", + contents: contentsRar2, + packer: packRar2, + doesNotSupportModTime: true, }, { - name: "7z", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, - {Name: "link", Linktarget: "sub/test", Mode: fs.ModeSymlink | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: uid, Gid: gid}, - }, - packer: pack7z2, + name: "7z", + contents: contents7z2, + packer: pack7z2, }, } @@ -1402,6 +1400,13 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { t.Fatalf("expected file mode %v, got %v, file %s", c.Mode.Perm(), stat.Mode().Perm(), c.Name) } } + if tc.doesNotSupportModTime { + continue + } + modTimeDiff := abs(stat.ModTime().Sub(c.ModTime).Nanoseconds()) + if modTimeDiff > int64(time.Microsecond) { + t.Fatalf("expected mod time %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), c.Name, modTimeDiff) + } } }) } @@ -1649,6 +1654,13 @@ func pack7z(t *testing.T, _ []archiveContent) []byte { // drwxr-xr-x 3 503 20 96B 6 Dez 14:12 sub/ // -rw-r--r-- 1 503 20 27B 6 Dez 14:12 sub/test // lrwxr-xr-x 1 503 20 8B 6 Dez 14:12 link@ -> sub/test +var contents7z2 = []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 42, 315443500, time.Local), Uid: 503, Gid: 20}, + {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 49, 378600200, time.Local), Uid: 503, Gid: 20}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 49, 378790200, time.Local), Uid: 503, Gid: 20}, + {Name: "link", Linktarget: "sub/test", Mode: fs.ModeSymlink | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 54, 532031200, time.Local), Uid: 503, Gid: 20}, +} + func pack7z2(t *testing.T, _ []archiveContent) []byte { t.Helper() b, err := hex.DecodeString("377abcaf271c00042d5fc057b50000000000000022000000000000004e8d3aa1e0003d00285d00399d486415d3bb7a709d8c05b9a4f8a601c485ca32a1ba56fbed0277df127ac8b5849a02ef89b000000000813307ae0fd100d43ca090a0775ec540189123d516c0a4234b6046777137a236d0c100afd4540a63bac5dbcdd5f4954e1321f89bc2fee32eda1ffebe24d8ec7f5495f31cb107f418f1a438bedfa190f8d5e9bd34f41831a3e85fb8590ee2d3eb6854856ce91c64623e7b1bec5c6bf403f9b195d06eb0810540f173e9abd2005e6a00001706300109808500070b01000123030101055d001000000c80ae0a01d53cb2d70000") @@ -1676,6 +1688,12 @@ func packRar(t *testing.T, _ []archiveContent) []byte { // -rw-r--r-- 1 503 20 27B 6 Dez 14:07 test // drwxr-xr-x 3 503 20 96B 6 Dez 14:08 sub/ // -rw-r--r-- 1 503 20 27B 6 Dez 14:08 sub/test +var contentsRar2 = []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 7, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: 503, Gid: 20}, + {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 7, 8, 0, time.Local), Uid: 503, Gid: 20}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 8, 0, 0, time.Local), Uid: 503, Gid: 20}, +} + func packRar2(t *testing.T, _ []archiveContent) []byte { t.Helper() b, err := hex.DecodeString("526172211a0701003392b5e50a010506000501018080003afe2e322202030b9b00049b00a48302032d6c9680000104746573740a03132ff752678a911e136861736869207361797320686920746f2074686520776f726c640a7db74f802602030b9b00049b00a48302032d6c96800001087375622f746573740a031334f752672333f02b6861736869207361797320686920746f2074686520776f726c640a5311ba9e1b02030b000100ed8301800001037375620a031334f752673549ed2b1d77565103050400") From fd3a38540c5cfd94b3b1feb83c051c0b3db92bc2 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 10 Dec 2024 10:58:29 +0100 Subject: [PATCH 08/48] removed code comment and fixed timestamp of archive file --- target_memory_test.go | 7 ------- unpack_test.go | 8 ++++---- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/target_memory_test.go b/target_memory_test.go index a620976a..d037975a 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -895,13 +895,6 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { if err := extract.UnpackTo(ctx, m, "", src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) } - // entries, err := m.Glob("**") - // if err != nil { - // t.Fatalf("error globbing files: %v", err) - // } - // for _, e := range entries { - // t.Logf("entry: %s", e) - // } for _, c := range tc.contents { parts := strings.Split(c.Name, "/") // create system specific path diff --git a/unpack_test.go b/unpack_test.go index 65e488ab..7e3f6b30 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1655,10 +1655,10 @@ func pack7z(t *testing.T, _ []archiveContent) []byte { // -rw-r--r-- 1 503 20 27B 6 Dez 14:12 sub/test // lrwxr-xr-x 1 503 20 8B 6 Dez 14:12 link@ -> sub/test var contents7z2 = []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 42, 315443500, time.Local), Uid: 503, Gid: 20}, - {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 49, 378600200, time.Local), Uid: 503, Gid: 20}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 49, 378790200, time.Local), Uid: 503, Gid: 20}, - {Name: "link", Linktarget: "sub/test", Mode: fs.ModeSymlink | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 14, 12, 54, 532031200, time.Local), Uid: 503, Gid: 20}, + {Name: "test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 13, 12, 42, 315443500, time.UTC), Uid: 503, Gid: 20}, + {Name: "sub", Mode: fs.ModeDir | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 13, 12, 49, 378600200, time.UTC), Uid: 503, Gid: 20}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 13, 12, 49, 378790200, time.UTC), Uid: 503, Gid: 20}, + {Name: "link", Linktarget: "sub/test", Mode: fs.ModeSymlink | 0755, AccessTime: time.Date(2024, 12, 6, 14, 12, 0, 0, time.Local), ModTime: time.Date(2024, 12, 6, 13, 12, 54, 532031200, time.UTC), Uid: 503, Gid: 20}, } func pack7z2(t *testing.T, _ []archiveContent) []byte { From 1ded8619500542a6f3c9e41940061370bdf9afba Mon Sep 17 00:00:00 2001 From: Sam Salisbury Date: Tue, 10 Dec 2024 13:12:42 +0000 Subject: [PATCH 09/48] rely on built-in unix.Timeval conversion --- target_disk_darwin.go | 32 ---------- target_disk_linux32.go | 32 ---------- target_disk_others.go | 3 +- target_disk_linux64.go => target_disk_unix.go | 15 +++-- target_disk_unix_test.go | 59 +++++++++++++++++++ 5 files changed, 71 insertions(+), 70 deletions(-) delete mode 100644 target_disk_darwin.go delete mode 100644 target_disk_linux32.go rename target_disk_linux64.go => target_disk_unix.go (67%) create mode 100644 target_disk_unix_test.go diff --git a/target_disk_darwin.go b/target_disk_darwin.go deleted file mode 100644 index 97e74c97..00000000 --- a/target_disk_darwin.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -//go:build darwin - -package extract - -import ( - "time" - - "golang.org/x/sys/unix" -) - -// Lchtimes modifies the access and modified timestamps on a target path -// This capability is only available on unix as of now. -func Lchtimes(path string, atime, mtime time.Time) error { - return unix.Lutimes(path, []unix.Timeval{ - {Sec: atime.Unix(), Usec: int32(atime.Nanosecond() / 1e3 % 1e6)}, - {Sec: mtime.Unix(), Usec: int32(mtime.Nanosecond() / 1e3 % 1e6)}, - }) -} - -// CanMaintainSymlinkTimestamps determines whether is is possible to change -// timestamps on symlinks for the the current platform. For regular files -// and directories, attempts are made to restore permissions and timestamps -// after extraction. But for symbolic links, go's cross-platform -// packages (Chmod and Chtimes) are not capable of changing symlink info -// because those methods follow the symlinks. However, a platform-dependent option -// is provided for unix (see Lchtimes) -func CanMaintainSymlinkTimestamps() bool { - return true -} diff --git a/target_disk_linux32.go b/target_disk_linux32.go deleted file mode 100644 index 8dac293f..00000000 --- a/target_disk_linux32.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -//go:build linux && 386 - -package extract - -import ( - "time" - - "golang.org/x/sys/unix" -) - -// Lchtimes modifies the access and modified timestamps on a target path -// This capability is only available on unix as of now. -func Lchtimes(path string, atime, mtime time.Time) error { - return unix.Lutimes(path, []unix.Timeval{ - {Sec: int32(atime.Unix()), Usec: int32(atime.Nanosecond() / 1e3 % 1e6)}, - {Sec: int32(mtime.Unix()), Usec: int32(mtime.Nanosecond() / 1e3 % 1e6)}}, - ) -} - -// CanMaintainSymlinkTimestamps determines whether is is possible to change -// timestamps on symlinks for the the current platform. For regular files -// and directories, attempts are made to restore permissions and timestamps -// after extraction. But for symbolic links, go's cross-platform -// packages (Chmod and Chtimes) are not capable of changing symlink info -// because those methods follow the symlinks. However, a platform-dependent option -// is provided for unix (see Lchtimes) -func CanMaintainSymlinkTimestamps() bool { - return true -} diff --git a/target_disk_others.go b/target_disk_others.go index 3d4ed04a..3add4157 100644 --- a/target_disk_others.go +++ b/target_disk_others.go @@ -1,8 +1,7 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 -//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd -// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd +//go:build !unix package extract diff --git a/target_disk_linux64.go b/target_disk_unix.go similarity index 67% rename from target_disk_linux64.go rename to target_disk_unix.go index 3b289164..9a76e718 100644 --- a/target_disk_linux64.go +++ b/target_disk_unix.go @@ -1,7 +1,7 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 -//go:build linux && (amd64 || arm64) +//go:build unix package extract @@ -14,13 +14,20 @@ import ( // Lchtimes modifies the access and modified timestamps on a target path // This capability is only available on unix as of now. func Lchtimes(path string, atime, mtime time.Time) error { - return unix.Lutimes(path, []unix.Timeval{ - {Sec: atime.Unix(), Usec: int64(atime.Nanosecond() / 1e3 % 1e6)}, - {Sec: mtime.Unix(), Usec: int64(mtime.Nanosecond() / 1e3 % 1e6)}, + unixTimeval(atime), + unixTimeval(mtime), }) } +// unixTimeval converts a time.Time to a unix.Timeval. Note that it always rounds +// up to the nearest microsecond, so even one nanosecond past the previous nanosecond +// will be rounded up to the next microsecond. +// See the implementation of unix.NsecToTimeval for details on how this happens. +func unixTimeval(t time.Time) unix.Timeval { + return unix.NsecToTimeval(t.UnixNano()) +} + // CanMaintainSymlinkTimestamps determines whether is is possible to change // timestamps on symlinks for the the current platform. For regular files // and directories, attempts are made to restore permissions and timestamps diff --git a/target_disk_unix_test.go b/target_disk_unix_test.go new file mode 100644 index 00000000..1fbf6241 --- /dev/null +++ b/target_disk_unix_test.go @@ -0,0 +1,59 @@ +//go:build unix + +package extract + +import ( + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func TestUnixTimeval(t *testing.T) { + tests := []struct { + input time.Time + want unix.Timeval + }{ + { + time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC), + unix.Timeval{Sec: 0, Usec: 0}, + }, + { + // Note: the single nanosecond is rounded up to the next microsecond. + time.Date(1970, 1, 1, 0, 0, 0, 1, time.UTC), + unix.Timeval{Sec: 0, Usec: 1}, + }, + { + // Note: the 100 nanoseconds are rounded up to the next microsecond. + time.Date(1970, 1, 1, 0, 0, 0, 100, time.UTC), + unix.Timeval{Sec: 0, Usec: 1}, + }, + { + // Note: exactly 1 microsecond is not rounded up. + time.Date(1970, 1, 1, 0, 0, 0, 1000, time.UTC), + unix.Timeval{Sec: 0, Usec: 1}, + }, + { + // Note: exactly 1 nanosecond past the microsecond is rounded up. + time.Date(1970, 1, 1, 0, 0, 0, 1001, time.UTC), + unix.Timeval{Sec: 0, Usec: 2}, + }, + { + time.Date(1970, 1, 1, 0, 0, 1, 1000, time.UTC), + unix.Timeval{Sec: 1, Usec: 1}, + }, + { + time.Date(1970, 1, 1, 0, 0, 1, 2000, time.UTC), + unix.Timeval{Sec: 1, Usec: 2}, + }, + } + + for _, test := range tests { + t.Run(test.input.String(), func(t *testing.T) { + got := unixTimeval(test.input) + if got != test.want { + t.Errorf("unixTimeval(%v) = %v; want %v", test.input, got, test.want) + } + }) + } +} From 03a7cb637658d37ed89ceb56767a850b2b2d4bb3 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 10 Dec 2024 15:36:57 +0100 Subject: [PATCH 10/48] unexpose internal functions --- target_disk.go | 4 ++-- target_disk_others.go | 8 ++++---- target_disk_unix.go | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/target_disk.go b/target_disk.go index 597715a3..72cb37ae 100644 --- a/target_disk.go +++ b/target_disk.go @@ -130,8 +130,8 @@ func (d *TargetDisk) Chown(name string, uid, gid int) error { // Lchtimes changes the access and modification times of the named file. func (d *TargetDisk) Lchtimes(name string, atime, mtime time.Time) error { - if CanMaintainSymlinkTimestamps() { - return Lchtimes(name, atime, mtime) + if canMaintainSymlinkTimestamps() { + return lchtimes(name, atime, mtime) } return nil } diff --git a/target_disk_others.go b/target_disk_others.go index 3add4157..f5c3348e 100644 --- a/target_disk_others.go +++ b/target_disk_others.go @@ -11,19 +11,19 @@ import ( "time" ) -// Lchtimes modifies the access and modified timestamps on a target path +// lchtimes modifies the access and modified timestamps on a target path // This capability is only available on unix as of now. -func Lchtimes(path string, atime, mtime time.Time) error { +func lchtimes(_ string, _, _ time.Time) error { return fmt.Errorf("Lchtimes is not supported on this platform (%s)", runtime.GOOS) } -// CanMaintainSymlinkTimestamps determines whether is is possible to change +// canMaintainSymlinkTimestamps determines whether is is possible to change // timestamps on symlinks for the the current platform. For regular files // and directories, attempts are made to restore permissions and timestamps // after extraction. But for symbolic links, go's cross-platform // packages (Chmod and Chtimes) are not capable of changing symlink info // because those methods follow the symlinks. However, a platform-dependent option // is provided for unix (see Lchtimes) -func CanMaintainSymlinkTimestamps() bool { +func canMaintainSymlinkTimestamps() bool { return false } diff --git a/target_disk_unix.go b/target_disk_unix.go index 9a76e718..f9a6f992 100644 --- a/target_disk_unix.go +++ b/target_disk_unix.go @@ -11,9 +11,9 @@ import ( "golang.org/x/sys/unix" ) -// Lchtimes modifies the access and modified timestamps on a target path +// lchtimes modifies the access and modified timestamps on a target path // This capability is only available on unix as of now. -func Lchtimes(path string, atime, mtime time.Time) error { +func lchtimes(path string, atime, mtime time.Time) error { return unix.Lutimes(path, []unix.Timeval{ unixTimeval(atime), unixTimeval(mtime), @@ -28,13 +28,13 @@ func unixTimeval(t time.Time) unix.Timeval { return unix.NsecToTimeval(t.UnixNano()) } -// CanMaintainSymlinkTimestamps determines whether is is possible to change +// canMaintainSymlinkTimestamps determines whether is is possible to change // timestamps on symlinks for the the current platform. For regular files // and directories, attempts are made to restore permissions and timestamps // after extraction. But for symbolic links, go's cross-platform // packages (Chmod and Chtimes) are not capable of changing symlink info // because those methods follow the symlinks. However, a platform-dependent option // is provided for unix (see Lchtimes) -func CanMaintainSymlinkTimestamps() bool { +func canMaintainSymlinkTimestamps() bool { return true } From fb7132a3a984ed2b2fb0deb17a19cd711b4b44bf Mon Sep 17 00:00:00 2001 From: Jan Harrie Date: Wed, 11 Dec 2024 13:52:29 +0100 Subject: [PATCH 11/48] Update target_memory_test.go Co-authored-by: Sam Salisbury --- target_memory_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target_memory_test.go b/target_memory_test.go index d037975a..fc858bd6 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -912,8 +912,7 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { continue } // calculate the time difference - modTimeDiff := abs(stat.ModTime().Sub(c.ModTime).Nanoseconds()) - if modTimeDiff > int64(time.Microsecond) { + if stat.ModTime().UnixMicro() != c.ModTime.UnixMIcro() { t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), path, modTimeDiff) } } From f9b271092265b5352bdfa94979b4335818225c02 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 11 Dec 2024 14:17:07 +0100 Subject: [PATCH 12/48] check for differences >= 1 micro second --- target_memory_test.go | 5 +++-- unpack_test.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/target_memory_test.go b/target_memory_test.go index fc858bd6..77f0d1e5 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -912,8 +912,9 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { continue } // calculate the time difference - if stat.ModTime().UnixMicro() != c.ModTime.UnixMIcro() { - t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), path, modTimeDiff) + modTimeDiff := abs(stat.ModTime().UnixNano() - c.ModTime.UnixNano()) + if modTimeDiff >= int64(time.Microsecond) { + t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", int64(c.ModTime.UnixMicro()), int64(stat.ModTime().UnixMicro()), path, modTimeDiff) } } }) diff --git a/unpack_test.go b/unpack_test.go index 7e3f6b30..d05481a9 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1403,8 +1403,8 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { if tc.doesNotSupportModTime { continue } - modTimeDiff := abs(stat.ModTime().Sub(c.ModTime).Nanoseconds()) - if modTimeDiff > int64(time.Microsecond) { + modTimeDiff := abs(stat.ModTime().UnixNano() - c.ModTime.UnixNano()) + if modTimeDiff >= int64(time.Microsecond) { t.Fatalf("expected mod time %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), c.Name, modTimeDiff) } } From 9e69c9df91a47d2f90a8c22572a7712de831472a Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 11 Dec 2024 14:20:12 +0100 Subject: [PATCH 13/48] simplified test print statement --- target_memory_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_memory_test.go b/target_memory_test.go index 77f0d1e5..3b636200 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -914,7 +914,7 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { // calculate the time difference modTimeDiff := abs(stat.ModTime().UnixNano() - c.ModTime.UnixNano()) if modTimeDiff >= int64(time.Microsecond) { - t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", int64(c.ModTime.UnixMicro()), int64(stat.ModTime().UnixMicro()), path, modTimeDiff) + t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), path, modTimeDiff) } } }) From 7fa9d28040ab3fd02221ccf5954b96b8164a726a Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 11 Dec 2024 14:35:30 +0100 Subject: [PATCH 14/48] invent testing as root to adjust ownership --- .github/workflows/testing.yml | 7 +++++++ Makefile | 5 +++++ unpack_test.go | 20 +++++++++++++++----- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 53fb8214..49caf181 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -30,6 +30,13 @@ jobs: run: | make test + - name: Test (as root) + id: sudo-test + run: | + make sudo_test + + + fuzzing: strategy: diff --git a/Makefile b/Makefile index d506cc78..da562541 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,11 @@ clean: test: go test ./... +sudo_test: + sudo -E go test ./... + + + test_coverage: go test ./... -coverprofile=coverage.out diff --git a/unpack_test.go b/unpack_test.go index d05481a9..e4002903 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -20,6 +20,7 @@ import ( "path/filepath" "runtime" "strings" + "syscall" "testing" "time" @@ -1343,12 +1344,13 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { contents []archiveContent packer func(*testing.T, []archiveContent) []byte doesNotSupportModTime bool + doesNotSupportOwner bool expectError bool }{ { name: "tar", contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, @@ -1363,7 +1365,8 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, }, - packer: packZip, + doesNotSupportOwner: true, + packer: packZip, }, { name: "rar", @@ -1372,9 +1375,10 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { doesNotSupportModTime: true, }, { - name: "7z", - contents: contents7z2, - packer: pack7z2, + name: "7z", + contents: contents7z2, + doesNotSupportOwner: true, + packer: pack7z2, }, } @@ -1407,6 +1411,12 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { if modTimeDiff >= int64(time.Microsecond) { t.Fatalf("expected mod time %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), c.Name, modTimeDiff) } + if os.Getuid() != 0 || tc.doesNotSupportOwner { + continue + } + if stat.Sys().(*syscall.Stat_t).Uid != uint32(c.Uid) { + t.Fatalf("expected uid %d, got %d, file %s", c.Uid, stat.Sys().(*syscall.Stat_t).Uid, c.Name) + } } }) } From 8498e88bfc0900766d4bd15cee6b3400093e7d29 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 11 Dec 2024 14:45:01 +0100 Subject: [PATCH 15/48] transformed canMaintainSymlinkTimestamps into a function and relocated unix specific tests --- target_disk.go | 2 +- target_disk_others.go | 4 +- target_disk_unix.go | 4 +- unpack_test.go | 91 ------------------------------------ unpack_unix_test.go | 104 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 98 deletions(-) create mode 100644 unpack_unix_test.go diff --git a/target_disk.go b/target_disk.go index 72cb37ae..c09b1510 100644 --- a/target_disk.go +++ b/target_disk.go @@ -130,7 +130,7 @@ func (d *TargetDisk) Chown(name string, uid, gid int) error { // Lchtimes changes the access and modification times of the named file. func (d *TargetDisk) Lchtimes(name string, atime, mtime time.Time) error { - if canMaintainSymlinkTimestamps() { + if canMaintainSymlinkTimestamps { return lchtimes(name, atime, mtime) } return nil diff --git a/target_disk_others.go b/target_disk_others.go index f5c3348e..b4dd1dc6 100644 --- a/target_disk_others.go +++ b/target_disk_others.go @@ -24,6 +24,4 @@ func lchtimes(_ string, _, _ time.Time) error { // packages (Chmod and Chtimes) are not capable of changing symlink info // because those methods follow the symlinks. However, a platform-dependent option // is provided for unix (see Lchtimes) -func canMaintainSymlinkTimestamps() bool { - return false -} +const canMaintainSymlinkTimestamps = false diff --git a/target_disk_unix.go b/target_disk_unix.go index f9a6f992..05c58b6b 100644 --- a/target_disk_unix.go +++ b/target_disk_unix.go @@ -35,6 +35,4 @@ func unixTimeval(t time.Time) unix.Timeval { // packages (Chmod and Chtimes) are not capable of changing symlink info // because those methods follow the symlinks. However, a platform-dependent option // is provided for unix (see Lchtimes) -func canMaintainSymlinkTimestamps() bool { - return true -} +const canMaintainSymlinkTimestamps = true diff --git a/unpack_test.go b/unpack_test.go index e4002903..edac4552 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -20,7 +20,6 @@ import ( "path/filepath" "runtime" "strings" - "syscall" "testing" "time" @@ -1332,96 +1331,6 @@ func abs(v int64) int64 { return int64(math.Abs(float64(v))) } -func TestUnpackWithPreserveFileAttributes(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("skipping test on windows systems") - } - - uid, gid := 503, 20 - baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) - testCases := []struct { - name string - contents []archiveContent - packer func(*testing.T, []archiveContent) []byte - doesNotSupportModTime bool - doesNotSupportOwner bool - expectError bool - }{ - { - name: "tar", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - packer: packTar, - }, - { - name: "zip", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - doesNotSupportOwner: true, - packer: packZip, - }, - { - name: "rar", - contents: contentsRar2, - packer: packRar2, - doesNotSupportModTime: true, - }, - { - name: "7z", - contents: contents7z2, - doesNotSupportOwner: true, - packer: pack7z2, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var ( - ctx = context.Background() - dst = t.TempDir() - src = asIoReader(t, tc.packer(t, tc.contents)) - cfg = extract.NewConfig(extract.WithPreserveFileAttributes(true)) - ) - if err := extract.Unpack(ctx, dst, src, cfg); err != nil { - t.Fatalf("error unpacking archive: %v", err) - } - for _, c := range tc.contents { - path := filepath.Join(dst, c.Name) - stat, err := os.Lstat(path) - if err != nil { - t.Fatalf("error getting file stats: %v", err) - } - if !(c.Mode&fs.ModeSymlink != 0) { // skip symlink checks - if stat.Mode().Perm() != c.Mode.Perm() { - t.Fatalf("expected file mode %v, got %v, file %s", c.Mode.Perm(), stat.Mode().Perm(), c.Name) - } - } - if tc.doesNotSupportModTime { - continue - } - modTimeDiff := abs(stat.ModTime().UnixNano() - c.ModTime.UnixNano()) - if modTimeDiff >= int64(time.Microsecond) { - t.Fatalf("expected mod time %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), c.Name, modTimeDiff) - } - if os.Getuid() != 0 || tc.doesNotSupportOwner { - continue - } - if stat.Sys().(*syscall.Stat_t).Uid != uint32(c.Uid) { - t.Fatalf("expected uid %d, got %d, file %s", c.Uid, stat.Sys().(*syscall.Stat_t).Uid, c.Name) - } - } - }) - } -} - func compressBrotli(t *testing.T, data []byte) []byte { t.Helper() b := new(bytes.Buffer) diff --git a/unpack_unix_test.go b/unpack_unix_test.go new file mode 100644 index 00000000..59cb5f2d --- /dev/null +++ b/unpack_unix_test.go @@ -0,0 +1,104 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build unix + +package extract_test + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/hashicorp/go-extract" +) + +func TestUnpackWithPreserveFileAttributes(t *testing.T) { + uid, gid := 503, 20 + baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) + testCases := []struct { + name string + contents []archiveContent + packer func(*testing.T, []archiveContent) []byte + doesNotSupportModTime bool + doesNotSupportOwner bool + expectError bool + }{ + { + name: "tar", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + packer: packTar, + }, + { + name: "zip", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + doesNotSupportOwner: true, + packer: packZip, + }, + { + name: "rar", + contents: contentsRar2, + packer: packRar2, + doesNotSupportModTime: true, + }, + { + name: "7z", + contents: contents7z2, + doesNotSupportOwner: true, + packer: pack7z2, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ( + ctx = context.Background() + dst = t.TempDir() + src = asIoReader(t, tc.packer(t, tc.contents)) + cfg = extract.NewConfig(extract.WithPreserveFileAttributes(true)) + ) + if err := extract.Unpack(ctx, dst, src, cfg); err != nil { + t.Fatalf("error unpacking archive: %v", err) + } + for _, c := range tc.contents { + path := filepath.Join(dst, c.Name) + stat, err := os.Lstat(path) + if err != nil { + t.Fatalf("error getting file stats: %v", err) + } + if !(c.Mode&fs.ModeSymlink != 0) { // skip symlink checks + if stat.Mode().Perm() != c.Mode.Perm() { + t.Fatalf("expected file mode %v, got %v, file %s", c.Mode.Perm(), stat.Mode().Perm(), c.Name) + } + } + if tc.doesNotSupportModTime { + continue + } + modTimeDiff := abs(stat.ModTime().UnixNano() - c.ModTime.UnixNano()) + if modTimeDiff >= int64(time.Microsecond) { + t.Fatalf("expected mod time %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), c.Name, modTimeDiff) + } + if os.Getuid() != 0 || tc.doesNotSupportOwner { + continue + } + if stat.Sys().(*syscall.Stat_t).Uid != uint32(c.Uid) { + t.Fatalf("expected uid %d, got %d, file %s", c.Uid, stat.Sys().(*syscall.Stat_t).Uid, c.Name) + } + } + }) + } +} From 900c4d0791215891d401d0b399280e25d2de8448 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 11 Dec 2024 14:49:27 +0100 Subject: [PATCH 16/48] skip sudo test on windows --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 49caf181..a859119d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -31,6 +31,7 @@ jobs: make test - name: Test (as root) + if: matrix.os != 'windows-latest' id: sudo-test run: | make sudo_test From 96659b8a432b44ec38fa45e0ff00407466da08ce Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 11 Dec 2024 14:53:05 +0100 Subject: [PATCH 17/48] added some error handling --- 7zip.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/7zip.go b/7zip.go index ae9cb2bb..66b99615 100644 --- a/7zip.go +++ b/7zip.go @@ -127,9 +127,15 @@ func (z *sevenZipEntry) Linkname() string { if !z.IsSymlink() { return "" } - f, _ := z.f.Open() + f, err := z.f.Open() + if err != nil { + return "" + } defer f.Close() - c, _ := io.ReadAll(f) + c, err := io.ReadAll(f) + if err != nil { + return "" + } return string(c) } From f4bfff1ffd2b2c289e99bdccc7e29efd18aadb06 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 09:58:08 +0100 Subject: [PATCH 18/48] adjusted comment that rar does not support uig/gid --- rar.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rar.go b/rar.go index 54d81fcd..ad95efd7 100644 --- a/rar.go +++ b/rar.go @@ -152,12 +152,14 @@ func (r *rarEntry) Sys() interface{} { return r.f } -// Gid returns the group ID of the file. +// Gid is not supported for Rar files. The used library does not provide +// this information. The function returns the group ID of the current process. func (r *rarEntry) Gid() int { return os.Getgid() } -// Uid returns the user ID of the file. +// Uid is not supported for Rar files. The used library does not provide +// this information. The function returns the user ID of the current process. func (r *rarEntry) Uid() int { return os.Getuid() } From e9c39415aa46333e44734c434648643fa1403828 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 09:59:34 +0100 Subject: [PATCH 19/48] located chown in platform specific implementation --- target_disk.go | 8 -------- target_disk_others.go | 5 +++++ target_disk_unix.go | 9 +++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/target_disk.go b/target_disk.go index c09b1510..170f1980 100644 --- a/target_disk.go +++ b/target_disk.go @@ -120,14 +120,6 @@ func (d *TargetDisk) Chtimes(name string, atime, mtime time.Time) error { return os.Chtimes(name, atime, mtime) } -// Chown changes the numeric uid and gid of the named file. -func (d *TargetDisk) Chown(name string, uid, gid int) error { - if os.Geteuid() != 0 { - return nil - } - return os.Lchown(name, uid, gid) -} - // Lchtimes changes the access and modification times of the named file. func (d *TargetDisk) Lchtimes(name string, atime, mtime time.Time) error { if canMaintainSymlinkTimestamps { diff --git a/target_disk_others.go b/target_disk_others.go index b4dd1dc6..9a2ab63d 100644 --- a/target_disk_others.go +++ b/target_disk_others.go @@ -25,3 +25,8 @@ func lchtimes(_ string, _, _ time.Time) error { // because those methods follow the symlinks. However, a platform-dependent option // is provided for unix (see Lchtimes) const canMaintainSymlinkTimestamps = false + +// Chown changes the numeric uid and gid of the named file. +func (d *TargetDisk) Chown(name string, uid, gid int) error { + return fmt.Errorf("Chown is not supported on this platform (%s)", runtime.GOOS) +} diff --git a/target_disk_unix.go b/target_disk_unix.go index 05c58b6b..47068477 100644 --- a/target_disk_unix.go +++ b/target_disk_unix.go @@ -6,11 +6,20 @@ package extract import ( + "os" "time" "golang.org/x/sys/unix" ) +// Chown changes the numeric uid and gid of the named file. +func (d *TargetDisk) Chown(name string, uid, gid int) error { + if os.Geteuid() != 0 { + return nil + } + return os.Lchown(name, uid, gid) +} + // lchtimes modifies the access and modified timestamps on a target path // This capability is only available on unix as of now. func lchtimes(path string, atime, mtime time.Time) error { From 9ab3635987c7fd3a334e0f404e66c91873a22046 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 10:00:10 +0100 Subject: [PATCH 20/48] marked in test case that rar does not support file ownership --- unpack_unix_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unpack_unix_test.go b/unpack_unix_test.go index 59cb5f2d..154caae6 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -52,8 +52,9 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { { name: "rar", contents: contentsRar2, - packer: packRar2, doesNotSupportModTime: true, + doesNotSupportOwner: true, + packer: packRar2, }, { name: "7z", From 00f38ebd6e55448bd8bf961f7add882ef1d2d4ae Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 11:50:48 +0100 Subject: [PATCH 21/48] separated ownership preservation in an own flag --- 7zip.go | 10 +- README.md | 4 +- cmd/run.go | 4 +- config.go | 17 +++ extractor.go | 309 +++++++++++++++++++++--------------------- rar.go | 4 +- target_disk_unix.go | 3 +- target_memory.go | 2 + target_memory_test.go | 72 ++++------ unpack_unix_test.go | 145 +++++++++++++------- zip.go | 10 +- 11 files changed, 321 insertions(+), 259 deletions(-) diff --git a/7zip.go b/7zip.go index 66b99615..b579952e 100644 --- a/7zip.go +++ b/7zip.go @@ -180,13 +180,15 @@ func (z *sevenZipEntry) Sys() interface{} { return z.f.FileInfo().Sys() } -// Gid returns the group ID of the 7zip entry +// Gid is not supported for 7zip files. The used library does not provide +// this information. The function returns the group ID of the current process. func (z *sevenZipEntry) Gid() int { - return os.Getgid() + return 0 } -// Uid returns the user ID of the 7zip entry +// Uid is not supported for 7zip files. The used library does not provide +// this information. The function returns the user ID of the current process. func (z *sevenZipEntry) Uid() int { // get current uid - return os.Getuid() + return 0 } diff --git a/README.md b/README.md index 520cf0eb..48d34a51 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,8 @@ Flags: -N, --no-untar-after-decompression Disable combined extraction of tar.gz. -O, --overwrite Overwrite if exist. -P, --pattern=PATTERN,... Extracted objects need to match shell file name pattern. - -p, --preserve-file-attributes Preserve file attributes from archive (access and modification time, file permissions and owner/group). + -p, --preserve-file-attributes Preserve file attributes from archive (access and modification time & file permissions). + -o, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix systems). -T, --telemetry Print telemetry data to log after extraction. -t, --type="" Type of archive. (7z, br, bz2, gz, lz4, rar, sz, tar, tgz, xz, zip, zst, zz) -v, --verbose Verbose logging. @@ -105,6 +106,7 @@ When calling the `extract.Unpack(..)` function, we need to provide `config` obje extract.WithOverwrite(..), extract.WithPatterns(..), extract.WithPreserveFileAttributes(..), + extract.WithPreserveOwner(..), extract.WithTelemetryHook(..), ) diff --git a/cmd/run.go b/cmd/run.go index 15e56252..b5db3a02 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -38,7 +38,8 @@ type CLI struct { NoUntarAfterDecompression bool `short:"N" optional:"" default:"false" help:"Disable combined extraction of tar.gz."` Overwrite bool `short:"O" help:"Overwrite if exist."` Pattern []string `short:"P" optional:"" name:"pattern" help:"Extracted objects need to match shell file name pattern."` - PreserveFileAttributes bool `short:"p" help:"Preserve file attributes from archive (access and modification time, file permissions and owner/group)."` + PreserveFileAttributes bool `short:"p" help:"Preserve file attributes from archive (access and modification time & file permissions)."` + PreserveOwner bool `short:"o" help:"Preserve owner and group of files from archive (only root/uid:0 on unix systems)."` Telemetry bool `short:"T" optional:"" default:"false" help:"Print telemetry data to log after extraction."` Type string `short:"t" optional:"" default:"${default_type}" name:"type" help:"Type of archive. (${valid_types})"` Verbose bool `short:"v" optional:"" help:"Verbose logging."` @@ -99,6 +100,7 @@ func Run(version, commit, date string) { extract.WithOverwrite(cli.Overwrite), extract.WithPatterns(cli.Pattern...), extract.WithPreserveFileAttributes(cli.PreserveFileAttributes), + extract.WithPreserveOwner(cli.PreserveOwner), extract.WithTelemetryHook(telemetryDataToLog), ) diff --git a/config.go b/config.go index 7e170ee9..06017561 100644 --- a/config.go +++ b/config.go @@ -79,6 +79,9 @@ type Config struct { // preserveFileAttributes is a flag to preserve the file attributes of the extracted files preserveFileAttributes bool + + // preserveOwner is a flag to preserve the owner of the extracted files + preserveOwner bool } // ContinueOnError returns true if the extraction should continue on error. @@ -209,6 +212,11 @@ func (c *Config) PreserveFileAttributes() bool { return c.preserveFileAttributes } +// PreserveOwner returns true if the owner of the extracted files should be preserved. +func (c *Config) PreserveOwner() bool { + return c.preserveOwner +} + // SetNoUntarAfterDecompression sets the noUntarAfterDecompression flag. If true, tar.gz files // are not untared after decompression. func (c *Config) SetNoUntarAfterDecompression(b bool) { @@ -240,6 +248,7 @@ const ( defaultNoUntarAfterDecompression = false // untar after decompression defaultOverwrite = false // do not overwrite existing files defaultPreserveFileAttributes = false // do not preserve file attributes + defaultPreserveOwner = false // do not preserve owner defaultTraverseSymlinks = false // do not traverse symlinks ) @@ -276,6 +285,7 @@ func NewConfig(opts ...ConfigOption) *Config { traverseSymlinks: defaultTraverseSymlinks, noUntarAfterDecompression: defaultNoUntarAfterDecompression, preserveFileAttributes: defaultPreserveFileAttributes, + preserveOwner: defaultPreserveOwner, } // Loop through each option @@ -422,6 +432,13 @@ func WithPreserveFileAttributes(preserve bool) ConfigOption { } } +// WithPreserveOwner options pattern function to preserve the owner of the extracted files. +func WithPreserveOwner(preserve bool) ConfigOption { + return func(c *Config) { + c.preserveOwner = preserve + } +} + // WithTelemetryHook options pattern function to set a [telemetry.TelemetryHook], which is called after extraction. func WithTelemetryHook(hook TelemetryHook) ConfigOption { return func(c *Config) { diff --git a/extractor.go b/extractor.go index bafa6233..1f3c8609 100644 --- a/extractor.go +++ b/extractor.go @@ -270,205 +270,212 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * var fileCounter int64 var extractionSize int64 - // check if attributes should be preserved, but as non-root user - if _, ok := t.(*TargetDisk); ok { - if cfg.PreserveFileAttributes() && os.Geteuid() != 0 { - cfg.Logger().Warn("cannot fully preserve file attributes as non-root user: cannot set file ownership", "uid", os.Geteuid()) - } - } - - // set attributes after all modification are done to ensure that - // the timestamps are set correctly + // collect extracted entries if file attributes should be preserved + collectEntries := cfg.PreserveFileAttributes() || cfg.PreserveOwner() var extractedEntries []archiveEntry - if cfg.PreserveFileAttributes() { - defer func() { - for _, ae := range extractedEntries { - path := filepath.Join(dst, ae.Name()) - if err := setFileAttributes(t, path, ae); err != nil { - cfg.Logger().Error("failed to set file attributes", "path", path, "error", err) - } + + // iterate over all files in archive + err := func() error { + for { + // check if context is canceled + if ctx.Err() != nil { + return ctx.Err() } - }() - } - for { - // check if context is canceled - if ctx.Err() != nil { - return ctx.Err() - } + // get next file + ae, err := src.Next() - // get next file - ae, err := src.Next() + switch { - switch { + // if no more files are found exit loop + case err == io.EOF: + // extraction finished + return nil - // if no more files are found exit loop - case err == io.EOF: - // extraction finished - return nil + // handle other errors and end extraction or continue + case err != nil: + if err := handleError(cfg, td, "error reading", err); err != nil { + return err + } + continue - // handle other errors and end extraction or continue - case err != nil: - if err := handleError(cfg, td, "error reading", err); err != nil { - return err + // if the header is nil, just skip it (not sure how this happens) + case ae == nil: + continue } - continue - // if the header is nil, just skip it (not sure how this happens) - case ae == nil: - continue - } + // check for to many files (including folder and symlinks) in archive + fileCounter++ - // check for to many files (including folder and symlinks) in archive - fileCounter++ + // check if maximum of files (including folder and symlinks) is exceeded + if err := cfg.CheckMaxFiles(fileCounter); err != nil { + return handleError(cfg, td, "max objects check failed", err) + } - // check if maximum of files (including folder and symlinks) is exceeded - if err := cfg.CheckMaxFiles(fileCounter); err != nil { - return handleError(cfg, td, "max objects check failed", err) - } + // check if file needs to match patterns + match, err := checkPatterns(cfg.Patterns(), ae.Name()) + if err != nil { + return handleError(cfg, td, "cannot check pattern", err) + } + if !match { + cfg.Logger().Info("skipping file (pattern mismatch)", "name", ae.Name()) + td.PatternMismatches++ + continue + } - // check if file needs to match patterns - match, err := checkPatterns(cfg.Patterns(), ae.Name()) - if err != nil { - return handleError(cfg, td, "cannot check pattern", err) - } - if !match { - cfg.Logger().Info("skipping file (pattern mismatch)", "name", ae.Name()) - td.PatternMismatches++ - continue - } + cfg.Logger().Debug("extract", "name", ae.Name()) + switch { - cfg.Logger().Debug("extract", "name", ae.Name()) - switch { + // if its a dir and it doesn't exist create it + case ae.IsDir(): - // if its a dir and it doesn't exist create it - case ae.IsDir(): + // handle directory + if err := createDir(t, dst, ae.Name(), ae.Mode(), cfg); err != nil { + if err := handleError(cfg, td, "failed to create safe directory", err); err != nil { + return err + } - // handle directory - if err := createDir(t, dst, ae.Name(), ae.Mode(), cfg); err != nil { - if err := handleError(cfg, td, "failed to create safe directory", err); err != nil { - return err + // do not end on error + continue + } + if collectEntries { + extractedEntries = append(extractedEntries, ae) } - // do not end on error - continue - } - if cfg.PreserveFileAttributes() { - extractedEntries = append(extractedEntries, ae) - } - - // store telemetry and continue - td.ExtractedDirs++ - - // if it's a file create it - case ae.IsRegular(): + // store telemetry and continue + td.ExtractedDirs++ - // check extraction size forecast - if err := cfg.CheckExtractionSize(extractionSize + ae.Size()); err != nil { - return handleError(cfg, td, "max extraction size exceeded", err) - } + // if it's a file create it + case ae.IsRegular(): - // open file inm archive - err, fileCreated := func() (error, bool) { - fin, err := ae.Open() - if err != nil { - return handleError(cfg, td, "failed to open file", err), false + // check extraction size forecast + if err := cfg.CheckExtractionSize(extractionSize + ae.Size()); err != nil { + return handleError(cfg, td, "max extraction size exceeded", err) } - defer fin.Close() - // create file - n, err := createFile(t, dst, ae.Name(), fin, ae.Mode(), cfg.MaxExtractionSize()-extractionSize, cfg) - extractionSize = extractionSize + n - td.ExtractionSize = extractionSize + // open file inm archive + err, fileCreated := func() (error, bool) { + fin, err := ae.Open() + if err != nil { + return handleError(cfg, td, "failed to open file", err), false + } + defer fin.Close() + + // create file + n, err := createFile(t, dst, ae.Name(), fin, ae.Mode(), cfg.MaxExtractionSize()-extractionSize, cfg) + extractionSize = extractionSize + n + td.ExtractionSize = extractionSize + if err != nil { + + // increase error counter, set error and end if necessary + return handleError(cfg, td, "failed to create safe file", err), false + } + + // do not end on error + return nil, true + }() if err != nil { + return err + } - // increase error counter, set error and end if necessary - return handleError(cfg, td, "failed to create safe file", err), false + // store telemetry + if fileCreated { + td.ExtractedFiles++ + if collectEntries { + extractedEntries = append(extractedEntries, ae) + } } - // do not end on error - return nil, true - }() - if err != nil { - return err - } + // its a symlink !! + case ae.IsSymlink(): - // store telemetry - if fileCreated { - td.ExtractedFiles++ - if cfg.PreserveFileAttributes() { - extractedEntries = append(extractedEntries, ae) + // check if symlinks are allowed + if cfg.DenySymlinkExtraction() { + + err := unsupportedFile(ae.Name()) + if err := handleError(cfg, td, "symlink extraction disabled", err); err != nil { + return err + } + + // do not end on error + continue } - } - // its a symlink !! - case ae.IsSymlink(): + // create link + if err := createSymlink(t, dst, ae.Name(), ae.Linkname(), cfg); err != nil { - // check if symlinks are allowed - if cfg.DenySymlinkExtraction() { + // increase error counter, set error and end if necessary + if err := handleError(cfg, td, "failed to create safe symlink", err); err != nil { + return err + } - err := unsupportedFile(ae.Name()) - if err := handleError(cfg, td, "symlink extraction disabled", err); err != nil { - return err + // do not end on error + continue + } + if collectEntries { + extractedEntries = append(extractedEntries, ae) } - // do not end on error - continue - } + // store telemetry and continue + td.ExtractedSymlinks++ + + default: - // create link - if err := createSymlink(t, dst, ae.Name(), ae.Linkname(), cfg); err != nil { + // tar specific: check for git comment file `pax_global_header` from type `67` and skip + if ae.Type()&tar.TypeXGlobalHeader == tar.TypeXGlobalHeader && ae.Name() == "pax_global_header" { + continue + } - // increase error counter, set error and end if necessary - if err := handleError(cfg, td, "failed to create safe symlink", err); err != nil { + err := unsupportedFile(ae.Name()) + msg := fmt.Sprintf("unsupported filetype in archive (%x)", ae.Mode()) + if err := handleError(cfg, td, msg, err); err != nil { return err } // do not end on error continue } - if cfg.PreserveFileAttributes() { - extractedEntries = append(extractedEntries, ae) - } - - // store telemetry and continue - td.ExtractedSymlinks++ - - default: - - // tar specific: check for git comment file `pax_global_header` from type `67` and skip - if ae.Type()&tar.TypeXGlobalHeader == tar.TypeXGlobalHeader && ae.Name() == "pax_global_header" { - continue - } + } + }() + if err != nil { + return err + } - err := unsupportedFile(ae.Name()) - msg := fmt.Sprintf("unsupported filetype in archive (%x)", ae.Mode()) - if err := handleError(cfg, td, msg, err); err != nil { - return err + // set attributes after all modification are done to ensure that + // the timestamps are set correctly + if collectEntries { + for _, ae := range extractedEntries { + path := filepath.Join(dst, ae.Name()) + if err := setFileAttributesAndOwner(t, path, ae, cfg.PreserveFileAttributes(), cfg.PreserveOwner()); err != nil { + return fmt.Errorf("failed to set file attributes: %w", err) } - - // do not end on error - continue } } + + // extraction finished + return nil } -// setFileAttributes sets the file attributes for the given path and archive entry. -func setFileAttributes(t Target, path string, ae archiveEntry) error { - if ae.IsSymlink() { // only time attributes are supported for symlinks - if err := t.Lchtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { - return fmt.Errorf("failed to lchtimes symlink: %w", err) +// setFileAttributesAndOwner sets the file attributes for the given path and archive entry. +func setFileAttributesAndOwner(t Target, path string, ae archiveEntry, fileAttributes bool, owner bool) error { + if fileAttributes { + if ae.IsSymlink() { // only time attributes are supported for symlinks + if err := t.Lchtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { + return fmt.Errorf("failed to lchtimes symlink: %w", err) + } + return nil + } + if err := t.Chmod(path, ae.Mode().Perm()); err != nil { + return fmt.Errorf("failed to chmod file: %w", err) + } + if err := t.Chtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { + return fmt.Errorf("failed to chtimes file: %w", err) } - return nil - } - if err := t.Chown(path, ae.Uid(), ae.Gid()); err != nil { - return fmt.Errorf("failed to chown file: %w", err) - } - if err := t.Chmod(path, ae.Mode().Perm()); err != nil { - return fmt.Errorf("failed to chmod file: %w", err) } - if err := t.Chtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { - return fmt.Errorf("failed to chtimes file: %w", err) + if owner { + if err := t.Chown(path, ae.Uid(), ae.Gid()); err != nil { + return fmt.Errorf("failed to chown file: %w", err) + } } return nil } diff --git a/rar.go b/rar.go index ad95efd7..89603aa9 100644 --- a/rar.go +++ b/rar.go @@ -155,11 +155,11 @@ func (r *rarEntry) Sys() interface{} { // Gid is not supported for Rar files. The used library does not provide // this information. The function returns the group ID of the current process. func (r *rarEntry) Gid() int { - return os.Getgid() + return 0 } // Uid is not supported for Rar files. The used library does not provide // this information. The function returns the user ID of the current process. func (r *rarEntry) Uid() int { - return os.Getuid() + return 0 } diff --git a/target_disk_unix.go b/target_disk_unix.go index 47068477..4ab5b30f 100644 --- a/target_disk_unix.go +++ b/target_disk_unix.go @@ -6,6 +6,7 @@ package extract import ( + "io/fs" "os" "time" @@ -15,7 +16,7 @@ import ( // Chown changes the numeric uid and gid of the named file. func (d *TargetDisk) Chown(name string, uid, gid int) error { if os.Geteuid() != 0 { - return nil + return &fs.PathError{Op: "chown", Path: name, Err: os.ErrPermission} } return os.Lchown(name, uid, gid) } diff --git a/target_memory.go b/target_memory.go index 597b9461..69061360 100644 --- a/target_memory.go +++ b/target_memory.go @@ -524,6 +524,8 @@ func (m *TargetMemory) Lstat(path string) (fs.FileInfo, error) { mode: mfi.Mode(), accessTime: mfi.AccessTime(), modTime: mfi.ModTime(), + gid: mfi.Gid(), + uid: mfi.Uid(), }, nil } diff --git a/target_memory_test.go b/target_memory_test.go index 3b636200..55c2bc13 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -841,56 +841,21 @@ func TestCreateSymlink(t *testing.T) { } } -func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { - uid, gid := 503, 20 - baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) - testCases := []struct { - name string - contents []archiveContent - packer func(*testing.T, []archiveContent) []byte - doesNotSupportModTime bool - expectError bool - }{ - { - name: "unpack tar with preserve file attributes", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - // {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - packer: packTar, - }, - { - name: "unpack zip with preserve file attributes", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - // {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - packer: packZip, - }, - { - name: "unpack rar with preserve file attributes", - contents: contentsRar2, - doesNotSupportModTime: true, - packer: packRar2, - }, - { - name: "unpack z7 with preserve file attributes", - contents: contents7z2, - packer: pack7z2, - }, +func TestUnpackToMemoryWithPreserveFileAttributesAndOwner(t *testing.T) { + type ownershipAccessor interface { + Uid() int + Gid() int } - for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var ( ctx = context.Background() m = extract.NewTargetMemory() src = asIoReader(t, tc.packer(t, tc.contents)) - cfg = extract.NewConfig(extract.WithPreserveFileAttributes(true)) + cfg = extract.NewConfig( + extract.WithPreserveFileAttributes(true), + extract.WithPreserveOwner(true), + ) ) if err := extract.UnpackTo(ctx, m, "", src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) @@ -908,13 +873,22 @@ func TestUnpackToMemoryWithPreserveFileAttributes(t *testing.T) { t.Fatalf("expected file mode %v, got %v, file %s", c.Mode.Perm(), stat.Mode().Perm(), path) } } - if tc.doesNotSupportModTime { - continue + if !tc.doesNotSupportModTime { + // calculate the time difference + modTimeDiff := abs(stat.ModTime().UnixNano() - c.ModTime.UnixNano()) + if modTimeDiff >= int64(time.Microsecond) { + t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), path, modTimeDiff) + } } - // calculate the time difference - modTimeDiff := abs(stat.ModTime().UnixNano() - c.ModTime.UnixNano()) - if modTimeDiff >= int64(time.Microsecond) { - t.Fatalf("expected file modtime %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), path, modTimeDiff) + if !tc.doesNotSupportOwner { + if oa, ok := stat.(ownershipAccessor); ok { + if oa.Uid() != c.Uid { + t.Fatalf("expected file uid %v, got %v, file %s", c.Uid, oa.Uid(), path) + } + if oa.Gid() != c.Gid { + t.Fatalf("expected file gid %v, got %v, file %s", c.Gid, oa.Gid(), path) + } + } } } }) diff --git a/unpack_unix_test.go b/unpack_unix_test.go index 154caae6..75258709 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -17,53 +17,56 @@ import ( "github.com/hashicorp/go-extract" ) -func TestUnpackWithPreserveFileAttributes(t *testing.T) { - uid, gid := 503, 20 - baseTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) - testCases := []struct { - name string - contents []archiveContent - packer func(*testing.T, []archiveContent) []byte - doesNotSupportModTime bool - doesNotSupportOwner bool - expectError bool - }{ - { - name: "tar", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - packer: packTar, - }, - { - name: "zip", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - doesNotSupportOwner: true, - packer: packZip, - }, - { - name: "rar", - contents: contentsRar2, - doesNotSupportModTime: true, - doesNotSupportOwner: true, - packer: packRar2, +var ( + uid, gid = 503, 20 + baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) +) + +var testCases = []struct { + name string + contents []archiveContent + packer func(*testing.T, []archiveContent) []byte + doesNotSupportModTime bool + doesNotSupportOwner bool + expectError bool +}{ + { + name: "tar", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, }, - { - name: "7z", - contents: contents7z2, - doesNotSupportOwner: true, - packer: pack7z2, + packer: packTar, + }, + { + name: "zip", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, }, - } + doesNotSupportOwner: true, + packer: packZip, + }, + { + name: "rar", + contents: contentsRar2, + doesNotSupportOwner: true, + doesNotSupportModTime: true, + packer: packRar2, + }, + { + name: "7z", + contents: contents7z2, + doesNotSupportOwner: true, + packer: pack7z2, + }, +} +func TestUnpackWithPreserveFileAttributes(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var ( @@ -93,8 +96,58 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { if modTimeDiff >= int64(time.Microsecond) { t.Fatalf("expected mod time %v, got %v, file %s, diff %v", c.ModTime, stat.ModTime(), c.Name, modTimeDiff) } - if os.Getuid() != 0 || tc.doesNotSupportOwner { - continue + } + }) + } +} + +func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { + + if os.Getuid() == 0 { + t.Skip("test requires non-root privileges") + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ( + ctx = context.Background() + dst = t.TempDir() + src = asIoReader(t, tc.packer(t, tc.contents)) + cfg = extract.NewConfig(extract.WithPreserveOwner(true)) + ) + // fail always, bc/ root needed to set ownership + err := extract.Unpack(ctx, dst, src, cfg) + if err == nil { + t.Fatalf("expected error, got nil") + } + }) + } +} +func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { + + if os.Getuid() != 0 { + t.Skip("test requires root privileges") + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ( + ctx = context.Background() + dst = t.TempDir() + src = asIoReader(t, tc.packer(t, tc.contents)) + cfg = extract.NewConfig(extract.WithPreserveOwner(true)) + ) + if err := extract.Unpack(ctx, dst, src, cfg); err != nil { + t.Fatalf("error unpacking archive: %v", err) + } + if tc.doesNotSupportOwner { + t.Skipf("archive type %s does not store ownership information", tc.name) + } + for _, c := range tc.contents { + path := filepath.Join(dst, c.Name) + stat, err := os.Lstat(path) + if err != nil { + t.Fatalf("error getting file stats: %v", err) } if stat.Sys().(*syscall.Stat_t).Uid != uint32(c.Uid) { t.Fatalf("expected uid %d, got %d, file %s", c.Uid, stat.Sys().(*syscall.Stat_t).Uid, c.Name) diff --git a/zip.go b/zip.go index cbc578c5..9dd016b4 100644 --- a/zip.go +++ b/zip.go @@ -171,12 +171,14 @@ func (z *zipEntry) Sys() interface{} { return z.zf.FileHeader } -// Gid returns the group ID of the entry +// Gid is not supported for zip files. The used library does not provide +// this information. The function returns the group ID of the current process. func (z *zipEntry) Gid() int { - return os.Getgid() + return 0 } -// Uid returns the user ID of the entry +// Uid is not supported for zip files. The used library does not provide +// this information. The function returns the user ID of the current process. func (z *zipEntry) Uid() int { - return os.Getuid() + return 0 } From cb45ca4399ff3f85eb5f17074d7845e077983e17 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 11:56:13 +0100 Subject: [PATCH 22/48] relocated test data --- unpack_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++++ unpack_unix_test.go | 49 --------------------------------------------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index edac4552..eb1af931 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1622,6 +1622,55 @@ func packRar2(t *testing.T, _ []archiveContent) []byte { return b } +var ( + uid, gid = 503, 20 + baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) +) + +var testCases = []struct { + name string + contents []archiveContent + packer func(*testing.T, []archiveContent) []byte + doesNotSupportModTime bool + doesNotSupportOwner bool + expectError bool +}{ + { + name: "tar", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + packer: packTar, + }, + { + name: "zip", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + }, + doesNotSupportOwner: true, + packer: packZip, + }, + { + name: "rar", + contents: contentsRar2, + doesNotSupportOwner: true, + doesNotSupportModTime: true, + packer: packRar2, + }, + { + name: "7z", + contents: contents7z2, + doesNotSupportOwner: true, + packer: pack7z2, + }, +} + // openFile is a helper function to "open" a file, // but it returns an in-memory reader for example purposes. func openFile(_ string) io.ReadCloser { diff --git a/unpack_unix_test.go b/unpack_unix_test.go index 75258709..e90376ed 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -17,55 +17,6 @@ import ( "github.com/hashicorp/go-extract" ) -var ( - uid, gid = 503, 20 - baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) -) - -var testCases = []struct { - name string - contents []archiveContent - packer func(*testing.T, []archiveContent) []byte - doesNotSupportModTime bool - doesNotSupportOwner bool - expectError bool -}{ - { - name: "tar", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - packer: packTar, - }, - { - name: "zip", - contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, - }, - doesNotSupportOwner: true, - packer: packZip, - }, - { - name: "rar", - contents: contentsRar2, - doesNotSupportOwner: true, - doesNotSupportModTime: true, - packer: packRar2, - }, - { - name: "7z", - contents: contents7z2, - doesNotSupportOwner: true, - packer: pack7z2, - }, -} - func TestUnpackWithPreserveFileAttributes(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { From 2e7c38075670e21e0f918c4151b274cc2d41bdb3 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 12:00:15 +0100 Subject: [PATCH 23/48] added copyright header --- target_disk_unix_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/target_disk_unix_test.go b/target_disk_unix_test.go index 1fbf6241..4fba3668 100644 --- a/target_disk_unix_test.go +++ b/target_disk_unix_test.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + //go:build unix package extract From dacbd03817461471ee8f9a33efc16e206f64a806 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 12:43:15 +0100 Subject: [PATCH 24/48] updated comments to remark that ownership is only transported by tar archives and preserve-able as root on unix --- README.md | 2 +- cmd/run.go | 2 +- config.go | 11 ++++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 48d34a51..f5a9a6ff 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ Flags: -O, --overwrite Overwrite if exist. -P, --pattern=PATTERN,... Extracted objects need to match shell file name pattern. -p, --preserve-file-attributes Preserve file attributes from archive (access and modification time & file permissions). - -o, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix systems). + -o, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files). -T, --telemetry Print telemetry data to log after extraction. -t, --type="" Type of archive. (7z, br, bz2, gz, lz4, rar, sz, tar, tgz, xz, zip, zst, zz) -v, --verbose Verbose logging. diff --git a/cmd/run.go b/cmd/run.go index b5db3a02..49c0b0da 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -39,7 +39,7 @@ type CLI struct { Overwrite bool `short:"O" help:"Overwrite if exist."` Pattern []string `short:"P" optional:"" name:"pattern" help:"Extracted objects need to match shell file name pattern."` PreserveFileAttributes bool `short:"p" help:"Preserve file attributes from archive (access and modification time & file permissions)."` - PreserveOwner bool `short:"o" help:"Preserve owner and group of files from archive (only root/uid:0 on unix systems)."` + PreserveOwner bool `short:"o" help:"Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files)."` Telemetry bool `short:"T" optional:"" default:"false" help:"Print telemetry data to log after extraction."` Type string `short:"t" optional:"" default:"${default_type}" name:"type" help:"Type of archive. (${valid_types})"` Verbose bool `short:"v" optional:"" help:"Verbose logging."` diff --git a/config.go b/config.go index 06017561..edf88132 100644 --- a/config.go +++ b/config.go @@ -212,7 +212,9 @@ func (c *Config) PreserveFileAttributes() bool { return c.preserveFileAttributes } -// PreserveOwner returns true if the owner of the extracted files should be preserved. +// PreserveOwner returns true if the owner of the extracted files should +// be preserved. This option is only available on Unix systems requiring +// root privileges and tar archives as input. func (c *Config) PreserveOwner() bool { return c.preserveOwner } @@ -425,14 +427,17 @@ func WithPatterns(pattern ...string) ConfigOption { } } -// WithPreserveFileAttributes options pattern function to preserve the file attributes of the extracted files. +// WithPreserveFileAttributes options pattern function to preserve the +// file attributes of the extracted files. func WithPreserveFileAttributes(preserve bool) ConfigOption { return func(c *Config) { c.preserveFileAttributes = preserve } } -// WithPreserveOwner options pattern function to preserve the owner of the extracted files. +// WithPreserveOwner options pattern function to preserve the owner of +// the extracted files. This option is only available on Unix systems +// requiring root privileges and tar archives as input. func WithPreserveOwner(preserve bool) ConfigOption { return func(c *Config) { c.preserveOwner = preserve From 3fb2d38baf66d6f12507c0eb6891a16b3f0f411e Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 16 Dec 2024 13:03:03 +0100 Subject: [PATCH 25/48] verified overwrite behaviour for double-symlinks in archive --- unpack_test.go | 277 ++++++++++++++++++++++++++----------------------- 1 file changed, 145 insertions(+), 132 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index eb1af931..1307da3e 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -373,7 +373,10 @@ func TestUnpackArchive(t *testing.T) { ctx = context.Background() dst = t.TempDir() src = cacheFunction(t, tc.src) - cfg = extract.NewConfig(extract.WithCreateDestination(true), extract.WithContinueOnUnsupportedFiles(true)) + cfg = extract.NewConfig( + extract.WithCreateDestination(true), + extract.WithContinueOnUnsupportedFiles(true), + ) ) if err := extract.Unpack(ctx, dst, src, cfg); err != nil { @@ -701,7 +704,7 @@ func TestUnpackWithConfig(t *testing.T) { expectError: true, }, { - name: "unpack with overwrite enabled", + name: "unpack with overwrite enabled (files)", testArchive: []archiveContent{ {Name: "test", Content: []byte("hello world"), Mode: 0644}, {Name: "test", Content: []byte("hello world"), Mode: 0644}, @@ -709,6 +712,16 @@ func TestUnpackWithConfig(t *testing.T) { cfg: extract.NewConfig(extract.WithOverwrite(true)), expectError: false, }, + { + name: "unpack with overwrite enabled (symlink)", + testArchive: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0644}, + {Name: "link", Mode: fs.ModeSymlink | 0755, Linktarget: "test"}, + {Name: "link", Mode: fs.ModeSymlink | 0755, Linktarget: "test"}, + }, + cfg: extract.NewConfig(extract.WithOverwrite(true)), + expectError: false, + }, { name: "traverse symlink disabled", testArchive: []archiveContent{ @@ -1327,6 +1340,136 @@ func TestHasKnownArchiveExtension(t *testing.T) { } } +func TestWithCustomMode(t *testing.T) { + umask := sniffUmask(t) + + tests := []struct { + name string + data []byte + dst string + cfg *extract.Config + expected map[string]fs.FileMode + expectError bool + }{ + { + name: "dir with 0755 and file with 0644", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "sub/file", + Mode: fs.FileMode(0644), // 420 + }, + })), + cfg: extract.NewConfig( + extract.WithCustomCreateDirMode(fs.FileMode(0755)), // 493 + ), + expected: map[string]fs.FileMode{ + "sub": fs.FileMode(0755), // 493 + "sub/file": fs.FileMode(0644), // 420 + }, + }, + { + name: "decompress with custom mode", + data: compressGzip(t, []byte("foobar content")), + dst: "out", // specify decompressed file name + cfg: extract.NewConfig( + extract.WithCustomDecompressFileMode(fs.FileMode(0666)), // 438 + ), + expected: map[string]fs.FileMode{ + "out": fs.FileMode(0666), // 438 + }, + }, + { + name: "dir with 0755 and file with 0777", + data: compressGzip(t, []byte("foobar content")), + dst: "foo/out", + cfg: extract.NewConfig( + extract.WithCreateDestination(true), // create destination^ + extract.WithCustomCreateDirMode(fs.FileMode(0750)), // 488 + extract.WithCustomDecompressFileMode(fs.FileMode(0777)), // 511 + ), + expected: map[string]fs.FileMode{ + "foo": fs.FileMode(0750), // 488 + "foo/out": fs.FileMode(0777), // 511 + }, + }, + { + name: "dir with 0777 and file with 0777", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "sub/file", + Mode: fs.FileMode(0777), // 511 + }, + })), + cfg: extract.NewConfig( + extract.WithCustomCreateDirMode(fs.FileMode(0777)), // 511 + ), + expected: map[string]fs.FileMode{ + "sub": fs.FileMode(0777), // 511 + "sub/file": fs.FileMode(0777), // 511 + }, + }, + { + name: "file with 0000 permissions", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "file", + Mode: fs.FileMode(0000), // 0 + }, + { + Name: "dir/", + Mode: fs.ModeDir, // 000 permission + }, + })), + cfg: extract.NewConfig(), + expected: map[string]fs.FileMode{ + "file": fs.FileMode(0000), // 0 + "dir": fs.FileMode(0000), // 0 + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.cfg == nil { + test.cfg = extract.NewConfig() + } + var ( + ctx = context.Background() + tmp = t.TempDir() + dst = filepath.Join(tmp, test.dst) + src = asIoReader(t, test.data) + cfg = test.cfg + ) + err := extract.Unpack(ctx, dst, src, cfg) + if test.expectError && err == nil { + t.Fatalf("expected error, got nil") + } + if !test.expectError && err != nil { + t.Fatalf("unexpected error: %v", err) + } + for name, expectedMode := range test.expected { + stat, err := os.Stat(filepath.Join(tmp, name)) + if err != nil { + t.Fatalf("error getting file stats: %s", err) + } + + if runtime.GOOS == "windows" { + if stat.IsDir() { + continue // Skip directory checks on Windows + } + expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) + } else { + expectedMode &= ^umask // Adjust for umask on non-Windows systems + } + + if stat.Mode().Perm() != expectedMode.Perm() { + t.Fatalf("expected directory/file to have mode %s, but got: %s", expectedMode.Perm(), stat.Mode().Perm()) + } + } + }) + } +} + func abs(v int64) int64 { return int64(math.Abs(float64(v))) } @@ -1737,136 +1880,6 @@ func asIoReader(t *testing.T, b []byte) io.Reader { return r } -func TestWithCustomMode(t *testing.T) { - umask := sniffUmask(t) - - tests := []struct { - name string - data []byte - dst string - cfg *extract.Config - expected map[string]fs.FileMode - expectError bool - }{ - { - name: "dir with 0755 and file with 0644", - data: compressGzip(t, packTar(t, []archiveContent{ - { - Name: "sub/file", - Mode: fs.FileMode(0644), // 420 - }, - })), - cfg: extract.NewConfig( - extract.WithCustomCreateDirMode(fs.FileMode(0755)), // 493 - ), - expected: map[string]fs.FileMode{ - "sub": fs.FileMode(0755), // 493 - "sub/file": fs.FileMode(0644), // 420 - }, - }, - { - name: "decompress with custom mode", - data: compressGzip(t, []byte("foobar content")), - dst: "out", // specify decompressed file name - cfg: extract.NewConfig( - extract.WithCustomDecompressFileMode(fs.FileMode(0666)), // 438 - ), - expected: map[string]fs.FileMode{ - "out": fs.FileMode(0666), // 438 - }, - }, - { - name: "dir with 0755 and file with 0777", - data: compressGzip(t, []byte("foobar content")), - dst: "foo/out", - cfg: extract.NewConfig( - extract.WithCreateDestination(true), // create destination^ - extract.WithCustomCreateDirMode(fs.FileMode(0750)), // 488 - extract.WithCustomDecompressFileMode(fs.FileMode(0777)), // 511 - ), - expected: map[string]fs.FileMode{ - "foo": fs.FileMode(0750), // 488 - "foo/out": fs.FileMode(0777), // 511 - }, - }, - { - name: "dir with 0777 and file with 0777", - data: compressGzip(t, packTar(t, []archiveContent{ - { - Name: "sub/file", - Mode: fs.FileMode(0777), // 511 - }, - })), - cfg: extract.NewConfig( - extract.WithCustomCreateDirMode(fs.FileMode(0777)), // 511 - ), - expected: map[string]fs.FileMode{ - "sub": fs.FileMode(0777), // 511 - "sub/file": fs.FileMode(0777), // 511 - }, - }, - { - name: "file with 0000 permissions", - data: compressGzip(t, packTar(t, []archiveContent{ - { - Name: "file", - Mode: fs.FileMode(0000), // 0 - }, - { - Name: "dir/", - Mode: fs.ModeDir, // 000 permission - }, - })), - cfg: extract.NewConfig(), - expected: map[string]fs.FileMode{ - "file": fs.FileMode(0000), // 0 - "dir": fs.FileMode(0000), // 0 - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if test.cfg == nil { - test.cfg = extract.NewConfig() - } - var ( - ctx = context.Background() - tmp = t.TempDir() - dst = filepath.Join(tmp, test.dst) - src = asIoReader(t, test.data) - cfg = test.cfg - ) - err := extract.Unpack(ctx, dst, src, cfg) - if test.expectError && err == nil { - t.Fatalf("expected error, got nil") - } - if !test.expectError && err != nil { - t.Fatalf("unexpected error: %v", err) - } - for name, expectedMode := range test.expected { - stat, err := os.Stat(filepath.Join(tmp, name)) - if err != nil { - t.Fatalf("error getting file stats: %s", err) - } - - if runtime.GOOS == "windows" { - if stat.IsDir() { - continue // Skip directory checks on Windows - } - expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) - } else { - expectedMode &= ^umask // Adjust for umask on non-Windows systems - } - - if stat.Mode().Perm() != expectedMode.Perm() { - t.Fatalf("expected directory/file to have mode %s, but got: %s", expectedMode.Perm(), stat.Mode().Perm()) - } - } - }) - } -} - // sniffUmask is a helper function to get the umask func sniffUmask(t *testing.T) fs.FileMode { t.Helper() From 668eb6b4cadd39a53728752a685d69ce2a8f5543 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 08:23:42 +0100 Subject: [PATCH 26/48] adjusted error handling and wrapping --- target_disk_unix.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target_disk_unix.go b/target_disk_unix.go index 4ab5b30f..5d98ad64 100644 --- a/target_disk_unix.go +++ b/target_disk_unix.go @@ -15,10 +15,10 @@ import ( // Chown changes the numeric uid and gid of the named file. func (d *TargetDisk) Chown(name string, uid, gid int) error { - if os.Geteuid() != 0 { - return &fs.PathError{Op: "chown", Path: name, Err: os.ErrPermission} + if err := os.Lchown(name, uid, gid); err != nil { + return &fs.PathError{Op: "chown", Path: name, Err: err} } - return os.Lchown(name, uid, gid) + return nil } // lchtimes modifies the access and modified timestamps on a target path From e1ce17d6308871fc5e11a68b909bed14ea553537 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 10:01:25 +0100 Subject: [PATCH 27/48] Return current Uid/Gid if archive does not support carriage of ownership (zip/rar/7zip) Added various test cases that verifies the behaviour. --- 7zip.go | 5 ++--- extractor.go | 18 +++++++++++------- rar.go | 4 ++-- unpack_test.go | 43 +++++++++++++++++++++++++++++++++---------- unpack_unix_test.go | 16 +++++++++++++--- zip.go | 4 ++-- 6 files changed, 63 insertions(+), 27 deletions(-) diff --git a/7zip.go b/7zip.go index b579952e..7c1455f1 100644 --- a/7zip.go +++ b/7zip.go @@ -183,12 +183,11 @@ func (z *sevenZipEntry) Sys() interface{} { // Gid is not supported for 7zip files. The used library does not provide // this information. The function returns the group ID of the current process. func (z *sevenZipEntry) Gid() int { - return 0 + return os.Getegid() } // Uid is not supported for 7zip files. The used library does not provide // this information. The function returns the user ID of the current process. func (z *sevenZipEntry) Uid() int { - // get current uid - return 0 + return os.Getuid() } diff --git a/extractor.go b/extractor.go index 1f3c8609..22d9b203 100644 --- a/extractor.go +++ b/extractor.go @@ -274,6 +274,10 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * collectEntries := cfg.PreserveFileAttributes() || cfg.PreserveOwner() var extractedEntries []archiveEntry + if cfg.PreserveOwner() && src.Type() != fileExtensionTar { + cfg.Logger().Info("owner preservation is only supported for tar archives", "type", src.Type()) + } + // iterate over all files in archive err := func() error { for { @@ -463,13 +467,13 @@ func setFileAttributesAndOwner(t Target, path string, ae archiveEntry, fileAttri if err := t.Lchtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { return fmt.Errorf("failed to lchtimes symlink: %w", err) } - return nil - } - if err := t.Chmod(path, ae.Mode().Perm()); err != nil { - return fmt.Errorf("failed to chmod file: %w", err) - } - if err := t.Chtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { - return fmt.Errorf("failed to chtimes file: %w", err) + } else { + if err := t.Chmod(path, ae.Mode().Perm()); err != nil { + return fmt.Errorf("failed to chmod file: %w", err) + } + if err := t.Chtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { + return fmt.Errorf("failed to chtimes file: %w", err) + } } } if owner { diff --git a/rar.go b/rar.go index 89603aa9..945771fd 100644 --- a/rar.go +++ b/rar.go @@ -155,11 +155,11 @@ func (r *rarEntry) Sys() interface{} { // Gid is not supported for Rar files. The used library does not provide // this information. The function returns the group ID of the current process. func (r *rarEntry) Gid() int { - return 0 + return os.Getegid() } // Uid is not supported for Rar files. The used library does not provide // this information. The function returns the user ID of the current process. func (r *rarEntry) Uid() int { - return 0 + return os.Geteuid() } diff --git a/unpack_test.go b/unpack_test.go index 1307da3e..cf36b8c3 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1766,8 +1766,9 @@ func packRar2(t *testing.T, _ []archiveContent) []byte { } var ( - uid, gid = 503, 20 - baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) + testDataUid, testDataGid = 1337, 42 + testDataRootUid, testDataWheelGid = 0, 0 + baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) ) var testCases = []struct { @@ -1777,31 +1778,51 @@ var testCases = []struct { doesNotSupportModTime bool doesNotSupportOwner bool expectError bool + uid int + gid int }{ { name: "tar", contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: 0, Gid: 0}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataUid, Gid: testDataGid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataUid, Gid: testDataGid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataUid, Gid: testDataGid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataUid, Gid: testDataGid}, }, + uid: testDataUid, + gid: testDataGid, + packer: packTar, + }, + { + name: "root-tar", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataRootUid, Gid: testDataWheelGid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataRootUid, Gid: testDataWheelGid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataRootUid, Gid: testDataWheelGid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataRootUid, Gid: testDataWheelGid}, + }, + uid: testDataRootUid, + gid: testDataWheelGid, packer: packTar, }, { name: "zip", contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: uid, Gid: gid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime}, + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, }, + uid: os.Getgid(), + gid: os.Getgid(), doesNotSupportOwner: true, packer: packZip, }, { name: "rar", contents: contentsRar2, + uid: os.Getuid(), + gid: os.Getgid(), doesNotSupportOwner: true, doesNotSupportModTime: true, packer: packRar2, @@ -1809,6 +1830,8 @@ var testCases = []struct { { name: "7z", contents: contents7z2, + uid: os.Getuid(), + gid: os.Getgid(), doesNotSupportOwner: true, packer: pack7z2, }, diff --git a/unpack_unix_test.go b/unpack_unix_test.go index e90376ed..bdb09ddb 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -66,10 +66,20 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { src = asIoReader(t, tc.packer(t, tc.contents)) cfg = extract.NewConfig(extract.WithPreserveOwner(true)) ) - // fail always, bc/ root needed to set ownership + // compare archive uid/gid with current user + archiveEntriesEqualCurrentOwner := tc.doesNotSupportOwner + + // If the archive supports owner information, check if the current user is the owner of the test data + if !tc.doesNotSupportOwner && (os.Getuid() != tc.uid || os.Getgid() != tc.gid) { + archiveEntriesEqualCurrentOwner = false + } + + // Unpack should fail if the user is not root and err := extract.Unpack(ctx, dst, src, cfg) - if err == nil { - t.Fatalf("expected error, got nil") + + // chown will only fail if the user is not root and the uid/gid is different + if archiveEntriesEqualCurrentOwner && err != nil { + t.Fatalf("error unpacking archive: %v", err) } }) } diff --git a/zip.go b/zip.go index 9dd016b4..d0a77cde 100644 --- a/zip.go +++ b/zip.go @@ -174,11 +174,11 @@ func (z *zipEntry) Sys() interface{} { // Gid is not supported for zip files. The used library does not provide // this information. The function returns the group ID of the current process. func (z *zipEntry) Gid() int { - return 0 + return os.Getegid() } // Uid is not supported for zip files. The used library does not provide // this information. The function returns the user ID of the current process. func (z *zipEntry) Uid() int { - return 0 + return os.Getuid() } From 03a9e842b7a9d00b2e0aee0e9009fa25d0a47e65 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:02:35 +0100 Subject: [PATCH 28/48] inverted logic to preserve file attributes (mod/access time and file permissions) by default --- cmd/run.go | 6 +- config.go | 22 ++--- extractor.go | 4 +- target.go | 6 -- target_memory_test.go | 2 +- unpack_other_test.go | 189 ++++++++++++++++++++++++++++++++++++++++++ unpack_test.go | 130 ----------------------------- unpack_unix_test.go | 141 ++++++++++++++++++++++++++++++- 8 files changed, 346 insertions(+), 154 deletions(-) create mode 100644 unpack_other_test.go diff --git a/cmd/run.go b/cmd/run.go index 49c0b0da..1c7928df 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -35,10 +35,10 @@ type CLI struct { MaxExtractionSize int64 `optional:"" default:"${default_max_extraction_size}" help:"Maximum extraction size that allowed is (in bytes). (disable check: -1)"` MaxExtractionTime int64 `optional:"" default:"${default_max_extraction_time}" help:"Maximum time that an extraction should take (in seconds). (disable check: -1)"` MaxInputSize int64 `optional:"" default:"${default_max_input_size}" help:"Maximum input size that allowed is (in bytes). (disable check: -1)"` + NoPreserveFileAttributes bool `short:"P" help:"Do not preserve file attributes (mode, modtime, access time)."` NoUntarAfterDecompression bool `short:"N" optional:"" default:"false" help:"Disable combined extraction of tar.gz."` Overwrite bool `short:"O" help:"Overwrite if exist."` - Pattern []string `short:"P" optional:"" name:"pattern" help:"Extracted objects need to match shell file name pattern."` - PreserveFileAttributes bool `short:"p" help:"Preserve file attributes from archive (access and modification time & file permissions)."` + Pattern []string `optional:"" name:"pattern" help:"Extracted objects need to match shell file name pattern."` PreserveOwner bool `short:"o" help:"Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files)."` Telemetry bool `short:"T" optional:"" default:"false" help:"Print telemetry data to log after extraction."` Type string `short:"t" optional:"" default:"${default_type}" name:"type" help:"Type of archive. (${valid_types})"` @@ -96,10 +96,10 @@ func Run(version, commit, date string) { extract.WithMaxExtractionSize(cli.MaxExtractionSize), extract.WithMaxFiles(cli.MaxFiles), extract.WithMaxInputSize(cli.MaxInputSize), + extract.WithNoPreserveFileAttributes(cli.NoPreserveFileAttributes), extract.WithNoUntarAfterDecompression(cli.NoUntarAfterDecompression), extract.WithOverwrite(cli.Overwrite), extract.WithPatterns(cli.Pattern...), - extract.WithPreserveFileAttributes(cli.PreserveFileAttributes), extract.WithPreserveOwner(cli.PreserveOwner), extract.WithTelemetryHook(telemetryDataToLog), ) diff --git a/config.go b/config.go index edf88132..aaf88977 100644 --- a/config.go +++ b/config.go @@ -68,6 +68,9 @@ type Config struct { // Important: do not adjust this value after extraction started telemetryHook TelemetryHook + // noPreserveFileAttributes is a flag to not preserve the file attributes of the extracted files + noPreserveFileAttributes bool + // noUntarAfterDecompression offers the option to enable/disable combined tar.gz extraction noUntarAfterDecompression bool @@ -77,9 +80,6 @@ type Config struct { // patterns is a list of file patterns to match files to extract patterns []string - // preserveFileAttributes is a flag to preserve the file attributes of the extracted files - preserveFileAttributes bool - // preserveOwner is a flag to preserve the owner of the extracted files preserveOwner bool } @@ -207,9 +207,9 @@ func (c *Config) Patterns() []string { return c.patterns } -// PreserveFileAttributes returns true if the file attributes of the extracted files should be preserved. -func (c *Config) PreserveFileAttributes() bool { - return c.preserveFileAttributes +// NoPreserveFileAttributes returns true if the file attributes of the extracted files should *NOT* be preserved. +func (c *Config) NoPreserveFileAttributes() bool { + return c.noPreserveFileAttributes } // PreserveOwner returns true if the owner of the extracted files should @@ -249,7 +249,7 @@ const ( defaultMaxInputSize = 1 << (10 * 3) // 1 Gb defaultNoUntarAfterDecompression = false // untar after decompression defaultOverwrite = false // do not overwrite existing files - defaultPreserveFileAttributes = false // do not preserve file attributes + defaultNoPreserveFileAttributes = false // dont preserve file attributes from archive (inverse wording) defaultPreserveOwner = false // do not preserve owner defaultTraverseSymlinks = false // do not traverse symlinks @@ -285,8 +285,8 @@ func NewConfig(opts ...ConfigOption) *Config { overwrite: defaultOverwrite, telemetryHook: defaultTelemetryHook, traverseSymlinks: defaultTraverseSymlinks, + noPreserveFileAttributes: defaultNoPreserveFileAttributes, noUntarAfterDecompression: defaultNoUntarAfterDecompression, - preserveFileAttributes: defaultPreserveFileAttributes, preserveOwner: defaultPreserveOwner, } @@ -427,11 +427,11 @@ func WithPatterns(pattern ...string) ConfigOption { } } -// WithPreserveFileAttributes options pattern function to preserve the +// WithNoPreserveFileAttributes options pattern function to not preserve the // file attributes of the extracted files. -func WithPreserveFileAttributes(preserve bool) ConfigOption { +func WithNoPreserveFileAttributes(noPreserve bool) ConfigOption { return func(c *Config) { - c.preserveFileAttributes = preserve + c.noPreserveFileAttributes = noPreserve } } diff --git a/extractor.go b/extractor.go index 22d9b203..d033abb6 100644 --- a/extractor.go +++ b/extractor.go @@ -271,7 +271,7 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * var extractionSize int64 // collect extracted entries if file attributes should be preserved - collectEntries := cfg.PreserveFileAttributes() || cfg.PreserveOwner() + collectEntries := (!cfg.NoPreserveFileAttributes()) || cfg.PreserveOwner() var extractedEntries []archiveEntry if cfg.PreserveOwner() && src.Type() != fileExtensionTar { @@ -450,7 +450,7 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * if collectEntries { for _, ae := range extractedEntries { path := filepath.Join(dst, ae.Name()) - if err := setFileAttributesAndOwner(t, path, ae, cfg.PreserveFileAttributes(), cfg.PreserveOwner()); err != nil { + if err := setFileAttributesAndOwner(t, path, ae, (!cfg.NoPreserveFileAttributes()), cfg.PreserveOwner()); err != nil { return fmt.Errorf("failed to set file attributes: %w", err) } } diff --git a/target.go b/target.go index 298b6bf3..d6d88890 100644 --- a/target.go +++ b/target.go @@ -172,12 +172,6 @@ func createSymlink(t Target, dst string, name string, linkTarget string, cfg *Co // Check if link target is absolute path if filepath.IsAbs(linkTarget) { - // continue on error? - if cfg.ContinueOnError() { - cfg.Logger().Info("skip link target with absolute path", "link target", linkTarget) - return nil - } - // return error return fmt.Errorf("symlink with absolute path as target: %s", linkTarget) } diff --git a/target_memory_test.go b/target_memory_test.go index 55c2bc13..338b267b 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -853,7 +853,7 @@ func TestUnpackToMemoryWithPreserveFileAttributesAndOwner(t *testing.T) { m = extract.NewTargetMemory() src = asIoReader(t, tc.packer(t, tc.contents)) cfg = extract.NewConfig( - extract.WithPreserveFileAttributes(true), + extract.WithNoPreserveFileAttributes(false), extract.WithPreserveOwner(true), ) ) diff --git a/unpack_other_test.go b/unpack_other_test.go new file mode 100644 index 00000000..56461981 --- /dev/null +++ b/unpack_other_test.go @@ -0,0 +1,189 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !unix + +package extract_test + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/hashicorp/go-extract" +) + +func TestToWindowsFileMode(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("skipping test on non-windows systems") + } + otherMasks := []int{00, 01, 02, 03, 04, 05, 06, 07} + groupMasks := []int{00, 010, 020, 030, 040, 050, 060, 070} + userMasks := []int{00, 0100, 0200, 0300, 0400, 0500, 0600, 0700} + for _, dir := range []bool{true, false} { + for _, o := range otherMasks { + for _, g := range groupMasks { + for _, u := range userMasks { + var ( + path = filepath.Join(t.TempDir(), "test") + mode = fs.FileMode(u | g | o) + ) + if err := func() error { + if dir { + return os.Mkdir(path, mode) + } + return os.WriteFile(path, []byte("foobar content"), mode) + }(); err != nil { + t.Fatalf("error creating test resource: %s", err) + } + stat, err := os.Stat(path) + if err != nil { + t.Fatalf("error getting file stats: %s", err) + } + calculated := toWindowsFileMode(dir, mode) + if stat.Mode().Perm() != calculated.Perm() { + t.Errorf("toWindowsFileMode(%t, %s) calculated mode mode %s, but actual windows mode: %s", dir, mode, calculated.Perm(), stat.Mode().Perm()) + } + } + } + } + } +} + +func TestWithCustomMode(t *testing.T) { + + if runtime.GOOS != "windows" { + t.Skip("test only runs on Windows") + } + + umask := sniffUmask(t) + + tests := []struct { + name string + data []byte + dst string + cfg *extract.Config + expected map[string]fs.FileMode + expectError bool + }{ + { + name: "dir with 0755 and file with 0644", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "sub/file", + Mode: fs.FileMode(0644), // 420 + }, + })), + cfg: extract.NewConfig( + extract.WithCustomCreateDirMode(fs.FileMode(0755)), // 493 + ), + expected: map[string]fs.FileMode{ + "sub": fs.FileMode(0755), // 493 + "sub/file": fs.FileMode(0644), // 420 + }, + }, + { + name: "decompress with custom mode", + data: compressGzip(t, []byte("foobar content")), + dst: "out", // specify decompressed file name + cfg: extract.NewConfig( + extract.WithCustomDecompressFileMode(fs.FileMode(0666)), // 438 + ), + expected: map[string]fs.FileMode{ + "out": fs.FileMode(0666), // 438 + }, + }, + { + name: "dir with 0755 and file with 0777", + data: compressGzip(t, []byte("foobar content")), + dst: "foo/out", + cfg: extract.NewConfig( + extract.WithCreateDestination(true), // create destination^ + extract.WithCustomCreateDirMode(fs.FileMode(0750)), // 488 + extract.WithCustomDecompressFileMode(fs.FileMode(0777)), // 511 + ), + expected: map[string]fs.FileMode{ + "foo": fs.FileMode(0750), // 488 + "foo/out": fs.FileMode(0777), // 511 + }, + }, + { + name: "dir with 0777 and file with 0777", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "sub/file", + Mode: fs.FileMode(0777), // 511 + }, + })), + cfg: extract.NewConfig( + extract.WithCustomCreateDirMode(fs.FileMode(0777)), // 511 + ), + expected: map[string]fs.FileMode{ + "sub": fs.FileMode(0777), // 511 + "sub/file": fs.FileMode(0777), // 511 + }, + }, + { + name: "file with 0000 permissions", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "file", + Mode: fs.FileMode(0000), // 0 + }, + { + Name: "dir/", + Mode: fs.ModeDir, // 000 permission + }, + })), + cfg: extract.NewConfig(), + expected: map[string]fs.FileMode{ + "file": fs.FileMode(0000), // 0 + "dir": fs.FileMode(0000), // 0 + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.cfg == nil { + test.cfg = extract.NewConfig() + } + var ( + ctx = context.Background() + tmp = t.TempDir() + dst = filepath.Join(tmp, test.dst) + src = asIoReader(t, test.data) + cfg = test.cfg + ) + err := extract.Unpack(ctx, dst, src, cfg) + if test.expectError && err == nil { + t.Fatalf("expected error, got nil") + } + if !test.expectError && err != nil { + t.Fatalf("unexpected error: %v", err) + } + for name, expectedMode := range test.expected { + stat, err := os.Stat(filepath.Join(tmp, name)) + if err != nil { + t.Fatalf("error getting file stats: %s", err) + } + + if runtime.GOOS == "windows" { + if stat.IsDir() { + continue // Skip directory checks on Windows + } + expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) + } else { + expectedMode &= ^umask // Adjust for umask on non-Windows systems + } + + if stat.Mode().Perm() != expectedMode.Perm() { + t.Fatalf("expected directory/file to have mode %s, but got: %s", expectedMode.Perm(), stat.Mode().Perm()) + } + } + }) + } +} diff --git a/unpack_test.go b/unpack_test.go index cf36b8c3..776d1d77 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1340,136 +1340,6 @@ func TestHasKnownArchiveExtension(t *testing.T) { } } -func TestWithCustomMode(t *testing.T) { - umask := sniffUmask(t) - - tests := []struct { - name string - data []byte - dst string - cfg *extract.Config - expected map[string]fs.FileMode - expectError bool - }{ - { - name: "dir with 0755 and file with 0644", - data: compressGzip(t, packTar(t, []archiveContent{ - { - Name: "sub/file", - Mode: fs.FileMode(0644), // 420 - }, - })), - cfg: extract.NewConfig( - extract.WithCustomCreateDirMode(fs.FileMode(0755)), // 493 - ), - expected: map[string]fs.FileMode{ - "sub": fs.FileMode(0755), // 493 - "sub/file": fs.FileMode(0644), // 420 - }, - }, - { - name: "decompress with custom mode", - data: compressGzip(t, []byte("foobar content")), - dst: "out", // specify decompressed file name - cfg: extract.NewConfig( - extract.WithCustomDecompressFileMode(fs.FileMode(0666)), // 438 - ), - expected: map[string]fs.FileMode{ - "out": fs.FileMode(0666), // 438 - }, - }, - { - name: "dir with 0755 and file with 0777", - data: compressGzip(t, []byte("foobar content")), - dst: "foo/out", - cfg: extract.NewConfig( - extract.WithCreateDestination(true), // create destination^ - extract.WithCustomCreateDirMode(fs.FileMode(0750)), // 488 - extract.WithCustomDecompressFileMode(fs.FileMode(0777)), // 511 - ), - expected: map[string]fs.FileMode{ - "foo": fs.FileMode(0750), // 488 - "foo/out": fs.FileMode(0777), // 511 - }, - }, - { - name: "dir with 0777 and file with 0777", - data: compressGzip(t, packTar(t, []archiveContent{ - { - Name: "sub/file", - Mode: fs.FileMode(0777), // 511 - }, - })), - cfg: extract.NewConfig( - extract.WithCustomCreateDirMode(fs.FileMode(0777)), // 511 - ), - expected: map[string]fs.FileMode{ - "sub": fs.FileMode(0777), // 511 - "sub/file": fs.FileMode(0777), // 511 - }, - }, - { - name: "file with 0000 permissions", - data: compressGzip(t, packTar(t, []archiveContent{ - { - Name: "file", - Mode: fs.FileMode(0000), // 0 - }, - { - Name: "dir/", - Mode: fs.ModeDir, // 000 permission - }, - })), - cfg: extract.NewConfig(), - expected: map[string]fs.FileMode{ - "file": fs.FileMode(0000), // 0 - "dir": fs.FileMode(0000), // 0 - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if test.cfg == nil { - test.cfg = extract.NewConfig() - } - var ( - ctx = context.Background() - tmp = t.TempDir() - dst = filepath.Join(tmp, test.dst) - src = asIoReader(t, test.data) - cfg = test.cfg - ) - err := extract.Unpack(ctx, dst, src, cfg) - if test.expectError && err == nil { - t.Fatalf("expected error, got nil") - } - if !test.expectError && err != nil { - t.Fatalf("unexpected error: %v", err) - } - for name, expectedMode := range test.expected { - stat, err := os.Stat(filepath.Join(tmp, name)) - if err != nil { - t.Fatalf("error getting file stats: %s", err) - } - - if runtime.GOOS == "windows" { - if stat.IsDir() { - continue // Skip directory checks on Windows - } - expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) - } else { - expectedMode &= ^umask // Adjust for umask on non-Windows systems - } - - if stat.Mode().Perm() != expectedMode.Perm() { - t.Fatalf("expected directory/file to have mode %s, but got: %s", expectedMode.Perm(), stat.Mode().Perm()) - } - } - }) - } -} - func abs(v int64) int64 { return int64(math.Abs(float64(v))) } diff --git a/unpack_unix_test.go b/unpack_unix_test.go index bdb09ddb..f55e976b 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -24,7 +24,7 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { ctx = context.Background() dst = t.TempDir() src = asIoReader(t, tc.packer(t, tc.contents)) - cfg = extract.NewConfig(extract.WithPreserveFileAttributes(true)) + cfg = extract.NewConfig(extract.WithNoPreserveFileAttributes(false)) ) if err := extract.Unpack(ctx, dst, src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) @@ -117,3 +117,142 @@ func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { }) } } + +func TestWithCustomMode(t *testing.T) { + umask := sniffUmask(t) + // expectedMode &= ^umask // Adjust for umask on non-Windows systems + + tests := []struct { + name string + data []byte + dst string + cfg *extract.Config + expected map[string]fs.FileMode + expectError bool + }{ + { + name: "dir with 0755 and file with 0644", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "sub/file", + Mode: fs.FileMode(0644), // 420 + }, + })), + cfg: extract.NewConfig( + extract.WithCustomCreateDirMode(fs.FileMode(0757 & ^umask)), // 493 & ^umask + ), + expected: map[string]fs.FileMode{ + "sub": fs.FileMode(0757 & ^umask), // 493 & ^umask <-- implicit created dir + "sub/file": fs.FileMode(0644), // 420 + }, + }, + { + name: "decompress with custom mode", + data: compressGzip(t, []byte("foobar content")), + dst: "out", // specify decompressed file name + cfg: extract.NewConfig( + extract.WithCustomDecompressFileMode(fs.FileMode(0666)), // 438 + umask is applied while file creation + ), + expected: map[string]fs.FileMode{ + "out": 0666 & ^umask, // 438 & ^umask + }, + }, + { + name: "dir with 0755 and file with 0777", + data: compressGzip(t, []byte("foobar content")), + dst: "foo/out", + cfg: extract.NewConfig( + extract.WithCreateDestination(true), // create destination^ + extract.WithCustomCreateDirMode(fs.FileMode(0750)), // 488 + umask is applied while dir creation + extract.WithCustomDecompressFileMode(fs.FileMode(0777)), // 511 + umask is applied while file creation + ), + expected: map[string]fs.FileMode{ + "foo": fs.FileMode(0750 & ^umask), // 488 & ^umask + "foo/out": fs.FileMode(0777 & ^umask), // 511 & ^umask + }, + }, + { + name: "dir with 0777 and file with 0777", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "sub/file", + Mode: fs.FileMode(0777), // 511 + }, + })), + cfg: extract.NewConfig( + extract.WithCustomCreateDirMode(fs.FileMode(0777)), // 511 + umask is applied while dir creation + ), + expected: map[string]fs.FileMode{ + "sub": fs.FileMode(0777 & ^umask), // 511 + "sub/file": fs.FileMode(0777), // 511 <-- is preserved from the archive and umask is not applied + }, + }, + { + name: "file with 0000 permissions", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "file", + Mode: fs.FileMode(0000), // 0 + }, + { + Name: "dir/", + Mode: fs.ModeDir, // 000 permission + }, + })), + cfg: extract.NewConfig(), + expected: map[string]fs.FileMode{ + "file": fs.FileMode(0000), // 0 + "dir": fs.FileMode(0000), // 0 + }, + }, + { + name: "dir with 777 and file with 777 but no file attribute mode preservation", + data: compressGzip(t, packTar(t, []archiveContent{ + { + Name: "file", + Mode: fs.FileMode(0777), // 511 + }, + { + Name: "dir", + Mode: fs.ModeDir | 0777, // 511 + }, + })), + cfg: extract.NewConfig(extract.WithNoPreserveFileAttributes(true)), + expected: map[string]fs.FileMode{ + "file": fs.FileMode(0777 & ^umask), // 438 + "dir": fs.FileMode(0777 & ^umask), // 438 + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.cfg == nil { + test.cfg = extract.NewConfig() + } + var ( + ctx = context.Background() + tmp = t.TempDir() + dst = filepath.Join(tmp, test.dst) + src = asIoReader(t, test.data) + cfg = test.cfg + ) + err := extract.Unpack(ctx, dst, src, cfg) + if test.expectError && err == nil { + t.Fatalf("expected error, got nil") + } + if !test.expectError && err != nil { + t.Fatalf("unexpected error: %v", err) + } + for name, expectedMode := range test.expected { + stat, err := os.Stat(filepath.Join(tmp, name)) + if err != nil { + t.Fatalf("error getting file stats: %s", err) + } + if stat.Mode().Perm() != expectedMode.Perm() { + t.Fatalf("expected %s to have mode %s, but got: %s", name, expectedMode.Perm(), stat.Mode().Perm()) + } + } + }) + } +} From f5b7936c78e6bbddc6431d7193113026805680f9 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:11:47 +0100 Subject: [PATCH 29/48] removed windows specifc code --- unpack_test.go | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index 776d1d77..62a8dc7b 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1146,43 +1146,6 @@ func TestUnpackWithTypes(t *testing.T) { } } -func TestToWindowsFileMode(t *testing.T) { - if runtime.GOOS != "windows" { - t.Skip("skipping test on non-windows systems") - } - otherMasks := []int{00, 01, 02, 03, 04, 05, 06, 07} - groupMasks := []int{00, 010, 020, 030, 040, 050, 060, 070} - userMasks := []int{00, 0100, 0200, 0300, 0400, 0500, 0600, 0700} - for _, dir := range []bool{true, false} { - for _, o := range otherMasks { - for _, g := range groupMasks { - for _, u := range userMasks { - var ( - path = filepath.Join(t.TempDir(), "test") - mode = fs.FileMode(u | g | o) - ) - if err := func() error { - if dir { - return os.Mkdir(path, mode) - } - return os.WriteFile(path, []byte("foobar content"), mode) - }(); err != nil { - t.Fatalf("error creating test resource: %s", err) - } - stat, err := os.Stat(path) - if err != nil { - t.Fatalf("error getting file stats: %s", err) - } - calculated := toWindowsFileMode(dir, mode) - if stat.Mode().Perm() != calculated.Perm() { - t.Errorf("toWindowsFileMode(%t, %s) calculated mode mode %s, but actual windows mode: %s", dir, mode, calculated.Perm(), stat.Mode().Perm()) - } - } - } - } - } -} - func TestUnsupportedArchiveNames(t *testing.T) { // test testCases testCases := []struct { From ccf382ea41d7cc203e86d4d036a623b1df98d008 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:17:36 +0100 Subject: [PATCH 30/48] adjusted test cases file split --- unpack_other_test.go | 30 ++++++++++++++++++------------ unpack_test.go | 17 ----------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/unpack_other_test.go b/unpack_other_test.go index 56461981..af9b9cb6 100644 --- a/unpack_other_test.go +++ b/unpack_other_test.go @@ -59,8 +59,6 @@ func TestWithCustomMode(t *testing.T) { t.Skip("test only runs on Windows") } - umask := sniffUmask(t) - tests := []struct { name string data []byte @@ -170,16 +168,7 @@ func TestWithCustomMode(t *testing.T) { if err != nil { t.Fatalf("error getting file stats: %s", err) } - - if runtime.GOOS == "windows" { - if stat.IsDir() { - continue // Skip directory checks on Windows - } - expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) - } else { - expectedMode &= ^umask // Adjust for umask on non-Windows systems - } - + expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) if stat.Mode().Perm() != expectedMode.Perm() { t.Fatalf("expected directory/file to have mode %s, but got: %s", expectedMode.Perm(), stat.Mode().Perm()) } @@ -187,3 +176,20 @@ func TestWithCustomMode(t *testing.T) { }) } } + +// toWindowsFileMode converts a fs.FileMode to a windows file mode +func toWindowsFileMode(isDir bool, mode fs.FileMode) fs.FileMode { + + // handle special case + if isDir { + return fs.FileMode(0777) + } + + // check for write permission + if mode&0200 != 0 { + return fs.FileMode(0666) + } + + // return the mode + return fs.FileMode(0444) +} diff --git a/unpack_test.go b/unpack_test.go index 62a8dc7b..1adad454 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1760,20 +1760,3 @@ func sniffUmask(t *testing.T) fs.FileMode { // return the umask return umask } - -// toWindowsFileMode converts a fs.FileMode to a windows file mode -func toWindowsFileMode(isDir bool, mode fs.FileMode) fs.FileMode { - - // handle special case - if isDir { - return fs.FileMode(0777) - } - - // check for write permission - if mode&0200 != 0 { - return fs.FileMode(0666) - } - - // return the mode - return fs.FileMode(0444) -} From 111a61f57bb8f58398b2b0d546ed8b725e2e2c0e Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:29:18 +0100 Subject: [PATCH 31/48] adjusted test output log --- unpack_other_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unpack_other_test.go b/unpack_other_test.go index af9b9cb6..b21e8e85 100644 --- a/unpack_other_test.go +++ b/unpack_other_test.go @@ -170,7 +170,7 @@ func TestWithCustomMode(t *testing.T) { } expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) if stat.Mode().Perm() != expectedMode.Perm() { - t.Fatalf("expected directory/file to have mode %s, but got: %s", expectedMode.Perm(), stat.Mode().Perm()) + t.Fatalf("expected %s to have mode %s, but got: %s", name, expectedMode.Perm(), stat.Mode().Perm()) } } }) From b4c67e90bab0758cc8bb37474261a19ab2c3c89c Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:37:42 +0100 Subject: [PATCH 32/48] drop write permissions on directory if no access is granted --- unpack_other_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unpack_other_test.go b/unpack_other_test.go index b21e8e85..16c69b36 100644 --- a/unpack_other_test.go +++ b/unpack_other_test.go @@ -182,7 +182,10 @@ func toWindowsFileMode(isDir bool, mode fs.FileMode) fs.FileMode { // handle special case if isDir { - return fs.FileMode(0777) + if mode&0222 > 0 { // drop write permission if not set + return fs.FileMode(0777) + } + return fs.FileMode(0777 & ^fs.FileMode(0222)) } // check for write permission From 7671e158756884b3876b886b0497c952258662fa Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:46:58 +0100 Subject: [PATCH 33/48] drop permission check for directories on windows, bc/ they create unstready results --- unpack_other_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/unpack_other_test.go b/unpack_other_test.go index 16c69b36..b2252a97 100644 --- a/unpack_other_test.go +++ b/unpack_other_test.go @@ -43,6 +43,9 @@ func TestToWindowsFileMode(t *testing.T) { if err != nil { t.Fatalf("error getting file stats: %s", err) } + if dir { + continue // skip directory tests, as they are not supported on windows und create unpredictable results + } calculated := toWindowsFileMode(dir, mode) if stat.Mode().Perm() != calculated.Perm() { t.Errorf("toWindowsFileMode(%t, %s) calculated mode mode %s, but actual windows mode: %s", dir, mode, calculated.Perm(), stat.Mode().Perm()) @@ -168,6 +171,9 @@ func TestWithCustomMode(t *testing.T) { if err != nil { t.Fatalf("error getting file stats: %s", err) } + if stat.IsDir() { + continue // skip directory tests, as they are not supported on windows und create unpredictable results + } expectedMode = toWindowsFileMode(stat.IsDir(), expectedMode) if stat.Mode().Perm() != expectedMode.Perm() { t.Fatalf("expected %s to have mode %s, but got: %s", name, expectedMode.Perm(), stat.Mode().Perm()) @@ -182,10 +188,7 @@ func toWindowsFileMode(isDir bool, mode fs.FileMode) fs.FileMode { // handle special case if isDir { - if mode&0222 > 0 { // drop write permission if not set - return fs.FileMode(0777) - } - return fs.FileMode(0777 & ^fs.FileMode(0222)) + return fs.FileMode(0777) } // check for write permission From 879edcd19a7ef71d5b88463a094e722e60e99bd2 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:55:33 +0100 Subject: [PATCH 34/48] updated test case comment --- unpack_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index 1adad454..a042afbb 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1646,16 +1646,16 @@ var testCases = []struct { {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, }, - uid: os.Getgid(), - gid: os.Getgid(), + uid: os.Getgid(), // bc/ of the way zip works, we can't set the uid and the created files are owned by the user running the test + gid: os.Getgid(), // bc/ of the way zip works, we can't set the gid and the created files are owned by the user running the test doesNotSupportOwner: true, packer: packZip, }, { name: "rar", contents: contentsRar2, - uid: os.Getuid(), - gid: os.Getgid(), + uid: os.Getuid(), // bc/ of the way rar works, we can't set the uid and the created files are owned by the user running the test + gid: os.Getgid(), // bc/ of the way rar works, we can't set the gid and the created files are owned by the user running the test doesNotSupportOwner: true, doesNotSupportModTime: true, packer: packRar2, @@ -1663,8 +1663,8 @@ var testCases = []struct { { name: "7z", contents: contents7z2, - uid: os.Getuid(), - gid: os.Getgid(), + uid: os.Getuid(), // bc/ of the way 7z works, we can't set the uid and the created files are owned by the user running the test + gid: os.Getgid(), // bc/ of the way 7z works, we can't set the gid and the created files are owned by the user running the test doesNotSupportOwner: true, packer: pack7z2, }, From 450aba362b6fe1e760dbdd8cbfced9c1b7410c98 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 11:58:35 +0100 Subject: [PATCH 35/48] clarified test in comment --- unpack_unix_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unpack_unix_test.go b/unpack_unix_test.go index f55e976b..aa1efc74 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -74,7 +74,9 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { archiveEntriesEqualCurrentOwner = false } - // Unpack should fail if the user is not root and + // Unpack should fail if the user is not root and the uid/gid + // in the archive is different from the current user (only + // if the archive supports owner information) err := extract.Unpack(ctx, dst, src, cfg) // chown will only fail if the user is not root and the uid/gid is different From 3c7c310952128d3d1dfbc5f547d8f876068569b1 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Tue, 17 Dec 2024 12:06:55 +0100 Subject: [PATCH 36/48] updated readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f5a9a6ff..dd29a197 100644 --- a/README.md +++ b/README.md @@ -73,10 +73,10 @@ Flags: --max-extraction-size=1073741824 Maximum extraction size that allowed is (in bytes). (disable check: -1) --max-extraction-time=60 Maximum time that an extraction should take (in seconds). (disable check: -1) --max-input-size=1073741824 Maximum input size that allowed is (in bytes). (disable check: -1) + -P, --no-preserve-file-attributes Do not preserve file attributes (mode, modtime, access time). -N, --no-untar-after-decompression Disable combined extraction of tar.gz. -O, --overwrite Overwrite if exist. - -P, --pattern=PATTERN,... Extracted objects need to match shell file name pattern. - -p, --preserve-file-attributes Preserve file attributes from archive (access and modification time & file permissions). + --pattern=PATTERN,... Extracted objects need to match shell file name pattern. -o, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files). -T, --telemetry Print telemetry data to log after extraction. -t, --type="" Type of archive. (7z, br, bz2, gz, lz4, rar, sz, tar, tgz, xz, zip, zst, zz) @@ -102,10 +102,10 @@ When calling the `extract.Unpack(..)` function, we need to provide `config` obje extract.WithMaxExtractionSize(..), extract.WithMaxFiles(..), extract.WithMaxInputSize(..), + extract.WithNoPreserveFileAttributes(..), extract.WithNoUntarAfterDecompression(..), extract.WithOverwrite(..), extract.WithPatterns(..), - extract.WithPreserveFileAttributes(..), extract.WithPreserveOwner(..), extract.WithTelemetryHook(..), ) From 5ff62798be4f469be5c42b1c77c3a23f40ca5a5d Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 18 Dec 2024 10:02:11 +0100 Subject: [PATCH 37/48] renamed config.noPreserveFileAttributes to config.dropFileAttributes --- README.md | 15 +++++++++------ cmd/run.go | 6 +++--- config.go | 36 ++++++++++++++++++------------------ extractor.go | 10 +++++----- target_memory_test.go | 2 +- unpack_unix_test.go | 4 ++-- 6 files changed, 38 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index dd29a197..51da3eec 100644 --- a/README.md +++ b/README.md @@ -65,19 +65,22 @@ Flags: -C, --continue-on-error Continue extraction on error. -S, --continue-on-unsupported-files Skip extraction of unsupported files. -c, --create-destination Create destination directory if it does not exist. - --custom-create-dir-mode=750 File mode for created directories, which are not listed in the archive. (respecting umask) + --custom-create-dir-mode=750 File mode for created directories, which are not listed in the archive. + (respecting umask) --custom-decompress-file-mode=640 File mode for decompressed files. (respecting umask) -D, --deny-symlinks Deny symlink extraction. + -d, --drop-file-attributes Drop file attributes (mode, modtime, access time). --insecure-traverse-symlinks Traverse symlinks to directories during extraction. - --max-files=100000 Maximum files (including folder and symlinks) that are extracted before stop. (disable check: -1) + --max-files=100000 Maximum files (including folder and symlinks) that are extracted before stop. + (disable check: -1) --max-extraction-size=1073741824 Maximum extraction size that allowed is (in bytes). (disable check: -1) --max-extraction-time=60 Maximum time that an extraction should take (in seconds). (disable check: -1) --max-input-size=1073741824 Maximum input size that allowed is (in bytes). (disable check: -1) - -P, --no-preserve-file-attributes Do not preserve file attributes (mode, modtime, access time). -N, --no-untar-after-decompression Disable combined extraction of tar.gz. -O, --overwrite Overwrite if exist. - --pattern=PATTERN,... Extracted objects need to match shell file name pattern. - -o, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files). + -P, --pattern=PATTERN,... Extracted objects need to match shell file name pattern. + -o, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix + systems for tar files). -T, --telemetry Print telemetry data to log after extraction. -t, --type="" Type of archive. (7z, br, bz2, gz, lz4, rar, sz, tar, tgz, xz, zip, zst, zz) -v, --verbose Verbose logging. @@ -96,13 +99,13 @@ When calling the `extract.Unpack(..)` function, we need to provide `config` obje extract.WithCustomCreateDirMode(..), extract.WithCustomDecompressFileMode(..), extract.WithDenySymlinkExtraction(..), + extract.WithDropFileAttributes(..), extract.WithExtractType(..), extract.WithInsecureTraverseSymlinks(..), extract.WithLogger(..), extract.WithMaxExtractionSize(..), extract.WithMaxFiles(..), extract.WithMaxInputSize(..), - extract.WithNoPreserveFileAttributes(..), extract.WithNoUntarAfterDecompression(..), extract.WithOverwrite(..), extract.WithPatterns(..), diff --git a/cmd/run.go b/cmd/run.go index 1c7928df..dfd4762f 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -30,15 +30,15 @@ type CLI struct { CustomDecompressFileMode int `optional:"" default:"640" help:"File mode for decompressed files. (respecting umask)"` DenySymlinks bool `short:"D" help:"Deny symlink extraction."` Destination string `arg:"" name:"destination" default:"." help:"Output directory/file."` + DropFileAttributes bool `short:"d" help:"Drop file attributes (mode, modtime, access time)."` InsecureTraverseSymlinks bool `help:"Traverse symlinks to directories during extraction."` MaxFiles int64 `optional:"" default:"${default_max_files}" help:"Maximum files (including folder and symlinks) that are extracted before stop. (disable check: -1)"` MaxExtractionSize int64 `optional:"" default:"${default_max_extraction_size}" help:"Maximum extraction size that allowed is (in bytes). (disable check: -1)"` MaxExtractionTime int64 `optional:"" default:"${default_max_extraction_time}" help:"Maximum time that an extraction should take (in seconds). (disable check: -1)"` MaxInputSize int64 `optional:"" default:"${default_max_input_size}" help:"Maximum input size that allowed is (in bytes). (disable check: -1)"` - NoPreserveFileAttributes bool `short:"P" help:"Do not preserve file attributes (mode, modtime, access time)."` NoUntarAfterDecompression bool `short:"N" optional:"" default:"false" help:"Disable combined extraction of tar.gz."` Overwrite bool `short:"O" help:"Overwrite if exist."` - Pattern []string `optional:"" name:"pattern" help:"Extracted objects need to match shell file name pattern."` + Pattern []string `optional:"" short:"P" name:"pattern" help:"Extracted objects need to match shell file name pattern."` PreserveOwner bool `short:"o" help:"Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files)."` Telemetry bool `short:"T" optional:"" default:"false" help:"Print telemetry data to log after extraction."` Type string `short:"t" optional:"" default:"${default_type}" name:"type" help:"Type of archive. (${valid_types})"` @@ -96,7 +96,7 @@ func Run(version, commit, date string) { extract.WithMaxExtractionSize(cli.MaxExtractionSize), extract.WithMaxFiles(cli.MaxFiles), extract.WithMaxInputSize(cli.MaxInputSize), - extract.WithNoPreserveFileAttributes(cli.NoPreserveFileAttributes), + extract.WithDropFileAttributes(cli.DropFileAttributes), extract.WithNoUntarAfterDecompression(cli.NoUntarAfterDecompression), extract.WithOverwrite(cli.Overwrite), extract.WithPatterns(cli.Pattern...), diff --git a/config.go b/config.go index aaf88977..d0a4fb87 100644 --- a/config.go +++ b/config.go @@ -43,6 +43,9 @@ type Config struct { // denySymlinkExtraction offers the option to enable/disable the extraction of symlinks denySymlinkExtraction bool + // dropFileAttributes is a flag drop the file attributes of the extracted files + dropFileAttributes bool + // extractionType is the type of extraction algorithm extractionType string @@ -68,9 +71,6 @@ type Config struct { // Important: do not adjust this value after extraction started telemetryHook TelemetryHook - // noPreserveFileAttributes is a flag to not preserve the file attributes of the extracted files - noPreserveFileAttributes bool - // noUntarAfterDecompression offers the option to enable/disable combined tar.gz extraction noUntarAfterDecompression bool @@ -161,6 +161,11 @@ func (c *Config) DenySymlinkExtraction() bool { return c.denySymlinkExtraction } +// DropFileAttributes returns true if the file attributes should be dropped. +func (c *Config) DropFileAttributes() bool { + return c.dropFileAttributes +} + // ExtractType returns the specified extraction type. func (c *Config) ExtractType() string { return c.extractionType @@ -207,11 +212,6 @@ func (c *Config) Patterns() []string { return c.patterns } -// NoPreserveFileAttributes returns true if the file attributes of the extracted files should *NOT* be preserved. -func (c *Config) NoPreserveFileAttributes() bool { - return c.noPreserveFileAttributes -} - // PreserveOwner returns true if the owner of the extracted files should // be preserved. This option is only available on Unix systems requiring // root privileges and tar archives as input. @@ -243,13 +243,13 @@ const ( defaultCustomCreateDirMode = 0750 // default directory permissions rwxr-x--- defaultCustomDecompressFileMode = 0640 // default decompression permissions rw-r----- defaultDenySymlinkExtraction = false // allow symlink extraction + defaultDropFileAttributes = false // drop file attributes from archive defaultExtractionType = "" // do not limit extraction type defaultMaxFiles = 100000 // 100k files defaultMaxExtractionSize = 1 << (10 * 3) // 1 Gb defaultMaxInputSize = 1 << (10 * 3) // 1 Gb defaultNoUntarAfterDecompression = false // untar after decompression defaultOverwrite = false // do not overwrite existing files - defaultNoPreserveFileAttributes = false // dont preserve file attributes from archive (inverse wording) defaultPreserveOwner = false // do not preserve owner defaultTraverseSymlinks = false // do not traverse symlinks @@ -277,6 +277,7 @@ func NewConfig(opts ...ConfigOption) *Config { customCreateDirMode: defaultCustomCreateDirMode, customDecompressFileMode: defaultCustomDecompressFileMode, denySymlinkExtraction: defaultDenySymlinkExtraction, + dropFileAttributes: defaultDropFileAttributes, extractionType: defaultExtractionType, logger: defaultLogger, maxFiles: defaultMaxFiles, @@ -285,7 +286,6 @@ func NewConfig(opts ...ConfigOption) *Config { overwrite: defaultOverwrite, telemetryHook: defaultTelemetryHook, traverseSymlinks: defaultTraverseSymlinks, - noPreserveFileAttributes: defaultNoPreserveFileAttributes, noUntarAfterDecompression: defaultNoUntarAfterDecompression, preserveOwner: defaultPreserveOwner, } @@ -358,6 +358,14 @@ func WithDenySymlinkExtraction(deny bool) ConfigOption { } } +// WithDropFileAttributes options pattern function to drop the +// file attributes of the extracted files. +func WithDropFileAttributes(drop bool) ConfigOption { + return func(c *Config) { + c.dropFileAttributes = drop + } +} + // WithExtractType options pattern function to set the extraction type in the [Config]. func WithExtractType(extractionType string) ConfigOption { return func(c *Config) { @@ -427,14 +435,6 @@ func WithPatterns(pattern ...string) ConfigOption { } } -// WithNoPreserveFileAttributes options pattern function to not preserve the -// file attributes of the extracted files. -func WithNoPreserveFileAttributes(noPreserve bool) ConfigOption { - return func(c *Config) { - c.noPreserveFileAttributes = noPreserve - } -} - // WithPreserveOwner options pattern function to preserve the owner of // the extracted files. This option is only available on Unix systems // requiring root privileges and tar archives as input. diff --git a/extractor.go b/extractor.go index d033abb6..ca3bc2f1 100644 --- a/extractor.go +++ b/extractor.go @@ -271,7 +271,7 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * var extractionSize int64 // collect extracted entries if file attributes should be preserved - collectEntries := (!cfg.NoPreserveFileAttributes()) || cfg.PreserveOwner() + collectEntries := (!cfg.DropFileAttributes()) || cfg.PreserveOwner() var extractedEntries []archiveEntry if cfg.PreserveOwner() && src.Type() != fileExtensionTar { @@ -450,7 +450,7 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * if collectEntries { for _, ae := range extractedEntries { path := filepath.Join(dst, ae.Name()) - if err := setFileAttributesAndOwner(t, path, ae, (!cfg.NoPreserveFileAttributes()), cfg.PreserveOwner()); err != nil { + if err := setFileAttributesAndOwner(t, path, ae, cfg.DropFileAttributes(), cfg.PreserveOwner()); err != nil { return fmt.Errorf("failed to set file attributes: %w", err) } } @@ -461,8 +461,8 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * } // setFileAttributesAndOwner sets the file attributes for the given path and archive entry. -func setFileAttributesAndOwner(t Target, path string, ae archiveEntry, fileAttributes bool, owner bool) error { - if fileAttributes { +func setFileAttributesAndOwner(t Target, path string, ae archiveEntry, dropFileAttributes bool, owner bool) error { + if !dropFileAttributes { // preserve file attributes if ae.IsSymlink() { // only time attributes are supported for symlinks if err := t.Lchtimes(path, ae.AccessTime(), ae.ModTime()); err != nil { return fmt.Errorf("failed to lchtimes symlink: %w", err) @@ -476,7 +476,7 @@ func setFileAttributesAndOwner(t Target, path string, ae archiveEntry, fileAttri } } } - if owner { + if owner { // preserve owner and group if err := t.Chown(path, ae.Uid(), ae.Gid()); err != nil { return fmt.Errorf("failed to chown file: %w", err) } diff --git a/target_memory_test.go b/target_memory_test.go index 338b267b..9761e812 100644 --- a/target_memory_test.go +++ b/target_memory_test.go @@ -853,7 +853,7 @@ func TestUnpackToMemoryWithPreserveFileAttributesAndOwner(t *testing.T) { m = extract.NewTargetMemory() src = asIoReader(t, tc.packer(t, tc.contents)) cfg = extract.NewConfig( - extract.WithNoPreserveFileAttributes(false), + extract.WithDropFileAttributes(false), extract.WithPreserveOwner(true), ) ) diff --git a/unpack_unix_test.go b/unpack_unix_test.go index aa1efc74..e7cb2eae 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -24,7 +24,7 @@ func TestUnpackWithPreserveFileAttributes(t *testing.T) { ctx = context.Background() dst = t.TempDir() src = asIoReader(t, tc.packer(t, tc.contents)) - cfg = extract.NewConfig(extract.WithNoPreserveFileAttributes(false)) + cfg = extract.NewConfig() ) if err := extract.Unpack(ctx, dst, src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) @@ -219,7 +219,7 @@ func TestWithCustomMode(t *testing.T) { Mode: fs.ModeDir | 0777, // 511 }, })), - cfg: extract.NewConfig(extract.WithNoPreserveFileAttributes(true)), + cfg: extract.NewConfig(extract.WithDropFileAttributes(true)), expected: map[string]fs.FileMode{ "file": fs.FileMode(0777 & ^umask), // 438 "dir": fs.FileMode(0777 & ^umask), // 438 From 668a07eb97f1d063bb3ce8cabce5151cc2f9fe6b Mon Sep 17 00:00:00 2001 From: Jan Harrie Date: Wed, 18 Dec 2024 10:11:34 +0100 Subject: [PATCH 38/48] Update target_disk_unix.go Co-authored-by: Sam Salisbury --- target_disk_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_disk_unix.go b/target_disk_unix.go index 5d98ad64..57eb8acb 100644 --- a/target_disk_unix.go +++ b/target_disk_unix.go @@ -16,7 +16,7 @@ import ( // Chown changes the numeric uid and gid of the named file. func (d *TargetDisk) Chown(name string, uid, gid int) error { if err := os.Lchown(name, uid, gid); err != nil { - return &fs.PathError{Op: "chown", Path: name, Err: err} + return fmt.Errorf("chown failed: %w", err) } return nil } From 49171ebff11e4ef18036231ba38980f0da87f1a4 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 18 Dec 2024 10:13:28 +0100 Subject: [PATCH 39/48] fixed import --- target_disk_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_disk_unix.go b/target_disk_unix.go index 57eb8acb..0500e6da 100644 --- a/target_disk_unix.go +++ b/target_disk_unix.go @@ -6,7 +6,7 @@ package extract import ( - "io/fs" + "fmt" "os" "time" From 9d10a84d088e2c65797984c6c1e1280f28b099c7 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 18 Dec 2024 10:21:10 +0100 Subject: [PATCH 40/48] adjusted short version of flag PreserveOwner from -o to -p , bc/ it is more intuitive --- README.md | 9 +++------ cmd/run.go | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 51da3eec..6aea05f8 100644 --- a/README.md +++ b/README.md @@ -65,22 +65,19 @@ Flags: -C, --continue-on-error Continue extraction on error. -S, --continue-on-unsupported-files Skip extraction of unsupported files. -c, --create-destination Create destination directory if it does not exist. - --custom-create-dir-mode=750 File mode for created directories, which are not listed in the archive. - (respecting umask) + --custom-create-dir-mode=750 File mode for created directories, which are not listed in the archive. (respecting umask) --custom-decompress-file-mode=640 File mode for decompressed files. (respecting umask) -D, --deny-symlinks Deny symlink extraction. -d, --drop-file-attributes Drop file attributes (mode, modtime, access time). --insecure-traverse-symlinks Traverse symlinks to directories during extraction. - --max-files=100000 Maximum files (including folder and symlinks) that are extracted before stop. - (disable check: -1) + --max-files=100000 Maximum files (including folder and symlinks) that are extracted before stop. (disable check: -1) --max-extraction-size=1073741824 Maximum extraction size that allowed is (in bytes). (disable check: -1) --max-extraction-time=60 Maximum time that an extraction should take (in seconds). (disable check: -1) --max-input-size=1073741824 Maximum input size that allowed is (in bytes). (disable check: -1) -N, --no-untar-after-decompression Disable combined extraction of tar.gz. -O, --overwrite Overwrite if exist. -P, --pattern=PATTERN,... Extracted objects need to match shell file name pattern. - -o, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix - systems for tar files). + -p, --preserve-owner Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files). -T, --telemetry Print telemetry data to log after extraction. -t, --type="" Type of archive. (7z, br, bz2, gz, lz4, rar, sz, tar, tgz, xz, zip, zst, zz) -v, --verbose Verbose logging. diff --git a/cmd/run.go b/cmd/run.go index dfd4762f..88a3968d 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -39,7 +39,7 @@ type CLI struct { NoUntarAfterDecompression bool `short:"N" optional:"" default:"false" help:"Disable combined extraction of tar.gz."` Overwrite bool `short:"O" help:"Overwrite if exist."` Pattern []string `optional:"" short:"P" name:"pattern" help:"Extracted objects need to match shell file name pattern."` - PreserveOwner bool `short:"o" help:"Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files)."` + PreserveOwner bool `short:"p" help:"Preserve owner and group of files from archive (only root/uid:0 on unix systems for tar files)."` Telemetry bool `short:"T" optional:"" default:"false" help:"Print telemetry data to log after extraction."` Type string `short:"t" optional:"" default:"${default_type}" name:"type" help:"Type of archive. (${valid_types})"` Verbose bool `short:"v" optional:"" help:"Verbose logging."` From 5580e05a86899ffd7c3876c5a94a8572bb2d4a87 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 18 Dec 2024 10:48:45 +0100 Subject: [PATCH 41/48] adjusted comments --- config.go | 10 +++++----- extractor.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config.go b/config.go index d0a4fb87..2ff0456b 100644 --- a/config.go +++ b/config.go @@ -239,19 +239,19 @@ const ( defaultCacheInMemory = false // cache on disk defaultContinueOnError = false // stop on error and return error defaultContinueOnUnsupportedFiles = false // stop on unsupported files and return error - defaultCreateDestination = false // do not create destination directory + defaultCreateDestination = false // don't create destination directory defaultCustomCreateDirMode = 0750 // default directory permissions rwxr-x--- defaultCustomDecompressFileMode = 0640 // default decompression permissions rw-r----- defaultDenySymlinkExtraction = false // allow symlink extraction defaultDropFileAttributes = false // drop file attributes from archive - defaultExtractionType = "" // do not limit extraction type + defaultExtractionType = "" // don't limit extraction type defaultMaxFiles = 100000 // 100k files defaultMaxExtractionSize = 1 << (10 * 3) // 1 Gb defaultMaxInputSize = 1 << (10 * 3) // 1 Gb defaultNoUntarAfterDecompression = false // untar after decompression - defaultOverwrite = false // do not overwrite existing files - defaultPreserveOwner = false // do not preserve owner - defaultTraverseSymlinks = false // do not traverse symlinks + defaultOverwrite = false // don't overwrite existing files + defaultPreserveOwner = false // don't preserve owner + defaultTraverseSymlinks = false // don't traverse symlinks ) diff --git a/extractor.go b/extractor.go index ca3bc2f1..021a4925 100644 --- a/extractor.go +++ b/extractor.go @@ -357,7 +357,7 @@ func extract(ctx context.Context, t Target, dst string, src archiveWalker, cfg * return handleError(cfg, td, "max extraction size exceeded", err) } - // open file inm archive + // open file in archive err, fileCreated := func() (error, bool) { fin, err := ae.Open() if err != nil { From bf3804a37808102f734f93e2cac76f5803b9ea31 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Mon, 6 Jan 2025 13:03:18 +0100 Subject: [PATCH 42/48] removed unused field from test --- unpack_other_test.go | 16 ++++++---------- unpack_unix_test.go | 16 ++++++---------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/unpack_other_test.go b/unpack_other_test.go index b2252a97..90dcb96e 100644 --- a/unpack_other_test.go +++ b/unpack_other_test.go @@ -63,12 +63,11 @@ func TestWithCustomMode(t *testing.T) { } tests := []struct { - name string - data []byte - dst string - cfg *extract.Config - expected map[string]fs.FileMode - expectError bool + name string + data []byte + dst string + cfg *extract.Config + expected map[string]fs.FileMode }{ { name: "dir with 0755 and file with 0644", @@ -160,10 +159,7 @@ func TestWithCustomMode(t *testing.T) { cfg = test.cfg ) err := extract.Unpack(ctx, dst, src, cfg) - if test.expectError && err == nil { - t.Fatalf("expected error, got nil") - } - if !test.expectError && err != nil { + if err != nil { t.Fatalf("unexpected error: %v", err) } for name, expectedMode := range test.expected { diff --git a/unpack_unix_test.go b/unpack_unix_test.go index e7cb2eae..8c4b2b50 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -125,12 +125,11 @@ func TestWithCustomMode(t *testing.T) { // expectedMode &= ^umask // Adjust for umask on non-Windows systems tests := []struct { - name string - data []byte - dst string - cfg *extract.Config - expected map[string]fs.FileMode - expectError bool + name string + data []byte + dst string + cfg *extract.Config + expected map[string]fs.FileMode }{ { name: "dir with 0755 and file with 0644", @@ -240,10 +239,7 @@ func TestWithCustomMode(t *testing.T) { cfg = test.cfg ) err := extract.Unpack(ctx, dst, src, cfg) - if test.expectError && err == nil { - t.Fatalf("expected error, got nil") - } - if !test.expectError && err != nil { + if err != nil { t.Fatalf("unexpected error: %v", err) } for name, expectedMode := range test.expected { From f59ea351a20f99ee05fd5e3a11ed1a1224655cc3 Mon Sep 17 00:00:00 2001 From: Jan Harrie Date: Wed, 8 Jan 2025 12:55:23 +0100 Subject: [PATCH 43/48] Update unpack_unix_test.go Co-authored-by: Sam Salisbury --- unpack_unix_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/unpack_unix_test.go b/unpack_unix_test.go index 8c4b2b50..6a2ea463 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -122,7 +122,6 @@ func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { func TestWithCustomMode(t *testing.T) { umask := sniffUmask(t) - // expectedMode &= ^umask // Adjust for umask on non-Windows systems tests := []struct { name string From 6c8d47a3b219495b7b8e137027246829e677d335 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 8 Jan 2025 13:50:08 +0100 Subject: [PATCH 44/48] dropped uid/gid from upper test case level and aligned the test case --- unpack_test.go | 12 ------------ unpack_unix_test.go | 11 ++--------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index a042afbb..da13b8b6 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1611,8 +1611,6 @@ var testCases = []struct { doesNotSupportModTime bool doesNotSupportOwner bool expectError bool - uid int - gid int }{ { name: "tar", @@ -1622,8 +1620,6 @@ var testCases = []struct { {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataUid, Gid: testDataGid}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataUid, Gid: testDataGid}, }, - uid: testDataUid, - gid: testDataGid, packer: packTar, }, { @@ -1634,8 +1630,6 @@ var testCases = []struct { {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataRootUid, Gid: testDataWheelGid}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataRootUid, Gid: testDataWheelGid}, }, - uid: testDataRootUid, - gid: testDataWheelGid, packer: packTar, }, { @@ -1646,16 +1640,12 @@ var testCases = []struct { {Name: "sub/test", Content: []byte("hello world"), Mode: 0644, AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: os.Getuid(), Gid: os.Getgid()}, }, - uid: os.Getgid(), // bc/ of the way zip works, we can't set the uid and the created files are owned by the user running the test - gid: os.Getgid(), // bc/ of the way zip works, we can't set the gid and the created files are owned by the user running the test doesNotSupportOwner: true, packer: packZip, }, { name: "rar", contents: contentsRar2, - uid: os.Getuid(), // bc/ of the way rar works, we can't set the uid and the created files are owned by the user running the test - gid: os.Getgid(), // bc/ of the way rar works, we can't set the gid and the created files are owned by the user running the test doesNotSupportOwner: true, doesNotSupportModTime: true, packer: packRar2, @@ -1663,8 +1653,6 @@ var testCases = []struct { { name: "7z", contents: contents7z2, - uid: os.Getuid(), // bc/ of the way 7z works, we can't set the uid and the created files are owned by the user running the test - gid: os.Getgid(), // bc/ of the way 7z works, we can't set the gid and the created files are owned by the user running the test doesNotSupportOwner: true, packer: pack7z2, }, diff --git a/unpack_unix_test.go b/unpack_unix_test.go index 6a2ea463..e7563d1f 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -66,21 +66,14 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { src = asIoReader(t, tc.packer(t, tc.contents)) cfg = extract.NewConfig(extract.WithPreserveOwner(true)) ) - // compare archive uid/gid with current user - archiveEntriesEqualCurrentOwner := tc.doesNotSupportOwner - - // If the archive supports owner information, check if the current user is the owner of the test data - if !tc.doesNotSupportOwner && (os.Getuid() != tc.uid || os.Getgid() != tc.gid) { - archiveEntriesEqualCurrentOwner = false - } // Unpack should fail if the user is not root and the uid/gid // in the archive is different from the current user (only // if the archive supports owner information) err := extract.Unpack(ctx, dst, src, cfg) - // chown will only fail if the user is not root and the uid/gid is different - if archiveEntriesEqualCurrentOwner && err != nil { + // chown will only fail if the user is not root + if !tc.doesNotSupportOwner && err == nil { t.Fatalf("error unpacking archive: %v", err) } }) From edfd38fe5f555aba9a72ec9ab9c3fa0d62f337aa Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 8 Jan 2025 15:25:11 +0100 Subject: [PATCH 45/48] simplified chown check --- unpack_test.go | 18 +++++++++++++---- unpack_unix_test.go | 47 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index da13b8b6..16c83b98 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1599,9 +1599,10 @@ func packRar2(t *testing.T, _ []archiveContent) []byte { } var ( - testDataUid, testDataGid = 1337, 42 - testDataRootUid, testDataWheelGid = 0, 0 - baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) + testDataUid, testDataGid = 1337, 42 + testDataRootUid, testDataWheelGid = 0, 0 + testDataCurrentUid, testDataCurrentGid = os.Getuid(), os.Getgid() + baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) ) var testCases = []struct { @@ -1610,7 +1611,6 @@ var testCases = []struct { packer func(*testing.T, []archiveContent) []byte doesNotSupportModTime bool doesNotSupportOwner bool - expectError bool }{ { name: "tar", @@ -1622,6 +1622,16 @@ var testCases = []struct { }, packer: packTar, }, + { + name: "current-user-tar", + contents: []archiveContent{ + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, + }, + packer: packTar, + }, { name: "root-tar", contents: []archiveContent{ diff --git a/unpack_unix_test.go b/unpack_unix_test.go index e7563d1f..fdf319a4 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -60,11 +60,20 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + + // skip test if the archive does not store ownership information + if tc.doesNotSupportOwner { + t.Skipf("archive %s does not store ownership information", tc.name) + } + var ( ctx = context.Background() dst = t.TempDir() src = asIoReader(t, tc.packer(t, tc.contents)) cfg = extract.NewConfig(extract.WithPreserveOwner(true)) + + // check if the archive contains files with different ownership + expectChown = containsDifferentOwnerThenCurrentUser(t, tc.contents) ) // Unpack should fail if the user is not root and the uid/gid @@ -72,13 +81,35 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { // if the archive supports owner information) err := extract.Unpack(ctx, dst, src, cfg) - // chown will only fail if the user is not root - if !tc.doesNotSupportOwner && err == nil { + // unpack should succeed if the archive does not store ownership + if !expectChown { + if err != nil { + t.Fatalf("error unpacking archive: %v", err) + } + return + } + + // chown will only fail if the user is not root and the + // uid/gid in the archive is different from the current user + if err == nil { t.Fatalf("error unpacking archive: %v", err) } }) } } + +// containsDifferentOwnerThenCurrentUser returns true if the ownership of the files in the +// archive is different from the current user. +func containsDifferentOwnerThenCurrentUser(t *testing.T, a []archiveContent) bool { + t.Helper() + for _, c := range a { + if c.Uid != os.Getuid() || c.Gid != os.Getgid() { + return true + } + } + return false +} + func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { if os.Getuid() != 0 { @@ -87,18 +118,24 @@ func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + + // skip test if the archive does not store ownership information + if tc.doesNotSupportOwner { + t.Skipf("archive type %s does not store ownership information", tc.name) + } + var ( ctx = context.Background() dst = t.TempDir() src = asIoReader(t, tc.packer(t, tc.contents)) cfg = extract.NewConfig(extract.WithPreserveOwner(true)) ) + if err := extract.Unpack(ctx, dst, src, cfg); err != nil { t.Fatalf("error unpacking archive: %v", err) } - if tc.doesNotSupportOwner { - t.Skipf("archive type %s does not store ownership information", tc.name) - } + + // check ownership of files for _, c := range tc.contents { path := filepath.Join(dst, c.Name) stat, err := os.Lstat(path) From 064184a0fdfff8a0f2efbce8d3378092fc0df042 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 8 Jan 2025 15:36:49 +0100 Subject: [PATCH 46/48] adjusted the current-user-tar test-case to invalid-uid-tar so that every test-case need a chown --- unpack_test.go | 12 ++++++------ unpack_unix_test.go | 23 ----------------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index 16c83b98..34d4fc9d 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1601,7 +1601,7 @@ func packRar2(t *testing.T, _ []archiveContent) []byte { var ( testDataUid, testDataGid = 1337, 42 testDataRootUid, testDataWheelGid = 0, 0 - testDataCurrentUid, testDataCurrentGid = os.Getuid(), os.Getgid() + testDataInvalidUid, testDataInvalidGid = -1, -2 baseTime = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) ) @@ -1623,12 +1623,12 @@ var testCases = []struct { packer: packTar, }, { - name: "current-user-tar", + name: "invalid-uid-tar", contents: []archiveContent{ - {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, - {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, - {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, - {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataCurrentUid, Gid: testDataCurrentGid}, + {Name: "test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataInvalidUid, Gid: testDataInvalidGid}, + {Name: "sub", Mode: fs.ModeDir | 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataInvalidUid, Gid: testDataInvalidGid}, + {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataInvalidUid, Gid: testDataInvalidGid}, + {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataInvalidUid, Gid: testDataInvalidGid}, }, packer: packTar, }, diff --git a/unpack_unix_test.go b/unpack_unix_test.go index fdf319a4..5cd2cf0b 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -71,9 +71,6 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { dst = t.TempDir() src = asIoReader(t, tc.packer(t, tc.contents)) cfg = extract.NewConfig(extract.WithPreserveOwner(true)) - - // check if the archive contains files with different ownership - expectChown = containsDifferentOwnerThenCurrentUser(t, tc.contents) ) // Unpack should fail if the user is not root and the uid/gid @@ -81,14 +78,6 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { // if the archive supports owner information) err := extract.Unpack(ctx, dst, src, cfg) - // unpack should succeed if the archive does not store ownership - if !expectChown { - if err != nil { - t.Fatalf("error unpacking archive: %v", err) - } - return - } - // chown will only fail if the user is not root and the // uid/gid in the archive is different from the current user if err == nil { @@ -98,18 +87,6 @@ func TestUnpackWithPreserveOwnershipAsNonRoot(t *testing.T) { } } -// containsDifferentOwnerThenCurrentUser returns true if the ownership of the files in the -// archive is different from the current user. -func containsDifferentOwnerThenCurrentUser(t *testing.T, a []archiveContent) bool { - t.Helper() - for _, c := range a { - if c.Uid != os.Getuid() || c.Gid != os.Getgid() { - return true - } - } - return false -} - func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { if os.Getuid() != 0 { From 45cb8517d148eba2352c35b85656d09cba5e6f71 Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 8 Jan 2025 16:14:10 +0100 Subject: [PATCH 47/48] adjusted test for invalid uid/gid --- unpack_test.go | 4 +++- unpack_unix_test.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/unpack_test.go b/unpack_test.go index 34d4fc9d..42e9780f 100644 --- a/unpack_test.go +++ b/unpack_test.go @@ -1611,6 +1611,7 @@ var testCases = []struct { packer func(*testing.T, []archiveContent) []byte doesNotSupportModTime bool doesNotSupportOwner bool + invalidUidGid bool }{ { name: "tar", @@ -1630,7 +1631,8 @@ var testCases = []struct { {Name: "sub/test", Content: []byte("hello world"), Mode: 0777, AccessTime: baseTime, ModTime: baseTime, Uid: testDataInvalidUid, Gid: testDataInvalidGid}, {Name: "link", Mode: fs.ModeSymlink | 0777, Linktarget: "sub/test", AccessTime: baseTime, ModTime: baseTime, Uid: testDataInvalidUid, Gid: testDataInvalidGid}, }, - packer: packTar, + packer: packTar, + invalidUidGid: true, }, { name: "root-tar", diff --git a/unpack_unix_test.go b/unpack_unix_test.go index 5cd2cf0b..69b2f589 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -113,13 +113,15 @@ func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { } // check ownership of files + expectUidMatch := !tc.invalidUidGid for _, c := range tc.contents { path := filepath.Join(dst, c.Name) stat, err := os.Lstat(path) if err != nil { t.Fatalf("error getting file stats: %v", err) } - if stat.Sys().(*syscall.Stat_t).Uid != uint32(c.Uid) { + uidMatch := stat.Sys().(*syscall.Stat_t).Uid != uint32(c.Uid) + if expectUidMatch != uidMatch { t.Fatalf("expected uid %d, got %d, file %s", c.Uid, stat.Sys().(*syscall.Stat_t).Uid, c.Name) } } From 5c74796d2fc87c09df98a4fa1bd9bc2227b8ffbf Mon Sep 17 00:00:00 2001 From: NodyHub Date: Wed, 8 Jan 2025 16:19:17 +0100 Subject: [PATCH 48/48] adjusted uid casting --- unpack_unix_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unpack_unix_test.go b/unpack_unix_test.go index 69b2f589..5c45707b 100644 --- a/unpack_unix_test.go +++ b/unpack_unix_test.go @@ -120,7 +120,7 @@ func TestUnpackWithPreserveOwnershipAsRoot(t *testing.T) { if err != nil { t.Fatalf("error getting file stats: %v", err) } - uidMatch := stat.Sys().(*syscall.Stat_t).Uid != uint32(c.Uid) + uidMatch := c.Uid == int(stat.Sys().(*syscall.Stat_t).Uid) if expectUidMatch != uidMatch { t.Fatalf("expected uid %d, got %d, file %s", c.Uid, stat.Sys().(*syscall.Stat_t).Uid, c.Name) }