From f2b5b8864bfb8b7be0cb2f8b1b54ff148ad467dd Mon Sep 17 00:00:00 2001 From: Carlos Lapao Date: Fri, 6 Dec 2024 16:38:28 +0000 Subject: [PATCH 1/2] Fixed the sparse compatibility - Added the ability to extract sparce and symlinks --- src/catalog/main.go | 56 ++++++++++++++++++++++++++++++++++++++- src/cmd/test_providers.go | 10 +++++++ src/helpers/os.go | 10 ++++++- 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/src/catalog/main.go b/src/catalog/main.go index 566c9f7..2c9f22d 100644 --- a/src/catalog/main.go +++ b/src/catalog/main.go @@ -367,6 +367,10 @@ func (s *CatalogManifestService) detectFileType(filepath string) (string, error) return "unknown", errors.New("file format not recognized as gzip or tar") } +func (s *CatalogManifestService) Unzip(ctx basecontext.ApiContext, machineFilePath string, destination string) error { + return s.decompressMachine(ctx, machineFilePath, destination) +} + func (s *CatalogManifestService) decompressMachine(ctx basecontext.ApiContext, machineFilePath string, destination string) error { staringTime := time.Now() filePath := filepath.Clean(machineFilePath) @@ -432,21 +436,39 @@ func (s *CatalogManifestService) decompressMachine(ctx basecontext.ApiContext, m switch header.Typeflag { case tar.TypeDir: + ctx.LogDebugf("Directory type found for file %v (byte %v, rune %v)", machineFilePath, header.Typeflag, string(header.Typeflag)) if _, err := os.Stat(machineFilePath); os.IsNotExist(err) { if err := os.MkdirAll(machineFilePath, os.FileMode(header.Mode)); err != nil { return err } } case tar.TypeReg: + ctx.LogDebugf("HardFile type found for file %v (byte %v, rune %v): size %v", machineFilePath, header.Typeflag, string(header.Typeflag), header.Size) file, err := os.OpenFile(filepath.Clean(machineFilePath), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(header.Mode)) if err != nil { return err } defer file.Close() - if err := helpers.CopyTarChunks(file, tarReader); err != nil { + if err := helpers.CopyTarChunks(file, tarReader, header.Size); err != nil { return err } + case tar.TypeGNUSparse: + ctx.LogDebugf("Sparse File type found for file %v (byte %v, rune %v): size %v", machineFilePath, header.Typeflag, string(header.Typeflag), header.Size) + file, err := os.OpenFile(filepath.Clean(machineFilePath), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(header.Mode)) + if err != nil { + return err + } + defer file.Close() + + if err := helpers.CopyTarChunks(file, tarReader, header.Size); err != nil { + return err + } + case tar.TypeSymlink: + ctx.LogDebugf("Symlink File type found for file %v (byte %v, rune %v)", machineFilePath, header.Typeflag, string(header.Typeflag)) + os.Symlink(header.Linkname, machineFilePath) + default: + ctx.LogWarnf("Unknown type found for file %v, ignoring (byte %v, rune %v)", machineFilePath, header.Typeflag, string(header.Typeflag)) } } @@ -455,6 +477,38 @@ func (s *CatalogManifestService) decompressMachine(ctx basecontext.ApiContext, m return nil } +func handleSparseFile(header *tar.Header, tarReader *tar.Reader, destDir string) error { + outFile, err := os.Create(destDir + "/" + header.Name) + if err != nil { + return fmt.Errorf("error creating file: %v", err) + } + defer outFile.Close() + + fmt.Printf("Writing sparse file: %s (%d bytes)\n", header.Name, header.Size) + + // Copy exactly the size of the sparse file + if _, err := io.CopyN(outFile, tarReader, header.Size); err != nil && err != io.EOF { + return fmt.Errorf("error writing sparse file content: %v", err) + } + return nil +} + +func handleRegularFile(header *tar.Header, tarReader *tar.Reader, destDir string) error { + outFile, err := os.Create(destDir + "/" + header.Name) + if err != nil { + return fmt.Errorf("error creating file: %v", err) + } + defer outFile.Close() + + fmt.Printf("Writing regular file: %s (%d bytes)\n", header.Name, header.Size) + + // Copy exactly the size of the regular file + if _, err := io.CopyN(outFile, tarReader, header.Size); err != nil && err != io.EOF { + return fmt.Errorf("error writing file content: %v", err) + } + return nil +} + func (s *CatalogManifestService) sendPullStepInfo(r *models.PullCatalogManifestRequest, msg string) { if r.StepChannel != nil { r.StepChannel <- msg diff --git a/src/cmd/test_providers.go b/src/cmd/test_providers.go index f6ac835..caa1ead 100644 --- a/src/cmd/test_providers.go +++ b/src/cmd/test_providers.go @@ -5,6 +5,7 @@ import ( "os" "github.com/Parallels/prl-devops-service/basecontext" + "github.com/Parallels/prl-devops-service/catalog" "github.com/Parallels/prl-devops-service/constants" "github.com/Parallels/prl-devops-service/tests" "github.com/cjlapao/common-go/helper" @@ -20,6 +21,15 @@ func processTestProviders(ctx basecontext.ApiContext, cmd string) { ctx.LogErrorf(err.Error()) os.Exit(1) } + case "unzip": + rootctx := basecontext.NewBaseContext() + filename := helper.GetFlagValue("zip-file", "") + destination := helper.GetFlagValue("destination", "") + if destination == "" { + destination = "/tmp" + } + cSrv := catalog.NewManifestService(rootctx) + cSrv.Unzip(rootctx, filename, destination) default: processHelp(constants.TEST_COMMAND) diff --git a/src/helpers/os.go b/src/helpers/os.go index 9bd4177..509e524 100644 --- a/src/helpers/os.go +++ b/src/helpers/os.go @@ -307,7 +307,9 @@ func GetFileMD5Checksum(path string) (string, error) { return checksum, nil } -func CopyTarChunks(file *os.File, reader *tar.Reader) error { +func CopyTarChunks(file *os.File, reader *tar.Reader, fileSize int64) error { + // extractedSize := int64(0) + // lastPrintTime := time.Now() for { _, err := io.CopyN(file, reader, 1024) if err != nil { @@ -316,6 +318,12 @@ func CopyTarChunks(file *os.File, reader *tar.Reader) error { } return err } + // extractedSize += 1024 + // percentage := float64(extractedSize) / float64(fileSize) * 100 + // if time.Since(lastPrintTime) >= 10*time.Second { + // fmt.Printf("\rExtracted: %.2f%%", percentage) + // lastPrintTime = time.Now() + // } } return nil From 015d60210f972c1d3172386cbe97680549c7214f Mon Sep 17 00:00:00 2001 From: Carlos Lapao Date: Fri, 6 Dec 2024 16:49:20 +0000 Subject: [PATCH 2/2] fix security issue --- src/catalog/main.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/catalog/main.go b/src/catalog/main.go index 2c9f22d..dc8885c 100644 --- a/src/catalog/main.go +++ b/src/catalog/main.go @@ -467,6 +467,19 @@ func (s *CatalogManifestService) decompressMachine(ctx basecontext.ApiContext, m case tar.TypeSymlink: ctx.LogDebugf("Symlink File type found for file %v (byte %v, rune %v)", machineFilePath, header.Typeflag, string(header.Typeflag)) os.Symlink(header.Linkname, machineFilePath) + realLinkPath, err := filepath.EvalSymlinks(filepath.Join(destination, header.Linkname)) + if err != nil { + ctx.LogWarnf("Error resolving symlink path: %v", header.Linkname) + if err := os.Remove(machineFilePath); err != nil { + return fmt.Errorf("failed to remove invalid symlink: %v", err) + } + } else { + relLinkPath, err := filepath.Rel(destination, realLinkPath) + if err != nil || strings.HasPrefix(filepath.Clean(relLinkPath), "..") { + return fmt.Errorf("invalid symlink path: %v", header.Linkname) + } + os.Symlink(realLinkPath, machineFilePath) + } default: ctx.LogWarnf("Unknown type found for file %v, ignoring (byte %v, rune %v)", machineFilePath, header.Typeflag, string(header.Typeflag)) }