add tests

Signed-off-by: Alexandr Stefurishin <[email protected]>

Author: Alexandr Stefurishin

go.mod

@@ -96,6 +96,7 @@ require (
 	go.uber.org/zap v1.19.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.22.0
 	golang.org/x/oauth2 v0.22.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect

go.sum

@@ -451,6 +451,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
+golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/nfs/controllerserver.go

@@ -401,7 +401,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
 	dstPath := filepath.Join(snapInternalVolPath, snapshot.archiveName())
 
 	klog.V(2).Infof("tar %v -> %v", srcPath, dstPath)
-	err = tarPack(dstPath, srcPath, true)
+	err = TarPack(srcPath, dstPath, true)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to create archive for snapshot: %v", err)
 	}
@@ -557,7 +557,7 @@ func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.Creat
 	dstPath := getInternalVolumePath(cs.Driver.workingMountDir, dstVol)
 	klog.V(2).Infof("copy volume from snapshot %v -> %v", snapPath, dstPath)
 
-	err = tarUnpack(snapPath, dstPath, true)
+	err = TarUnpack(snapPath, dstPath, true)
 	if err != nil {
 		return status.Errorf(codes.Internal, "failed to copy volume for snapshot: %v", err)
 	}

pkg/nfs/tar.go

@@ -8,32 +8,33 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 )
 
-func tarPack(dstFilePath, srcPath string, enableCompression bool) error {
+func TarPack(srcDirPath string, dstPath string, enableCompression bool) error {
 	// normalize all paths to be absolute and clean
-	dstFilePath, err := filepath.Abs(dstFilePath)
+	dstPath, err := filepath.Abs(dstPath)
 	if err != nil {
 		return fmt.Errorf("normalizing destination path: %w", err)
 	}
 
-	srcPath, err = filepath.Abs(srcPath)
+	srcDirPath, err = filepath.Abs(srcDirPath)
 	if err != nil {
 		return fmt.Errorf("normalizing source path: %w", err)
 	}
 
-	if strings.Index(dstFilePath, srcPath) == 0 {
-		return fmt.Errorf("destination file %s cannot be under source directory %s", dstFilePath, srcPath)
+	if strings.Index(path.Dir(dstPath), srcDirPath) == 0 {
+		return fmt.Errorf("destination file %s cannot be under source directory %s", dstPath, srcDirPath)
 	}
 
-	tarFile, err := os.Create(dstFilePath)
+	tarFile, err := os.Create(dstPath)
 	if err != nil {
 		return fmt.Errorf("creating destination file: %w", err)
 	}
 	defer func() {
-		err = errors.Join(err, closeAndWrapErr(tarFile, "closing destination file %s: %w", dstFilePath))
+		err = errors.Join(err, closeAndWrapErr(tarFile, "closing destination file %s: %w", dstPath))
 	}()
 
 	var tarDst io.Writer = tarFile
@@ -52,11 +53,13 @@ func tarPack(dstFilePath, srcPath string, enableCompression bool) error {
 
 	// recursively visit every file and write it
 	if err = filepath.Walk(
-		srcPath,
+		srcDirPath,
 		func(srcSubPath string, fileInfo fs.FileInfo, walkErr error) error {
-			return tarVisitFileToPack(tarWriter, srcPath, srcSubPath, fileInfo, walkErr)
+			return tarVisitFileToPack(tarWriter, srcDirPath, srcSubPath, fileInfo, walkErr)
 		},
 	); err != nil {
+		ee := err.Error()
+		_ = ee
 		return fmt.Errorf("walking source directory: %w", err)
 	}
 
@@ -115,24 +118,24 @@ func tarVisitFileToPack(
 	return nil
 }
 
-func tarUnpack(archivePath, dstPath string, enableCompression bool) (err error) {
+func TarUnpack(srcPath, dstDirPath string, enableCompression bool) (err error) {
 	// normalize all paths to be absolute and clean
-	archivePath, err = filepath.Abs(archivePath)
+	srcPath, err = filepath.Abs(srcPath)
 	if err != nil {
 		return fmt.Errorf("normalizing archive path: %w", err)
 	}
 
-	dstPath, err = filepath.Abs(dstPath)
+	dstDirPath, err = filepath.Abs(dstDirPath)
 	if err != nil {
 		return fmt.Errorf("normalizing archive destination path: %w", err)
 	}
 
-	tarFile, err := os.Open(archivePath)
+	tarFile, err := os.Open(srcPath)
 	if err != nil {
-		return fmt.Errorf("opening archive %s: %w", archivePath, err)
+		return fmt.Errorf("opening archive %s: %w", srcPath, err)
 	}
 	defer func() {
-		err = errors.Join(err, closeAndWrapErr(tarFile, "closing archive %s: %w", archivePath))
+		err = errors.Join(err, closeAndWrapErr(tarFile, "closing archive %s: %w", srcPath))
 	}()
 
 	var tarDst io.Reader = tarFile
@@ -146,7 +149,7 @@ func tarUnpack(archivePath, dstPath string, enableCompression bool) (err error)
 			err = errors.Join(err, closeAndWrapErr(gzipReader, "closing gzip reader: %w"))
 		}()
 
-		tarDst = tar.NewReader(gzipReader)
+		tarDst = gzipReader
 	}
 
 	tarReader := tar.NewReader(tarDst)
@@ -158,12 +161,12 @@ func tarUnpack(archivePath, dstPath string, enableCompression bool) (err error)
 			break
 		}
 		if err != nil {
-			return fmt.Errorf("reading tar header of %s: %w", archivePath, err)
+			return fmt.Errorf("reading tar header of %s: %w", srcPath, err)
 		}
 
 		fileInfo := tarHeader.FileInfo()
 
-		filePath := filepath.Join(dstPath, tarHeader.Name)
+		filePath := filepath.Join(dstDirPath, tarHeader.Name)
 
 		fileDirPath := filePath
 		if !fileInfo.Mode().IsDir() {

test/tar/tar_test.go

@@ -0,0 +1,187 @@
+package tar
+
+import (
+	"bytes"
+	"fmt"
+	"maps"
+	"math"
+	"os"
+	"os/exec"
+	"path"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/kubernetes-csi/csi-driver-nfs/pkg/nfs"
+	"golang.org/x/mod/sumdb/dirhash"
+)
+
+const (
+	code packApi = '0'
+	cli  packApi = '1'
+)
+
+type packApi byte
+
+const archiveFileExt = ".tar.gz"
+
+func TestPackUnpack(t *testing.T) {
+	inputPath := t.TempDir()
+	generateFileSystem(t, inputPath)
+
+	outputPath := t.TempDir()
+
+	// produced file names (without extensions) have a suffix,
+	// which determine the last operation:
+	// "0" means that it was produced from code
+	// "1" means that it was produced from CLI
+	// e.g.: "testdata011.tar.gz" - was packed from code,
+	// then unpacked from cli and packed again from cli
+
+	pathsBySuffix := make(map[string]string)
+
+	// number of pack/unpack operations
+	opNum := 4
+
+	// generate all operation combinations
+	fileNum := int(math.Pow(2, float64(opNum)))
+	for i := 0; i < fileNum; i++ {
+		binStr := fmt.Sprintf("%b", i)
+
+		// left-pad with zeroes
+		binStr = strings.Repeat("0", opNum-len(binStr)) + binStr
+
+		// copy slices to satisfy type system
+		ops := make([]packApi, opNum)
+		for opIdx := 0; opIdx < opNum; opIdx++ {
+			ops[opIdx] = packApi(binStr[opIdx])
+		}
+
+		// produce folders and archives
+		produce(t, pathsBySuffix, inputPath, outputPath, ops...)
+	}
+
+	// byte-compare archives with the same last step
+	paths := slices.Collect(maps.Values(pathsBySuffix))
+	assertUnpackedFilesEqual(t, inputPath, paths)
+}
+
+func produce(
+	t *testing.T,
+	results map[string]string,
+	inputDirPath string,
+	outputDirPath string,
+	ops ...packApi,
+) {
+	baseName := path.Base(inputDirPath)
+
+	for i := 0; i < len(ops); i++ {
+		packing := i%2 == 0
+
+		srcPath := inputDirPath
+		if i > 0 {
+			prevSuffix := string(ops[:i])
+			srcPath = path.Join(outputDirPath, baseName+prevSuffix)
+			if !packing {
+				srcPath += archiveFileExt
+			}
+		}
+
+		suffix := string(ops[:i+1])
+		dstPath := path.Join(outputDirPath, baseName+suffix)
+		if packing {
+			dstPath += archiveFileExt
+		}
+
+		if _, ok := results[suffix]; ok {
+			continue
+		}
+
+		switch {
+		case packing && ops[i] == code:
+			// packing from code
+			if err := nfs.TarPack(srcPath, dstPath, true); err != nil {
+				t.Fatalf("packing '%s' with TarPack into '%s': %v", srcPath, dstPath, err)
+			}
+		case packing && ops[i] == cli:
+			// packing from CLI
+			if out, err := exec.Command("tar", "-C", srcPath, "-czvf", dstPath, ".").CombinedOutput(); err != nil {
+				t.Log("TAR OUTPUT:", string(out))
+				t.Fatalf("packing '%s' with tar into '%s': %v", srcPath, dstPath, err)
+			}
+		case !packing && ops[i] == code:
+			// unpacking from code
+			if err := nfs.TarUnpack(srcPath, dstPath, true); err != nil {
+				t.Fatalf("unpacking '%s' with TarUnpack into '%s': %v", srcPath, dstPath, err)
+			}
+		case !packing && ops[i] == cli:
+			// unpacking from CLI
+			// tar requires destination directory to exist
+			if err := os.MkdirAll(dstPath, 0755); err != nil {
+				t.Fatalf("making dir '%s' for unpacking with tar: %v", dstPath, err)
+			}
+			if out, err := exec.Command("tar", "-xzvf", srcPath, "-C", dstPath).CombinedOutput(); err != nil {
+				t.Log("TAR OUTPUT:", string(out))
+				t.Fatalf("unpacking '%s' with tar into '%s': %v", srcPath, dstPath, err)
+			}
+		default:
+			t.Fatalf("unknown suffix: %s", string(ops[i]))
+		}
+
+		results[suffix] = dstPath
+	}
+}
+
+func assertUnpackedFilesEqual(t *testing.T, originalDir string, paths []string) {
+	originalDirHash, err := dirhash.HashDir(originalDir, "_", dirhash.DefaultHash)
+	if err != nil {
+		t.Fatal("failed hashing original dir ", err)
+	}
+
+	for _, p := range paths {
+		if strings.HasSuffix(p, archiveFileExt) {
+			// archive, not a directory
+			continue
+		}
+
+		// unpacked directory
+		hs, err := dirhash.HashDir(p, "_", dirhash.DefaultHash)
+		if err != nil {
+			t.Fatal("failed hashing dir ", err)
+		}
+
+		if hs != originalDirHash {
+			t.Errorf("expected '%s' to have the same hash as '%s', got different", originalDir, p)
+		}
+	}
+}
+
+func generateFileSystem(t *testing.T, inputPath string) {
+	// empty directory
+	if err := os.MkdirAll(path.Join(inputPath, "empty_dir"), 0755); err != nil {
+		t.Fatalf("generating empty directory: %v", err)
+	}
+
+	// deep empty directories
+	deepEmptyDirPath := path.Join(inputPath, "deep_empty_dir", strings.Repeat("/0/1/2", 20))
+	if err := os.MkdirAll(deepEmptyDirPath, 0755); err != nil {
+		t.Fatalf("generating deep empty directory '%s': %v", deepEmptyDirPath, err)
+	}
+
+	// empty file
+	f, err := os.Create(path.Join(inputPath, "empty_file"))
+	if err != nil {
+		t.Fatalf("generating empty file: %v", err)
+	}
+	f.Close()
+
+	// big (100MB) file
+	bigFilePath := path.Join(inputPath, "big_file")
+	for i := byte(0); i < 100; i++ {
+		// write 1MB
+		err := os.WriteFile(bigFilePath, bytes.Repeat([]byte{i}, 1024*1024), 0755)
+		if err != nil {
+			t.Fatalf("generating empty file: %v", err)
+		}
+	}
+}