From 73bc5a420b46fd8525727400c2ac9257a245bf49 Mon Sep 17 00:00:00 2001 From: Alex Angelini Date: Fri, 28 Oct 2022 15:18:47 -0400 Subject: [PATCH] Remove dir fuzz error --- cmd/fuzz-test/main.go | 146 ++++++++++++++++++++++++----------- test/client_combined_test.go | 64 +++++++++++++++ test/shared_test.go | 10 +++ 3 files changed, 173 insertions(+), 47 deletions(-) diff --git a/cmd/fuzz-test/main.go b/cmd/fuzz-test/main.go index 0f26283..4dceef4 100644 --- a/cmd/fuzz-test/main.go +++ b/cmd/fuzz-test/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "io/fs" stdlog "log" @@ -180,7 +179,7 @@ func (o AddFileOp) Apply() error { } func (o AddFileOp) String() string { - return fmt.Sprintf("AddFileOp(%s, %d)", Join(o.dir, o.name), len(o.content)) + return fmt.Sprintf("AddFile(%s, %d)", Join(o.dir, o.name), len(o.content)) } type UpdateFileOp struct { @@ -198,7 +197,7 @@ func newUpdateFileOp(base string, project int64) Operation { return SkipOp{} } - return AddFileOp{ + return UpdateFileOp{ base: base, dir: dir, name: files[rand.Intn(len(files))], @@ -211,7 +210,7 @@ func (o UpdateFileOp) Apply() error { } func (o UpdateFileOp) String() string { - return fmt.Sprintf("UpdateFileOp(%s, %d)", Join(o.dir, o.name), len(o.content)) + return fmt.Sprintf("UpdateFile(%s, %d)", Join(o.dir, o.name), len(o.content)) } type AddDirOp struct { @@ -277,7 +276,36 @@ func (o RemoveFileOp) Apply() error { } func (o RemoveFileOp) String() string { - return fmt.Sprintf("RemoveFileOp(%s)", Join(o.dir, o.name)) + return fmt.Sprintf("RemoveFile(%s)", Join(o.dir, o.name)) +} + +type RemoveDirOp struct { + base string + dir string + name string +} + +func newRemoveDirOp(base string, project int64) Operation { + dir := fmt.Sprint(project) + dirs := objectFilter(walkDir(Join(base, dir)), typeDirectory) + + if len(dirs) == 0 { + return SkipOp{} + } + + return RemoveDirOp{ + base: base, + dir: dir, + name: dirs[rand.Intn(len(dirs))], + } +} + +func (o RemoveDirOp) Apply() error { + return os.RemoveAll(Join(o.base, o.dir, o.name)) +} + +func (o RemoveDirOp) String() string { + return fmt.Sprintf("RemoveDir(%s)", Join(o.dir, o.name)) } type AddSymlinkOp struct { @@ -320,12 +348,12 @@ func (o AddSymlinkOp) Apply() error { } func (o AddSymlinkOp) String() string { - return fmt.Sprintf("AddSymlinkOp(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name)) + return fmt.Sprintf("AddSymlink(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name)) } type OpConstructor func(dir string, project int64) Operation -var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newAddSymlinkOp} +var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newRemoveDirOp, newAddSymlinkOp} func randomOperation(baseDir string, project int64) Operation { var operation Operation = SkipOp{} @@ -340,71 +368,97 @@ func randomOperation(baseDir string, project int64) Operation { return operation } -func createDirs(projects int) (string, string, string, string, error) { +type Directories struct { + base string + reset string + previous string + step string +} + +func createDirectories(projects int) (*Directories, error) { var dirs []string - for _, name := range []string{"base", "continue", "reset", "step"} { + for _, name := range []string{"base", "reset", "previous", "step"} { dir, err := os.MkdirTemp("", fmt.Sprintf("dl-ft-%s-", name)) if err != nil { - return "", "", "", "", fmt.Errorf("cannot create tmp dir: %w", err) + return nil, fmt.Errorf("cannot create tmp dir: %w", err) } for projectIdx := 1; projectIdx <= projects; projectIdx++ { err = os.MkdirAll(filepath.Join(dir, fmt.Sprint(projectIdx)), 0755) if err != nil { - return "", "", "", "", fmt.Errorf("cannot create project dir: %w", err) + return nil, fmt.Errorf("cannot create project dir: %w", err) } } dirs = append(dirs, dir) } - return dirs[0], dirs[1], dirs[2], dirs[3], nil + return &Directories{ + base: dirs[0], + reset: dirs[1], + previous: dirs[2], + step: dirs[3], + }, nil +} + +func (d *Directories) Log(ctx context.Context) { + logger.Info(ctx, "base", zap.String("path", d.base)) + logger.Info(ctx, "reset", zap.String("path", d.reset)) + logger.Info(ctx, "prev", zap.String("path", d.previous)) + logger.Info(ctx, "step", zap.String("path", d.step)) } -func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, baseDir, continueDir, resetDir, stepDir string) error { +func (d *Directories) RemoveAll() { + os.RemoveAll(d.base) + os.RemoveAll(d.reset) + os.RemoveAll(d.previous) + os.RemoveAll(d.step) +} + +func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, dirs *Directories) (int64, error) { err := operation.Apply() if err != nil { - return fmt.Errorf("failed to apply operation %s: %w", operation.String(), err) + return -1, fmt.Errorf("failed to apply operation %s: %w", operation.String(), err) } - version, _, err := client.Update(ctx, project, projectDir(baseDir, project)) + version, _, err := client.Update(ctx, project, projectDir(dirs.base, project)) if err != nil { - return fmt.Errorf("failed to update project %d: %w", project, err) + return -1, fmt.Errorf("failed to update project %d: %w", project, err) } - _, _, err = client.Rebuild(ctx, project, "", nil, projectDir(continueDir, project), "") + os.RemoveAll(projectDir(dirs.reset, project)) + err = os.MkdirAll(projectDir(dirs.reset, project), 0755) if err != nil { - return fmt.Errorf("failed to rebuild continue project %d: %w", project, err) + return -1, fmt.Errorf("failed to create reset dir %s: %w", projectDir(dirs.reset, project), err) } - os.RemoveAll(projectDir(resetDir, project)) - err = os.MkdirAll(projectDir(resetDir, project), 0755) + _, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.reset, project), "") if err != nil { - return fmt.Errorf("failed to create reset dir %s: %w", projectDir(resetDir, project), err) + return -1, fmt.Errorf("failed to rebuild from reset, project %d: %w", project, err) } - _, _, err = client.Rebuild(ctx, project, "", nil, projectDir(resetDir, project), "") + _, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.previous, project), "") if err != nil { - return fmt.Errorf("failed to rebuild reset project %d: %w", project, err) + return -1, fmt.Errorf("failed to rebuild from previous version, project %d: %w", project, err) } - os.RemoveAll(projectDir(stepDir, project)) - err = os.MkdirAll(projectDir(stepDir, project), 0755) + os.RemoveAll(projectDir(dirs.step, project)) + err = os.MkdirAll(projectDir(dirs.step, project), 0755) if err != nil { - return fmt.Errorf("failed to create step dir %s: %w", projectDir(stepDir, project), err) + return -1, fmt.Errorf("failed to create step dir %s: %w", projectDir(dirs.step, project), err) } stepVersion := int64(rand.Intn(int(version))) - _, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(stepDir, project), "") + _, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(dirs.step, project), "") if err != nil { - return fmt.Errorf("failed to rebuild step project %d: %w", project, err) + return -1, fmt.Errorf("failed to rebuild step to version %v, project %d: %w", stepVersion, project, err) } - _, _, err = client.Rebuild(ctx, project, "", &version, projectDir(stepDir, project), "") + _, _, err = client.Rebuild(ctx, project, "", &version, projectDir(dirs.step, project), "") if err != nil { - return fmt.Errorf("failed to rebuild step project %d: %w", project, err) + return -1, fmt.Errorf("failed to rebuild step from version %v, project %d: %w", stepVersion, project, err) } - return nil + return stepVersion, nil } type MatchError struct { @@ -507,35 +561,35 @@ func logOpLog(ctx context.Context, opLog []Operation) { } } -func verifyDirs(ctx context.Context, projects int, baseDir, continueDir, resetDir, stepDir string) error { +func verifyDirs(ctx context.Context, projects int, dirs *Directories, stepVersion int64) error { for projectIdx := 1; projectIdx <= projects; projectIdx++ { project := int64(projectIdx) - matchErrors, err := compareDirs(project, projectDir(baseDir, project), projectDir(resetDir, project)) + matchErrors, err := compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.reset, project)) if err != nil { return fmt.Errorf("failed to compare base & reset dirs: %w", err) } if len(matchErrors) > 0 { logMatchErrors(ctx, matchErrors) - return errors.New("reset directory match error") + return fmt.Errorf("reset directory match error, project %d", project) } - matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(continueDir, project)) + matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.previous, project)) if err != nil { - return fmt.Errorf("failed to compare base & continue dirs: %w", err) + return fmt.Errorf("failed to compare base & previous dirs: %w", err) } if len(matchErrors) > 0 { logMatchErrors(ctx, matchErrors) - return errors.New("continue directory match error") + return fmt.Errorf("previous version directory match error, project %d", project) } - matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(stepDir, project)) + matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.step, project)) if err != nil { return fmt.Errorf("failed to compare base & step dirs: %w", err) } if len(matchErrors) > 0 { logMatchErrors(ctx, matchErrors) - return errors.New("step directory match error") + return fmt.Errorf("step from verion %d directory match error, project %d", stepVersion, project) } } return nil @@ -552,29 +606,27 @@ func fuzzTest(ctx context.Context, client *dlc.Client, projects, iterations int) } } - baseDir, continueDir, resetDir, stepDir, err := createDirs(projects) + dirs, err := createDirectories(projects) if err != nil { return err } - defer os.RemoveAll(baseDir) - defer os.RemoveAll(continueDir) - defer os.RemoveAll(resetDir) - defer os.RemoveAll(stepDir) + // defer dirs.RemoveAll() + dirs.Log(ctx) var opLog []Operation for iterIdx := 0; iterIdx < iterations; iterIdx++ { project := int64(rand.Intn(projects) + 1) - operation := randomOperation(baseDir, project) + operation := randomOperation(dirs.base, project) opLog = append(opLog, operation) - err := runIteration(ctx, client, project, operation, baseDir, continueDir, resetDir, stepDir) + stepVersion, err := runIteration(ctx, client, project, operation, dirs) if err != nil { return fmt.Errorf("failed to run iteration %d: %w", iterIdx, err) } - err = verifyDirs(ctx, projects, baseDir, continueDir, resetDir, stepDir) + err = verifyDirs(ctx, projects, dirs, stepVersion) if err != nil { logOpLog(ctx, opLog) return err diff --git a/test/client_combined_test.go b/test/client_combined_test.go index 46b8e07..9347a88 100644 --- a/test/client_combined_test.go +++ b/test/client_combined_test.go @@ -621,3 +621,67 @@ func TestCombinedWithPrefixDirectoryBug(t *testing.T) { "abc": {content: "abc v2"}, }) } + +func TestCombinedAddAndRemoveInDirBug(t *testing.T) { + tc := util.NewTestCtx(t, auth.Project, 1) + defer tc.Close() + + writeProject(tc, 1, 1) + writeEmptyDir(tc, 1, 1, nil, "a/") + + c, fs, close := createTestClient(tc) + defer close() + + tmpDir := emptyTmpDir(t) + defer os.RemoveAll(tmpDir) + + rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{ + version: 1, + count: 1, + }) + + verifyDir(t, tmpDir, 1, map[string]expectedFile{ + "a/": {content: "", fileType: typeDirectory}, + }) + + writeFile(t, tmpDir, "a/b", "a/b v1") + + update(tc, c, 1, tmpDir, expectedResponse{ + version: 2, + count: 1, + }) + + removeFile(t, tmpDir, "a/b") + + update(tc, c, 1, tmpDir, expectedResponse{ + version: 3, + count: 2, + }) + + updateStream := newMockUpdateServer(tc.Context(), 1, map[string]expectedObject{ + "b": {content: "b v4"}, + "a/": {deleted: true}, + }) + err := fs.Update(updateStream) + require.NoError(t, err, "fs.Update") + + os.RemoveAll(tmpDir) + err = os.MkdirAll(tmpDir, 0755) + require.NoError(t, err, "recreate tmp dir %v", tmpDir) + + rebuild(tc, c, 1, i(2), tmpDir, nil, expectedResponse{ + version: 2, + count: 1, + }) + + debugObjects(tc) + + rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{ + version: 4, + count: 2, + }) + + verifyDir(t, tmpDir, 4, map[string]expectedFile{ + "b": {content: "b v4"}, + }) +} diff --git a/test/shared_test.go b/test/shared_test.go index d7c0ece..7663e62 100644 --- a/test/shared_test.go +++ b/test/shared_test.go @@ -243,6 +243,16 @@ func writeFile(t *testing.T, dir string, path string, content string) { require.NoError(t, err, "write file %v", path) } +func removeFile(t *testing.T, dir string, path string) { + err := os.Remove(filepath.Join(dir, path)) + require.NoError(t, err, "remove file %v", path) +} + +func removeDir(t *testing.T, dir string, path string) { + err := os.RemoveAll(filepath.Join(dir, path)) + require.NoError(t, err, "remove dir %v", path) +} + func emptyTmpDir(t *testing.T) string { dir, err := os.MkdirTemp("", "dateilager_tests_") require.NoError(t, err, "create temp dir")