package archive import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "os" "path/filepath" "reflect" "runtime" "strings" "sync" "sync/atomic" "testing" "time" "github.com/gogf/gf/v2/errors/gerror" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract" ) func TestPublishRunIsDurableDeterministicAndSummaryOnly(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-1")) service := New(root, nil, nil, repository) work := contract.ArchiveWork{RunID: "run-1", Version: "version-1"} staleDir := root if err := os.MkdirAll(staleDir, 0o755); err != nil { t.Fatal(err) } stalePath := filepath.Join(staleDir, ".run-1-stale.tmp") if err := os.WriteFile(stalePath, []byte("incomplete"), 0o600); err != nil { t.Fatal(err) } receipt, err := service.PublishRun(context.Background(), work, repository.snapshot) if err != nil { t.Fatalf("PublishRun() error = %v", err) } if receipt.Path != "run-run-1.json" { t.Fatalf("receipt path = %q", receipt.Path) } if err := receipt.Validate(); err != nil { t.Fatalf("receipt validation failed: %v", err) } archivePath := filepath.Join(root, filepath.FromSlash(receipt.Path)) info, err := os.Stat(archivePath) if err != nil { t.Fatal(err) } if runtime.GOOS != "windows" { if got := info.Mode().Perm(); got != 0o644 { t.Fatalf("archive mode = %o, want 644", got) } } data, err := os.ReadFile(archivePath) if err != nil { t.Fatal(err) } sum := sha256.Sum256(data) if got := hex.EncodeToString(sum[:]); got != receipt.Checksum { t.Fatalf("checksum = %q, want %q", receipt.Checksum, got) } for _, forbidden := range [][]byte{[]byte(`"archiveStatus"`), []byte(`"archivePath"`), []byte(`"taskId"`), []byte(`"card"`), []byte(`"message"`)} { if bytes.Contains(data, forbidden) { t.Fatalf("archive contains forbidden detail %s: %s", forbidden, data) } } var decoded contract.RunArchive if err := json.Unmarshal(data, &decoded); err != nil { t.Fatal(err) } if len(decoded.Rounds) != 2 || decoded.Rounds[0].Round != 1 || decoded.Rounds[1].Round != 2 { t.Fatalf("rounds were not deterministically sorted: %+v", decoded.Rounds) } if decoded.Rounds[1].ErrorCounts[contract.ErrorRateLimited] != 1 { t.Fatalf("error categories were not normalized: %+v", decoded.Rounds[1].ErrorCounts) } repeated, err := service.PublishRun(context.Background(), work, repository.snapshot) if err != nil { t.Fatalf("repeated PublishRun() error = %v", err) } if !reflect.DeepEqual(repeated, receipt) { t.Fatalf("repeated receipt changed:\nfirst: %+v\nsecond: %+v", receipt, repeated) } if _, err := os.Stat(stalePath); err != nil { t.Fatalf("stale temp residue should not block publication: %v", err) } repository.mu.Lock() defer repository.mu.Unlock() if len(repository.receipts) != 2 || !reflect.DeepEqual(repository.receipts[0], repository.receipts[1]) { t.Fatalf("registered receipts = %+v", repository.receipts) } } func TestArchiveActionPublishesAndRegistersFencedReceipt(t *testing.T) { root := t.TempDir() action := testReplayAction("action-1") action.Error = strings.Repeat("x", contract.MaxErrorMessageBytes+50) repository := newFakeReplayRepository(action) service := NewServiceWithReplay( Config{Root: root}, nil, nil, nil, ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository}, ) if err := service.ArchiveAction(context.Background(), action); err != nil { t.Fatalf("ArchiveAction() error = %v", err) } repository.mu.Lock() update := repository.update released := repository.released repository.mu.Unlock() if update == nil || update.Status != contract.ActionCompleted || update.Receipt == nil || update.CompletedAt == nil { t.Fatalf("registered update = %+v", update) } if update.ExpectedRevision != action.Revision || update.Receipt.ArchivedAt != *update.CompletedAt { t.Fatalf("receipt registration lost fencing/time: %+v", update) } if update.Lease.Token != "lease-token-2" { t.Fatalf("receipt registration used the unrenewed lease: %+v", update.Lease) } if released.Token != "lease-token-2" { t.Fatalf("archive released the stale lease: %+v", released) } if update.Receipt.Path != "action-action-1.json" { t.Fatalf("action receipt path = %q", update.Receipt.Path) } data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(update.Receipt.Path))) if err != nil { t.Fatal(err) } for _, forbidden := range [][]byte{[]byte("idempotencyKey"), []byte("fingerprint"), []byte("leaseOwner"), []byte("leaseToken"), []byte("revision"), []byte(`"receipt"`), []byte("completedAt")} { if bytes.Contains(data, forbidden) { t.Fatalf("action archive contains internal field %q: %s", forbidden, data) } } var decoded contract.ReplayAction if err := json.Unmarshal(data, &decoded); err != nil { t.Fatal(err) } if decoded.Status != contract.ActionReceiptPending || len(decoded.Error) != contract.MaxErrorMessageBytes { t.Fatalf("archived action was not bounded receipt_pending summary: %+v", decoded) } repeated, err := service.PublishAction(context.Background(), action) if err != nil { t.Fatalf("repeated PublishAction() error = %v", err) } if !reflect.DeepEqual(repeated, *update.Receipt) { t.Fatalf("repeated action receipt changed: %+v != %+v", repeated, *update.Receipt) } } func TestPublishRunCleansTempAfterRenameFailure(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-rename")) service := New(root, nil, nil, repository) renameErr := gerror.New("injected rename failure") service.replaceFile = func(string, string) error { return renameErr } _, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: "run-rename", Version: "v1"}, repository.snapshot) if !errors.Is(err, renameErr) { t.Fatalf("PublishRun() error = %v, want rename failure", err) } if matches, globErr := filepath.Glob(filepath.Join(root, ".run-rename-*.tmp")); globErr != nil || len(matches) != 0 { t.Fatalf("temporary files after failure = %v, error = %v", matches, globErr) } if _, statErr := os.Stat(filepath.Join(root, "run-run-rename.json")); !errors.Is(statErr, os.ErrNotExist) { t.Fatalf("final file exists after failed rename: %v", statErr) } if len(repository.receipts) != 0 { t.Fatalf("registered receipt after failed rename: %+v", repository.receipts) } } func TestChecksumFailureIsRecoveredByExactPathReplacement(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-checksum")) service := New(root, nil, nil, repository) service.afterReplace = func(path string) error { return os.WriteFile(path, []byte("corrupt"), 0o600) } work := contract.ArchiveWork{RunID: "run-checksum", Version: "v1"} if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, errChecksumMismatch) { t.Fatalf("PublishRun() error = %v, want checksum mismatch", err) } if len(repository.receipts) != 0 { t.Fatalf("registered receipt after checksum failure: %+v", repository.receipts) } service.afterReplace = func(string) error { return nil } receipt, err := service.PublishRun(context.Background(), work, repository.snapshot) if err != nil { t.Fatalf("recovery PublishRun() error = %v", err) } if receipt.Path != "run-run-checksum.json" { t.Fatalf("replacement used different path: %q", receipt.Path) } data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(receipt.Path))) if err != nil { t.Fatal(err) } if string(data) == "corrupt" { t.Fatal("corrupt final file was not replaced") } } func TestRegistrationRetryReusesIdenticalReceipt(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-register")) repository.registerErr = gerror.New("registration result was uncertain") service := New(root, nil, nil, repository) work := contract.ArchiveWork{RunID: "run-register", Version: "v1"} if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, repository.registerErr) { t.Fatalf("first PublishRun() error = %v", err) } repository.mu.Lock() firstReceipt := repository.receipts[0] repository.registerErr = nil repository.mu.Unlock() receipt, err := service.PublishRun(context.Background(), work, repository.snapshot) if err != nil { t.Fatalf("retry PublishRun() error = %v", err) } if !reflect.DeepEqual(receipt, firstReceipt) { t.Fatalf("retry changed uncertain receipt:\nfirst: %+v\nretry: %+v", firstReceipt, receipt) } } func TestPublishRunRejectsDifferentValidContentAtSamePath(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-conflict")) service := New(root, nil, nil, repository) work := contract.ArchiveWork{RunID: "run-conflict", Version: "v1"} first, err := service.PublishRun(context.Background(), work, repository.snapshot) if err != nil { t.Fatal(err) } changed := repository.snapshot changed.Run.TriggerSource = "different-valid-summary" if _, err = service.PublishRun(context.Background(), work, changed); !errors.Is(err, contract.ErrConflict) { t.Fatalf("PublishRun() error = %v, want conflict", err) } data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(first.Path))) if err != nil { t.Fatal(err) } sum := sha256.Sum256(data) if got := hex.EncodeToString(sum[:]); got != first.Checksum { t.Fatalf("conflicting publication replaced the original file: %q != %q", got, first.Checksum) } } func TestScannerRecoversFailedRunAfterRestart(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-restart")) repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) { return contract.RunArchive{}, gerror.New("source temporarily unavailable") } first := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository) if err := first.Start(context.Background()); err != nil { t.Fatal(err) } waitSignal(t, repository.failed, "initial archive failure") if err := first.Stop(); err != nil { t.Fatal(err) } repository.mu.Lock() repository.load = nil repository.mu.Unlock() second := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository) if err := second.Start(context.Background()); err != nil { t.Fatal(err) } waitSignal(t, repository.succeeded, "restart archive success") if err := second.Stop(); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(root, "run-run-restart.json")); err != nil { t.Fatalf("recovered archive file missing: %v", err) } } func TestPeriodicScannerRetriesPendingRun(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-periodic")) var calls atomic.Int32 repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) { if calls.Add(1) == 1 { return contract.RunArchive{}, gerror.New("first scan fails") } return repository.snapshot, nil } service := NewService(Config{Root: root, Interval: 10 * time.Millisecond, MaxAttempts: 1}, repository, repository, repository) if err := service.Start(context.Background()); err != nil { t.Fatal(err) } waitSignal(t, repository.succeeded, "periodic archive success") if err := service.Stop(); err != nil { t.Fatal(err) } if calls.Load() < 2 { t.Fatalf("loader calls = %d, want at least 2", calls.Load()) } } func TestStartupScannerArchivesReceiptPendingAction(t *testing.T) { action := testReplayAction("action-startup") repository := newFakeReplayRepository(action) service := NewServiceWithReplay( Config{Root: t.TempDir(), Interval: time.Hour, MaxAttempts: 1}, nil, nil, nil, ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository}, ) if err := service.Start(context.Background()); err != nil { t.Fatal(err) } waitSignal(t, repository.completed, "startup replay archive success") if err := service.Stop(); err != nil { t.Fatal(err) } } func TestCancellationStopsBlockedScannerAndPublication(t *testing.T) { root := t.TempDir() repository := newFakeRunRepository(testRunArchive("run-cancel")) started := make(chan struct{}) repository.load = func(ctx context.Context, _ contract.ArchiveWork) (contract.RunArchive, error) { close(started) <-ctx.Done() return contract.RunArchive{}, ctx.Err() } service := NewService(Config{Root: root, Interval: time.Hour, InitialBackoff: time.Hour}, repository, repository, repository) if err := service.Start(context.Background()); err != nil { t.Fatal(err) } waitSignal(t, started, "blocked loader start") stopped := make(chan struct{}) go func() { _ = service.Stop() close(stopped) }() waitSignal(t, stopped, "scanner cancellation") canceled, cancel := context.WithCancel(context.Background()) cancel() _, err := service.PublishRun(canceled, contract.ArchiveWork{RunID: "run-cancel", Version: "v1"}, repository.snapshot) if !errors.Is(err, context.Canceled) { t.Fatalf("canceled PublishRun() error = %v", err) } if _, err := os.Stat(filepath.Join(root, "run-run-cancel.json")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("canceled publication wrote final file: %v", err) } } func TestCancellationAfterRenameStillVerifiesPublicationAndReleasesLease(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) action := testReplayAction("action-cancel-after-rename") repository := newFakeReplayRepository(action) service := NewServiceWithReplay( Config{Root: t.TempDir()}, nil, nil, nil, ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository}, ) service.afterReplace = func(string) error { cancel() return nil } if err := service.ArchiveAction(ctx, action); err != nil { t.Fatalf("ArchiveAction() error = %v", err) } repository.mu.Lock() releaseCtxErr := repository.releaseCtxErr repository.mu.Unlock() if releaseCtxErr != nil { t.Fatalf("lease release inherited canceled context: %v", releaseCtxErr) } } func TestArchiveRejectsTraversalAndUnsafeFilesystemIDs(t *testing.T) { root := t.TempDir() service := New(root, nil, nil, newFakeRunRepository(contract.RunArchive{})) for _, id := range []string{"../escape", `..\escape`, "/absolute", `C:\escape`, "drive:name", ".", "..", "{hash}", "CON"} { t.Run(strings.ReplaceAll(id, "/", "_"), func(t *testing.T) { snapshot := testRunArchive(id) _, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: id, Version: "v1"}, snapshot) if !errors.Is(err, contract.ErrInvalidArgument) { t.Fatalf("PublishRun(%q) error = %v, want invalid argument", id, err) } }) } entries, err := os.ReadDir(root) if err != nil { t.Fatal(err) } if len(entries) != 0 { t.Fatalf("unsafe IDs created archive entries: %+v", entries) } } func testRunArchive(runID string) contract.RunArchive { created := time.Date(2026, time.August, 31, 23, 0, 0, 0, time.FixedZone("test", 8*60*60)) completed := time.Date(2026, time.September, 20, 16, 0, 0, 0, time.FixedZone("test", 8*60*60)) archived := completed.Add(time.Minute) return contract.RunArchive{ Version: contract.ArchiveSchemaVersion, Run: contract.RunMeta{ RunID: runID, Status: contract.RunCompletedWithFailures, ProducerDone: true, CurrentRound: 2, MaxRound: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1, PendingFailureCount: 1, CreatedAt: created, ProducerCompletedAt: completed, CompletedAt: completed, ArchiveStatus: contract.ArchiveFailed, ArchiveVersion: 99, ArchivePath: "old.json", ArchiveChecksum: strings.Repeat("a", 64), ArchivedAt: &archived, }, Rounds: []contract.RoundStats{ {RunID: runID, Round: 2, EnqueuedCount: 1, FailedCount: 1, ErrorCounts: contract.ErrorCounts{"mobile_busy": 1}}, {RunID: runID, Round: 1, EnqueuedCount: 2, SuccessCount: 2, ErrorCounts: nil}, }, } } func testReplayAction(actionID string) contract.ReplayAction { created := time.Date(2026, time.September, 20, 12, 0, 0, 0, time.FixedZone("test", 8*60*60)) return contract.ReplayAction{ ActionID: actionID, SourceRunID: "source-1", TargetRunID: "target-1", RequestedBy: "operator", IdempotencyKey: "secret-key", Fingerprint: "secret-fingerprint", Status: contract.ActionReceiptPending, ReplayCycle: 1, Progress: contract.ReplayProgress{DiscoveredCount: 2, EnqueuedCount: 2, CleanedCount: 2}, LeaseOwner: "engine", LeaseToken: "secret-token", LeaseUntil: created.Add(time.Hour), Revision: 7, CreatedAt: created, UpdatedAt: created.Add(time.Minute), } } type fakeRunRepository struct { mu sync.Mutex work contract.ArchiveWork snapshot contract.RunArchive load func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) registered bool receipts []contract.ArchiveReceipt failures []string registerErr error succeeded chan struct{} failed chan struct{} } func newFakeRunRepository(snapshot contract.RunArchive) *fakeRunRepository { return &fakeRunRepository{ work: contract.ArchiveWork{RunID: snapshot.Run.RunID, Version: "repository-version"}, snapshot: snapshot, succeeded: make(chan struct{}, 8), failed: make(chan struct{}, 8), } } func (r *fakeRunRepository) ListArchiveWork(ctx context.Context, _ contract.PageRequest) (contract.ArchiveWorkPage, error) { if err := ctx.Err(); err != nil { return contract.ArchiveWorkPage{}, err } r.mu.Lock() defer r.mu.Unlock() if r.registered { return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}, nil } return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{r.work}}, nil } func (r *fakeRunRepository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) { r.mu.Lock() load := r.load snapshot := r.snapshot r.mu.Unlock() if load != nil { return load(ctx, work) } return snapshot, nil } func (r *fakeRunRepository) RegisterArchive(_ context.Context, _ contract.ArchiveWork, receipt contract.ArchiveReceipt) error { r.mu.Lock() r.receipts = append(r.receipts, receipt) err := r.registerErr if err == nil { r.registered = true } r.mu.Unlock() if err != nil { return err } notify(r.succeeded) return nil } func (r *fakeRunRepository) RegisterArchiveFailure(_ context.Context, _ contract.ArchiveWork, message string) error { r.mu.Lock() r.failures = append(r.failures, message) r.mu.Unlock() notify(r.failed) return nil } type fakeReplayRepository struct { mu sync.Mutex action contract.ReplayAction update *contract.ReplayActionUpdate released contract.ReplayLease releaseCtxErr error completed chan struct{} } func newFakeReplayRepository(action contract.ReplayAction) *fakeReplayRepository { return &fakeReplayRepository{action: action, completed: make(chan struct{}, 4)} } func (r *fakeReplayRepository) ListReplayWork(ctx context.Context, _ contract.PageRequest) (contract.ReplayActionPage, error) { if err := ctx.Err(); err != nil { return contract.ReplayActionPage{}, err } r.mu.Lock() defer r.mu.Unlock() if r.update != nil { return contract.ReplayActionPage{Items: []contract.ReplayAction{}}, nil } return contract.ReplayActionPage{Items: []contract.ReplayAction{r.action}}, nil } func (r *fakeReplayRepository) AcquireReplayLease(_ context.Context, actionID, owner string, ttl time.Duration) (contract.ReplayLease, error) { return contract.ReplayLease{ActionID: actionID, Owner: owner, Token: "lease-token", Until: time.Now().Add(ttl)}, nil } func (r *fakeReplayRepository) RenewReplayLease(_ context.Context, lease contract.ReplayLease, ttl time.Duration) (contract.ReplayLease, error) { lease.Token = "lease-token-2" lease.Until = time.Now().Add(ttl) return lease, nil } func (r *fakeReplayRepository) ReleaseReplayLease(ctx context.Context, lease contract.ReplayLease) error { r.mu.Lock() r.released = lease r.releaseCtxErr = ctx.Err() r.mu.Unlock() return nil } func (r *fakeReplayRepository) UpdateReplayAction(_ context.Context, update contract.ReplayActionUpdate) (contract.ReplayAction, error) { r.mu.Lock() copyOfUpdate := update r.update = ©OfUpdate action := r.action action.Status = update.Status action.Receipt = update.Receipt action.CompletedAt = update.CompletedAt r.mu.Unlock() notify(r.completed) return action, nil } func notify(ch chan struct{}) { select { case ch <- struct{}{}: default: } } func waitSignal(t *testing.T, ch <-chan struct{}, operation string) { t.Helper() select { case <-ch: case <-time.After(3 * time.Second): t.Fatalf("timed out waiting for %s", operation) } }