iotlcm/internal/fullsync/archive/archive_test.go

package archive

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/gogf/gf/v2/errors/gerror"

	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
)

func TestPublishRunIsDurableDeterministicAndSummaryOnly(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-1"))
	service := New(root, nil, nil, repository)
	work := contract.ArchiveWork{RunID: "run-1", Version: "version-1"}

	staleDir := root
	if err := os.MkdirAll(staleDir, 0o755); err != nil {
		t.Fatal(err)
	}
	stalePath := filepath.Join(staleDir, ".run-1-stale.tmp")
	if err := os.WriteFile(stalePath, []byte("incomplete"), 0o600); err != nil {
		t.Fatal(err)
	}

	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
	if err != nil {
		t.Fatalf("PublishRun() error = %v", err)
	}
	if receipt.Path != "run-run-1.json" {
		t.Fatalf("receipt path = %q", receipt.Path)
	}
	if err := receipt.Validate(); err != nil {
		t.Fatalf("receipt validation failed: %v", err)
	}

	archivePath := filepath.Join(root, filepath.FromSlash(receipt.Path))
	info, err := os.Stat(archivePath)
	if err != nil {
		t.Fatal(err)
	}
	if runtime.GOOS != "windows" {
		if got := info.Mode().Perm(); got != 0o644 {
			t.Fatalf("archive mode = %o, want 644", got)
		}
	}
	data, err := os.ReadFile(archivePath)
	if err != nil {
		t.Fatal(err)
	}
	sum := sha256.Sum256(data)
	if got := hex.EncodeToString(sum[:]); got != receipt.Checksum {
		t.Fatalf("checksum = %q, want %q", receipt.Checksum, got)
	}
	for _, forbidden := range [][]byte{[]byte(`"archiveStatus"`), []byte(`"archivePath"`), []byte(`"taskId"`), []byte(`"card"`), []byte(`"message"`)} {
		if bytes.Contains(data, forbidden) {
			t.Fatalf("archive contains forbidden detail %s: %s", forbidden, data)
		}
	}
	var decoded contract.RunArchive
	if err := json.Unmarshal(data, &decoded); err != nil {
		t.Fatal(err)
	}
	if len(decoded.Rounds) != 2 || decoded.Rounds[0].Round != 1 || decoded.Rounds[1].Round != 2 {
		t.Fatalf("rounds were not deterministically sorted: %+v", decoded.Rounds)
	}
	if decoded.Rounds[1].ErrorCounts[contract.ErrorRateLimited] != 1 {
		t.Fatalf("error categories were not normalized: %+v", decoded.Rounds[1].ErrorCounts)
	}

	repeated, err := service.PublishRun(context.Background(), work, repository.snapshot)
	if err != nil {
		t.Fatalf("repeated PublishRun() error = %v", err)
	}
	if !reflect.DeepEqual(repeated, receipt) {
		t.Fatalf("repeated receipt changed:\nfirst:  %+v\nsecond: %+v", receipt, repeated)
	}
	if _, err := os.Stat(stalePath); err != nil {
		t.Fatalf("stale temp residue should not block publication: %v", err)
	}
	repository.mu.Lock()
	defer repository.mu.Unlock()
	if len(repository.receipts) != 2 || !reflect.DeepEqual(repository.receipts[0], repository.receipts[1]) {
		t.Fatalf("registered receipts = %+v", repository.receipts)
	}
}

func TestArchiveActionPublishesAndRegistersFencedReceipt(t *testing.T) {
	root := t.TempDir()
	action := testReplayAction("action-1")
	action.Error = strings.Repeat("x", contract.MaxErrorMessageBytes+50)
	repository := newFakeReplayRepository(action)
	service := NewServiceWithReplay(
		Config{Root: root},
		nil,
		nil,
		nil,
		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
	)

	if err := service.ArchiveAction(context.Background(), action); err != nil {
		t.Fatalf("ArchiveAction() error = %v", err)
	}
	repository.mu.Lock()
	update := repository.update
	released := repository.released
	repository.mu.Unlock()
	if update == nil || update.Status != contract.ActionCompleted || update.Receipt == nil || update.CompletedAt == nil {
		t.Fatalf("registered update = %+v", update)
	}
	if update.ExpectedRevision != action.Revision || update.Receipt.ArchivedAt != *update.CompletedAt {
		t.Fatalf("receipt registration lost fencing/time: %+v", update)
	}
	if update.Lease.Token != "lease-token-2" {
		t.Fatalf("receipt registration used the unrenewed lease: %+v", update.Lease)
	}
	if released.Token != "lease-token-2" {
		t.Fatalf("archive released the stale lease: %+v", released)
	}
	if update.Receipt.Path != "action-action-1.json" {
		t.Fatalf("action receipt path = %q", update.Receipt.Path)
	}

	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(update.Receipt.Path)))
	if err != nil {
		t.Fatal(err)
	}
	for _, forbidden := range [][]byte{[]byte("idempotencyKey"), []byte("fingerprint"), []byte("leaseOwner"), []byte("leaseToken"), []byte("revision"), []byte(`"receipt"`), []byte("completedAt")} {
		if bytes.Contains(data, forbidden) {
			t.Fatalf("action archive contains internal field %q: %s", forbidden, data)
		}
	}
	var decoded contract.ReplayAction
	if err := json.Unmarshal(data, &decoded); err != nil {
		t.Fatal(err)
	}
	if decoded.Status != contract.ActionReceiptPending || len(decoded.Error) != contract.MaxErrorMessageBytes {
		t.Fatalf("archived action was not bounded receipt_pending summary: %+v", decoded)
	}
	repeated, err := service.PublishAction(context.Background(), action)
	if err != nil {
		t.Fatalf("repeated PublishAction() error = %v", err)
	}
	if !reflect.DeepEqual(repeated, *update.Receipt) {
		t.Fatalf("repeated action receipt changed: %+v != %+v", repeated, *update.Receipt)
	}
}

func TestPublishRunCleansTempAfterRenameFailure(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-rename"))
	service := New(root, nil, nil, repository)
	renameErr := gerror.New("injected rename failure")
	service.replaceFile = func(string, string) error { return renameErr }

	_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: "run-rename", Version: "v1"}, repository.snapshot)
	if !errors.Is(err, renameErr) {
		t.Fatalf("PublishRun() error = %v, want rename failure", err)
	}
	if matches, globErr := filepath.Glob(filepath.Join(root, ".run-rename-*.tmp")); globErr != nil || len(matches) != 0 {
		t.Fatalf("temporary files after failure = %v, error = %v", matches, globErr)
	}
	if _, statErr := os.Stat(filepath.Join(root, "run-run-rename.json")); !errors.Is(statErr, os.ErrNotExist) {
		t.Fatalf("final file exists after failed rename: %v", statErr)
	}
	if len(repository.receipts) != 0 {
		t.Fatalf("registered receipt after failed rename: %+v", repository.receipts)
	}
}

func TestChecksumFailureIsRecoveredByExactPathReplacement(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-checksum"))
	service := New(root, nil, nil, repository)
	service.afterReplace = func(path string) error {
		return os.WriteFile(path, []byte("corrupt"), 0o600)
	}
	work := contract.ArchiveWork{RunID: "run-checksum", Version: "v1"}

	if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, errChecksumMismatch) {
		t.Fatalf("PublishRun() error = %v, want checksum mismatch", err)
	}
	if len(repository.receipts) != 0 {
		t.Fatalf("registered receipt after checksum failure: %+v", repository.receipts)
	}
	service.afterReplace = func(string) error { return nil }
	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
	if err != nil {
		t.Fatalf("recovery PublishRun() error = %v", err)
	}
	if receipt.Path != "run-run-checksum.json" {
		t.Fatalf("replacement used different path: %q", receipt.Path)
	}
	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(receipt.Path)))
	if err != nil {
		t.Fatal(err)
	}
	if string(data) == "corrupt" {
		t.Fatal("corrupt final file was not replaced")
	}
}

func TestRegistrationRetryReusesIdenticalReceipt(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-register"))
	repository.registerErr = gerror.New("registration result was uncertain")
	service := New(root, nil, nil, repository)
	work := contract.ArchiveWork{RunID: "run-register", Version: "v1"}

	if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, repository.registerErr) {
		t.Fatalf("first PublishRun() error = %v", err)
	}
	repository.mu.Lock()
	firstReceipt := repository.receipts[0]
	repository.registerErr = nil
	repository.mu.Unlock()

	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
	if err != nil {
		t.Fatalf("retry PublishRun() error = %v", err)
	}
	if !reflect.DeepEqual(receipt, firstReceipt) {
		t.Fatalf("retry changed uncertain receipt:\nfirst: %+v\nretry: %+v", firstReceipt, receipt)
	}
}

func TestPublishRunRejectsDifferentValidContentAtSamePath(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-conflict"))
	service := New(root, nil, nil, repository)
	work := contract.ArchiveWork{RunID: "run-conflict", Version: "v1"}

	first, err := service.PublishRun(context.Background(), work, repository.snapshot)
	if err != nil {
		t.Fatal(err)
	}
	changed := repository.snapshot
	changed.Run.TriggerSource = "different-valid-summary"
	if _, err = service.PublishRun(context.Background(), work, changed); !errors.Is(err, contract.ErrConflict) {
		t.Fatalf("PublishRun() error = %v, want conflict", err)
	}
	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(first.Path)))
	if err != nil {
		t.Fatal(err)
	}
	sum := sha256.Sum256(data)
	if got := hex.EncodeToString(sum[:]); got != first.Checksum {
		t.Fatalf("conflicting publication replaced the original file: %q != %q", got, first.Checksum)
	}
}

func TestScannerRecoversFailedRunAfterRestart(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-restart"))
	repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
		return contract.RunArchive{}, gerror.New("source temporarily unavailable")
	}
	first := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
	if err := first.Start(context.Background()); err != nil {
		t.Fatal(err)
	}
	waitSignal(t, repository.failed, "initial archive failure")
	if err := first.Stop(); err != nil {
		t.Fatal(err)
	}

	repository.mu.Lock()
	repository.load = nil
	repository.mu.Unlock()
	second := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
	if err := second.Start(context.Background()); err != nil {
		t.Fatal(err)
	}
	waitSignal(t, repository.succeeded, "restart archive success")
	if err := second.Stop(); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(filepath.Join(root, "run-run-restart.json")); err != nil {
		t.Fatalf("recovered archive file missing: %v", err)
	}
}

func TestPeriodicScannerRetriesPendingRun(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-periodic"))
	var calls atomic.Int32
	repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
		if calls.Add(1) == 1 {
			return contract.RunArchive{}, gerror.New("first scan fails")
		}
		return repository.snapshot, nil
	}
	service := NewService(Config{Root: root, Interval: 10 * time.Millisecond, MaxAttempts: 1}, repository, repository, repository)
	if err := service.Start(context.Background()); err != nil {
		t.Fatal(err)
	}
	waitSignal(t, repository.succeeded, "periodic archive success")
	if err := service.Stop(); err != nil {
		t.Fatal(err)
	}
	if calls.Load() < 2 {
		t.Fatalf("loader calls = %d, want at least 2", calls.Load())
	}
}

func TestStartupScannerArchivesReceiptPendingAction(t *testing.T) {
	action := testReplayAction("action-startup")
	repository := newFakeReplayRepository(action)
	service := NewServiceWithReplay(
		Config{Root: t.TempDir(), Interval: time.Hour, MaxAttempts: 1},
		nil,
		nil,
		nil,
		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
	)
	if err := service.Start(context.Background()); err != nil {
		t.Fatal(err)
	}
	waitSignal(t, repository.completed, "startup replay archive success")
	if err := service.Stop(); err != nil {
		t.Fatal(err)
	}
}

func TestCancellationStopsBlockedScannerAndPublication(t *testing.T) {
	root := t.TempDir()
	repository := newFakeRunRepository(testRunArchive("run-cancel"))
	started := make(chan struct{})
	repository.load = func(ctx context.Context, _ contract.ArchiveWork) (contract.RunArchive, error) {
		close(started)
		<-ctx.Done()
		return contract.RunArchive{}, ctx.Err()
	}
	service := NewService(Config{Root: root, Interval: time.Hour, InitialBackoff: time.Hour}, repository, repository, repository)
	if err := service.Start(context.Background()); err != nil {
		t.Fatal(err)
	}
	waitSignal(t, started, "blocked loader start")
	stopped := make(chan struct{})
	go func() {
		_ = service.Stop()
		close(stopped)
	}()
	waitSignal(t, stopped, "scanner cancellation")

	canceled, cancel := context.WithCancel(context.Background())
	cancel()
	_, err := service.PublishRun(canceled, contract.ArchiveWork{RunID: "run-cancel", Version: "v1"}, repository.snapshot)
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("canceled PublishRun() error = %v", err)
	}
	if _, err := os.Stat(filepath.Join(root, "run-run-cancel.json")); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("canceled publication wrote final file: %v", err)
	}
}

func TestCancellationAfterRenameStillVerifiesPublicationAndReleasesLease(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	action := testReplayAction("action-cancel-after-rename")
	repository := newFakeReplayRepository(action)
	service := NewServiceWithReplay(
		Config{Root: t.TempDir()},
		nil,
		nil,
		nil,
		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
	)
	service.afterReplace = func(string) error {
		cancel()
		return nil
	}

	if err := service.ArchiveAction(ctx, action); err != nil {
		t.Fatalf("ArchiveAction() error = %v", err)
	}
	repository.mu.Lock()
	releaseCtxErr := repository.releaseCtxErr
	repository.mu.Unlock()
	if releaseCtxErr != nil {
		t.Fatalf("lease release inherited canceled context: %v", releaseCtxErr)
	}
}

func TestArchiveRejectsTraversalAndUnsafeFilesystemIDs(t *testing.T) {
	root := t.TempDir()
	service := New(root, nil, nil, newFakeRunRepository(contract.RunArchive{}))
	for _, id := range []string{"../escape", `..\escape`, "/absolute", `C:\escape`, "drive:name", ".", "..", "{hash}", "CON"} {
		t.Run(strings.ReplaceAll(id, "/", "_"), func(t *testing.T) {
			snapshot := testRunArchive(id)
			_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: id, Version: "v1"}, snapshot)
			if !errors.Is(err, contract.ErrInvalidArgument) {
				t.Fatalf("PublishRun(%q) error = %v, want invalid argument", id, err)
			}
		})
	}
	entries, err := os.ReadDir(root)
	if err != nil {
		t.Fatal(err)
	}
	if len(entries) != 0 {
		t.Fatalf("unsafe IDs created archive entries: %+v", entries)
	}
}

func testRunArchive(runID string) contract.RunArchive {
	created := time.Date(2026, time.August, 31, 23, 0, 0, 0, time.FixedZone("test", 8*60*60))
	completed := time.Date(2026, time.September, 20, 16, 0, 0, 0, time.FixedZone("test", 8*60*60))
	archived := completed.Add(time.Minute)
	return contract.RunArchive{
		Version: contract.ArchiveSchemaVersion,
		Run: contract.RunMeta{
			RunID: runID, Status: contract.RunCompletedWithFailures, ProducerDone: true,
			CurrentRound: 2, MaxRound: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1, PendingFailureCount: 1,
			CreatedAt: created, ProducerCompletedAt: completed, CompletedAt: completed,
			ArchiveStatus: contract.ArchiveFailed, ArchiveVersion: 99, ArchivePath: "old.json",
			ArchiveChecksum: strings.Repeat("a", 64), ArchivedAt: &archived,
		},
		Rounds: []contract.RoundStats{
			{RunID: runID, Round: 2, EnqueuedCount: 1, FailedCount: 1, ErrorCounts: contract.ErrorCounts{"mobile_busy": 1}},
			{RunID: runID, Round: 1, EnqueuedCount: 2, SuccessCount: 2, ErrorCounts: nil},
		},
	}
}

func testReplayAction(actionID string) contract.ReplayAction {
	created := time.Date(2026, time.September, 20, 12, 0, 0, 0, time.FixedZone("test", 8*60*60))
	return contract.ReplayAction{
		ActionID: actionID, SourceRunID: "source-1", TargetRunID: "target-1", RequestedBy: "operator",
		IdempotencyKey: "secret-key", Fingerprint: "secret-fingerprint", Status: contract.ActionReceiptPending,
		ReplayCycle: 1, Progress: contract.ReplayProgress{DiscoveredCount: 2, EnqueuedCount: 2, CleanedCount: 2},
		LeaseOwner: "engine", LeaseToken: "secret-token", LeaseUntil: created.Add(time.Hour), Revision: 7,
		CreatedAt: created, UpdatedAt: created.Add(time.Minute),
	}
}

type fakeRunRepository struct {
	mu          sync.Mutex
	work        contract.ArchiveWork
	snapshot    contract.RunArchive
	load        func(context.Context, contract.ArchiveWork) (contract.RunArchive, error)
	registered  bool
	receipts    []contract.ArchiveReceipt
	failures    []string
	registerErr error
	succeeded   chan struct{}
	failed      chan struct{}
}

func newFakeRunRepository(snapshot contract.RunArchive) *fakeRunRepository {
	return &fakeRunRepository{
		work:      contract.ArchiveWork{RunID: snapshot.Run.RunID, Version: "repository-version"},
		snapshot:  snapshot,
		succeeded: make(chan struct{}, 8),
		failed:    make(chan struct{}, 8),
	}
}

func (r *fakeRunRepository) ListArchiveWork(ctx context.Context, _ contract.PageRequest) (contract.ArchiveWorkPage, error) {
	if err := ctx.Err(); err != nil {
		return contract.ArchiveWorkPage{}, err
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.registered {
		return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}, nil
	}
	return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{r.work}}, nil
}

func (r *fakeRunRepository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
	r.mu.Lock()
	load := r.load
	snapshot := r.snapshot
	r.mu.Unlock()
	if load != nil {
		return load(ctx, work)
	}
	return snapshot, nil
}

func (r *fakeRunRepository) RegisterArchive(_ context.Context, _ contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
	r.mu.Lock()
	r.receipts = append(r.receipts, receipt)
	err := r.registerErr
	if err == nil {
		r.registered = true
	}
	r.mu.Unlock()
	if err != nil {
		return err
	}
	notify(r.succeeded)
	return nil
}

func (r *fakeRunRepository) RegisterArchiveFailure(_ context.Context, _ contract.ArchiveWork, message string) error {
	r.mu.Lock()
	r.failures = append(r.failures, message)
	r.mu.Unlock()
	notify(r.failed)
	return nil
}

type fakeReplayRepository struct {
	mu            sync.Mutex
	action        contract.ReplayAction
	update        *contract.ReplayActionUpdate
	released      contract.ReplayLease
	releaseCtxErr error
	completed     chan struct{}
}

func newFakeReplayRepository(action contract.ReplayAction) *fakeReplayRepository {
	return &fakeReplayRepository{action: action, completed: make(chan struct{}, 4)}
}

func (r *fakeReplayRepository) ListReplayWork(ctx context.Context, _ contract.PageRequest) (contract.ReplayActionPage, error) {
	if err := ctx.Err(); err != nil {
		return contract.ReplayActionPage{}, err
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.update != nil {
		return contract.ReplayActionPage{Items: []contract.ReplayAction{}}, nil
	}
	return contract.ReplayActionPage{Items: []contract.ReplayAction{r.action}}, nil
}

func (r *fakeReplayRepository) AcquireReplayLease(_ context.Context, actionID, owner string, ttl time.Duration) (contract.ReplayLease, error) {
	return contract.ReplayLease{ActionID: actionID, Owner: owner, Token: "lease-token", Until: time.Now().Add(ttl)}, nil
}

func (r *fakeReplayRepository) RenewReplayLease(_ context.Context, lease contract.ReplayLease, ttl time.Duration) (contract.ReplayLease, error) {
	lease.Token = "lease-token-2"
	lease.Until = time.Now().Add(ttl)
	return lease, nil
}

func (r *fakeReplayRepository) ReleaseReplayLease(ctx context.Context, lease contract.ReplayLease) error {
	r.mu.Lock()
	r.released = lease
	r.releaseCtxErr = ctx.Err()
	r.mu.Unlock()
	return nil
}

func (r *fakeReplayRepository) UpdateReplayAction(_ context.Context, update contract.ReplayActionUpdate) (contract.ReplayAction, error) {
	r.mu.Lock()
	copyOfUpdate := update
	r.update = &copyOfUpdate
	action := r.action
	action.Status = update.Status
	action.Receipt = update.Receipt
	action.CompletedAt = update.CompletedAt
	r.mu.Unlock()
	notify(r.completed)
	return action, nil
}

func notify(ch chan struct{}) {
	select {
	case ch <- struct{}{}:
	default:
	}
}

func waitSignal(t *testing.T, ch <-chan struct{}, operation string) {
	t.Helper()
	select {
	case <-ch:
	case <-time.After(3 * time.Second):
		t.Fatalf("timed out waiting for %s", operation)
	}
}