// Package archive durably stores summary-only receipts for completed full-sync // runs and replay actions. It depends only on the shared full-sync contracts; // repository implementations and service wiring live elsewhere. package archive import ( "bufio" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "io" "os" "path/filepath" "sort" "strings" "sync" "time" "github.com/gogf/gf/v2/errors/gerror" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract" ) const ( defaultInterval = time.Minute defaultBackoff = time.Second defaultMaxBackoff = time.Minute defaultMaxAttempts = 4 defaultReplayLeaseTTL = time.Minute defaultReplayLeaseName = "fullsync-archive" defaultReleaseTimeout = 5 * time.Second maxRounds = 10000 ) var errChecksumMismatch = gerror.New("archive checksum verification failed") // Config controls the recovery scanner. Root is the archive directory. Zero // timing values select conservative defaults. type Config struct { Root string Interval time.Duration InitialBackoff time.Duration MaxBackoff time.Duration MaxAttempts int ReplayLeaseOwner string ReplayLeaseTTL time.Duration Now func() time.Time } // ReplayRepositories are the contract boundaries used to recover replay // actions left in receipt_pending after a process restart. type ReplayRepositories struct { Discovery contract.ReplayWorkDiscovery Leases contract.ReplayLeaseStore Progress contract.ReplayProgressStore } // Service publishes archive files and scans durable repository indexes at // startup and periodically. A Service may be configured for runs, actions, or // both. type Service struct { root string runDiscovery contract.ArchiveWorkDiscovery runLoader contract.ArchiveWorkLoader runRegistrar contract.ArchiveRegistrar replay ReplayRepositories interval time.Duration initialBackoff time.Duration maxBackoff time.Duration maxAttempts int leaseOwner string leaseTTL time.Duration now func() time.Time // Publications are serialized so repeat writes in this process observe the // same final file timestamp and therefore produce the same receipt. publishMu sync.Mutex mu sync.Mutex running bool cancel context.CancelFunc done chan struct{} // These hooks keep failure-path tests deterministic without weakening the // production filesystem path. replaceFile func(string, string) error afterReplace func(string) error } // New constructs a run-archive service with default scanner settings. func New(root string, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service { return NewService(Config{Root: root}, discovery, loader, registrar) } // NewService constructs a run-archive service. Use NewServiceWithReplay when // replay receipt recovery is also required. func NewService(cfg Config, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service { return NewServiceWithReplay(cfg, discovery, loader, registrar, ReplayRepositories{}) } // NewServiceWithReplay constructs a lifecycle-enabled run and replay archive // service using only shared contract interfaces. func NewServiceWithReplay( cfg Config, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar, replay ReplayRepositories, ) *Service { if cfg.Interval <= 0 { cfg.Interval = defaultInterval } if cfg.InitialBackoff <= 0 { cfg.InitialBackoff = defaultBackoff } if cfg.MaxBackoff <= 0 { cfg.MaxBackoff = defaultMaxBackoff } if cfg.MaxBackoff < cfg.InitialBackoff { cfg.MaxBackoff = cfg.InitialBackoff } if cfg.MaxAttempts <= 0 { cfg.MaxAttempts = defaultMaxAttempts } if strings.TrimSpace(cfg.ReplayLeaseOwner) == "" { cfg.ReplayLeaseOwner = defaultReplayLeaseName } if cfg.ReplayLeaseTTL <= 0 { cfg.ReplayLeaseTTL = defaultReplayLeaseTTL } if cfg.Now == nil { cfg.Now = time.Now } return &Service{ root: cfg.Root, runDiscovery: discovery, runLoader: loader, runRegistrar: registrar, replay: replay, interval: cfg.Interval, initialBackoff: cfg.InitialBackoff, maxBackoff: cfg.MaxBackoff, maxAttempts: cfg.MaxAttempts, leaseOwner: cfg.ReplayLeaseOwner, leaseTTL: cfg.ReplayLeaseTTL, now: cfg.Now, replaceFile: atomicReplace, afterReplace: func(string) error { return nil }, } } // Start launches an immediate recovery scan followed by periodic scans. It is // idempotent while the service is running. func (s *Service) Start(parent context.Context) error { if parent == nil { return gerror.New("archive parent context is nil") } if err := s.validateConfiguration(); err != nil { return err } s.mu.Lock() if s.running { s.mu.Unlock() return nil } ctx, cancel := context.WithCancel(parent) done := make(chan struct{}) s.cancel, s.done, s.running = cancel, done, true s.mu.Unlock() go func() { defer func() { s.mu.Lock() if s.done == done { s.running = false s.cancel = nil } s.mu.Unlock() close(done) }() s.scanLoop(ctx) }() return nil } // Stop cancels the scanner and waits for it to finish. It is safe to call more // than once. func (s *Service) Stop() error { s.mu.Lock() if !s.running { s.mu.Unlock() return nil } cancel, done := s.cancel, s.done s.mu.Unlock() cancel() <-done return nil } // Close implements the common service lifecycle convention. func (s *Service) Close() error { return s.Stop() } // Done is closed when the current scanner invocation exits. It returns nil // before the service has been started. func (s *Service) Done() <-chan struct{} { s.mu.Lock() defer s.mu.Unlock() return s.done } // RunOnce performs one recovery pass. It is useful for explicit lifecycle // wiring and deterministic integration tests. func (s *Service) RunOnce(ctx context.Context) error { if ctx == nil { return gerror.New("archive scan context is nil") } if err := s.validateConfiguration(); err != nil { return err } var scanErrors []error if s.runDiscovery != nil { if err := s.scanRuns(ctx); err != nil { scanErrors = append(scanErrors, gerror.Wrap(err, "scan run archives")) } } if s.replay.Discovery != nil && ctx.Err() == nil { if err := s.scanActions(ctx); err != nil { scanErrors = append(scanErrors, gerror.Wrap(err, "scan replay action archives")) } } return errors.Join(scanErrors...) } // PublishRun atomically writes a run summary and then conditionally registers // the durable receipt against the work version. func (s *Service) PublishRun(ctx context.Context, work contract.ArchiveWork, snapshot contract.RunArchive) (contract.ArchiveReceipt, error) { if err := validateWork(work); err != nil { return contract.ArchiveReceipt{}, err } normalized, err := normalizeRunArchive(work, snapshot) if err != nil { return contract.ArchiveReceipt{}, err } pathTime := normalized.Run.CompletedAt if pathTime.IsZero() { pathTime = normalized.Run.CreatedAt } receipt, err := s.publishJSON(ctx, "runs", work.RunID, pathTime, normalized) if err != nil { return contract.ArchiveReceipt{}, err } if s.runRegistrar != nil { if err := s.runRegistrar.RegisterArchive(ctx, work, receipt); err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "register run archive") } } return receipt, nil } // PublishAction atomically writes a receipt_pending replay action summary. The // repository update is performed by ArchiveAction because it requires a fenced // replay lease. func (s *Service) PublishAction(ctx context.Context, action contract.ReplayAction) (contract.ArchiveReceipt, error) { normalized, err := normalizeAction(action) if err != nil { return contract.ArchiveReceipt{}, err } return s.publishJSON(ctx, "actions", normalized.ActionID, normalized.CreatedAt, normalized) } // ArchiveRun loads, publishes, and registers one durable run work item. func (s *Service) ArchiveRun(ctx context.Context, work contract.ArchiveWork) error { if s.runLoader == nil { return gerror.New("archive run loader is nil") } if s.runRegistrar == nil { return gerror.New("archive run registrar is nil") } snapshot, err := s.runLoader.LoadArchiveWork(ctx, work) if err != nil { s.recordRunFailure(ctx, work, err) return gerror.Wrap(err, "load run archive work") } if _, err = s.PublishRun(ctx, work, snapshot); err != nil { s.recordRunFailure(ctx, work, err) return err } return nil } // ArchiveAction acquires a replay lease, atomically publishes the action // summary, and fences the completed receipt update through ReplayProgressStore. func (s *Service) ArchiveAction(ctx context.Context, action contract.ReplayAction) error { if s.replay.Leases == nil || s.replay.Progress == nil { return gerror.New("replay archive repositories are incomplete") } if action.Status != contract.ActionReceiptPending { return gerror.Wrap(contract.ErrInvalidArgument, "replay action is not receipt_pending") } if _, err := normalizeAction(action); err != nil { return err } lease, err := s.replay.Leases.AcquireReplayLease(ctx, action.ActionID, s.leaseOwner, s.leaseTTL) if err != nil { return gerror.Wrap(err, "acquire replay archive lease") } defer func() { releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultReleaseTimeout) defer cancel() _ = s.replay.Leases.ReleaseReplayLease(releaseCtx, lease) }() receipt, err := s.PublishAction(ctx, action) if err != nil { return err } lease, err = s.replay.Leases.RenewReplayLease(ctx, lease, s.leaseTTL) if err != nil { return gerror.Wrap(err, "renew replay archive lease") } completedAt := receipt.ArchivedAt _, err = s.replay.Progress.UpdateReplayAction(ctx, contract.ReplayActionUpdate{ Lease: lease, ExpectedRevision: action.Revision, Status: contract.ActionCompleted, Progress: action.Progress, Receipt: &receipt, CompletedAt: &completedAt, }) if err != nil { return gerror.Wrap(err, "register replay action archive") } return nil } func (s *Service) scanLoop(ctx context.Context) { _ = s.RunOnce(ctx) ticker := time.NewTicker(s.interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: _ = s.RunOnce(ctx) } } } func (s *Service) scanRuns(ctx context.Context) error { page := contract.PageRequest{Limit: contract.DefaultPageLimit} seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}} var itemErrors []error for { if err := ctx.Err(); err != nil { return err } result, err := s.runDiscovery.ListArchiveWork(ctx, page) if err != nil { return err } for _, work := range result.Items { err := s.retry(ctx, func() error { return s.ArchiveRun(ctx, work) }) if err != nil { if ctx.Err() != nil { return ctx.Err() } itemErrors = append(itemErrors, gerror.Wrapf(err, "archive run %q", work.RunID)) } } if result.NextCursor == "" { return errors.Join(itemErrors...) } if _, repeated := seenCursors[result.NextCursor]; repeated { return errors.Join(append(itemErrors, gerror.New("run archive pagination cursor did not advance"))...) } seenCursors[result.NextCursor] = struct{}{} page.Cursor = result.NextCursor } } func (s *Service) scanActions(ctx context.Context) error { page := contract.PageRequest{Limit: contract.DefaultPageLimit} seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}} var itemErrors []error for { if err := ctx.Err(); err != nil { return err } result, err := s.replay.Discovery.ListReplayWork(ctx, page) if err != nil { return err } for _, action := range result.Items { if action.Status != contract.ActionReceiptPending { continue } err := s.retry(ctx, func() error { return s.ArchiveAction(ctx, action) }) if errors.Is(err, contract.ErrBusy) { continue } if err != nil { if ctx.Err() != nil { return ctx.Err() } itemErrors = append(itemErrors, gerror.Wrapf(err, "archive replay action %q", action.ActionID)) } } if result.NextCursor == "" { return errors.Join(itemErrors...) } if _, repeated := seenCursors[result.NextCursor]; repeated { return errors.Join(append(itemErrors, gerror.New("replay archive pagination cursor did not advance"))...) } seenCursors[result.NextCursor] = struct{}{} page.Cursor = result.NextCursor } } func (s *Service) retry(ctx context.Context, fn func() error) error { delay := s.initialBackoff var lastErr error for attempt := 0; attempt < s.maxAttempts; attempt++ { if err := ctx.Err(); err != nil { return err } lastErr = fn() if lastErr == nil || errors.Is(lastErr, contract.ErrBusy) { return lastErr } if attempt+1 == s.maxAttempts { break } timer := time.NewTimer(delay) select { case <-ctx.Done(): if !timer.Stop() { <-timer.C } return ctx.Err() case <-timer.C: } if delay <= s.maxBackoff/2 { delay *= 2 } else { delay = s.maxBackoff } } return lastErr } func (s *Service) recordRunFailure(ctx context.Context, work contract.ArchiveWork, archiveErr error) { if s.runRegistrar == nil || ctx.Err() != nil { return } _ = s.runRegistrar.RegisterArchiveFailure(ctx, work, contract.BoundedErrorMessage(archiveErr.Error())) } func (s *Service) publishJSON(ctx context.Context, kind, id string, pathTime time.Time, value any) (contract.ArchiveReceipt, error) { if ctx == nil { return contract.ArchiveReceipt{}, gerror.New("archive publication context is nil") } if err := ctx.Err(); err != nil { return contract.ArchiveReceipt{}, err } dir, final, relative, err := s.storagePath(kind, id, pathTime) if err != nil { return contract.ArchiveReceipt{}, err } s.publishMu.Lock() defer s.publishMu.Unlock() if err := os.MkdirAll(dir, 0o755); err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive directory") } if err := ctx.Err(); err != nil { return contract.ArchiveReceipt{}, err } temp, err := os.CreateTemp(dir, "."+id+"-*.tmp") if err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive temporary file") } tempName := temp.Name() defer func() { _ = os.Remove(tempName) }() hasher := sha256.New() buffer := bufio.NewWriter(io.MultiWriter(contextWriter{ctx: ctx, writer: temp}, hasher)) encoder := json.NewEncoder(buffer) if err = encoder.Encode(value); err == nil { err = buffer.Flush() } if err == nil { err = temp.Chmod(0o644) } if err == nil { err = temp.Sync() } closeErr := temp.Close() if err == nil { err = closeErr } if err != nil { _ = temp.Close() return contract.ArchiveReceipt{}, gerror.Wrap(err, "encode archive temporary file") } expectedChecksum := hex.EncodeToString(hasher.Sum(nil)) if receipt, ok, existingErr := s.existingReceipt(final, relative, kind, id, expectedChecksum); existingErr != nil { return contract.ArchiveReceipt{}, existingErr } else if ok { if err := syncDirectory(dir); err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory") } return receipt, nil } if err := ctx.Err(); err != nil { return contract.ArchiveReceipt{}, err } if err := s.replaceFile(tempName, final); err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "publish archive file") } if err := syncDirectory(dir); err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory") } if err := s.afterReplace(final); err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "archive post-publish verification hook") } stored, err := os.ReadFile(final) if err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "read published archive file") } storedSum := sha256.Sum256(stored) if hex.EncodeToString(storedSum[:]) != expectedChecksum { return contract.ArchiveReceipt{}, errChecksumMismatch } info, err := os.Stat(final) if err != nil { return contract.ArchiveReceipt{}, gerror.Wrap(err, "stat published archive file") } receipt := contract.ArchiveReceipt{ Version: contract.ArchiveSchemaVersion, Path: relative, Checksum: expectedChecksum, ArchivedAt: archiveModTime(info, s.now), } if err := receipt.Validate(); err != nil { return contract.ArchiveReceipt{}, err } return receipt, nil } func (s *Service) existingReceipt(final, relative, kind, id, expectedChecksum string) (contract.ArchiveReceipt, bool, error) { info, err := os.Lstat(final) if errors.Is(err, os.ErrNotExist) { return contract.ArchiveReceipt{}, false, nil } if err != nil { return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "inspect existing archive file") } if !info.Mode().IsRegular() { return contract.ArchiveReceipt{}, false, nil } stored, err := os.ReadFile(final) if err != nil { return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "read existing archive file") } storedSum := sha256.Sum256(stored) if hex.EncodeToString(storedSum[:]) != expectedChecksum { if validExistingArchive(kind, id, stored) { return contract.ArchiveReceipt{}, false, gerror.Wrap(contract.ErrConflict, "archive path already contains different valid content") } return contract.ArchiveReceipt{}, false, nil } receipt := contract.ArchiveReceipt{ Version: contract.ArchiveSchemaVersion, Path: relative, Checksum: expectedChecksum, ArchivedAt: archiveModTime(info, s.now), } if err := receipt.Validate(); err != nil { return contract.ArchiveReceipt{}, false, err } return receipt, true, nil } func validExistingArchive(kind, id string, data []byte) bool { switch kind { case "runs": var snapshot contract.RunArchive if json.Unmarshal(data, &snapshot) != nil { return false } _, err := normalizeRunArchive(contract.ArchiveWork{RunID: id, Version: "existing"}, snapshot) return err == nil case "actions": var action contract.ReplayAction if json.Unmarshal(data, &action) != nil || action.ActionID != id { return false } _, err := normalizeAction(action) return err == nil default: return false } } func archiveModTime(info os.FileInfo, now func() time.Time) time.Time { archivedAt := info.ModTime().UTC() if archivedAt.IsZero() { archivedAt = now().UTC() } return archivedAt } func (s *Service) storagePath(kind, id string, pathTime time.Time) (string, string, string, error) { if err := validateStorageID(id); err != nil { return "", "", "", err } if kind != "runs" && kind != "actions" { return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "unknown archive kind") } if strings.TrimSpace(s.root) == "" { return "", "", "", gerror.New("archive root is empty") } if pathTime.IsZero() { pathTime = s.now().UTC() } else { pathTime = pathTime.UTC() } if pathTime.Year() < 1 || pathTime.Year() > 9999 { return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive timestamp is outside the supported year range") } root := filepath.Clean(s.root) prefix := "run-" if kind == "actions" { prefix = "action-" } relativeOS := prefix + id + ".json" final := filepath.Join(root, relativeOS) contained, err := filepath.Rel(root, final) if err != nil || filepath.IsAbs(contained) || contained == ".." || strings.HasPrefix(contained, ".."+string(filepath.Separator)) { return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive path escapes root") } return filepath.Dir(final), final, filepath.ToSlash(relativeOS), nil } func (s *Service) validateConfiguration() error { if strings.TrimSpace(s.root) == "" { return gerror.New("archive root is empty") } if s.runDiscovery != nil && (s.runLoader == nil || s.runRegistrar == nil) { return gerror.New("run archive repositories are incomplete") } if s.replay.Discovery != nil && (s.replay.Leases == nil || s.replay.Progress == nil) { return gerror.New("replay archive repositories are incomplete") } return nil } func validateWork(work contract.ArchiveWork) error { if err := validateStorageID(work.RunID); err != nil { return err } if strings.TrimSpace(work.Version) == "" { return gerror.Wrap(contract.ErrInvalidArgument, "archive work version is required") } return nil } func validateStorageID(id string) error { if err := contract.ValidateID(id); err != nil { return err } if id == "." || id == ".." || filepath.VolumeName(id) != "" || strings.ContainsRune(id, ':') || filepath.Base(id) != id { return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is not safe for archive storage") } name := strings.ToUpper(strings.TrimSuffix(id, filepath.Ext(id))) switch name { case "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9": return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is reserved by the filesystem") } return nil } func normalizeRunArchive(work contract.ArchiveWork, snapshot contract.RunArchive) (contract.RunArchive, error) { if snapshot.Version == 0 { snapshot.Version = contract.ArchiveSchemaVersion } if snapshot.Version != contract.ArchiveSchemaVersion { return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "invalid archive schema version") } if snapshot.Run.RunID != work.RunID { return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archive run ID does not match work") } if !snapshot.Run.Status.Terminal() { return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "only terminal runs can be archived") } if err := snapshot.Run.Validate(); err != nil { return contract.RunArchive{}, err } if len(snapshot.Rounds) > maxRounds { return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "too many archived rounds") } snapshot.Run = normalizeRunMeta(snapshot.Run) snapshot.Rounds = append([]contract.RoundStats(nil), snapshot.Rounds...) if snapshot.Rounds == nil { snapshot.Rounds = make([]contract.RoundStats, 0) } seen := make(map[int]struct{}, len(snapshot.Rounds)) for i := range snapshot.Rounds { round := &snapshot.Rounds[i] if round.RunID != work.RunID { return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archived round belongs to another run") } if _, duplicate := seen[round.Round]; duplicate { return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "duplicate archived round") } seen[round.Round] = struct{}{} if err := round.Validate(); err != nil { return contract.RunArchive{}, err } normalizedCounts, err := round.ErrorCounts.Normalized() if err != nil { return contract.RunArchive{}, err } round.ErrorCounts = normalizedCounts round.StartedAt = utcTimePtr(round.StartedAt) round.CompletedAt = utcTimePtr(round.CompletedAt) } sort.Slice(snapshot.Rounds, func(i, j int) bool { return snapshot.Rounds[i].Round < snapshot.Rounds[j].Round }) return snapshot, nil } func normalizeRunMeta(meta contract.RunMeta) contract.RunMeta { meta.CreatedAt = meta.CreatedAt.UTC() meta.ProducerCompletedAt = meta.ProducerCompletedAt.UTC() meta.CompletedAt = meta.CompletedAt.UTC() meta.StartedAt = utcTimePtr(meta.StartedAt) meta.ScanCompletedAt = utcTimePtr(meta.ScanCompletedAt) // Archive state describes the repository registration and is intentionally // excluded from the file whose receipt populates that state. meta.ArchiveStatus = "" meta.ArchiveVersion = 0 meta.ArchivePath = "" meta.ArchiveChecksum = "" meta.ArchivedAt = nil return meta } func normalizeAction(action contract.ReplayAction) (contract.ReplayAction, error) { if err := validateStorageID(action.ActionID); err != nil { return contract.ReplayAction{}, err } if action.Status != contract.ActionReceiptPending { return contract.ReplayAction{}, gerror.Wrap(contract.ErrInvalidArgument, "only receipt_pending replay actions can be archived") } action.Error = contract.BoundedErrorMessage(action.Error) action.CreatedAt = action.CreatedAt.UTC() action.UpdatedAt = action.UpdatedAt.UTC() action.CompletedAt = nil action.Receipt = nil if err := action.Validate(); err != nil { return contract.ReplayAction{}, err } return action, nil } func utcTimePtr(value *time.Time) *time.Time { if value == nil { return nil } utc := value.UTC() return &utc } type contextWriter struct { ctx context.Context writer io.Writer } func (w contextWriter) Write(p []byte) (int, error) { if err := w.ctx.Err(); err != nil { return 0, err } return w.writer.Write(p) }