iotlcm/internal/fullsync/archive/archive.go

// Package archive durably stores summary-only receipts for completed full-sync
// runs and replay actions. It depends only on the shared full-sync contracts;
// repository implementations and service wiring live elsewhere.
package archive

import (
	"bufio"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/gogf/gf/v2/errors/gerror"

	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
)

const (
	defaultInterval        = time.Minute
	defaultBackoff         = time.Second
	defaultMaxBackoff      = time.Minute
	defaultMaxAttempts     = 4
	defaultReplayLeaseTTL  = time.Minute
	defaultReplayLeaseName = "fullsync-archive"
	defaultReleaseTimeout  = 5 * time.Second
	maxRounds              = 10000
)

var errChecksumMismatch = gerror.New("archive checksum verification failed")

// Config controls the recovery scanner. Root is the archive directory. Zero
// timing values select conservative defaults.
type Config struct {
	Root             string
	Interval         time.Duration
	InitialBackoff   time.Duration
	MaxBackoff       time.Duration
	MaxAttempts      int
	ReplayLeaseOwner string
	ReplayLeaseTTL   time.Duration
	Now              func() time.Time
}

// ReplayRepositories are the contract boundaries used to recover replay
// actions left in receipt_pending after a process restart.
type ReplayRepositories struct {
	Discovery contract.ReplayWorkDiscovery
	Leases    contract.ReplayLeaseStore
	Progress  contract.ReplayProgressStore
}

// Service publishes archive files and scans durable repository indexes at
// startup and periodically. A Service may be configured for runs, actions, or
// both.
type Service struct {
	root           string
	runDiscovery   contract.ArchiveWorkDiscovery
	runLoader      contract.ArchiveWorkLoader
	runRegistrar   contract.ArchiveRegistrar
	replay         ReplayRepositories
	interval       time.Duration
	initialBackoff time.Duration
	maxBackoff     time.Duration
	maxAttempts    int
	leaseOwner     string
	leaseTTL       time.Duration
	now            func() time.Time

	// Publications are serialized so repeat writes in this process observe the
	// same final file timestamp and therefore produce the same receipt.
	publishMu sync.Mutex

	mu      sync.Mutex
	running bool
	cancel  context.CancelFunc
	done    chan struct{}

	// These hooks keep failure-path tests deterministic without weakening the
	// production filesystem path.
	replaceFile  func(string, string) error
	afterReplace func(string) error
}

// New constructs a run-archive service with default scanner settings.
func New(root string, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
	return NewService(Config{Root: root}, discovery, loader, registrar)
}

// NewService constructs a run-archive service. Use NewServiceWithReplay when
// replay receipt recovery is also required.
func NewService(cfg Config, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
	return NewServiceWithReplay(cfg, discovery, loader, registrar, ReplayRepositories{})
}

// NewServiceWithReplay constructs a lifecycle-enabled run and replay archive
// service using only shared contract interfaces.
func NewServiceWithReplay(
	cfg Config,
	discovery contract.ArchiveWorkDiscovery,
	loader contract.ArchiveWorkLoader,
	registrar contract.ArchiveRegistrar,
	replay ReplayRepositories,
) *Service {
	if cfg.Interval <= 0 {
		cfg.Interval = defaultInterval
	}
	if cfg.InitialBackoff <= 0 {
		cfg.InitialBackoff = defaultBackoff
	}
	if cfg.MaxBackoff <= 0 {
		cfg.MaxBackoff = defaultMaxBackoff
	}
	if cfg.MaxBackoff < cfg.InitialBackoff {
		cfg.MaxBackoff = cfg.InitialBackoff
	}
	if cfg.MaxAttempts <= 0 {
		cfg.MaxAttempts = defaultMaxAttempts
	}
	if strings.TrimSpace(cfg.ReplayLeaseOwner) == "" {
		cfg.ReplayLeaseOwner = defaultReplayLeaseName
	}
	if cfg.ReplayLeaseTTL <= 0 {
		cfg.ReplayLeaseTTL = defaultReplayLeaseTTL
	}
	if cfg.Now == nil {
		cfg.Now = time.Now
	}

	return &Service{
		root:           cfg.Root,
		runDiscovery:   discovery,
		runLoader:      loader,
		runRegistrar:   registrar,
		replay:         replay,
		interval:       cfg.Interval,
		initialBackoff: cfg.InitialBackoff,
		maxBackoff:     cfg.MaxBackoff,
		maxAttempts:    cfg.MaxAttempts,
		leaseOwner:     cfg.ReplayLeaseOwner,
		leaseTTL:       cfg.ReplayLeaseTTL,
		now:            cfg.Now,
		replaceFile:    atomicReplace,
		afterReplace:   func(string) error { return nil },
	}
}

// Start launches an immediate recovery scan followed by periodic scans. It is
// idempotent while the service is running.
func (s *Service) Start(parent context.Context) error {
	if parent == nil {
		return gerror.New("archive parent context is nil")
	}
	if err := s.validateConfiguration(); err != nil {
		return err
	}

	s.mu.Lock()
	if s.running {
		s.mu.Unlock()
		return nil
	}
	ctx, cancel := context.WithCancel(parent)
	done := make(chan struct{})
	s.cancel, s.done, s.running = cancel, done, true
	s.mu.Unlock()

	go func() {
		defer func() {
			s.mu.Lock()
			if s.done == done {
				s.running = false
				s.cancel = nil
			}
			s.mu.Unlock()
			close(done)
		}()
		s.scanLoop(ctx)
	}()
	return nil
}

// Stop cancels the scanner and waits for it to finish. It is safe to call more
// than once.
func (s *Service) Stop() error {
	s.mu.Lock()
	if !s.running {
		s.mu.Unlock()
		return nil
	}
	cancel, done := s.cancel, s.done
	s.mu.Unlock()

	cancel()
	<-done
	return nil
}

// Close implements the common service lifecycle convention.
func (s *Service) Close() error { return s.Stop() }

// Done is closed when the current scanner invocation exits. It returns nil
// before the service has been started.
func (s *Service) Done() <-chan struct{} {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.done
}

// RunOnce performs one recovery pass. It is useful for explicit lifecycle
// wiring and deterministic integration tests.
func (s *Service) RunOnce(ctx context.Context) error {
	if ctx == nil {
		return gerror.New("archive scan context is nil")
	}
	if err := s.validateConfiguration(); err != nil {
		return err
	}

	var scanErrors []error
	if s.runDiscovery != nil {
		if err := s.scanRuns(ctx); err != nil {
			scanErrors = append(scanErrors, gerror.Wrap(err, "scan run archives"))
		}
	}
	if s.replay.Discovery != nil && ctx.Err() == nil {
		if err := s.scanActions(ctx); err != nil {
			scanErrors = append(scanErrors, gerror.Wrap(err, "scan replay action archives"))
		}
	}
	return errors.Join(scanErrors...)
}

// PublishRun atomically writes a run summary and then conditionally registers
// the durable receipt against the work version.
func (s *Service) PublishRun(ctx context.Context, work contract.ArchiveWork, snapshot contract.RunArchive) (contract.ArchiveReceipt, error) {
	if err := validateWork(work); err != nil {
		return contract.ArchiveReceipt{}, err
	}
	normalized, err := normalizeRunArchive(work, snapshot)
	if err != nil {
		return contract.ArchiveReceipt{}, err
	}

	pathTime := normalized.Run.CompletedAt
	if pathTime.IsZero() {
		pathTime = normalized.Run.CreatedAt
	}
	receipt, err := s.publishJSON(ctx, "runs", work.RunID, pathTime, normalized)
	if err != nil {
		return contract.ArchiveReceipt{}, err
	}
	if s.runRegistrar != nil {
		if err := s.runRegistrar.RegisterArchive(ctx, work, receipt); err != nil {
			return contract.ArchiveReceipt{}, gerror.Wrap(err, "register run archive")
		}
	}
	return receipt, nil
}

// PublishAction atomically writes a receipt_pending replay action summary. The
// repository update is performed by ArchiveAction because it requires a fenced
// replay lease.
func (s *Service) PublishAction(ctx context.Context, action contract.ReplayAction) (contract.ArchiveReceipt, error) {
	normalized, err := normalizeAction(action)
	if err != nil {
		return contract.ArchiveReceipt{}, err
	}
	return s.publishJSON(ctx, "actions", normalized.ActionID, normalized.CreatedAt, normalized)
}

// ArchiveRun loads, publishes, and registers one durable run work item.
func (s *Service) ArchiveRun(ctx context.Context, work contract.ArchiveWork) error {
	if s.runLoader == nil {
		return gerror.New("archive run loader is nil")
	}
	if s.runRegistrar == nil {
		return gerror.New("archive run registrar is nil")
	}

	snapshot, err := s.runLoader.LoadArchiveWork(ctx, work)
	if err != nil {
		s.recordRunFailure(ctx, work, err)
		return gerror.Wrap(err, "load run archive work")
	}
	if _, err = s.PublishRun(ctx, work, snapshot); err != nil {
		s.recordRunFailure(ctx, work, err)
		return err
	}
	return nil
}

// ArchiveAction acquires a replay lease, atomically publishes the action
// summary, and fences the completed receipt update through ReplayProgressStore.
func (s *Service) ArchiveAction(ctx context.Context, action contract.ReplayAction) error {
	if s.replay.Leases == nil || s.replay.Progress == nil {
		return gerror.New("replay archive repositories are incomplete")
	}
	if action.Status != contract.ActionReceiptPending {
		return gerror.Wrap(contract.ErrInvalidArgument, "replay action is not receipt_pending")
	}
	if _, err := normalizeAction(action); err != nil {
		return err
	}

	lease, err := s.replay.Leases.AcquireReplayLease(ctx, action.ActionID, s.leaseOwner, s.leaseTTL)
	if err != nil {
		return gerror.Wrap(err, "acquire replay archive lease")
	}
	defer func() {
		releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultReleaseTimeout)
		defer cancel()
		_ = s.replay.Leases.ReleaseReplayLease(releaseCtx, lease)
	}()

	receipt, err := s.PublishAction(ctx, action)
	if err != nil {
		return err
	}
	lease, err = s.replay.Leases.RenewReplayLease(ctx, lease, s.leaseTTL)
	if err != nil {
		return gerror.Wrap(err, "renew replay archive lease")
	}
	completedAt := receipt.ArchivedAt
	_, err = s.replay.Progress.UpdateReplayAction(ctx, contract.ReplayActionUpdate{
		Lease:            lease,
		ExpectedRevision: action.Revision,
		Status:           contract.ActionCompleted,
		Progress:         action.Progress,
		Receipt:          &receipt,
		CompletedAt:      &completedAt,
	})
	if err != nil {
		return gerror.Wrap(err, "register replay action archive")
	}
	return nil
}

func (s *Service) scanLoop(ctx context.Context) {
	_ = s.RunOnce(ctx)
	ticker := time.NewTicker(s.interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			_ = s.RunOnce(ctx)
		}
	}
}

func (s *Service) scanRuns(ctx context.Context) error {
	page := contract.PageRequest{Limit: contract.DefaultPageLimit}
	seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
	var itemErrors []error
	for {
		if err := ctx.Err(); err != nil {
			return err
		}
		result, err := s.runDiscovery.ListArchiveWork(ctx, page)
		if err != nil {
			return err
		}
		for _, work := range result.Items {
			err := s.retry(ctx, func() error { return s.ArchiveRun(ctx, work) })
			if err != nil {
				if ctx.Err() != nil {
					return ctx.Err()
				}
				itemErrors = append(itemErrors, gerror.Wrapf(err, "archive run %q", work.RunID))
			}
		}
		if result.NextCursor == "" {
			return errors.Join(itemErrors...)
		}
		if _, repeated := seenCursors[result.NextCursor]; repeated {
			return errors.Join(append(itemErrors, gerror.New("run archive pagination cursor did not advance"))...)
		}
		seenCursors[result.NextCursor] = struct{}{}
		page.Cursor = result.NextCursor
	}
}

func (s *Service) scanActions(ctx context.Context) error {
	page := contract.PageRequest{Limit: contract.DefaultPageLimit}
	seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
	var itemErrors []error
	for {
		if err := ctx.Err(); err != nil {
			return err
		}
		result, err := s.replay.Discovery.ListReplayWork(ctx, page)
		if err != nil {
			return err
		}
		for _, action := range result.Items {
			if action.Status != contract.ActionReceiptPending {
				continue
			}
			err := s.retry(ctx, func() error { return s.ArchiveAction(ctx, action) })
			if errors.Is(err, contract.ErrBusy) {
				continue
			}
			if err != nil {
				if ctx.Err() != nil {
					return ctx.Err()
				}
				itemErrors = append(itemErrors, gerror.Wrapf(err, "archive replay action %q", action.ActionID))
			}
		}
		if result.NextCursor == "" {
			return errors.Join(itemErrors...)
		}
		if _, repeated := seenCursors[result.NextCursor]; repeated {
			return errors.Join(append(itemErrors, gerror.New("replay archive pagination cursor did not advance"))...)
		}
		seenCursors[result.NextCursor] = struct{}{}
		page.Cursor = result.NextCursor
	}
}

func (s *Service) retry(ctx context.Context, fn func() error) error {
	delay := s.initialBackoff
	var lastErr error
	for attempt := 0; attempt < s.maxAttempts; attempt++ {
		if err := ctx.Err(); err != nil {
			return err
		}
		lastErr = fn()
		if lastErr == nil || errors.Is(lastErr, contract.ErrBusy) {
			return lastErr
		}
		if attempt+1 == s.maxAttempts {
			break
		}
		timer := time.NewTimer(delay)
		select {
		case <-ctx.Done():
			if !timer.Stop() {
				<-timer.C
			}
			return ctx.Err()
		case <-timer.C:
		}
		if delay <= s.maxBackoff/2 {
			delay *= 2
		} else {
			delay = s.maxBackoff
		}
	}
	return lastErr
}

func (s *Service) recordRunFailure(ctx context.Context, work contract.ArchiveWork, archiveErr error) {
	if s.runRegistrar == nil || ctx.Err() != nil {
		return
	}
	_ = s.runRegistrar.RegisterArchiveFailure(ctx, work, contract.BoundedErrorMessage(archiveErr.Error()))
}

func (s *Service) publishJSON(ctx context.Context, kind, id string, pathTime time.Time, value any) (contract.ArchiveReceipt, error) {
	if ctx == nil {
		return contract.ArchiveReceipt{}, gerror.New("archive publication context is nil")
	}
	if err := ctx.Err(); err != nil {
		return contract.ArchiveReceipt{}, err
	}
	dir, final, relative, err := s.storagePath(kind, id, pathTime)
	if err != nil {
		return contract.ArchiveReceipt{}, err
	}

	s.publishMu.Lock()
	defer s.publishMu.Unlock()

	if err := os.MkdirAll(dir, 0o755); err != nil {
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive directory")
	}
	if err := ctx.Err(); err != nil {
		return contract.ArchiveReceipt{}, err
	}

	temp, err := os.CreateTemp(dir, "."+id+"-*.tmp")
	if err != nil {
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive temporary file")
	}
	tempName := temp.Name()
	defer func() { _ = os.Remove(tempName) }()

	hasher := sha256.New()
	buffer := bufio.NewWriter(io.MultiWriter(contextWriter{ctx: ctx, writer: temp}, hasher))
	encoder := json.NewEncoder(buffer)
	if err = encoder.Encode(value); err == nil {
		err = buffer.Flush()
	}
	if err == nil {
		err = temp.Chmod(0o644)
	}
	if err == nil {
		err = temp.Sync()
	}
	closeErr := temp.Close()
	if err == nil {
		err = closeErr
	}
	if err != nil {
		_ = temp.Close()
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "encode archive temporary file")
	}
	expectedChecksum := hex.EncodeToString(hasher.Sum(nil))

	if receipt, ok, existingErr := s.existingReceipt(final, relative, kind, id, expectedChecksum); existingErr != nil {
		return contract.ArchiveReceipt{}, existingErr
	} else if ok {
		if err := syncDirectory(dir); err != nil {
			return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
		}
		return receipt, nil
	}

	if err := ctx.Err(); err != nil {
		return contract.ArchiveReceipt{}, err
	}
	if err := s.replaceFile(tempName, final); err != nil {
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "publish archive file")
	}
	if err := syncDirectory(dir); err != nil {
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
	}
	if err := s.afterReplace(final); err != nil {
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "archive post-publish verification hook")
	}

	stored, err := os.ReadFile(final)
	if err != nil {
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "read published archive file")
	}
	storedSum := sha256.Sum256(stored)
	if hex.EncodeToString(storedSum[:]) != expectedChecksum {
		return contract.ArchiveReceipt{}, errChecksumMismatch
	}
	info, err := os.Stat(final)
	if err != nil {
		return contract.ArchiveReceipt{}, gerror.Wrap(err, "stat published archive file")
	}
	receipt := contract.ArchiveReceipt{
		Version:    contract.ArchiveSchemaVersion,
		Path:       relative,
		Checksum:   expectedChecksum,
		ArchivedAt: archiveModTime(info, s.now),
	}
	if err := receipt.Validate(); err != nil {
		return contract.ArchiveReceipt{}, err
	}
	return receipt, nil
}

func (s *Service) existingReceipt(final, relative, kind, id, expectedChecksum string) (contract.ArchiveReceipt, bool, error) {
	info, err := os.Lstat(final)
	if errors.Is(err, os.ErrNotExist) {
		return contract.ArchiveReceipt{}, false, nil
	}
	if err != nil {
		return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "inspect existing archive file")
	}
	if !info.Mode().IsRegular() {
		return contract.ArchiveReceipt{}, false, nil
	}
	stored, err := os.ReadFile(final)
	if err != nil {
		return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "read existing archive file")
	}
	storedSum := sha256.Sum256(stored)
	if hex.EncodeToString(storedSum[:]) != expectedChecksum {
		if validExistingArchive(kind, id, stored) {
			return contract.ArchiveReceipt{}, false, gerror.Wrap(contract.ErrConflict, "archive path already contains different valid content")
		}
		return contract.ArchiveReceipt{}, false, nil
	}
	receipt := contract.ArchiveReceipt{
		Version:    contract.ArchiveSchemaVersion,
		Path:       relative,
		Checksum:   expectedChecksum,
		ArchivedAt: archiveModTime(info, s.now),
	}
	if err := receipt.Validate(); err != nil {
		return contract.ArchiveReceipt{}, false, err
	}
	return receipt, true, nil
}

func validExistingArchive(kind, id string, data []byte) bool {
	switch kind {
	case "runs":
		var snapshot contract.RunArchive
		if json.Unmarshal(data, &snapshot) != nil {
			return false
		}
		_, err := normalizeRunArchive(contract.ArchiveWork{RunID: id, Version: "existing"}, snapshot)
		return err == nil
	case "actions":
		var action contract.ReplayAction
		if json.Unmarshal(data, &action) != nil || action.ActionID != id {
			return false
		}
		_, err := normalizeAction(action)
		return err == nil
	default:
		return false
	}
}

func archiveModTime(info os.FileInfo, now func() time.Time) time.Time {
	archivedAt := info.ModTime().UTC()
	if archivedAt.IsZero() {
		archivedAt = now().UTC()
	}
	return archivedAt
}

func (s *Service) storagePath(kind, id string, pathTime time.Time) (string, string, string, error) {
	if err := validateStorageID(id); err != nil {
		return "", "", "", err
	}
	if kind != "runs" && kind != "actions" {
		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "unknown archive kind")
	}
	if strings.TrimSpace(s.root) == "" {
		return "", "", "", gerror.New("archive root is empty")
	}
	if pathTime.IsZero() {
		pathTime = s.now().UTC()
	} else {
		pathTime = pathTime.UTC()
	}
	if pathTime.Year() < 1 || pathTime.Year() > 9999 {
		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive timestamp is outside the supported year range")
	}

	root := filepath.Clean(s.root)
	prefix := "run-"
	if kind == "actions" {
		prefix = "action-"
	}
	relativeOS := prefix + id + ".json"
	final := filepath.Join(root, relativeOS)
	contained, err := filepath.Rel(root, final)
	if err != nil || filepath.IsAbs(contained) || contained == ".." || strings.HasPrefix(contained, ".."+string(filepath.Separator)) {
		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive path escapes root")
	}
	return filepath.Dir(final), final, filepath.ToSlash(relativeOS), nil
}

func (s *Service) validateConfiguration() error {
	if strings.TrimSpace(s.root) == "" {
		return gerror.New("archive root is empty")
	}
	if s.runDiscovery != nil && (s.runLoader == nil || s.runRegistrar == nil) {
		return gerror.New("run archive repositories are incomplete")
	}
	if s.replay.Discovery != nil && (s.replay.Leases == nil || s.replay.Progress == nil) {
		return gerror.New("replay archive repositories are incomplete")
	}
	return nil
}

func validateWork(work contract.ArchiveWork) error {
	if err := validateStorageID(work.RunID); err != nil {
		return err
	}
	if strings.TrimSpace(work.Version) == "" {
		return gerror.Wrap(contract.ErrInvalidArgument, "archive work version is required")
	}
	return nil
}

func validateStorageID(id string) error {
	if err := contract.ValidateID(id); err != nil {
		return err
	}
	if id == "." || id == ".." || filepath.VolumeName(id) != "" || strings.ContainsRune(id, ':') || filepath.Base(id) != id {
		return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is not safe for archive storage")
	}
	name := strings.ToUpper(strings.TrimSuffix(id, filepath.Ext(id)))
	switch name {
	case "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9":
		return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is reserved by the filesystem")
	}
	return nil
}

func normalizeRunArchive(work contract.ArchiveWork, snapshot contract.RunArchive) (contract.RunArchive, error) {
	if snapshot.Version == 0 {
		snapshot.Version = contract.ArchiveSchemaVersion
	}
	if snapshot.Version != contract.ArchiveSchemaVersion {
		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "invalid archive schema version")
	}
	if snapshot.Run.RunID != work.RunID {
		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archive run ID does not match work")
	}
	if !snapshot.Run.Status.Terminal() {
		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "only terminal runs can be archived")
	}
	if err := snapshot.Run.Validate(); err != nil {
		return contract.RunArchive{}, err
	}
	if len(snapshot.Rounds) > maxRounds {
		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "too many archived rounds")
	}

	snapshot.Run = normalizeRunMeta(snapshot.Run)
	snapshot.Rounds = append([]contract.RoundStats(nil), snapshot.Rounds...)
	if snapshot.Rounds == nil {
		snapshot.Rounds = make([]contract.RoundStats, 0)
	}
	seen := make(map[int]struct{}, len(snapshot.Rounds))
	for i := range snapshot.Rounds {
		round := &snapshot.Rounds[i]
		if round.RunID != work.RunID {
			return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archived round belongs to another run")
		}
		if _, duplicate := seen[round.Round]; duplicate {
			return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "duplicate archived round")
		}
		seen[round.Round] = struct{}{}
		if err := round.Validate(); err != nil {
			return contract.RunArchive{}, err
		}
		normalizedCounts, err := round.ErrorCounts.Normalized()
		if err != nil {
			return contract.RunArchive{}, err
		}
		round.ErrorCounts = normalizedCounts
		round.StartedAt = utcTimePtr(round.StartedAt)
		round.CompletedAt = utcTimePtr(round.CompletedAt)
	}
	sort.Slice(snapshot.Rounds, func(i, j int) bool { return snapshot.Rounds[i].Round < snapshot.Rounds[j].Round })
	return snapshot, nil
}

func normalizeRunMeta(meta contract.RunMeta) contract.RunMeta {
	meta.CreatedAt = meta.CreatedAt.UTC()
	meta.ProducerCompletedAt = meta.ProducerCompletedAt.UTC()
	meta.CompletedAt = meta.CompletedAt.UTC()
	meta.StartedAt = utcTimePtr(meta.StartedAt)
	meta.ScanCompletedAt = utcTimePtr(meta.ScanCompletedAt)
	// Archive state describes the repository registration and is intentionally
	// excluded from the file whose receipt populates that state.
	meta.ArchiveStatus = ""
	meta.ArchiveVersion = 0
	meta.ArchivePath = ""
	meta.ArchiveChecksum = ""
	meta.ArchivedAt = nil
	return meta
}

func normalizeAction(action contract.ReplayAction) (contract.ReplayAction, error) {
	if err := validateStorageID(action.ActionID); err != nil {
		return contract.ReplayAction{}, err
	}
	if action.Status != contract.ActionReceiptPending {
		return contract.ReplayAction{}, gerror.Wrap(contract.ErrInvalidArgument, "only receipt_pending replay actions can be archived")
	}
	action.Error = contract.BoundedErrorMessage(action.Error)
	action.CreatedAt = action.CreatedAt.UTC()
	action.UpdatedAt = action.UpdatedAt.UTC()
	action.CompletedAt = nil
	action.Receipt = nil
	if err := action.Validate(); err != nil {
		return contract.ReplayAction{}, err
	}
	return action, nil
}

func utcTimePtr(value *time.Time) *time.Time {
	if value == nil {
		return nil
	}
	utc := value.UTC()
	return &utc
}

type contextWriter struct {
	ctx    context.Context
	writer io.Writer
}

func (w contextWriter) Write(p []byte) (int, error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}
	return w.writer.Write(p)
}