chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）

worker-svr/internal/fullsync/archive/archive.go

@@ -0,0 +1,802 @@
+// Package archive durably stores summary-only receipts for completed full-sync
+// runs and replay actions. It depends only on the shared full-sync contracts;
+// repository implementations and service wiring live elsewhere.
+package archive
+
+import (
+	"bufio"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+const (
+	defaultInterval        = time.Minute
+	defaultBackoff         = time.Second
+	defaultMaxBackoff      = time.Minute
+	defaultMaxAttempts     = 4
+	defaultReplayLeaseTTL  = time.Minute
+	defaultReplayLeaseName = "fullsync-archive"
+	defaultReleaseTimeout  = 5 * time.Second
+	maxRounds              = 10000
+)
+
+var errChecksumMismatch = gerror.New("archive checksum verification failed")
+
+// Config controls the recovery scanner. Root is the archive directory. Zero
+// timing values select conservative defaults.
+type Config struct {
+	Root             string
+	Interval         time.Duration
+	InitialBackoff   time.Duration
+	MaxBackoff       time.Duration
+	MaxAttempts      int
+	ReplayLeaseOwner string
+	ReplayLeaseTTL   time.Duration
+	Now              func() time.Time
+}
+
+// ReplayRepositories are the contract boundaries used to recover replay
+// actions left in receipt_pending after a process restart.
+type ReplayRepositories struct {
+	Discovery contract.ReplayWorkDiscovery
+	Leases    contract.ReplayLeaseStore
+	Progress  contract.ReplayProgressStore
+}
+
+// Service publishes archive files and scans durable repository indexes at
+// startup and periodically. A Service may be configured for runs, actions, or
+// both.
+type Service struct {
+	root           string
+	runDiscovery   contract.ArchiveWorkDiscovery
+	runLoader      contract.ArchiveWorkLoader
+	runRegistrar   contract.ArchiveRegistrar
+	replay         ReplayRepositories
+	interval       time.Duration
+	initialBackoff time.Duration
+	maxBackoff     time.Duration
+	maxAttempts    int
+	leaseOwner     string
+	leaseTTL       time.Duration
+	now            func() time.Time
+
+	// Publications are serialized so repeat writes in this process observe the
+	// same final file timestamp and therefore produce the same receipt.
+	publishMu sync.Mutex
+
+	mu      sync.Mutex
+	running bool
+	cancel  context.CancelFunc
+	done    chan struct{}
+
+	// These hooks keep failure-path tests deterministic without weakening the
+	// production filesystem path.
+	replaceFile  func(string, string) error
+	afterReplace func(string) error
+}
+
+// New constructs a run-archive service with default scanner settings.
+func New(root string, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
+	return NewService(Config{Root: root}, discovery, loader, registrar)
+}
+
+// NewService constructs a run-archive service. Use NewServiceWithReplay when
+// replay receipt recovery is also required.
+func NewService(cfg Config, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
+	return NewServiceWithReplay(cfg, discovery, loader, registrar, ReplayRepositories{})
+}
+
+// NewServiceWithReplay constructs a lifecycle-enabled run and replay archive
+// service using only shared contract interfaces.
+func NewServiceWithReplay(
+	cfg Config,
+	discovery contract.ArchiveWorkDiscovery,
+	loader contract.ArchiveWorkLoader,
+	registrar contract.ArchiveRegistrar,
+	replay ReplayRepositories,
+) *Service {
+	if cfg.Interval <= 0 {
+		cfg.Interval = defaultInterval
+	}
+	if cfg.InitialBackoff <= 0 {
+		cfg.InitialBackoff = defaultBackoff
+	}
+	if cfg.MaxBackoff <= 0 {
+		cfg.MaxBackoff = defaultMaxBackoff
+	}
+	if cfg.MaxBackoff < cfg.InitialBackoff {
+		cfg.MaxBackoff = cfg.InitialBackoff
+	}
+	if cfg.MaxAttempts <= 0 {
+		cfg.MaxAttempts = defaultMaxAttempts
+	}
+	if strings.TrimSpace(cfg.ReplayLeaseOwner) == "" {
+		cfg.ReplayLeaseOwner = defaultReplayLeaseName
+	}
+	if cfg.ReplayLeaseTTL <= 0 {
+		cfg.ReplayLeaseTTL = defaultReplayLeaseTTL
+	}
+	if cfg.Now == nil {
+		cfg.Now = time.Now
+	}
+
+	return &Service{
+		root:           cfg.Root,
+		runDiscovery:   discovery,
+		runLoader:      loader,
+		runRegistrar:   registrar,
+		replay:         replay,
+		interval:       cfg.Interval,
+		initialBackoff: cfg.InitialBackoff,
+		maxBackoff:     cfg.MaxBackoff,
+		maxAttempts:    cfg.MaxAttempts,
+		leaseOwner:     cfg.ReplayLeaseOwner,
+		leaseTTL:       cfg.ReplayLeaseTTL,
+		now:            cfg.Now,
+		replaceFile:    atomicReplace,
+		afterReplace:   func(string) error { return nil },
+	}
+}
+
+// Start launches an immediate recovery scan followed by periodic scans. It is
+// idempotent while the service is running.
+func (s *Service) Start(parent context.Context) error {
+	if parent == nil {
+		return gerror.New("archive parent context is nil")
+	}
+	if err := s.validateConfiguration(); err != nil {
+		return err
+	}
+
+	s.mu.Lock()
+	if s.running {
+		s.mu.Unlock()
+		return nil
+	}
+	ctx, cancel := context.WithCancel(parent)
+	done := make(chan struct{})
+	s.cancel, s.done, s.running = cancel, done, true
+	s.mu.Unlock()
+
+	go func() {
+		defer func() {
+			s.mu.Lock()
+			if s.done == done {
+				s.running = false
+				s.cancel = nil
+			}
+			s.mu.Unlock()
+			close(done)
+		}()
+		s.scanLoop(ctx)
+	}()
+	return nil
+}
+
+// Stop cancels the scanner and waits for it to finish. It is safe to call more
+// than once.
+func (s *Service) Stop() error {
+	s.mu.Lock()
+	if !s.running {
+		s.mu.Unlock()
+		return nil
+	}
+	cancel, done := s.cancel, s.done
+	s.mu.Unlock()
+
+	cancel()
+	<-done
+	return nil
+}
+
+// Close implements the common service lifecycle convention.
+func (s *Service) Close() error { return s.Stop() }
+
+// Done is closed when the current scanner invocation exits. It returns nil
+// before the service has been started.
+func (s *Service) Done() <-chan struct{} {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.done
+}
+
+// RunOnce performs one recovery pass. It is useful for explicit lifecycle
+// wiring and deterministic integration tests.
+func (s *Service) RunOnce(ctx context.Context) error {
+	if ctx == nil {
+		return gerror.New("archive scan context is nil")
+	}
+	if err := s.validateConfiguration(); err != nil {
+		return err
+	}
+
+	var scanErrors []error
+	if s.runDiscovery != nil {
+		if err := s.scanRuns(ctx); err != nil {
+			scanErrors = append(scanErrors, gerror.Wrap(err, "scan run archives"))
+		}
+	}
+	if s.replay.Discovery != nil && ctx.Err() == nil {
+		if err := s.scanActions(ctx); err != nil {
+			scanErrors = append(scanErrors, gerror.Wrap(err, "scan replay action archives"))
+		}
+	}
+	return errors.Join(scanErrors...)
+}
+
+// PublishRun atomically writes a run summary and then conditionally registers
+// the durable receipt against the work version.
+func (s *Service) PublishRun(ctx context.Context, work contract.ArchiveWork, snapshot contract.RunArchive) (contract.ArchiveReceipt, error) {
+	if err := validateWork(work); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	normalized, err := normalizeRunArchive(work, snapshot)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+
+	pathTime := normalized.Run.CompletedAt
+	if pathTime.IsZero() {
+		pathTime = normalized.Run.CreatedAt
+	}
+	receipt, err := s.publishJSON(ctx, "runs", work.RunID, pathTime, normalized)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	if s.runRegistrar != nil {
+		if err := s.runRegistrar.RegisterArchive(ctx, work, receipt); err != nil {
+			return contract.ArchiveReceipt{}, gerror.Wrap(err, "register run archive")
+		}
+	}
+	return receipt, nil
+}
+
+// PublishAction atomically writes a receipt_pending replay action summary. The
+// repository update is performed by ArchiveAction because it requires a fenced
+// replay lease.
+func (s *Service) PublishAction(ctx context.Context, action contract.ReplayAction) (contract.ArchiveReceipt, error) {
+	normalized, err := normalizeAction(action)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	return s.publishJSON(ctx, "actions", normalized.ActionID, normalized.CreatedAt, normalized)
+}
+
+// ArchiveRun loads, publishes, and registers one durable run work item.
+func (s *Service) ArchiveRun(ctx context.Context, work contract.ArchiveWork) error {
+	if s.runLoader == nil {
+		return gerror.New("archive run loader is nil")
+	}
+	if s.runRegistrar == nil {
+		return gerror.New("archive run registrar is nil")
+	}
+
+	snapshot, err := s.runLoader.LoadArchiveWork(ctx, work)
+	if err != nil {
+		s.recordRunFailure(ctx, work, err)
+		return gerror.Wrap(err, "load run archive work")
+	}
+	if _, err = s.PublishRun(ctx, work, snapshot); err != nil {
+		s.recordRunFailure(ctx, work, err)
+		return err
+	}
+	return nil
+}
+
+// ArchiveAction acquires a replay lease, atomically publishes the action
+// summary, and fences the completed receipt update through ReplayProgressStore.
+func (s *Service) ArchiveAction(ctx context.Context, action contract.ReplayAction) error {
+	if s.replay.Leases == nil || s.replay.Progress == nil {
+		return gerror.New("replay archive repositories are incomplete")
+	}
+	if action.Status != contract.ActionReceiptPending {
+		return gerror.Wrap(contract.ErrInvalidArgument, "replay action is not receipt_pending")
+	}
+	if _, err := normalizeAction(action); err != nil {
+		return err
+	}
+
+	lease, err := s.replay.Leases.AcquireReplayLease(ctx, action.ActionID, s.leaseOwner, s.leaseTTL)
+	if err != nil {
+		return gerror.Wrap(err, "acquire replay archive lease")
+	}
+	defer func() {
+		releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultReleaseTimeout)
+		defer cancel()
+		_ = s.replay.Leases.ReleaseReplayLease(releaseCtx, lease)
+	}()
+
+	receipt, err := s.PublishAction(ctx, action)
+	if err != nil {
+		return err
+	}
+	lease, err = s.replay.Leases.RenewReplayLease(ctx, lease, s.leaseTTL)
+	if err != nil {
+		return gerror.Wrap(err, "renew replay archive lease")
+	}
+	completedAt := receipt.ArchivedAt
+	_, err = s.replay.Progress.UpdateReplayAction(ctx, contract.ReplayActionUpdate{
+		Lease:            lease,
+		ExpectedRevision: action.Revision,
+		Status:           contract.ActionCompleted,
+		Progress:         action.Progress,
+		Receipt:          &receipt,
+		CompletedAt:      &completedAt,
+	})
+	if err != nil {
+		return gerror.Wrap(err, "register replay action archive")
+	}
+	return nil
+}
+
+func (s *Service) scanLoop(ctx context.Context) {
+	_ = s.RunOnce(ctx)
+	ticker := time.NewTicker(s.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			_ = s.RunOnce(ctx)
+		}
+	}
+}
+
+func (s *Service) scanRuns(ctx context.Context) error {
+	page := contract.PageRequest{Limit: contract.DefaultPageLimit}
+	seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
+	var itemErrors []error
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		result, err := s.runDiscovery.ListArchiveWork(ctx, page)
+		if err != nil {
+			return err
+		}
+		for _, work := range result.Items {
+			err := s.retry(ctx, func() error { return s.ArchiveRun(ctx, work) })
+			if err != nil {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+				itemErrors = append(itemErrors, gerror.Wrapf(err, "archive run %q", work.RunID))
+			}
+		}
+		if result.NextCursor == "" {
+			return errors.Join(itemErrors...)
+		}
+		if _, repeated := seenCursors[result.NextCursor]; repeated {
+			return errors.Join(append(itemErrors, gerror.New("run archive pagination cursor did not advance"))...)
+		}
+		seenCursors[result.NextCursor] = struct{}{}
+		page.Cursor = result.NextCursor
+	}
+}
+
+func (s *Service) scanActions(ctx context.Context) error {
+	page := contract.PageRequest{Limit: contract.DefaultPageLimit}
+	seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
+	var itemErrors []error
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		result, err := s.replay.Discovery.ListReplayWork(ctx, page)
+		if err != nil {
+			return err
+		}
+		for _, action := range result.Items {
+			if action.Status != contract.ActionReceiptPending {
+				continue
+			}
+			err := s.retry(ctx, func() error { return s.ArchiveAction(ctx, action) })
+			if errors.Is(err, contract.ErrBusy) {
+				continue
+			}
+			if err != nil {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+				itemErrors = append(itemErrors, gerror.Wrapf(err, "archive replay action %q", action.ActionID))
+			}
+		}
+		if result.NextCursor == "" {
+			return errors.Join(itemErrors...)
+		}
+		if _, repeated := seenCursors[result.NextCursor]; repeated {
+			return errors.Join(append(itemErrors, gerror.New("replay archive pagination cursor did not advance"))...)
+		}
+		seenCursors[result.NextCursor] = struct{}{}
+		page.Cursor = result.NextCursor
+	}
+}
+
+func (s *Service) retry(ctx context.Context, fn func() error) error {
+	delay := s.initialBackoff
+	var lastErr error
+	for attempt := 0; attempt < s.maxAttempts; attempt++ {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		lastErr = fn()
+		if lastErr == nil || errors.Is(lastErr, contract.ErrBusy) {
+			return lastErr
+		}
+		if attempt+1 == s.maxAttempts {
+			break
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			return ctx.Err()
+		case <-timer.C:
+		}
+		if delay <= s.maxBackoff/2 {
+			delay *= 2
+		} else {
+			delay = s.maxBackoff
+		}
+	}
+	return lastErr
+}
+
+func (s *Service) recordRunFailure(ctx context.Context, work contract.ArchiveWork, archiveErr error) {
+	if s.runRegistrar == nil || ctx.Err() != nil {
+		return
+	}
+	_ = s.runRegistrar.RegisterArchiveFailure(ctx, work, contract.BoundedErrorMessage(archiveErr.Error()))
+}
+
+func (s *Service) publishJSON(ctx context.Context, kind, id string, pathTime time.Time, value any) (contract.ArchiveReceipt, error) {
+	if ctx == nil {
+		return contract.ArchiveReceipt{}, gerror.New("archive publication context is nil")
+	}
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	dir, final, relative, err := s.storagePath(kind, id, pathTime)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+
+	s.publishMu.Lock()
+	defer s.publishMu.Unlock()
+
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive directory")
+	}
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+
+	temp, err := os.CreateTemp(dir, "."+id+"-*.tmp")
+	if err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive temporary file")
+	}
+	tempName := temp.Name()
+	defer func() { _ = os.Remove(tempName) }()
+
+	hasher := sha256.New()
+	buffer := bufio.NewWriter(io.MultiWriter(contextWriter{ctx: ctx, writer: temp}, hasher))
+	encoder := json.NewEncoder(buffer)
+	if err = encoder.Encode(value); err == nil {
+		err = buffer.Flush()
+	}
+	if err == nil {
+		err = temp.Chmod(0o644)
+	}
+	if err == nil {
+		err = temp.Sync()
+	}
+	closeErr := temp.Close()
+	if err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		_ = temp.Close()
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "encode archive temporary file")
+	}
+	expectedChecksum := hex.EncodeToString(hasher.Sum(nil))
+
+	if receipt, ok, existingErr := s.existingReceipt(final, relative, kind, id, expectedChecksum); existingErr != nil {
+		return contract.ArchiveReceipt{}, existingErr
+	} else if ok {
+		if err := syncDirectory(dir); err != nil {
+			return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
+		}
+		return receipt, nil
+	}
+
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	if err := s.replaceFile(tempName, final); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "publish archive file")
+	}
+	if err := syncDirectory(dir); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
+	}
+	if err := s.afterReplace(final); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "archive post-publish verification hook")
+	}
+
+	stored, err := os.ReadFile(final)
+	if err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "read published archive file")
+	}
+	storedSum := sha256.Sum256(stored)
+	if hex.EncodeToString(storedSum[:]) != expectedChecksum {
+		return contract.ArchiveReceipt{}, errChecksumMismatch
+	}
+	info, err := os.Stat(final)
+	if err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "stat published archive file")
+	}
+	receipt := contract.ArchiveReceipt{
+		Version:    contract.ArchiveSchemaVersion,
+		Path:       relative,
+		Checksum:   expectedChecksum,
+		ArchivedAt: archiveModTime(info, s.now),
+	}
+	if err := receipt.Validate(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	return receipt, nil
+}
+
+func (s *Service) existingReceipt(final, relative, kind, id, expectedChecksum string) (contract.ArchiveReceipt, bool, error) {
+	info, err := os.Lstat(final)
+	if errors.Is(err, os.ErrNotExist) {
+		return contract.ArchiveReceipt{}, false, nil
+	}
+	if err != nil {
+		return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "inspect existing archive file")
+	}
+	if !info.Mode().IsRegular() {
+		return contract.ArchiveReceipt{}, false, nil
+	}
+	stored, err := os.ReadFile(final)
+	if err != nil {
+		return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "read existing archive file")
+	}
+	storedSum := sha256.Sum256(stored)
+	if hex.EncodeToString(storedSum[:]) != expectedChecksum {
+		if validExistingArchive(kind, id, stored) {
+			return contract.ArchiveReceipt{}, false, gerror.Wrap(contract.ErrConflict, "archive path already contains different valid content")
+		}
+		return contract.ArchiveReceipt{}, false, nil
+	}
+	receipt := contract.ArchiveReceipt{
+		Version:    contract.ArchiveSchemaVersion,
+		Path:       relative,
+		Checksum:   expectedChecksum,
+		ArchivedAt: archiveModTime(info, s.now),
+	}
+	if err := receipt.Validate(); err != nil {
+		return contract.ArchiveReceipt{}, false, err
+	}
+	return receipt, true, nil
+}
+
+func validExistingArchive(kind, id string, data []byte) bool {
+	switch kind {
+	case "runs":
+		var snapshot contract.RunArchive
+		if json.Unmarshal(data, &snapshot) != nil {
+			return false
+		}
+		_, err := normalizeRunArchive(contract.ArchiveWork{RunID: id, Version: "existing"}, snapshot)
+		return err == nil
+	case "actions":
+		var action contract.ReplayAction
+		if json.Unmarshal(data, &action) != nil || action.ActionID != id {
+			return false
+		}
+		_, err := normalizeAction(action)
+		return err == nil
+	default:
+		return false
+	}
+}
+
+func archiveModTime(info os.FileInfo, now func() time.Time) time.Time {
+	archivedAt := info.ModTime().UTC()
+	if archivedAt.IsZero() {
+		archivedAt = now().UTC()
+	}
+	return archivedAt
+}
+
+func (s *Service) storagePath(kind, id string, pathTime time.Time) (string, string, string, error) {
+	if err := validateStorageID(id); err != nil {
+		return "", "", "", err
+	}
+	if kind != "runs" && kind != "actions" {
+		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "unknown archive kind")
+	}
+	if strings.TrimSpace(s.root) == "" {
+		return "", "", "", gerror.New("archive root is empty")
+	}
+	if pathTime.IsZero() {
+		pathTime = s.now().UTC()
+	} else {
+		pathTime = pathTime.UTC()
+	}
+	if pathTime.Year() < 1 || pathTime.Year() > 9999 {
+		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive timestamp is outside the supported year range")
+	}
+
+	root := filepath.Clean(s.root)
+	prefix := "run-"
+	if kind == "actions" {
+		prefix = "action-"
+	}
+	relativeOS := prefix + id + ".json"
+	final := filepath.Join(root, relativeOS)
+	contained, err := filepath.Rel(root, final)
+	if err != nil || filepath.IsAbs(contained) || contained == ".." || strings.HasPrefix(contained, ".."+string(filepath.Separator)) {
+		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive path escapes root")
+	}
+	return filepath.Dir(final), final, filepath.ToSlash(relativeOS), nil
+}
+
+func (s *Service) validateConfiguration() error {
+	if strings.TrimSpace(s.root) == "" {
+		return gerror.New("archive root is empty")
+	}
+	if s.runDiscovery != nil && (s.runLoader == nil || s.runRegistrar == nil) {
+		return gerror.New("run archive repositories are incomplete")
+	}
+	if s.replay.Discovery != nil && (s.replay.Leases == nil || s.replay.Progress == nil) {
+		return gerror.New("replay archive repositories are incomplete")
+	}
+	return nil
+}
+
+func validateWork(work contract.ArchiveWork) error {
+	if err := validateStorageID(work.RunID); err != nil {
+		return err
+	}
+	if strings.TrimSpace(work.Version) == "" {
+		return gerror.Wrap(contract.ErrInvalidArgument, "archive work version is required")
+	}
+	return nil
+}
+
+func validateStorageID(id string) error {
+	if err := contract.ValidateID(id); err != nil {
+		return err
+	}
+	if id == "." || id == ".." || filepath.VolumeName(id) != "" || strings.ContainsRune(id, ':') || filepath.Base(id) != id {
+		return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is not safe for archive storage")
+	}
+	name := strings.ToUpper(strings.TrimSuffix(id, filepath.Ext(id)))
+	switch name {
+	case "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9":
+		return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is reserved by the filesystem")
+	}
+	return nil
+}
+
+func normalizeRunArchive(work contract.ArchiveWork, snapshot contract.RunArchive) (contract.RunArchive, error) {
+	if snapshot.Version == 0 {
+		snapshot.Version = contract.ArchiveSchemaVersion
+	}
+	if snapshot.Version != contract.ArchiveSchemaVersion {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "invalid archive schema version")
+	}
+	if snapshot.Run.RunID != work.RunID {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archive run ID does not match work")
+	}
+	if !snapshot.Run.Status.Terminal() {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "only terminal runs can be archived")
+	}
+	if err := snapshot.Run.Validate(); err != nil {
+		return contract.RunArchive{}, err
+	}
+	if len(snapshot.Rounds) > maxRounds {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "too many archived rounds")
+	}
+
+	snapshot.Run = normalizeRunMeta(snapshot.Run)
+	snapshot.Rounds = append([]contract.RoundStats(nil), snapshot.Rounds...)
+	if snapshot.Rounds == nil {
+		snapshot.Rounds = make([]contract.RoundStats, 0)
+	}
+	seen := make(map[int]struct{}, len(snapshot.Rounds))
+	for i := range snapshot.Rounds {
+		round := &snapshot.Rounds[i]
+		if round.RunID != work.RunID {
+			return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archived round belongs to another run")
+		}
+		if _, duplicate := seen[round.Round]; duplicate {
+			return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "duplicate archived round")
+		}
+		seen[round.Round] = struct{}{}
+		if err := round.Validate(); err != nil {
+			return contract.RunArchive{}, err
+		}
+		normalizedCounts, err := round.ErrorCounts.Normalized()
+		if err != nil {
+			return contract.RunArchive{}, err
+		}
+		round.ErrorCounts = normalizedCounts
+		round.StartedAt = utcTimePtr(round.StartedAt)
+		round.CompletedAt = utcTimePtr(round.CompletedAt)
+	}
+	sort.Slice(snapshot.Rounds, func(i, j int) bool { return snapshot.Rounds[i].Round < snapshot.Rounds[j].Round })
+	return snapshot, nil
+}
+
+func normalizeRunMeta(meta contract.RunMeta) contract.RunMeta {
+	meta.CreatedAt = meta.CreatedAt.UTC()
+	meta.ProducerCompletedAt = meta.ProducerCompletedAt.UTC()
+	meta.CompletedAt = meta.CompletedAt.UTC()
+	meta.StartedAt = utcTimePtr(meta.StartedAt)
+	meta.ScanCompletedAt = utcTimePtr(meta.ScanCompletedAt)
+	// Archive state describes the repository registration and is intentionally
+	// excluded from the file whose receipt populates that state.
+	meta.ArchiveStatus = ""
+	meta.ArchiveVersion = 0
+	meta.ArchivePath = ""
+	meta.ArchiveChecksum = ""
+	meta.ArchivedAt = nil
+	return meta
+}
+
+func normalizeAction(action contract.ReplayAction) (contract.ReplayAction, error) {
+	if err := validateStorageID(action.ActionID); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	if action.Status != contract.ActionReceiptPending {
+		return contract.ReplayAction{}, gerror.Wrap(contract.ErrInvalidArgument, "only receipt_pending replay actions can be archived")
+	}
+	action.Error = contract.BoundedErrorMessage(action.Error)
+	action.CreatedAt = action.CreatedAt.UTC()
+	action.UpdatedAt = action.UpdatedAt.UTC()
+	action.CompletedAt = nil
+	action.Receipt = nil
+	if err := action.Validate(); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	return action, nil
+}
+
+func utcTimePtr(value *time.Time) *time.Time {
+	if value == nil {
+		return nil
+	}
+	utc := value.UTC()
+	return &utc
+}
+
+type contextWriter struct {
+	ctx    context.Context
+	writer io.Writer
+}
+
+func (w contextWriter) Write(p []byte) (int, error) {
+	if err := w.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return w.writer.Write(p)
+}

worker-svr/internal/fullsync/archive/archive_test.go

@@ -0,0 +1,588 @@
+package archive
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestPublishRunIsDurableDeterministicAndSummaryOnly(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-1"))
+	service := New(root, nil, nil, repository)
+	work := contract.ArchiveWork{RunID: "run-1", Version: "version-1"}
+
+	staleDir := root
+	if err := os.MkdirAll(staleDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	stalePath := filepath.Join(staleDir, ".run-1-stale.tmp")
+	if err := os.WriteFile(stalePath, []byte("incomplete"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("PublishRun() error = %v", err)
+	}
+	if receipt.Path != "run-run-1.json" {
+		t.Fatalf("receipt path = %q", receipt.Path)
+	}
+	if err := receipt.Validate(); err != nil {
+		t.Fatalf("receipt validation failed: %v", err)
+	}
+
+	archivePath := filepath.Join(root, filepath.FromSlash(receipt.Path))
+	info, err := os.Stat(archivePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runtime.GOOS != "windows" {
+		if got := info.Mode().Perm(); got != 0o644 {
+			t.Fatalf("archive mode = %o, want 644", got)
+		}
+	}
+	data, err := os.ReadFile(archivePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(data)
+	if got := hex.EncodeToString(sum[:]); got != receipt.Checksum {
+		t.Fatalf("checksum = %q, want %q", receipt.Checksum, got)
+	}
+	for _, forbidden := range [][]byte{[]byte(`"archiveStatus"`), []byte(`"archivePath"`), []byte(`"taskId"`), []byte(`"card"`), []byte(`"message"`)} {
+		if bytes.Contains(data, forbidden) {
+			t.Fatalf("archive contains forbidden detail %s: %s", forbidden, data)
+		}
+	}
+	var decoded contract.RunArchive
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded.Rounds) != 2 || decoded.Rounds[0].Round != 1 || decoded.Rounds[1].Round != 2 {
+		t.Fatalf("rounds were not deterministically sorted: %+v", decoded.Rounds)
+	}
+	if decoded.Rounds[1].ErrorCounts[contract.ErrorRateLimited] != 1 {
+		t.Fatalf("error categories were not normalized: %+v", decoded.Rounds[1].ErrorCounts)
+	}
+
+	repeated, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("repeated PublishRun() error = %v", err)
+	}
+	if !reflect.DeepEqual(repeated, receipt) {
+		t.Fatalf("repeated receipt changed:\nfirst:  %+v\nsecond: %+v", receipt, repeated)
+	}
+	if _, err := os.Stat(stalePath); err != nil {
+		t.Fatalf("stale temp residue should not block publication: %v", err)
+	}
+	repository.mu.Lock()
+	defer repository.mu.Unlock()
+	if len(repository.receipts) != 2 || !reflect.DeepEqual(repository.receipts[0], repository.receipts[1]) {
+		t.Fatalf("registered receipts = %+v", repository.receipts)
+	}
+}
+
+func TestArchiveActionPublishesAndRegistersFencedReceipt(t *testing.T) {
+	root := t.TempDir()
+	action := testReplayAction("action-1")
+	action.Error = strings.Repeat("x", contract.MaxErrorMessageBytes+50)
+	repository := newFakeReplayRepository(action)
+	service := NewServiceWithReplay(
+		Config{Root: root},
+		nil,
+		nil,
+		nil,
+		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
+	)
+
+	if err := service.ArchiveAction(context.Background(), action); err != nil {
+		t.Fatalf("ArchiveAction() error = %v", err)
+	}
+	repository.mu.Lock()
+	update := repository.update
+	released := repository.released
+	repository.mu.Unlock()
+	if update == nil || update.Status != contract.ActionCompleted || update.Receipt == nil || update.CompletedAt == nil {
+		t.Fatalf("registered update = %+v", update)
+	}
+	if update.ExpectedRevision != action.Revision || update.Receipt.ArchivedAt != *update.CompletedAt {
+		t.Fatalf("receipt registration lost fencing/time: %+v", update)
+	}
+	if update.Lease.Token != "lease-token-2" {
+		t.Fatalf("receipt registration used the unrenewed lease: %+v", update.Lease)
+	}
+	if released.Token != "lease-token-2" {
+		t.Fatalf("archive released the stale lease: %+v", released)
+	}
+	if update.Receipt.Path != "action-action-1.json" {
+		t.Fatalf("action receipt path = %q", update.Receipt.Path)
+	}
+
+	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(update.Receipt.Path)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, forbidden := range [][]byte{[]byte("idempotencyKey"), []byte("fingerprint"), []byte("leaseOwner"), []byte("leaseToken"), []byte("revision"), []byte(`"receipt"`), []byte("completedAt")} {
+		if bytes.Contains(data, forbidden) {
+			t.Fatalf("action archive contains internal field %q: %s", forbidden, data)
+		}
+	}
+	var decoded contract.ReplayAction
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.Status != contract.ActionReceiptPending || len(decoded.Error) != contract.MaxErrorMessageBytes {
+		t.Fatalf("archived action was not bounded receipt_pending summary: %+v", decoded)
+	}
+	repeated, err := service.PublishAction(context.Background(), action)
+	if err != nil {
+		t.Fatalf("repeated PublishAction() error = %v", err)
+	}
+	if !reflect.DeepEqual(repeated, *update.Receipt) {
+		t.Fatalf("repeated action receipt changed: %+v != %+v", repeated, *update.Receipt)
+	}
+}
+
+func TestPublishRunCleansTempAfterRenameFailure(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-rename"))
+	service := New(root, nil, nil, repository)
+	renameErr := gerror.New("injected rename failure")
+	service.replaceFile = func(string, string) error { return renameErr }
+
+	_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: "run-rename", Version: "v1"}, repository.snapshot)
+	if !errors.Is(err, renameErr) {
+		t.Fatalf("PublishRun() error = %v, want rename failure", err)
+	}
+	if matches, globErr := filepath.Glob(filepath.Join(root, ".run-rename-*.tmp")); globErr != nil || len(matches) != 0 {
+		t.Fatalf("temporary files after failure = %v, error = %v", matches, globErr)
+	}
+	if _, statErr := os.Stat(filepath.Join(root, "run-run-rename.json")); !errors.Is(statErr, os.ErrNotExist) {
+		t.Fatalf("final file exists after failed rename: %v", statErr)
+	}
+	if len(repository.receipts) != 0 {
+		t.Fatalf("registered receipt after failed rename: %+v", repository.receipts)
+	}
+}
+
+func TestChecksumFailureIsRecoveredByExactPathReplacement(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-checksum"))
+	service := New(root, nil, nil, repository)
+	service.afterReplace = func(path string) error {
+		return os.WriteFile(path, []byte("corrupt"), 0o600)
+	}
+	work := contract.ArchiveWork{RunID: "run-checksum", Version: "v1"}
+
+	if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, errChecksumMismatch) {
+		t.Fatalf("PublishRun() error = %v, want checksum mismatch", err)
+	}
+	if len(repository.receipts) != 0 {
+		t.Fatalf("registered receipt after checksum failure: %+v", repository.receipts)
+	}
+	service.afterReplace = func(string) error { return nil }
+	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("recovery PublishRun() error = %v", err)
+	}
+	if receipt.Path != "run-run-checksum.json" {
+		t.Fatalf("replacement used different path: %q", receipt.Path)
+	}
+	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(receipt.Path)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) == "corrupt" {
+		t.Fatal("corrupt final file was not replaced")
+	}
+}
+
+func TestRegistrationRetryReusesIdenticalReceipt(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-register"))
+	repository.registerErr = gerror.New("registration result was uncertain")
+	service := New(root, nil, nil, repository)
+	work := contract.ArchiveWork{RunID: "run-register", Version: "v1"}
+
+	if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, repository.registerErr) {
+		t.Fatalf("first PublishRun() error = %v", err)
+	}
+	repository.mu.Lock()
+	firstReceipt := repository.receipts[0]
+	repository.registerErr = nil
+	repository.mu.Unlock()
+
+	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("retry PublishRun() error = %v", err)
+	}
+	if !reflect.DeepEqual(receipt, firstReceipt) {
+		t.Fatalf("retry changed uncertain receipt:\nfirst: %+v\nretry: %+v", firstReceipt, receipt)
+	}
+}
+
+func TestPublishRunRejectsDifferentValidContentAtSamePath(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-conflict"))
+	service := New(root, nil, nil, repository)
+	work := contract.ArchiveWork{RunID: "run-conflict", Version: "v1"}
+
+	first, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	changed := repository.snapshot
+	changed.Run.TriggerSource = "different-valid-summary"
+	if _, err = service.PublishRun(context.Background(), work, changed); !errors.Is(err, contract.ErrConflict) {
+		t.Fatalf("PublishRun() error = %v, want conflict", err)
+	}
+	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(first.Path)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(data)
+	if got := hex.EncodeToString(sum[:]); got != first.Checksum {
+		t.Fatalf("conflicting publication replaced the original file: %q != %q", got, first.Checksum)
+	}
+}
+
+func TestScannerRecoversFailedRunAfterRestart(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-restart"))
+	repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
+		return contract.RunArchive{}, gerror.New("source temporarily unavailable")
+	}
+	first := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
+	if err := first.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.failed, "initial archive failure")
+	if err := first.Stop(); err != nil {
+		t.Fatal(err)
+	}
+
+	repository.mu.Lock()
+	repository.load = nil
+	repository.mu.Unlock()
+	second := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
+	if err := second.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.succeeded, "restart archive success")
+	if err := second.Stop(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(root, "run-run-restart.json")); err != nil {
+		t.Fatalf("recovered archive file missing: %v", err)
+	}
+}
+
+func TestPeriodicScannerRetriesPendingRun(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-periodic"))
+	var calls atomic.Int32
+	repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
+		if calls.Add(1) == 1 {
+			return contract.RunArchive{}, gerror.New("first scan fails")
+		}
+		return repository.snapshot, nil
+	}
+	service := NewService(Config{Root: root, Interval: 10 * time.Millisecond, MaxAttempts: 1}, repository, repository, repository)
+	if err := service.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.succeeded, "periodic archive success")
+	if err := service.Stop(); err != nil {
+		t.Fatal(err)
+	}
+	if calls.Load() < 2 {
+		t.Fatalf("loader calls = %d, want at least 2", calls.Load())
+	}
+}
+
+func TestStartupScannerArchivesReceiptPendingAction(t *testing.T) {
+	action := testReplayAction("action-startup")
+	repository := newFakeReplayRepository(action)
+	service := NewServiceWithReplay(
+		Config{Root: t.TempDir(), Interval: time.Hour, MaxAttempts: 1},
+		nil,
+		nil,
+		nil,
+		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
+	)
+	if err := service.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.completed, "startup replay archive success")
+	if err := service.Stop(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCancellationStopsBlockedScannerAndPublication(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-cancel"))
+	started := make(chan struct{})
+	repository.load = func(ctx context.Context, _ contract.ArchiveWork) (contract.RunArchive, error) {
+		close(started)
+		<-ctx.Done()
+		return contract.RunArchive{}, ctx.Err()
+	}
+	service := NewService(Config{Root: root, Interval: time.Hour, InitialBackoff: time.Hour}, repository, repository, repository)
+	if err := service.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, started, "blocked loader start")
+	stopped := make(chan struct{})
+	go func() {
+		_ = service.Stop()
+		close(stopped)
+	}()
+	waitSignal(t, stopped, "scanner cancellation")
+
+	canceled, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := service.PublishRun(canceled, contract.ArchiveWork{RunID: "run-cancel", Version: "v1"}, repository.snapshot)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("canceled PublishRun() error = %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(root, "run-run-cancel.json")); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("canceled publication wrote final file: %v", err)
+	}
+}
+
+func TestCancellationAfterRenameStillVerifiesPublicationAndReleasesLease(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	action := testReplayAction("action-cancel-after-rename")
+	repository := newFakeReplayRepository(action)
+	service := NewServiceWithReplay(
+		Config{Root: t.TempDir()},
+		nil,
+		nil,
+		nil,
+		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
+	)
+	service.afterReplace = func(string) error {
+		cancel()
+		return nil
+	}
+
+	if err := service.ArchiveAction(ctx, action); err != nil {
+		t.Fatalf("ArchiveAction() error = %v", err)
+	}
+	repository.mu.Lock()
+	releaseCtxErr := repository.releaseCtxErr
+	repository.mu.Unlock()
+	if releaseCtxErr != nil {
+		t.Fatalf("lease release inherited canceled context: %v", releaseCtxErr)
+	}
+}
+
+func TestArchiveRejectsTraversalAndUnsafeFilesystemIDs(t *testing.T) {
+	root := t.TempDir()
+	service := New(root, nil, nil, newFakeRunRepository(contract.RunArchive{}))
+	for _, id := range []string{"../escape", `..\escape`, "/absolute", `C:\escape`, "drive:name", ".", "..", "{hash}", "CON"} {
+		t.Run(strings.ReplaceAll(id, "/", "_"), func(t *testing.T) {
+			snapshot := testRunArchive(id)
+			_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: id, Version: "v1"}, snapshot)
+			if !errors.Is(err, contract.ErrInvalidArgument) {
+				t.Fatalf("PublishRun(%q) error = %v, want invalid argument", id, err)
+			}
+		})
+	}
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("unsafe IDs created archive entries: %+v", entries)
+	}
+}
+
+func testRunArchive(runID string) contract.RunArchive {
+	created := time.Date(2026, time.August, 31, 23, 0, 0, 0, time.FixedZone("test", 8*60*60))
+	completed := time.Date(2026, time.September, 20, 16, 0, 0, 0, time.FixedZone("test", 8*60*60))
+	archived := completed.Add(time.Minute)
+	return contract.RunArchive{
+		Version: contract.ArchiveSchemaVersion,
+		Run: contract.RunMeta{
+			RunID: runID, Status: contract.RunCompletedWithFailures, ProducerDone: true,
+			CurrentRound: 2, MaxRound: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1, PendingFailureCount: 1,
+			CreatedAt: created, ProducerCompletedAt: completed, CompletedAt: completed,
+			ArchiveStatus: contract.ArchiveFailed, ArchiveVersion: 99, ArchivePath: "old.json",
+			ArchiveChecksum: strings.Repeat("a", 64), ArchivedAt: &archived,
+		},
+		Rounds: []contract.RoundStats{
+			{RunID: runID, Round: 2, EnqueuedCount: 1, FailedCount: 1, ErrorCounts: contract.ErrorCounts{"mobile_busy": 1}},
+			{RunID: runID, Round: 1, EnqueuedCount: 2, SuccessCount: 2, ErrorCounts: nil},
+		},
+	}
+}
+
+func testReplayAction(actionID string) contract.ReplayAction {
+	created := time.Date(2026, time.September, 20, 12, 0, 0, 0, time.FixedZone("test", 8*60*60))
+	return contract.ReplayAction{
+		ActionID: actionID, SourceRunID: "source-1", TargetRunID: "target-1", RequestedBy: "operator",
+		IdempotencyKey: "secret-key", Fingerprint: "secret-fingerprint", Status: contract.ActionReceiptPending,
+		ReplayCycle: 1, Progress: contract.ReplayProgress{DiscoveredCount: 2, EnqueuedCount: 2, CleanedCount: 2},
+		LeaseOwner: "engine", LeaseToken: "secret-token", LeaseUntil: created.Add(time.Hour), Revision: 7,
+		CreatedAt: created, UpdatedAt: created.Add(time.Minute),
+	}
+}
+
+type fakeRunRepository struct {
+	mu          sync.Mutex
+	work        contract.ArchiveWork
+	snapshot    contract.RunArchive
+	load        func(context.Context, contract.ArchiveWork) (contract.RunArchive, error)
+	registered  bool
+	receipts    []contract.ArchiveReceipt
+	failures    []string
+	registerErr error
+	succeeded   chan struct{}
+	failed      chan struct{}
+}
+
+func newFakeRunRepository(snapshot contract.RunArchive) *fakeRunRepository {
+	return &fakeRunRepository{
+		work:      contract.ArchiveWork{RunID: snapshot.Run.RunID, Version: "repository-version"},
+		snapshot:  snapshot,
+		succeeded: make(chan struct{}, 8),
+		failed:    make(chan struct{}, 8),
+	}
+}
+
+func (r *fakeRunRepository) ListArchiveWork(ctx context.Context, _ contract.PageRequest) (contract.ArchiveWorkPage, error) {
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.registered {
+		return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}, nil
+	}
+	return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{r.work}}, nil
+}
+
+func (r *fakeRunRepository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
+	r.mu.Lock()
+	load := r.load
+	snapshot := r.snapshot
+	r.mu.Unlock()
+	if load != nil {
+		return load(ctx, work)
+	}
+	return snapshot, nil
+}
+
+func (r *fakeRunRepository) RegisterArchive(_ context.Context, _ contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
+	r.mu.Lock()
+	r.receipts = append(r.receipts, receipt)
+	err := r.registerErr
+	if err == nil {
+		r.registered = true
+	}
+	r.mu.Unlock()
+	if err != nil {
+		return err
+	}
+	notify(r.succeeded)
+	return nil
+}
+
+func (r *fakeRunRepository) RegisterArchiveFailure(_ context.Context, _ contract.ArchiveWork, message string) error {
+	r.mu.Lock()
+	r.failures = append(r.failures, message)
+	r.mu.Unlock()
+	notify(r.failed)
+	return nil
+}
+
+type fakeReplayRepository struct {
+	mu            sync.Mutex
+	action        contract.ReplayAction
+	update        *contract.ReplayActionUpdate
+	released      contract.ReplayLease
+	releaseCtxErr error
+	completed     chan struct{}
+}
+
+func newFakeReplayRepository(action contract.ReplayAction) *fakeReplayRepository {
+	return &fakeReplayRepository{action: action, completed: make(chan struct{}, 4)}
+}
+
+func (r *fakeReplayRepository) ListReplayWork(ctx context.Context, _ contract.PageRequest) (contract.ReplayActionPage, error) {
+	if err := ctx.Err(); err != nil {
+		return contract.ReplayActionPage{}, err
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.update != nil {
+		return contract.ReplayActionPage{Items: []contract.ReplayAction{}}, nil
+	}
+	return contract.ReplayActionPage{Items: []contract.ReplayAction{r.action}}, nil
+}
+
+func (r *fakeReplayRepository) AcquireReplayLease(_ context.Context, actionID, owner string, ttl time.Duration) (contract.ReplayLease, error) {
+	return contract.ReplayLease{ActionID: actionID, Owner: owner, Token: "lease-token", Until: time.Now().Add(ttl)}, nil
+}
+
+func (r *fakeReplayRepository) RenewReplayLease(_ context.Context, lease contract.ReplayLease, ttl time.Duration) (contract.ReplayLease, error) {
+	lease.Token = "lease-token-2"
+	lease.Until = time.Now().Add(ttl)
+	return lease, nil
+}
+
+func (r *fakeReplayRepository) ReleaseReplayLease(ctx context.Context, lease contract.ReplayLease) error {
+	r.mu.Lock()
+	r.released = lease
+	r.releaseCtxErr = ctx.Err()
+	r.mu.Unlock()
+	return nil
+}
+
+func (r *fakeReplayRepository) UpdateReplayAction(_ context.Context, update contract.ReplayActionUpdate) (contract.ReplayAction, error) {
+	r.mu.Lock()
+	copyOfUpdate := update
+	r.update = &copyOfUpdate
+	action := r.action
+	action.Status = update.Status
+	action.Receipt = update.Receipt
+	action.CompletedAt = update.CompletedAt
+	r.mu.Unlock()
+	notify(r.completed)
+	return action, nil
+}
+
+func notify(ch chan struct{}) {
+	select {
+	case ch <- struct{}{}:
+	default:
+	}
+}
+
+func waitSignal(t *testing.T, ch <-chan struct{}, operation string) {
+	t.Helper()
+	select {
+	case <-ch:
+	case <-time.After(3 * time.Second):
+		t.Fatalf("timed out waiting for %s", operation)
+	}
+}

worker-svr/internal/fullsync/archive/replace_other.go

@@ -0,0 +1,21 @@
+//go:build !windows
+
+package archive
+
+import "os"
+
+func atomicReplace(from, to string) error {
+	return os.Rename(from, to)
+}
+
+func syncDirectory(path string) error {
+	dir, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	if err := dir.Sync(); err != nil {
+		_ = dir.Close()
+		return err
+	}
+	return dir.Close()
+}

worker-svr/internal/fullsync/archive/replace_windows.go

@@ -0,0 +1,21 @@
+//go:build windows
+
+package archive
+
+import "golang.org/x/sys/windows"
+
+func atomicReplace(from, to string) error {
+	fromUTF16, err := windows.UTF16PtrFromString(from)
+	if err != nil {
+		return err
+	}
+	toUTF16, err := windows.UTF16PtrFromString(to)
+	if err != nil {
+		return err
+	}
+	return windows.MoveFileEx(fromUTF16, toUTF16, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH)
+}
+
+// Windows has no portable directory fsync operation. MoveFileEx with
+// MOVEFILE_WRITE_THROUGH provides the strongest supported replacement flush.
+func syncDirectory(string) error { return nil }

worker-svr/internal/fullsync/contract/README.md

@@ -0,0 +1,124 @@
+# Full-sync shared contracts
+
+This package establishes the boundary for later queue accounting, archive,
+replay, and HTTP work. It does not change running service behavior. Production
+code in this package does not import the queue package; compatibility tests
+compare its models and keys with the existing queue implementation.
+
+## Compatibility and counters
+
+- `RunMeta` retains all existing JSON fields and adds optional fields. Existing
+  `zset.RunMeta`, `zset.Task`, and `zset.FailureRecord` remain unchanged; adapters
+  are a later integration step. The new `RunStatus` is a named string type.
+- `Validate` accepts missing extension fields on old records.
+  `ValidateAccounting` is opt-in for runs with known extended counters and
+  enforces `enqueuedCount = successCount + failedCount + activeCount + nextRoundCount`.
+  `inflightCount` is a subset of `activeCount`.
+- Source cleanup changes `pendingFailureCount`, never historical `failedCount`.
+- Round statistics have no task/card/channel dimensions. `ErrorCounts`
+  normalizes both JSON input and output to eight canonical categories, merging
+  unknown names into `other` and rejecting negative/overflowing totals.
+- Existing failure payloads retain their free-form classification and message.
+  The HTTP `Failure` projection normalizes categories and bounds messages to
+  2,048 UTF-8 bytes. Historical blocked tasks may have round zero.
+
+## Status domains
+
+| Domain | Wire values |
+| --- | --- |
+| Run | `producing`, `running`, `completed`, `completed_with_failures` |
+| Sync type | `normal`, `replay` |
+| Archive | `disabled`, `pending`, `processing`, `succeeded`, `failed` |
+| Source replay disposition | `none`, `pending`, `processing`, `completed`, `needs_attention` |
+| Replay action | `accepted`, `processing`, `cleanup_pending`, `receipt_pending`, `completed`, `needs_attention` |
+| Error category | `timeout`, `rate_limited`, `authentication`, `validation`, `upstream`, `transport`, `internal`, `other` |
+
+The source field is `replayDisposition`, distinct from the replay action's
+`status`. Lease tokens, record versions, action revision, idempotency key, and
+request fingerprint are internal and excluded from HTTP JSON.
+
+## Interfaces and persistence obligations
+
+HTTP consumers depend on `RunReader`, `FailurePager`, `ReplayRequester`, and
+`ActionReader`. Separate live/archive readers support fallback only after a live
+`ErrNotFound`; storage failures must remain visible. An archived summary cannot
+provide per-card details: use `ErrFailureDetailsUnavailable` when live details
+are gone. Repositories and HTTP both apply the default page limit 100 and
+maximum 500. Cursors remain opaque and are bound to the original query. An empty
+scan page with a next cursor does not signal completion.
+
+Archive discovery, scheduling, snapshot loading, and result registration use
+separate narrow interfaces. Registration is conditional on an opaque work
+version. Archives contain run and round aggregates only. Archive receipts use
+a schema version, storage-owned path, lowercase SHA-256 checksum of stored
+bytes, and publication time. Archive schema version starts at 1.
+
+Replay acceptance atomically reserves `(sourceRunId, idempotencyKey)` with a
+SHA-256 fingerprint of the source and exact requestedBy body value. Repeating
+the request returns its original action/target; changing the request returns
+`ErrConflict`. The action fixes its target ID before target creation/enqueue.
+
+Replay work discovery, lease management, progress updates, source pagination,
+target creation/enqueue, and source cleanup are separate interfaces. Progress
+updates are fenced and use a revision check. Target enqueue distinguishes
+durably present duplicates from rejected writes; a zero insert count by itself
+does not authorize source cleanup. Cleanup checks exact source record versions
+and target acceptance evidence. Implementations must persist enough evidence
+to recover partial batches; a scan cursor by itself is insufficient. Source
+pagination must remain correct while already-enqueued entries are deleted.
+
+`ErrNotFound`, `ErrConflict`, `ErrInvalidArgument`,
+`ErrFailureDetailsUnavailable`, `ErrLeaseLost`, and `ErrBusy` support
+`errors.Is` through wrapping. HTTP error envelopes and status mappings remain
+the HTTP implementation's responsibility.
+
+## Redis naming
+
+All names are prefixed with the configured namespace (`flow:` by default;
+`sim-status:` for status sync). Existing run/round/task-state/failure names are
+preserved. New keys include:
+
+| Suffix after namespace | Scope |
+| --- | --- |
+| `{runId}:round:N:stats` | Run slot |
+| `{runId}:archive` | Run slot |
+| `runs:all` | Global index |
+| `archive:pending` | Global index |
+| `{replay}:action:actionId:meta` | Replay slot |
+| `{replay}:action:actionId:lease` | Replay slot |
+| `{replay}:idempotency:SHA256(sourceRunId,key)` | Replay slot; tuple encoded as a JSON array before hashing |
+| `{replay}:pending` | Replay slot |
+
+Replay action creation, idempotency, and discovery can be atomic within their
+shared slot. Source/target runs and global indexes can occupy different slots;
+their coordination requires recoverable registrations/receipts rather than a
+single cross-slot Lua transaction. New identifier validation rejects hash-tag
+or path delimiters without rewriting legacy JSON on decode.
+
+## HTTP surface
+
+| Method | Route | Response |
+| --- | --- | --- |
+| GET | `/internal/v1/full-sync-runs?syncKind=&limit=` | `RunList` merged from active Redis Runs and `run-<runId>.json` archives |
+| GET | `/internal/v1/full-sync-runs/{runId}` | `RunView` with `source`, `run`, optional `rounds` |
+| GET | `/internal/v1/full-sync-runs/{runId}/failures?cursor=&limit=` | `FailurePage` with `items`, `nextCursor` |
+| POST | `/internal/v1/full-sync-runs/{runId}/replay` | HTTP 202 `ReplayAccepted` |
+| GET | `/internal/v1/full-sync-actions/{actionId}` | `ReplayAction` |
+
+POST requires `Idempotency-Key` and JSON `{"requestedBy":"..."}`. Authentication,
+HTTP handlers, storage, replay execution, archive files, and service wiring are
+intentionally left to subsequent implementations.
+
+## Verification
+
+```text
+gofmt -l internal/fullsync/contract
+go test ./internal/fullsync/contract
+go test ./internal/queue/zset ./internal/consumer/zset ./internal/producer/simstatus ./internal/producer/monthlyusage ./internal/service/simstatus ./internal/service/monthlyusage
+```
+
+Tests cover legacy schema and JSON compatibility, existing key compatibility,
+cluster key boundaries, bounded categories/messages, overflow, run accounting,
+pagination limits, status domains, replay identity, internal field exclusion,
+and receipt/progress validation. Regression tests use the repository's isolated
+Redis helper and require `redis-server` or `REDIS_SERVER_BINARY`.

worker-svr/internal/fullsync/contract/archive.go

@@ -0,0 +1,46 @@
+package contract
+
+import "time"
+
+const ArchiveSchemaVersion = 1
+
+// ArchiveReceipt identifies a durably published archive object. Path is a
+// storage-owned reference, never a client-supplied destination. Checksum is
+// lowercase hex SHA-256 of the final stored bytes; version is the schema.
+type ArchiveReceipt struct {
+	Version    int       `json:"version"`
+	Path       string    `json:"path"`
+	Checksum   string    `json:"checksum"`
+	ArchivedAt time.Time `json:"archivedAt"`
+}
+
+func (r ArchiveReceipt) Validate() error {
+	if r.Version < 1 || r.Path == "" || r.ArchivedAt.IsZero() || len(r.Checksum) != 64 {
+		return invalid("incomplete archive receipt")
+	}
+	for _, c := range r.Checksum {
+		if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
+			return invalid("archive checksum must be lowercase SHA-256 hex")
+		}
+	}
+	return nil
+}
+
+// RunArchive is summary-only: never embed per-card task/failure records.
+type RunArchive struct {
+	Version int          `json:"version"`
+	Run     RunMeta      `json:"run"`
+	Rounds  []RoundStats `json:"rounds"`
+}
+
+// ArchiveWork.Version is an opaque concurrency token for registration; it is
+// distinct from the archive schema version. Work discovery must survive restarts.
+type ArchiveWork struct {
+	RunID   string `json:"runId"`
+	Version string `json:"-"`
+}
+
+type ArchiveWorkPage struct {
+	Items      []ArchiveWork `json:"items"`
+	NextCursor Cursor        `json:"nextCursor"`
+}

worker-svr/internal/fullsync/contract/compatibility_test.go

@@ -0,0 +1,107 @@
+package contract_test
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+// Check every old field, including zero/omitempty behavior, so future adapters
+// can adopt the contract without silently changing the persisted wire format.
+func TestLegacyFieldSchemas(t *testing.T) {
+	for _, pair := range []struct{ old, next any }{
+		{zset.RunMeta{}, contract.RunMeta{}},
+		{zset.FailureRecord{}, contract.FailureRecord{}},
+		{zset.Task{}, contract.Task{}},
+	} {
+		old, next := reflect.TypeOf(pair.old), reflect.TypeOf(pair.next)
+		for i := 0; i < old.NumField(); i++ {
+			field := old.Field(i)
+			replacement, ok := next.FieldByName(field.Name)
+			if !ok || replacement.Tag.Get("json") != field.Tag.Get("json") || replacement.Type.Kind() != field.Type.Kind() {
+				t.Errorf("%s.%s lost its legacy schema: old=%+v new=%+v", old.Name(), field.Name, field, replacement)
+			}
+		}
+	}
+}
+
+func TestLegacyRunJSONRoundTrip(t *testing.T) {
+	now := time.Date(2026, 9, 20, 1, 2, 3, 456, time.UTC)
+	legacy := zset.RunMeta{
+		RunID: "legacy", Operation: "current_month_usage_full_sync", QueryMonth: "202609",
+		Status: "completed_with_failures", ProducerDone: true, CurrentRound: 5,
+		SuccessCount: 9, FailedCount: 1, CreatedAt: now, ProducerCompletedAt: now, CompletedAt: now,
+	}
+	data := mustJSON(t, legacy)
+	var decoded contract.RunMeta
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if err := decoded.Validate(); err != nil {
+		t.Fatalf("legacy run rejected: %v", err)
+	}
+	if got := string(mustJSON(t, decoded)); got != string(data) {
+		t.Fatalf("legacy wire format changed:\n%s\n%s", data, got)
+	}
+	// New fields do not prevent old readers from consuming the same metadata.
+	decoded.SyncType = contract.SyncReplay
+	decoded.SourceRunID = "source"
+	decoded.ReplayCycle = 1
+	decoded.EnqueuedCount = 10
+	decoded.PendingFailureCount = 1
+	var oldReader zset.RunMeta
+	if err := json.Unmarshal(mustJSON(t, decoded), &oldReader); err != nil {
+		t.Fatal(err)
+	}
+	oldReader.SyncType = ""
+	oldReader.SourceRunID = ""
+	oldReader.ReplayCycle = 0
+	oldReader.EnqueuedCount = 0
+	oldReader.PendingFailureCount = 0
+	if !reflect.DeepEqual(oldReader, legacy) {
+		t.Fatalf("old reader lost fields: %+v", oldReader)
+	}
+}
+
+func TestLegacyFailureAndTaskJSON(t *testing.T) {
+	legacy := zset.FailureRecord{
+		Version: 1, TaskID: "source:op:42", OriginalRunID: "source", LatestRunID: "latest",
+		QueryMonth: "202609", Operation: "op", CardID: 42, ICCID: "iccid", MSISDN: "msisdn", IMSI: "imsi",
+		ChanKey: "chan", TempKey: "template", Carrier: "mobile", FailureType: "mobile_busy_exhausted",
+		Round: 5, Attempts: 7, Code: 13000, ErrCode: "BUSY", Message: "try again",
+		ReviewStatus: "pending", ReviewRemark: "remark", ReviewedBy: "operator", ReviewedAt: "2026-09-20",
+		RetryCycle: 2, ReplayRunID: "replayed", FirstFailedAt: time.Unix(1, 0).UTC(), LastFailedAt: time.Unix(2, 0).UTC(),
+	}
+	var record contract.FailureRecord
+	if err := json.Unmarshal(mustJSON(t, legacy), &record); err != nil {
+		t.Fatal(err)
+	}
+	if string(mustJSON(t, record)) != string(mustJSON(t, legacy)) {
+		t.Fatal("failure record wire format changed")
+	}
+	if summary := record.Summary(); summary.ErrorCategory != contract.ErrorRateLimited || summary.TaskID != legacy.TaskID || summary.FailedAt != legacy.LastFailedAt {
+		t.Fatalf("incorrect failure projection: %+v", summary)
+	}
+	task := zset.Task{TaskID: "r:op:42", RunID: "r", Operation: "op", QueryMonth: "202609", ChanKey: "c", TempKey: "t", Operator: 2, CardID: 42, ICCID: "i", MSISDN: "m", IMSI: "s", NextExecuteAt: 1234}
+	var newTask contract.Task
+	if err := json.Unmarshal(mustJSON(t, task), &newTask); err != nil {
+		t.Fatal(err)
+	}
+	newTask.NextExecuteAt = 9999
+	if string(mustJSON(t, newTask)) != string(mustJSON(t, task)) {
+		t.Fatal("task wire format or scheduling exclusion changed")
+	}
+}
+
+func mustJSON(t *testing.T, value any) []byte {
+	t.Helper()
+	data, err := json.Marshal(value)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}

worker-svr/internal/fullsync/contract/doc.go

@@ -0,0 +1,14 @@
+// Package contract defines the shared full-sync domain and service boundaries.
+// It contains no Redis, filesystem, replay execution, or HTTP handler code.
+//
+// Legacy queue models remain in internal/queue/zset until adapters adopt this
+// package. Their existing JSON fields are retained here; new run fields are
+// optional so historical records still decode. Missing extended counters mean
+// unknown, not a reconstructed total. FailedCount is the historical terminal
+// count for a run; cleaning source failures changes PendingFailureCount only.
+//
+// Implementations must preserve these boundaries: run-local Redis keys share
+// one hash tag, global indexes do not; archive summaries never contain card
+// details; cursors are opaque and scoped to the original query; replay writes
+// are idempotent and fenced by a lease before source cleanup is allowed.
+package contract

worker-svr/internal/fullsync/contract/errors.go

@@ -0,0 +1,112 @@
+package contract
+
+import (
+	"encoding/json"
+	"math"
+	"strings"
+	"unicode/utf8"
+)
+
+type ErrorCategory string
+
+const (
+	ErrorTimeout         ErrorCategory = "timeout"
+	ErrorRateLimited     ErrorCategory = "rate_limited"
+	ErrorAuthentication  ErrorCategory = "authentication"
+	ErrorValidation      ErrorCategory = "validation"
+	ErrorUpstream        ErrorCategory = "upstream"
+	ErrorTransport       ErrorCategory = "transport"
+	ErrorInternal        ErrorCategory = "internal"
+	ErrorOther           ErrorCategory = "other"
+	MaxErrorCategories                 = 8
+	MaxErrorMessageBytes               = 2048
+)
+
+// NormalizeErrorCategory maps the existing consumer classifications into the
+// approved, bounded vocabulary. Never use an upstream message/code as a key.
+func NormalizeErrorCategory(value string) ErrorCategory {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "timeout":
+		return ErrorTimeout
+	case "rate_limited", "mobile_busy", "mobile_busy_exhausted", "channel_limited", "channel_blocked":
+		return ErrorRateLimited
+	case "authentication":
+		return ErrorAuthentication
+	case "validation":
+		return ErrorValidation
+	case "upstream", "business", "carrier_retry":
+		return ErrorUpstream
+	case "transport", "network":
+		return ErrorTransport
+	case "internal", "canceled":
+		return ErrorInternal
+	default:
+		return ErrorOther
+	}
+}
+
+func (c ErrorCategory) MarshalJSON() ([]byte, error) {
+	return json.Marshal(string(NormalizeErrorCategory(string(c))))
+}
+
+func (c *ErrorCategory) UnmarshalJSON(data []byte) error {
+	var value string
+	if err := json.Unmarshal(data, &value); err != nil {
+		return invalid("error category must be a string: %v", err)
+	}
+	*c = NormalizeErrorCategory(value)
+	return nil
+}
+
+// ErrorCounts is normalized on both JSON input and output, including map keys
+// (encoding/json does not invoke ErrorCategory.MarshalJSON for map keys).
+// Counts are per category, not card IDs or individual error messages.
+type ErrorCounts map[ErrorCategory]int64
+
+func (counts ErrorCounts) Normalized() (ErrorCounts, error) {
+	normalized := make(ErrorCounts)
+	for category, count := range counts {
+		key := NormalizeErrorCategory(string(category))
+		if count < 0 || count > math.MaxInt64-normalized[key] {
+			return nil, invalid("error counts must be nonnegative and must not overflow")
+		}
+		normalized[key] += count
+	}
+	return normalized, nil
+}
+
+func (counts ErrorCounts) MarshalJSON() ([]byte, error) {
+	normalized, err := counts.Normalized()
+	if err != nil {
+		return nil, err
+	}
+	type wireCounts map[ErrorCategory]int64
+	return json.Marshal(wireCounts(normalized))
+}
+
+func (counts *ErrorCounts) UnmarshalJSON(data []byte) error {
+	var decoded map[ErrorCategory]int64
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return invalid("invalid error counts: %v", err)
+	}
+	normalized, err := ErrorCounts(decoded).Normalized()
+	if err != nil {
+		return err
+	}
+	*counts = normalized
+	return nil
+}
+
+// BoundedErrorMessage makes new summary/action error messages valid UTF-8 and
+// caps their byte length. It does not modify legacy FailureRecord.Message.
+func BoundedErrorMessage(message string) string {
+	message = strings.ToValidUTF8(message, "\uFFFD")
+	if len(message) <= MaxErrorMessageBytes {
+		return message
+	}
+	end := MaxErrorMessageBytes
+	for !utf8.RuneStart(message[end]) {
+		end--
+	}
+	return message[:end]
+}

worker-svr/internal/fullsync/contract/errors_test.go

@@ -0,0 +1,91 @@
+package contract_test
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestBoundedErrorCategories(t *testing.T) {
+	for raw, want := range map[string]contract.ErrorCategory{
+		"timeout": contract.ErrorTimeout, " RATE_LIMITED ": contract.ErrorRateLimited,
+		"authentication": contract.ErrorAuthentication, "validation": contract.ErrorValidation,
+		"upstream": contract.ErrorUpstream, "transport": contract.ErrorTransport,
+		"internal": contract.ErrorInternal, "other": contract.ErrorOther,
+		"network": contract.ErrorTransport, "mobile_busy": contract.ErrorRateLimited,
+		"mobile_busy_exhausted": contract.ErrorRateLimited, "channel_blocked": contract.ErrorRateLimited,
+		"channel_limited": contract.ErrorRateLimited, "business": contract.ErrorUpstream,
+		"carrier_retry": contract.ErrorUpstream, "canceled": contract.ErrorInternal,
+		"": contract.ErrorOther, "card-898600-unique-error": contract.ErrorOther,
+	} {
+		if got := contract.NormalizeErrorCategory(raw); got != want {
+			t.Errorf("normalize %q = %q, want %q", raw, got, want)
+		}
+		var decoded contract.ErrorCategory
+		if err := json.Unmarshal(mustJSON(t, raw), &decoded); err != nil || decoded != want {
+			t.Errorf("decode %q = %q, err %v", raw, decoded, err)
+		}
+		if got := string(mustJSON(t, contract.ErrorCategory(raw))); got != string(mustJSON(t, string(want))) {
+			t.Errorf("encode %q = %s", raw, got)
+		}
+	}
+}
+
+func TestErrorCountsCannotGrowWithCardinality(t *testing.T) {
+	counts := contract.ErrorCounts{contract.ErrorTimeout: 2, contract.ErrorOther: 3, "network": 4}
+	for i := 0; i < 1000; i++ {
+		counts[contract.ErrorCategory(fmt.Sprintf("card-error-%d", i))] = 1
+	}
+	var decoded contract.ErrorCounts
+	if err := json.Unmarshal(mustJSON(t, counts), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded) != 3 || len(decoded) > contract.MaxErrorCategories || decoded[contract.ErrorOther] != 1003 || decoded[contract.ErrorTransport] != 4 {
+		t.Fatalf("unbounded or lost counts: %+v", decoded)
+	}
+	if err := json.Unmarshal([]byte(`{"unknown":2,"different":3,"other":4,"network":5}`), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded) != 2 || decoded[contract.ErrorOther] != 9 || decoded[contract.ErrorTransport] != 5 {
+		t.Fatalf("input normalization failed: %+v", decoded)
+	}
+}
+
+func TestErrorCountsRejectInvalidNumbers(t *testing.T) {
+	for _, raw := range []string{`{"timeout":-1}`, `{"one":9223372036854775807,"two":1}`, `{"other":1.5}`, `{"other":"1"}`} {
+		var counts contract.ErrorCounts
+		if err := json.Unmarshal([]byte(raw), &counts); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("invalid input %s returned %v", raw, err)
+		}
+	}
+	for _, counts := range []contract.ErrorCounts{{"timeout": -1}, {"one": math.MaxInt64, "two": 1}} {
+		if _, err := json.Marshal(counts); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("invalid output counts returned %v", err)
+		}
+	}
+}
+
+func TestBoundedFailureMessage(t *testing.T) {
+	for _, message := range []string{"short", strings.Repeat("x", 3000), strings.Repeat("卡", 1000), strings.Repeat("x", 2047) + "卡", "invalid\xfftext"} {
+		got := contract.BoundedErrorMessage(message)
+		if len(got) > contract.MaxErrorMessageBytes || !utf8.ValidString(got) {
+			t.Fatalf("invalid bounded output: bytes=%d", len(got))
+		}
+		if utf8.ValidString(message) && len(message) <= contract.MaxErrorMessageBytes && got != message {
+			t.Fatal("short message changed")
+		}
+	}
+	record := contract.FailureRecord{TaskID: "task", Message: strings.Repeat("卡", 1000)}
+	if err := record.Summary().Validate(); err != nil {
+		t.Fatal(err)
+	}
+	if len(record.Message) != 3000 {
+		t.Fatal("summary mutated legacy failure message")
+	}
+}

worker-svr/internal/fullsync/contract/failure.go

@@ -0,0 +1,129 @@
+package contract
+
+import (
+	"time"
+	"unicode/utf8"
+)
+
+// Cursor is an opaque repository-issued continuation token. Clients must not
+// parse, increment, or reuse it for another run/filter. Empty starts a scan;
+// an empty nextCursor ends it. An empty page with a cursor is not end-of-scan.
+// Repositories validate token contents and query binding with ErrInvalidArgument.
+type Cursor string
+
+const (
+	DefaultPageLimit = 100
+	MaxPageLimit     = 500
+)
+
+type PageRequest struct {
+	Cursor Cursor `json:"cursor,omitempty"`
+	Limit  int    `json:"limit,omitempty"`
+}
+
+// Normalized must be called at both HTTP and repository boundaries. Limits
+// outside the bounds are rejected, not silently truncated.
+func (p PageRequest) Normalized() (PageRequest, error) {
+	if p.Limit == 0 {
+		p.Limit = DefaultPageLimit
+	}
+	if p.Limit < 1 || p.Limit > MaxPageLimit {
+		return PageRequest{}, invalid("limit must be between 1 and %d", MaxPageLimit)
+	}
+	return p, nil
+}
+
+// Failure is the bounded public projection, not the replay payload. Legacy
+// free-form failureType/message fields remain available in FailureRecord.
+type Failure struct {
+	TaskID        string        `json:"taskId"`
+	CardID        int64         `json:"cardId"`
+	ErrorCategory ErrorCategory `json:"errorCategory"`
+	ErrorMessage  string        `json:"errorMessage"`
+	FailedAt      time.Time     `json:"failedAt"`
+	Round         int           `json:"round"`
+}
+
+func (f Failure) Validate() error {
+	if f.TaskID == "" || f.Round < 0 || len(f.ErrorMessage) > MaxErrorMessageBytes || !utf8.ValidString(f.ErrorMessage) {
+		return invalid("invalid failure summary")
+	}
+	return nil
+}
+
+type FailurePage struct {
+	Items      []Failure `json:"items"`
+	NextCursor Cursor    `json:"nextCursor"`
+}
+
+// FailureRecord preserves the existing persisted payload without changing
+// review semantics or discarding fields needed to reconstruct replay tasks.
+type FailureRecord struct {
+	Version       int    `json:"version"`
+	TaskID        string `json:"taskId"`
+	OriginalRunID string `json:"originalRunId"`
+	LatestRunID   string `json:"latestRunId"`
+	QueryMonth    string `json:"queryMonth"`
+	Operation     string `json:"operation"`
+	CardID        int64  `json:"cardId"`
+	ICCID         string `json:"iccid"`
+	MSISDN        string `json:"msisdn"`
+	IMSI          string `json:"imsi"`
+	ChanKey       string `json:"chanKey"`
+	TempKey       string `json:"tempKey"`
+	// PackageCode is additive; older failure records decode it as empty.
+	PackageCode   string    `json:"packageCode,omitempty"`
+	Carrier       string    `json:"carrier"`
+	FailureType   string    `json:"failureType"`
+	Round         int       `json:"round"`
+	Attempts      int       `json:"attempts"`
+	Code          int       `json:"code"`
+	ErrCode       string    `json:"errCode"`
+	Message       string    `json:"message"`
+	ReviewStatus  string    `json:"reviewStatus"`
+	ReviewRemark  string    `json:"reviewRemark"`
+	ReviewedBy    string    `json:"reviewedBy"`
+	ReviewedAt    string    `json:"reviewedAt"`
+	RetryCycle    int       `json:"retryCycle"`
+	ReplayRunID   string    `json:"replayRunId"`
+	FirstFailedAt time.Time `json:"firstFailedAt"`
+	LastFailedAt  time.Time `json:"lastFailedAt"`
+}
+
+func (f FailureRecord) Summary() Failure {
+	return Failure{
+		TaskID: f.TaskID, CardID: f.CardID, ErrorCategory: NormalizeErrorCategory(f.FailureType),
+		ErrorMessage: BoundedErrorMessage(f.Message), FailedAt: f.LastFailedAt, Round: f.Round,
+	}
+}
+
+// FailureEntry carries an opaque version for conditional source cleanup. It
+// must identify the precise record read, not just its reusable task ID.
+type FailureEntry struct {
+	Record  FailureRecord `json:"record"`
+	Version string        `json:"-"`
+}
+
+type FailureRecordPage struct {
+	Items      []FailureEntry `json:"items"`
+	NextCursor Cursor         `json:"nextCursor"`
+}
+
+// Task keeps the existing queue member serialization. The scheduling score
+// must never be part of the JSON member or producer retries lose idempotency.
+type Task struct {
+	TaskID     string `json:"taskId"`
+	RunID      string `json:"runId"`
+	Operation  string `json:"operation"`
+	QueryMonth string `json:"queryMonth"`
+	ChanKey    string `json:"chanKey"`
+	TempKey    string `json:"tempKey"`
+	Operator   int    `json:"operator,omitempty"`
+	CardID     int64  `json:"cardId"`
+	ICCID      string `json:"iccid"`
+	MSISDN     string `json:"msisdn"`
+	IMSI       string `json:"imsi"`
+	// PackageCode is additive; older members decode it as empty.
+	PackageCode   string `json:"packageCode,omitempty"`
+	NextExecuteAt int64  `json:"-"`
+}

worker-svr/internal/fullsync/contract/http.go

@@ -0,0 +1,28 @@
+package contract
+
+import "net/url"
+
+const (
+	ListRunsRoute     = "/internal/v1/full-sync-runs"
+	GetRunRoute       = "/internal/v1/full-sync-runs/{runId}"
+	GetFailuresRoute  = "/internal/v1/full-sync-runs/{runId}/failures"
+	PostReplayRoute   = "/internal/v1/full-sync-runs/{runId}/replay"
+	GetActionRoute    = "/internal/v1/full-sync-actions/{actionId}"
+	IdempotencyHeader = "Idempotency-Key"
+)
+
+// ReplayAccepted is the 202 response from a successful ReplayRequester call.
+// A repeated request returns the original action and its current status.
+type ReplayAccepted struct {
+	ActionID    string       `json:"actionId"`
+	TargetRunID string       `json:"targetRunId"`
+	Status      ActionStatus `json:"status"`
+	StatusURL   string       `json:"statusUrl"`
+}
+
+func ActionStatusURL(actionID string) (string, error) {
+	if err := ValidateID(actionID); err != nil {
+		return "", err
+	}
+	return "/internal/v1/full-sync-actions/" + url.PathEscape(actionID), nil
+}

worker-svr/internal/fullsync/contract/interfaces.go

@@ -0,0 +1,145 @@
+package contract
+
+import (
+	"context"
+	"time"
+)
+
+// RunReader is the HTTP lookup boundary. Fall back to archive only when the
+// live reader returns ErrNotFound, not on transient live storage failures.
+type RunReader interface {
+	GetRun(ctx context.Context, runID string) (RunView, error)
+}
+
+type RunListFilter struct {
+	SyncKind SyncKind
+	Limit    int
+}
+
+type RunList struct {
+	Items []RunView `json:"items"`
+	Total int       `json:"total"`
+}
+
+type RunLister interface {
+	ListRuns(ctx context.Context, filter RunListFilter) (RunList, error)
+}
+
+type LiveRunReader interface {
+	GetRunMeta(ctx context.Context, runID string) (RunMeta, error)
+}
+
+type ArchivedRunReader interface {
+	GetArchivedRun(ctx context.Context, runID string) (RunArchive, error)
+}
+
+type RoundStatsReader interface {
+	GetRoundStats(ctx context.Context, runID string) ([]RoundStats, error)
+}
+
+// FailurePager returns live details only. An archived summary without live
+// details returns ErrFailureDetailsUnavailable, never a misleading empty page.
+// Items must serialize as [] on empty pages; NextCursor alone signals EOF.
+type FailurePager interface {
+	ListFailures(ctx context.Context, runID string, page PageRequest) (FailurePage, error)
+}
+
+type ReplayRequester interface {
+	RequestReplay(ctx context.Context, request ReplayRequest) (ReplayAccepted, error)
+}
+
+type ActionReader interface {
+	GetReplayAction(ctx context.Context, actionID string) (ReplayAction, error)
+}
+
+// ArchiveWorkDiscovery returns pending/retryable work from a durable index.
+// Terminal run registration must not depend on an in-memory notification.
+type ArchiveWorkDiscovery interface {
+	ListArchiveWork(ctx context.Context, page PageRequest) (ArchiveWorkPage, error)
+}
+
+// RegisterArchiveWork idempotently schedules a terminal run/version. Since the
+// index can occupy another Redis slot, callers must reconcile registration
+// after a crash between completing the run and updating the index.
+type ArchiveWorkRegistrar interface {
+	RegisterArchiveWork(ctx context.Context, work ArchiveWork) error
+}
+
+type ArchiveWorkLoader interface {
+	LoadArchiveWork(ctx context.Context, work ArchiveWork) (RunArchive, error)
+}
+
+// ArchiveRegistrar conditionally records a durable publication against the
+// work version. Repeating the same receipt is a no-op; a different version or
+// conflicting receipt returns ErrConflict. Failure retains discoverable work.
+type ArchiveRegistrar interface {
+	RegisterArchive(ctx context.Context, work ArchiveWork, receipt ArchiveReceipt) error
+	RegisterArchiveFailure(ctx context.Context, work ArchiveWork, message string) error
+}
+
+// ReplayActionCreator atomically reserves (sourceRunId, idempotencyKey) with
+// its fingerprint and action. Same key/request returns the original action
+// regardless of candidate IDs; different request returns ErrConflict. The
+// candidate starts accepted and fixes the target ID before any target writes.
+type ReplayActionCreator interface {
+	GetOrCreateReplayAction(ctx context.Context, request ReplayRequest, candidate ReplayAction) (ReplayActionCreation, error)
+}
+
+// Discovery includes accepted, processing, cleanup_pending and receipt_pending
+// actions after crashes. Completed/needs_attention actions require no automatic
+// work. Lease acquisition, not listing, chooses an exclusive owner.
+type ReplayWorkDiscovery interface {
+	ListReplayWork(ctx context.Context, page PageRequest) (ReplayActionPage, error)
+}
+
+type ReplayLeaseStore interface {
+	AcquireReplayLease(ctx context.Context, actionID, owner string, ttl time.Duration) (ReplayLease, error)
+	RenewReplayLease(ctx context.Context, lease ReplayLease, ttl time.Duration) (ReplayLease, error)
+	ReleaseReplayLease(ctx context.Context, lease ReplayLease) error
+}
+
+// All updates are fenced and compare ExpectedRevision. Expired/replaced leases
+// return ErrLeaseLost; revision conflicts return ErrConflict. Implementations
+// reject progress regression and preserve immutable action identity.
+type ReplayProgressStore interface {
+	UpdateReplayAction(ctx context.Context, update ReplayActionUpdate) (ReplayAction, error)
+}
+
+// The implementation must preserve scan continuity while source entries are
+// cleaned. This can use a durable selection/snapshot; deleting records must not
+// make a later cursor skip an unprocessed failure. Tokens are bound to actionID.
+type ReplayFailurePager interface {
+	ListReplayFailures(ctx context.Context, actionID string, page PageRequest) (FailureRecordPage, error)
+}
+
+// TargetRunCreator is idempotent: retrying the same target preserves counters;
+// incompatible source/operation/month/cycle returns ErrConflict. Replay targets
+// are distinct runs and must not mutate the source's historical FailedCount.
+type TargetRunCreator interface {
+	EnsureTargetRun(ctx context.Context, meta RunMeta) (RunMeta, error)
+}
+
+// TargetEnqueueResult distinguishes a durable duplicate from a rejected write.
+// A zero NewlyEnqueued count alone is never sufficient evidence for cleanup.
+// Every input TaskID must be durably present on nil error; otherwise return an
+// error and let the caller safely retry the identical batch. Both counts are
+// unique task counts, and their sum equals the number of distinct input IDs.
+type TargetEnqueueResult struct {
+	NewlyEnqueued  int64
+	AlreadyPresent int64
+}
+
+type TargetRunEnqueuer interface {
+	// Tasks must match the target run and channel and use stable logical IDs.
+	EnqueueTargetBatch(ctx context.Context, lease ReplayLease, runID, chanKey string, tasks []Task, nextExecuteAt time.Time) (TargetEnqueueResult, error)
+	MarkTargetProducerDone(ctx context.Context, lease ReplayLease, runID string) error
+}
+
+// Cleanup is idempotent and conditional on the lease, exact source versions,
+// and durable evidence of target acceptance. It changes pending failures only,
+// never historical failedCount. Conflicts must remain visible for attention.
+// Source keys and action/target keys may occupy different Redis Cluster slots;
+// implementations need recoverable receipts, not cross-slot Lua assumptions.
+type ReplayFailureCleaner interface {
+	CleanupReplayFailures(ctx context.Context, cleanup ReplayCleanup) (ReplayCleanupResult, error)
+}

worker-svr/internal/fullsync/contract/keys.go

@@ -0,0 +1,116 @@
+package contract
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const (
+	DefaultFlowPrefix      = "flow:"
+	DefaultSimStatusPrefix = "sim-status:"
+)
+
+// Keys is a naming contract only. It does not create indexes or storage.
+// Existing run names are byte-for-byte compatible with zset.Store. Global
+// run/archive indexes are outside run hash slots and cannot join run-local Lua.
+type Keys struct{ prefix string }
+
+func NewKeys(prefix string) (Keys, error) {
+	if strings.TrimSpace(prefix) == "" {
+		prefix = DefaultFlowPrefix
+	}
+	if strings.ContainsAny(prefix, "{}") {
+		return Keys{}, invalid("prefix must not contain Redis hash tags")
+	}
+	return Keys{prefix: prefix}, nil
+}
+
+// The zero value uses the legacy flow prefix, just like NewKeys("").
+func (k Keys) Prefix() string {
+	if k.prefix == "" {
+		return DefaultFlowPrefix
+	}
+	return k.prefix
+}
+
+func (k Keys) runKey(runID, suffix string) (string, error) {
+	if err := ValidateID(runID); err != nil {
+		return "", err
+	}
+	return k.Prefix() + "{" + runID + "}:" + suffix, nil
+}
+
+func (k Keys) RunMetaKey(runID string) (string, error)      { return k.runKey(runID, "meta") }
+func (k Keys) RunChannelsKey(runID string) (string, error)  { return k.runKey(runID, "channels") }
+func (k Keys) TaskStateKey(runID string) (string, error)    { return k.runKey(runID, "task-state") }
+func (k Keys) FailedKey(runID string) (string, error)       { return k.runKey(runID, "failed") }
+func (k Keys) ArchiveStateKey(runID string) (string, error) { return k.runKey(runID, "archive") }
+
+func (k Keys) roundKey(runID string, round int, suffix string) (string, error) {
+	if round < 1 {
+		return "", invalid("round must be positive")
+	}
+	return k.runKey(runID, fmt.Sprintf("round:%d:%s", round, suffix))
+}
+
+func (k Keys) RoundStatsKey(runID string, round int) (string, error) {
+	return k.roundKey(runID, round, "stats")
+}
+
+func (k Keys) roundChannelKey(runID string, round int, kind, chanKey string) (string, error) {
+	if strings.TrimSpace(chanKey) == "" {
+		return "", invalid("chanKey is empty")
+	}
+	return k.roundKey(runID, round, kind+":"+chanKey)
+}
+
+func (k Keys) RoundActiveKey(runID string, round int, chanKey string) (string, error) {
+	return k.roundChannelKey(runID, round, "active", chanKey)
+}
+
+func (k Keys) RoundWaitingKey(runID string, round int, chanKey string) (string, error) {
+	return k.roundChannelKey(runID, round, "waiting", chanKey)
+}
+
+func (k Keys) RoundLeaseKey(runID string, round int, chanKey string) (string, error) {
+	return k.roundChannelKey(runID, round, "lease", chanKey)
+}
+
+func (k Keys) FailedRunsKey() string      { return k.Prefix() + "failed:runs" }
+func (k Keys) ActiveRunsKey() string      { return k.Prefix() + "runs:active" }
+func (k Keys) AllRunsKey() string         { return k.Prefix() + "runs:all" }
+func (k Keys) PendingArchivesKey() string { return k.Prefix() + "archive:pending" }
+
+// Replay action state, leases, idempotency reservations and pending index share
+// a dedicated slot, allowing atomic acceptance/discovery. Source/target run
+// writes remain in their own slots and require a recoverable replay protocol.
+func (k Keys) replayKey(actionID, suffix string) (string, error) {
+	if err := ValidateID(actionID); err != nil {
+		return "", err
+	}
+	return k.Prefix() + "{replay}:action:" + actionID + ":" + suffix, nil
+}
+
+func (k Keys) ReplayActionKey(actionID string) (string, error) { return k.replayKey(actionID, "meta") }
+func (k Keys) ReplayLeaseKey(actionID string) (string, error)  { return k.replayKey(actionID, "lease") }
+func (k Keys) PendingReplayActionsKey() string                 { return k.Prefix() + "{replay}:pending" }
+
+// Hash the complete tuple so neither delimiter collisions nor arbitrary
+// Idempotency-Key characters can change the Redis key or hash slot.
+func (k Keys) ReplayIdempotencyKey(sourceRunID, idempotencyKey string) (string, error) {
+	if err := ValidateID(sourceRunID); err != nil {
+		return "", err
+	}
+	if err := ValidateIdempotencyKey(idempotencyKey); err != nil {
+		return "", err
+	}
+	data, err := json.Marshal([2]string{sourceRunID, idempotencyKey})
+	if err != nil {
+		return "", invalid("cannot encode idempotency scope: %v", err)
+	}
+	digest := sha256.Sum256(data)
+	return k.Prefix() + "{replay}:idempotency:" + hex.EncodeToString(digest[:]), nil
+}

worker-svr/internal/fullsync/contract/keys_test.go

@@ -0,0 +1,116 @@
+package contract_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+
+	"github.com/gogf/gf/v2/container/gvar"
+)
+
+type unusedRedis struct{}
+
+func (unusedRedis) Do(context.Context, string, ...any) (*gvar.Var, error) {
+	panic("naming contracts must not access Redis")
+}
+
+func (unusedRedis) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	panic("naming contracts must not access Redis")
+}
+
+func TestKeysMatchExistingQueue(t *testing.T) {
+	for _, prefix := range []string{"", "flow:", "sim-status:", "custom-prefix"} {
+		keys, err := contract.NewKeys(prefix)
+		if err != nil {
+			t.Fatal(err)
+		}
+		store, err := zset.New(unusedRedis{}, prefix)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, pair := range []struct {
+			old, next func(string) (string, error)
+		}{
+			{store.RunMetaKey, keys.RunMetaKey}, {store.RunChannelsKey, keys.RunChannelsKey},
+			{store.TaskStateKey, keys.TaskStateKey}, {store.FailedKey, keys.FailedKey},
+		} {
+			old, oldErr := pair.old("run:1")
+			next, nextErr := pair.next("run:1")
+			if oldErr != nil || nextErr != nil || old != next || !strings.Contains(next, "{run:1}") {
+				t.Errorf("run naming changed: old=%q new=%q errors=%v/%v", old, next, oldErr, nextErr)
+			}
+		}
+		for _, pair := range []struct {
+			old, next func(string, int, string) (string, error)
+		}{
+			{store.RoundActiveKey, keys.RoundActiveKey}, {store.RoundWaitingKey, keys.RoundWaitingKey},
+			{store.RoundLeaseKey, keys.RoundLeaseKey},
+		} {
+			old, oldErr := pair.old("run:1", 3, "channel")
+			next, nextErr := pair.next("run:1", 3, "channel")
+			if oldErr != nil || nextErr != nil || old != next {
+				t.Errorf("round naming changed: old=%q new=%q errors=%v/%v", old, next, oldErr, nextErr)
+			}
+		}
+		if keys.ActiveRunsKey() != store.ActiveRunsKey() || keys.FailedRunsKey() != store.FailedRunsKey() {
+			t.Fatal("existing global index names changed")
+		}
+	}
+}
+
+func TestNewKeysAndClusterBoundaries(t *testing.T) {
+	keys, _ := contract.NewKeys("flow:")
+	stats, _ := keys.RoundStatsKey("run", 2)
+	archive, _ := keys.ArchiveStateKey("run")
+	if stats != "flow:{run}:round:2:stats" || archive != "flow:{run}:archive" {
+		t.Fatalf("new run keys changed: %s, %s", stats, archive)
+	}
+	action, _ := keys.ReplayActionKey("a")
+	lease, _ := keys.ReplayLeaseKey("a")
+	idempotency, _ := keys.ReplayIdempotencyKey("source", "a{different-slot}:key")
+	for _, key := range []string{action, lease, idempotency, keys.PendingReplayActionsKey()} {
+		if !strings.HasPrefix(key, "flow:{replay}:") || strings.Count(key, "{") != 1 {
+			t.Fatalf("replay key escaped action slot: %s", key)
+		}
+	}
+	if action != "flow:{replay}:action:a:meta" || lease != "flow:{replay}:action:a:lease" {
+		t.Fatal("action naming changed")
+	}
+	for _, key := range []string{keys.ActiveRunsKey(), keys.FailedRunsKey(), keys.AllRunsKey(), keys.PendingArchivesKey()} {
+		if strings.ContainsAny(key, "{}") {
+			t.Fatalf("global index unexpectedly shares run slot: %s", key)
+		}
+	}
+	first, _ := keys.ReplayIdempotencyKey("a:b", "c")
+	second, _ := keys.ReplayIdempotencyKey("a", "b:c")
+	repeat, _ := keys.ReplayIdempotencyKey("a:b", "c")
+	otherSource, _ := keys.ReplayIdempotencyKey("other", "c")
+	if first == second || first == otherSource || first != repeat {
+		t.Fatal("idempotency scope is ambiguous or unstable")
+	}
+}
+
+func TestInvalidKeyParts(t *testing.T) {
+	keys := contract.Keys{}
+	for _, id := range []string{"", " ", "trim ", "{run}", "run}", "run/path", "run\\path", "run\n", "invalid\xff"} {
+		if _, err := keys.RunMetaKey(id); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("invalid ID %q returned %v", id, err)
+		}
+	}
+	if _, err := contract.NewKeys("prefix:{wrong}:"); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("invalid prefix returned %v", err)
+	}
+	if _, err := keys.RoundStatsKey("run", 0); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("invalid round returned %v", err)
+	}
+	if _, err := keys.RoundActiveKey("run", 1, " "); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("invalid channel returned %v", err)
+	}
+	if key, err := keys.RunMetaKey("run"); err != nil || key != "flow:{run}:meta" {
+		t.Fatalf("zero-value keys differ from legacy default: %s %v", key, err)
+	}
+}

worker-svr/internal/fullsync/contract/replay.go

@@ -0,0 +1,172 @@
+package contract
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+const MaxIdempotencyKeyBytes = 256
+
+// ReplayRequest combines the path, header and body. Only requestedBy is in the
+// HTTP body. Idempotency keys are scoped to the source run; fingerprints cover
+// the source and exact request body, not the key or a generated target run ID.
+type ReplayRequest struct {
+	SourceRunID    string `json:"-"`
+	IdempotencyKey string `json:"-"`
+	RequestedBy    string `json:"requestedBy"`
+}
+
+func ValidateIdempotencyKey(key string) error {
+	if strings.TrimSpace(key) == "" || len(key) > MaxIdempotencyKeyBytes || !utf8.ValidString(key) || strings.ContainsFunc(key, unicode.IsControl) {
+		return invalid("invalid Idempotency-Key")
+	}
+	return nil
+}
+
+func (r ReplayRequest) Validate() error {
+	if err := ValidateID(r.SourceRunID); err != nil {
+		return err
+	}
+	if err := ValidateIdempotencyKey(r.IdempotencyKey); err != nil {
+		return err
+	}
+	if strings.TrimSpace(r.RequestedBy) == "" || !utf8.ValidString(r.RequestedBy) {
+		return invalid("requestedBy is required")
+	}
+	return nil
+}
+
+func (r ReplayRequest) Fingerprint() (string, error) {
+	if err := r.Validate(); err != nil {
+		return "", err
+	}
+	// JSON avoids ambiguous delimiter concatenation and fixes field ordering.
+	data, err := json.Marshal(struct {
+		SourceRunID string `json:"sourceRunId"`
+		RequestedBy string `json:"requestedBy"`
+	}{r.SourceRunID, r.RequestedBy})
+	if err != nil {
+		return "", invalid("cannot encode replay request: %v", err)
+	}
+	digest := sha256.Sum256(data)
+	return hex.EncodeToString(digest[:]), nil
+}
+
+// Counters describe distinct source failures, not scan visits or retry calls.
+// ScanCursor alone is not an enqueue/cleanup checkpoint: repositories must
+// retain sufficient durable per-batch evidence to resume partial writes.
+type ReplayProgress struct {
+	ScanCursor      Cursor `json:"scanCursor"`
+	DiscoveredCount int64  `json:"discoveredCount"`
+	EnqueuedCount   int64  `json:"enqueuedCount"`
+	CleanedCount    int64  `json:"cleanedCount"`
+	FailureCount    int64  `json:"failureCount"`
+}
+
+func (p ReplayProgress) Validate() error {
+	if err := nonnegative(p.DiscoveredCount, p.EnqueuedCount, p.CleanedCount, p.FailureCount); err != nil {
+		return err
+	}
+	if p.EnqueuedCount > p.DiscoveredCount || p.CleanedCount > p.EnqueuedCount {
+		return invalid("replay progress must satisfy cleaned <= enqueued <= discovered")
+	}
+	return nil
+}
+
+type ReplayAction struct {
+	ActionID       string          `json:"actionId"`
+	SourceRunID    string          `json:"sourceRunId"`
+	TargetRunID    string          `json:"targetRunId"`
+	RequestedBy    string          `json:"requestedBy"`
+	IdempotencyKey string          `json:"-"`
+	Fingerprint    string          `json:"-"`
+	Status         ActionStatus    `json:"status"`
+	ReplayCycle    int             `json:"replayCycle"`
+	Progress       ReplayProgress  `json:"progress"`
+	LeaseOwner     string          `json:"-"`
+	LeaseToken     string          `json:"-"`
+	LeaseUntil     time.Time       `json:"-"`
+	Revision       uint64          `json:"-"`
+	Error          string          `json:"error,omitempty"`
+	CreatedAt      time.Time       `json:"createdAt"`
+	UpdatedAt      time.Time       `json:"updatedAt"`
+	CompletedAt    *time.Time      `json:"completedAt,omitempty"`
+	Receipt        *ArchiveReceipt `json:"receipt,omitempty"`
+}
+
+func (a ReplayAction) Validate() error {
+	for _, id := range []string{a.ActionID, a.SourceRunID, a.TargetRunID} {
+		if err := ValidateID(id); err != nil {
+			return err
+		}
+	}
+	if a.SourceRunID == a.TargetRunID || a.ReplayCycle < 1 || !a.Status.Valid() || strings.TrimSpace(a.RequestedBy) == "" {
+		return invalid("invalid replay action identity or status")
+	}
+	if len(a.Error) > MaxErrorMessageBytes || !utf8.ValidString(a.Error) {
+		return invalid("action error exceeds message bounds")
+	}
+	if err := a.Progress.Validate(); err != nil {
+		return err
+	}
+	if a.Receipt != nil {
+		if err := a.Receipt.Validate(); err != nil {
+			return err
+		}
+	}
+	if a.Status == ActionCompleted && (a.Receipt == nil || a.CompletedAt == nil || a.CompletedAt.IsZero()) {
+		return invalid("completed action requires a durable receipt and completedAt")
+	}
+	return nil
+}
+
+type ReplayActionPage struct {
+	Items      []ReplayAction `json:"items"`
+	NextCursor Cursor         `json:"nextCursor"`
+}
+
+type ReplayActionCreation struct {
+	Action  ReplayAction
+	Created bool
+}
+
+// Lease token fences a particular acquisition, even if the same owner name is
+// reused. Repository methods check expiry against their authoritative clock.
+type ReplayLease struct {
+	ActionID string
+	Owner    string
+	Token    string
+	Until    time.Time
+}
+
+// ReplayActionUpdate is a compare-and-swap of mutable action state. Identity,
+// provenance, request fingerprint, and target run must never be changed.
+type ReplayActionUpdate struct {
+	Lease            ReplayLease
+	ExpectedRevision uint64
+	Status           ActionStatus
+	Progress         ReplayProgress
+	Error            string
+	Receipt          *ArchiveReceipt
+	CompletedAt      *time.Time
+}
+
+// Cleanup contains the exact source versions durably accepted by the target.
+// The engine must not request cleanup until target enqueue is confirmed.
+type ReplayCleanup struct {
+	Lease       ReplayLease
+	SourceRunID string
+	TargetRunID string
+	Entries     []FailureEntry
+}
+
+type ReplayCleanupResult struct {
+	CleanedCount        int64
+	AlreadyCleanedCount int64
+	ConflictCount       int64
+}

worker-svr/internal/fullsync/contract/replay_test.go

@@ -0,0 +1,103 @@
+package contract_test
+
+import (
+	"encoding/json"
+	"errors"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestReplayRequestIdentity(t *testing.T) {
+	request := contract.ReplayRequest{SourceRunID: "source", IdempotencyKey: "key", RequestedBy: "operator"}
+	fingerprint, err := request.Fingerprint()
+	if err != nil || len(fingerprint) != 64 {
+		t.Fatalf("fingerprint=%q, err=%v", fingerprint, err)
+	}
+	for name, mutate := range map[string]func(*contract.ReplayRequest){
+		"same":   func(*contract.ReplayRequest) {},
+		"key":    func(r *contract.ReplayRequest) { r.IdempotencyKey = "another-key" },
+		"source": func(r *contract.ReplayRequest) { r.SourceRunID = "another-source" },
+		"body":   func(r *contract.ReplayRequest) { r.RequestedBy = "another-operator" },
+	} {
+		copy := request
+		mutate(&copy)
+		got, err := copy.Fingerprint()
+		if err != nil || (got == fingerprint) != (name == "same" || name == "key") {
+			t.Errorf("%s fingerprint=%s err=%v", name, got, err)
+		}
+	}
+	if got := string(mustJSON(t, request)); got != `{"requestedBy":"operator"}` {
+		t.Fatalf("path/header leaked into body: %s", got)
+	}
+	for _, bad := range []contract.ReplayRequest{
+		{SourceRunID: "source", RequestedBy: "operator"},
+		{SourceRunID: "source", IdempotencyKey: "key", RequestedBy: " "},
+		{SourceRunID: "{source}", IdempotencyKey: "key", RequestedBy: "operator"},
+		{SourceRunID: "source", IdempotencyKey: "key\r\ninjected", RequestedBy: "operator"},
+		{SourceRunID: "source", IdempotencyKey: strings.Repeat("x", 257), RequestedBy: "operator"},
+	} {
+		if _, err := bad.Fingerprint(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("bad request returned %v", err)
+		}
+	}
+}
+
+func TestActionProgressAndReceiptValidation(t *testing.T) {
+	for _, progress := range []contract.ReplayProgress{
+		{DiscoveredCount: -1}, {DiscoveredCount: 1, EnqueuedCount: 2},
+		{DiscoveredCount: 2, EnqueuedCount: 1, CleanedCount: 2}, {FailureCount: -1},
+	} {
+		if err := progress.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Fatalf("bad progress %+v returned %v", progress, err)
+		}
+	}
+	now := time.Now().UTC()
+	action := contract.ReplayAction{ActionID: "action", SourceRunID: "source", TargetRunID: "target", RequestedBy: "operator", ReplayCycle: 1, Status: contract.ActionAccepted}
+	if err := action.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	action.Status = contract.ActionCompleted
+	if err := action.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("completed action without receipt returned %v", err)
+	}
+	action.Receipt = &contract.ArchiveReceipt{Version: 1, Path: "receipt.json", Checksum: strings.Repeat("a", 64), ArchivedAt: now}
+	action.CompletedAt = &now
+	if err := action.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	action.Receipt.Checksum = "not-a-checksum"
+	if err := action.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("bad receipt returned %v", err)
+	}
+}
+
+func TestActionWireFormatHidesInternalState(t *testing.T) {
+	action := contract.ReplayAction{
+		ActionID: "action", SourceRunID: "source", TargetRunID: "target", RequestedBy: "operator", ReplayCycle: 1,
+		Status: contract.ActionCleanupPending, IdempotencyKey: "secret", Fingerprint: "internal-fingerprint",
+		LeaseOwner: "worker", LeaseToken: "token", LeaseUntil: time.Now(), Revision: 9,
+		Progress: contract.ReplayProgress{ScanCursor: "opaque", DiscoveredCount: 10, EnqueuedCount: 8, CleanedCount: 7, FailureCount: 2},
+	}
+	data := mustJSON(t, action)
+	for _, hidden := range []string{"idempotency", "fingerprint", "lease", "revision", "secret", "token"} {
+		if strings.Contains(strings.ToLower(string(data)), hidden) {
+			t.Errorf("internal %s leaked: %s", hidden, data)
+		}
+	}
+	var decoded contract.ReplayAction
+	if err := json.Unmarshal(data, &decoded); err != nil || !reflect.DeepEqual(decoded.Progress, action.Progress) || decoded.Status != contract.ActionCleanupPending {
+		t.Fatalf("action roundtrip lost state: %+v, %v", decoded, err)
+	}
+	statusURL, err := contract.ActionStatusURL("action%1")
+	if err != nil || statusURL != "/internal/v1/full-sync-actions/action%251" {
+		t.Fatalf("status URL=%s, %v", statusURL, err)
+	}
+	accepted := contract.ReplayAccepted{ActionID: "action", TargetRunID: "target", Status: contract.ActionAccepted, StatusURL: "/internal/v1/full-sync-actions/action"}
+	if got := string(mustJSON(t, accepted)); got != `{"actionId":"action","targetRunId":"target","status":"accepted","statusUrl":"/internal/v1/full-sync-actions/action"}` {
+		t.Fatalf("202 contract changed: %s", got)
+	}
+}

worker-svr/internal/fullsync/contract/run.go

@@ -0,0 +1,146 @@
+package contract
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
+	"time"
+)
+
+// RunMeta retains every existing queue metadata field and JSON tag. The
+// optional fields are additive; zero extension fields are legal on old runs.
+type RunMeta struct {
+	SupplyObservation   *supply.Snapshot `json:"supplyObservation,omitempty"`
+	RunID               string           `json:"runId"`
+	Operation           string           `json:"operation"`
+	QueryMonth          string           `json:"queryMonth"`
+	Status              RunStatus        `json:"status"`
+	ProducerDone        bool             `json:"producerDone"`
+	CurrentRound        int              `json:"currentRound"`
+	ActiveCount         int64            `json:"activeCount"`
+	InflightCount       int64            `json:"inflightCount"`
+	NextRoundCount      int64            `json:"nextRoundCount"`
+	SuccessCount        int64            `json:"successCount"`
+	FailedCount         int64            `json:"failedCount"`
+	CreatedAt           time.Time        `json:"createdAt"`
+	ProducerCompletedAt time.Time        `json:"producerCompletedAt"`
+	CompletedAt         time.Time        `json:"completedAt"`
+
+	SyncType             SyncType          `json:"syncType,omitempty"`
+	SourceRunID          string            `json:"sourceRunId,omitempty"`
+	ReplayCycle          int               `json:"replayCycle,omitempty"`
+	TriggerSource        string            `json:"triggerSource,omitempty"`
+	ScannedCount         int64             `json:"scannedCount,omitempty"`
+	EnqueuedCount        int64             `json:"enqueuedCount,omitempty"`
+	UpstreamAttemptCount int64             `json:"upstreamAttemptCount,omitempty"`
+	BatchCount           int64             `json:"batchCount,omitempty"`
+	MaxRound             int               `json:"maxRound,omitempty"`
+	PendingFailureCount  int64             `json:"pendingFailureCount,omitempty"`
+	StartedAt            *time.Time        `json:"startedAt,omitempty"`
+	ScanCompletedAt      *time.Time        `json:"scanCompletedAt,omitempty"`
+	ArchiveStatus        ArchiveStatus     `json:"archiveStatus,omitempty"`
+	ArchiveVersion       int               `json:"archiveVersion,omitempty"`
+	ArchivePath          string            `json:"archivePath,omitempty"`
+	ArchiveChecksum      string            `json:"archiveChecksum,omitempty"`
+	ArchivedAt           *time.Time        `json:"archivedAt,omitempty"`
+	ReplayDisposition    ReplayDisposition `json:"replayDisposition,omitempty"`
+	// Scope fields mirror zset.RunMeta. ScopeChannels == 0 means a full sync.
+	ScopeChannels       int          `json:"scopeChannels,omitempty"`
+	ScopeDigest         string       `json:"scopeDigest,omitempty"`
+	ScopeChannelsSample string       `json:"scopeChannelsSample,omitempty"`
+	RoundStats          []RoundStats `json:"rounds,omitempty"`
+}
+
+func (m RunMeta) Validate() error {
+	if err := ValidateID(m.RunID); err != nil {
+		return err
+	}
+	if m.Status != "" && !m.Status.Valid() {
+		return invalid("unknown run status")
+	}
+	if m.SyncType != "" && !m.SyncType.Valid() {
+		return invalid("unknown sync type")
+	}
+	if m.ArchiveStatus != "" && !m.ArchiveStatus.Valid() {
+		return invalid("unknown archive status")
+	}
+	if m.ReplayDisposition != "" && !m.ReplayDisposition.Valid() {
+		return invalid("unknown replay disposition")
+	}
+	if m.SourceRunID != "" {
+		if err := ValidateID(m.SourceRunID); err != nil {
+			return err
+		}
+		if m.SourceRunID == m.RunID {
+			return invalid("a run cannot replay itself")
+		}
+	}
+	if m.SyncType == SyncReplay && (m.SourceRunID == "" || m.ReplayCycle < 1) {
+		return invalid("replay requires sourceRunId and a positive replayCycle")
+	}
+	if m.SyncType == SyncNormal && (m.SourceRunID != "" || m.ReplayCycle != 0) {
+		return invalid("normal runs cannot have replay provenance")
+	}
+	if m.MaxRound > 0 && m.CurrentRound > m.MaxRound {
+		return invalid("currentRound exceeds maxRound")
+	}
+	return nonnegative(int64(m.CurrentRound), int64(m.MaxRound), int64(m.ReplayCycle), int64(m.ArchiveVersion), m.ActiveCount, m.InflightCount, m.NextRoundCount, m.SuccessCount, m.FailedCount, m.ScannedCount, m.EnqueuedCount, m.UpstreamAttemptCount, m.BatchCount, m.PendingFailureCount, int64(m.ScopeChannels))
+}
+
+// ValidateAccounting is opt-in for runs whose extended counters are known.
+// Do not apply it to legacy metadata with a missing enqueuedCount. Inflight is
+// a subset of active and is never added to the completion equation.
+func (m RunMeta) ValidateAccounting() error {
+	if err := m.Validate(); err != nil {
+		return err
+	}
+	remaining := m.EnqueuedCount
+	for _, count := range []int64{m.SuccessCount, m.FailedCount, m.ActiveCount, m.NextRoundCount} {
+		if count > remaining {
+			return invalid("enqueuedCount does not balance run outcomes")
+		}
+		remaining -= count
+	}
+	if remaining != 0 || m.InflightCount > m.ActiveCount || m.PendingFailureCount > m.FailedCount {
+		return invalid("inconsistent run counters")
+	}
+	if m.Status.Terminal() && (!m.ProducerDone || m.ActiveCount != 0 || m.NextRoundCount != 0) {
+		return invalid("terminal run still has producer or queue work")
+	}
+	return nil
+}
+
+// RoundStats contains only bounded aggregate fields. It must never grow with
+// the number of cards, channels, task IDs, upstream codes, or error messages.
+type RoundStats struct {
+	RunID                string      `json:"runId"`
+	Round                int         `json:"round"`
+	EnqueuedCount        int64       `json:"enqueuedCount"`
+	ActiveCount          int64       `json:"activeCount"`
+	NextRoundCount       int64       `json:"nextRoundCount"`
+	SuccessCount         int64       `json:"successCount"`
+	FailedCount          int64       `json:"failedCount"`
+	UpstreamAttemptCount int64       `json:"upstreamAttemptCount"`
+	BatchCount           int64       `json:"batchCount"`
+	StartedAt            *time.Time  `json:"startedAt,omitempty"`
+	CompletedAt          *time.Time  `json:"completedAt,omitempty"`
+	ErrorCounts          ErrorCounts `json:"errorCounts"`
+}
+
+func (s RoundStats) Validate() error {
+	if err := ValidateID(s.RunID); err != nil {
+		return err
+	}
+	if s.Round < 1 {
+		return invalid("round must be positive")
+	}
+	if err := nonnegative(s.EnqueuedCount, s.ActiveCount, s.NextRoundCount, s.SuccessCount, s.FailedCount, s.UpstreamAttemptCount, s.BatchCount); err != nil {
+		return err
+	}
+	_, err := s.ErrorCounts.Normalized()
+	return err
+}
+
+type RunView struct {
+	Source RunSource    `json:"source"`
+	Run    RunMeta      `json:"run"`
+	Rounds []RoundStats `json:"rounds,omitempty"`
+}

worker-svr/internal/fullsync/contract/status.go

@@ -0,0 +1,137 @@
+package contract
+
+type RunStatus string
+
+const (
+	RunProducing             RunStatus = "producing"
+	RunRunning               RunStatus = "running"
+	RunCompleted             RunStatus = "completed"
+	RunCompletedWithFailures RunStatus = "completed_with_failures"
+)
+
+func (s RunStatus) Valid() bool {
+	return s == RunProducing || s == RunRunning || s.Terminal()
+}
+
+func (s RunStatus) Terminal() bool { return s == RunCompleted || s == RunCompletedWithFailures }
+
+type SyncType string
+
+const (
+	SyncNormal SyncType = "normal"
+	SyncReplay SyncType = "replay"
+)
+
+func (s SyncType) Valid() bool { return s == SyncNormal || s == SyncReplay }
+
+type SyncKind string
+
+const (
+	SyncKindTrafficUsage        SyncKind = "traffic_usage"
+	SyncKindSimStatus           SyncKind = "sim_status"
+	SyncKindTrafficUsageDaily   SyncKind = "traffic_usage_daily"
+	SyncKindTrafficUsageMonthly SyncKind = "traffic_usage_monthly"
+	SyncKindImei                SyncKind = "imei"
+)
+
+func (s SyncKind) Valid() bool {
+	return s == SyncKindTrafficUsage || s == SyncKindSimStatus || s == SyncKindTrafficUsageDaily || s == SyncKindTrafficUsageMonthly || s == SyncKindImei
+}
+
+func SyncKindForOperation(operation string) SyncKind {
+	switch operation {
+	case "current_month_usage_full_sync":
+		return SyncKindTrafficUsage
+	case "sim_status_queue_full_sync":
+		return SyncKindSimStatus
+	case "previous_day_usage_full_sync":
+		return SyncKindTrafficUsageDaily
+	case "previous_month_usage_full_sync":
+		return SyncKindTrafficUsageMonthly
+	case "imei_full_sync":
+		return SyncKindImei
+	default:
+		return ""
+	}
+}
+
+type ArchiveStatus string
+
+const (
+	ArchiveDisabled   ArchiveStatus = "disabled"
+	ArchivePending    ArchiveStatus = "pending"
+	ArchiveProcessing ArchiveStatus = "processing"
+	ArchiveSucceeded  ArchiveStatus = "succeeded"
+	ArchiveFailed     ArchiveStatus = "failed"
+)
+
+func (s ArchiveStatus) Valid() bool {
+	switch s {
+	case ArchiveDisabled, ArchivePending, ArchiveProcessing, ArchiveSucceeded, ArchiveFailed:
+		return true
+	}
+	return false
+}
+
+// ReplayDisposition describes a source run, not an individual replay action.
+type ReplayDisposition string
+
+const (
+	ReplayNone           ReplayDisposition = "none"
+	ReplayPending        ReplayDisposition = "pending"
+	ReplayProcessing     ReplayDisposition = "processing"
+	ReplayCompleted      ReplayDisposition = "completed"
+	ReplayNeedsAttention ReplayDisposition = "needs_attention"
+)
+
+func (s ReplayDisposition) Valid() bool {
+	switch s {
+	case ReplayNone, ReplayPending, ReplayProcessing, ReplayCompleted, ReplayNeedsAttention:
+		return true
+	}
+	return false
+}
+
+type ActionStatus string
+
+const (
+	ActionAccepted       ActionStatus = "accepted"
+	ActionProcessing     ActionStatus = "processing"
+	ActionCleanupPending ActionStatus = "cleanup_pending"
+	ActionReceiptPending ActionStatus = "receipt_pending"
+	ActionCompleted      ActionStatus = "completed"
+	ActionNeedsAttention ActionStatus = "needs_attention"
+)
+
+func (s ActionStatus) Valid() bool {
+	switch s {
+	case ActionAccepted, ActionProcessing, ActionCleanupPending, ActionReceiptPending, ActionCompleted, ActionNeedsAttention:
+		return true
+	}
+	return false
+}
+
+// Terminal means automatic work has stopped; needs_attention may be resumed
+// only by an explicit recovery policy, never by assuming it succeeded.
+func (s ActionStatus) Terminal() bool { return s == ActionCompleted || s == ActionNeedsAttention }
+
+type RunSource string
+
+const (
+	SourceLive    RunSource = "live"
+	SourceArchive RunSource = "archive"
+)
+
+func (s RunSource) Valid() bool { return s == SourceLive || s == SourceArchive }
+
+// These values are already persisted by the queue protocol.
+const (
+	TaskQueued              = "queued"
+	TaskClaimed             = "claimed"
+	TaskWaitingNextRound    = "waiting_next_round"
+	TaskSucceeded           = "succeeded"
+	TaskCovered             = "covered"
+	TaskFailedPendingReview = "failed_pending_review"
+	TaskChannelBlocked      = "channel_blocked"
+	ReviewPending           = "pending"
+)

worker-svr/internal/fullsync/contract/status_test.go

@@ -0,0 +1,57 @@
+package contract_test
+
+import (
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+// TestSyncKindForDailyUsageOperation guards the mapping that makes a daily run
+// discoverable. Catalog.ListRuns drops any run whose operation maps to an empty
+// kind, so losing this case would make daily runs invisible in the monitor
+// without any error.
+func TestSyncKindForDailyUsageOperation(t *testing.T) {
+	if got := contract.SyncKindForOperation(model.OperationPreviousDayUsageFullSync); got != contract.SyncKindTrafficUsageDaily {
+		t.Fatalf("SyncKindForOperation(%q) = %q, want %q",
+			model.OperationPreviousDayUsageFullSync, got, contract.SyncKindTrafficUsageDaily)
+	}
+	if !contract.SyncKindTrafficUsageDaily.Valid() {
+		t.Fatal("SyncKindTrafficUsageDaily must be a valid kind")
+	}
+	if contract.SyncKindForOperation("some_unknown_operation") != "" {
+		t.Fatal("an unknown operation must not be assigned a sync kind")
+	}
+}
+
+// TestSyncKindForMonthlyHistoryOperation guards the mapping that makes a monthly
+// history run discoverable. Catalog.ListRuns drops any run whose operation maps
+// to an empty kind, so losing this case would make monthly history runs
+// invisible in the monitor without any error.
+func TestSyncKindForMonthlyHistoryOperation(t *testing.T) {
+	if got := contract.SyncKindForOperation(model.OperationPreviousMonthUsageFullSync); got != contract.SyncKindTrafficUsageMonthly {
+		t.Fatalf("SyncKindForOperation(%q) = %q, want %q",
+			model.OperationPreviousMonthUsageFullSync, got, contract.SyncKindTrafficUsageMonthly)
+	}
+}
+
+// TestDailyAndMonthlyUsageHaveDistinctSyncKinds keeps the usage runs separately
+// filterable. There are three distinct usage runs now: the MySQL current-month
+// snapshot, the TDengine daily history, and the TDengine monthly history.
+func TestDailyAndMonthlyUsageHaveDistinctSyncKinds(t *testing.T) {
+	monthly := contract.SyncKindForOperation("current_month_usage_full_sync")
+	daily := contract.SyncKindForOperation(model.OperationPreviousDayUsageFullSync)
+	monthlyHistory := contract.SyncKindForOperation(model.OperationPreviousMonthUsageFullSync)
+	if monthly == daily {
+		t.Fatalf("current-month snapshot and daily history share the sync kind %q", daily)
+	}
+	if monthly == monthlyHistory {
+		t.Fatalf("current-month snapshot and monthly history share the sync kind %q", monthlyHistory)
+	}
+	if daily == monthlyHistory {
+		t.Fatalf("daily and monthly history share the sync kind %q", daily)
+	}
+	if !monthlyHistory.Valid() {
+		t.Fatal("SyncKindTrafficUsageMonthly must be a valid kind")
+	}
+}

worker-svr/internal/fullsync/contract/sync_kind_test.go

@@ -0,0 +1,85 @@
+package contract_test
+
+import (
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+// TestSyncKindForOperationCoversEveryChain pins the operation→kind mapping for
+// all five full-sync chains. A missing case makes the run invisible in the
+// admin monitor, because the catalog filters runs by SyncKind.
+func TestSyncKindForOperationCoversEveryChain(t *testing.T) {
+	cases := []struct {
+		operation string
+		want      contract.SyncKind
+	}{
+		{"current_month_usage_full_sync", contract.SyncKindTrafficUsage},
+		{"sim_status_queue_full_sync", contract.SyncKindSimStatus},
+		{"previous_day_usage_full_sync", contract.SyncKindTrafficUsageDaily},
+		{"previous_month_usage_full_sync", contract.SyncKindTrafficUsageMonthly},
+		{"imei_full_sync", contract.SyncKindImei},
+	}
+	for _, tc := range cases {
+		got := contract.SyncKindForOperation(tc.operation)
+		if got != tc.want {
+			t.Fatalf("SyncKindForOperation(%q) = %q, want %q", tc.operation, got, tc.want)
+		}
+		if !got.Valid() {
+			t.Fatalf("%q 应通过 Valid()", got)
+		}
+	}
+}
+
+// TestSyncKindForOperationRejectsUnknown 未知 operation 返回空 kind，
+// 由调用方决定是跳过还是报错。
+func TestSyncKindForOperationRejectsUnknown(t *testing.T) {
+	for _, operation := range []string{"", "unknown", "IMEI_FULL_SYNC"} {
+		got := contract.SyncKindForOperation(operation)
+		if got != "" {
+			t.Fatalf("SyncKindForOperation(%q) = %q, want empty", operation, got)
+		}
+		if got.Valid() {
+			t.Fatalf("空 kind 不应通过 Valid()")
+		}
+	}
+}
+
+// TestSyncKindValidRejectsUnknown 只有五个已注册的 kind 有效。
+func TestSyncKindValidRejectsUnknown(t *testing.T) {
+	valid := []contract.SyncKind{
+		contract.SyncKindTrafficUsage,
+		contract.SyncKindSimStatus,
+		contract.SyncKindTrafficUsageDaily,
+		contract.SyncKindTrafficUsageMonthly,
+		contract.SyncKindImei,
+	}
+	for _, kind := range valid {
+		if !kind.Valid() {
+			t.Fatalf("%q 应有效", kind)
+		}
+	}
+	invalid := []contract.SyncKind{"", "imei_full_sync", "IMEI", "traffic", "sim-status"}
+	for _, kind := range invalid {
+		if kind.Valid() {
+			t.Fatalf("%q 不应有效", kind)
+		}
+	}
+}
+
+// TestSyncKindValuesAreStable 枚举字符串是持久化契约（写入 Run 元数据、
+// 归档 JSON 与 admin 查询参数），不得随意改动。
+func TestSyncKindValuesAreStable(t *testing.T) {
+	cases := map[contract.SyncKind]string{
+		contract.SyncKindTrafficUsage:        "traffic_usage",
+		contract.SyncKindSimStatus:           "sim_status",
+		contract.SyncKindTrafficUsageDaily:   "traffic_usage_daily",
+		contract.SyncKindTrafficUsageMonthly: "traffic_usage_monthly",
+		contract.SyncKindImei:                "imei",
+	}
+	for kind, want := range cases {
+		if string(kind) != want {
+			t.Fatalf("kind = %q, want %q（持久化契约不得改动）", kind, want)
+		}
+	}
+}

worker-svr/internal/fullsync/contract/validation.go

@@ -0,0 +1,42 @@
+package contract
+
+import (
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+// Implementations wrap these errors so callers can use errors.Is without
+// inspecting error text. HTTP adapters choose the response envelope themselves.
+var (
+	ErrNotFound                  = gerror.New("full-sync resource not found")
+	ErrConflict                  = gerror.New("full-sync conflict")
+	ErrInvalidArgument           = gerror.New("invalid full-sync argument")
+	ErrFailureDetailsUnavailable = gerror.New("live failure details unavailable")
+	ErrLeaseLost                 = gerror.New("full-sync lease lost")
+	ErrBusy                      = gerror.New("full-sync resource busy")
+)
+
+func invalid(format string, args ...any) error {
+	return gerror.Wrapf(ErrInvalidArgument, format, args...)
+}
+
+// ValidateID checks new resource identifiers used both as Redis hash tags and
+// HTTP path segments. It does not rewrite historical identifiers during decode.
+func ValidateID(id string) error {
+	if id == "" || strings.TrimSpace(id) != id || !utf8.ValidString(id) || strings.ContainsAny(id, "{}/\\") || strings.ContainsFunc(id, unicode.IsControl) {
+		return invalid("invalid resource identifier")
+	}
+	return nil
+}
+
+func nonnegative(values ...int64) error {
+	for _, value := range values {
+		if value < 0 {
+			return invalid("counts must be nonnegative")
+		}
+	}
+	return nil
+}

worker-svr/internal/fullsync/contract/validation_test.go

@@ -0,0 +1,121 @@
+package contract_test
+
+import (
+	"errors"
+	"math"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestRunAccounting(t *testing.T) {
+	valid := contract.RunMeta{
+		RunID: "run", Status: contract.RunRunning, CurrentRound: 1, MaxRound: 5,
+		EnqueuedCount: 10, ActiveCount: 3, InflightCount: 2, NextRoundCount: 2,
+		SuccessCount: 4, FailedCount: 1, PendingFailureCount: 1,
+	}
+	if err := valid.ValidateAccounting(); err != nil {
+		t.Fatal(err)
+	}
+	for name, mutate := range map[string]func(*contract.RunMeta){
+		"unbalanced": func(m *contract.RunMeta) { m.EnqueuedCount++ },
+		"negative":   func(m *contract.RunMeta) { m.BatchCount = -1 },
+		"overflow": func(m *contract.RunMeta) {
+			m.EnqueuedCount = math.MaxInt64
+			m.SuccessCount = math.MaxInt64
+			m.FailedCount = math.MaxInt64
+		},
+		"inflight double count":        func(m *contract.RunMeta) { m.InflightCount = m.ActiveCount + 1 },
+		"pending exceeds history":      func(m *contract.RunMeta) { m.PendingFailureCount = 2 },
+		"terminal with work":           func(m *contract.RunMeta) { m.Status = contract.RunCompletedWithFailures; m.ProducerDone = true },
+		"unknown status":               func(m *contract.RunMeta) { m.Status = "succeeded" },
+		"unknown sync type":            func(m *contract.RunMeta) { m.SyncType = "manual" },
+		"action status in archive":     func(m *contract.RunMeta) { m.ArchiveStatus = "completed" },
+		"action status in disposition": func(m *contract.RunMeta) { m.ReplayDisposition = "cleanup_pending" },
+		"round out of range":           func(m *contract.RunMeta) { m.CurrentRound = 6 },
+		"replay without source":        func(m *contract.RunMeta) { m.SyncType = contract.SyncReplay },
+		"self replay": func(m *contract.RunMeta) {
+			m.SyncType = contract.SyncReplay
+			m.SourceRunID = m.RunID
+			m.ReplayCycle = 1
+		},
+		"normal with provenance": func(m *contract.RunMeta) { m.SyncType = contract.SyncNormal; m.SourceRunID = "source" },
+	} {
+		t.Run(name, func(t *testing.T) {
+			meta := valid
+			mutate(&meta)
+			if err := meta.ValidateAccounting(); !errors.Is(err, contract.ErrInvalidArgument) {
+				t.Fatalf("invalid accounting returned %v", err)
+			}
+		})
+	}
+	// Cleaned source failures never subtract from the historical failed total.
+	completed := contract.RunMeta{RunID: "source", Status: contract.RunCompletedWithFailures, ProducerDone: true, EnqueuedCount: 5, SuccessCount: 3, FailedCount: 2, PendingFailureCount: 0}
+	if err := completed.ValidateAccounting(); err != nil {
+		t.Fatal(err)
+	}
+	completed.ProducerDone = false
+	if err := completed.ValidateAccounting(); err == nil {
+		t.Fatal("accepted terminal run with unfinished producer")
+	}
+	if err := (contract.RunMeta{RunID: "old", Status: contract.RunRunning, SuccessCount: 42}).Validate(); err != nil {
+		t.Fatalf("historical run without new counters rejected: %v", err)
+	}
+}
+
+func TestRoundStatsValidation(t *testing.T) {
+	valid := contract.RoundStats{RunID: "run", Round: 1, ErrorCounts: contract.ErrorCounts{"unrecognized": 1}}
+	if err := valid.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	for _, stats := range []contract.RoundStats{
+		{RunID: "run", Round: 0}, {RunID: "run", Round: 1, BatchCount: -1},
+		{RunID: "run", Round: 1, ErrorCounts: contract.ErrorCounts{contract.ErrorOther: -1}},
+	} {
+		if err := stats.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Fatalf("invalid round stats returned %v", err)
+		}
+	}
+}
+
+func TestPageLimitsAndOpaqueCursor(t *testing.T) {
+	for _, limit := range []int{0, 1, 100, 500} {
+		request, err := (contract.PageRequest{Limit: limit, Cursor: "opaque+/=:%?next"}).Normalized()
+		if err != nil || request.Cursor != "opaque+/=:%?next" || request.Limit < 1 || request.Limit > contract.MaxPageLimit {
+			t.Fatalf("unexpected page: %+v, %v", request, err)
+		}
+		if limit == 0 && request.Limit != 100 {
+			t.Fatal("default limit changed")
+		}
+	}
+	for _, limit := range []int{-1, 501, math.MaxInt} {
+		if _, err := (contract.PageRequest{Limit: limit}).Normalized(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Fatalf("limit %d returned %v", limit, err)
+		}
+	}
+	page := contract.FailurePage{Items: []contract.Failure{}, NextCursor: "continue"}
+	if got := string(mustJSON(t, page)); got != `{"items":[],"nextCursor":"continue"}` {
+		t.Fatalf("empty scan page lost its continuation: %s", got)
+	}
+}
+
+func TestStatusVocabulary(t *testing.T) {
+	for _, status := range []contract.ActionStatus{"accepted", "processing", "cleanup_pending", "receipt_pending", "completed", "needs_attention"} {
+		if !status.Valid() || status.Terminal() != (status == "completed" || status == "needs_attention") {
+			t.Errorf("invalid action status %q", status)
+		}
+	}
+	for _, status := range []contract.ArchiveStatus{"disabled", "pending", "processing", "succeeded", "failed"} {
+		if !status.Valid() {
+			t.Errorf("invalid archive status %q", status)
+		}
+	}
+	for _, status := range []contract.ReplayDisposition{"none", "pending", "processing", "completed", "needs_attention"} {
+		if !status.Valid() {
+			t.Errorf("invalid disposition %q", status)
+		}
+	}
+	if contract.ActionStatus("succeeded").Valid() || contract.ArchiveStatus("completed").Valid() || contract.ReplayDisposition("receipt_pending").Valid() {
+		t.Fatal("status domains were conflated")
+	}
+}

worker-svr/internal/fullsync/httpapi/README.md

@@ -0,0 +1,99 @@
+# Full-sync HTTP control API
+
+The handler imports no queue or storage implementations. Inject the five
+contract interfaces in `Dependencies`: `RunLister`, `RunReader`,
+`FailurePager`, `ReplayRequester`, and `ActionReader`. The replay service owns durable
+idempotency and execution; the HTTP layer does not cache requests or launch work.
+
+Use `NewRunReader(live, archive, rounds)` to compose `LiveRunReader`,
+`ArchivedRunReader`, and `RoundStatsReader` into a `RunReader`. Live metadata
+wins and returns `source: "live"`; only `ErrNotFound` from live metadata permits
+archive fallback, returning `source: "archive"`. Live storage errors and round
+lookup errors propagate. Archive and round readers may be nil to disable those
+lookups. A failure listing always calls `FailurePager` directly, never archives.
+
+## Routes and responses
+
+All routes require `Authorization: Bearer <configured token>`. Missing, wrong,
+or ambiguous credentials return 401 with a `WWW-Authenticate` challenge. An
+empty configured token is a startup configuration error. Responses use JSON and
+`Cache-Control: no-store`.
+
+| Method | Route | Success body |
+| --- | --- | --- |
+| GET | `/internal/v1/full-sync-runs?syncKind=&limit=` | 200 `contract.RunList` |
+| GET | `/internal/v1/full-sync-runs/{runId}` | 200 `contract.RunView` |
+| GET | `/internal/v1/full-sync-runs/{runId}/failures?cursor=&limit=` | 200 `contract.FailurePage` |
+| POST | `/internal/v1/full-sync-runs/{runId}/replay` | 202 `contract.ReplayAccepted` |
+| GET | `/internal/v1/full-sync-actions/{actionId}` | 200 `contract.ReplayAction` |
+
+Run list results merge active Redis metadata with flat `run-<runId>.json`
+archives under the configured archive root. Active Redis metadata wins when an
+ID exists in both sources. `syncKind` accepts `traffic_usage` or `sim_status`;
+`limit` defaults to 50 and is capped at 100.
+
+Failure limits use the contract rules: absent, empty, or zero selects 100;
+positive values up to 500 are accepted; other values are rejected. Repeated
+cursor/limit parameters and malformed query encoding are rejected. Cursors are
+opaque and passed unchanged after URL decoding. The pager validates cursor
+contents and run binding. Empty pages serialize `items: []` and preserve
+`nextCursor`; only an empty next cursor signals completion.
+
+Replay requires a single `Idempotency-Key` header and a JSON body such as
+`{"requestedBy":"operator"}`. The body is limited to 64 KiB, must contain one
+JSON object with no unknown fields, and must satisfy `ReplayRequest.Validate`.
+If supplied, Content-Type must be `application/json` (parameters are allowed).
+The exact `requestedBy` value is preserved for fingerprinting. Repeated accepted
+requests return the service's existing action/target and current status with
+202. The adapter constructs `statusUrl` and the `Location` header from actionId.
+
+All application errors use this envelope, without storage error details:
+
+```json
+{"error":{"code":"invalid_argument","message":"Invalid request"}}
+```
+
+| Condition | HTTP status | Error code |
+| --- | --- | --- |
+| Invalid path, query, header, body, or `ErrInvalidArgument` | 400 | `invalid_argument` |
+| Missing/wrong bearer credentials | 401 | `unauthorized` |
+| Unknown route/resource or `ErrNotFound` | 404 | `not_found` |
+| Unsupported route method | 405 | `method_not_allowed` |
+| `ErrConflict` (including changed idempotent request), `ErrLeaseLost` | 409 | `conflict` |
+| `ErrFailureDetailsUnavailable` | 410 | `failure_details_unavailable` |
+| Oversized replay body | 413 | `body_too_large` |
+| `ErrBusy` | 503 | `busy` |
+| Other service errors or unencodable service output | 500 | `internal_error` |
+
+## Lifecycle
+
+`NewHandler(token, deps)` can be mounted by an existing HTTP server. For an
+independent listener, construct `NewServer(Config{Addr: address, BearerToken:
+token}, deps)` and call `Start(workerCtx)`. `Start` binds synchronously and
+returns address-in-use and other startup errors directly. `Addr()` exposes the
+bound address, including an OS-assigned port when configured with port zero.
+
+Call `Wait()` after Start, normally in the worker's supervision goroutine. It
+returns serving or shutdown failures; it also returns startup failures if Start
+failed. The server is single-use and rejects a second Start. Wait can have
+multiple callers but must not be called before Start.
+
+Canceling `workerCtx` closes the listener and drains active HTTP requests.
+Active request contexts stay usable during the drain. `ShutdownTimeout` defaults
+to ten seconds; if it expires, active connections are closed and Wait returns
+an error wrapping `context.DeadlineExceeded`. Normal cancellation with a
+successful drain returns nil. No worker wiring or configuration-file changes
+are part of this package.
+
+## Verification
+
+```text
+gofmt -l internal/fullsync/httpapi
+go test ./internal/fullsync/...
+go test ./internal/fullsync/httpapi -cover
+go vet ./internal/fullsync/...
+```
+
+Tests use contract mocks and local HTTP listeners, requiring no Redis or archive
+files. `go test -race ./internal/fullsync/...` additionally requires a supported
+C compiler and CGO enabled.

worker-svr/internal/fullsync/httpapi/SIMPLE_REPLAY.md

@@ -0,0 +1,10 @@
+# Simple full-sync replay
+
+The Worker control server listens on `:18081` inside the private Docker network. Do not publish port 18081 to the host or the internet. Admin already enforces user JWT authentication and replay permissions and calls `http://iotlcm-worker-svr:18081` directly; no additional Worker URL or bearer token environment variables are needed.
+
+- `POST /api/v1/iot/fullSyncRuns/replay` with `{ "sourceRunId": "..." }` returns `replayRunId` and `accepted`. Admin requires `cpm:iot:fullSyncRuns:replay`.
+- `GET /api/v1/iot/fullSyncRuns/replay/{replayRunId}` reports admission (`accepted`, `processing`, `needs_attention`, `completed`), remaining source records, enqueued count, and the independent target Run status. Requires `cpm:iot:fullSyncRuns:replay:view`.
+
+All failures in a completed `flow:` or `sim-status:` Run are selected. No retryability filter; the target Run has exactly one business round. Source failure entries are deleted only after target task-state confirms enqueue; original outcome counts and historical JSON remain unchanged. Repeat requests to the same source return the same target. After repairing a `needs_attention` condition, repeat the request to resume it. To retry again after completion, use the new Replay Run as the source.
+
+No deployment or live Replay request is part of this change.

worker-svr/internal/fullsync/httpapi/handler.go

@@ -0,0 +1,321 @@
+// Package httpapi exposes the internal full-sync control API through contract
+// interfaces. Persistence and replay execution belong to the injected services.
+package httpapi
+
+import (
+	"bytes"
+	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"io"
+	"mime"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	fullsyncruntime "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/runtime"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+const maxReplayBodyBytes = 64 << 10
+
+// Dependencies are the only application services used by the HTTP handlers.
+type Dependencies struct {
+	Runs     contract.RunReader
+	List     contract.RunLister
+	Failures contract.FailurePager
+	Replays  contract.ReplayRequester
+	Actions  contract.ActionReader
+	Simple   *fullsyncruntime.SimpleReplay
+}
+
+// ErrorResponse is shared by authentication, routing, validation and service
+// failures. Messages deliberately exclude underlying storage error details.
+type ErrorResponse struct {
+	Error APIError `json:"error"`
+}
+
+type APIError struct {
+	Code    string `json:"code"`
+	Message string `json:"message"`
+}
+
+// NewHandler exposes the control routes on the private Docker network. Admin
+// performs the user JWT and permission checks before calling this listener.
+func NewHandler(bearerToken string, deps Dependencies) (http.Handler, error) {
+	if strings.ContainsFunc(bearerToken, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) {
+		return nil, gerror.New("invalid full-sync HTTP token")
+	}
+	if deps.Runs == nil || deps.List == nil || deps.Failures == nil || deps.Replays == nil || deps.Actions == nil {
+		return nil, gerror.New("full-sync HTTP requires run list, run, failure, replay and action services")
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc(contract.ListRunsRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		query, err := url.ParseQuery(r.URL.RawQuery)
+		if err != nil || len(query["limit"]) > 1 || len(query["syncKind"]) > 1 {
+			writeError(w, contract.ErrInvalidArgument)
+			return
+		}
+		filter := contract.RunListFilter{SyncKind: contract.SyncKind(query.Get("syncKind"))}
+		if filter.SyncKind != "" && !filter.SyncKind.Valid() {
+			writeError(w, contract.ErrInvalidArgument)
+			return
+		}
+		if query.Get("limit") != "" {
+			filter.Limit, err = strconv.Atoi(query.Get("limit"))
+			if err != nil || filter.Limit < 1 || filter.Limit > 100 {
+				writeError(w, contract.ErrInvalidArgument)
+				return
+			}
+		}
+		result, err := deps.List.ListRuns(r.Context(), filter)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, result)
+	}))
+	mux.HandleFunc(contract.GetRunRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		id := r.PathValue("runId")
+		if err := contract.ValidateID(id); err != nil {
+			writeError(w, err)
+			return
+		}
+		view, err := deps.Runs.GetRun(r.Context(), id)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, view)
+	}))
+	mux.HandleFunc(contract.GetFailuresRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		id := r.PathValue("runId")
+		if err := contract.ValidateID(id); err != nil {
+			writeError(w, err)
+			return
+		}
+		page, err := parsePage(r.URL.RawQuery)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		result, err := deps.Failures.ListFailures(r.Context(), id, page)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		if result.Items == nil {
+			result.Items = []contract.Failure{}
+		}
+		writeJSON(w, http.StatusOK, result)
+	}))
+	mux.HandleFunc(contract.PostReplayRoute, method(http.MethodPost, func(w http.ResponseWriter, r *http.Request) {
+		if deps.Simple != nil {
+			id := r.PathValue("runId")
+			if err := contract.ValidateID(id); err != nil {
+				writeError(w, err)
+				return
+			}
+			var request struct {
+				RequestedBy string `json:"requestedBy"`
+				MaxRounds   int    `json:"maxRounds"`
+			}
+			decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxReplayBodyBytes))
+			decoder.DisallowUnknownFields()
+			if err := decoder.Decode(&request); err != nil {
+				writeError(w, contract.ErrInvalidArgument)
+				return
+			}
+			if err := decoder.Decode(new(any)); err != io.EOF {
+				writeError(w, contract.ErrInvalidArgument)
+				return
+			}
+			accepted, err := deps.Simple.Request(r.Context(), id, request.RequestedBy, request.MaxRounds)
+			if errors.Is(err, fullsyncruntime.ErrNoFailures) {
+				writeAPIError(w, http.StatusUnprocessableEntity, "no_failures", "Run has no failures to replay")
+				return
+			}
+			if err != nil {
+				writeError(w, err)
+				return
+			}
+			writeJSON(w, http.StatusAccepted, accepted)
+			return
+		}
+		request, err := parseReplay(w, r)
+		if err != nil {
+			var tooLarge *http.MaxBytesError
+			if errors.As(err, &tooLarge) {
+				writeAPIError(w, http.StatusRequestEntityTooLarge, "body_too_large", "Replay request body is too large")
+			} else {
+				writeError(w, err)
+			}
+			return
+		}
+		accepted, err := deps.Replays.RequestReplay(r.Context(), request)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		// Construct a local status URL from the returned identity, never an
+		// externally supplied redirect. Invalid service output is a server error.
+		accepted.StatusURL, err = contract.ActionStatusURL(accepted.ActionID)
+		if err != nil || contract.ValidateID(accepted.TargetRunID) != nil || !accepted.Status.Valid() {
+			writeAPIError(w, http.StatusInternalServerError, "internal_error", "Internal server error")
+			return
+		}
+		w.Header().Set("Location", accepted.StatusURL)
+		writeJSON(w, http.StatusAccepted, accepted)
+	}))
+	mux.HandleFunc("/internal/v1/full-sync-replays/{replayRunId}", method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		if deps.Simple == nil {
+			writeError(w, contract.ErrNotFound)
+			return
+		}
+		status, err := deps.Simple.Status(r.Context(), r.PathValue("replayRunId"))
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, status)
+	}))
+	mux.HandleFunc(contract.GetActionRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		if deps.Simple != nil {
+			writeError(w, contract.ErrNotFound)
+			return
+		}
+		id := r.PathValue("actionId")
+		if err := contract.ValidateID(id); err != nil {
+			writeError(w, err)
+			return
+		}
+		action, err := deps.Actions.GetReplayAction(r.Context(), id)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, action)
+	}))
+	mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
+		writeError(w, contract.ErrNotFound)
+	})
+	if bearerToken == "" {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Cache-Control", "no-store")
+			mux.ServeHTTP(w, r)
+		}), nil
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Cache-Control", "no-store")
+		values := r.Header.Values("Authorization")
+		if len(values) == 1 {
+			parts := strings.Fields(values[0])
+			if len(parts) == 2 && strings.EqualFold(parts[0], "Bearer") && subtle.ConstantTimeCompare([]byte(parts[1]), []byte(bearerToken)) == 1 {
+				mux.ServeHTTP(w, r)
+				return
+			}
+		}
+		w.Header().Set("WWW-Authenticate", `Bearer realm="full-sync"`)
+		writeAPIError(w, http.StatusUnauthorized, "unauthorized", "Valid bearer authentication is required")
+	}), nil
+}
+
+func method(expected string, next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != expected {
+			w.Header().Set("Allow", expected)
+			writeAPIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "Method not allowed")
+			return
+		}
+		next(w, r)
+	}
+}
+
+func parsePage(rawQuery string) (contract.PageRequest, error) {
+	query, err := url.ParseQuery(rawQuery)
+	if err != nil || len(query["cursor"]) > 1 || len(query["limit"]) > 1 {
+		return contract.PageRequest{}, gerror.Wrap(contract.ErrInvalidArgument, "invalid pagination query")
+	}
+	page := contract.PageRequest{Cursor: contract.Cursor(query.Get("cursor"))}
+	if value := query.Get("limit"); value != "" {
+		page.Limit, err = strconv.Atoi(value)
+		if err != nil {
+			return contract.PageRequest{}, gerror.Wrap(contract.ErrInvalidArgument, "limit must be an integer")
+		}
+	}
+	return page.Normalized()
+}
+
+func parseReplay(w http.ResponseWriter, r *http.Request) (contract.ReplayRequest, error) {
+	var request contract.ReplayRequest
+	if value := r.Header.Get("Content-Type"); value != "" {
+		mediaType, _, err := mime.ParseMediaType(value)
+		if err != nil || mediaType != "application/json" {
+			return request, gerror.Wrap(contract.ErrInvalidArgument, "replay body must be JSON")
+		}
+	}
+	data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxReplayBodyBytes))
+	if err != nil {
+		var tooLarge *http.MaxBytesError
+		if errors.As(err, &tooLarge) {
+			return request, err
+		}
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "cannot read replay body")
+	}
+	if !utf8.Valid(data) {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "replay body must be UTF-8")
+	}
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&request); err != nil {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "invalid replay JSON")
+	}
+	if err := decoder.Decode(new(any)); err != io.EOF {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "replay body must contain exactly one JSON object")
+	}
+	if len(r.Header.Values(contract.IdempotencyHeader)) != 1 {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "exactly one Idempotency-Key is required")
+	}
+	request.SourceRunID = r.PathValue("runId")
+	request.IdempotencyKey = r.Header.Get(contract.IdempotencyHeader)
+	return request, request.Validate()
+}
+
+func writeError(w http.ResponseWriter, err error) {
+	switch {
+	case errors.Is(err, contract.ErrInvalidArgument):
+		writeAPIError(w, http.StatusBadRequest, "invalid_argument", "Invalid request")
+	case errors.Is(err, contract.ErrNotFound):
+		writeAPIError(w, http.StatusNotFound, "not_found", "Resource not found")
+	case errors.Is(err, contract.ErrFailureDetailsUnavailable):
+		writeAPIError(w, http.StatusGone, "failure_details_unavailable", "Live failure details are no longer available")
+	case errors.Is(err, contract.ErrConflict), errors.Is(err, contract.ErrLeaseLost):
+		writeAPIError(w, http.StatusConflict, "conflict", "Request conflicts with the current resource")
+	case errors.Is(err, contract.ErrBusy):
+		writeAPIError(w, http.StatusServiceUnavailable, "busy", "Resource is busy; retry later")
+	default:
+		writeAPIError(w, http.StatusInternalServerError, "internal_error", "Internal server error")
+	}
+}
+
+func writeAPIError(w http.ResponseWriter, status int, code, message string) {
+	writeJSON(w, status, ErrorResponse{Error: APIError{Code: code, Message: message}})
+}
+
+func writeJSON(w http.ResponseWriter, status int, value any) {
+	// Marshal before writing headers so an invalid service payload cannot
+	// produce a successful status with truncated JSON.
+	data, err := json.Marshal(value)
+	if err != nil {
+		status = http.StatusInternalServerError
+		data = []byte(`{"error":{"code":"internal_error","message":"Internal server error"}}`)
+	}
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(status)
+	_, _ = w.Write(append(data, '\n'))
+}

worker-svr/internal/fullsync/httpapi/handler_test.go

@@ -0,0 +1,437 @@
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+const (
+	testToken   = "test-secret"
+	runPath     = "/internal/v1/full-sync-runs/run-1"
+	failurePath = runPath + "/failures"
+	replayPath  = runPath + "/replay"
+	actionPath  = "/internal/v1/full-sync-actions/action-1"
+)
+
+type runFunc func(context.Context, string) (contract.RunView, error)
+
+func (f runFunc) GetRun(ctx context.Context, id string) (contract.RunView, error) {
+	return f(ctx, id)
+}
+
+type runListFunc func(context.Context, contract.RunListFilter) (contract.RunList, error)
+
+func (f runListFunc) ListRuns(ctx context.Context, filter contract.RunListFilter) (contract.RunList, error) {
+	return f(ctx, filter)
+}
+
+type failureFunc func(context.Context, string, contract.PageRequest) (contract.FailurePage, error)
+
+func (f failureFunc) ListFailures(ctx context.Context, id string, page contract.PageRequest) (contract.FailurePage, error) {
+	return f(ctx, id, page)
+}
+
+type replayFunc func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error)
+
+func (f replayFunc) RequestReplay(ctx context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
+	return f(ctx, req)
+}
+
+type actionFunc func(context.Context, string) (contract.ReplayAction, error)
+
+func (f actionFunc) GetReplayAction(ctx context.Context, id string) (contract.ReplayAction, error) {
+	return f(ctx, id)
+}
+
+func testDependencies() Dependencies {
+	return Dependencies{
+		Runs: runFunc(func(context.Context, string) (contract.RunView, error) {
+			return contract.RunView{}, contract.ErrNotFound
+		}),
+		List: runListFunc(func(context.Context, contract.RunListFilter) (contract.RunList, error) {
+			return contract.RunList{}, contract.ErrNotFound
+		}),
+		Failures: failureFunc(func(context.Context, string, contract.PageRequest) (contract.FailurePage, error) {
+			return contract.FailurePage{}, contract.ErrNotFound
+		}),
+		Replays: replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+			return contract.ReplayAccepted{}, contract.ErrNotFound
+		}),
+		Actions: actionFunc(func(context.Context, string) (contract.ReplayAction, error) {
+			return contract.ReplayAction{}, contract.ErrNotFound
+		}),
+	}
+}
+
+func testHandler(t *testing.T, deps Dependencies) http.Handler {
+	t.Helper()
+	handler, err := NewHandler(testToken, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return handler
+}
+
+func request(handler http.Handler, method, path, body string, headers ...string) *httptest.ResponseRecorder {
+	req := httptest.NewRequest(method, path, strings.NewReader(body))
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	for i := 0; i < len(headers); i += 2 {
+		req.Header.Set(headers[i], headers[i+1])
+	}
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, req)
+	return recorder
+}
+
+func assertError(t *testing.T, response *httptest.ResponseRecorder, status int, code string) {
+	t.Helper()
+	if response.Code != status {
+		t.Fatalf("status = %d, want %d: %s", response.Code, status, response.Body)
+	}
+	if !strings.HasPrefix(response.Header().Get("Content-Type"), "application/json") {
+		t.Fatalf("missing JSON content type: %v", response.Header())
+	}
+	var body ErrorResponse
+	if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+		t.Fatal(err)
+	}
+	if body.Error.Code != code || body.Error.Message == "" {
+		t.Fatalf("error = %+v, want code %q and message", body, code)
+	}
+}
+
+func TestAuthenticationProtectsEveryRoute(t *testing.T) {
+	deps := testDependencies()
+	deps.Runs = runFunc(func(context.Context, string) (contract.RunView, error) {
+		t.Fatal("unauthenticated request reached service")
+		return contract.RunView{}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, path := range []string{runPath, failurePath, replayPath, actionPath, "/unknown"} {
+		for _, auth := range []string{"", "Bearer", "Bearer wrong", "Basic " + testToken, "Bearer " + testToken + " extra"} {
+			t.Run(path+"/"+auth, func(t *testing.T) {
+				response := request(handler, http.MethodGet, path, "", "Authorization", auth)
+				assertError(t, response, 401, "unauthorized")
+				if response.Header().Get("WWW-Authenticate") == "" {
+					t.Fatal("missing authentication challenge")
+				}
+			})
+		}
+	}
+	req := httptest.NewRequest(http.MethodGet, runPath, nil)
+	req.Header.Add("Authorization", "Bearer "+testToken)
+	req.Header.Add("Authorization", "Bearer "+testToken)
+	response := httptest.NewRecorder()
+	handler.ServeHTTP(response, req)
+	assertError(t, response, 401, "unauthorized")
+}
+
+func TestRunListQuery(t *testing.T) {
+	deps := testDependencies()
+	deps.List = runListFunc(func(_ context.Context, filter contract.RunListFilter) (contract.RunList, error) {
+		if filter.SyncKind != contract.SyncKindTrafficUsage || filter.Limit != 7 {
+			t.Fatalf("filter = %+v", filter)
+		}
+		return contract.RunList{Items: []contract.RunView{{Source: contract.SourceLive, Run: contract.RunMeta{RunID: "run-1", Operation: "current_month_usage_full_sync"}}}, Total: 1}, nil
+	})
+	response := request(testHandler(t, deps), http.MethodGet, "/internal/v1/full-sync-runs?syncKind=traffic_usage&limit=7", "")
+	if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), "run-1") {
+		t.Fatalf("response = %d %s", response.Code, response.Body)
+	}
+	for _, query := range []string{"syncKind=invalid", "limit=101", "limit=abc", "limit=1&limit=2"} {
+		assertError(t, request(testHandler(t, deps), http.MethodGet, "/internal/v1/full-sync-runs?"+query, ""), 400, "invalid_argument")
+	}
+}
+
+func TestRoutesAndValidation(t *testing.T) {
+	handler := testHandler(t, testDependencies())
+	for _, route := range []struct{ method, path string }{
+		{http.MethodGet, runPath}, {http.MethodGet, failurePath},
+		{http.MethodPost, replayPath}, {http.MethodGet, actionPath},
+	} {
+		t.Run(route.path, func(t *testing.T) {
+			response := request(handler, route.method, route.path, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key")
+			assertError(t, response, 404, "not_found")
+			wrongMethod := http.MethodPost
+			if route.method == http.MethodPost {
+				wrongMethod = http.MethodGet
+			}
+			response = request(handler, wrongMethod, route.path, "")
+			assertError(t, response, 405, "method_not_allowed")
+			if response.Header().Get("Allow") != route.method {
+				t.Fatalf("Allow = %q", response.Header().Get("Allow"))
+			}
+		})
+	}
+	for _, path := range []string{"/unknown", "/internal/v1/full-sync-runs", runPath + "/other", runPath + "/"} {
+		assertError(t, request(handler, http.MethodGet, path, ""), 404, "not_found")
+	}
+	for _, path := range []string{
+		"/internal/v1/full-sync-runs/%20run",
+		"/internal/v1/full-sync-runs/a%2Fb/failures",
+		"/internal/v1/full-sync-actions/%7Baction%7D",
+	} {
+		assertError(t, request(handler, http.MethodGet, path, ""), 400, "invalid_argument")
+	}
+	assertError(t, request(handler, http.MethodPost, "/internal/v1/full-sync-runs/a%5Cb/replay", `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key"), 400, "invalid_argument")
+}
+
+func TestRunAndActionResponsesPropagateContext(t *testing.T) {
+	type contextKey struct{}
+	deps := testDependencies()
+	deps.Runs = runFunc(func(ctx context.Context, id string) (contract.RunView, error) {
+		if id != "run-1" || ctx.Value(contextKey{}) != "request" {
+			t.Fatal("run identity or request context lost")
+		}
+		return contract.RunView{Source: contract.SourceLive, Run: contract.RunMeta{RunID: id}}, nil
+	})
+	deps.Actions = actionFunc(func(ctx context.Context, id string) (contract.ReplayAction, error) {
+		if id != "action-1" || ctx.Value(contextKey{}) != "request" {
+			t.Fatal("action identity or request context lost")
+		}
+		return contract.ReplayAction{ActionID: id, Status: contract.ActionProcessing, IdempotencyKey: "private-key", Fingerprint: "private-fingerprint", LeaseToken: "private-lease", Revision: 99}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, path := range []string{runPath, actionPath} {
+		req := httptest.NewRequest(http.MethodGet, path, nil).WithContext(context.WithValue(context.Background(), contextKey{}, "request"))
+		req.Header.Set("Authorization", "bearer "+testToken)
+		response := httptest.NewRecorder()
+		handler.ServeHTTP(response, req)
+		if response.Code != 200 || response.Header().Get("Cache-Control") != "no-store" {
+			t.Fatalf("response = %d %s", response.Code, response.Body)
+		}
+		if strings.Contains(response.Body.String(), "private-") || strings.Contains(response.Body.String(), "revision") {
+			t.Fatalf("internal fields leaked: %s", response.Body)
+		}
+	}
+}
+
+func TestFailurePagination(t *testing.T) {
+	for _, test := range []struct {
+		query  string
+		limit  int
+		cursor contract.Cursor
+	}{
+		{"", 100, ""}, {"?limit=", 100, ""}, {"?limit=0", 100, ""},
+		{"?limit=1", 1, ""}, {"?limit=500", 500, ""},
+		{"?limit=17&cursor=" + url.QueryEscape("opaque/+ token==:雪"), 17, "opaque/+ token==:雪"},
+	} {
+		t.Run(test.query, func(t *testing.T) {
+			deps := testDependencies()
+			calls := 0
+			deps.Failures = failureFunc(func(_ context.Context, id string, page contract.PageRequest) (contract.FailurePage, error) {
+				calls++
+				if id != "run-1" || page.Limit != test.limit || page.Cursor != test.cursor {
+					t.Fatalf("query = %q %+v", id, page)
+				}
+				return contract.FailurePage{NextCursor: "next:opaque/+="}, nil
+			})
+			response := request(testHandler(t, deps), http.MethodGet, failurePath+test.query, "")
+			if response.Code != 200 || calls != 1 {
+				t.Fatalf("response = %d, calls = %d", response.Code, calls)
+			}
+			var page contract.FailurePage
+			if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil {
+				t.Fatal(err)
+			}
+			if page.Items == nil || len(page.Items) != 0 || page.NextCursor != "next:opaque/+=" {
+				t.Fatalf("empty scan page lost array or continuation: %s", response.Body)
+			}
+		})
+	}
+	deps := testDependencies()
+	deps.Failures = failureFunc(func(context.Context, string, contract.PageRequest) (contract.FailurePage, error) {
+		t.Fatal("invalid pagination reached service")
+		return contract.FailurePage{}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, query := range []string{"limit=-1", "limit=501", "limit=abc", "limit=1.5", "limit=999999999999999999999999", "limit=1&limit=2", "cursor=a&cursor=b", "cursor=%ZZ", "cursor=a;b"} {
+		assertError(t, request(handler, http.MethodGet, failurePath+"?"+query, ""), 400, "invalid_argument")
+	}
+}
+
+func TestFailureCursorValidationAndLiveOnly(t *testing.T) {
+	deps := testDependencies()
+	deps.Runs = runFunc(func(context.Context, string) (contract.RunView, error) {
+		t.Fatal("failure route must not query run summaries or fall back to archives")
+		return contract.RunView{}, nil
+	})
+	deps.Failures = failureFunc(func(_ context.Context, _ string, page contract.PageRequest) (contract.FailurePage, error) {
+		if page.Cursor == "another-run-token" {
+			return contract.FailurePage{}, fmt.Errorf("wrong query binding: %w", contract.ErrInvalidArgument)
+		}
+		return contract.FailurePage{}, fmt.Errorf("expired details: %w", contract.ErrFailureDetailsUnavailable)
+	})
+	handler := testHandler(t, deps)
+	assertError(t, request(handler, http.MethodGet, failurePath+"?cursor=another-run-token", ""), 400, "invalid_argument")
+	assertError(t, request(handler, http.MethodGet, failurePath, ""), 410, "failure_details_unavailable")
+}
+
+func TestReplayValidation(t *testing.T) {
+	deps := testDependencies()
+	deps.Replays = replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+		t.Fatal("invalid replay reached service")
+		return contract.ReplayAccepted{}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, body := range []string{"", "{", "null", "[]", `{}`, `{"requestedBy":""}`, `{"requestedBy":"  "}`, `{"requestedBy":12}`, `{"requestedBy":"a","targetRunId":"injected"}`, `{"requestedBy":"a"} {}`, `{"requestedBy":"a"} trailing`, "{\"requestedBy\":\"\xff\"}"} {
+		t.Run(body, func(t *testing.T) {
+			assertError(t, request(handler, http.MethodPost, replayPath, body, contract.IdempotencyHeader, "key"), 400, "invalid_argument")
+		})
+	}
+	for _, key := range []string{"", " ", strings.Repeat("x", contract.MaxIdempotencyKeyBytes+1), "bad\tkey"} {
+		assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, key), 400, "invalid_argument")
+	}
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`), 400, "invalid_argument")
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key", "Content-Type", "text/plain"), 400, "invalid_argument")
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"`+strings.Repeat("x", maxReplayBodyBytes)+`"}`, contract.IdempotencyHeader, "key"), 413, "body_too_large")
+	req := httptest.NewRequest(http.MethodPost, replayPath, strings.NewReader(`{"requestedBy":"operator"}`))
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	req.Header.Add(contract.IdempotencyHeader, "key")
+	req.Header.Add(contract.IdempotencyHeader, "key")
+	response := httptest.NewRecorder()
+	handler.ServeHTTP(response, req)
+	assertError(t, response, 400, "invalid_argument")
+}
+
+// The mock owns idempotency; the HTTP adapter must pass exact request identity
+// and return the service's existing action/target even across handler instances.
+func TestReplayAcceptedIdempotencyAndConflict(t *testing.T) {
+	deps := testDependencies()
+	var (
+		stored contract.ReplayRequest
+		calls  int
+	)
+	deps.Replays = replayFunc(func(_ context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
+		calls++
+		if calls == 1 {
+			stored = req
+		}
+		if req != stored {
+			return contract.ReplayAccepted{}, fmt.Errorf("changed fingerprint: %w", contract.ErrConflict)
+		}
+		return contract.ReplayAccepted{ActionID: "action-1", TargetRunID: "replay-1", Status: contract.ActionAccepted, StatusURL: "https://untrusted.invalid"}, nil
+	})
+	var first contract.ReplayAccepted
+	for i := 0; i < 2; i++ {
+		response := request(testHandler(t, deps), http.MethodPost, replayPath, "{\"requestedBy\":\" operator \"}\n", contract.IdempotencyHeader, "key", "Content-Type", "application/json; charset=utf-8")
+		if response.Code != 202 {
+			t.Fatalf("response = %d %s", response.Code, response.Body)
+		}
+		var accepted contract.ReplayAccepted
+		if err := json.Unmarshal(response.Body.Bytes(), &accepted); err != nil {
+			t.Fatal(err)
+		}
+		if accepted.ActionID != "action-1" || accepted.TargetRunID != "replay-1" || accepted.StatusURL != actionPath || response.Header().Get("Location") != actionPath {
+			t.Fatalf("acceptance = %+v, headers = %v", accepted, response.Header())
+		}
+		if i == 0 {
+			first = accepted
+		} else if !reflect.DeepEqual(first, accepted) {
+			t.Fatalf("repeat changed identity: %+v vs %+v", first, accepted)
+		}
+	}
+	if stored.SourceRunID != "run-1" || stored.RequestedBy != " operator " || stored.IdempotencyKey != "key" {
+		t.Fatalf("request identity changed: %+v", stored)
+	}
+	assertError(t, request(testHandler(t, deps), http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key"), 409, "conflict")
+	if calls != 3 {
+		t.Fatalf("replay calls = %d", calls)
+	}
+}
+
+func TestWrappedServiceErrorsUseConsistentEnvelope(t *testing.T) {
+	for _, test := range []struct {
+		err    error
+		status int
+		code   string
+	}{
+		{contract.ErrInvalidArgument, 400, "invalid_argument"},
+		{contract.ErrNotFound, 404, "not_found"},
+		{contract.ErrConflict, 409, "conflict"},
+		{contract.ErrLeaseLost, 409, "conflict"},
+		{contract.ErrFailureDetailsUnavailable, 410, "failure_details_unavailable"},
+		{contract.ErrBusy, 503, "busy"},
+		{fmt.Errorf("private database password"), 500, "internal_error"},
+	} {
+		t.Run(test.code, func(t *testing.T) {
+			err := fmt.Errorf("private database detail: %w", test.err)
+			deps := Dependencies{
+				Runs: runFunc(func(context.Context, string) (contract.RunView, error) { return contract.RunView{}, err }),
+				List: runListFunc(func(context.Context, contract.RunListFilter) (contract.RunList, error) {
+					return contract.RunList{}, err
+				}),
+				Failures: failureFunc(func(context.Context, string, contract.PageRequest) (contract.FailurePage, error) {
+					return contract.FailurePage{}, err
+				}),
+				Replays: replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+					return contract.ReplayAccepted{}, err
+				}),
+				Actions: actionFunc(func(context.Context, string) (contract.ReplayAction, error) { return contract.ReplayAction{}, err }),
+			}
+			handler := testHandler(t, deps)
+			for _, path := range []string{runPath, failurePath, actionPath, replayPath} {
+				method := http.MethodGet
+				if path == replayPath {
+					method = http.MethodPost
+				}
+				response := request(handler, method, path, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key")
+				assertError(t, response, test.status, test.code)
+				if strings.Contains(response.Body.String(), "private") {
+					t.Fatal("service details leaked")
+				}
+			}
+		})
+	}
+}
+
+func TestInvalidServiceOutputReturns500(t *testing.T) {
+	deps := testDependencies()
+	deps.Replays = replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+		return contract.ReplayAccepted{ActionID: "bad/id"}, nil
+	})
+	deps.Runs = runFunc(func(context.Context, string) (contract.RunView, error) {
+		return contract.RunView{Rounds: []contract.RoundStats{{ErrorCounts: contract.ErrorCounts{contract.ErrorOther: -1}}}}, nil
+	})
+	handler := testHandler(t, deps)
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key"), 500, "internal_error")
+	assertError(t, request(handler, http.MethodGet, runPath, ""), 500, "internal_error")
+}
+
+func TestNewHandlerRejectsInvalidConfiguration(t *testing.T) {
+	if _, err := NewHandler("", testDependencies()); err != nil {
+		t.Fatalf("empty token should use Docker-network-only access: %v", err)
+	}
+	for _, token := range []string{" ", "a b", "token\n"} {
+		if _, err := NewHandler(token, testDependencies()); err == nil {
+			t.Errorf("accepted token %q", token)
+		}
+	}
+	for i := 0; i < 5; i++ {
+		deps := testDependencies()
+		switch i {
+		case 0:
+			deps.Runs = nil
+		case 1:
+			deps.Failures = nil
+		case 2:
+			deps.Replays = nil
+		case 3:
+			deps.Actions = nil
+		case 4:
+			deps.List = nil
+		}
+		if _, err := NewHandler(testToken, deps); err == nil {
+			t.Errorf("accepted missing dependency %d", i)
+		}
+	}
+}

worker-svr/internal/fullsync/httpapi/run_reader.go

@@ -0,0 +1,48 @@
+package httpapi
+
+import (
+	"context"
+	"errors"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+type runReader struct {
+	live    contract.LiveRunReader
+	archive contract.ArchivedRunReader
+	rounds  contract.RoundStatsReader
+}
+
+// NewRunReader composes live-first run queries without knowing either storage
+// implementation. Archive and rounds are optional (nil disables that lookup).
+// Only absent live metadata permits archive fallback; live round errors do not.
+func NewRunReader(live contract.LiveRunReader, archive contract.ArchivedRunReader, rounds contract.RoundStatsReader) (contract.RunReader, error) {
+	if live == nil {
+		return nil, gerror.New("full-sync run reader requires a live reader")
+	}
+	return &runReader{live: live, archive: archive, rounds: rounds}, nil
+}
+
+func (r *runReader) GetRun(ctx context.Context, runID string) (contract.RunView, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunView{}, err
+	}
+	meta, err := r.live.GetRunMeta(ctx, runID)
+	if err == nil {
+		view := contract.RunView{Source: contract.SourceLive, Run: meta}
+		if r.rounds != nil {
+			view.Rounds, err = r.rounds.GetRoundStats(ctx, runID)
+		}
+		return view, err
+	}
+	if !errors.Is(err, contract.ErrNotFound) || r.archive == nil {
+		return contract.RunView{}, err
+	}
+	archive, err := r.archive.GetArchivedRun(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	return contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}, nil
+}

worker-svr/internal/fullsync/httpapi/run_reader_test.go

@@ -0,0 +1,125 @@
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+type liveFunc func(context.Context, string) (contract.RunMeta, error)
+
+func (f liveFunc) GetRunMeta(ctx context.Context, id string) (contract.RunMeta, error) {
+	return f(ctx, id)
+}
+
+type archiveFunc func(context.Context, string) (contract.RunArchive, error)
+
+func (f archiveFunc) GetArchivedRun(ctx context.Context, id string) (contract.RunArchive, error) {
+	return f(ctx, id)
+}
+
+type roundsFunc func(context.Context, string) ([]contract.RoundStats, error)
+
+func (f roundsFunc) GetRoundStats(ctx context.Context, id string) ([]contract.RoundStats, error) {
+	return f(ctx, id)
+}
+
+func TestRunLookupLiveFirstAndArchiveFallback(t *testing.T) {
+	for _, test := range []struct {
+		name         string
+		liveErr      error
+		roundsErr    error
+		archiveErr   error
+		wantStatus   int
+		wantSource   contract.RunSource
+		wantArchives int
+		wantRounds   int
+	}{
+		{name: "live", wantStatus: 200, wantSource: contract.SourceLive, wantRounds: 1},
+		{name: "archive", liveErr: fmt.Errorf("expired: %w", contract.ErrNotFound), wantStatus: 200, wantSource: contract.SourceArchive, wantArchives: 1},
+		{name: "unknown", liveErr: contract.ErrNotFound, archiveErr: contract.ErrNotFound, wantStatus: 404, wantArchives: 1},
+		{name: "live failure never falls back", liveErr: fmt.Errorf("live offline"), wantStatus: 500},
+		{name: "round error never falls back", roundsErr: fmt.Errorf("rounds offline"), wantStatus: 500, wantRounds: 1},
+		{name: "missing rounds never fall back", roundsErr: contract.ErrNotFound, wantStatus: 404, wantRounds: 1},
+		{name: "archive error", liveErr: contract.ErrNotFound, archiveErr: fmt.Errorf("archive offline"), wantStatus: 500, wantArchives: 1},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var liveCalls, archiveCalls, roundsCalls int
+			reader, err := NewRunReader(
+				liveFunc(func(_ context.Context, id string) (contract.RunMeta, error) {
+					liveCalls++
+					if id != "run-1" {
+						t.Fatalf("live ID = %q", id)
+					}
+					return contract.RunMeta{RunID: id, CurrentRound: 2}, test.liveErr
+				}),
+				archiveFunc(func(_ context.Context, id string) (contract.RunArchive, error) {
+					archiveCalls++
+					if id != "run-1" {
+						t.Fatalf("archive ID = %q", id)
+					}
+					return contract.RunArchive{Run: contract.RunMeta{RunID: id, CurrentRound: 3}, Rounds: []contract.RoundStats{{RunID: id, Round: 3}}}, test.archiveErr
+				}),
+				roundsFunc(func(_ context.Context, id string) ([]contract.RoundStats, error) {
+					roundsCalls++
+					if id != "run-1" {
+						t.Fatalf("rounds ID = %q", id)
+					}
+					return []contract.RoundStats{{RunID: id, Round: 2}}, test.roundsErr
+				}),
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+			deps := testDependencies()
+			deps.Runs = reader
+			response := request(testHandler(t, deps), http.MethodGet, runPath, "")
+			if response.Code != test.wantStatus {
+				t.Fatalf("response = %d %s", response.Code, response.Body)
+			}
+			if liveCalls != 1 || archiveCalls != test.wantArchives || roundsCalls != test.wantRounds {
+				t.Fatalf("calls: live=%d archive=%d rounds=%d", liveCalls, archiveCalls, roundsCalls)
+			}
+			if response.Code == 200 {
+				var view contract.RunView
+				if err := json.Unmarshal(response.Body.Bytes(), &view); err != nil {
+					t.Fatal(err)
+				}
+				if view.Source != test.wantSource || view.Run.RunID != "run-1" || len(view.Rounds) != 1 || view.Rounds[0].Round != view.Run.CurrentRound {
+					t.Fatalf("view = %+v", view)
+				}
+			}
+		})
+	}
+}
+
+func TestRunReaderOptionalServicesAndValidation(t *testing.T) {
+	if _, err := NewRunReader(nil, nil, nil); err == nil {
+		t.Fatal("accepted nil live reader")
+	}
+	calls := 0
+	reader, err := NewRunReader(liveFunc(func(_ context.Context, id string) (contract.RunMeta, error) {
+		calls++
+		if id == "missing" {
+			return contract.RunMeta{}, contract.ErrNotFound
+		}
+		return contract.RunMeta{RunID: id}, nil
+	}), nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := reader.GetRun(context.Background(), "bad/id"); err == nil || calls != 0 {
+		t.Fatal("invalid ID reached live reader")
+	}
+	view, err := reader.GetRun(context.Background(), "run-1")
+	if err != nil || view.Source != contract.SourceLive || view.Rounds != nil {
+		t.Fatalf("view = %+v, err = %v", view, err)
+	}
+	if _, err := reader.GetRun(context.Background(), "missing"); err != contract.ErrNotFound {
+		t.Fatalf("error = %v", err)
+	}
+}

worker-svr/internal/fullsync/httpapi/server.go

@@ -0,0 +1,119 @@
+package httpapi
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+// Config is deliberately independent of worker/config-file wiring. Addr must
+// be explicit; use 127.0.0.1:0 to let the OS allocate a local test port.
+type Config struct {
+	Addr            string
+	BearerToken     string // Optional for standalone callers; production Worker leaves it empty.
+	ShutdownTimeout time.Duration
+}
+
+// Server is single-use. Start reports bind errors synchronously; after a
+// successful Start, Wait reports serve or shutdown errors. Canceling the Start
+// context stops accepting requests and drains active requests up to the timeout.
+type Server struct {
+	server          *http.Server
+	shutdownTimeout time.Duration
+	mu              sync.Mutex
+	started         bool
+	addr            net.Addr
+	done            chan struct{}
+	err             error
+}
+
+func NewServer(config Config, deps Dependencies) (*Server, error) {
+	if config.Addr == "" || config.ShutdownTimeout < 0 {
+		return nil, gerror.New("full-sync HTTP requires an address and a nonnegative shutdown timeout")
+	}
+	handler, err := NewHandler(config.BearerToken, deps)
+	if err != nil {
+		return nil, err
+	}
+	if config.ShutdownTimeout == 0 {
+		config.ShutdownTimeout = 10 * time.Second
+	}
+	return &Server{
+		server: &http.Server{
+			Addr: config.Addr, Handler: handler,
+			ReadHeaderTimeout: 5 * time.Second,
+			ReadTimeout:       30 * time.Second,
+			WriteTimeout:      30 * time.Second,
+			IdleTimeout:       60 * time.Second,
+		},
+		shutdownTimeout: config.ShutdownTimeout,
+		done:            make(chan struct{}),
+	}, nil
+}
+
+func (s *Server) Start(ctx context.Context) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.started {
+		return gerror.New("full-sync HTTP server has already been started")
+	}
+	s.started = true
+	if err := ctx.Err(); err != nil {
+		s.err = err
+		close(s.done)
+		return err
+	}
+	listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", s.server.Addr)
+	if err != nil {
+		s.err = gerror.Wrap(err, "listen for full-sync HTTP")
+		close(s.done)
+		return s.err
+	}
+	s.addr = listener.Addr()
+	go s.run(ctx, listener)
+	return nil
+}
+
+// Addr returns the bound address, or nil before a successful Start.
+func (s *Server) Addr() net.Addr {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.addr
+}
+
+// Wait may be called concurrently after Start (including a failed Start).
+func (s *Server) Wait() error {
+	<-s.done
+	return s.err
+}
+
+func (s *Server) run(ctx context.Context, listener net.Listener) {
+	serveDone := make(chan error, 1)
+	go func() { serveDone <- s.server.Serve(listener) }()
+	var err error
+	select {
+	case err = <-serveDone:
+		// An unexpected listener failure also needs to drain active requests.
+	case <-ctx.Done():
+	}
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), s.shutdownTimeout)
+	shutdownErr := s.server.Shutdown(shutdownCtx)
+	cancel()
+	if shutdownErr != nil {
+		// Shutdown does not close active connections when its deadline expires.
+		_ = s.server.Close()
+	}
+	if err == nil {
+		err = <-serveDone
+	}
+	if errors.Is(err, http.ErrServerClosed) {
+		err = nil
+	}
+	s.err = errors.Join(err, shutdownErr)
+	close(s.done)
+}

worker-svr/internal/fullsync/httpapi/server_test.go

@@ -0,0 +1,226 @@
+package httpapi
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func awaitServer(t *testing.T, done <-chan error) error {
+	t.Helper()
+	select {
+	case err := <-done:
+		return err
+	case <-time.After(5 * time.Second):
+		t.Fatal("server did not stop")
+		return nil
+	}
+}
+
+func startTestServer(t *testing.T, deps Dependencies, timeout time.Duration) (*Server, context.CancelFunc, <-chan error) {
+	t.Helper()
+	server, err := NewServer(Config{Addr: "127.0.0.1:0", BearerToken: testToken, ShutdownTimeout: timeout}, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if server.Addr() != nil {
+		t.Fatal("address before Start")
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	if err := server.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+	done := make(chan error, 1)
+	go func() { done <- server.Wait() }()
+	return server, cancel, done
+}
+
+func serverRequest(server *Server) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, "http://"+server.Addr().String()+runPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	client := &http.Client{Timeout: 4 * time.Second}
+	return client.Do(req)
+}
+
+func TestServerStartServeAndCancel(t *testing.T) {
+	server, cancel, done := startTestServer(t, testDependencies(), 0)
+	response, err := serverRequest(server)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = response.Body.Close()
+	if response.StatusCode != 404 {
+		t.Fatalf("status = %d", response.StatusCode)
+	}
+	if err := server.Start(context.Background()); err == nil {
+		t.Fatal("duplicate Start succeeded")
+	}
+	cancel()
+	if err := awaitServer(t, done); err != nil {
+		t.Fatal(err)
+	}
+	if err := server.Wait(); err != nil {
+		t.Fatal(err)
+	}
+	connection, err := net.DialTimeout("tcp", server.Addr().String(), time.Second)
+	if err == nil {
+		_ = connection.Close()
+		t.Fatal("listener still accepts after shutdown")
+	}
+}
+
+func TestServerGracefulShutdownDrainsActiveRequest(t *testing.T) {
+	var (
+		entered = make(chan struct{})
+		release = make(chan struct{})
+		once    sync.Once
+	)
+	t.Cleanup(func() { once.Do(func() { close(release) }) })
+	deps := testDependencies()
+	deps.Runs = runFunc(func(ctx context.Context, id string) (contract.RunView, error) {
+		close(entered)
+		select {
+		case <-release:
+			return contract.RunView{Source: contract.SourceLive, Run: contract.RunMeta{RunID: id}}, nil
+		case <-ctx.Done():
+			return contract.RunView{}, ctx.Err()
+		}
+	})
+	server, cancel, done := startTestServer(t, deps, 2*time.Second)
+	requestDone := make(chan error, 1)
+	go func() {
+		response, err := serverRequest(server)
+		if err == nil {
+			_, err = io.ReadAll(response.Body)
+			_ = response.Body.Close()
+			if response.StatusCode != 200 {
+				err = errors.New("active request did not complete successfully")
+			}
+		}
+		requestDone <- err
+	}()
+	select {
+	case <-entered:
+	case <-time.After(3 * time.Second):
+		t.Fatal("request did not enter handler")
+	}
+	cancel()
+	select {
+	case err := <-done:
+		t.Fatalf("shutdown returned before draining: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+	once.Do(func() { close(release) })
+	if err := awaitServer(t, requestDone); err != nil {
+		t.Fatal(err)
+	}
+	if err := awaitServer(t, done); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestServerShutdownDeadlineClosesActiveConnections(t *testing.T) {
+	var (
+		entered = make(chan struct{})
+		exited  = make(chan struct{})
+	)
+	deps := testDependencies()
+	deps.Runs = runFunc(func(ctx context.Context, _ string) (contract.RunView, error) {
+		close(entered)
+		<-ctx.Done()
+		close(exited)
+		return contract.RunView{}, ctx.Err()
+	})
+	server, cancel, done := startTestServer(t, deps, 50*time.Millisecond)
+	requestDone := make(chan error, 1)
+	go func() {
+		response, err := serverRequest(server)
+		if response != nil {
+			_ = response.Body.Close()
+		}
+		requestDone <- err
+	}()
+	select {
+	case <-entered:
+	case <-time.After(3 * time.Second):
+		t.Fatal("request did not enter handler")
+	}
+	cancel()
+	if err := awaitServer(t, done); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("shutdown error = %v", err)
+	}
+	select {
+	case <-exited:
+	case <-time.After(3 * time.Second):
+		t.Fatal("forced close did not cancel request context")
+	}
+	_ = awaitServer(t, requestDone)
+}
+
+func TestServerStartupFailureIsSynchronous(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = listener.Close() })
+	server, err := NewServer(Config{Addr: listener.Addr().String(), BearerToken: testToken}, testDependencies())
+	if err != nil {
+		t.Fatal(err)
+	}
+	startErr := server.Start(context.Background())
+	if startErr == nil {
+		t.Fatal("Start did not report occupied port")
+	}
+	if server.Wait() != startErr || server.Addr() != nil {
+		t.Fatal("failed startup state was lost")
+	}
+	var netErr *net.OpError
+	if !errors.As(startErr, &netErr) {
+		t.Fatalf("network error cause lost: %v", startErr)
+	}
+}
+
+func TestServerCanceledBeforeStartAndImmediateCancellation(t *testing.T) {
+	server, err := NewServer(Config{Addr: "127.0.0.1:0", BearerToken: testToken}, testDependencies())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := server.Start(ctx); !errors.Is(err, context.Canceled) {
+		t.Fatalf("Start = %v", err)
+	}
+	if err := server.Wait(); !errors.Is(err, context.Canceled) {
+		t.Fatalf("Wait = %v", err)
+	}
+	// Cancellation can win the race before Serve starts accepting requests.
+	for i := 0; i < 10; i++ {
+		_, cancel, done := startTestServer(t, testDependencies(), time.Second)
+		cancel()
+		if err := awaitServer(t, done); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestNewServerRejectsInvalidConfiguration(t *testing.T) {
+	for _, config := range []Config{
+		{BearerToken: testToken},
+		{Addr: "127.0.0.1:0", BearerToken: testToken, ShutdownTimeout: -time.Second},
+	} {
+		if _, err := NewServer(config, testDependencies()); err == nil {
+			t.Fatalf("accepted config %+v", config)
+		}
+	}
+}

worker-svr/internal/fullsync/httpapi/simple_replay_test.go

@@ -0,0 +1,59 @@
+package httpapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/runtime"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayHTTPPrivateNetworkAcceptance(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := zset.New(redis, "sim-status:")
+	service, _ := runtime.NewSimpleReplay(store)
+	ctx := context.Background()
+	key, _ := store.RunMetaKey("source")
+	failed, _ := store.FailedKey("source")
+	_, _ = redis.Do(ctx, "HSET", key, "runId", "source", "status", "completed_with_failures", "producerDone", "1", "currentRound", "1")
+	record := zset.FailureRecord{TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "ch", TempKey: "template"}
+	value, _ := json.Marshal(record)
+	_, _ = redis.Do(ctx, "HSET", failed, "task", string(value))
+	deps := testDependencies()
+	deps.Simple = service
+	handler, err := NewHandler("", deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := httptest.NewRequest(http.MethodPost, "/internal/v1/full-sync-runs/source/replay", bytes.NewBufferString(`{"requestedBy":"42"}`))
+	w := httptest.NewRecorder()
+	handler.ServeHTTP(w, req)
+	if w.Code != 202 {
+		t.Fatalf("accepted: %d %s", w.Code, w.Body.String())
+	}
+	var receipt runtime.ReplayReceipt
+	if err := json.Unmarshal(w.Body.Bytes(), &receipt); err != nil || receipt.ReplayRunID == "" {
+		t.Fatalf("receipt: %+v %v", receipt, err)
+	}
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		w := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodGet, "/internal/v1/full-sync-replays/"+receipt.ReplayRunID, nil)
+		handler.ServeHTTP(w, req)
+		if w.Code == 200 {
+			var status runtime.ReplayStatus
+			_ = json.Unmarshal(w.Body.Bytes(), &status)
+			if status.Status == "completed" && status.Enqueued == 1 {
+				return
+			}
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("replay HTTP never completed")
+}

worker-svr/internal/fullsync/runtime/catalog.go

@@ -0,0 +1,211 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+const (
+	defaultRunListLimit = 50
+	maxRunListLimit     = 100
+)
+
+// Catalog aggregates the flow and sim-status Redis namespaces while keeping
+// each run's storage protocol inside its existing Repository.
+type Catalog struct {
+	repositories []*Repository
+}
+
+func NewCatalog(repositories ...*Repository) (*Catalog, error) {
+	if len(repositories) == 0 {
+		return nil, errors.New("full-sync catalog requires repositories")
+	}
+	for _, repository := range repositories {
+		if repository == nil {
+			return nil, errors.New("full-sync catalog repository is nil")
+		}
+	}
+	return &Catalog{repositories: repositories}, nil
+}
+
+func (c *Catalog) GetRun(ctx context.Context, runID string) (contract.RunView, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunView{}, err
+	}
+	for _, repository := range c.repositories {
+		meta, err := repository.GetRunMeta(ctx, runID)
+		if err == nil {
+			rounds, roundErr := repository.GetRoundStats(ctx, runID)
+			if roundErr != nil {
+				return contract.RunView{}, roundErr
+			}
+			return contract.RunView{Source: contract.SourceLive, Run: meta, Rounds: rounds}, nil
+		}
+		if !errors.Is(err, contract.ErrNotFound) {
+			return contract.RunView{}, err
+		}
+	}
+	for _, repository := range c.repositories {
+		archive, err := repository.GetArchivedRun(ctx, runID)
+		if err == nil {
+			return contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}, nil
+		}
+		if !errors.Is(err, contract.ErrNotFound) {
+			return contract.RunView{}, err
+		}
+	}
+	return contract.RunView{}, contract.ErrNotFound
+}
+
+func (c *Catalog) ListRuns(ctx context.Context, filter contract.RunListFilter) (contract.RunList, error) {
+	if filter.Limit == 0 {
+		filter.Limit = defaultRunListLimit
+	}
+	if filter.Limit < 1 || filter.Limit > maxRunListLimit {
+		return contract.RunList{}, contract.ErrInvalidArgument
+	}
+	byID := make(map[string]contract.RunView)
+	for _, repository := range c.repositories {
+		active, err := repository.store.ActiveRuns(ctx)
+		if err != nil {
+			return contract.RunList{}, err
+		}
+		for _, runID := range active {
+			view, getErr := repository.liveRun(ctx, runID)
+			if errors.Is(getErr, contract.ErrNotFound) {
+				continue
+			}
+			if getErr != nil {
+				return contract.RunList{}, getErr
+			}
+			byID[runID] = view
+		}
+	}
+	if err := c.listArchives(ctx, byID); err != nil {
+		return contract.RunList{}, err
+	}
+	items := make([]contract.RunView, 0, len(byID))
+	for _, item := range byID {
+		kind := contract.SyncKindForOperation(item.Run.Operation)
+		if kind == "" || (filter.SyncKind != "" && kind != filter.SyncKind) {
+			continue
+		}
+		items = append(items, item)
+	}
+	sort.SliceStable(items, func(i, j int) bool {
+		return runSortTime(items[i].Run).After(runSortTime(items[j].Run))
+	})
+	total := len(items)
+	if len(items) > filter.Limit {
+		items = items[:filter.Limit]
+	}
+	if items == nil {
+		items = []contract.RunView{}
+	}
+	return contract.RunList{Items: items, Total: total}, nil
+}
+
+func (c *Catalog) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.FailurePage{}, err
+	}
+	for _, repository := range c.repositories {
+		meta, err := repository.GetRunMeta(ctx, runID)
+		if err == nil {
+			result, listErr := repository.ListFailures(ctx, runID, page)
+			if listErr != nil {
+				return contract.FailurePage{}, listErr
+			}
+			if meta.FailedCount > 0 && len(result.Items) == 0 && result.NextCursor == "" {
+				failedKey, _ := repository.store.FailedKey(runID)
+				size, sizeErr := repository.redis.Do(ctx, "HLEN", failedKey)
+				if sizeErr != nil {
+					return contract.FailurePage{}, sizeErr
+				}
+				if size.Int64() == 0 {
+					return contract.FailurePage{}, contract.ErrFailureDetailsUnavailable
+				}
+			}
+			return result, nil
+		} else if !errors.Is(err, contract.ErrNotFound) {
+			return contract.FailurePage{}, err
+		}
+	}
+	view, err := c.GetRun(ctx, runID)
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	if view.Run.FailedCount == 0 {
+		return contract.FailurePage{Items: []contract.Failure{}}, nil
+	}
+	for _, repository := range c.repositories {
+		failedKey, _ := repository.store.FailedKey(runID)
+		size, sizeErr := repository.redis.Do(ctx, "HLEN", failedKey)
+		if sizeErr != nil {
+			return contract.FailurePage{}, sizeErr
+		}
+		if size.Int64() > 0 {
+			return repository.ListFailures(ctx, runID, page)
+		}
+	}
+	return contract.FailurePage{}, contract.ErrFailureDetailsUnavailable
+}
+
+func (c *Catalog) listArchives(ctx context.Context, byID map[string]contract.RunView) error {
+	root := c.repositories[0].root
+	if strings.TrimSpace(root) == "" {
+		return nil
+	}
+	matches, err := filepath.Glob(filepath.Join(root, "run-*.json"))
+	if err != nil {
+		return err
+	}
+	for _, path := range matches {
+		data, readErr := os.ReadFile(path)
+		if readErr != nil {
+			g.Log().Warningf(ctx, "event=full_sync_list_archive status=skip path=%s error=%q", path, readErr.Error())
+			continue
+		}
+		var archive contract.RunArchive
+		if decodeErr := json.Unmarshal(data, &archive); decodeErr != nil || archive.Run.RunID == "" || filepath.Base(path) != "run-"+archive.Run.RunID+".json" {
+			g.Log().Warningf(ctx, "event=full_sync_list_archive status=skip path=%s error=%v", path, decodeErr)
+			continue
+		}
+		if _, exists := byID[archive.Run.RunID]; !exists {
+			byID[archive.Run.RunID] = contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}
+		}
+	}
+	return nil
+}
+
+func (r *Repository) liveRun(ctx context.Context, runID string) (contract.RunView, error) {
+	meta, err := r.GetRunMeta(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	rounds, err := r.GetRoundStats(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	return contract.RunView{Source: contract.SourceLive, Run: meta, Rounds: rounds}, nil
+}
+
+func runSortTime(run contract.RunMeta) (value time.Time) {
+	if run.StartedAt != nil && !run.StartedAt.IsZero() {
+		return *run.StartedAt
+	}
+	if !run.CompletedAt.IsZero() {
+		return run.CompletedAt
+	}
+	return run.CreatedAt
+}

worker-svr/internal/fullsync/runtime/catalog_test.go

@@ -0,0 +1,123 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type catalogRedisStub struct {
+	responses []*gvar.Var
+	calls     []string
+}
+
+func (s *catalogRedisStub) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
+	s.calls = append(s.calls, command)
+	if len(s.responses) == 0 {
+		return gvar.New(nil), nil
+	}
+	response := s.responses[0]
+	s.responses = s.responses[1:]
+	return response, nil
+}
+
+func (s *catalogRedisStub) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	return gvar.New(int64(1)), nil
+}
+
+func TestCatalogListRunsMergesLiveAndArchive(t *testing.T) {
+	root := t.TempDir()
+	now := time.Date(2026, 9, 28, 1, 0, 0, 0, time.UTC)
+	archive := contract.RunArchive{Version: 1, Run: contract.RunMeta{
+		RunID: "archived", Operation: "sim_status_queue_full_sync", Status: contract.RunCompleted,
+		CreatedAt: now.Add(-time.Hour), CompletedAt: now.Add(-time.Minute),
+	}}
+	data, err := json.Marshal(archive)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "run-archived.json"), data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	metaResponse := gvar.New([]any{"runId", "live", "operation", "current_month_usage_full_sync", "status", "running", "currentRound", "1", "createdAt", now.Format(time.RFC3339Nano), "roundStats", "[]"})
+	redis := &catalogRedisStub{responses: []*gvar.Var{
+		gvar.New([]any{"live"}), metaResponse, metaResponse,
+		gvar.New([]any{"live"}), metaResponse, metaResponse,
+	}}
+	store, err := zset.New(redis, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	catalog, err := NewCatalog(repository)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := catalog.ListRuns(context.Background(), contract.RunListFilter{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Total != 2 || len(result.Items) != 2 {
+		t.Fatalf("result = %+v", result)
+	}
+	if result.Items[0].Run.RunID != "live" || result.Items[1].Run.RunID != "archived" {
+		t.Fatalf("items = %+v", result.Items)
+	}
+	filtered, err := catalog.ListRuns(context.Background(), contract.RunListFilter{SyncKind: contract.SyncKindTrafficUsage, Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if filtered.Total != 1 || filtered.Items[0].Run.RunID != "live" {
+		t.Fatalf("filtered = %+v", filtered)
+	}
+	view, err := catalog.GetRun(context.Background(), "archived")
+	if err != nil || view.Source != contract.SourceArchive || view.Run.RunID != "archived" {
+		t.Fatalf("archived view = %+v, err = %v", view, err)
+	}
+}
+
+func TestCatalogPrefersLiveAndSkipsBrokenArchive(t *testing.T) {
+	root := t.TempDir()
+	for name, payload := range map[string]string{
+		"run-live.json":   `{"version":1,"run":{"runId":"live","operation":"sim_status_queue_full_sync","status":"completed"}}`,
+		"run-broken.json": `{not-json}`,
+	} {
+		if err := os.WriteFile(filepath.Join(root, name), []byte(payload), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	redis := &catalogRedisStub{responses: []*gvar.Var{
+		gvar.New([]any{"live"}),
+		gvar.New([]any{"runId", "live", "operation", "sim_status_queue_full_sync", "status", "running"}),
+		gvar.New([]any{"runId", "live", "operation", "sim_status_queue_full_sync", "status", "running"}),
+	}}
+	store, err := zset.New(redis, "sim-status:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	catalog, err := NewCatalog(repository)
+	if err != nil {
+		t.Fatal(err)
+	}
+	result, err := catalog.ListRuns(context.Background(), contract.RunListFilter{Limit: 10})
+	if err != nil || result.Total != 1 || result.Items[0].Source != contract.SourceLive || result.Items[0].Run.Status != contract.RunRunning {
+		t.Fatalf("result = %+v, err = %v", result, err)
+	}
+}

worker-svr/internal/fullsync/runtime/runtime.go

@@ -0,0 +1,497 @@
+// Package runtime provides the small concrete storage adapters used by the
+// full-sync HTTP and archive services. It intentionally keeps queue protocol
+// transitions in queue/zset and only reads or records durable metadata here.
+package runtime
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type Repository struct {
+	store *zset.Store
+	redis zset.RedisClient
+	keys  contract.Keys
+	root  string
+}
+
+func (r *Repository) GetRoundStats(ctx context.Context, runID string) ([]contract.RoundStats, error) {
+	stats, err := r.store.GetRoundStats(ctx, runID)
+	if err != nil {
+		return nil, wrapNotFound(err)
+	}
+	out := make([]contract.RoundStats, 0, len(stats))
+	for _, stat := range stats {
+		out = append(out, contract.RoundStats{
+			RunID:                stat.RunID,
+			Round:                stat.Round,
+			EnqueuedCount:        stat.EnqueuedCount,
+			ActiveCount:          stat.ActiveCount,
+			NextRoundCount:       stat.NextRoundCount,
+			SuccessCount:         stat.SuccessCount,
+			FailedCount:          stat.FailedCount,
+			UpstreamAttemptCount: stat.UpstreamAttemptCount,
+			BatchCount:           stat.BatchCount,
+			StartedAt:            stat.StartedAt,
+			CompletedAt:          stat.CompletedAt,
+		})
+	}
+	return out, nil
+}
+
+func (r *Repository) RequestReplay(ctx context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
+	if err := req.Validate(); err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	fp, err := req.Fingerprint()
+	if err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	key, err := r.keys.ReplayIdempotencyKey(req.SourceRunID, req.IdempotencyKey)
+	if err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	actionID := "replay-" + fp[:24]
+	actionKey, _ := r.keys.ReplayActionKey(actionID)
+	targetID := "replay-run-" + fp[:24]
+	now := time.Now().UTC()
+	action := contract.ReplayAction{ActionID: actionID, SourceRunID: req.SourceRunID, TargetRunID: targetID, RequestedBy: req.RequestedBy, IdempotencyKey: req.IdempotencyKey, Fingerprint: fp, Status: contract.ActionAccepted, ReplayCycle: 1, Progress: contract.ReplayProgress{}, CreatedAt: now, UpdatedAt: now}
+	b, _ := json.Marshal(action)
+	// Reserve idempotency first. A repeat reads the original action; a changed
+	// request under the same key is rejected by fingerprint comparison.
+	set, e := r.redis.Do(ctx, "SET", key, actionID, "NX", "EX", 604800)
+	if e != nil {
+		return contract.ReplayAccepted{}, e
+	}
+	if set.String() == "" || strings.EqualFold(set.String(), "nil") {
+		reserved, e := r.redis.Do(ctx, "GET", key)
+		if e != nil {
+			return contract.ReplayAccepted{}, e
+		}
+		reservedID := reserved.String()
+		old, e := r.GetReplayAction(ctx, reservedID)
+		if e != nil {
+			return contract.ReplayAccepted{}, e
+		}
+		if old.Fingerprint != fp {
+			return contract.ReplayAccepted{}, contract.ErrConflict
+		}
+		u, _ := contract.ActionStatusURL(old.ActionID)
+		return contract.ReplayAccepted{ActionID: old.ActionID, TargetRunID: old.TargetRunID, Status: old.Status, StatusURL: u}, nil
+	}
+	if _, e = r.redis.Do(ctx, "SET", actionKey, string(b), "EX", 604800); e != nil {
+		return contract.ReplayAccepted{}, e
+	}
+	_, _ = r.redis.Do(ctx, "SADD", r.keys.PendingReplayActionsKey(), actionID)
+	go r.executeReplay(actionID)
+	u, _ := contract.ActionStatusURL(actionID)
+	return contract.ReplayAccepted{ActionID: actionID, TargetRunID: targetID, Status: action.Status, StatusURL: u}, nil
+}
+
+// executeReplay is deliberately small: it snapshots the source failure hash,
+// enqueues equivalent tasks into one target run, then removes only entries
+// that were accepted. Redis action JSON is the durable retry checkpoint.
+func (r *Repository) executeReplay(actionID string) {
+	ctx := context.Background()
+	action, err := r.GetReplayAction(ctx, actionID)
+	if err != nil {
+		return
+	}
+	action.Status = contract.ActionProcessing
+	action.UpdatedAt = time.Now().UTC()
+	_ = r.saveAction(ctx, action)
+	source, err := r.GetRunMeta(ctx, action.SourceRunID)
+	if err != nil {
+		action.Error = err.Error()
+		action.Status = contract.ActionNeedsAttention
+		action.UpdatedAt = time.Now().UTC()
+		_ = r.saveAction(ctx, action)
+		return
+	}
+	target := zset.RunMeta{RunID: action.TargetRunID, Operation: source.Operation, QueryMonth: source.QueryMonth, Status: "producing", CurrentRound: 1, CreatedAt: time.Now().UTC(), SyncType: string(contract.SyncReplay), SourceRunID: action.SourceRunID, ReplayCycle: action.ReplayCycle, TriggerSource: "replay"}
+	if err := r.store.CreateRun(ctx, target); err != nil {
+		if _, getErr := r.store.GetRunMeta(ctx, target.RunID); getErr != nil {
+			action.Error = err.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+	}
+	fkey, _ := r.store.FailedKey(action.SourceRunID)
+	var records []struct {
+		field string
+		rec   zset.FailureRecord
+	}
+	var cursor uint64
+	for {
+		v, e := r.redis.Do(ctx, "HSCAN", fkey, cursor, "COUNT", 500)
+		if e != nil {
+			action.Error = e.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+		cursor, pairs, e := parseScanResult(v)
+		if e != nil {
+			action.Error = e.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+		for i := 0; i < len(pairs); i += 2 {
+			var rec zset.FailureRecord
+			if json.Unmarshal([]byte(pairs[i+1].String()), &rec) == nil {
+				records = append(records, struct {
+					field string
+					rec   zset.FailureRecord
+				}{pairs[i].String(), rec})
+			}
+		}
+		if cursor == 0 {
+			break
+		}
+	}
+	groups := map[string][]zset.Task{}
+	for _, item := range records {
+		rec := item.rec
+		chanKey := rec.ChanKey
+		if chanKey == "" {
+			chanKey = "default"
+		}
+		groups[chanKey] = append(groups[chanKey], zset.Task{TaskID: action.TargetRunID + ":" + rec.TaskID, RunID: action.TargetRunID, Operation: rec.Operation, QueryMonth: rec.QueryMonth, ChanKey: chanKey, TempKey: rec.TempKey, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI})
+	}
+	for chanKey, tasks := range groups {
+		if _, err := r.store.EnqueueRunBatch(ctx, action.TargetRunID, chanKey, tasks, time.Now()); err != nil {
+			action.Error = err.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+	}
+	for _, item := range records {
+		_, _ = r.redis.Do(ctx, "HDEL", fkey, item.field)
+	}
+	_ = r.store.MarkProducerDone(ctx, action.TargetRunID)
+	action.Progress = contract.ReplayProgress{DiscoveredCount: int64(len(records)), EnqueuedCount: int64(len(records)), CleanedCount: int64(len(records)), FailureCount: int64(len(records))}
+	action.Status = contract.ActionCompleted
+	action.UpdatedAt = time.Now().UTC()
+	completed := action.UpdatedAt
+	action.CompletedAt = &completed
+	digest := sha256.Sum256([]byte(action.ActionID + action.TargetRunID))
+	action.Receipt = &contract.ArchiveReceipt{Version: 1, Path: "replay://" + action.ActionID, Checksum: hex.EncodeToString(digest[:]), ArchivedAt: completed}
+	_ = r.saveAction(ctx, action)
+}
+
+func (r *Repository) saveAction(ctx context.Context, action contract.ReplayAction) error {
+	b, err := json.Marshal(action)
+	if err != nil {
+		return err
+	}
+	key, err := r.keys.ReplayActionKey(action.ActionID)
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "SET", key, string(b), "EX", 604800)
+	return err
+}
+
+func (r *Repository) GetReplayAction(ctx context.Context, actionID string) (contract.ReplayAction, error) {
+	if err := contract.ValidateID(actionID); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	key, _ := r.keys.ReplayActionKey(actionID)
+	v, err := r.redis.Do(ctx, "GET", key)
+	if err != nil {
+		return contract.ReplayAction{}, err
+	}
+	if v == nil || v.IsNil() || strings.TrimSpace(v.String()) == "" {
+		return contract.ReplayAction{}, contract.ErrNotFound
+	}
+	var action contract.ReplayAction
+	if err := json.Unmarshal([]byte(v.String()), &action); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	return action, nil
+}
+
+func New(store *zset.Store, archiveRoot string) (*Repository, error) {
+	if store == nil {
+		return nil, errors.New("full-sync runtime store is nil")
+	}
+	keys, err := contract.NewKeys(store.Prefix())
+	if err != nil {
+		return nil, err
+	}
+	return &Repository{store: store, redis: store.RedisClient(), keys: keys, root: archiveRoot}, nil
+}
+
+func (r *Repository) GetRunMeta(ctx context.Context, runID string) (contract.RunMeta, error) {
+	m, err := r.store.GetRunMeta(ctx, runID)
+	if err != nil {
+		return contract.RunMeta{}, wrapNotFound(err)
+	}
+	return toContractMeta(m), nil
+}
+
+func (r *Repository) GetArchivedRun(ctx context.Context, runID string) (contract.RunArchive, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunArchive{}, err
+	}
+	if strings.TrimSpace(r.root) == "" {
+		return contract.RunArchive{}, contract.ErrNotFound
+	}
+	b, err := os.ReadFile(filepath.Join(r.root, "run-"+runID+".json"))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return contract.RunArchive{}, contract.ErrNotFound
+		}
+		return contract.RunArchive{}, err
+	}
+	var archive contract.RunArchive
+	if err := json.Unmarshal(b, &archive); err != nil {
+		return contract.RunArchive{}, fmt.Errorf("decode run archive: %w", err)
+	}
+	if archive.Run.RunID != runID {
+		return contract.RunArchive{}, fmt.Errorf("archive run ID does not match filename")
+	}
+	return archive, nil
+}
+
+func (r *Repository) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) {
+	p, err := page.Normalized()
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	key, err := r.store.FailedKey(runID)
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	start := uint64(0)
+	if p.Cursor != "" {
+		start, err = decodeCursor(p.Cursor, runID)
+		if err != nil {
+			return contract.FailurePage{}, err
+		}
+	}
+	var items []contract.Failure
+	var next uint64
+	for {
+		v, e := r.redis.Do(ctx, "HSCAN", key, start, "COUNT", p.Limit)
+		if e != nil {
+			return contract.FailurePage{}, e
+		}
+		next, pairs, e := parseScanResult(v)
+		if e != nil {
+			return contract.FailurePage{}, e
+		}
+		for i := 1; i < len(pairs); i += 2 {
+			var rec zset.FailureRecord
+			if json.Unmarshal([]byte(pairs[i].String()), &rec) == nil {
+				items = append(items, toFailure(rec))
+				if len(items) >= p.Limit {
+					break
+				}
+			}
+		}
+		if len(items) >= p.Limit || next == 0 {
+			break
+		}
+		start = next
+	}
+	result := contract.FailurePage{Items: items}
+	if next != 0 {
+		result.NextCursor = encodeCursor(runID, next)
+	}
+	if result.Items == nil {
+		result.Items = []contract.Failure{}
+	}
+	return result, nil
+}
+
+func (r *Repository) ListArchiveWork(ctx context.Context, page contract.PageRequest) (contract.ArchiveWorkPage, error) {
+	p, err := page.Normalized()
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	start := uint64(0)
+	if p.Cursor != "" {
+		start, err = decodeCursor(p.Cursor, "archive")
+		if err != nil {
+			return contract.ArchiveWorkPage{}, err
+		}
+	}
+	v, err := r.redis.Do(ctx, "ZSCAN", r.keys.PendingArchivesKey(), start, "COUNT", p.Limit)
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	out := contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}
+	next, pairs, err := parseScanResult(v)
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	for i := 0; i < len(pairs); i += 2 {
+		out.Items = append(out.Items, contract.ArchiveWork{RunID: pairs[i].String(), Version: pairs[i+1].String()})
+	}
+	if next != 0 {
+		out.NextCursor = encodeCursor("archive", next)
+	}
+	return out, nil
+}
+
+func parseScanResult(value *gvar.Var) (uint64, []*gvar.Var, error) {
+	if value == nil {
+		return 0, nil, errors.New("redis scan returned nil response")
+	}
+	values := value.Vars()
+	if len(values) != 2 {
+		return 0, nil, fmt.Errorf("unexpected Redis scan response length: %d", len(values))
+	}
+	pairs := values[1].Vars()
+	if len(pairs)%2 != 0 {
+		return 0, nil, fmt.Errorf("redis scan returned odd number of entry values: %d", len(pairs))
+	}
+	return uint64(values[0].Int64()), pairs, nil
+}
+
+func (r *Repository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
+	if archived, err := r.GetArchivedRun(ctx, work.RunID); err == nil {
+		return archived, nil
+	}
+	meta, err := r.GetRunMeta(ctx, work.RunID)
+	if err != nil {
+		return contract.RunArchive{}, err
+	}
+	rounds, err := r.GetRoundStats(ctx, work.RunID)
+	if err != nil {
+		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=read error=%q", work.RunID, err.Error())
+		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
+	}
+	if err := validateRoundAccounting(meta, rounds); err != nil {
+		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=validate error=%q", work.RunID, err.Error())
+		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
+	}
+	return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: rounds}, nil
+}
+
+func validateRoundAccounting(meta contract.RunMeta, rounds []contract.RoundStats) error {
+	if len(rounds) == 0 {
+		return nil
+	}
+	var success, failed int64
+	seen := make(map[int]struct{}, len(rounds))
+	for _, round := range rounds {
+		if round.RunID != meta.RunID {
+			return fmt.Errorf("round %d belongs to run %q", round.Round, round.RunID)
+		}
+		if _, ok := seen[round.Round]; ok {
+			return fmt.Errorf("duplicate round %d", round.Round)
+		}
+		seen[round.Round] = struct{}{}
+		if round.EnqueuedCount != round.SuccessCount+round.NextRoundCount+round.FailedCount {
+			return fmt.Errorf("round %d counters do not close", round.Round)
+		}
+		success += round.SuccessCount
+		failed += round.FailedCount
+	}
+	if success != meta.SuccessCount || failed != meta.FailedCount {
+		return fmt.Errorf("run counters do not match round totals: success=%d/%d failed=%d/%d", success, meta.SuccessCount, failed, meta.FailedCount)
+	}
+	return nil
+}
+
+func (r *Repository) RegisterArchiveWork(ctx context.Context, work contract.ArchiveWork) error {
+	if err := contract.ValidateID(work.RunID); err != nil {
+		return err
+	}
+	if strings.TrimSpace(work.Version) == "" {
+		return contract.ErrInvalidArgument
+	}
+	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), work.Version, work.RunID)
+	return err
+}
+
+func (r *Repository) RegisterArchive(ctx context.Context, work contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
+	if err := receipt.Validate(); err != nil {
+		return err
+	}
+	key, err := r.keys.ArchiveStateKey(work.RunID)
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "HSET", key, "version", receipt.Version, "path", receipt.Path, "checksum", receipt.Checksum, "archivedAt", receipt.ArchivedAt.UTC().Format(time.RFC3339Nano))
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "ZREM", r.keys.PendingArchivesKey(), work.RunID)
+	return err
+}
+
+func (r *Repository) RegisterArchiveFailure(ctx context.Context, work contract.ArchiveWork, message string) error {
+	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), time.Now().UnixMilli(), work.RunID)
+	return err
+}
+
+func toContractMeta(m zset.RunMeta) contract.RunMeta {
+	syncType := contract.SyncType(m.SyncType)
+	if syncType == "" {
+		syncType = contract.SyncNormal
+	}
+	return contract.RunMeta{SupplyObservation: m.SupplyObservation, RunID: m.RunID, Operation: m.Operation, QueryMonth: m.QueryMonth, Status: contract.RunStatus(m.Status), ProducerDone: m.ProducerDone, CurrentRound: m.CurrentRound, ActiveCount: m.ActiveCount, InflightCount: m.InflightCount, NextRoundCount: m.NextRoundCount, SuccessCount: m.SuccessCount, FailedCount: m.FailedCount, CreatedAt: m.CreatedAt, ProducerCompletedAt: m.ProducerCompletedAt, CompletedAt: m.CompletedAt, SyncType: syncType, SourceRunID: m.SourceRunID, ReplayCycle: m.ReplayCycle, TriggerSource: m.TriggerSource, ScannedCount: m.ScannedCount, EnqueuedCount: m.EnqueuedCount, UpstreamAttemptCount: m.UpstreamAttemptCount, BatchCount: m.BatchCount, MaxRound: m.MaxRound, PendingFailureCount: m.PendingFailureCount, ArchiveStatus: contract.ArchiveStatus(m.ArchiveStatus), ArchiveVersion: m.ArchiveVersion, ArchivePath: m.ArchivePath, ArchiveChecksum: m.ArchiveChecksum, ReplayDisposition: contract.ReplayDisposition(m.ReplayDisposition), ScopeChannels: m.ScopeChannels, ScopeDigest: m.ScopeDigest, ScopeChannelsSample: m.ScopeChannelsSample, StartedAt: m.StartedAt, ScanCompletedAt: m.ScanCompletedAt, ArchivedAt: m.ArchivedAt}
+}
+func toFailure(f zset.FailureRecord) contract.Failure {
+	return contract.Failure{TaskID: f.TaskID, CardID: f.CardID, ErrorCategory: contract.NormalizeErrorCategory(f.FailureType), ErrorMessage: contract.BoundedErrorMessage(f.Message), FailedAt: f.LastFailedAt, Round: f.Round}
+}
+func wrapNotFound(err error) error {
+	if strings.Contains(strings.ToLower(err.Error()), "not found") {
+		return contract.ErrNotFound
+	}
+	return err
+}
+func encodeCursor(scope string, n uint64) contract.Cursor {
+	return contract.Cursor(base64.RawURLEncoding.EncodeToString([]byte(scope + ":" + strconv.FormatUint(n, 10))))
+}
+func decodeCursor(c contract.Cursor, scope string) (uint64, error) {
+	b, e := base64.RawURLEncoding.DecodeString(string(c))
+	if e != nil {
+		return 0, contract.ErrInvalidArgument
+	}
+	p := strings.Split(string(b), ":")
+	if len(p) != 2 || p[0] != scope {
+		return 0, contract.ErrInvalidArgument
+	}
+	n, e := strconv.ParseUint(p[1], 10, 64)
+	if e != nil {
+		return 0, contract.ErrInvalidArgument
+	}
+	return n, nil
+}
+
+var _ contract.LiveRunReader = (*Repository)(nil)
+var _ contract.ArchivedRunReader = (*Repository)(nil)
+var _ contract.FailurePager = (*Repository)(nil)
+var _ contract.ArchiveWorkDiscovery = (*Repository)(nil)
+var _ contract.ArchiveWorkLoader = (*Repository)(nil)
+var _ contract.ArchiveWorkRegistrar = (*Repository)(nil)
+var _ contract.ArchiveRegistrar = (*Repository)(nil)
+var _ contract.ReplayRequester = (*Repository)(nil)
+var _ contract.ActionReader = (*Repository)(nil)

worker-svr/internal/fullsync/runtime/runtime_test.go

@@ -0,0 +1,143 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type runtimeRedisStub struct {
+	responses []*gvar.Var
+}
+
+func (s *runtimeRedisStub) Do(context.Context, string, ...any) (*gvar.Var, error) {
+	if len(s.responses) == 0 {
+		return gvar.New(nil), nil
+	}
+	response := s.responses[0]
+	s.responses = s.responses[1:]
+	return response, nil
+}
+
+func (s *runtimeRedisStub) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	return gvar.New(int64(1)), nil
+}
+
+func newRuntimeRepository(t *testing.T, responses ...*gvar.Var) *Repository {
+	t.Helper()
+	store, err := zset.New(&runtimeRedisStub{responses: responses}, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return repository
+}
+
+func TestListArchiveWorkParsesNestedRedisScanResult(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"run-1", "17", "run-2", "18"},
+	}))
+
+	page, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 2 {
+		t.Fatalf("archive work count = %d, want 2", len(page.Items))
+	}
+	if page.Items[0].RunID != "run-1" || page.Items[0].Version != "17" {
+		t.Fatalf("first archive work = %+v", page.Items[0])
+	}
+	if page.Items[1].RunID != "run-2" || page.Items[1].Version != "18" {
+		t.Fatalf("second archive work = %+v", page.Items[1])
+	}
+}
+
+func TestGetRoundStatsReadsRunHashJSON(t *testing.T) {
+	raw, err := json.Marshal([]zset.RoundStats{{RunID: "run-1", Round: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"runId", "run-1", "status", "completed", "roundStats", string(raw),
+	}))
+
+	got, err := repository.GetRoundStats(context.Background(), "run-1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Round != 2 || got[0].SuccessCount != 2 || got[0].FailedCount != 1 {
+		t.Fatalf("unexpected round stats: %+v", got)
+	}
+}
+
+func TestValidateRoundAccountingRejectsUnbalancedRound(t *testing.T) {
+	err := validateRoundAccounting(contract.RunMeta{RunID: "run-1", SuccessCount: 1, FailedCount: 0}, []contract.RoundStats{{RunID: "run-1", Round: 1, EnqueuedCount: 2, SuccessCount: 1}})
+	if err == nil {
+		t.Fatal("expected unbalanced round error")
+	}
+}
+
+func TestListArchiveWorkHandlesEmptyRedisScanResult(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{"0", []any{}}))
+
+	page, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 0 {
+		t.Fatalf("archive work = %+v, want empty", page.Items)
+	}
+}
+
+func TestListFailuresParsesNestedRedisScanResult(t *testing.T) {
+	failedAt := time.Date(2026, 9, 21, 6, 0, 0, 0, time.UTC)
+	record := zset.FailureRecord{
+		TaskID:       "task-1",
+		CardID:       42,
+		FailureType:  "business",
+		Message:      "upstream rejected request",
+		Round:        3,
+		LastFailedAt: failedAt,
+	}
+	payload, err := json.Marshal(record)
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"task-1", string(payload)},
+	}))
+
+	page, err := repository.ListFailures(context.Background(), "run-1", contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 1 {
+		t.Fatalf("failure count = %d, want 1", len(page.Items))
+	}
+	if page.Items[0].TaskID != "task-1" || page.Items[0].CardID != 42 || page.Items[0].Round != 3 {
+		t.Fatalf("failure = %+v", page.Items[0])
+	}
+}
+
+func TestListArchiveWorkRejectsOddRedisScanEntries(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"run-without-score"},
+	}))
+
+	if _, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10}); err == nil {
+		t.Fatal("ListArchiveWork() error = nil, want malformed response error")
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay.go

@@ -0,0 +1,474 @@
+package runtime
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+// SimpleReplay hands every source failure to a separate Run. The source hash
+// is the checkpoint: a record is removed only after the target's task-state
+// confirms that EnqueueRunBatch accepted it. No cursor is retained across
+// mutations of the source hash.
+//
+// The target's retry-round budget is taken from the caller when supplied and
+// falls back to the configured default otherwise. It is persisted on the
+// source reservation because a worker restart re-enters through reconcile(),
+// which no longer has the caller's arguments.
+type SimpleReplay struct {
+	stores []*zset.Store
+	redis  zset.RedisClient
+	// defaultMaxRounds applies when a replay request does not specify one.
+	defaultMaxRounds int
+}
+
+// Replay round bounds. MaxReplayRounds is a server-side guard, not a capacity
+// limit: the observed chain of 10-09 needed 74 rounds to reach 98% recovery,
+// so a low ceiling would cut off a legitimate convergence path.
+//
+// DefaultReplayMaxRounds is only the fallback for a request that omits a
+// budget; callers may ask for more, up to MaxReplayRounds.
+const (
+	DefaultReplayMaxRounds = 3
+	MaxReplayRounds        = 100
+	legacyReplayMaxRounds  = 1
+)
+
+type ReplayReceipt struct {
+	ReplayRunID string `json:"replayRunId"`
+	Status      string `json:"status"`
+}
+
+type ReplayStatus struct {
+	ReplayRunID string `json:"replayRunId"`
+	SourceRunID string `json:"sourceRunId"`
+	Status      string `json:"status"`
+	Remaining   int64  `json:"remaining"`
+	Enqueued    int64  `json:"enqueued"`
+	RunStatus   string `json:"runStatus,omitempty"`
+	Error       string `json:"error,omitempty"`
+}
+
+var ErrNoFailures = errors.New("run has no failures to replay")
+
+func NewSimpleReplay(stores ...*zset.Store) (*SimpleReplay, error) {
+	return NewSimpleReplayWithDefault(DefaultReplayMaxRounds, stores...)
+}
+
+// NewSimpleReplayWithDefault builds a replay service whose unspecified replay
+// requests inherit defaultMaxRounds. A non-positive value falls back to
+// DefaultReplayMaxRounds so a missing config key cannot disable the budget.
+func NewSimpleReplayWithDefault(defaultMaxRounds int, stores ...*zset.Store) (*SimpleReplay, error) {
+	if len(stores) == 0 || stores[0] == nil {
+		return nil, fmt.Errorf("replay requires Redis stores")
+	}
+	for _, store := range stores {
+		if store == nil {
+			return nil, fmt.Errorf("replay store is nil")
+		}
+	}
+	if defaultMaxRounds <= 0 {
+		defaultMaxRounds = DefaultReplayMaxRounds
+	}
+	if defaultMaxRounds > MaxReplayRounds {
+		defaultMaxRounds = MaxReplayRounds
+	}
+	return &SimpleReplay{stores: stores, redis: stores[0].RedisClient(), defaultMaxRounds: defaultMaxRounds}, nil
+}
+
+func (r *SimpleReplay) find(ctx context.Context, id string) (*zset.Store, zset.RunMeta, error) {
+	if err := contract.ValidateID(id); err != nil || len(id) > 128 {
+		return nil, zset.RunMeta{}, contract.ErrInvalidArgument
+	}
+	var found *zset.Store
+	var meta zset.RunMeta
+	for _, store := range r.stores {
+		key, _ := store.RunMetaKey(id)
+		v, err := r.redis.Do(ctx, "EXISTS", key)
+		if err != nil {
+			return nil, meta, err
+		}
+		if v.Int() == 0 {
+			continue
+		}
+		if found != nil {
+			return nil, meta, contract.ErrConflict
+		}
+		found = store
+		meta, err = store.GetRunMeta(ctx, id)
+		if err != nil {
+			return nil, meta, err
+		}
+	}
+	if found == nil {
+		return nil, meta, contract.ErrNotFound
+	}
+	return found, meta, nil
+}
+
+// Request accepts a replay of the source run's failures. maxRounds == 0 means
+// "use the configured default"; requests above MaxReplayRounds are rejected.
+func (r *SimpleReplay) Request(ctx context.Context, sourceID, requestedBy string, maxRounds int) (ReplayReceipt, error) {
+	if len(requestedBy) == 0 || len(requestedBy) > 64 {
+		return ReplayReceipt{}, contract.ErrInvalidArgument
+	}
+	if maxRounds < 0 || maxRounds > MaxReplayRounds {
+		return ReplayReceipt{}, contract.ErrInvalidArgument
+	}
+	if maxRounds == 0 {
+		maxRounds = r.defaultMaxRounds
+	}
+	store, _, err := r.find(ctx, sourceID)
+	if err != nil {
+		return ReplayReceipt{}, err
+	}
+	metaKey, _ := store.RunMetaKey(sourceID)
+	failedKey, _ := store.FailedKey(sourceID)
+	var random [16]byte
+	if _, err := rand.Read(random[:]); err != nil {
+		return ReplayReceipt{}, err
+	}
+	targetID := "replay-" + hex.EncodeToString(random[:])
+	// An index written before the reservation makes the pending source
+	// discoverable even if the process stops immediately after the Lua call.
+	if _, err = r.redis.Do(ctx, "SADD", store.Prefix()+"replay:pending", sourceID); err != nil {
+		return ReplayReceipt{}, err
+	}
+	v, err := r.redis.Eval(ctx, reserveReplay, 2, []string{metaKey, failedKey}, []any{targetID, requestedBy, maxRounds})
+	if err != nil {
+		return ReplayReceipt{}, err
+	}
+	result := v.String()
+	// An unsuccessful reservation leaves an inert discovery entry. Never
+	// remove it here: a concurrent caller might have reserved the source
+	// after this script returned, and that reservation needs recovery.
+	switch result {
+	case "missing":
+		return ReplayReceipt{}, contract.ErrNotFound
+	case "running":
+		return ReplayReceipt{}, contract.ErrConflict
+	case "empty":
+		return ReplayReceipt{}, ErrNoFailures
+	}
+	if !strings.HasPrefix(result, "replay-") {
+		return ReplayReceipt{}, fmt.Errorf("unexpected replay reservation result")
+	}
+	go r.process(store, sourceID)
+	return ReplayReceipt{ReplayRunID: result, Status: "accepted"}, nil
+}
+
+// Start reconciles pending sources after restarts and missed goroutine wakes.
+// Every process may run it: the per-source Redis lease elects one writer.
+func (r *SimpleReplay) Start(ctx context.Context) {
+	go func() {
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		for {
+			r.reconcile(ctx)
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+}
+
+func (r *SimpleReplay) reconcile(ctx context.Context) {
+	for _, store := range r.stores {
+		v, err := r.redis.Do(ctx, "SMEMBERS", store.Prefix()+"replay:pending")
+		if err != nil {
+			continue
+		}
+		for _, id := range v.Strings() {
+			if ctx.Err() != nil {
+				return
+			}
+			key, keyErr := store.RunMetaKey(id)
+			if keyErr != nil {
+				continue
+			}
+			reservation, getErr := r.redis.Do(ctx, "HGET", key, "replayTargetId")
+			// SADD happens before the reservation so even a crash during
+			// acceptance cannot lose work. Unreserved entries are inert.
+			if getErr != nil || reservation == nil || reservation.String() == "" {
+				continue
+			}
+			state, stateErr := r.redis.Do(ctx, "HGET", key, "replayState")
+			if stateErr != nil || state == nil || state.String() == "needs_attention" {
+				continue
+			}
+			go r.process(store, id)
+		}
+	}
+}
+
+func (r *SimpleReplay) Status(ctx context.Context, targetID string) (ReplayStatus, error) {
+	store, target, err := r.find(ctx, targetID)
+	if err != nil {
+		// The target may not exist yet: locate its durable source reservation.
+		if !errors.Is(err, contract.ErrNotFound) {
+			return ReplayStatus{}, err
+		}
+		for _, candidate := range r.stores {
+			v, e := r.redis.Do(ctx, "SMEMBERS", candidate.Prefix()+"replay:pending")
+			if e != nil {
+				return ReplayStatus{}, e
+			}
+			for _, sourceID := range v.Strings() {
+				key, _ := candidate.RunMetaKey(sourceID)
+				value, e := r.redis.Do(ctx, "HGET", key, "replayTargetId")
+				if e == nil && value != nil && value.String() == targetID {
+					store = candidate
+					target.SourceRunID = sourceID
+					break
+				}
+			}
+		}
+		if store == nil {
+			return ReplayStatus{}, contract.ErrNotFound
+		}
+	}
+	if target.SourceRunID == "" {
+		return ReplayStatus{}, contract.ErrNotFound
+	}
+	sourceKey, _ := store.RunMetaKey(target.SourceRunID)
+	failureKey, _ := store.FailedKey(target.SourceRunID)
+	values, err := r.redis.Do(ctx, "HMGET", sourceKey, "replayTargetId", "replayState", "replayError", "replayEnqueued")
+	if err != nil {
+		return ReplayStatus{}, err
+	}
+	fields := values.Vars()
+	if len(fields) != 4 || fields[0].String() != targetID {
+		return ReplayStatus{}, contract.ErrNotFound
+	}
+	remaining, err := r.redis.Do(ctx, "HLEN", failureKey)
+	if err != nil {
+		return ReplayStatus{}, err
+	}
+	return ReplayStatus{ReplayRunID: targetID, SourceRunID: target.SourceRunID, Status: fields[1].String(), Error: fields[2].String(), Enqueued: fields[3].Int64(), Remaining: remaining.Int64(), RunStatus: target.Status}, nil
+}
+
+func (r *SimpleReplay) process(store *zset.Store, sourceID string) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
+	defer cancel()
+	// Keep one process per source; batches are bounded so a short lease is
+	// renewed before it expires. A lost lease stops cleanup at the next batch.
+	lockKey := store.Prefix() + "{" + sourceID + "}:replay-lock"
+	var token [16]byte
+	if _, err := rand.Read(token[:]); err != nil {
+		return
+	}
+	owner := hex.EncodeToString(token[:])
+	locked, err := store.AcquireLock(ctx, lockKey, owner, time.Minute)
+	if err != nil || !locked {
+		return
+	}
+	defer func() { _, _ = store.ReleaseLock(context.Background(), lockKey, owner) }()
+	if err := r.handoff(ctx, store, sourceID, lockKey, owner); err != nil {
+		// The time slice ended, not the replay: the reconciler will resume
+		// from remaining source records under a new lease.
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+			return
+		}
+		sourceKey, _ := store.RunMetaKey(sourceID)
+		// A stale owner must not overwrite a newer worker's status.
+		_, _ = r.redis.Eval(context.Background(), replayAttention, 2, []string{sourceKey, lockKey}, []any{owner})
+	}
+}
+
+func (r *SimpleReplay) handoff(ctx context.Context, store *zset.Store, sourceID, lockKey, owner string) error {
+	sourceKey, _ := store.RunMetaKey(sourceID)
+	failedKey, _ := store.FailedKey(sourceID)
+	meta, err := store.GetRunMeta(ctx, sourceID)
+	if err != nil {
+		return err
+	}
+	v, err := r.redis.Do(ctx, "HGET", sourceKey, "replayTargetId")
+	if err != nil {
+		return err
+	}
+	if v == nil || v.String() == "" {
+		return fmt.Errorf("replay reservation missing")
+	}
+	targetID := v.String()
+	// New reservations persist their budget. Reservations made before this
+	// field existed used one round; a restart must not apply today's default
+	// to those already accepted runs.
+	storedMaxRounds, err := r.redis.Do(ctx, "HGET", sourceKey, "replayMaxRound")
+	if err != nil {
+		return err
+	}
+	maxRounds := legacyReplayMaxRounds
+	if storedMaxRounds != nil && storedMaxRounds.String() != "" {
+		maxRounds = storedMaxRounds.Int()
+	}
+	if maxRounds <= 0 {
+		return fmt.Errorf("replay reservation has invalid round budget")
+	}
+	if maxRounds > MaxReplayRounds {
+		return fmt.Errorf("replay reservation exceeds round budget limit")
+	}
+	// CreateRun is idempotent for the main hash but writes extension fields
+	// again. Read and validate an existing target BEFORE CreateRun, otherwise
+	// recovery could silently replace its accepted maxRound.
+	existing, err := store.GetRunMeta(ctx, targetID)
+	if err != nil && !errors.Is(err, zset.ErrRunNotFound) {
+		return err
+	}
+	if err == nil {
+		if storedMaxRounds == nil || storedMaxRounds.String() == "" {
+			maxRounds = existing.MaxRound
+		}
+		if maxRounds <= 0 || maxRounds > MaxReplayRounds || existing.SourceRunID != sourceID || existing.SyncType != "replay" || existing.MaxRound != maxRounds || existing.Operation != meta.Operation || existing.QueryMonth != meta.QueryMonth {
+			return fmt.Errorf("replay target metadata is inconsistent")
+		}
+	}
+	target := zset.RunMeta{RunID: targetID, Operation: meta.Operation, QueryMonth: meta.QueryMonth, SyncType: "replay", SourceRunID: sourceID, ReplayCycle: meta.ReplayCycle + 1, TriggerSource: "replay", MaxRound: maxRounds}
+	if err := store.CreateRun(ctx, target); err != nil {
+		return err
+	}
+	actual, err := store.GetRunMeta(ctx, targetID)
+	if err != nil || actual.SourceRunID != sourceID || actual.SyncType != "replay" || actual.MaxRound != maxRounds || actual.Operation != meta.Operation || actual.QueryMonth != meta.QueryMonth {
+		return fmt.Errorf("replay target metadata is inconsistent")
+	}
+	// The target task-state is the admission receipt during recovery. It may
+	// not expire while source failures are still being transferred.
+	targetKey, _ := store.RunMetaKey(targetID)
+	stateKey, _ := store.TaskStateKey(targetID)
+	if !actual.ProducerDone {
+		if _, err := r.redis.Do(ctx, "PERSIST", targetKey); err != nil {
+			return err
+		}
+		if _, err := r.redis.Do(ctx, "PERSIST", stateKey); err != nil {
+			return err
+		}
+	}
+	_, err = r.redis.Do(ctx, "HSET", sourceKey, "replayState", "processing", "replayError", "")
+	if err != nil {
+		return err
+	}
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		valid, err := store.RenewLock(ctx, lockKey, owner, time.Minute)
+		if err != nil || !valid {
+			return fmt.Errorf("replay lease lost: %w", err)
+		}
+		// HSCAN is not a snapshot. Complete a bounded cursor pass, then
+		// recheck HLEN and repeat from zero if deletion changed the scan.
+		var cursor uint64
+		for {
+			v, e := r.redis.Do(ctx, "HSCAN", failedKey, cursor, "COUNT", 100)
+			if e != nil {
+				return e
+			}
+			var scanPairs []*gvar.Var
+			cursor, scanPairs, e = parseScanResult(v)
+			if e != nil {
+				return e
+			}
+			if len(scanPairs) > 0 {
+				for i := 0; i < len(scanPairs); i += 2 {
+					valid, renewErr := store.RenewLock(ctx, lockKey, owner, time.Minute)
+					if renewErr != nil || !valid {
+						return fmt.Errorf("replay lease lost")
+					}
+					field, raw := scanPairs[i].String(), scanPairs[i+1].String()
+					var rec zset.FailureRecord
+					if json.Unmarshal([]byte(raw), &rec) != nil || rec.TaskID == "" || rec.CardID <= 0 || rec.ICCID == "" || rec.ChanKey == "" || rec.TempKey == "" || field != rec.TaskID || (rec.LatestRunID != "" && rec.LatestRunID != sourceID) {
+						return fmt.Errorf("source failure record cannot be replayed")
+					}
+					operation := rec.Operation
+					if operation == "" {
+						operation = meta.Operation
+					}
+					// Source task identity (not just card ID) prevents collisions
+					// between distinct failures for the same card.
+					taskID := targetID + ":" + rec.TaskID
+					queryMonth := rec.QueryMonth
+					if queryMonth == "" {
+						queryMonth = meta.QueryMonth
+					}
+					task := zset.Task{RunID: targetID, TaskID: taskID, Operation: operation, QueryMonth: queryMonth, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI, ChanKey: rec.ChanKey, TempKey: rec.TempKey, PackageCode: rec.PackageCode}
+					if _, e = store.EnqueueRunBatch(ctx, targetID, rec.ChanKey, []zset.Task{task}, time.Now()); e != nil {
+						return e
+					}
+					confirmed, e := r.redis.Do(ctx, "HEXISTS", stateKey, taskID)
+					if e != nil || confirmed == nil || confirmed.Int() != 1 {
+						return fmt.Errorf("target enqueue unconfirmed: %w", e)
+					}
+					removed, e := r.redis.Eval(ctx, cleanupReplayFailure, 3, []string{sourceKey, failedKey, lockKey}, []any{field, raw, targetID, owner})
+					if e != nil {
+						return e
+					}
+					if removed.Int() == -1 {
+						return fmt.Errorf("source failure changed during replay")
+					}
+				}
+			}
+			if cursor == 0 {
+				break
+			}
+		}
+		remaining, err := r.redis.Do(ctx, "HLEN", failedKey)
+		if err != nil {
+			return err
+		}
+		if remaining.Int64() == 0 {
+			break
+		}
+	}
+	// Persist producerDone only after every source record was confirmed.
+	if err := store.MarkProducerDone(ctx, targetID); err != nil {
+		return err
+	}
+	if _, err := r.redis.Do(ctx, "HSET", sourceKey, "replayState", "completed", "replayError", ""); err != nil {
+		return err
+	}
+	// The source summary has already been archived. Keep the small source
+	// metadata for lookup but restore a bounded retention after handoff.
+	if _, err := r.redis.Do(ctx, "EXPIRE", sourceKey, 30*24*3600); err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "SREM", store.Prefix()+"replay:pending", sourceID)
+	return err
+}
+
+const replayAttention = `
+if redis.call('GET',KEYS[2])~=ARGV[1] then return 0 end
+redis.call('HSET',KEYS[1],'replayState','needs_attention','replayError','Replay handoff stopped; check Worker logs')
+return 1
+`
+
+const reserveReplay = `
+local id=redis.call('HGET',KEYS[1],'replayTargetId')
+if id then return id end
+local status=redis.call('HGET',KEYS[1],'status')
+if not status then return 'missing' end
+if status~='completed' and status~='completed_with_failures' then return 'running' end
+if redis.call('HLEN',KEYS[2])==0 then return 'empty' end
+redis.call('HSET',KEYS[1],'replayTargetId',ARGV[1],'replayState','accepted','replayRequestedBy',ARGV[2],'replayEnqueued',0,'replayMaxRound',ARGV[3])
+redis.call('PERSIST',KEYS[1]); redis.call('PERSIST',KEYS[2])
+return ARGV[1]
+`
+const cleanupReplayFailure = `
+if redis.call('HGET',KEYS[1],'replayTargetId')~=ARGV[3] or redis.call('GET',KEYS[3])~=ARGV[4] then return -1 end
+local value=redis.call('HGET',KEYS[2],ARGV[1])
+if not value then return 0 end
+if value~=ARGV[2] then return -1 end
+redis.call('HDEL',KEYS[2],ARGV[1]); redis.call('HINCRBY',KEYS[1],'replayEnqueued',1)
+redis.call('HSET',KEYS[1],'pendingFailureCount',redis.call('HLEN',KEYS[2]))
+return 1
+`

worker-svr/internal/fullsync/runtime/simple_replay_invalid_test.go

@@ -0,0 +1,59 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayRetainsInvalidFailureAndResumesSameTarget(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "sim-status:")
+	key, _ := store.RunMetaKey("source")
+	failed, _ := store.FailedKey("source")
+	_, _ = redis.Do(ctx, "HSET", key, "runId", "source", "status", "completed_with_failures", "producerDone", "1", "currentRound", "1", "failedCount", "2")
+	valid := zset.FailureRecord{TaskID: "a", CardID: 1, ICCID: "card", ChanKey: "ch", TempKey: "template"}
+	payload, _ := json.Marshal(valid)
+	_, _ = redis.Do(ctx, "HSET", failed, "a", string(payload), "bad", `{"taskId":"bad"}`)
+	svc, _ := NewSimpleReplay(store)
+	accepted, err := svc.Request(ctx, "source", "42", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		state, err := svc.Status(ctx, accepted.ReplayRunID)
+		if err == nil && state.Status == "needs_attention" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	state, _ := svc.Status(ctx, accepted.ReplayRunID)
+	if state.Status != "needs_attention" || state.Remaining < 1 {
+		t.Fatalf("unexpected status: %+v", state)
+	}
+	// No unconfirmed record may be deleted. Retry the SAME target after repair.
+	repaired := zset.FailureRecord{TaskID: "bad", CardID: 2, ICCID: "card2", ChanKey: "ch", TempKey: "template"}
+	payload, _ = json.Marshal(repaired)
+	_, _ = redis.Do(ctx, "HSET", failed, "bad", string(payload))
+	repeat, err := svc.Request(ctx, "source", "42", 0)
+	if err != nil || repeat.ReplayRunID != accepted.ReplayRunID {
+		t.Fatalf("repeat: %+v %v", repeat, err)
+	}
+	deadline = time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		state, _ = svc.Status(ctx, accepted.ReplayRunID)
+		if state.Status == "completed" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if state.Status != "completed" || state.Enqueued != 2 || state.Remaining != 0 {
+		t.Fatalf("resume: %+v", state)
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_max_rounds_test.go

@@ -0,0 +1,121 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+// seedReplaySource writes a completed source run with n failures into store.
+func seedReplaySource(t *testing.T, redis zset.RedisClient, store *zset.Store, name string, n int) {
+	t.Helper()
+	ctx := context.Background()
+	key, _ := store.RunMetaKey(name)
+	failed, _ := store.FailedKey(name)
+	if _, err := redis.Do(ctx, "HSET", key, "runId", name, "operation", "usage", "status", "completed_with_failures", "producerDone", "1", "currentRound", "1", "failedCount", fmt.Sprint(n)); err != nil {
+		t.Fatal(err)
+	}
+	for i := 1; i <= n; i++ {
+		rec := zset.FailureRecord{TaskID: fmt.Sprintf("task-%d", i), CardID: int64(i), ICCID: fmt.Sprintf("iccid-%d", i), ChanKey: "ch", TempKey: "template", Operation: "usage", FailureType: "mobile_busy"}
+		payload, _ := json.Marshal(rec)
+		if _, err := redis.Do(ctx, "HSET", failed, rec.TaskID, string(payload)); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// awaitTargetMaxRound waits for the handoff to create the target and returns
+// the persisted MaxRound of the replayed Run.
+func awaitTargetMaxRound(t *testing.T, svc *SimpleReplay, store *zset.Store, replayRunID string) int {
+	t.Helper()
+	ctx := context.Background()
+	deadline := time.Now().Add(4 * time.Second)
+	for time.Now().Before(deadline) {
+		meta, err := store.GetRunMeta(ctx, replayRunID)
+		if err == nil && meta.MaxRound > 0 && meta.ProducerDone {
+			return meta.MaxRound
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	meta, err := store.GetRunMeta(ctx, replayRunID)
+	t.Fatalf("target never settled: %+v %v", meta, err)
+	return 0
+}
+
+func TestReplayMaxRoundsPrefersRequestThenConfig(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+
+	// Use a default that differs from DefaultReplayMaxRounds so the assertion
+	// proves the injected config value is used, not the built-in constant.
+	svc, err := NewSimpleReplayWithDefault(5, store)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	seedReplaySource(t, redis, store, "flow:default-source", 2)
+	receipt, err := svc.Request(ctx, "flow:default-source", "42", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := awaitTargetMaxRound(t, svc, store, receipt.ReplayRunID); got != 5 {
+		t.Fatalf("unspecified request must inherit config default 5, got %d", got)
+	}
+
+	seedReplaySource(t, redis, store, "flow:override-source", 2)
+	override, err := svc.Request(ctx, "flow:override-source", "42", 7)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := awaitTargetMaxRound(t, svc, store, override.ReplayRunID); got != 7 {
+		t.Fatalf("explicit request must win over config, got %d", got)
+	}
+}
+
+func TestReplayMaxRoundsRejectsOutOfRange(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	svc, _ := NewSimpleReplay(store)
+	seedReplaySource(t, redis, store, "flow:source", 1)
+
+	for _, requested := range []int{-1, MaxReplayRounds + 1} {
+		if _, err := svc.Request(ctx, "flow:source", "42", requested); err != contract.ErrInvalidArgument {
+			t.Fatalf("maxRounds=%d must be rejected, got %v", requested, err)
+		}
+	}
+	// The rejected requests must not have reserved the source.
+	if got, err := redis.Do(ctx, "HGET", "flow:{flow:source}:meta", "replayTargetId"); err == nil && got != nil && got.String() != "" {
+		t.Fatalf("rejected request must not reserve the source: %v", got.String())
+	}
+}
+
+// A worker restart re-enters through reconcile() without caller arguments, so
+// the budget must survive on the reservation rather than come from the caller.
+func TestReplayMaxRoundsSurvivesRestartRecovery(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	seedReplaySource(t, redis, store, "flow:source", 2)
+
+	// Reserve with an explicit budget, then simulate a restart by building a
+	// fresh service whose configured default differs from the reservation.
+	first, _ := NewSimpleReplayWithDefault(1, store)
+	receipt, err := first.Request(ctx, "flow:source", "42", 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restarted, _ := NewSimpleReplayWithDefault(9, store)
+	restarted.reconcile(ctx)
+
+	if got := awaitTargetMaxRound(t, restarted, store, receipt.ReplayRunID); got != 5 {
+		t.Fatalf("recovery must use the persisted budget 5, not the new default 9: got %d", got)
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_package_test.go

@@ -0,0 +1,82 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+// Exercise the real task -> failure pool -> replayed task path for both
+// TDengine usage chains; a producer-only assertion misses lost package codes.
+func TestReplayDailyAndMonthlyFailurePoolPreservesPackageCode(t *testing.T) {
+	for _, tc := range []struct {
+		name, operation, period string
+	}{
+		{"daily", "previous_day_usage_full_sync", "20261009"},
+		{"monthly", "previous_month_usage_full_sync", "202609"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			redis := testutil.Redis(t)
+			ctx := context.Background()
+			store, err := zset.New(redis, tc.name+":")
+			if err != nil {
+				t.Fatal(err)
+			}
+			sourceID := tc.name + "-source"
+			if err := store.CreateRun(ctx, zset.RunMeta{RunID: sourceID, Operation: tc.operation, QueryMonth: tc.period}); err != nil {
+				t.Fatal(err)
+			}
+			task := zset.Task{TaskID: sourceID + ":card-42", RunID: sourceID, Operation: tc.operation, QueryMonth: tc.period, CardID: 42, ICCID: "89860042", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "PKG-42"}
+			if n, err := store.EnqueueRunBatch(ctx, sourceID, task.ChanKey, []zset.Task{task}, time.Now().Add(-time.Second)); err != nil || n != 1 {
+				t.Fatalf("enqueue source task: %d, %v", n, err)
+			}
+			if err := store.MarkProducerDone(ctx, sourceID); err != nil {
+				t.Fatal(err)
+			}
+			now := time.Now()
+			claimed, err := store.ClaimRun(ctx, sourceID, task.ChanKey, now, now.Add(time.Minute), 1)
+			if err != nil || len(claimed) != 1 {
+				t.Fatalf("claim source task: %d, %v", len(claimed), err)
+			}
+			if done, err := store.FinalizeRunTask(ctx, claimed[0], zset.TaskOutcome{ErrorClass: "business", Code: 500, Message: "retry by pool"}); err != nil || !done {
+				t.Fatalf("write failure pool: %v, %v", done, err)
+			}
+			failedKey, _ := store.FailedKey(sourceID)
+			value, err := redis.Do(ctx, "HGET", failedKey, task.TaskID)
+			if err != nil || value == nil {
+				t.Fatalf("failure record missing: %v", err)
+			}
+			var failure zset.FailureRecord
+			if err := json.Unmarshal([]byte(value.String()), &failure); err != nil || failure.PackageCode != task.PackageCode {
+				t.Fatalf("failure pool package code: %+v, %v", failure, err)
+			}
+			if done, err := store.CompleteRunIfIdle(ctx, sourceID); err != nil || !done {
+				t.Fatalf("complete failed source: %v, %v", done, err)
+			}
+			svc, err := NewSimpleReplay(store)
+			if err != nil {
+				t.Fatal(err)
+			}
+			receipt, err := svc.Request(ctx, sourceID, "42", 0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := awaitTargetMaxRound(t, svc, store, receipt.ReplayRunID); got != DefaultReplayMaxRounds {
+				t.Fatalf("unexpected replay budget: %d", got)
+			}
+			now = time.Now()
+			replayed, err := store.ClaimRun(ctx, receipt.ReplayRunID, task.ChanKey, now, now.Add(time.Minute), 1)
+			if err != nil || len(replayed) != 1 {
+				t.Fatalf("claim replayed task: %d, %v", len(replayed), err)
+			}
+			got := replayed[0].Task
+			if got.PackageCode != task.PackageCode || got.QueryMonth != tc.period || got.Operation != tc.operation {
+				t.Fatalf("replayed task lost package or period: %+v", got)
+			}
+		})
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_recovery_test.go

@@ -0,0 +1,64 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayRecoversEnqueuedButNotCleanedFailure(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	sourceKey, _ := store.RunMetaKey("source")
+	failedKey, _ := store.FailedKey("source")
+	_, _ = redis.Do(ctx, "HSET", sourceKey, "runId", "source", "status", "completed_with_failures", "producerDone", "1", "operation", "usage", "currentRound", "1", "failedCount", "1", "replayTargetId", "replay-existing", "replayState", "processing", "replayEnqueued", "0")
+	rec := zset.FailureRecord{TaskID: "source:usage:1", CardID: 1, ICCID: "card", ChanKey: "ch", TempKey: "template", Operation: "usage", LatestRunID: "source"}
+	raw, _ := json.Marshal(rec)
+	_, _ = redis.Do(ctx, "HSET", failedKey, rec.TaskID, string(raw))
+	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "replay-existing", Operation: "usage", SyncType: "replay", SourceRunID: "source", ReplayCycle: 1, MaxRound: 1}); err != nil {
+		t.Fatal(err)
+	}
+	if v, err := redis.Do(ctx, "HGET", sourceKey, "replayMaxRound"); err != nil || (v != nil && v.String() != "") {
+		t.Fatalf("legacy reservation unexpectedly has a stored budget: %v, %v", v, err)
+	}
+	task := zset.Task{RunID: "replay-existing", TaskID: "replay-existing:" + rec.TaskID, CardID: rec.CardID, ICCID: rec.ICCID, Operation: "usage", ChanKey: "ch", TempKey: "template"}
+	if count, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []zset.Task{task}, time.Now()); err != nil || count != 1 {
+		t.Fatalf("seed enqueue = %d, %v", count, err)
+	}
+	svc, _ := NewSimpleReplayWithDefault(9, store)
+	svc.process(store, "source")
+	meta, err := store.GetRunMeta(ctx, "replay-existing")
+	if err != nil || meta.ActiveCount != 1 || !meta.ProducerDone || meta.MaxRound != 1 {
+		t.Fatalf("legacy target budget or enqueue changed after recovery: %+v %v", meta, err)
+	}
+	state, err := svc.Status(ctx, "replay-existing")
+	if err != nil || state.Status != "completed" || state.Enqueued != 1 || state.Remaining != 0 {
+		t.Fatalf("recovery state: %+v %v", state, err)
+	}
+}
+
+func TestSimpleReplayRecoversLegacyReservationBeforeTargetCreation(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	seedReplaySource(t, redis, store, "source", 1)
+	sourceKey, _ := store.RunMetaKey("source")
+	if _, err := redis.Do(ctx, "HSET", sourceKey, "replayTargetId", "replay-old-reservation", "replayState", "accepted", "replayEnqueued", "0"); err != nil {
+		t.Fatal(err)
+	}
+	svc, _ := NewSimpleReplayWithDefault(9, store)
+	svc.process(store, "source")
+	meta, err := store.GetRunMeta(ctx, "replay-old-reservation")
+	if err != nil || meta.MaxRound != 1 || !meta.ProducerDone || meta.ActiveCount != 1 {
+		t.Fatalf("legacy reservation must retain the old one-round budget: %+v %v", meta, err)
+	}
+	state, err := svc.Status(ctx, meta.RunID)
+	if err != nil || state.Status != "completed" || state.Enqueued != 1 || state.Remaining != 0 {
+		t.Fatalf("legacy reservation did not complete: %+v %v", state, err)
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_test.go

@@ -0,0 +1,96 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayBothNamespacesAndRecovery(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	traffic, _ := zset.New(redis, "flow:")
+	status, _ := zset.New(redis, "sim-status:")
+	for _, store := range []*zset.Store{traffic, status} {
+		name := store.Prefix() + "source"
+		key, _ := store.RunMetaKey(name)
+		failed, _ := store.FailedKey(name)
+		_, err := redis.Do(ctx, "HSET", key, "runId", name, "operation", "operation", "status", "completed_with_failures", "producerDone", "1", "failedCount", "3", "currentRound", "1")
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 1; i <= 3; i++ {
+			rec := zset.FailureRecord{TaskID: fmt.Sprintf("task-%d", i), CardID: int64(i), ICCID: fmt.Sprintf("iccid-%d", i), ChanKey: "ch", TempKey: "template", Operation: "operation", FailureType: "business"}
+			payload, _ := json.Marshal(rec)
+			if _, err := redis.Do(ctx, "HSET", failed, rec.TaskID, string(payload)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	svc, _ := NewSimpleReplay(traffic, status)
+	for _, prefix := range []string{"flow:", "sim-status:"} {
+		source := prefix + "source"
+		receipt, err := svc.Request(ctx, source, "42", 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		repeat, err := svc.Request(ctx, source, "42", 0)
+		if err != nil || repeat.ReplayRunID != receipt.ReplayRunID {
+			t.Fatalf("repeat = %+v, %v", repeat, err)
+		}
+		// Recovery competes with the initial goroutine but must keep its ID.
+		svc.reconcile(ctx)
+		deadline := time.Now().Add(4 * time.Second)
+		for time.Now().Before(deadline) {
+			state, err := svc.Status(ctx, receipt.ReplayRunID)
+			if err == nil && state.Status == "completed" {
+				if state.Remaining != 0 || state.Enqueued != 3 {
+					t.Fatalf("state = %+v", state)
+				}
+				store := traffic
+				if prefix == "sim-status:" {
+					store = status
+				}
+				meta, err := store.GetRunMeta(ctx, receipt.ReplayRunID)
+				if err != nil || meta.MaxRound != DefaultReplayMaxRounds || meta.SourceRunID != source || !meta.ProducerDone || meta.ActiveCount != 3 {
+					t.Fatalf("target meta = %+v, %v", meta, err)
+				}
+				sourceMeta, err := store.GetRunMeta(ctx, source)
+				if err != nil || sourceMeta.FailedCount != 3 {
+					t.Fatalf("source meta = %+v, %v", sourceMeta, err)
+				}
+				break
+			}
+			time.Sleep(20 * time.Millisecond)
+		}
+		state, _ := svc.Status(ctx, receipt.ReplayRunID)
+		if state.Status != "completed" {
+			t.Fatalf("handoff timed out: %+v", state)
+		}
+	}
+}
+
+func TestSimpleReplayRejectsRunningAndEmpty(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := zset.New(redis, "flow:")
+	svc, _ := NewSimpleReplay(store)
+	ctx := context.Background()
+	if _, err := svc.Request(ctx, "missing", "1", 0); err != contract.ErrNotFound {
+		t.Fatal(err)
+	}
+	key, _ := store.RunMetaKey("source")
+	_, _ = redis.Do(ctx, "HSET", key, "status", "running")
+	if _, err := svc.Request(ctx, "source", "1", 0); err != contract.ErrConflict {
+		t.Fatal(err)
+	}
+	_, _ = redis.Do(ctx, "HSET", key, "status", "completed_with_failures")
+	if _, err := svc.Request(ctx, "source", "1", 0); err != ErrNoFailures {
+		t.Fatal(err)
+	}
+}

worker-svr/internal/fullsync/supply/observation.go

@@ -0,0 +1,287 @@
+// Package supply holds bounded, per-channel telemetry independent of card volume.
+package supply
+
+import (
+	"sort"
+	"sync"
+	"time"
+)
+
+const SchemaVersion = 1
+
+type Wait struct {
+	Count     int64 `json:"count"`
+	TotalMS   int64 `json:"totalMs"`
+	LongestMS int64 `json:"longestMs"`
+}
+type Channel struct {
+	Key            string           `json:"chanKey"`
+	FirstQueryMS   *int64           `json:"firstQueryMs"`
+	FirstEnqueueMS *int64           `json:"firstEnqueueMs"`
+	Startup        Wait             `json:"startupWait"`
+	Confirmed      Wait             `json:"confirmedWait"`
+	Unconfirmed    Wait             `json:"unconfirmedIdle"`
+	Consumer       Wait             `json:"consumerStartupWait"`
+	UnknownMS      int64            `json:"unknownMs"`
+	EmptyClaims    int64            `json:"emptyClaims"`
+	ValidSamples   int64            `json:"validSamples"`
+	MissingSamples int64            `json:"missingSamples"`
+	Stages         map[string]int64 `json:"producerStageMs"`
+	Completeness   string           `json:"completeness"`
+	Termination    string           `json:"termination,omitempty"`
+	CoverageStart  time.Time        `json:"coverageStart"`
+	CoverageEnd    time.Time        `json:"coverageEnd"`
+}
+type Summary struct {
+	Targets       int    `json:"targetChannels"`
+	Valid         int    `json:"validChannels"`
+	Confirmed     int    `json:"confirmedChannels"`
+	Unknown       int    `json:"unknownChannels"`
+	UnknownMS     int64  `json:"unknownChannelMs"`
+	UnconfirmedMS int64  `json:"unconfirmedChannelMs"`
+	ConsumerMS    int64  `json:"consumerStartupChannelMs"`
+	TotalMS       int64  `json:"confirmedChannelMs"`
+	LongestMS     int64  `json:"longestMs"`
+	Worst         string `json:"worstChannel,omitempty"`
+}
+type Snapshot struct {
+	SchemaVersion int       `json:"schemaVersion"`
+	Attempt       string    `json:"attempt"`
+	ResolutionMS  int       `json:"resolutionMs"`
+	Partial       bool      `json:"partial"`
+	Truncated     bool      `json:"truncated"`
+	Summary       Summary   `json:"summary"`
+	Channels      []Channel `json:"channels"`
+}
+type View struct {
+	Version    uint64
+	Pending    int
+	Processing int
+	Stage      string
+	Ended      bool
+}
+type Sample struct {
+	Ready    int64
+	Leases   int64
+	Future   int64
+	Eligible bool
+	Known    bool
+}
+type state struct {
+	Channel
+	View
+	last       time.Time
+	kind       string
+	continuous int64
+	open       bool
+}
+type Observer struct {
+	mu        sync.Mutex
+	started   time.Time
+	attempt   string
+	partial   bool
+	retained  Summary
+	truncated bool
+	channels  map[string]*state
+}
+
+func New(attempt string, started time.Time, keys []string) *Observer {
+	o := &Observer{attempt: attempt, started: started, channels: make(map[string]*state, len(keys))}
+	for _, k := range keys {
+		o.channels[k] = &state{Channel: Channel{Key: k, Stages: map[string]int64{}, CoverageStart: started}, View: View{Stage: "initializing"}}
+	}
+	return o
+}
+func (o *Observer) Event(key, stage string, pending int, added bool, ended bool, now time.Time) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	s := o.channels[key]
+	if s == nil {
+		return
+	}
+	s.Version++
+	s.Stage = stage
+	s.Pending = pending
+	if stage == "sql_query" && s.FirstQueryMS == nil {
+		v := now.Sub(o.started).Milliseconds()
+		s.FirstQueryMS = &v
+	}
+	if added && s.FirstEnqueueMS == nil {
+		v := now.Sub(o.started).Milliseconds()
+		s.FirstEnqueueMS = &v
+		s.close()
+	}
+	if ended {
+		s.Ended = true
+		s.Termination = stage
+		s.CoverageEnd = now
+		s.close()
+	}
+}
+func (o *Observer) Empty(key string) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	if s := o.channels[key]; s != nil && !s.Ended {
+		s.EmptyClaims++
+	}
+}
+func (o *Observer) Processing(key string, delta int) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	if s := o.channels[key]; s != nil {
+		s.Version++
+		s.Processing = max(0, s.Processing+delta)
+		if delta > 0 {
+			s.close()
+		}
+	}
+}
+func (o *Observer) Views() map[string]View {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	out := make(map[string]View, len(o.channels))
+	for k, s := range o.channels {
+		out[k] = s.View
+	}
+	return out
+}
+func (s *state) close()      { s.kind = ""; s.continuous = 0; s.open = false; s.last = time.Time{} }
+func (o *Observer) Partial() { o.mu.Lock(); o.partial = true; o.mu.Unlock() }
+func (o *Observer) Apply(key string, before View, sample Sample, now time.Time) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	s := o.channels[key]
+	if s == nil || s.Ended {
+		return
+	}
+	kind := "unknown"
+	if sample.Known && before.Version == s.Version {
+		s.ValidSamples++
+		switch {
+		case !sample.Eligible:
+			s.close()
+			s.CoverageEnd = now
+			return
+		case sample.Ready > 0:
+			if s.Processing == 0 && sample.Leases == 0 {
+				kind = "consumer"
+			} else {
+				kind = "stock"
+			}
+		case s.Processing > 0 || sample.Leases > 0:
+			kind = "processing"
+		case sample.Future > 0:
+			kind = "scheduled"
+		case s.Pending > 0:
+			if s.FirstEnqueueMS == nil {
+				kind = "startup"
+			} else {
+				kind = "confirmed"
+			}
+		default:
+			kind = "unconfirmed"
+		}
+	} else {
+		s.MissingSamples++
+		o.partial = true
+	}
+	elapsed := now.Sub(s.last)
+	if !s.last.IsZero() && elapsed > 2*time.Second {
+		s.MissingSamples += int64(elapsed/time.Second) - 1
+		o.partial = true
+		s.close()
+	}
+	if s.kind == kind && !s.last.IsZero() {
+		ms := elapsed.Milliseconds()
+		if ms < 0 {
+			ms = 0
+		}
+		s.continuous += ms
+		var w *Wait
+		switch kind {
+		case "startup":
+			w = &s.Startup
+		case "confirmed":
+			w = &s.Confirmed
+		case "unconfirmed":
+			w = &s.Unconfirmed
+		case "consumer":
+			w = &s.Consumer
+		case "unknown":
+			s.UnknownMS += ms
+		}
+		if w != nil {
+			if !s.open {
+				w.Count++
+				s.open = true
+			}
+			w.TotalMS += ms
+			if s.continuous > w.LongestMS {
+				w.LongestMS = s.continuous
+			}
+			if kind == "startup" || kind == "confirmed" {
+				s.Stages[s.Stage] += ms
+			}
+		}
+	} else {
+		s.continuous = 0
+		s.open = false
+	}
+	s.kind = kind
+	s.last = now
+	s.CoverageEnd = now
+}
+func (o *Observer) Snapshot() Snapshot {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	out := Snapshot{SchemaVersion: 1, Attempt: o.attempt, ResolutionMS: 1000, Partial: o.partial, Truncated: o.truncated, Channels: make([]Channel, 0, len(o.channels))}
+	out.Summary.Targets = len(o.channels)
+	for _, s := range o.channels {
+		c := s.Channel
+		c.Completeness = "observing"
+		if s.Ended {
+			c.Completeness = "complete"
+		}
+		if c.MissingSamples > 0 || c.UnknownMS > 0 || c.ValidSamples == 0 || c.Termination == "cancelled" {
+			c.Completeness = "partial"
+		}
+		c.Stages = make(map[string]int64, len(s.Stages))
+		for k, v := range s.Stages {
+			c.Stages[k] = v
+		}
+		out.Channels = append(out.Channels, c)
+		if c.ValidSamples == 0 {
+			out.Partial = true
+		}
+		out.Summary.UnknownMS += c.UnknownMS
+		out.Summary.UnconfirmedMS += c.Unconfirmed.TotalMS
+		out.Summary.ConsumerMS += c.Consumer.TotalMS
+		if c.ValidSamples > 0 {
+			out.Summary.Valid++
+		}
+		total := c.Startup.TotalMS + c.Confirmed.TotalMS
+		longest := max(c.Startup.LongestMS, c.Confirmed.LongestMS)
+		if total > 0 {
+			out.Summary.Confirmed++
+		}
+		if c.MissingSamples > 0 || c.UnknownMS > 0 || c.ValidSamples == 0 {
+			out.Summary.Unknown++
+		}
+		out.Summary.TotalMS += total
+		if longest > out.Summary.LongestMS {
+			out.Summary.LongestMS = longest
+			out.Summary.Worst = c.Key
+		}
+	}
+	out.Summary.Confirmed = min(out.Summary.Targets, out.Summary.Confirmed+o.retained.Confirmed)
+	out.Summary.TotalMS += o.retained.TotalMS
+	out.Summary.UnknownMS += o.retained.UnknownMS
+	out.Summary.UnconfirmedMS += o.retained.UnconfirmedMS
+	out.Summary.ConsumerMS += o.retained.ConsumerMS
+	if o.retained.LongestMS > out.Summary.LongestMS {
+		out.Summary.LongestMS = o.retained.LongestMS
+		out.Summary.Worst = o.retained.Worst
+	}
+	sort.Slice(out.Channels, func(i, j int) bool { return out.Channels[i].Key < out.Channels[j].Key })
+	return out
+}

worker-svr/internal/fullsync/supply/observation_test.go

@@ -0,0 +1,134 @@
+package supply
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestSnapshotLimitPreservesSummaryAndRanking(t *testing.T) {
+	keys := make([]string, 700)
+	for i := range keys {
+		keys[i] = fmt.Sprintf("%04d-%s", i, strings.Repeat("x", 200))
+	}
+	start := time.Unix(100, 0)
+	o := New("attempt", start, keys)
+	for i, k := range keys {
+		o.Event(k, "global_capacity", 1, false, false, start)
+		o.Apply(k, o.Views()[k], Sample{Known: true, Eligible: true}, start.Add(time.Second))
+		o.Apply(k, o.Views()[k], Sample{Known: true, Eligible: true}, start.Add(time.Duration(1000+i)*time.Millisecond+time.Second))
+	}
+	body, err := o.Encode()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var s Snapshot
+	if err = json.Unmarshal(body, &s); err != nil {
+		t.Fatal(err)
+	}
+	if len(body) > 256*1024 || !s.Truncated || len(s.Channels) != 100 || s.Summary.Targets != 700 || s.Channels[0].Key != keys[699] {
+		t.Fatalf("invalid bounded snapshot bytes=%d %+v", len(body), s.Summary)
+	}
+	restored := New("new", start.Add(time.Hour), keys)
+	restored.Restore(s)
+	r := restored.Snapshot()
+	if !r.Partial || !r.Truncated || r.Summary.TotalMS != s.Summary.TotalMS {
+		t.Fatalf("lost historical totals %+v %+v", s.Summary, r.Summary)
+	}
+}
+
+func TestConfirmedIntervalsAndStages(t *testing.T) {
+	start := time.Unix(100, 0)
+	o := New("attempt", start, []string{"a", "empty"})
+	o.Event("a", "global_capacity", 2, false, false, start)
+	tick := func(n int) {
+		v := o.Views()["a"]
+		o.Apply("a", v, Sample{Known: true, Eligible: true}, start.Add(time.Duration(n)*time.Second))
+	}
+	tick(1)
+	tick(2)
+	o.Event("a", "redis_write", 2, false, false, start.Add(2*time.Second))
+	tick(3)
+	s := o.Snapshot()
+	c := s.Channels[0]
+	if c.Startup.Count != 1 || c.Startup.TotalMS != 2000 || c.Stages["global_capacity"] != 1000 || c.Stages["redis_write"] != 1000 {
+		t.Fatalf("unexpected %+v", c)
+	}
+	o.Processing("a", 1)
+	tick(4)
+	tick(5)
+	if o.Snapshot().Channels[0].Startup.TotalMS != 2000 {
+		t.Fatal("processing counted as idle")
+	}
+}
+func TestUncertainAndGap(t *testing.T) {
+	start := time.Unix(100, 0)
+	o := New("a", start, []string{"a"})
+	v := o.Views()["a"]
+	o.Event("a", "sql_query", 0, false, false, start)
+	o.Apply("a", v, Sample{Known: true, Eligible: true}, start.Add(time.Second))
+	o.Apply("a", o.Views()["a"], Sample{Known: false}, start.Add(2*time.Second))
+	o.Apply("a", o.Views()["a"], Sample{Known: true, Eligible: true}, start.Add(6*time.Second))
+	s := o.Snapshot()
+	if !s.Partial || s.Channels[0].UnknownMS != 1000 || s.Channels[0].MissingSamples != 5 || s.Summary.Confirmed != 0 {
+		t.Fatalf("unexpected %+v", s)
+	}
+}
+func TestSupplyExclusions(t *testing.T) {
+	for _, tc := range []struct {
+		name       string
+		sample     Sample
+		processing bool
+	}{{"stock", Sample{Known: true, Eligible: true, Ready: 1}, false}, {"lease", Sample{Known: true, Eligible: true, Leases: 1}, false}, {"ended_round_or_blocked", Sample{Known: true, Eligible: false}, false}, {"consumer", Sample{Known: true, Eligible: true}, true}, {"unknown", Sample{Eligible: true}, false}} {
+		t.Run(tc.name, func(t *testing.T) {
+			start := time.Unix(100, 0)
+			o := New("a", start, []string{"a"})
+			o.Event("a", "global_capacity", 100, false, false, start)
+			if tc.processing {
+				o.Processing("a", 1)
+			}
+			for i := 1; i <= 3; i++ {
+				o.Apply("a", o.Views()["a"], tc.sample, start.Add(time.Duration(i)*time.Second))
+			}
+			if o.Snapshot().Summary.Confirmed != 0 {
+				t.Fatal("excluded sample counted as supply idle")
+			}
+		})
+	}
+}
+func TestConcurrentEvents(t *testing.T) {
+	start := time.Unix(100, 0)
+	o := New("a", start, []string{"a"})
+	var wg sync.WaitGroup
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for n := 0; n < 100; n++ {
+				o.Event("a", "sql_query", 0, false, false, start)
+				o.Processing("a", 1)
+				o.Empty("a")
+				o.Apply("a", o.Views()["a"], Sample{Known: true, Eligible: true}, start)
+				o.Snapshot()
+				o.Processing("a", -1)
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func TestEmptyNeverConfirms(t *testing.T) {
+	start := time.Unix(100, 0)
+	o := New("a", start, []string{"a"})
+	for i := 1; i <= 3; i++ {
+		o.Apply("a", o.Views()["a"], Sample{Known: true, Eligible: true}, start.Add(time.Duration(i)*time.Second))
+	}
+	o.Event("a", "empty", 0, false, true, start.Add(3*time.Second))
+	s := o.Snapshot()
+	if s.Summary.Confirmed != 0 || s.Channels[0].Unconfirmed.TotalMS != 2000 {
+		t.Fatalf("unexpected %+v", s)
+	}
+}

worker-svr/internal/fullsync/supply/tracker.go

@@ -0,0 +1,90 @@
+package supply
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"sync"
+	"sync/atomic"
+)
+
+// Tracker routes optional consumer events to the local producer attempt.
+type Tracker struct {
+	mu             sync.RWMutex
+	runs           map[string]*Observer
+	LocalConsumer  bool
+	ConsumerActive atomic.Bool
+}
+
+func NewTracker(localConsumer bool) *Tracker {
+	return &Tracker{runs: map[string]*Observer{}, LocalConsumer: localConsumer}
+}
+func (t *Tracker) Put(run string, o *Observer) { t.mu.Lock(); t.runs[run] = o; t.mu.Unlock() }
+func (t *Tracker) Remove(run string)           { t.mu.Lock(); delete(t.runs, run); t.mu.Unlock() }
+func (t *Tracker) Empty(run, key string) {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	if o := t.runs[run]; o != nil {
+		o.Empty(key)
+	}
+}
+func (t *Tracker) Processing(run, key string, delta int) {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	if o := t.runs[run]; o != nil {
+		o.Processing(key, delta)
+	}
+}
+func (o *Observer) Restore(previous Snapshot) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.partial = true
+	o.retained = previous.Summary
+	o.truncated = previous.Truncated
+	for _, c := range previous.Channels {
+		if s := o.channels[c.Key]; s != nil {
+			if c.Startup.TotalMS+c.Confirmed.TotalMS > 0 {
+				o.retained.Confirmed--
+			}
+			o.retained.TotalMS -= c.Startup.TotalMS + c.Confirmed.TotalMS
+			o.retained.UnknownMS -= c.UnknownMS
+			o.retained.UnconfirmedMS -= c.Unconfirmed.TotalMS
+			o.retained.ConsumerMS -= c.Consumer.TotalMS
+			s.Channel = c
+			if s.Stages == nil {
+				s.Stages = map[string]int64{}
+			}
+			s.Termination = ""
+			s.MissingSamples++
+			s.close()
+		}
+	}
+}
+func (o *Observer) Encode() ([]byte, error) {
+	snapshot := o.Snapshot()
+	encoded, err := json.Marshal(snapshot)
+	if err != nil || len(encoded) <= 256*1024 {
+		return encoded, err
+	}
+	snapshot.Truncated = true
+	snapshot.Partial = true
+	sort.Slice(snapshot.Channels, func(i, j int) bool {
+		a, b := snapshot.Channels[i], snapshot.Channels[j]
+		av, bv := max(a.Startup.LongestMS, a.Confirmed.LongestMS), max(b.Startup.LongestMS, b.Confirmed.LongestMS)
+		if av == bv {
+			return a.Key < b.Key
+		}
+		return av > bv
+	})
+	snapshot.Channels = snapshot.Channels[:min(100, len(snapshot.Channels))]
+	for {
+		encoded, err = json.Marshal(snapshot)
+		if err != nil || len(encoded) <= 256*1024 {
+			return encoded, err
+		}
+		if len(snapshot.Channels) == 0 {
+			return nil, fmt.Errorf("supply summary exceeds snapshot limit")
+		}
+		snapshot.Channels = snapshot.Channels[:len(snapshot.Channels)-1]
+	}
+}

worker-svr/internal/fullsync/supply/validation.go

@@ -0,0 +1,27 @@
+package supply
+
+// Valid checks only telemetry. Callers must degrade invalid observations without
+// rejecting the business run or its archive.
+func (s Snapshot) Valid() bool {
+	if s.SchemaVersion != 1 || s.ResolutionMS != 1000 || s.Summary.Targets < 0 || s.Summary.Valid < 0 || s.Summary.Valid > s.Summary.Targets || s.Summary.Confirmed < 0 || s.Summary.Confirmed > s.Summary.Targets || s.Summary.Unknown < 0 || s.Summary.Unknown > s.Summary.Targets || s.Summary.TotalMS < 0 || s.Summary.LongestMS < 0 || s.Summary.UnknownMS < 0 || s.Summary.UnconfirmedMS < 0 || s.Summary.ConsumerMS < 0 {
+		return false
+	}
+	seen := make(map[string]bool, len(s.Channels))
+	for _, c := range s.Channels {
+		if c.Key == "" || seen[c.Key] || c.ValidSamples < 0 || c.MissingSamples < 0 || c.UnknownMS < 0 {
+			return false
+		}
+		seen[c.Key] = true
+		for _, w := range []Wait{c.Startup, c.Confirmed, c.Unconfirmed, c.Consumer} {
+			if w.Count < 0 || w.TotalMS < 0 || w.LongestMS < 0 || w.LongestMS > w.TotalMS {
+				return false
+			}
+		}
+		for _, v := range c.Stages {
+			if v < 0 {
+				return false
+			}
+		}
+	}
+	return len(s.Channels) <= s.Summary.Targets
+}