chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）

worker-svr/internal/fullsync/runtime/catalog.go

@@ -0,0 +1,211 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+const (
+	defaultRunListLimit = 50
+	maxRunListLimit     = 100
+)
+
+// Catalog aggregates the flow and sim-status Redis namespaces while keeping
+// each run's storage protocol inside its existing Repository.
+type Catalog struct {
+	repositories []*Repository
+}
+
+func NewCatalog(repositories ...*Repository) (*Catalog, error) {
+	if len(repositories) == 0 {
+		return nil, errors.New("full-sync catalog requires repositories")
+	}
+	for _, repository := range repositories {
+		if repository == nil {
+			return nil, errors.New("full-sync catalog repository is nil")
+		}
+	}
+	return &Catalog{repositories: repositories}, nil
+}
+
+func (c *Catalog) GetRun(ctx context.Context, runID string) (contract.RunView, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunView{}, err
+	}
+	for _, repository := range c.repositories {
+		meta, err := repository.GetRunMeta(ctx, runID)
+		if err == nil {
+			rounds, roundErr := repository.GetRoundStats(ctx, runID)
+			if roundErr != nil {
+				return contract.RunView{}, roundErr
+			}
+			return contract.RunView{Source: contract.SourceLive, Run: meta, Rounds: rounds}, nil
+		}
+		if !errors.Is(err, contract.ErrNotFound) {
+			return contract.RunView{}, err
+		}
+	}
+	for _, repository := range c.repositories {
+		archive, err := repository.GetArchivedRun(ctx, runID)
+		if err == nil {
+			return contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}, nil
+		}
+		if !errors.Is(err, contract.ErrNotFound) {
+			return contract.RunView{}, err
+		}
+	}
+	return contract.RunView{}, contract.ErrNotFound
+}
+
+func (c *Catalog) ListRuns(ctx context.Context, filter contract.RunListFilter) (contract.RunList, error) {
+	if filter.Limit == 0 {
+		filter.Limit = defaultRunListLimit
+	}
+	if filter.Limit < 1 || filter.Limit > maxRunListLimit {
+		return contract.RunList{}, contract.ErrInvalidArgument
+	}
+	byID := make(map[string]contract.RunView)
+	for _, repository := range c.repositories {
+		active, err := repository.store.ActiveRuns(ctx)
+		if err != nil {
+			return contract.RunList{}, err
+		}
+		for _, runID := range active {
+			view, getErr := repository.liveRun(ctx, runID)
+			if errors.Is(getErr, contract.ErrNotFound) {
+				continue
+			}
+			if getErr != nil {
+				return contract.RunList{}, getErr
+			}
+			byID[runID] = view
+		}
+	}
+	if err := c.listArchives(ctx, byID); err != nil {
+		return contract.RunList{}, err
+	}
+	items := make([]contract.RunView, 0, len(byID))
+	for _, item := range byID {
+		kind := contract.SyncKindForOperation(item.Run.Operation)
+		if kind == "" || (filter.SyncKind != "" && kind != filter.SyncKind) {
+			continue
+		}
+		items = append(items, item)
+	}
+	sort.SliceStable(items, func(i, j int) bool {
+		return runSortTime(items[i].Run).After(runSortTime(items[j].Run))
+	})
+	total := len(items)
+	if len(items) > filter.Limit {
+		items = items[:filter.Limit]
+	}
+	if items == nil {
+		items = []contract.RunView{}
+	}
+	return contract.RunList{Items: items, Total: total}, nil
+}
+
+func (c *Catalog) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.FailurePage{}, err
+	}
+	for _, repository := range c.repositories {
+		meta, err := repository.GetRunMeta(ctx, runID)
+		if err == nil {
+			result, listErr := repository.ListFailures(ctx, runID, page)
+			if listErr != nil {
+				return contract.FailurePage{}, listErr
+			}
+			if meta.FailedCount > 0 && len(result.Items) == 0 && result.NextCursor == "" {
+				failedKey, _ := repository.store.FailedKey(runID)
+				size, sizeErr := repository.redis.Do(ctx, "HLEN", failedKey)
+				if sizeErr != nil {
+					return contract.FailurePage{}, sizeErr
+				}
+				if size.Int64() == 0 {
+					return contract.FailurePage{}, contract.ErrFailureDetailsUnavailable
+				}
+			}
+			return result, nil
+		} else if !errors.Is(err, contract.ErrNotFound) {
+			return contract.FailurePage{}, err
+		}
+	}
+	view, err := c.GetRun(ctx, runID)
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	if view.Run.FailedCount == 0 {
+		return contract.FailurePage{Items: []contract.Failure{}}, nil
+	}
+	for _, repository := range c.repositories {
+		failedKey, _ := repository.store.FailedKey(runID)
+		size, sizeErr := repository.redis.Do(ctx, "HLEN", failedKey)
+		if sizeErr != nil {
+			return contract.FailurePage{}, sizeErr
+		}
+		if size.Int64() > 0 {
+			return repository.ListFailures(ctx, runID, page)
+		}
+	}
+	return contract.FailurePage{}, contract.ErrFailureDetailsUnavailable
+}
+
+func (c *Catalog) listArchives(ctx context.Context, byID map[string]contract.RunView) error {
+	root := c.repositories[0].root
+	if strings.TrimSpace(root) == "" {
+		return nil
+	}
+	matches, err := filepath.Glob(filepath.Join(root, "run-*.json"))
+	if err != nil {
+		return err
+	}
+	for _, path := range matches {
+		data, readErr := os.ReadFile(path)
+		if readErr != nil {
+			g.Log().Warningf(ctx, "event=full_sync_list_archive status=skip path=%s error=%q", path, readErr.Error())
+			continue
+		}
+		var archive contract.RunArchive
+		if decodeErr := json.Unmarshal(data, &archive); decodeErr != nil || archive.Run.RunID == "" || filepath.Base(path) != "run-"+archive.Run.RunID+".json" {
+			g.Log().Warningf(ctx, "event=full_sync_list_archive status=skip path=%s error=%v", path, decodeErr)
+			continue
+		}
+		if _, exists := byID[archive.Run.RunID]; !exists {
+			byID[archive.Run.RunID] = contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}
+		}
+	}
+	return nil
+}
+
+func (r *Repository) liveRun(ctx context.Context, runID string) (contract.RunView, error) {
+	meta, err := r.GetRunMeta(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	rounds, err := r.GetRoundStats(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	return contract.RunView{Source: contract.SourceLive, Run: meta, Rounds: rounds}, nil
+}
+
+func runSortTime(run contract.RunMeta) (value time.Time) {
+	if run.StartedAt != nil && !run.StartedAt.IsZero() {
+		return *run.StartedAt
+	}
+	if !run.CompletedAt.IsZero() {
+		return run.CompletedAt
+	}
+	return run.CreatedAt
+}

worker-svr/internal/fullsync/runtime/catalog_test.go

@@ -0,0 +1,123 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type catalogRedisStub struct {
+	responses []*gvar.Var
+	calls     []string
+}
+
+func (s *catalogRedisStub) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
+	s.calls = append(s.calls, command)
+	if len(s.responses) == 0 {
+		return gvar.New(nil), nil
+	}
+	response := s.responses[0]
+	s.responses = s.responses[1:]
+	return response, nil
+}
+
+func (s *catalogRedisStub) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	return gvar.New(int64(1)), nil
+}
+
+func TestCatalogListRunsMergesLiveAndArchive(t *testing.T) {
+	root := t.TempDir()
+	now := time.Date(2026, 9, 28, 1, 0, 0, 0, time.UTC)
+	archive := contract.RunArchive{Version: 1, Run: contract.RunMeta{
+		RunID: "archived", Operation: "sim_status_queue_full_sync", Status: contract.RunCompleted,
+		CreatedAt: now.Add(-time.Hour), CompletedAt: now.Add(-time.Minute),
+	}}
+	data, err := json.Marshal(archive)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "run-archived.json"), data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	metaResponse := gvar.New([]any{"runId", "live", "operation", "current_month_usage_full_sync", "status", "running", "currentRound", "1", "createdAt", now.Format(time.RFC3339Nano), "roundStats", "[]"})
+	redis := &catalogRedisStub{responses: []*gvar.Var{
+		gvar.New([]any{"live"}), metaResponse, metaResponse,
+		gvar.New([]any{"live"}), metaResponse, metaResponse,
+	}}
+	store, err := zset.New(redis, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	catalog, err := NewCatalog(repository)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := catalog.ListRuns(context.Background(), contract.RunListFilter{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Total != 2 || len(result.Items) != 2 {
+		t.Fatalf("result = %+v", result)
+	}
+	if result.Items[0].Run.RunID != "live" || result.Items[1].Run.RunID != "archived" {
+		t.Fatalf("items = %+v", result.Items)
+	}
+	filtered, err := catalog.ListRuns(context.Background(), contract.RunListFilter{SyncKind: contract.SyncKindTrafficUsage, Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if filtered.Total != 1 || filtered.Items[0].Run.RunID != "live" {
+		t.Fatalf("filtered = %+v", filtered)
+	}
+	view, err := catalog.GetRun(context.Background(), "archived")
+	if err != nil || view.Source != contract.SourceArchive || view.Run.RunID != "archived" {
+		t.Fatalf("archived view = %+v, err = %v", view, err)
+	}
+}
+
+func TestCatalogPrefersLiveAndSkipsBrokenArchive(t *testing.T) {
+	root := t.TempDir()
+	for name, payload := range map[string]string{
+		"run-live.json":   `{"version":1,"run":{"runId":"live","operation":"sim_status_queue_full_sync","status":"completed"}}`,
+		"run-broken.json": `{not-json}`,
+	} {
+		if err := os.WriteFile(filepath.Join(root, name), []byte(payload), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	redis := &catalogRedisStub{responses: []*gvar.Var{
+		gvar.New([]any{"live"}),
+		gvar.New([]any{"runId", "live", "operation", "sim_status_queue_full_sync", "status", "running"}),
+		gvar.New([]any{"runId", "live", "operation", "sim_status_queue_full_sync", "status", "running"}),
+	}}
+	store, err := zset.New(redis, "sim-status:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	catalog, err := NewCatalog(repository)
+	if err != nil {
+		t.Fatal(err)
+	}
+	result, err := catalog.ListRuns(context.Background(), contract.RunListFilter{Limit: 10})
+	if err != nil || result.Total != 1 || result.Items[0].Source != contract.SourceLive || result.Items[0].Run.Status != contract.RunRunning {
+		t.Fatalf("result = %+v, err = %v", result, err)
+	}
+}

worker-svr/internal/fullsync/runtime/runtime.go

@@ -0,0 +1,497 @@
+// Package runtime provides the small concrete storage adapters used by the
+// full-sync HTTP and archive services. It intentionally keeps queue protocol
+// transitions in queue/zset and only reads or records durable metadata here.
+package runtime
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type Repository struct {
+	store *zset.Store
+	redis zset.RedisClient
+	keys  contract.Keys
+	root  string
+}
+
+func (r *Repository) GetRoundStats(ctx context.Context, runID string) ([]contract.RoundStats, error) {
+	stats, err := r.store.GetRoundStats(ctx, runID)
+	if err != nil {
+		return nil, wrapNotFound(err)
+	}
+	out := make([]contract.RoundStats, 0, len(stats))
+	for _, stat := range stats {
+		out = append(out, contract.RoundStats{
+			RunID:                stat.RunID,
+			Round:                stat.Round,
+			EnqueuedCount:        stat.EnqueuedCount,
+			ActiveCount:          stat.ActiveCount,
+			NextRoundCount:       stat.NextRoundCount,
+			SuccessCount:         stat.SuccessCount,
+			FailedCount:          stat.FailedCount,
+			UpstreamAttemptCount: stat.UpstreamAttemptCount,
+			BatchCount:           stat.BatchCount,
+			StartedAt:            stat.StartedAt,
+			CompletedAt:          stat.CompletedAt,
+		})
+	}
+	return out, nil
+}
+
+func (r *Repository) RequestReplay(ctx context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
+	if err := req.Validate(); err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	fp, err := req.Fingerprint()
+	if err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	key, err := r.keys.ReplayIdempotencyKey(req.SourceRunID, req.IdempotencyKey)
+	if err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	actionID := "replay-" + fp[:24]
+	actionKey, _ := r.keys.ReplayActionKey(actionID)
+	targetID := "replay-run-" + fp[:24]
+	now := time.Now().UTC()
+	action := contract.ReplayAction{ActionID: actionID, SourceRunID: req.SourceRunID, TargetRunID: targetID, RequestedBy: req.RequestedBy, IdempotencyKey: req.IdempotencyKey, Fingerprint: fp, Status: contract.ActionAccepted, ReplayCycle: 1, Progress: contract.ReplayProgress{}, CreatedAt: now, UpdatedAt: now}
+	b, _ := json.Marshal(action)
+	// Reserve idempotency first. A repeat reads the original action; a changed
+	// request under the same key is rejected by fingerprint comparison.
+	set, e := r.redis.Do(ctx, "SET", key, actionID, "NX", "EX", 604800)
+	if e != nil {
+		return contract.ReplayAccepted{}, e
+	}
+	if set.String() == "" || strings.EqualFold(set.String(), "nil") {
+		reserved, e := r.redis.Do(ctx, "GET", key)
+		if e != nil {
+			return contract.ReplayAccepted{}, e
+		}
+		reservedID := reserved.String()
+		old, e := r.GetReplayAction(ctx, reservedID)
+		if e != nil {
+			return contract.ReplayAccepted{}, e
+		}
+		if old.Fingerprint != fp {
+			return contract.ReplayAccepted{}, contract.ErrConflict
+		}
+		u, _ := contract.ActionStatusURL(old.ActionID)
+		return contract.ReplayAccepted{ActionID: old.ActionID, TargetRunID: old.TargetRunID, Status: old.Status, StatusURL: u}, nil
+	}
+	if _, e = r.redis.Do(ctx, "SET", actionKey, string(b), "EX", 604800); e != nil {
+		return contract.ReplayAccepted{}, e
+	}
+	_, _ = r.redis.Do(ctx, "SADD", r.keys.PendingReplayActionsKey(), actionID)
+	go r.executeReplay(actionID)
+	u, _ := contract.ActionStatusURL(actionID)
+	return contract.ReplayAccepted{ActionID: actionID, TargetRunID: targetID, Status: action.Status, StatusURL: u}, nil
+}
+
+// executeReplay is deliberately small: it snapshots the source failure hash,
+// enqueues equivalent tasks into one target run, then removes only entries
+// that were accepted. Redis action JSON is the durable retry checkpoint.
+func (r *Repository) executeReplay(actionID string) {
+	ctx := context.Background()
+	action, err := r.GetReplayAction(ctx, actionID)
+	if err != nil {
+		return
+	}
+	action.Status = contract.ActionProcessing
+	action.UpdatedAt = time.Now().UTC()
+	_ = r.saveAction(ctx, action)
+	source, err := r.GetRunMeta(ctx, action.SourceRunID)
+	if err != nil {
+		action.Error = err.Error()
+		action.Status = contract.ActionNeedsAttention
+		action.UpdatedAt = time.Now().UTC()
+		_ = r.saveAction(ctx, action)
+		return
+	}
+	target := zset.RunMeta{RunID: action.TargetRunID, Operation: source.Operation, QueryMonth: source.QueryMonth, Status: "producing", CurrentRound: 1, CreatedAt: time.Now().UTC(), SyncType: string(contract.SyncReplay), SourceRunID: action.SourceRunID, ReplayCycle: action.ReplayCycle, TriggerSource: "replay"}
+	if err := r.store.CreateRun(ctx, target); err != nil {
+		if _, getErr := r.store.GetRunMeta(ctx, target.RunID); getErr != nil {
+			action.Error = err.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+	}
+	fkey, _ := r.store.FailedKey(action.SourceRunID)
+	var records []struct {
+		field string
+		rec   zset.FailureRecord
+	}
+	var cursor uint64
+	for {
+		v, e := r.redis.Do(ctx, "HSCAN", fkey, cursor, "COUNT", 500)
+		if e != nil {
+			action.Error = e.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+		cursor, pairs, e := parseScanResult(v)
+		if e != nil {
+			action.Error = e.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+		for i := 0; i < len(pairs); i += 2 {
+			var rec zset.FailureRecord
+			if json.Unmarshal([]byte(pairs[i+1].String()), &rec) == nil {
+				records = append(records, struct {
+					field string
+					rec   zset.FailureRecord
+				}{pairs[i].String(), rec})
+			}
+		}
+		if cursor == 0 {
+			break
+		}
+	}
+	groups := map[string][]zset.Task{}
+	for _, item := range records {
+		rec := item.rec
+		chanKey := rec.ChanKey
+		if chanKey == "" {
+			chanKey = "default"
+		}
+		groups[chanKey] = append(groups[chanKey], zset.Task{TaskID: action.TargetRunID + ":" + rec.TaskID, RunID: action.TargetRunID, Operation: rec.Operation, QueryMonth: rec.QueryMonth, ChanKey: chanKey, TempKey: rec.TempKey, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI})
+	}
+	for chanKey, tasks := range groups {
+		if _, err := r.store.EnqueueRunBatch(ctx, action.TargetRunID, chanKey, tasks, time.Now()); err != nil {
+			action.Error = err.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+	}
+	for _, item := range records {
+		_, _ = r.redis.Do(ctx, "HDEL", fkey, item.field)
+	}
+	_ = r.store.MarkProducerDone(ctx, action.TargetRunID)
+	action.Progress = contract.ReplayProgress{DiscoveredCount: int64(len(records)), EnqueuedCount: int64(len(records)), CleanedCount: int64(len(records)), FailureCount: int64(len(records))}
+	action.Status = contract.ActionCompleted
+	action.UpdatedAt = time.Now().UTC()
+	completed := action.UpdatedAt
+	action.CompletedAt = &completed
+	digest := sha256.Sum256([]byte(action.ActionID + action.TargetRunID))
+	action.Receipt = &contract.ArchiveReceipt{Version: 1, Path: "replay://" + action.ActionID, Checksum: hex.EncodeToString(digest[:]), ArchivedAt: completed}
+	_ = r.saveAction(ctx, action)
+}
+
+func (r *Repository) saveAction(ctx context.Context, action contract.ReplayAction) error {
+	b, err := json.Marshal(action)
+	if err != nil {
+		return err
+	}
+	key, err := r.keys.ReplayActionKey(action.ActionID)
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "SET", key, string(b), "EX", 604800)
+	return err
+}
+
+func (r *Repository) GetReplayAction(ctx context.Context, actionID string) (contract.ReplayAction, error) {
+	if err := contract.ValidateID(actionID); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	key, _ := r.keys.ReplayActionKey(actionID)
+	v, err := r.redis.Do(ctx, "GET", key)
+	if err != nil {
+		return contract.ReplayAction{}, err
+	}
+	if v == nil || v.IsNil() || strings.TrimSpace(v.String()) == "" {
+		return contract.ReplayAction{}, contract.ErrNotFound
+	}
+	var action contract.ReplayAction
+	if err := json.Unmarshal([]byte(v.String()), &action); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	return action, nil
+}
+
+func New(store *zset.Store, archiveRoot string) (*Repository, error) {
+	if store == nil {
+		return nil, errors.New("full-sync runtime store is nil")
+	}
+	keys, err := contract.NewKeys(store.Prefix())
+	if err != nil {
+		return nil, err
+	}
+	return &Repository{store: store, redis: store.RedisClient(), keys: keys, root: archiveRoot}, nil
+}
+
+func (r *Repository) GetRunMeta(ctx context.Context, runID string) (contract.RunMeta, error) {
+	m, err := r.store.GetRunMeta(ctx, runID)
+	if err != nil {
+		return contract.RunMeta{}, wrapNotFound(err)
+	}
+	return toContractMeta(m), nil
+}
+
+func (r *Repository) GetArchivedRun(ctx context.Context, runID string) (contract.RunArchive, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunArchive{}, err
+	}
+	if strings.TrimSpace(r.root) == "" {
+		return contract.RunArchive{}, contract.ErrNotFound
+	}
+	b, err := os.ReadFile(filepath.Join(r.root, "run-"+runID+".json"))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return contract.RunArchive{}, contract.ErrNotFound
+		}
+		return contract.RunArchive{}, err
+	}
+	var archive contract.RunArchive
+	if err := json.Unmarshal(b, &archive); err != nil {
+		return contract.RunArchive{}, fmt.Errorf("decode run archive: %w", err)
+	}
+	if archive.Run.RunID != runID {
+		return contract.RunArchive{}, fmt.Errorf("archive run ID does not match filename")
+	}
+	return archive, nil
+}
+
+func (r *Repository) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) {
+	p, err := page.Normalized()
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	key, err := r.store.FailedKey(runID)
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	start := uint64(0)
+	if p.Cursor != "" {
+		start, err = decodeCursor(p.Cursor, runID)
+		if err != nil {
+			return contract.FailurePage{}, err
+		}
+	}
+	var items []contract.Failure
+	var next uint64
+	for {
+		v, e := r.redis.Do(ctx, "HSCAN", key, start, "COUNT", p.Limit)
+		if e != nil {
+			return contract.FailurePage{}, e
+		}
+		next, pairs, e := parseScanResult(v)
+		if e != nil {
+			return contract.FailurePage{}, e
+		}
+		for i := 1; i < len(pairs); i += 2 {
+			var rec zset.FailureRecord
+			if json.Unmarshal([]byte(pairs[i].String()), &rec) == nil {
+				items = append(items, toFailure(rec))
+				if len(items) >= p.Limit {
+					break
+				}
+			}
+		}
+		if len(items) >= p.Limit || next == 0 {
+			break
+		}
+		start = next
+	}
+	result := contract.FailurePage{Items: items}
+	if next != 0 {
+		result.NextCursor = encodeCursor(runID, next)
+	}
+	if result.Items == nil {
+		result.Items = []contract.Failure{}
+	}
+	return result, nil
+}
+
+func (r *Repository) ListArchiveWork(ctx context.Context, page contract.PageRequest) (contract.ArchiveWorkPage, error) {
+	p, err := page.Normalized()
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	start := uint64(0)
+	if p.Cursor != "" {
+		start, err = decodeCursor(p.Cursor, "archive")
+		if err != nil {
+			return contract.ArchiveWorkPage{}, err
+		}
+	}
+	v, err := r.redis.Do(ctx, "ZSCAN", r.keys.PendingArchivesKey(), start, "COUNT", p.Limit)
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	out := contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}
+	next, pairs, err := parseScanResult(v)
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	for i := 0; i < len(pairs); i += 2 {
+		out.Items = append(out.Items, contract.ArchiveWork{RunID: pairs[i].String(), Version: pairs[i+1].String()})
+	}
+	if next != 0 {
+		out.NextCursor = encodeCursor("archive", next)
+	}
+	return out, nil
+}
+
+func parseScanResult(value *gvar.Var) (uint64, []*gvar.Var, error) {
+	if value == nil {
+		return 0, nil, errors.New("redis scan returned nil response")
+	}
+	values := value.Vars()
+	if len(values) != 2 {
+		return 0, nil, fmt.Errorf("unexpected Redis scan response length: %d", len(values))
+	}
+	pairs := values[1].Vars()
+	if len(pairs)%2 != 0 {
+		return 0, nil, fmt.Errorf("redis scan returned odd number of entry values: %d", len(pairs))
+	}
+	return uint64(values[0].Int64()), pairs, nil
+}
+
+func (r *Repository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
+	if archived, err := r.GetArchivedRun(ctx, work.RunID); err == nil {
+		return archived, nil
+	}
+	meta, err := r.GetRunMeta(ctx, work.RunID)
+	if err != nil {
+		return contract.RunArchive{}, err
+	}
+	rounds, err := r.GetRoundStats(ctx, work.RunID)
+	if err != nil {
+		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=read error=%q", work.RunID, err.Error())
+		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
+	}
+	if err := validateRoundAccounting(meta, rounds); err != nil {
+		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=validate error=%q", work.RunID, err.Error())
+		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
+	}
+	return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: rounds}, nil
+}
+
+func validateRoundAccounting(meta contract.RunMeta, rounds []contract.RoundStats) error {
+	if len(rounds) == 0 {
+		return nil
+	}
+	var success, failed int64
+	seen := make(map[int]struct{}, len(rounds))
+	for _, round := range rounds {
+		if round.RunID != meta.RunID {
+			return fmt.Errorf("round %d belongs to run %q", round.Round, round.RunID)
+		}
+		if _, ok := seen[round.Round]; ok {
+			return fmt.Errorf("duplicate round %d", round.Round)
+		}
+		seen[round.Round] = struct{}{}
+		if round.EnqueuedCount != round.SuccessCount+round.NextRoundCount+round.FailedCount {
+			return fmt.Errorf("round %d counters do not close", round.Round)
+		}
+		success += round.SuccessCount
+		failed += round.FailedCount
+	}
+	if success != meta.SuccessCount || failed != meta.FailedCount {
+		return fmt.Errorf("run counters do not match round totals: success=%d/%d failed=%d/%d", success, meta.SuccessCount, failed, meta.FailedCount)
+	}
+	return nil
+}
+
+func (r *Repository) RegisterArchiveWork(ctx context.Context, work contract.ArchiveWork) error {
+	if err := contract.ValidateID(work.RunID); err != nil {
+		return err
+	}
+	if strings.TrimSpace(work.Version) == "" {
+		return contract.ErrInvalidArgument
+	}
+	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), work.Version, work.RunID)
+	return err
+}
+
+func (r *Repository) RegisterArchive(ctx context.Context, work contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
+	if err := receipt.Validate(); err != nil {
+		return err
+	}
+	key, err := r.keys.ArchiveStateKey(work.RunID)
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "HSET", key, "version", receipt.Version, "path", receipt.Path, "checksum", receipt.Checksum, "archivedAt", receipt.ArchivedAt.UTC().Format(time.RFC3339Nano))
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "ZREM", r.keys.PendingArchivesKey(), work.RunID)
+	return err
+}
+
+func (r *Repository) RegisterArchiveFailure(ctx context.Context, work contract.ArchiveWork, message string) error {
+	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), time.Now().UnixMilli(), work.RunID)
+	return err
+}
+
+func toContractMeta(m zset.RunMeta) contract.RunMeta {
+	syncType := contract.SyncType(m.SyncType)
+	if syncType == "" {
+		syncType = contract.SyncNormal
+	}
+	return contract.RunMeta{SupplyObservation: m.SupplyObservation, RunID: m.RunID, Operation: m.Operation, QueryMonth: m.QueryMonth, Status: contract.RunStatus(m.Status), ProducerDone: m.ProducerDone, CurrentRound: m.CurrentRound, ActiveCount: m.ActiveCount, InflightCount: m.InflightCount, NextRoundCount: m.NextRoundCount, SuccessCount: m.SuccessCount, FailedCount: m.FailedCount, CreatedAt: m.CreatedAt, ProducerCompletedAt: m.ProducerCompletedAt, CompletedAt: m.CompletedAt, SyncType: syncType, SourceRunID: m.SourceRunID, ReplayCycle: m.ReplayCycle, TriggerSource: m.TriggerSource, ScannedCount: m.ScannedCount, EnqueuedCount: m.EnqueuedCount, UpstreamAttemptCount: m.UpstreamAttemptCount, BatchCount: m.BatchCount, MaxRound: m.MaxRound, PendingFailureCount: m.PendingFailureCount, ArchiveStatus: contract.ArchiveStatus(m.ArchiveStatus), ArchiveVersion: m.ArchiveVersion, ArchivePath: m.ArchivePath, ArchiveChecksum: m.ArchiveChecksum, ReplayDisposition: contract.ReplayDisposition(m.ReplayDisposition), ScopeChannels: m.ScopeChannels, ScopeDigest: m.ScopeDigest, ScopeChannelsSample: m.ScopeChannelsSample, StartedAt: m.StartedAt, ScanCompletedAt: m.ScanCompletedAt, ArchivedAt: m.ArchivedAt}
+}
+func toFailure(f zset.FailureRecord) contract.Failure {
+	return contract.Failure{TaskID: f.TaskID, CardID: f.CardID, ErrorCategory: contract.NormalizeErrorCategory(f.FailureType), ErrorMessage: contract.BoundedErrorMessage(f.Message), FailedAt: f.LastFailedAt, Round: f.Round}
+}
+func wrapNotFound(err error) error {
+	if strings.Contains(strings.ToLower(err.Error()), "not found") {
+		return contract.ErrNotFound
+	}
+	return err
+}
+func encodeCursor(scope string, n uint64) contract.Cursor {
+	return contract.Cursor(base64.RawURLEncoding.EncodeToString([]byte(scope + ":" + strconv.FormatUint(n, 10))))
+}
+func decodeCursor(c contract.Cursor, scope string) (uint64, error) {
+	b, e := base64.RawURLEncoding.DecodeString(string(c))
+	if e != nil {
+		return 0, contract.ErrInvalidArgument
+	}
+	p := strings.Split(string(b), ":")
+	if len(p) != 2 || p[0] != scope {
+		return 0, contract.ErrInvalidArgument
+	}
+	n, e := strconv.ParseUint(p[1], 10, 64)
+	if e != nil {
+		return 0, contract.ErrInvalidArgument
+	}
+	return n, nil
+}
+
+var _ contract.LiveRunReader = (*Repository)(nil)
+var _ contract.ArchivedRunReader = (*Repository)(nil)
+var _ contract.FailurePager = (*Repository)(nil)
+var _ contract.ArchiveWorkDiscovery = (*Repository)(nil)
+var _ contract.ArchiveWorkLoader = (*Repository)(nil)
+var _ contract.ArchiveWorkRegistrar = (*Repository)(nil)
+var _ contract.ArchiveRegistrar = (*Repository)(nil)
+var _ contract.ReplayRequester = (*Repository)(nil)
+var _ contract.ActionReader = (*Repository)(nil)

worker-svr/internal/fullsync/runtime/runtime_test.go

@@ -0,0 +1,143 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type runtimeRedisStub struct {
+	responses []*gvar.Var
+}
+
+func (s *runtimeRedisStub) Do(context.Context, string, ...any) (*gvar.Var, error) {
+	if len(s.responses) == 0 {
+		return gvar.New(nil), nil
+	}
+	response := s.responses[0]
+	s.responses = s.responses[1:]
+	return response, nil
+}
+
+func (s *runtimeRedisStub) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	return gvar.New(int64(1)), nil
+}
+
+func newRuntimeRepository(t *testing.T, responses ...*gvar.Var) *Repository {
+	t.Helper()
+	store, err := zset.New(&runtimeRedisStub{responses: responses}, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return repository
+}
+
+func TestListArchiveWorkParsesNestedRedisScanResult(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"run-1", "17", "run-2", "18"},
+	}))
+
+	page, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 2 {
+		t.Fatalf("archive work count = %d, want 2", len(page.Items))
+	}
+	if page.Items[0].RunID != "run-1" || page.Items[0].Version != "17" {
+		t.Fatalf("first archive work = %+v", page.Items[0])
+	}
+	if page.Items[1].RunID != "run-2" || page.Items[1].Version != "18" {
+		t.Fatalf("second archive work = %+v", page.Items[1])
+	}
+}
+
+func TestGetRoundStatsReadsRunHashJSON(t *testing.T) {
+	raw, err := json.Marshal([]zset.RoundStats{{RunID: "run-1", Round: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"runId", "run-1", "status", "completed", "roundStats", string(raw),
+	}))
+
+	got, err := repository.GetRoundStats(context.Background(), "run-1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Round != 2 || got[0].SuccessCount != 2 || got[0].FailedCount != 1 {
+		t.Fatalf("unexpected round stats: %+v", got)
+	}
+}
+
+func TestValidateRoundAccountingRejectsUnbalancedRound(t *testing.T) {
+	err := validateRoundAccounting(contract.RunMeta{RunID: "run-1", SuccessCount: 1, FailedCount: 0}, []contract.RoundStats{{RunID: "run-1", Round: 1, EnqueuedCount: 2, SuccessCount: 1}})
+	if err == nil {
+		t.Fatal("expected unbalanced round error")
+	}
+}
+
+func TestListArchiveWorkHandlesEmptyRedisScanResult(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{"0", []any{}}))
+
+	page, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 0 {
+		t.Fatalf("archive work = %+v, want empty", page.Items)
+	}
+}
+
+func TestListFailuresParsesNestedRedisScanResult(t *testing.T) {
+	failedAt := time.Date(2026, 9, 21, 6, 0, 0, 0, time.UTC)
+	record := zset.FailureRecord{
+		TaskID:       "task-1",
+		CardID:       42,
+		FailureType:  "business",
+		Message:      "upstream rejected request",
+		Round:        3,
+		LastFailedAt: failedAt,
+	}
+	payload, err := json.Marshal(record)
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"task-1", string(payload)},
+	}))
+
+	page, err := repository.ListFailures(context.Background(), "run-1", contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 1 {
+		t.Fatalf("failure count = %d, want 1", len(page.Items))
+	}
+	if page.Items[0].TaskID != "task-1" || page.Items[0].CardID != 42 || page.Items[0].Round != 3 {
+		t.Fatalf("failure = %+v", page.Items[0])
+	}
+}
+
+func TestListArchiveWorkRejectsOddRedisScanEntries(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"run-without-score"},
+	}))
+
+	if _, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10}); err == nil {
+		t.Fatal("ListArchiveWork() error = nil, want malformed response error")
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay.go

@@ -0,0 +1,474 @@
+package runtime
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+// SimpleReplay hands every source failure to a separate Run. The source hash
+// is the checkpoint: a record is removed only after the target's task-state
+// confirms that EnqueueRunBatch accepted it. No cursor is retained across
+// mutations of the source hash.
+//
+// The target's retry-round budget is taken from the caller when supplied and
+// falls back to the configured default otherwise. It is persisted on the
+// source reservation because a worker restart re-enters through reconcile(),
+// which no longer has the caller's arguments.
+type SimpleReplay struct {
+	stores []*zset.Store
+	redis  zset.RedisClient
+	// defaultMaxRounds applies when a replay request does not specify one.
+	defaultMaxRounds int
+}
+
+// Replay round bounds. MaxReplayRounds is a server-side guard, not a capacity
+// limit: the observed chain of 10-09 needed 74 rounds to reach 98% recovery,
+// so a low ceiling would cut off a legitimate convergence path.
+//
+// DefaultReplayMaxRounds is only the fallback for a request that omits a
+// budget; callers may ask for more, up to MaxReplayRounds.
+const (
+	DefaultReplayMaxRounds = 3
+	MaxReplayRounds        = 100
+	legacyReplayMaxRounds  = 1
+)
+
+type ReplayReceipt struct {
+	ReplayRunID string `json:"replayRunId"`
+	Status      string `json:"status"`
+}
+
+type ReplayStatus struct {
+	ReplayRunID string `json:"replayRunId"`
+	SourceRunID string `json:"sourceRunId"`
+	Status      string `json:"status"`
+	Remaining   int64  `json:"remaining"`
+	Enqueued    int64  `json:"enqueued"`
+	RunStatus   string `json:"runStatus,omitempty"`
+	Error       string `json:"error,omitempty"`
+}
+
+var ErrNoFailures = errors.New("run has no failures to replay")
+
+func NewSimpleReplay(stores ...*zset.Store) (*SimpleReplay, error) {
+	return NewSimpleReplayWithDefault(DefaultReplayMaxRounds, stores...)
+}
+
+// NewSimpleReplayWithDefault builds a replay service whose unspecified replay
+// requests inherit defaultMaxRounds. A non-positive value falls back to
+// DefaultReplayMaxRounds so a missing config key cannot disable the budget.
+func NewSimpleReplayWithDefault(defaultMaxRounds int, stores ...*zset.Store) (*SimpleReplay, error) {
+	if len(stores) == 0 || stores[0] == nil {
+		return nil, fmt.Errorf("replay requires Redis stores")
+	}
+	for _, store := range stores {
+		if store == nil {
+			return nil, fmt.Errorf("replay store is nil")
+		}
+	}
+	if defaultMaxRounds <= 0 {
+		defaultMaxRounds = DefaultReplayMaxRounds
+	}
+	if defaultMaxRounds > MaxReplayRounds {
+		defaultMaxRounds = MaxReplayRounds
+	}
+	return &SimpleReplay{stores: stores, redis: stores[0].RedisClient(), defaultMaxRounds: defaultMaxRounds}, nil
+}
+
+func (r *SimpleReplay) find(ctx context.Context, id string) (*zset.Store, zset.RunMeta, error) {
+	if err := contract.ValidateID(id); err != nil || len(id) > 128 {
+		return nil, zset.RunMeta{}, contract.ErrInvalidArgument
+	}
+	var found *zset.Store
+	var meta zset.RunMeta
+	for _, store := range r.stores {
+		key, _ := store.RunMetaKey(id)
+		v, err := r.redis.Do(ctx, "EXISTS", key)
+		if err != nil {
+			return nil, meta, err
+		}
+		if v.Int() == 0 {
+			continue
+		}
+		if found != nil {
+			return nil, meta, contract.ErrConflict
+		}
+		found = store
+		meta, err = store.GetRunMeta(ctx, id)
+		if err != nil {
+			return nil, meta, err
+		}
+	}
+	if found == nil {
+		return nil, meta, contract.ErrNotFound
+	}
+	return found, meta, nil
+}
+
+// Request accepts a replay of the source run's failures. maxRounds == 0 means
+// "use the configured default"; requests above MaxReplayRounds are rejected.
+func (r *SimpleReplay) Request(ctx context.Context, sourceID, requestedBy string, maxRounds int) (ReplayReceipt, error) {
+	if len(requestedBy) == 0 || len(requestedBy) > 64 {
+		return ReplayReceipt{}, contract.ErrInvalidArgument
+	}
+	if maxRounds < 0 || maxRounds > MaxReplayRounds {
+		return ReplayReceipt{}, contract.ErrInvalidArgument
+	}
+	if maxRounds == 0 {
+		maxRounds = r.defaultMaxRounds
+	}
+	store, _, err := r.find(ctx, sourceID)
+	if err != nil {
+		return ReplayReceipt{}, err
+	}
+	metaKey, _ := store.RunMetaKey(sourceID)
+	failedKey, _ := store.FailedKey(sourceID)
+	var random [16]byte
+	if _, err := rand.Read(random[:]); err != nil {
+		return ReplayReceipt{}, err
+	}
+	targetID := "replay-" + hex.EncodeToString(random[:])
+	// An index written before the reservation makes the pending source
+	// discoverable even if the process stops immediately after the Lua call.
+	if _, err = r.redis.Do(ctx, "SADD", store.Prefix()+"replay:pending", sourceID); err != nil {
+		return ReplayReceipt{}, err
+	}
+	v, err := r.redis.Eval(ctx, reserveReplay, 2, []string{metaKey, failedKey}, []any{targetID, requestedBy, maxRounds})
+	if err != nil {
+		return ReplayReceipt{}, err
+	}
+	result := v.String()
+	// An unsuccessful reservation leaves an inert discovery entry. Never
+	// remove it here: a concurrent caller might have reserved the source
+	// after this script returned, and that reservation needs recovery.
+	switch result {
+	case "missing":
+		return ReplayReceipt{}, contract.ErrNotFound
+	case "running":
+		return ReplayReceipt{}, contract.ErrConflict
+	case "empty":
+		return ReplayReceipt{}, ErrNoFailures
+	}
+	if !strings.HasPrefix(result, "replay-") {
+		return ReplayReceipt{}, fmt.Errorf("unexpected replay reservation result")
+	}
+	go r.process(store, sourceID)
+	return ReplayReceipt{ReplayRunID: result, Status: "accepted"}, nil
+}
+
+// Start reconciles pending sources after restarts and missed goroutine wakes.
+// Every process may run it: the per-source Redis lease elects one writer.
+func (r *SimpleReplay) Start(ctx context.Context) {
+	go func() {
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		for {
+			r.reconcile(ctx)
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+}
+
+func (r *SimpleReplay) reconcile(ctx context.Context) {
+	for _, store := range r.stores {
+		v, err := r.redis.Do(ctx, "SMEMBERS", store.Prefix()+"replay:pending")
+		if err != nil {
+			continue
+		}
+		for _, id := range v.Strings() {
+			if ctx.Err() != nil {
+				return
+			}
+			key, keyErr := store.RunMetaKey(id)
+			if keyErr != nil {
+				continue
+			}
+			reservation, getErr := r.redis.Do(ctx, "HGET", key, "replayTargetId")
+			// SADD happens before the reservation so even a crash during
+			// acceptance cannot lose work. Unreserved entries are inert.
+			if getErr != nil || reservation == nil || reservation.String() == "" {
+				continue
+			}
+			state, stateErr := r.redis.Do(ctx, "HGET", key, "replayState")
+			if stateErr != nil || state == nil || state.String() == "needs_attention" {
+				continue
+			}
+			go r.process(store, id)
+		}
+	}
+}
+
+func (r *SimpleReplay) Status(ctx context.Context, targetID string) (ReplayStatus, error) {
+	store, target, err := r.find(ctx, targetID)
+	if err != nil {
+		// The target may not exist yet: locate its durable source reservation.
+		if !errors.Is(err, contract.ErrNotFound) {
+			return ReplayStatus{}, err
+		}
+		for _, candidate := range r.stores {
+			v, e := r.redis.Do(ctx, "SMEMBERS", candidate.Prefix()+"replay:pending")
+			if e != nil {
+				return ReplayStatus{}, e
+			}
+			for _, sourceID := range v.Strings() {
+				key, _ := candidate.RunMetaKey(sourceID)
+				value, e := r.redis.Do(ctx, "HGET", key, "replayTargetId")
+				if e == nil && value != nil && value.String() == targetID {
+					store = candidate
+					target.SourceRunID = sourceID
+					break
+				}
+			}
+		}
+		if store == nil {
+			return ReplayStatus{}, contract.ErrNotFound
+		}
+	}
+	if target.SourceRunID == "" {
+		return ReplayStatus{}, contract.ErrNotFound
+	}
+	sourceKey, _ := store.RunMetaKey(target.SourceRunID)
+	failureKey, _ := store.FailedKey(target.SourceRunID)
+	values, err := r.redis.Do(ctx, "HMGET", sourceKey, "replayTargetId", "replayState", "replayError", "replayEnqueued")
+	if err != nil {
+		return ReplayStatus{}, err
+	}
+	fields := values.Vars()
+	if len(fields) != 4 || fields[0].String() != targetID {
+		return ReplayStatus{}, contract.ErrNotFound
+	}
+	remaining, err := r.redis.Do(ctx, "HLEN", failureKey)
+	if err != nil {
+		return ReplayStatus{}, err
+	}
+	return ReplayStatus{ReplayRunID: targetID, SourceRunID: target.SourceRunID, Status: fields[1].String(), Error: fields[2].String(), Enqueued: fields[3].Int64(), Remaining: remaining.Int64(), RunStatus: target.Status}, nil
+}
+
+func (r *SimpleReplay) process(store *zset.Store, sourceID string) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
+	defer cancel()
+	// Keep one process per source; batches are bounded so a short lease is
+	// renewed before it expires. A lost lease stops cleanup at the next batch.
+	lockKey := store.Prefix() + "{" + sourceID + "}:replay-lock"
+	var token [16]byte
+	if _, err := rand.Read(token[:]); err != nil {
+		return
+	}
+	owner := hex.EncodeToString(token[:])
+	locked, err := store.AcquireLock(ctx, lockKey, owner, time.Minute)
+	if err != nil || !locked {
+		return
+	}
+	defer func() { _, _ = store.ReleaseLock(context.Background(), lockKey, owner) }()
+	if err := r.handoff(ctx, store, sourceID, lockKey, owner); err != nil {
+		// The time slice ended, not the replay: the reconciler will resume
+		// from remaining source records under a new lease.
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+			return
+		}
+		sourceKey, _ := store.RunMetaKey(sourceID)
+		// A stale owner must not overwrite a newer worker's status.
+		_, _ = r.redis.Eval(context.Background(), replayAttention, 2, []string{sourceKey, lockKey}, []any{owner})
+	}
+}
+
+func (r *SimpleReplay) handoff(ctx context.Context, store *zset.Store, sourceID, lockKey, owner string) error {
+	sourceKey, _ := store.RunMetaKey(sourceID)
+	failedKey, _ := store.FailedKey(sourceID)
+	meta, err := store.GetRunMeta(ctx, sourceID)
+	if err != nil {
+		return err
+	}
+	v, err := r.redis.Do(ctx, "HGET", sourceKey, "replayTargetId")
+	if err != nil {
+		return err
+	}
+	if v == nil || v.String() == "" {
+		return fmt.Errorf("replay reservation missing")
+	}
+	targetID := v.String()
+	// New reservations persist their budget. Reservations made before this
+	// field existed used one round; a restart must not apply today's default
+	// to those already accepted runs.
+	storedMaxRounds, err := r.redis.Do(ctx, "HGET", sourceKey, "replayMaxRound")
+	if err != nil {
+		return err
+	}
+	maxRounds := legacyReplayMaxRounds
+	if storedMaxRounds != nil && storedMaxRounds.String() != "" {
+		maxRounds = storedMaxRounds.Int()
+	}
+	if maxRounds <= 0 {
+		return fmt.Errorf("replay reservation has invalid round budget")
+	}
+	if maxRounds > MaxReplayRounds {
+		return fmt.Errorf("replay reservation exceeds round budget limit")
+	}
+	// CreateRun is idempotent for the main hash but writes extension fields
+	// again. Read and validate an existing target BEFORE CreateRun, otherwise
+	// recovery could silently replace its accepted maxRound.
+	existing, err := store.GetRunMeta(ctx, targetID)
+	if err != nil && !errors.Is(err, zset.ErrRunNotFound) {
+		return err
+	}
+	if err == nil {
+		if storedMaxRounds == nil || storedMaxRounds.String() == "" {
+			maxRounds = existing.MaxRound
+		}
+		if maxRounds <= 0 || maxRounds > MaxReplayRounds || existing.SourceRunID != sourceID || existing.SyncType != "replay" || existing.MaxRound != maxRounds || existing.Operation != meta.Operation || existing.QueryMonth != meta.QueryMonth {
+			return fmt.Errorf("replay target metadata is inconsistent")
+		}
+	}
+	target := zset.RunMeta{RunID: targetID, Operation: meta.Operation, QueryMonth: meta.QueryMonth, SyncType: "replay", SourceRunID: sourceID, ReplayCycle: meta.ReplayCycle + 1, TriggerSource: "replay", MaxRound: maxRounds}
+	if err := store.CreateRun(ctx, target); err != nil {
+		return err
+	}
+	actual, err := store.GetRunMeta(ctx, targetID)
+	if err != nil || actual.SourceRunID != sourceID || actual.SyncType != "replay" || actual.MaxRound != maxRounds || actual.Operation != meta.Operation || actual.QueryMonth != meta.QueryMonth {
+		return fmt.Errorf("replay target metadata is inconsistent")
+	}
+	// The target task-state is the admission receipt during recovery. It may
+	// not expire while source failures are still being transferred.
+	targetKey, _ := store.RunMetaKey(targetID)
+	stateKey, _ := store.TaskStateKey(targetID)
+	if !actual.ProducerDone {
+		if _, err := r.redis.Do(ctx, "PERSIST", targetKey); err != nil {
+			return err
+		}
+		if _, err := r.redis.Do(ctx, "PERSIST", stateKey); err != nil {
+			return err
+		}
+	}
+	_, err = r.redis.Do(ctx, "HSET", sourceKey, "replayState", "processing", "replayError", "")
+	if err != nil {
+		return err
+	}
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		valid, err := store.RenewLock(ctx, lockKey, owner, time.Minute)
+		if err != nil || !valid {
+			return fmt.Errorf("replay lease lost: %w", err)
+		}
+		// HSCAN is not a snapshot. Complete a bounded cursor pass, then
+		// recheck HLEN and repeat from zero if deletion changed the scan.
+		var cursor uint64
+		for {
+			v, e := r.redis.Do(ctx, "HSCAN", failedKey, cursor, "COUNT", 100)
+			if e != nil {
+				return e
+			}
+			var scanPairs []*gvar.Var
+			cursor, scanPairs, e = parseScanResult(v)
+			if e != nil {
+				return e
+			}
+			if len(scanPairs) > 0 {
+				for i := 0; i < len(scanPairs); i += 2 {
+					valid, renewErr := store.RenewLock(ctx, lockKey, owner, time.Minute)
+					if renewErr != nil || !valid {
+						return fmt.Errorf("replay lease lost")
+					}
+					field, raw := scanPairs[i].String(), scanPairs[i+1].String()
+					var rec zset.FailureRecord
+					if json.Unmarshal([]byte(raw), &rec) != nil || rec.TaskID == "" || rec.CardID <= 0 || rec.ICCID == "" || rec.ChanKey == "" || rec.TempKey == "" || field != rec.TaskID || (rec.LatestRunID != "" && rec.LatestRunID != sourceID) {
+						return fmt.Errorf("source failure record cannot be replayed")
+					}
+					operation := rec.Operation
+					if operation == "" {
+						operation = meta.Operation
+					}
+					// Source task identity (not just card ID) prevents collisions
+					// between distinct failures for the same card.
+					taskID := targetID + ":" + rec.TaskID
+					queryMonth := rec.QueryMonth
+					if queryMonth == "" {
+						queryMonth = meta.QueryMonth
+					}
+					task := zset.Task{RunID: targetID, TaskID: taskID, Operation: operation, QueryMonth: queryMonth, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI, ChanKey: rec.ChanKey, TempKey: rec.TempKey, PackageCode: rec.PackageCode}
+					if _, e = store.EnqueueRunBatch(ctx, targetID, rec.ChanKey, []zset.Task{task}, time.Now()); e != nil {
+						return e
+					}
+					confirmed, e := r.redis.Do(ctx, "HEXISTS", stateKey, taskID)
+					if e != nil || confirmed == nil || confirmed.Int() != 1 {
+						return fmt.Errorf("target enqueue unconfirmed: %w", e)
+					}
+					removed, e := r.redis.Eval(ctx, cleanupReplayFailure, 3, []string{sourceKey, failedKey, lockKey}, []any{field, raw, targetID, owner})
+					if e != nil {
+						return e
+					}
+					if removed.Int() == -1 {
+						return fmt.Errorf("source failure changed during replay")
+					}
+				}
+			}
+			if cursor == 0 {
+				break
+			}
+		}
+		remaining, err := r.redis.Do(ctx, "HLEN", failedKey)
+		if err != nil {
+			return err
+		}
+		if remaining.Int64() == 0 {
+			break
+		}
+	}
+	// Persist producerDone only after every source record was confirmed.
+	if err := store.MarkProducerDone(ctx, targetID); err != nil {
+		return err
+	}
+	if _, err := r.redis.Do(ctx, "HSET", sourceKey, "replayState", "completed", "replayError", ""); err != nil {
+		return err
+	}
+	// The source summary has already been archived. Keep the small source
+	// metadata for lookup but restore a bounded retention after handoff.
+	if _, err := r.redis.Do(ctx, "EXPIRE", sourceKey, 30*24*3600); err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "SREM", store.Prefix()+"replay:pending", sourceID)
+	return err
+}
+
+const replayAttention = `
+if redis.call('GET',KEYS[2])~=ARGV[1] then return 0 end
+redis.call('HSET',KEYS[1],'replayState','needs_attention','replayError','Replay handoff stopped; check Worker logs')
+return 1
+`
+
+const reserveReplay = `
+local id=redis.call('HGET',KEYS[1],'replayTargetId')
+if id then return id end
+local status=redis.call('HGET',KEYS[1],'status')
+if not status then return 'missing' end
+if status~='completed' and status~='completed_with_failures' then return 'running' end
+if redis.call('HLEN',KEYS[2])==0 then return 'empty' end
+redis.call('HSET',KEYS[1],'replayTargetId',ARGV[1],'replayState','accepted','replayRequestedBy',ARGV[2],'replayEnqueued',0,'replayMaxRound',ARGV[3])
+redis.call('PERSIST',KEYS[1]); redis.call('PERSIST',KEYS[2])
+return ARGV[1]
+`
+const cleanupReplayFailure = `
+if redis.call('HGET',KEYS[1],'replayTargetId')~=ARGV[3] or redis.call('GET',KEYS[3])~=ARGV[4] then return -1 end
+local value=redis.call('HGET',KEYS[2],ARGV[1])
+if not value then return 0 end
+if value~=ARGV[2] then return -1 end
+redis.call('HDEL',KEYS[2],ARGV[1]); redis.call('HINCRBY',KEYS[1],'replayEnqueued',1)
+redis.call('HSET',KEYS[1],'pendingFailureCount',redis.call('HLEN',KEYS[2]))
+return 1
+`

worker-svr/internal/fullsync/runtime/simple_replay_invalid_test.go

@@ -0,0 +1,59 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayRetainsInvalidFailureAndResumesSameTarget(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "sim-status:")
+	key, _ := store.RunMetaKey("source")
+	failed, _ := store.FailedKey("source")
+	_, _ = redis.Do(ctx, "HSET", key, "runId", "source", "status", "completed_with_failures", "producerDone", "1", "currentRound", "1", "failedCount", "2")
+	valid := zset.FailureRecord{TaskID: "a", CardID: 1, ICCID: "card", ChanKey: "ch", TempKey: "template"}
+	payload, _ := json.Marshal(valid)
+	_, _ = redis.Do(ctx, "HSET", failed, "a", string(payload), "bad", `{"taskId":"bad"}`)
+	svc, _ := NewSimpleReplay(store)
+	accepted, err := svc.Request(ctx, "source", "42", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		state, err := svc.Status(ctx, accepted.ReplayRunID)
+		if err == nil && state.Status == "needs_attention" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	state, _ := svc.Status(ctx, accepted.ReplayRunID)
+	if state.Status != "needs_attention" || state.Remaining < 1 {
+		t.Fatalf("unexpected status: %+v", state)
+	}
+	// No unconfirmed record may be deleted. Retry the SAME target after repair.
+	repaired := zset.FailureRecord{TaskID: "bad", CardID: 2, ICCID: "card2", ChanKey: "ch", TempKey: "template"}
+	payload, _ = json.Marshal(repaired)
+	_, _ = redis.Do(ctx, "HSET", failed, "bad", string(payload))
+	repeat, err := svc.Request(ctx, "source", "42", 0)
+	if err != nil || repeat.ReplayRunID != accepted.ReplayRunID {
+		t.Fatalf("repeat: %+v %v", repeat, err)
+	}
+	deadline = time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		state, _ = svc.Status(ctx, accepted.ReplayRunID)
+		if state.Status == "completed" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if state.Status != "completed" || state.Enqueued != 2 || state.Remaining != 0 {
+		t.Fatalf("resume: %+v", state)
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_max_rounds_test.go

@@ -0,0 +1,121 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+// seedReplaySource writes a completed source run with n failures into store.
+func seedReplaySource(t *testing.T, redis zset.RedisClient, store *zset.Store, name string, n int) {
+	t.Helper()
+	ctx := context.Background()
+	key, _ := store.RunMetaKey(name)
+	failed, _ := store.FailedKey(name)
+	if _, err := redis.Do(ctx, "HSET", key, "runId", name, "operation", "usage", "status", "completed_with_failures", "producerDone", "1", "currentRound", "1", "failedCount", fmt.Sprint(n)); err != nil {
+		t.Fatal(err)
+	}
+	for i := 1; i <= n; i++ {
+		rec := zset.FailureRecord{TaskID: fmt.Sprintf("task-%d", i), CardID: int64(i), ICCID: fmt.Sprintf("iccid-%d", i), ChanKey: "ch", TempKey: "template", Operation: "usage", FailureType: "mobile_busy"}
+		payload, _ := json.Marshal(rec)
+		if _, err := redis.Do(ctx, "HSET", failed, rec.TaskID, string(payload)); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// awaitTargetMaxRound waits for the handoff to create the target and returns
+// the persisted MaxRound of the replayed Run.
+func awaitTargetMaxRound(t *testing.T, svc *SimpleReplay, store *zset.Store, replayRunID string) int {
+	t.Helper()
+	ctx := context.Background()
+	deadline := time.Now().Add(4 * time.Second)
+	for time.Now().Before(deadline) {
+		meta, err := store.GetRunMeta(ctx, replayRunID)
+		if err == nil && meta.MaxRound > 0 && meta.ProducerDone {
+			return meta.MaxRound
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	meta, err := store.GetRunMeta(ctx, replayRunID)
+	t.Fatalf("target never settled: %+v %v", meta, err)
+	return 0
+}
+
+func TestReplayMaxRoundsPrefersRequestThenConfig(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+
+	// Use a default that differs from DefaultReplayMaxRounds so the assertion
+	// proves the injected config value is used, not the built-in constant.
+	svc, err := NewSimpleReplayWithDefault(5, store)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	seedReplaySource(t, redis, store, "flow:default-source", 2)
+	receipt, err := svc.Request(ctx, "flow:default-source", "42", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := awaitTargetMaxRound(t, svc, store, receipt.ReplayRunID); got != 5 {
+		t.Fatalf("unspecified request must inherit config default 5, got %d", got)
+	}
+
+	seedReplaySource(t, redis, store, "flow:override-source", 2)
+	override, err := svc.Request(ctx, "flow:override-source", "42", 7)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := awaitTargetMaxRound(t, svc, store, override.ReplayRunID); got != 7 {
+		t.Fatalf("explicit request must win over config, got %d", got)
+	}
+}
+
+func TestReplayMaxRoundsRejectsOutOfRange(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	svc, _ := NewSimpleReplay(store)
+	seedReplaySource(t, redis, store, "flow:source", 1)
+
+	for _, requested := range []int{-1, MaxReplayRounds + 1} {
+		if _, err := svc.Request(ctx, "flow:source", "42", requested); err != contract.ErrInvalidArgument {
+			t.Fatalf("maxRounds=%d must be rejected, got %v", requested, err)
+		}
+	}
+	// The rejected requests must not have reserved the source.
+	if got, err := redis.Do(ctx, "HGET", "flow:{flow:source}:meta", "replayTargetId"); err == nil && got != nil && got.String() != "" {
+		t.Fatalf("rejected request must not reserve the source: %v", got.String())
+	}
+}
+
+// A worker restart re-enters through reconcile() without caller arguments, so
+// the budget must survive on the reservation rather than come from the caller.
+func TestReplayMaxRoundsSurvivesRestartRecovery(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	seedReplaySource(t, redis, store, "flow:source", 2)
+
+	// Reserve with an explicit budget, then simulate a restart by building a
+	// fresh service whose configured default differs from the reservation.
+	first, _ := NewSimpleReplayWithDefault(1, store)
+	receipt, err := first.Request(ctx, "flow:source", "42", 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restarted, _ := NewSimpleReplayWithDefault(9, store)
+	restarted.reconcile(ctx)
+
+	if got := awaitTargetMaxRound(t, restarted, store, receipt.ReplayRunID); got != 5 {
+		t.Fatalf("recovery must use the persisted budget 5, not the new default 9: got %d", got)
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_package_test.go

@@ -0,0 +1,82 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+// Exercise the real task -> failure pool -> replayed task path for both
+// TDengine usage chains; a producer-only assertion misses lost package codes.
+func TestReplayDailyAndMonthlyFailurePoolPreservesPackageCode(t *testing.T) {
+	for _, tc := range []struct {
+		name, operation, period string
+	}{
+		{"daily", "previous_day_usage_full_sync", "20261009"},
+		{"monthly", "previous_month_usage_full_sync", "202609"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			redis := testutil.Redis(t)
+			ctx := context.Background()
+			store, err := zset.New(redis, tc.name+":")
+			if err != nil {
+				t.Fatal(err)
+			}
+			sourceID := tc.name + "-source"
+			if err := store.CreateRun(ctx, zset.RunMeta{RunID: sourceID, Operation: tc.operation, QueryMonth: tc.period}); err != nil {
+				t.Fatal(err)
+			}
+			task := zset.Task{TaskID: sourceID + ":card-42", RunID: sourceID, Operation: tc.operation, QueryMonth: tc.period, CardID: 42, ICCID: "89860042", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "PKG-42"}
+			if n, err := store.EnqueueRunBatch(ctx, sourceID, task.ChanKey, []zset.Task{task}, time.Now().Add(-time.Second)); err != nil || n != 1 {
+				t.Fatalf("enqueue source task: %d, %v", n, err)
+			}
+			if err := store.MarkProducerDone(ctx, sourceID); err != nil {
+				t.Fatal(err)
+			}
+			now := time.Now()
+			claimed, err := store.ClaimRun(ctx, sourceID, task.ChanKey, now, now.Add(time.Minute), 1)
+			if err != nil || len(claimed) != 1 {
+				t.Fatalf("claim source task: %d, %v", len(claimed), err)
+			}
+			if done, err := store.FinalizeRunTask(ctx, claimed[0], zset.TaskOutcome{ErrorClass: "business", Code: 500, Message: "retry by pool"}); err != nil || !done {
+				t.Fatalf("write failure pool: %v, %v", done, err)
+			}
+			failedKey, _ := store.FailedKey(sourceID)
+			value, err := redis.Do(ctx, "HGET", failedKey, task.TaskID)
+			if err != nil || value == nil {
+				t.Fatalf("failure record missing: %v", err)
+			}
+			var failure zset.FailureRecord
+			if err := json.Unmarshal([]byte(value.String()), &failure); err != nil || failure.PackageCode != task.PackageCode {
+				t.Fatalf("failure pool package code: %+v, %v", failure, err)
+			}
+			if done, err := store.CompleteRunIfIdle(ctx, sourceID); err != nil || !done {
+				t.Fatalf("complete failed source: %v, %v", done, err)
+			}
+			svc, err := NewSimpleReplay(store)
+			if err != nil {
+				t.Fatal(err)
+			}
+			receipt, err := svc.Request(ctx, sourceID, "42", 0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := awaitTargetMaxRound(t, svc, store, receipt.ReplayRunID); got != DefaultReplayMaxRounds {
+				t.Fatalf("unexpected replay budget: %d", got)
+			}
+			now = time.Now()
+			replayed, err := store.ClaimRun(ctx, receipt.ReplayRunID, task.ChanKey, now, now.Add(time.Minute), 1)
+			if err != nil || len(replayed) != 1 {
+				t.Fatalf("claim replayed task: %d, %v", len(replayed), err)
+			}
+			got := replayed[0].Task
+			if got.PackageCode != task.PackageCode || got.QueryMonth != tc.period || got.Operation != tc.operation {
+				t.Fatalf("replayed task lost package or period: %+v", got)
+			}
+		})
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_recovery_test.go

@@ -0,0 +1,64 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayRecoversEnqueuedButNotCleanedFailure(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	sourceKey, _ := store.RunMetaKey("source")
+	failedKey, _ := store.FailedKey("source")
+	_, _ = redis.Do(ctx, "HSET", sourceKey, "runId", "source", "status", "completed_with_failures", "producerDone", "1", "operation", "usage", "currentRound", "1", "failedCount", "1", "replayTargetId", "replay-existing", "replayState", "processing", "replayEnqueued", "0")
+	rec := zset.FailureRecord{TaskID: "source:usage:1", CardID: 1, ICCID: "card", ChanKey: "ch", TempKey: "template", Operation: "usage", LatestRunID: "source"}
+	raw, _ := json.Marshal(rec)
+	_, _ = redis.Do(ctx, "HSET", failedKey, rec.TaskID, string(raw))
+	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "replay-existing", Operation: "usage", SyncType: "replay", SourceRunID: "source", ReplayCycle: 1, MaxRound: 1}); err != nil {
+		t.Fatal(err)
+	}
+	if v, err := redis.Do(ctx, "HGET", sourceKey, "replayMaxRound"); err != nil || (v != nil && v.String() != "") {
+		t.Fatalf("legacy reservation unexpectedly has a stored budget: %v, %v", v, err)
+	}
+	task := zset.Task{RunID: "replay-existing", TaskID: "replay-existing:" + rec.TaskID, CardID: rec.CardID, ICCID: rec.ICCID, Operation: "usage", ChanKey: "ch", TempKey: "template"}
+	if count, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []zset.Task{task}, time.Now()); err != nil || count != 1 {
+		t.Fatalf("seed enqueue = %d, %v", count, err)
+	}
+	svc, _ := NewSimpleReplayWithDefault(9, store)
+	svc.process(store, "source")
+	meta, err := store.GetRunMeta(ctx, "replay-existing")
+	if err != nil || meta.ActiveCount != 1 || !meta.ProducerDone || meta.MaxRound != 1 {
+		t.Fatalf("legacy target budget or enqueue changed after recovery: %+v %v", meta, err)
+	}
+	state, err := svc.Status(ctx, "replay-existing")
+	if err != nil || state.Status != "completed" || state.Enqueued != 1 || state.Remaining != 0 {
+		t.Fatalf("recovery state: %+v %v", state, err)
+	}
+}
+
+func TestSimpleReplayRecoversLegacyReservationBeforeTargetCreation(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "flow:")
+	seedReplaySource(t, redis, store, "source", 1)
+	sourceKey, _ := store.RunMetaKey("source")
+	if _, err := redis.Do(ctx, "HSET", sourceKey, "replayTargetId", "replay-old-reservation", "replayState", "accepted", "replayEnqueued", "0"); err != nil {
+		t.Fatal(err)
+	}
+	svc, _ := NewSimpleReplayWithDefault(9, store)
+	svc.process(store, "source")
+	meta, err := store.GetRunMeta(ctx, "replay-old-reservation")
+	if err != nil || meta.MaxRound != 1 || !meta.ProducerDone || meta.ActiveCount != 1 {
+		t.Fatalf("legacy reservation must retain the old one-round budget: %+v %v", meta, err)
+	}
+	state, err := svc.Status(ctx, meta.RunID)
+	if err != nil || state.Status != "completed" || state.Enqueued != 1 || state.Remaining != 0 {
+		t.Fatalf("legacy reservation did not complete: %+v %v", state, err)
+	}
+}

worker-svr/internal/fullsync/runtime/simple_replay_test.go

@@ -0,0 +1,96 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayBothNamespacesAndRecovery(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	traffic, _ := zset.New(redis, "flow:")
+	status, _ := zset.New(redis, "sim-status:")
+	for _, store := range []*zset.Store{traffic, status} {
+		name := store.Prefix() + "source"
+		key, _ := store.RunMetaKey(name)
+		failed, _ := store.FailedKey(name)
+		_, err := redis.Do(ctx, "HSET", key, "runId", name, "operation", "operation", "status", "completed_with_failures", "producerDone", "1", "failedCount", "3", "currentRound", "1")
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 1; i <= 3; i++ {
+			rec := zset.FailureRecord{TaskID: fmt.Sprintf("task-%d", i), CardID: int64(i), ICCID: fmt.Sprintf("iccid-%d", i), ChanKey: "ch", TempKey: "template", Operation: "operation", FailureType: "business"}
+			payload, _ := json.Marshal(rec)
+			if _, err := redis.Do(ctx, "HSET", failed, rec.TaskID, string(payload)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	svc, _ := NewSimpleReplay(traffic, status)
+	for _, prefix := range []string{"flow:", "sim-status:"} {
+		source := prefix + "source"
+		receipt, err := svc.Request(ctx, source, "42", 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		repeat, err := svc.Request(ctx, source, "42", 0)
+		if err != nil || repeat.ReplayRunID != receipt.ReplayRunID {
+			t.Fatalf("repeat = %+v, %v", repeat, err)
+		}
+		// Recovery competes with the initial goroutine but must keep its ID.
+		svc.reconcile(ctx)
+		deadline := time.Now().Add(4 * time.Second)
+		for time.Now().Before(deadline) {
+			state, err := svc.Status(ctx, receipt.ReplayRunID)
+			if err == nil && state.Status == "completed" {
+				if state.Remaining != 0 || state.Enqueued != 3 {
+					t.Fatalf("state = %+v", state)
+				}
+				store := traffic
+				if prefix == "sim-status:" {
+					store = status
+				}
+				meta, err := store.GetRunMeta(ctx, receipt.ReplayRunID)
+				if err != nil || meta.MaxRound != DefaultReplayMaxRounds || meta.SourceRunID != source || !meta.ProducerDone || meta.ActiveCount != 3 {
+					t.Fatalf("target meta = %+v, %v", meta, err)
+				}
+				sourceMeta, err := store.GetRunMeta(ctx, source)
+				if err != nil || sourceMeta.FailedCount != 3 {
+					t.Fatalf("source meta = %+v, %v", sourceMeta, err)
+				}
+				break
+			}
+			time.Sleep(20 * time.Millisecond)
+		}
+		state, _ := svc.Status(ctx, receipt.ReplayRunID)
+		if state.Status != "completed" {
+			t.Fatalf("handoff timed out: %+v", state)
+		}
+	}
+}
+
+func TestSimpleReplayRejectsRunningAndEmpty(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := zset.New(redis, "flow:")
+	svc, _ := NewSimpleReplay(store)
+	ctx := context.Background()
+	if _, err := svc.Request(ctx, "missing", "1", 0); err != contract.ErrNotFound {
+		t.Fatal(err)
+	}
+	key, _ := store.RunMetaKey("source")
+	_, _ = redis.Do(ctx, "HSET", key, "status", "running")
+	if _, err := svc.Request(ctx, "source", "1", 0); err != contract.ErrConflict {
+		t.Fatal(err)
+	}
+	_, _ = redis.Do(ctx, "HSET", key, "status", "completed_with_failures")
+	if _, err := svc.Request(ctx, "source", "1", 0); err != ErrNoFailures {
+		t.Fatal(err)
+	}
+}