chore: 初始开源发布

iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。

- Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件）
- 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、
  库名 xiujie_admin → iotlcm、哨兵名占位
- 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异，
  不破坏「校验 MSISDN 格式」等测试断言）
- 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像
  digest 记录
- Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23），
  移除写死的 git 凭据
- README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0
  LICENSE，LF 行尾 + .gitattributes

已知：3 个测试包在源仓同一提交上同样失败（client/upstream、
service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。

internal/batch/batch.go

@@ -0,0 +1,152 @@
+package batch
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gogf/gf/v2/os/gtime"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
+)
+
+const (
+	Type3   = 3
+	Type4   = 4
+	Type11  = 11
+	Type12  = 12
+	Type40  = 40
+	Type41  = 41
+	Type101 = 101
+	Type102 = 102
+	Type103 = 103
+	Type104 = 104
+	Type105 = 105
+	Type106 = 106
+	Type107 = 107
+	Type108 = 108
+	Type109 = 109
+	Type110 = 110
+
+	AsyncStatusPending    = 1
+	AsyncStatusProcessing = 2
+	AsyncStatusSuccess    = 3
+	AsyncStatusFailed     = 4
+	RunResultRunning      = 1
+	RunResultSuccess      = 2
+	RunResultFailed       = 3
+
+	DetailStatusPending = 0
+	DetailStatusRunning = 1
+	DetailStatusSuccess = 2
+	DetailStatusFailed  = 3
+)
+
+type Message struct {
+	BatchNo       string `json:"batchNo"`
+	BatchType     int    `json:"batchType"`
+	FailureReason string `json:"failureReason,omitempty"`
+	Retryable     bool   `json:"retryable,omitempty"`
+}
+
+// RetryableError marks an executor failure that may succeed after delayed redelivery.
+// The batch remains in processing until retries are exhausted and finalized.
+type RetryableError struct {
+	Err error
+}
+
+func (e *RetryableError) Error() string {
+	if e == nil || e.Err == nil {
+		return "retryable batch execution error"
+	}
+	return e.Err.Error()
+}
+
+func (e *RetryableError) Unwrap() error {
+	if e == nil {
+		return nil
+	}
+	return e.Err
+}
+
+func Retryable(err error) error {
+	if err == nil {
+		return nil
+	}
+	if IsRetryable(err) {
+		return err
+	}
+	return &RetryableError{Err: err}
+}
+
+func IsRetryable(err error) bool {
+	var retryableErr *RetryableError
+	return errors.As(err, &retryableErr)
+}
+
+// FailureMarkError 表示执行已结束，但失败终态尚未成功写入数据库。
+// 消费者重投这类消息时只能补写失败终态，不能再次执行具体业务。
+type FailureMarkError struct {
+	Reason string
+	Err    error
+}
+
+func (e *FailureMarkError) Error() string {
+	return "mark batch failed after executor error: " + e.Reason + ": " + e.Err.Error()
+}
+
+func (e *FailureMarkError) Unwrap() error {
+	return e.Err
+}
+
+type Executor interface {
+	Execute(ctx context.Context, batchNo string) error
+}
+
+type ProcessingExecutor interface {
+	CanExecuteProcessing(ctx context.Context, main *entity.BatchMain) (bool, error)
+}
+
+// SelfClaimingProcessingExecutor marks executors that perform their own
+// processing-state CAS before executing a re-entry phase.
+type SelfClaimingProcessingExecutor interface {
+	ClaimsProcessingReentry() bool
+}
+
+type ExecutionFailureHandler interface {
+	HandleExecutionFailure(ctx context.Context, batchNo string, batchType int, reason string) error
+}
+
+type ProcessingClaimer interface {
+	ReclaimProcessing(ctx context.Context, batchNo string, batchType int, startedAt *gtime.Time) (bool, error)
+}
+
+type Dispatcher struct {
+	executors map[int]Executor
+}
+
+func NewDispatcher(executor Executor) *Dispatcher {
+	return &Dispatcher{executors: map[int]Executor{
+		Type103: executor,
+		Type104: executor,
+	}}
+}
+
+func (d *Dispatcher) Register(batchType int, executor Executor) {
+	if d == nil || executor == nil {
+		return
+	}
+	d.executors[batchType] = executor
+}
+
+func (d *Dispatcher) Executor(batchType int) (Executor, bool) {
+	if d == nil {
+		return nil, false
+	}
+	executor, ok := d.executors[batchType]
+	return executor, ok && executor != nil
+}
+
+func (d *Dispatcher) Has(batchType int) bool {
+	_, ok := d.Executor(batchType)
+	return ok
+}

internal/batch/batch_test.go

@@ -0,0 +1,516 @@
+package batch
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gtime"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
+)
+
+type executorStub struct {
+	calls   int
+	batchNo string
+	err     error
+}
+
+func (s *executorStub) Execute(_ context.Context, batchNo string) error {
+	s.calls++
+	s.batchNo = batchNo
+	return s.err
+}
+
+type processingExecutorStub struct {
+	executorStub
+	canCalls  int
+	main      *entity.BatchMain
+	can       bool
+	canErr    error
+	selfClaim bool
+}
+
+type blockingExecutorStub struct {
+	calls int
+}
+
+func (s *blockingExecutorStub) Execute(ctx context.Context, _ string) error {
+	s.calls++
+	<-ctx.Done()
+	return ctx.Err()
+}
+
+func (s *processingExecutorStub) CanExecuteProcessing(_ context.Context, main *entity.BatchMain) (bool, error) {
+	s.canCalls++
+	s.main = main
+	return s.can, s.canErr
+}
+
+func (s *processingExecutorStub) ClaimsProcessingReentry() bool {
+	return s.selfClaim
+}
+
+type repositoryStub struct {
+	mains          []*entity.BatchMain
+	findErr        error
+	markErr        error
+	claimed        bool
+	findCalls      int
+	markCalls      int
+	markBatchNo    string
+	markType       int
+	failErr        error
+	failCalls      int
+	failBatchNo    string
+	failType       int
+	failReason     string
+	successErr     error
+	successCalls   int
+	successBatchNo string
+	successType    int
+	successCount   int
+	successFails   int
+	successReason  string
+	reclaimErr     error
+	reclaimSet     bool
+	reclaimed      bool
+	reclaimCalls   int
+	reclaimBatchNo string
+	reclaimType    int
+}
+
+func (s *repositoryStub) FindMain(context.Context, string) (*entity.BatchMain, error) {
+	s.findCalls++
+	if s.findErr != nil {
+		return nil, s.findErr
+	}
+	if len(s.mains) == 0 {
+		return nil, nil
+	}
+	index := s.findCalls - 1
+	if index >= len(s.mains) {
+		index = len(s.mains) - 1
+	}
+	return s.mains[index], nil
+}
+
+func (s *repositoryStub) MarkProcessing(_ context.Context, batchNo string, batchType int) (bool, error) {
+	s.markCalls++
+	s.markBatchNo = batchNo
+	s.markType = batchType
+	return s.claimed, s.markErr
+}
+
+func (s *repositoryStub) MarkFailed(_ context.Context, batchNo string, batchType int, reason string) (bool, error) {
+	s.failCalls++
+	s.failBatchNo = batchNo
+	s.failType = batchType
+	s.failReason = reason
+	return s.failErr == nil, s.failErr
+}
+
+func (s *repositoryStub) MarkSuccess(_ context.Context, batchNo string, batchType, success, fail int, reason string) (bool, error) {
+	s.successCalls++
+	s.successBatchNo = batchNo
+	s.successType = batchType
+	s.successCount = success
+	s.successFails = fail
+	s.successReason = reason
+	return s.successErr == nil, s.successErr
+}
+
+func (s *repositoryStub) ReclaimProcessing(_ context.Context, batchNo string, batchType int, _ *gtime.Time) (bool, error) {
+	s.reclaimCalls++
+	s.reclaimBatchNo = batchNo
+	s.reclaimType = batchType
+	if !s.reclaimSet {
+		return s.reclaimErr == nil, s.reclaimErr
+	}
+	return s.reclaimed, s.reclaimErr
+}
+
+func TestMessageJSONContract(t *testing.T) {
+	body, err := json.Marshal(Message{BatchNo: "BATCH-1", BatchType: Type103})
+	if err != nil {
+		t.Fatalf("marshal message: %v", err)
+	}
+	const expected = `{"batchNo":"BATCH-1","batchType":103}`
+	if string(body) != expected {
+		t.Fatalf("message JSON = %s, want %s", body, expected)
+	}
+}
+
+func TestDispatcherSupportsLifecycleTypesAndExplicitRegistration(t *testing.T) {
+	executor := &executorStub{}
+	dispatcher := NewDispatcher(executor)
+	for _, batchType := range []int{Type103, Type104} {
+		actual, ok := dispatcher.Executor(batchType)
+		if !ok || actual != executor {
+			t.Fatalf("batch type %d was not registered with shared executor", batchType)
+		}
+	}
+	if _, ok := dispatcher.Executor(102); ok {
+		t.Fatal("unexpected executor for unsupported batch type")
+	}
+
+	inbound := &executorStub{}
+	dispatcher.Register(1, inbound)
+	actual, ok := dispatcher.Executor(1)
+	if !ok || actual != inbound {
+		t.Fatal("registered inbound executor was not returned")
+	}
+
+	dataExecutor := &executorStub{}
+	dispatcher.Register(Type107, dataExecutor)
+	dispatcher.Register(Type109, dataExecutor)
+	dispatcher.Register(Type110, dataExecutor)
+	for _, batchType := range []int{Type107, Type109, Type110} {
+		actual, ok = dispatcher.Executor(batchType)
+		if !ok || actual != dataExecutor {
+			t.Fatalf("batch type %d was not registered with the data service executor", batchType)
+		}
+	}
+}
+
+func TestRuntimeAcknowledgesInvalidAndUnclaimableMessages(t *testing.T) {
+	g.Log().SetWriter(io.Discard)
+	tests := []struct {
+		name       string
+		message    Message
+		mains      []*entity.BatchMain
+		claimed    bool
+		wantFind   int
+		wantMarked int
+	}{
+		{name: "empty batch number", message: Message{BatchType: Type103}},
+		{name: "blank batch number", message: Message{BatchNo: "   ", BatchType: Type103}},
+		{name: "unsupported type", message: Message{BatchNo: "B-1", BatchType: 102}},
+		{name: "not found", message: Message{BatchNo: "B-1", BatchType: Type103}, wantFind: 1},
+		{name: "type mismatch", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}}, wantFind: 1},
+		{name: "processing", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusProcessing}}, wantFind: 1},
+		{name: "terminal", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: 3}}, wantFind: 1},
+		{name: "CAS lost", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusPending}, {BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusProcessing}}, wantFind: 2, wantMarked: 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			repository := &repositoryStub{mains: tt.mains, claimed: tt.claimed}
+			executor := &executorStub{}
+			runtime := NewRuntime(repository, NewDispatcher(executor))
+			if err := runtime.Handle(context.Background(), tt.message); err != nil {
+				t.Fatalf("Handle() error = %v", err)
+			}
+			if repository.findCalls != tt.wantFind || repository.markCalls != tt.wantMarked {
+				t.Fatalf("repository calls: find=%d mark=%d, want find=%d mark=%d", repository.findCalls, repository.markCalls, tt.wantFind, tt.wantMarked)
+			}
+			if executor.calls != 0 {
+				t.Fatalf("executor calls = %d, want 0", executor.calls)
+			}
+		})
+	}
+}
+
+func TestRuntimeExecutesRegisteredDataServiceBatch(t *testing.T) {
+	repository := &repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "B-109", BatchType: Type109, BatchAsyncStatus: AsyncStatusPending}},
+		claimed: true,
+	}
+	dataExecutor := &executorStub{}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(Type109, dataExecutor)
+	dispatcher.Register(Type110, dataExecutor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-109", BatchType: Type109}); err != nil {
+		t.Fatalf("Handle() error = %v", err)
+	}
+	if dataExecutor.calls != 1 || dataExecutor.batchNo != "B-109" {
+		t.Fatalf("data executor was not invoked: %+v", dataExecutor)
+	}
+}
+
+func TestRuntimeExecutesTrafficCutoffConfigWithDedicatedExecutor(t *testing.T) {
+	repository := &repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "B-41", BatchType: Type41, BatchAsyncStatus: AsyncStatusPending}},
+		claimed: true,
+	}
+	dataExecutor := &executorStub{err: errors.New("data executor must not be called")}
+	trafficCutoffConfigExecutor := &executorStub{}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(Type107, dataExecutor)
+	dispatcher.Register(Type41, trafficCutoffConfigExecutor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-41", BatchType: Type41}); err != nil {
+		t.Fatalf("Handle() error = %v", err)
+	}
+	if trafficCutoffConfigExecutor.calls != 1 || trafficCutoffConfigExecutor.batchNo != "B-41" {
+		t.Fatalf("traffic cutoff config executor was not invoked: %+v", trafficCutoffConfigExecutor)
+	}
+	if dataExecutor.calls != 0 {
+		t.Fatalf("data executor calls = %d, want 0", dataExecutor.calls)
+	}
+}
+
+func TestRuntimeIgnoresFinalizedTrafficCutoffConfigDuplicateMessage(t *testing.T) {
+	repository := &repositoryStub{
+		mains: []*entity.BatchMain{{BatchNo: "B-41-DONE", BatchType: Type41, BatchAsyncStatus: AsyncStatusSuccess}},
+	}
+	trafficCutoffConfigExecutor := &executorStub{err: errors.New("traffic cutoff config executor must not be called")}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(Type41, trafficCutoffConfigExecutor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-41-DONE", BatchType: Type41}); err != nil {
+		t.Fatalf("Handle() error = %v", err)
+	}
+	if trafficCutoffConfigExecutor.calls != 0 {
+		t.Fatalf("traffic cutoff config executor calls = %d, want 0", trafficCutoffConfigExecutor.calls)
+	}
+}
+
+func TestRuntimeMarksExecutorFailureAsFailed(t *testing.T) {
+	expectedErr := errors.New("execute failed")
+	repository := &repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "B-104", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}},
+		claimed: true,
+	}
+	executor := &executorStub{err: expectedErr}
+	runtime := NewRuntime(repository, NewDispatcher(executor))
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-104", BatchType: Type104}); err != nil {
+		t.Fatalf("Handle() error = %v, want nil", err)
+	}
+	if repository.markCalls != 1 || repository.markBatchNo != "B-104" || repository.markType != Type104 {
+		t.Fatalf("unexpected claim call: %+v", repository)
+	}
+	if executor.calls != 1 || executor.batchNo != "B-104" {
+		t.Fatalf("unexpected executor call: %+v", executor)
+	}
+	if repository.failCalls != 1 || repository.failBatchNo != "B-104" || repository.failType != Type104 || repository.failReason != expectedErr.Error() {
+		t.Fatalf("unexpected failure mark call: %+v", repository)
+	}
+}
+
+func TestRuntimeReturnsRetryableExecutorFailureWithoutFinalizing(t *testing.T) {
+	sourceErr := errors.New("source file is temporarily unavailable")
+	repository := &repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "INB-RETRY", BatchType: 1, BatchAsyncStatus: AsyncStatusPending}},
+		claimed: true,
+	}
+	executor := &executorStub{err: Retryable(sourceErr)}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(1, executor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	err := runtime.Handle(context.Background(), Message{BatchNo: "INB-RETRY", BatchType: 1})
+	if !errors.Is(err, sourceErr) || !IsRetryable(err) {
+		t.Fatalf("Handle() error = %v, want retryable wrapper of %v", err, sourceErr)
+	}
+	if repository.failCalls != 0 {
+		t.Fatalf("MarkFailed calls = %d, want 0", repository.failCalls)
+	}
+}
+
+func TestRuntimeReclaimsRetryableProcessingBatch(t *testing.T) {
+	repository := &repositoryStub{
+		mains:      []*entity.BatchMain{{BatchNo: "INB-RETRY", BatchType: 1, BatchAsyncStatus: AsyncStatusProcessing}},
+		reclaimSet: true,
+		reclaimed:  true,
+	}
+	executor := &executorStub{}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(1, executor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "INB-RETRY", BatchType: 1, Retryable: true}); err != nil {
+		t.Fatalf("Handle() error = %v", err)
+	}
+	if repository.reclaimCalls != 1 || repository.reclaimBatchNo != "INB-RETRY" || repository.reclaimType != 1 {
+		t.Fatalf("unexpected reclaim call: %+v", repository)
+	}
+	if executor.calls != 1 {
+		t.Fatalf("executor calls = %d, want 1", executor.calls)
+	}
+}
+
+func TestRuntimeMarksTimeoutAsFailed(t *testing.T) {
+	repository := &repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "B-TIMEOUT", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}},
+		claimed: true,
+	}
+	executor := &blockingExecutorStub{}
+	runtime := NewRuntime(
+		repository,
+		NewDispatcher(executor),
+		WithExecutionTimeout(time.Millisecond),
+		WithFinalizationTimeout(time.Second),
+	)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-TIMEOUT", BatchType: Type104}); err != nil {
+		t.Fatalf("Handle() error = %v, want nil", err)
+	}
+	if executor.calls != 1 {
+		t.Fatalf("executor calls = %d, want 1", executor.calls)
+	}
+	if repository.failCalls != 1 || !strings.Contains(repository.failReason, "timed out after 1ms") {
+		t.Fatalf("unexpected timeout failure mark: %+v", repository)
+	}
+}
+
+func TestRuntimeReturnsFailureMarkErrors(t *testing.T) {
+	executeErr := errors.New("execute failed")
+	markErr := errors.New("mark failed")
+	repository := &repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "B-104", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}},
+		claimed: true,
+		failErr: markErr,
+	}
+	runtime := NewRuntime(repository, NewDispatcher(&executorStub{err: executeErr}))
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-104", BatchType: Type104}); !errors.Is(err, markErr) {
+		t.Fatalf("Handle() error = %v, want %v", err, markErr)
+	}
+}
+
+func TestRuntimeCompensatesFailureMarkWithoutReexecutingBatch(t *testing.T) {
+	markErr := errors.New("mark failed")
+	repository := &repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "B-RETRY", BatchType: Type104, BatchAsyncStatus: AsyncStatusProcessing}},
+		failErr: markErr,
+	}
+	executor := &executorStub{err: errors.New("must not execute again")}
+	runtime := NewRuntime(repository, NewDispatcher(executor))
+
+	err := runtime.Handle(context.Background(), Message{
+		BatchNo:       "B-RETRY",
+		BatchType:     Type104,
+		FailureReason: "original executor failure",
+	})
+	if !errors.Is(err, markErr) {
+		t.Fatalf("Handle() error = %v, want %v", err, markErr)
+	}
+	if executor.calls != 0 {
+		t.Fatalf("executor calls = %d, want 0 for failure finalization retry", executor.calls)
+	}
+	if repository.markCalls != 0 {
+		t.Fatalf("MarkProcessing calls = %d, want 0 for failure finalization retry", repository.markCalls)
+	}
+	if repository.failCalls != 1 || repository.failReason != "original executor failure" {
+		t.Fatalf("unexpected failure finalization call: %+v", repository)
+	}
+}
+
+func TestRuntimeAllowsSelfClaimingProcessingReEntryWithoutRefreshingStartTime(t *testing.T) {
+	repository := &repositoryStub{
+		mains: []*entity.BatchMain{{BatchNo: "RST-1", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
+	}
+	executor := &processingExecutorStub{can: true, selfClaim: true}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(Type105, executor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-1", BatchType: Type105}); err != nil {
+		t.Fatalf("Handle() error = %v", err)
+	}
+	if executor.canCalls != 1 || executor.main == nil || executor.main.BatchNo != "RST-1" {
+		t.Fatalf("processing check not called with main: %+v", executor)
+	}
+	if repository.markCalls != 0 {
+		t.Fatalf("MarkProcessing calls = %d, want 0", repository.markCalls)
+	}
+	if repository.reclaimCalls != 0 {
+		t.Fatalf("ReclaimProcessing calls = %d, want 0 for self-claiming executor", repository.reclaimCalls)
+	}
+	if executor.calls != 1 || executor.batchNo != "RST-1" {
+		t.Fatalf("executor calls = %+v", executor.executorStub)
+	}
+}
+
+func TestRuntimeMarksProcessingReEntryFailureAsFailed(t *testing.T) {
+	expectedErr := errors.New("resume failed")
+	repository := &repositoryStub{
+		mains: []*entity.BatchMain{{BatchNo: "RST-1", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
+	}
+	executor := &processingExecutorStub{can: true}
+	executor.err = expectedErr
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(Type105, executor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-1", BatchType: Type105}); err != nil {
+		t.Fatalf("Handle() error = %v, want nil", err)
+	}
+	if repository.markCalls != 0 {
+		t.Fatalf("MarkProcessing calls = %d, want 0", repository.markCalls)
+	}
+	if repository.failCalls != 1 || repository.failBatchNo != "RST-1" || repository.failType != Type105 || repository.failReason != expectedErr.Error() {
+		t.Fatalf("unexpected failure mark call: %+v", repository)
+	}
+}
+
+func TestRuntimeIgnoresProcessingWhenOptInExecutorDeclines(t *testing.T) {
+	repository := &repositoryStub{
+		mains: []*entity.BatchMain{{BatchNo: "RST-2", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
+	}
+	executor := &processingExecutorStub{can: false}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(Type105, executor)
+	runtime := NewRuntime(repository, dispatcher)
+
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-2", BatchType: Type105}); err != nil {
+		t.Fatalf("Handle() error = %v", err)
+	}
+	if executor.canCalls != 1 {
+		t.Fatalf("processing check calls = %d, want 1", executor.canCalls)
+	}
+	if executor.calls != 0 {
+		t.Fatalf("executor calls = %d, want 0", executor.calls)
+	}
+}
+
+func TestRuntimeReturnsDatabaseErrors(t *testing.T) {
+	findErr := errors.New("find failed")
+	runtime := NewRuntime(&repositoryStub{findErr: findErr}, NewDispatcher(&executorStub{}))
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-1", BatchType: Type103}); !errors.Is(err, findErr) {
+		t.Fatalf("find error = %v, want %v", err, findErr)
+	}
+
+	markErr := errors.New("mark failed")
+	runtime = NewRuntime(&repositoryStub{
+		mains:   []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusPending}},
+		markErr: markErr,
+	}, NewDispatcher(&executorStub{}))
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "B-1", BatchType: Type103}); !errors.Is(err, markErr) {
+		t.Fatalf("mark error = %v, want %v", err, markErr)
+	}
+
+	checkErr := errors.New("check failed")
+	processing := &processingExecutorStub{canErr: checkErr}
+	dispatcher := NewDispatcher(&executorStub{})
+	dispatcher.Register(Type105, processing)
+	runtime = NewRuntime(&repositoryStub{
+		mains: []*entity.BatchMain{{BatchNo: "RST-3", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
+	}, dispatcher)
+	if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-3", BatchType: Type105}); !errors.Is(err, checkErr) {
+		t.Fatalf("processing check error = %v, want %v", err, checkErr)
+	}
+}
+
+func TestMessageHasOnlyContractFields(t *testing.T) {
+	typeOfMessage := reflect.TypeOf(Message{})
+	if typeOfMessage.NumField() != 4 {
+		t.Fatalf("Message has %d fields, want 4", typeOfMessage.NumField())
+	}
+	if field, ok := typeOfMessage.FieldByName("FailureReason"); !ok || field.Tag.Get("json") != "failureReason,omitempty" {
+		t.Fatalf("FailureReason JSON contract is missing or incorrect")
+	}
+	if field, ok := typeOfMessage.FieldByName("Retryable"); !ok || field.Tag.Get("json") != "retryable,omitempty" {
+		t.Fatalf("Retryable JSON contract is missing or incorrect")
+	}
+}

internal/batch/runtime.go

@@ -0,0 +1,197 @@
+package batch
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
+)
+
+type Repository interface {
+	FindMain(ctx context.Context, batchNo string) (*entity.BatchMain, error)
+	MarkProcessing(ctx context.Context, batchNo string, batchType int) (bool, error)
+	MarkFailed(ctx context.Context, batchNo string, batchType int, reason string) (bool, error)
+	MarkSuccess(ctx context.Context, batchNo string, batchType int, success, fail int, reason string) (bool, error)
+}
+
+type Runtime struct {
+	repository          Repository
+	dispatcher          *Dispatcher
+	executionTimeout    time.Duration
+	finalizationTimeout time.Duration
+}
+
+func NewRuntime(repository Repository, dispatcher *Dispatcher, options ...RuntimeOption) *Runtime {
+	runtime := &Runtime{
+		repository:          repository,
+		dispatcher:          dispatcher,
+		executionTimeout:    0,
+		finalizationTimeout: 10 * time.Second,
+	}
+	for _, option := range options {
+		option(runtime)
+	}
+	return runtime
+}
+
+type RuntimeOption func(*Runtime)
+
+func WithExecutionTimeout(timeout time.Duration) RuntimeOption {
+	return func(runtime *Runtime) {
+		if timeout >= 0 {
+			runtime.executionTimeout = timeout
+		}
+	}
+}
+
+func WithFinalizationTimeout(timeout time.Duration) RuntimeOption {
+	return func(runtime *Runtime) {
+		if timeout > 0 {
+			runtime.finalizationTimeout = timeout
+		}
+	}
+}
+
+func (r *Runtime) Handle(ctx context.Context, message Message) error {
+	if strings.TrimSpace(message.BatchNo) == "" {
+		g.Log().Warning(ctx, "batch message dropped, empty batchNo")
+		return nil
+	}
+	executor, ok := r.dispatcher.Executor(message.BatchType)
+	if !ok {
+		g.Log().Warningf(ctx, "batch message dropped, unsupported batchType:%d, batchNo:%s", message.BatchType, message.BatchNo)
+		return nil
+	}
+
+	main, err := r.repository.FindMain(ctx, message.BatchNo)
+	if err != nil {
+		return gerror.Wrap(err, "load batch main failed")
+	}
+	if main == nil {
+		g.Log().Warningf(ctx, "batch message ignored, batch not found, batchNo:%s", message.BatchNo)
+		return nil
+	}
+	if main.BatchType != message.BatchType {
+		g.Log().Warningf(ctx, "batch message ignored, type mismatch, batchNo:%s, messageType:%d, storedType:%d", message.BatchNo, message.BatchType, main.BatchType)
+		return nil
+	}
+	if strings.TrimSpace(message.FailureReason) != "" {
+		return r.markExecutionFailed(ctx, message, executor, message.FailureReason)
+	}
+	if main.BatchAsyncStatus != AsyncStatusPending {
+		if main.BatchAsyncStatus == AsyncStatusProcessing {
+			if message.Retryable {
+				return r.reclaimAndExecute(ctx, message, executor, main)
+			}
+			if processingExecutor, ok := executor.(ProcessingExecutor); ok {
+				canExecute, checkErr := processingExecutor.CanExecuteProcessing(ctx, main)
+				if checkErr != nil {
+					return gerror.Wrap(checkErr, "check processing batch re-entry failed")
+				}
+				if canExecute {
+					if selfClaimer, selfClaims := executor.(SelfClaimingProcessingExecutor); selfClaims && selfClaimer.ClaimsProcessingReentry() {
+						return r.execute(ctx, message, executor)
+					}
+					return r.reclaimAndExecute(ctx, message, executor, main)
+				}
+			}
+		}
+		logUnclaimable(ctx, main)
+		return nil
+	}
+
+	claimed, err := r.repository.MarkProcessing(ctx, message.BatchNo, message.BatchType)
+	if err != nil {
+		return gerror.Wrap(err, "mark batch processing failed")
+	}
+	if !claimed {
+		current, findErr := r.repository.FindMain(ctx, message.BatchNo)
+		if findErr != nil {
+			return gerror.Wrap(findErr, "reload batch main after CAS failed")
+		}
+		if current == nil {
+			g.Log().Warningf(ctx, "batch CAS lost, batch not found, batchNo:%s", message.BatchNo)
+			return nil
+		}
+		if current.BatchType != message.BatchType {
+			g.Log().Warningf(ctx, "batch CAS lost, type mismatch, batchNo:%s, messageType:%d, storedType:%d", message.BatchNo, message.BatchType, current.BatchType)
+			return nil
+		}
+		logUnclaimable(ctx, current)
+		return nil
+	}
+
+	g.Log().Infof(ctx, "batch claimed, batchNo:%s, batchType:%d", message.BatchNo, message.BatchType)
+	return r.execute(ctx, message, executor)
+}
+
+func (r *Runtime) reclaimAndExecute(ctx context.Context, message Message, executor Executor, main *entity.BatchMain) error {
+	if claimer, claimSupported := r.repository.(ProcessingClaimer); claimSupported {
+		reclaimed, claimErr := claimer.ReclaimProcessing(ctx, message.BatchNo, message.BatchType, main.StartTime)
+		if claimErr != nil {
+			return gerror.Wrap(claimErr, "claim stale processing batch failed")
+		}
+		if !reclaimed {
+			g.Log().Infof(ctx, "batch re-entry claim lost, batchNo:%s", message.BatchNo)
+			return nil
+		}
+	}
+	return r.execute(ctx, message, executor)
+}
+
+func (r *Runtime) execute(ctx context.Context, message Message, executor Executor) error {
+	executionCtx := ctx
+	var cancel context.CancelFunc = func() {}
+	if r.executionTimeout > 0 {
+		executionCtx, cancel = context.WithTimeout(ctx, r.executionTimeout)
+	}
+	defer cancel()
+
+	if err := executor.Execute(executionCtx, message.BatchNo); err != nil {
+		if IsRetryable(err) {
+			return err
+		}
+		reason := err.Error()
+		if r.executionTimeout > 0 && executionCtx.Err() == context.DeadlineExceeded {
+			reason = gerror.Newf("batch execution timed out after %s", r.executionTimeout).Error()
+		}
+		return r.markExecutionFailed(ctx, message, executor, reason)
+	}
+	if r.executionTimeout > 0 && executionCtx.Err() == context.DeadlineExceeded {
+		return r.markExecutionFailed(ctx, message, executor, gerror.Newf("batch execution timed out after %s", r.executionTimeout).Error())
+	}
+	return nil
+}
+
+func (r *Runtime) markExecutionFailed(ctx context.Context, message Message, executor Executor, reason string) error {
+	finalizationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), r.finalizationTimeout)
+	defer cancel()
+
+	failed, markErr := r.repository.MarkFailed(finalizationCtx, message.BatchNo, message.BatchType, reason)
+	if markErr != nil {
+		return &FailureMarkError{Reason: reason, Err: markErr}
+	}
+	if handler, ok := executor.(ExecutionFailureHandler); ok {
+		if err := handler.HandleExecutionFailure(finalizationCtx, message.BatchNo, message.BatchType, reason); err != nil {
+			return &FailureMarkError{Reason: reason, Err: err}
+		}
+	}
+	if !failed {
+		g.Log().Warningf(ctx, "batch execution failed but status was already finalized, batchNo:%s, reason:%s", message.BatchNo, reason)
+		return nil
+	}
+	g.Log().Errorf(ctx, "batch execution failed and was marked failed, batchNo:%s, reason:%s", message.BatchNo, reason)
+	return nil
+}
+
+func logUnclaimable(ctx context.Context, main *entity.BatchMain) {
+	if main.BatchAsyncStatus == AsyncStatusProcessing {
+		g.Log().Infof(ctx, "batch message ignored, batch already processing, batchNo:%s", main.BatchNo)
+		return
+	}
+	g.Log().Infof(ctx, "batch message ignored, batch is terminal or not pending, batchNo:%s, status:%d", main.BatchNo, main.BatchAsyncStatus)
+}

internal/client/upstream/client.go

@@ -0,0 +1,241 @@
+package upstream
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/net/gclient"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+const inboundBasicInfoClientNameEnv = "UPSTREAM_INBOUND_BASIC_INFO_CLIENT_NAME"
+const upstreamProxyMaxTimeoutConfigKey = "upstream.proxy.maxTimeout"
+
+type Client struct {
+	currentUsageClientName     string
+	currentUsageTimeout        time.Duration
+	inboundBasicInfoClientName string
+	imeiClientName             string
+	imeiTimeout                time.Duration
+}
+
+type response[T any] struct {
+	Code    int                   `json:"code"`
+	Message string                `json:"message"`
+	Data    []model.SimBaseRes[T] `json:"data"`
+}
+
+func New() *Client {
+	return &Client{}
+}
+
+func (c *Client) QuerySimBasicInfo(ctx context.Context, input *model.SimBasicInfoInput) ([]model.SimBaseRes[model.SimBasicInfoItem], error) {
+	address, err := c.inboundBasicInfoAddress(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return requestAllWithAddress[model.SimBasicInfoItem](ctx, "/sim/query/sim-basic-info", input, len(input.SimKeys), address)
+}
+
+// NewWithCurrentUsageClient keeps the full-sync current-usage call independent
+// from the configured upstream.proxy endpoint.
+func NewWithCurrentUsageClient(address string, timeout ...time.Duration) *Client {
+	var currentUsageTimeout time.Duration
+	if len(timeout) > 0 {
+		currentUsageTimeout = timeout[0]
+	}
+	return &Client{currentUsageClientName: address, currentUsageTimeout: currentUsageTimeout}
+}
+
+func NewWithInboundBasicInfoClient(address string) *Client {
+	return &Client{inboundBasicInfoClientName: address}
+}
+
+// NewWithImeiClient pins the IMEI full-sync path to an explicit chan endpoint
+// and timeout. An empty address falls back to the shared upstream.proxy
+// configuration, which is only intended for local runs.
+func NewWithImeiClient(address string, timeout ...time.Duration) *Client {
+	var imeiTimeout time.Duration
+	if len(timeout) > 0 {
+		imTimeout := timeout[0]
+		if imTimeout > 0 {
+			imeiTimeout = imTimeout
+		}
+	}
+	return &Client{imeiClientName: address, imeiTimeout: imeiTimeout}
+}
+
+func (c *Client) inboundBasicInfoAddress(ctx context.Context) (string, error) {
+	if strings.TrimSpace(c.inboundBasicInfoClientName) != "" {
+		return strings.TrimSpace(c.inboundBasicInfoClientName), nil
+	}
+	if address := strings.TrimSpace(os.Getenv(inboundBasicInfoClientNameEnv)); address != "" {
+		return address, nil
+	}
+	config, ok, err := optionalUpstreamConfig(ctx, "upstream.gateway")
+	if err != nil {
+		return "", err
+	}
+	if !ok || strings.TrimSpace(config.Address) == "" {
+		return "", gerror.Newf("upstream gateway address is empty, configure upstream.gateway.address or %s", inboundBasicInfoClientNameEnv)
+	}
+	return strings.TrimSpace(config.Address), nil
+}
+
+// QuerySimDataUsage calls Chan's current-month real-time usage endpoint. The
+// response keeps one item per requested card so the caller can classify
+// per-card errors without reissuing successful cards.
+func (c *Client) QuerySimDataUsage(ctx context.Context, input *model.SimDataUsageInput) ([]model.SimBaseRes[model.SimDataUsageItem], error) {
+	return requestAllWithAddressTimeout[model.SimDataUsageItem](ctx, "/sim/query/sim-datausage", input, len(input.SimKeys), c.currentUsageClientName, c.currentUsageTimeout)
+}
+
+// QuerySimCardStatus queries the carrier status for the supplied cards. The
+// status queue currently sends one card per request, while retaining the
+// response shape needed for future batch support.
+func (c *Client) QuerySimCardStatus(ctx context.Context, input *model.SimCardStatusInput) ([]model.SimBaseRes[model.SimCardStatusItem], error) {
+	return requestAll[model.SimCardStatusItem](ctx, "/sim/query/sim-status", input, len(input.SimKeys))
+}
+
+// QuerySimImei queries the terminal IMEI for the supplied cards. The IMEI queue
+// sends one card per request.
+//
+// It deliberately uses requestAll rather than request: a per-item carrier
+// failure (unsupported template, missing value) must reach the handler as an
+// item with a non-zero Code, not be silently dropped from the slice.
+func (c *Client) QuerySimImei(ctx context.Context, input *model.SimImeiInput) ([]model.SimBaseRes[model.SimImeiItem], error) {
+	return requestAllWithAddressTimeout[model.SimImeiItem](ctx, "/sim/query/sim-imei", input, len(input.SimKeys), c.imeiClientName, c.imeiTimeout)
+}
+
+func (c *Client) QuerySimDatausageDaily(ctx context.Context, input *model.SimDataUsageDailyInput) ([]model.SimBaseRes[model.SimDataUsageDailyOutputItem], error) {
+	return requestAll[model.SimDataUsageDailyOutputItem](ctx, "/sim/query/sim-datausage-daily", input, len(input.SimKeys))
+}
+
+func (c *Client) QuerySimDatausageMonthly(ctx context.Context, input *model.SimDataUsageMonthlyInput) ([]model.SimBaseRes[model.SimDataUsageMonthlyOutputItem], error) {
+	return requestAll[model.SimDataUsageMonthlyOutputItem](ctx, "/sim/query/sim-datausage-monthly", input, len(input.SimKeys))
+}
+
+// ChangeSimStatus returns one result for every upstream item, including failed items.
+// Lifecycle batch execution needs those failures to be persisted to the matching detail.
+func (c *Client) ChangeSimStatus(ctx context.Context, input *model.SimCardStatusChangeInput) ([]model.SimBaseRes[model.SimCardStatusChangeOutputItem], error) {
+	return requestAll[model.SimCardStatusChangeOutputItem](ctx, "/sim/change/sim-status", input, len(input.SimKeys))
+}
+
+// ChangeSimApnFunction returns one result for every upstream item, including failed items.
+// Data service batch execution needs those failures to be persisted to the matching detail.
+func (c *Client) ChangeSimApnFunction(ctx context.Context, input *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+	return requestAll[model.SimCardApnChangeOutputItem](ctx, "/sim/change/sim-apn", input, len(input.SimKeys))
+}
+
+func request[T any](ctx context.Context, path string, input any, itemCount int) ([]model.SimBaseRes[T], error) {
+	items, err := requestAll[T](ctx, path, input, itemCount)
+	if err != nil {
+		return nil, err
+	}
+
+	successItems := make([]model.SimBaseRes[T], 0, len(items))
+	for _, item := range items {
+		if item.Code == 0 {
+			successItems = append(successItems, item)
+		}
+	}
+	return successItems, nil
+}
+
+func requestAll[T any](ctx context.Context, path string, input any, itemCount int) ([]model.SimBaseRes[T], error) {
+	return requestAllWithAddress[T](ctx, path, input, itemCount, "")
+}
+
+func requestAllWithAddress[T any](ctx context.Context, path string, input any, itemCount int, address string) ([]model.SimBaseRes[T], error) {
+	return requestAllWithAddressTimeout[T](ctx, path, input, itemCount, address, 0)
+}
+
+func requestAllWithAddressTimeout[T any](ctx context.Context, path string, input any, itemCount int, address string, timeoutOverride time.Duration) ([]model.SimBaseRes[T], error) {
+	config, err := upstreamConfig(ctx, "upstream.proxy")
+	if err != nil {
+		return nil, err
+	}
+	if address == "" {
+		address = config.Address
+	}
+	if address == "" {
+		return nil, gerror.New("upstream proxy address is empty")
+	}
+	if itemCount < 1 {
+		itemCount = 1
+	}
+
+	client := g.Client()
+	timeout := timeoutOverride
+	if timeout <= 0 {
+		timeout = config.Timeout
+	}
+	if timeout <= 0 {
+		timeout = 10 * time.Second
+	}
+	if itemCount > 1 {
+		timeout *= time.Duration(itemCount)
+	}
+	if maxTimeout := g.Cfg().MustGet(ctx, upstreamProxyMaxTimeoutConfigKey, 2*time.Minute).Duration(); maxTimeout > 0 && timeout > maxTimeout {
+		timeout = maxTimeout
+	}
+	setClientTimeout(client, timeout)
+	clientResponse, err := client.Post(ctx, fmt.Sprintf("http://%s%s", address, path), input)
+	if err != nil {
+		return nil, gerror.Wrap(err, "request upstream failed")
+	}
+	defer clientResponse.Close()
+
+	return decodeResponse[T](clientResponse.ReadAll())
+}
+
+func optionalUpstreamConfig(ctx context.Context, key string) (model.UpstreamChanConfig, bool, error) {
+	configValue, err := g.Cfg().Get(ctx, key)
+	if err != nil {
+		return model.UpstreamChanConfig{}, false, gerror.Wrapf(err, "read upstream configuration failed: %s", key)
+	}
+	if configValue == nil || configValue.IsNil() {
+		return model.UpstreamChanConfig{}, false, nil
+	}
+	config := model.UpstreamChanConfig{}
+	if err = configValue.Scan(&config); err != nil {
+		return model.UpstreamChanConfig{}, false, gerror.Wrapf(err, "parse upstream configuration failed: %s", key)
+	}
+	return config, true, nil
+}
+
+func upstreamConfig(ctx context.Context, key string) (model.UpstreamChanConfig, error) {
+	config, ok, err := optionalUpstreamConfig(ctx, key)
+	if err != nil {
+		return model.UpstreamChanConfig{}, err
+	}
+	if !ok {
+		return model.UpstreamChanConfig{}, gerror.Newf("upstream configuration is missing: %s", key)
+	}
+	return config, nil
+}
+
+func decodeResponse[T any](body []byte) ([]model.SimBaseRes[T], error) {
+	output := response[T]{}
+	if err := json.Unmarshal(body, &output); err != nil {
+		return nil, gerror.Wrap(err, "parse upstream response failed")
+	}
+	if output.Code != 0 {
+		return nil, gerror.New(output.Message)
+	}
+	return output.Data, nil
+}
+
+func setClientTimeout(client *gclient.Client, timeout time.Duration) {
+	client.SetTimeout(timeout)
+	if transport, ok := client.Transport.(*http.Transport); ok {
+		transport.ResponseHeaderTimeout = timeout
+	}
+}

internal/client/upstream/client_test.go

@@ -0,0 +1,133 @@
+package upstream
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func TestSimCardStatusChangeProtocol(t *testing.T) {
+	input := model.SimCardStatusChangeInput{
+		SimKeys: []*model.SimKey{{ChanKey: "channel-a", ICCID: "89860001"}},
+		ReqData: model.SimCardStatusChangeInputItem{OperType: "1"},
+	}
+	body, err := json.Marshal(input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	const want = `{"simKeys":[{"chanKey":"channel-a","iccid":"89860001","imsi":"","msisdn":""}],"reqData":{"operType":"1"}}`
+	if got := string(body); got != want {
+		t.Fatalf("request body = %s, want %s", got, want)
+	}
+}
+
+func TestCurrentUsageConsumerProtocol(t *testing.T) {
+	input := model.SimDataUsageInput{
+		SimKeys: []*model.SimKey{{ChanKey: "channel-a", ICCID: "89860001", IMSI: "460001", MSISDN: "13800000000"}},
+	}
+	body, err := json.Marshal(input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	const want = `{"simKeys":[{"chanKey":"channel-a","iccid":"89860001","imsi":"460001","msisdn":"13800000000"}],"reqData":{}}`
+	if got := string(body); got != want {
+		t.Fatalf("request body = %s, want %s", got, want)
+	}
+	client := NewWithCurrentUsageClient("iotlcmupstreamchan-http-svc:8110")
+	if client.currentUsageClientName != "iotlcmupstreamchan-http-svc:8110" {
+		t.Fatalf("current usage client = %q", client.currentUsageClientName)
+	}
+	if client.currentUsageTimeout != 0 {
+		t.Fatalf("default current usage timeout = %s, want unset", client.currentUsageTimeout)
+	}
+	configured := NewWithCurrentUsageClient("iotlcmupstreamchan-http-svc:8110", 90*time.Second)
+	if configured.currentUsageTimeout != 90*time.Second {
+		t.Fatalf("configured current usage timeout = %s, want 90s", configured.currentUsageTimeout)
+	}
+}
+
+func TestInboundBasicInfoClientCanUseDedicatedEndpoint(t *testing.T) {
+	client := NewWithInboundBasicInfoClient("iotlcm.example.com:8110")
+	clientName, err := client.inboundBasicInfoAddress(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if clientName != "iotlcm.example.com:8110" {
+		t.Fatalf("inbound basic info client = %q, want dedicated chan endpoint", clientName)
+	}
+}
+
+func TestInboundBasicInfoClientCanUseEnvEndpoint(t *testing.T) {
+	const endpoint = "iotlcm.example.com:8110"
+	t.Setenv(inboundBasicInfoClientNameEnv, endpoint)
+	client := New()
+	clientName, err := client.inboundBasicInfoAddress(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if clientName != endpoint {
+		t.Fatalf("inbound basic info client = %q, want env endpoint", clientName)
+	}
+}
+
+func TestConfiguredInboundBasicInfoClientUsesDedicatedEndpoint(t *testing.T) {
+	if value := os.Getenv(inboundBasicInfoClientNameEnv); value != "" {
+		t.Setenv(inboundBasicInfoClientNameEnv, "")
+	}
+	client := New()
+	clientName, err := client.inboundBasicInfoAddress(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(clientName, ":8110") {
+		t.Fatalf("inbound basic info client = %q, want dedicated chan endpoint", clientName)
+	}
+}
+
+func TestSimCardApnChangeProtocol(t *testing.T) {
+	input := model.SimCardApnChangeInput{
+		SimKeys: []*model.SimKey{{ChanKey: "channel-a", ICCID: "89860001"}},
+		ReqData: model.SimCardApnChangeInputItem{OperType: "0", ApnName: "CMIOT"},
+	}
+	body, err := json.Marshal(input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	const want = `{"simKeys":[{"chanKey":"channel-a","iccid":"89860001","imsi":"","msisdn":""}],"reqData":{"operType":"0","apnName":"CMIOT"}}`
+	if got := string(body); got != want {
+		t.Fatalf("request body = %s, want %s", got, want)
+	}
+}
+
+func TestDecodeSimCardApnChangeResponseKeepsFailedItems(t *testing.T) {
+	body := []byte(`{"code":0,"message":"ok","data":[{"code":0,"message":"success","simKey":{"iccid":"89860001"},"resData":{"orderNum":"order-1"}},{"code":4001,"message":"denied","simKey":{"iccid":"89860002"},"resData":{"orderNum":"order-2"}}]}`)
+	items, err := decodeResponse[model.SimCardApnChangeOutputItem](body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(items) != 2 {
+		t.Fatalf("item count = %d, want 2", len(items))
+	}
+	if items[1].Code != 4001 || items[1].ResData.OrderNum != "order-2" {
+		t.Fatalf("failed item was not preserved: %+v", items[1])
+	}
+}
+
+func TestDecodeSimCardStatusChangeResponseKeepsFailedItems(t *testing.T) {
+	body := []byte(`{"code":0,"message":"ok","data":[{"code":0,"message":"success","simKey":{"iccid":"89860001"},"resData":{"orderNum":"order-1"}},{"code":4001,"message":"denied","simKey":{"iccid":"89860002"},"resData":{"orderNum":"order-2"}}]}`)
+	items, err := decodeResponse[model.SimCardStatusChangeOutputItem](body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(items) != 2 {
+		t.Fatalf("item count = %d, want 2", len(items))
+	}
+	if items[1].Code != 4001 || items[1].ResData.OrderNum != "order-2" {
+		t.Fatalf("failed item was not preserved: %+v", items[1])
+	}
+}

internal/cmd/cmd.go

@@ -0,0 +1,18 @@
+package cmd
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/os/gcmd"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/worker"
+)
+
+var Main = gcmd.Command{
+	Name:  "main",
+	Usage: "main",
+	Brief: "start RabbitMQ worker",
+	Func: func(ctx context.Context, _ *gcmd.Parser) error {
+		return worker.Run(ctx)
+	},
+}

internal/consts/batch_submit.go

@@ -0,0 +1,11 @@
+package consts
+
+// BatchSubBatchSize 是批量开停数据 / 停复机执行时，单个上游通道每次请求携带的卡数量上限。
+// 原来一个通道整组一次性发给 upstream-chan，大批量会触发 worker 侧 HTTP 超时（默认 2 分钟上限）导致整批失败。
+// 按此值拆分为多个小请求后，每个请求都能在超时上限内返回。
+const BatchSubBatchSize = 100
+
+// BatchSubBatchConcurrency 是同一通道内子批次的并发度，受信号量限制：
+// 即最多同时向上游发起 BatchSubBatchConcurrency 个请求，等价于最多同时发起同样数量的运营商调用，
+// 既避免串行过慢，也避免无节制并发打爆运营商限流。如需更保守可调小该值。
+const BatchSubBatchConcurrency = 5

internal/consts/lifecycle.go

@@ -0,0 +1,154 @@
+package consts
+
+func FormalPackageTypes() []int { return []int{PackageTypeCard, PackageTypePool} }
+
+func IsAddPackageType(packageType int) bool {
+	return packageType == PackageTypeCardAddPackage || packageType == PackageTypePoolAddPackage
+}
+
+const (
+	BatchTypeInbound             = 1
+	BatchTypeAllocate            = 3
+	BatchTypeCardAddPackage      = 11
+	BatchTypePoolAddPackage      = 12
+	BatchTypeSimCardStart        = 103
+	BatchTypeSimCardStop         = 104
+	BatchTypeSimCardRestart      = 105
+	BatchTypeSimCardCancel       = 106
+	BatchTypeSimCardUnsub        = 108
+	BatchTypeDataCut             = 107
+	BatchTypeDataStart           = 109
+	BatchTypeDataStop            = 110
+	BatchTypeSimWarningConfig    = 40
+	BatchTypeTrafficCutoffConfig = 41
+	BatchTypeTrafficCutoff       = BatchTypeTrafficCutoffConfig // Deprecated: use BatchTypeTrafficCutoffConfig.
+
+	// 异步续费/变更扩展
+	BatchTypePrepare = 4   // 预处理（renew/change 共用 prepare）
+	BatchTypeRenew   = 101 // 卡续费
+	BatchTypeChange  = 102 // 卡变更套餐
+
+	// t_sim_card_bind_pkg.package_type 中，只有正式卡套餐和池套餐参与续费/变更判断。
+	PackageTypeCard           = 101
+	PackageTypePool           = 102
+	PackageTypeCardAddPackage = 103
+	PackageTypePoolAddPackage = 104
+
+	BatchSourceTypeBatchSelect = 0
+	BatchSourceTypeRange       = 1
+	BatchSourceTypeBatchInput  = 2
+	BatchSourceTypeFile        = 3
+	BatchSourceTypeScheduled   = 4
+
+	BatchSourceTypeExcel = BatchSourceTypeFile
+	BatchSourceTypeInput = BatchSourceTypeBatchInput
+
+	BatchNoPrefixCardAddPackage = "ADP"
+	BatchNoPrefixPoolAddPackage = "PAP"
+
+	SimCardStatusOperTypeStop  = "0"
+	SimCardStatusOperTypeStart = "1"
+
+	RestartPhaseCreated       = "created"
+	RestartPhaseWaitingResume = "waiting_resume"
+	RestartPhaseResuming      = "resuming"
+	RestartPhaseFinished      = "finished"
+
+	SimBossCardStatusActivated = 2
+	SimBossCardStatusStopped   = 4
+	SimBossCardStatusCanceled  = 9
+
+	SimApnOperTypeOpen = "0"
+	SimApnOperTypeStop = "1"
+
+	SimApnNameDefault = "CMIOT"
+
+	SimDataServiceStatusEnabled  = 1
+	SimDataServiceStatusDisabled = 0
+
+	BatchAsyncStatusPending        = 1
+	BatchAsyncStatusProcessing     = 2
+	BatchAsyncStatusSuccess        = 3
+	BatchAsyncStatusFailed         = 4
+	BatchAsyncStatusPartialSuccess = 5
+
+	BatchRunResultPending        = 0
+	BatchRunResultRunning        = 1
+	BatchRunResultSuccess        = 2
+	BatchRunResultFailed         = 3
+	BatchRunResultPartialSuccess = 4
+
+	BatchDetailStatusPending = 0
+	BatchDetailStatusRunning = 1
+	BatchDetailStatusSuccess = 2
+	BatchDetailStatusFailed  = 3
+
+	ResourceTypeExcel       = 1
+	DownloadStatusGenerated = 1
+
+	ResourceExportSceneCardAddPackage = 11
+	ResourceExportScenePoolAddPackage = 12
+
+	// 卡套餐有效状态
+	PackageStatusNone      = 0
+	PackageStatusPending   = 1
+	PackageStatusEffective = 2
+	PackageStatusExpired   = 3
+
+	// 订购记录有效状态（t_sim_card_bind_pkg.effective_status）
+	EffectiveStatusPending    = 0
+	EffectiveStatusEffective  = 1
+	EffectiveStatusExpired    = 2
+	EffectiveStatusTerminated = 3
+	EffectiveStatusVoided     = 4
+
+	// 退订状态（t_order.return_status）
+	OrderReturnStatusNone    = 0
+	OrderReturnStatusSuccess = 4
+	OrderReturnStatusPartial = 3
+
+	// Renew/change order contract. These values mirror the Admin cardpackage
+	// domain and the production t_order/t_order_item enum contract.
+	OrderTypeCardSubscribe = 1
+	OrderTypeCardRenew     = 2
+	OrderTypeCardChange    = 3
+	OrderStatusCompleted   = 5
+	CustomerTypeAgent      = 1
+	PayTypeOffline         = 3
+	PayTypeBalance         = 4
+	PayStatusPaid          = 2
+	ProductTypeCard        = 1
+	PayModeYearMonth       = 1
+	ShippingMethodNone     = 3
+
+	PackageBusinessTypeSubscribe = 0
+	PackageBusinessTypeChange    = 1
+	PackageBusinessTypeRenew     = 2
+
+	ResourceBatchDownloadTaskSimCardAddPackage = "单卡加油包结果"
+	ResourceBatchDownloadTaskSimPoolAddPackage = "流量池加油包结果"
+
+	SimCardAddPackageMaxFileSize = 50 << 20
+	SimPoolAddPackageMaxFileSize = 50 << 20
+
+	SimCardAddPackageSubDirImportExcel = "sim-card-add-package/import"
+	SimCardAddPackageSubDirResultExcel = "sim-card-add-package/result"
+	SimPoolAddPackageSubDirImportExcel = "sim-pool-add-package/import"
+	SimPoolAddPackageSubDirResultExcel = "sim-pool-add-package/result"
+
+	// 续费/变更结果目录
+	SimCardRenewResultSubDir  = "sim-card-renew/result"
+	SimCardChangeResultSubDir = "sim-card-change/result"
+
+	// 续费/变更结果任务名（沿用 Admin 约定）
+	ResultTaskSimCardRenew   = "卡续费结果"
+	ResultTaskSimCardChange  = "卡变更套餐结果"
+	ResultTaskSimCardPrepare = "卡续费/变更预处理结果"
+)
+
+// RenewPriceSource 续费/变更价格来源
+const (
+	RenewPriceSourceSystem       = "system"
+	RenewPriceSourceAgentDefault = "agentDefault"
+	RenewPriceSourceTarget       = "target"
+)

internal/consts/package_duration.go

@@ -0,0 +1,42 @@
+package consts
+
+import "strings"
+
+const (
+	BillingMethodMonth = 1
+	BillingMethodYear  = 2
+
+	RenewDurationUnitMonth = "month"
+	RenewDurationUnitYear  = "year"
+)
+
+func BillingMethodDurationUnit(billingMethod int) string {
+	if billingMethod == BillingMethodMonth {
+		return RenewDurationUnitMonth
+	}
+	return RenewDurationUnitYear
+}
+
+func NormalizeRenewDurationUnit(unit string) string {
+	if strings.EqualFold(strings.TrimSpace(unit), RenewDurationUnitMonth) {
+		return RenewDurationUnitMonth
+	}
+	return RenewDurationUnitYear
+}
+
+func RenewDurationUnitText(unit string) string {
+	if NormalizeRenewDurationUnit(unit) == RenewDurationUnitMonth {
+		return "个月"
+	}
+	return "年"
+}
+
+func DurationValueToMonths(unit string, value int) int {
+	if value <= 0 {
+		return 0
+	}
+	if NormalizeRenewDurationUnit(unit) == RenewDurationUnitMonth {
+		return value
+	}
+	return value * 12
+}

internal/consts/queue.go

@@ -0,0 +1,31 @@
+package consts
+
+const (
+	QueueTrafficUsageDailyRetry   = "iotlcm.traffic_usage.daily.retry"
+	QueueTrafficUsageMonthlyRetry = "iotlcm.traffic_usage.monthly.retry"
+	QueueTrafficUsageMonthlyStart = "iotlcm.traffic_usage.monthly.start"
+	// QueueTrafficUsageDailyStart carries one command per full daily usage run.
+	// Admin publishes it; Worker owns the card scan and Redis production.
+	QueueTrafficUsageDailyStart = "iotlcm.traffic_usage.daily.start"
+	// QueueTrafficUsageMonthlyHistoryStart carries one command per full monthly
+	// usage history run, which writes the TDengine monthly table. It is distinct
+	// from QueueTrafficUsageMonthlyStart, which fills the MySQL current-month
+	// snapshot.
+	QueueTrafficUsageMonthlyHistoryStart = "iotlcm.traffic_usage.monthly_history.start"
+	QueueSimStatusQueueStart             = "iotlcm.sim_status.queue.start"
+	QueueSimStatusQueueTask              = "iotlcm.sim_status.queue.task"
+	QueueImeiQueueStart                  = "iotlcm.imei.queue.start"
+	QueueWorkOrderAutoClose              = "iotlcm.work_order.auto_close"
+	QueueBalanceRechargeTimeout          = "iotlcm.balance.recharge.timeout"
+	QueueBalanceRechargeDelay            = "iotlcm.balance.recharge.delay"
+	QueueWorkerDiagnostic                = "iotlcm.worker.diagnostic"
+	QueueBatchExecute                    = "iotlcm.batch.execute"
+	QueueBatchExecuteRetry               = "iotlcm.batch.execute.retry"
+	QueueBatchExecuteDeadLetter          = "iotlcm.batch.execute.dlq"
+	QueueBatchRestartResumeDelay         = "iotlcm.batch.restart.resume.delay"
+	QueuePackageCustomerSync             = "iotlcm.package_customer.sync"
+	QueueChannelRefresh                  = "iotlcm.channel.refresh"
+	// QueueBalanceRechargeDeadLetter 保存重试次数耗尽后仍失败的充值超时消息，
+	// 供人工排查（例如支付宝凭据未配置导致的永久失败），不再自动重投。
+	QueueBalanceRechargeDeadLetter = "iotlcm.balance.recharge.dlq"
+)

internal/consumer/rabbitmq/balance_recharge.go

@@ -0,0 +1,103 @@
+package rabbitmq
+
+import (
+	"context"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+// defaultBalanceRechargeMaxRetry 是充值超时消息的默认最大重试次数。
+// 支付宝凭据未配置等永久性失败会让 CheckRechargeTimeout 一直返回错误，
+// 必须设置上限，否则消息会无限重投并刷屏日志。
+const defaultBalanceRechargeMaxRetry = 100
+
+// 包级变量便于测试注入发布行为，与 trafficUsagePublish 保持一致。
+var balanceRechargePublish = mq.Publish
+var balanceRechargePublishDelayed = mq.PublishDelayed
+
+func (c *Consumers) initBalanceRechargeTimeout(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "balance recharge timeout consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if c.balance == nil {
+		return gerror.New("balance recharge handler is not configured")
+	}
+	retryDelay, err := getBalanceRechargeRetryDelay(ctx)
+	if err != nil {
+		return err
+	}
+	maxRetry, err := getBalanceRechargeMaxRetry(ctx)
+	if err != nil {
+		return err
+	}
+	c.balanceRechargeRetryDelay = retryDelay
+	c.balanceRechargeMaxRetry = maxRetry
+	if err := mq.DeclareQueue(ctx, consts.QueueBalanceRechargeDeadLetter); err != nil {
+		return gerror.Wrap(err, "declare balance recharge dead-letter queue failed")
+	}
+	if err := mq.ConsumeJSON[model.BalanceRechargeTimeoutMessage](
+		ctx,
+		consts.QueueBalanceRechargeTimeout,
+		"iotlcm-balance-recharge-timeout",
+		c.handleBalanceRechargeTimeout,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "balance recharge timeout consumer started, queue:%s, retryDelay:%s, maxRetry:%d", consts.QueueBalanceRechargeTimeout, retryDelay, maxRetry)
+	return nil
+}
+
+func (c *Consumers) handleBalanceRechargeTimeout(ctx context.Context, message model.BalanceRechargeTimeoutMessage) error {
+	if message.OutTradeNo == "" {
+		g.Log().Errorf(ctx, "balance recharge timeout message dropped, invalid message:%+v", message)
+		return nil
+	}
+	if c.balance == nil {
+		return gerror.New("balance recharge handler is not configured")
+	}
+	if err := c.balance.CheckRechargeTimeout(ctx, message.OutTradeNo); err != nil {
+		message.RetryCount++
+		if shouldDeadLetterBalanceRecharge(message.RetryCount, c.balanceRechargeMaxRetry) {
+			if publishErr := balanceRechargePublish(ctx, consts.QueueBalanceRechargeDeadLetter, message); publishErr != nil {
+				return gerror.Wrap(publishErr, "publish balance recharge dead-letter message failed")
+			}
+			g.Log().Errorf(ctx, "balance recharge timeout retries exhausted and message sent to dead-letter queue, outTradeNo:%s, retryCount:%d, maxRetry:%d, err:%+v", message.OutTradeNo, message.RetryCount, c.balanceRechargeMaxRetry, err)
+			return nil
+		}
+		if retryErr := balanceRechargePublishDelayed(ctx, consts.QueueBalanceRechargeDelay, consts.QueueBalanceRechargeTimeout, c.balanceRechargeRetryDelay, message); retryErr != nil {
+			return gerror.Wrap(retryErr, "republish balance recharge timeout message failed")
+		}
+		g.Log().Warningf(ctx, "balance recharge timeout query failed and will retry, outTradeNo:%s, retryCount:%d, err:%+v", message.OutTradeNo, message.RetryCount, err)
+	}
+	return nil
+}
+
+// shouldDeadLetterBalanceRecharge 判断本次失败后是否放弃重试。
+// retryCount 是自增后的重试次数，因此 maxRetry 表示最多重投多少次。
+func shouldDeadLetterBalanceRecharge(retryCount, maxRetry int) bool {
+	return retryCount > maxRetry
+}
+
+func getBalanceRechargeRetryDelay(ctx context.Context) (time.Duration, error) {
+	value := g.Cfg().MustGet(ctx, "rabbitmq.balanceRecharge.retryDelay", "1m").String()
+	delay, err := time.ParseDuration(value)
+	if err != nil || delay <= 0 {
+		return 0, gerror.Newf("balance recharge retry delay is invalid: %q", value)
+	}
+	return delay, nil
+}
+
+func getBalanceRechargeMaxRetry(ctx context.Context) (int, error) {
+	maxRetry := g.Cfg().MustGet(ctx, "rabbitmq.balanceRecharge.maxRetry", defaultBalanceRechargeMaxRetry).Int()
+	if maxRetry <= 0 {
+		return 0, gerror.Newf("balance recharge max retry is invalid: %d", maxRetry)
+	}
+	return maxRetry, nil
+}

internal/consumer/rabbitmq/balance_recharge_test.go

@@ -0,0 +1,153 @@
+package rabbitmq
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gcfg"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func stubBalanceRechargePublish(t *testing.T) {
+	t.Helper()
+	originalPublish, originalDelayed := balanceRechargePublish, balanceRechargePublishDelayed
+	t.Cleanup(func() {
+		balanceRechargePublish, balanceRechargePublishDelayed = originalPublish, originalDelayed
+	})
+	balanceRechargePublish = func(context.Context, string, any) error { return nil }
+	balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error { return nil }
+}
+
+func setBalanceRechargeTestConfig(t *testing.T, content string) {
+	t.Helper()
+	oldAdapter := g.Cfg().GetAdapter()
+	adapter, err := gcfg.NewAdapterContent(content)
+	if err != nil {
+		t.Fatal(err)
+	}
+	g.Cfg().SetAdapter(adapter)
+	t.Cleanup(func() { g.Cfg().SetAdapter(oldAdapter) })
+}
+
+func TestHandleBalanceRechargeTimeoutRepublishesDelayedBelowMaxRetry(t *testing.T) {
+	stubBalanceRechargePublish(t)
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
+	consumers.balanceRechargeRetryDelay = time.Minute
+	consumers.balanceRechargeMaxRetry = 3
+
+	delayedCalls := 0
+	balanceRechargePublishDelayed = func(_ context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error {
+		delayedCalls++
+		if delayQueue != consts.QueueBalanceRechargeDelay || targetQueue != consts.QueueBalanceRechargeTimeout || delay != time.Minute {
+			t.Fatalf("unexpected delayed publish: %s -> %s, delay:%s", delayQueue, targetQueue, delay)
+		}
+		message, ok := data.(model.BalanceRechargeTimeoutMessage)
+		if !ok || message.OutTradeNo != "ALI-1" || message.RetryCount != 3 {
+			t.Fatalf("unexpected retry payload: %+v", data)
+		}
+		return nil
+	}
+	deadLetterCalls := 0
+	balanceRechargePublish = func(_ context.Context, queue string, _ any) error {
+		deadLetterCalls++
+		return nil
+	}
+
+	// 自增后达到上限仍未超过，应继续重投延迟队列。
+	if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 2}); err != nil {
+		t.Fatalf("retry below cap returned error: %v", err)
+	}
+	if delayedCalls != 1 || deadLetterCalls != 0 {
+		t.Fatalf("delayedCalls=%d deadLetterCalls=%d, want 1 and 0", delayedCalls, deadLetterCalls)
+	}
+}
+
+func TestHandleBalanceRechargeTimeoutDeadLettersAboveMaxRetry(t *testing.T) {
+	stubBalanceRechargePublish(t)
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
+	consumers.balanceRechargeRetryDelay = time.Minute
+	consumers.balanceRechargeMaxRetry = 3
+
+	delayedCalls := 0
+	balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error {
+		delayedCalls++
+		return nil
+	}
+	deadLetterCalls := 0
+	balanceRechargePublish = func(_ context.Context, queue string, data any) error {
+		deadLetterCalls++
+		if queue != consts.QueueBalanceRechargeDeadLetter {
+			t.Fatalf("dead letter queue = %s, want %s", queue, consts.QueueBalanceRechargeDeadLetter)
+		}
+		message, ok := data.(model.BalanceRechargeTimeoutMessage)
+		if !ok || message.OutTradeNo != "ALI-1" || message.RetryCount != 4 {
+			t.Fatalf("unexpected dead-letter payload: %+v", data)
+		}
+		return nil
+	}
+
+	if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 3}); err != nil {
+		t.Fatalf("dead-letter path returned error: %v", err)
+	}
+	if delayedCalls != 0 || deadLetterCalls != 1 {
+		t.Fatalf("delayedCalls=%d deadLetterCalls=%d, want 0 and 1", delayedCalls, deadLetterCalls)
+	}
+}
+
+func TestHandleBalanceRechargeTimeoutReturnsPublishErrors(t *testing.T) {
+	stubBalanceRechargePublish(t)
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
+	consumers.balanceRechargeRetryDelay = time.Minute
+	consumers.balanceRechargeMaxRetry = 3
+
+	expected := errors.New("broker unavailable")
+	balanceRechargePublish = func(context.Context, string, any) error { return expected }
+	balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error { return expected }
+
+	// 重投失败必须向上返回错误，让消费框架 Nack 重新入队，不能静默丢失消息。
+	if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 0}); !errors.Is(err, expected) {
+		t.Fatalf("delayed publish failure = %v, want %v", err, expected)
+	}
+	if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 3}); !errors.Is(err, expected) {
+		t.Fatalf("dead-letter publish failure = %v, want %v", err, expected)
+	}
+}
+
+func TestGetBalanceRechargeMaxRetry(t *testing.T) {
+	t.Run("defaults when unconfigured", func(t *testing.T) {
+		setBalanceRechargeTestConfig(t, "rabbitmq:\n  balanceRecharge:\n    retryDelay: \"1m\"\n")
+		maxRetry, err := getBalanceRechargeMaxRetry(context.Background())
+		if err != nil || maxRetry != defaultBalanceRechargeMaxRetry {
+			t.Fatalf("maxRetry = %d (err=%v), want %d", maxRetry, err, defaultBalanceRechargeMaxRetry)
+		}
+	})
+
+	t.Run("reads configured value", func(t *testing.T) {
+		setBalanceRechargeTestConfig(t, "rabbitmq:\n  balanceRecharge:\n    maxRetry: 7\n")
+		maxRetry, err := getBalanceRechargeMaxRetry(context.Background())
+		if err != nil || maxRetry != 7 {
+			t.Fatalf("maxRetry = %d (err=%v), want 7", maxRetry, err)
+		}
+	})
+
+	for _, test := range []struct {
+		name  string
+		value string
+	}{
+		{name: "zero", value: "0"},
+		{name: "negative", value: "-1"},
+		{name: "non-numeric", value: "\"abc\""},
+	} {
+		t.Run("rejects "+test.name, func(t *testing.T) {
+			setBalanceRechargeTestConfig(t, "rabbitmq:\n  balanceRecharge:\n    maxRetry: "+test.value+"\n")
+			if _, err := getBalanceRechargeMaxRetry(context.Background()); err == nil {
+				t.Fatalf("maxRetry=%s was accepted, want error", test.value)
+			}
+		})
+	}
+}

internal/consumer/rabbitmq/batch.go

@@ -0,0 +1,178 @@
+package rabbitmq
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+	amqp "github.com/rabbitmq/amqp091-go"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+)
+
+const batchRetryCountHeader = "x-iotlcm-retry-count"
+
+func (c *Consumers) initBatchExecute(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "batch execute consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if c.batch == nil {
+		return gerror.New("batch execute handler is not configured")
+	}
+	retryDelay, maxRetries, err := getBatchExecuteRetryConfig(ctx)
+	if err != nil {
+		return err
+	}
+	c.batchExecuteRetryDelay = retryDelay
+	c.batchExecuteMaxRetries = maxRetries
+	if err = mq.DeclareQueue(ctx, consts.QueueBatchExecuteDeadLetter); err != nil {
+		return gerror.Wrap(err, "declare batch execute dead-letter queue failed")
+	}
+	if err = mq.ConsumeJSON[batch.Message](
+		ctx,
+		consts.QueueBatchExecute,
+		"iotlcm-batch-execute",
+		c.handleBatchExecute,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "batch execute consumer started, queue:%s, retryDelay:%s, maxRetries:%d", consts.QueueBatchExecute, retryDelay, maxRetries)
+	return nil
+}
+
+func (c *Consumers) handleBatchExecute(ctx context.Context, message batch.Message) error {
+	if c.batch == nil {
+		return gerror.New("batch execute handler is not configured")
+	}
+	g.Log().Infof(ctx, "batch execute message received, batchNo:%s, batchType:%d", message.BatchNo, message.BatchType)
+	if err := c.batch.Handle(ctx, message); err != nil {
+		if _, delivered := mq.DeliveryMetadataFromContext(ctx); !delivered {
+			return err
+		}
+		if batch.IsRetryable(err) {
+			message.Retryable = true
+		}
+		var failureMarkErr *batch.FailureMarkError
+		if errors.As(err, &failureMarkErr) {
+			message.FailureReason = failureMarkErr.Reason
+		}
+		return c.retryOrDeadLetterBatchMessage(ctx, message, err)
+	}
+	return nil
+}
+
+func (c *Consumers) retryOrDeadLetterBatchMessage(ctx context.Context, message batch.Message, handlerErr error) error {
+	metadata, _ := mq.DeliveryMetadataFromContext(ctx)
+	retryCount := batchRetryCount(metadata.Headers)
+	headers := amqp.Table{batchRetryCountHeader: retryCount + 1}
+	if shouldDeadLetterBatch(retryCount, c.batchExecuteMaxRetries) {
+		if message.Retryable {
+			finalizationMessage := message
+			finalizationMessage.Retryable = false
+			finalizationMessage.FailureReason = handlerErr.Error()
+			if err := c.batch.Handle(ctx, finalizationMessage); err != nil {
+				return gerror.Wrap(err, "finalize retryable batch failure before dead-lettering")
+			}
+			message.FailureReason = handlerErr.Error()
+		}
+		deadLetterBody, marshalErr := json.Marshal(batchDeadLetterMessage{
+			Message:    message,
+			RetryCount: retryCount,
+			Error:      handlerErr.Error(),
+			FailedAt:   time.Now().Format(time.RFC3339),
+		})
+		if marshalErr != nil {
+			return gerror.Wrap(marshalErr, "marshal batch execute dead-letter message failed")
+		}
+		if err := mq.PublishBytesWithHeaders(ctx, consts.QueueBatchExecuteDeadLetter, deadLetterBody, nil); err != nil {
+			return gerror.Wrap(err, "publish batch execute dead-letter message failed")
+		}
+		g.Log().Errorf(ctx, "batch execute message sent to dead-letter queue, batchNo:%s, batchType:%d, retryCount:%d, err:%+v", message.BatchNo, message.BatchType, retryCount, handlerErr)
+		return nil
+	}
+	body, marshalErr := json.Marshal(message)
+	if marshalErr != nil {
+		return gerror.Wrap(marshalErr, "marshal batch execute retry message failed")
+	}
+	if err := mq.PublishDelayedBytesWithHeaders(
+		ctx,
+		consts.QueueBatchExecuteRetry,
+		consts.QueueBatchExecute,
+		c.batchExecuteRetryDelay,
+		body,
+		headers,
+	); err != nil {
+		return gerror.Wrap(err, "republish batch execute retry message failed")
+	}
+	g.Log().Warningf(ctx, "batch execute message scheduled for retry, batchNo:%s, batchType:%d, retryCount:%d, delay:%s, err:%+v", message.BatchNo, message.BatchType, retryCount+1, c.batchExecuteRetryDelay, handlerErr)
+	return nil
+}
+
+type batchDeadLetterMessage struct {
+	Message    batch.Message `json:"message"`
+	RetryCount int           `json:"retryCount"`
+	Error      string        `json:"error"`
+	FailedAt   string        `json:"failedAt"`
+}
+
+func shouldDeadLetterBatch(retryCount, maxRetries int) bool {
+	return retryCount >= maxRetries
+}
+
+func batchRetryCount(headers map[string]any) int {
+	if headers == nil {
+		return 0
+	}
+	value, ok := headers[batchRetryCountHeader]
+	if !ok {
+		return 0
+	}
+	switch count := value.(type) {
+	case int:
+		return max(count, 0)
+	case int8:
+		return max(int(count), 0)
+	case int16:
+		return max(int(count), 0)
+	case int32:
+		return max(int(count), 0)
+	case int64:
+		return max(int(count), 0)
+	case uint8:
+		return int(count)
+	case uint16:
+		return int(count)
+	case uint32:
+		return int(count)
+	case uint64:
+		if count > uint64(^uint(0)>>1) {
+			return int(^uint(0) >> 1)
+		}
+		return int(count)
+	case string:
+		parsed, err := strconv.Atoi(count)
+		if err == nil {
+			return max(parsed, 0)
+		}
+	}
+	return 0
+}
+
+func getBatchExecuteRetryConfig(ctx context.Context) (time.Duration, int, error) {
+	delay := g.Cfg().MustGet(ctx, "rabbitmq.batchExecute.retryDelay", "30s").Duration()
+	if delay <= 0 {
+		return 0, 0, gerror.New("batch execute retry delay must be positive")
+	}
+	maxRetries := g.Cfg().MustGet(ctx, "rabbitmq.batchExecute.maxRetries", 3).Int()
+	if maxRetries < 0 {
+		return 0, 0, gerror.New("batch execute max retries cannot be negative")
+	}
+	return delay, maxRetries, nil
+}

internal/consumer/rabbitmq/batch_test.go

@@ -0,0 +1,48 @@
+package rabbitmq
+
+import "testing"
+
+func TestShouldDeadLetterBatch(t *testing.T) {
+	tests := []struct {
+		name       string
+		retryCount int
+		maxRetries int
+		want       bool
+	}{
+		{name: "first attempt retries", retryCount: 0, maxRetries: 3, want: false},
+		{name: "second attempt retries", retryCount: 1, maxRetries: 3, want: false},
+		{name: "last configured retry retries", retryCount: 2, maxRetries: 3, want: false},
+		{name: "next failure dead letters", retryCount: 3, maxRetries: 3, want: true},
+		{name: "zero retries dead letters immediately", retryCount: 0, maxRetries: 0, want: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if actual := shouldDeadLetterBatch(tt.retryCount, tt.maxRetries); actual != tt.want {
+				t.Fatalf("shouldDeadLetterBatch(%d, %d) = %t, want %t", tt.retryCount, tt.maxRetries, actual, tt.want)
+			}
+		})
+	}
+}
+
+func TestBatchRetryCount(t *testing.T) {
+	tests := []struct {
+		name    string
+		headers map[string]any
+		want    int
+	}{
+		{name: "missing", want: 0},
+		{name: "int", headers: map[string]any{batchRetryCountHeader: 2}, want: 2},
+		{name: "int64", headers: map[string]any{batchRetryCountHeader: int64(3)}, want: 3},
+		{name: "uint", headers: map[string]any{batchRetryCountHeader: uint16(4)}, want: 4},
+		{name: "string", headers: map[string]any{batchRetryCountHeader: "5"}, want: 5},
+		{name: "negative", headers: map[string]any{batchRetryCountHeader: -1}, want: 0},
+		{name: "invalid", headers: map[string]any{batchRetryCountHeader: "not-a-number"}, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if actual := batchRetryCount(tt.headers); actual != tt.want {
+				t.Fatalf("batchRetryCount() = %d, want %d", actual, tt.want)
+			}
+		})
+	}
+}

internal/consumer/rabbitmq/channel_refresh.go

@@ -0,0 +1,243 @@
+package rabbitmq
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+const channelRefreshConsumer = "iotlcm-worker-channel-refresh"
+
+type channelRefreshConfig struct {
+	Address string        `json:"address"`
+	Timeout time.Duration `json:"timeout"`
+}
+
+type channelRefreshResponse struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+func (c *Consumers) initChannelRefresh(ctx context.Context) error {
+	if !rabbitmq.Enabled(ctx) {
+		g.Log().Info(ctx, "channel refresh consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := rabbitmq.ConsumeJSON[model.ChannelRefreshMessage](
+		ctx,
+		consts.QueueChannelRefresh,
+		channelRefreshConsumer,
+		c.handleChannelRefresh,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "channel refresh consumer started, queue:%s", consts.QueueChannelRefresh)
+	return nil
+}
+
+func (c *Consumers) handleChannelRefresh(ctx context.Context, message model.ChannelRefreshMessage) error {
+	switch strings.ToLower(strings.TrimSpace(message.Type)) {
+	case "channel":
+		if strings.TrimSpace(message.ChanKey) == "" {
+			g.Log().Errorf(ctx, "channel refresh message dropped, chanKey is empty")
+			return nil
+		}
+		if err := refreshChannelWithRetry(ctx, message.ChanKey); err != nil {
+			g.Log().Errorf(ctx, "channel refresh failed, chanKey=%s, err=%+v", message.ChanKey, err)
+			return err
+		}
+		return nil
+	case "template":
+		if strings.TrimSpace(message.TempKey) == "" {
+			g.Log().Errorf(ctx, "channel refresh message dropped, tempKey is empty")
+			return nil
+		}
+		return refreshTemplateChannels(ctx, message.TempKey)
+	case "channel_rename":
+		oldChanKey := strings.TrimSpace(message.OldChanKey)
+		newChanKey := strings.TrimSpace(message.NewChanKey)
+		if oldChanKey == "" || newChanKey == "" || oldChanKey == newChanKey {
+			g.Log().Errorf(ctx, "channel rename message dropped, oldChanKey=%q newChanKey=%q", oldChanKey, newChanKey)
+			return nil
+		}
+		if err := renameChannelWithRetry(ctx, oldChanKey, newChanKey); err != nil {
+			// 返回 error 让消费框架 Nack 并重新入队：rename 各步骤幂等，
+			// 重复投递安全；静默 Ack 会导致旧通道实例永久残留，只能重启恢复。
+			g.Log().Errorf(ctx, "channel rename failed after retries, will requeue, oldChanKey=%s newChanKey=%s err=%+v", oldChanKey, newChanKey, err)
+			return err
+		}
+		return nil
+	default:
+		g.Log().Errorf(ctx, "channel refresh message dropped, unsupported type=%s", message.Type)
+		return nil
+	}
+}
+
+func renameChannelWithRetry(ctx context.Context, oldChanKey, newChanKey string) error {
+	var err error
+	for attempt := 1; attempt <= 3; attempt++ {
+		err = renameChannel(ctx, oldChanKey, newChanKey)
+		if err == nil {
+			return nil
+		}
+		if attempt == 3 {
+			break
+		}
+		delay := time.Second
+		if attempt == 2 {
+			delay = 3 * time.Second
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(delay):
+		}
+	}
+	return gerror.Wrapf(err, "channel rename failed after %d attempts, oldChanKey=%s newChanKey=%s", 3, oldChanKey, newChanKey)
+}
+
+// renameChannel is ordered so that the old runtime remains available until
+// both upstreams have built the new instance. Card data and ICCID routes are
+// intentionally left untouched for operations staff to migrate manually.
+func renameChannel(ctx context.Context, oldChanKey, newChanKey string) error {
+	if err := refreshChannel(ctx, newChanKey); err != nil {
+		return gerror.Wrap(err, "refresh new channel failed")
+	}
+	if err := removeChannelTarget(ctx, "upstream.gateway", oldChanKey); err != nil {
+		return gerror.Wrap(err, "remove old gateway channel failed")
+	}
+	if err := removeChannelTarget(ctx, "upstream.proxy", oldChanKey); err != nil {
+		return gerror.Wrap(err, "remove old proxy channel failed")
+	}
+	g.Log().Infof(ctx, "channel instance rebuilt after rename, oldChanKey=%s newChanKey=%s", oldChanKey, newChanKey)
+	return nil
+}
+
+func refreshTemplateChannels(ctx context.Context, tempKey string) error {
+	var rows []struct {
+		ChanKey string `json:"chanKey"`
+	}
+	if err := g.DB().Model("t_sim_channels").Ctx(ctx).
+		Fields("chan_key").
+		Where("temp_key", strings.TrimSpace(tempKey)).
+		Scan(&rows); err != nil {
+		return gerror.Wrap(err, "query template channels failed")
+	}
+
+	var firstErr error
+	for _, row := range rows {
+		if err := refreshChannelWithRetry(ctx, row.ChanKey); err != nil {
+			g.Log().Errorf(ctx, "template channel refresh failed, tempKey=%s, chanKey=%s, err=%+v", tempKey, row.ChanKey, err)
+			if firstErr == nil {
+				firstErr = err
+			}
+		}
+	}
+	return firstErr
+}
+
+func refreshChannelWithRetry(ctx context.Context, chanKey string) error {
+	var err error
+	for attempt := 1; attempt <= 3; attempt++ {
+		err = refreshChannel(ctx, chanKey)
+		if err == nil {
+			return nil
+		}
+		if attempt == 3 {
+			break
+		}
+		delay := time.Second
+		if attempt == 2 {
+			delay = 3 * time.Second
+		}
+		select {
+		case <-ctx.Done():
+			return gerror.Wrapf(err, "channel refresh canceled after %d attempts, chanKey=%s", attempt, chanKey)
+		case <-time.After(delay):
+		}
+	}
+	return gerror.Wrapf(err, "channel refresh failed after %d attempts, chanKey=%s", 3, chanKey)
+}
+
+func refreshChannel(ctx context.Context, chanKey string) error {
+	chanKey = strings.TrimSpace(chanKey)
+	if chanKey == "" {
+		return gerror.New("chanKey is required")
+	}
+	if err := refreshChannelTarget(ctx, "upstream.gateway", chanKey); err != nil {
+		return gerror.Wrap(err, "refresh gateway channel failed")
+	}
+	if err := refreshChannelTarget(ctx, "upstream.proxy", chanKey); err != nil {
+		return gerror.Wrap(err, "refresh proxy channel failed")
+	}
+	return nil
+}
+
+func refreshChannelTarget(ctx context.Context, configKey, chanKey string) error {
+	return channelControlTarget(ctx, configKey, "/internal/channel/refresh", chanKey)
+}
+
+func removeChannelTarget(ctx context.Context, configKey, chanKey string) error {
+	return channelControlTarget(ctx, configKey, "/internal/channel/remove", chanKey)
+}
+
+func channelControlTarget(ctx context.Context, configKey, path, chanKey string) error {
+	configValue, err := g.Cfg().Get(ctx, configKey)
+	if err != nil {
+		return gerror.Wrapf(err, "read upstream configuration failed: %s", configKey)
+	}
+	if configValue == nil || configValue.IsNil() {
+		return gerror.Newf("upstream configuration is missing: %s", configKey)
+	}
+	config := channelRefreshConfig{}
+	if err = configValue.Scan(&config); err != nil {
+		return gerror.Wrapf(err, "parse upstream configuration failed: %s", configKey)
+	}
+	address := strings.TrimSpace(config.Address)
+	if address == "" {
+		return gerror.Newf("upstream address is empty: %s", configKey)
+	}
+	if config.Timeout <= 0 {
+		config.Timeout = 10 * time.Second
+	}
+	requestCtx, cancel := context.WithTimeout(ctx, config.Timeout)
+	defer cancel()
+
+	client := g.Client()
+	client.SetTimeout(config.Timeout)
+	controlURL := fmt.Sprintf("http://%s%s?chanKey=%s", address, path, url.QueryEscape(chanKey))
+	response, err := client.Post(requestCtx, controlURL, nil)
+	if err != nil {
+		return gerror.Wrapf(err, "request upstream channel control failed: target=%s path=%s", configKey, path)
+	}
+	defer response.Close()
+
+	output := channelRefreshResponse{}
+	statusCode := 0
+	if response.Response != nil {
+		statusCode = response.StatusCode
+	}
+	if err = json.Unmarshal(response.ReadAll(), &output); err != nil {
+		return gerror.Wrapf(err, "parse upstream channel control response failed: target=%s path=%s status=%d", configKey, path, statusCode)
+	}
+	if output.Code != 0 {
+		if output.Message == "" {
+			output.Message = "upstream channel control failed"
+		}
+		return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d message=%s", configKey, path, statusCode, output.Message)
+	}
+	if statusCode < 200 || statusCode >= 300 {
+		return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d", configKey, path, statusCode)
+	}
+	return nil
+}

internal/consumer/rabbitmq/channel_refresh_test.go

@@ -0,0 +1,134 @@
+package rabbitmq
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gcfg"
+)
+
+func setChannelRefreshTestConfig(t *testing.T, gateway, proxy string) {
+	t.Helper()
+	oldAdapter := g.Cfg().GetAdapter()
+	adapter, err := gcfg.NewAdapterContent(fmt.Sprintf(`
+upstream:
+  gateway:
+    address: %q
+    timeout: 1s
+  proxy:
+    address: %q
+    timeout: 1s
+`, gateway, proxy))
+	if err != nil {
+		t.Fatal(err)
+	}
+	g.Cfg().SetAdapter(adapter)
+	t.Cleanup(func() { g.Cfg().SetAdapter(oldAdapter) })
+}
+
+func testServerAddress(server *httptest.Server) string {
+	return strings.TrimPrefix(server.URL, "http://")
+}
+
+func TestRefreshChannelCallsGatewayAndProxy(t *testing.T) {
+	var gatewayCalls atomic.Int32
+	var proxyCalls atomic.Int32
+	newServer := func(counter *atomic.Int32) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+			counter.Add(1)
+			if request.URL.Path != "/internal/channel/refresh" || request.URL.Query().Get("chanKey") != "channel-a" {
+				t.Fatalf("unexpected request: %s", request.URL.String())
+			}
+			_, _ = w.Write([]byte(`{"code":0,"message":"success"}`))
+		}))
+	}
+	gateway := newServer(&gatewayCalls)
+	defer gateway.Close()
+	proxy := newServer(&proxyCalls)
+	defer proxy.Close()
+	setChannelRefreshTestConfig(t, testServerAddress(gateway), testServerAddress(proxy))
+
+	if err := refreshChannel(context.Background(), "channel-a"); err != nil {
+		t.Fatal(err)
+	}
+	if gatewayCalls.Load() != 1 || proxyCalls.Load() != 1 {
+		t.Fatalf("gateway calls=%d proxy calls=%d", gatewayCalls.Load(), proxyCalls.Load())
+	}
+}
+
+func TestRenameChannelOnlyRebuildsRuntimeInstances(t *testing.T) {
+	var (
+		mu     sync.Mutex
+		events []string
+	)
+	newServer := func(target string) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+			mu.Lock()
+			events = append(events, target+":"+request.URL.Path+":"+request.URL.Query().Get("chanKey"))
+			mu.Unlock()
+			_, _ = w.Write([]byte(`{"code":0,"message":"success"}`))
+		}))
+	}
+	gateway := newServer("gateway")
+	defer gateway.Close()
+	proxy := newServer("proxy")
+	defer proxy.Close()
+	setChannelRefreshTestConfig(t, testServerAddress(gateway), testServerAddress(proxy))
+
+	if err := renameChannel(context.Background(), "channel-old", "channel-new"); err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		"gateway:/internal/channel/refresh:channel-new",
+		"proxy:/internal/channel/refresh:channel-new",
+		"gateway:/internal/channel/remove:channel-old",
+		"proxy:/internal/channel/remove:channel-old",
+	}
+	if strings.Join(events, "\n") != strings.Join(want, "\n") {
+		t.Fatalf("rename events = %q, want %q", events, want)
+	}
+}
+
+func TestChannelControlErrorIncludesTargetStatusAndMessage(t *testing.T) {
+	gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadGateway)
+		_, _ = w.Write([]byte(`{"code":50,"message":"carrier unavailable"}`))
+	}))
+	defer gateway.Close()
+	setChannelRefreshTestConfig(t, testServerAddress(gateway), "127.0.0.1:1")
+
+	err := refreshChannelTarget(context.Background(), "upstream.gateway", "channel-a")
+	if err == nil {
+		t.Fatal("expected target error")
+	}
+	message := err.Error()
+	for _, part := range []string{"target=upstream.gateway", "status=502", "carrier unavailable"} {
+		if !strings.Contains(message, part) {
+			t.Fatalf("error %q does not contain %q", message, part)
+		}
+	}
+}
+
+func TestHandleChannelRefreshReturnsError(t *testing.T) {
+	gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"code":50,"message":"refresh rejected"}`))
+	}))
+	defer gateway.Close()
+	setChannelRefreshTestConfig(t, testServerAddress(gateway), "127.0.0.1:1")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	consumer := &Consumers{}
+	if err := consumer.handleChannelRefresh(ctx, model.ChannelRefreshMessage{Type: "channel", ChanKey: "channel-a"}); err == nil {
+		t.Fatal("refresh failure was acknowledged as success")
+	}
+}

internal/consumer/rabbitmq/consumer.go

@@ -0,0 +1,175 @@
+package rabbitmq
+
+import (
+	"context"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+type TrafficUsageHandler interface {
+	RetryDaily(context.Context, model.TrafficUsageDailyRetryMessage) error
+	RetryMonthly(context.Context, model.TrafficUsageMonthlyRetryMessage) error
+}
+
+type WorkOrderHandler interface {
+	AutoCloseResolved(context.Context, model.WorkOrderAutoCloseMessage) error
+}
+
+type BalanceRechargeHandler interface {
+	CheckRechargeTimeout(context.Context, string) error
+}
+
+type BatchHandler interface {
+	Handle(context.Context, batch.Message) error
+}
+
+type PackageCustomerSyncHandler interface {
+	SyncDeptPackages(context.Context, model.PackageCustomerSyncMessage) error
+}
+
+type TrafficUsageMonthlyStartHandler interface {
+	Produce(context.Context, model.TrafficUsageMonthlyStartMessage) error
+}
+
+type TrafficUsageDailyStartHandler interface {
+	Produce(context.Context, model.TrafficUsageDailyStartMessage) error
+}
+
+type TrafficUsageMonthlyHistoryStartHandler interface {
+	Produce(context.Context, model.TrafficUsageMonthlyHistoryStartMessage) error
+}
+
+type SimStatusQueueStartHandler interface {
+	Produce(context.Context, model.SimStatusQueueStartMessage) error
+}
+
+type SimStatusQueueTaskHandler interface {
+	EnqueueCard(context.Context, model.SimStatusQueueTaskMessage) error
+}
+
+type ImeiQueueStartHandler interface {
+	Produce(context.Context, model.ImeiQueueStartMessage) error
+}
+
+type Consumers struct {
+	trafficUsage                    TrafficUsageHandler
+	workOrder                       WorkOrderHandler
+	balance                         BalanceRechargeHandler
+	balanceRechargeRetryDelay       time.Duration
+	balanceRechargeMaxRetry         int
+	batch                           BatchHandler
+	batchExecuteRetryDelay          time.Duration
+	batchExecuteMaxRetries          int
+	packageCustomerSync             PackageCustomerSyncHandler
+	trafficUsageMonthlyStart        TrafficUsageMonthlyStartHandler
+	trafficUsageDailyStart          TrafficUsageDailyStartHandler
+	trafficUsageMonthlyHistoryStart TrafficUsageMonthlyHistoryStartHandler
+	simStatusQueueStart             SimStatusQueueStartHandler
+	simStatusQueueTask              SimStatusQueueTaskHandler
+	imeiQueueStart                  ImeiQueueStartHandler
+}
+
+// WithDailyUsageStart adds the previous-day usage producer without changing any
+// existing RabbitMQ consumer wiring.
+func (c *Consumers) WithDailyUsageStart(start TrafficUsageDailyStartHandler) *Consumers {
+	c.trafficUsageDailyStart = start
+	return c
+}
+
+// WithMonthlyHistoryStart adds the previous-month usage history producer without
+// changing any existing RabbitMQ consumer wiring.
+func (c *Consumers) WithMonthlyHistoryStart(start TrafficUsageMonthlyHistoryStartHandler) *Consumers {
+	c.trafficUsageMonthlyHistoryStart = start
+	return c
+}
+
+// WithSimStatusQueue adds the independent status queue handlers without
+// changing any existing RabbitMQ consumer wiring.
+func (c *Consumers) WithSimStatusQueue(start SimStatusQueueStartHandler, task SimStatusQueueTaskHandler) *Consumers {
+	c.simStatusQueueStart = start
+	c.simStatusQueueTask = task
+	return c
+}
+
+// WithImeiQueue adds the independent IMEI queue handler without changing any
+// existing RabbitMQ consumer wiring.
+func (c *Consumers) WithImeiQueue(start ImeiQueueStartHandler) *Consumers {
+	c.imeiQueueStart = start
+	return c
+}
+
+func New(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, batchHandlers ...BatchHandler) *Consumers {
+	consumers := &Consumers{
+		trafficUsage: trafficUsage,
+		workOrder:    workOrder,
+		balance:      balance,
+	}
+	if len(batchHandlers) > 0 {
+		consumers.batch = batchHandlers[0]
+	}
+	return consumers
+}
+
+func NewWithPackageCustomerSync(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, packageCustomerSync PackageCustomerSyncHandler, batchHandlers ...BatchHandler) *Consumers {
+	consumers := New(trafficUsage, workOrder, balance, batchHandlers...)
+	consumers.packageCustomerSync = packageCustomerSync
+	return consumers
+}
+
+func NewWithPackageCustomerSyncAndMonthlyUsage(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, packageCustomerSync PackageCustomerSyncHandler, monthlyStart TrafficUsageMonthlyStartHandler, batchHandlers ...BatchHandler) *Consumers {
+	consumers := NewWithPackageCustomerSync(trafficUsage, workOrder, balance, packageCustomerSync, batchHandlers...)
+	consumers.trafficUsageMonthlyStart = monthlyStart
+	return consumers
+}
+
+func (c *Consumers) Start(ctx context.Context) error {
+	initializers := c.initializers(g.Cfg().MustGet(ctx, "workerDiagnostic.enabled", false).Bool())
+	for _, initialize := range initializers {
+		if err := initialize(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Consumers) initializers(workerDiagnosticEnabled bool) []func(context.Context) error {
+	initializers := []func(context.Context) error{
+		c.initTrafficUsageDailyRetry,
+		c.initTrafficUsageMonthlyRetry,
+		c.initWorkOrderAutoClose,
+		c.initBalanceRechargeTimeout,
+		c.initBatchExecute,
+		c.initChannelRefresh,
+	}
+	if c.packageCustomerSync != nil {
+		initializers = append(initializers, c.initPackageCustomerSync)
+	}
+	if c.trafficUsageMonthlyStart != nil {
+		initializers = append(initializers, c.initTrafficUsageMonthlyStart)
+	}
+	if c.trafficUsageDailyStart != nil {
+		initializers = append(initializers, c.initTrafficUsageDailyStart)
+	}
+	if c.trafficUsageMonthlyHistoryStart != nil {
+		initializers = append(initializers, c.initTrafficUsageMonthlyHistoryStart)
+	}
+	if c.simStatusQueueStart != nil && c.simStatusQueueTask != nil {
+		initializers = append(initializers, c.initSimStatusQueueStart, c.initSimStatusQueueTask)
+	}
+	if c.imeiQueueStart != nil {
+		initializers = append(initializers, c.initImeiQueueStart)
+	}
+	if workerDiagnosticEnabled {
+		initializers = append(initializers, c.initWorkerDiagnostic)
+	}
+	return initializers
+}
+
+func (c *Consumers) Stop(ctx context.Context) error {
+	return mq.Close(ctx)
+}

internal/consumer/rabbitmq/consumer_test.go

@@ -0,0 +1,526 @@
+package rabbitmq
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+	amqp "github.com/rabbitmq/amqp091-go"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	dailyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
+	monthlyhistoryproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
+	packagecustomerservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/packagecustomer"
+)
+
+func TestMain(m *testing.M) {
+	g.Log().SetWriter(io.Discard)
+	os.Exit(m.Run())
+}
+
+type trafficUsageHandlerStub struct {
+	dailyCalls   int
+	monthlyCalls int
+	dailyMessage model.TrafficUsageDailyRetryMessage
+	monthMessage model.TrafficUsageMonthlyRetryMessage
+	dailyErr     error
+	monthlyErr   error
+}
+
+func (s *trafficUsageHandlerStub) RetryDaily(_ context.Context, message model.TrafficUsageDailyRetryMessage) error {
+	s.dailyCalls++
+	s.dailyMessage = message
+	return s.dailyErr
+}
+
+func (s *trafficUsageHandlerStub) RetryMonthly(_ context.Context, message model.TrafficUsageMonthlyRetryMessage) error {
+	s.monthlyCalls++
+	s.monthMessage = message
+	return s.monthlyErr
+}
+
+type workOrderHandlerStub struct {
+	calls   int
+	message model.WorkOrderAutoCloseMessage
+	err     error
+}
+
+type balanceRechargeHandlerStub struct {
+	calls      int
+	outTradeNo string
+	err        error
+}
+
+func (s *balanceRechargeHandlerStub) CheckRechargeTimeout(_ context.Context, outTradeNo string) error {
+	s.calls++
+	s.outTradeNo = outTradeNo
+	return s.err
+}
+
+type batchHandlerStub struct {
+	calls   int
+	message batch.Message
+	err     error
+}
+
+func (s *batchHandlerStub) Handle(_ context.Context, message batch.Message) error {
+	s.calls++
+	s.message = message
+	return s.err
+}
+
+type packageCustomerSyncHandlerStub struct {
+	calls   int
+	message model.PackageCustomerSyncMessage
+	err     error
+}
+
+type dailyStartHandlerStub struct {
+	calls   int
+	message model.TrafficUsageDailyStartMessage
+	err     error
+}
+
+func (s *dailyStartHandlerStub) Produce(_ context.Context, message model.TrafficUsageDailyStartMessage) error {
+	s.calls++
+	s.message = message
+	return s.err
+}
+
+type monthlyHistoryStartHandlerStub struct {
+	calls   int
+	message model.TrafficUsageMonthlyHistoryStartMessage
+	err     error
+}
+
+func (s *monthlyHistoryStartHandlerStub) Produce(_ context.Context, message model.TrafficUsageMonthlyHistoryStartMessage) error {
+	s.calls++
+	s.message = message
+	return s.err
+}
+
+func (s *packageCustomerSyncHandlerStub) SyncDeptPackages(_ context.Context, message model.PackageCustomerSyncMessage) error {
+	s.calls++
+	s.message = message
+	return s.err
+}
+
+func (s *workOrderHandlerStub) AutoCloseResolved(_ context.Context, message model.WorkOrderAutoCloseMessage) error {
+	s.calls++
+	s.message = message
+	return s.err
+}
+
+func TestHandleTrafficUsageDailyRetry(t *testing.T) {
+	stubTrafficPublish(t)
+	t.Run("invalid messages are dead lettered", func(t *testing.T) {
+		messages := []model.TrafficUsageDailyRetryMessage{
+			{},
+			{SimKey: &model.SimKey{}, Card: model.TrafficUsageSimCard{Iccid: "card"}},
+			{SimKey: &model.SimKey{ICCID: "sim"}},
+		}
+		for _, message := range messages {
+			handler := &trafficUsageHandlerStub{}
+			consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+			if err := consumers.handleTrafficUsageDailyRetry(context.Background(), message); err != nil {
+				t.Fatalf("invalid message returned error: %v", err)
+			}
+			if handler.dailyCalls != 0 {
+				t.Fatalf("invalid message called handler %d times", handler.dailyCalls)
+			}
+		}
+	})
+
+	t.Run("valid message is delegated", func(t *testing.T) {
+		expectedErr := errors.New("daily failed")
+		expected := model.TrafficUsageDailyRetryMessage{
+			SimKey:    &model.SimKey{ICCID: "sim"},
+			Card:      model.TrafficUsageSimCard{Iccid: "sim", PackageCode: "package"},
+			QueryDate: "20260714",
+		}
+		handler := &trafficUsageHandlerStub{dailyErr: expectedErr}
+		consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+		if err := consumers.handleTrafficUsageDailyRetry(context.Background(), expected); err != nil {
+			t.Fatalf("expected delegated error, got %v", err)
+		}
+		if handler.dailyCalls != 1 || !reflect.DeepEqual(handler.dailyMessage, expected) {
+			t.Fatalf("daily message was not delegated exactly once: %+v", handler.dailyMessage)
+		}
+	})
+}
+
+func TestHandleTrafficUsageMonthlyRetry(t *testing.T) {
+	stubTrafficPublish(t)
+	t.Run("invalid messages are dead lettered", func(t *testing.T) {
+		messages := []model.TrafficUsageMonthlyRetryMessage{
+			{},
+			{SimKey: &model.SimKey{}, Card: model.TrafficUsageSimCard{Iccid: "card"}},
+			{SimKey: &model.SimKey{ICCID: "sim"}},
+		}
+		for _, message := range messages {
+			handler := &trafficUsageHandlerStub{}
+			consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+			if err := consumers.handleTrafficUsageMonthlyRetry(context.Background(), message); err != nil {
+				t.Fatalf("invalid message returned error: %v", err)
+			}
+			if handler.monthlyCalls != 0 {
+				t.Fatalf("invalid message called handler %d times", handler.monthlyCalls)
+			}
+		}
+	})
+
+	t.Run("valid message is delegated", func(t *testing.T) {
+		expectedErr := errors.New("monthly failed")
+		expected := model.TrafficUsageMonthlyRetryMessage{
+			SimKey:    &model.SimKey{ICCID: "sim"},
+			Card:      model.TrafficUsageSimCard{Iccid: "sim", PackageCode: "package"},
+			QueryDate: "202607",
+		}
+		handler := &trafficUsageHandlerStub{monthlyErr: expectedErr}
+		consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+		if err := consumers.handleTrafficUsageMonthlyRetry(context.Background(), expected); err != nil {
+			t.Fatalf("expected delegated error, got %v", err)
+		}
+		if handler.monthlyCalls != 1 || !reflect.DeepEqual(handler.monthMessage, expected) {
+			t.Fatalf("monthly message was not delegated exactly once: %+v", handler.monthMessage)
+		}
+	})
+}
+
+func stubTrafficPublish(t *testing.T) {
+	t.Helper()
+	original, delayed := trafficUsagePublish, trafficUsagePublishDelayed
+	t.Cleanup(func() { trafficUsagePublish, trafficUsagePublishDelayed = original, delayed })
+	trafficUsagePublish = func(context.Context, string, []byte, amqp.Table) error { return nil }
+	trafficUsagePublishDelayed = func(context.Context, string, string, time.Duration, []byte, amqp.Table) error { return nil }
+}
+
+func TestHandleWorkOrderAutoClose(t *testing.T) {
+	handler := &workOrderHandlerStub{}
+	consumers := New(&trafficUsageHandlerStub{}, handler, &balanceRechargeHandlerStub{})
+	if err := consumers.handleWorkOrderAutoClose(context.Background(), model.WorkOrderAutoCloseMessage{}); err != nil {
+		t.Fatalf("invalid message returned error: %v", err)
+	}
+	if handler.calls != 0 {
+		t.Fatalf("invalid message called handler %d times", handler.calls)
+	}
+
+	expectedErr := errors.New("work order failed")
+	expected := model.WorkOrderAutoCloseMessage{WoNo: "WO-1", ResolveTimeTs: 1}
+	handler.err = expectedErr
+	if err := consumers.handleWorkOrderAutoClose(context.Background(), expected); !errors.Is(err, expectedErr) {
+		t.Fatalf("expected delegated error, got %v", err)
+	}
+	if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
+		t.Fatalf("work order message was not delegated exactly once: %+v", handler.message)
+	}
+}
+
+func TestHandleBalanceRechargeTimeout(t *testing.T) {
+	handler := &balanceRechargeHandlerStub{}
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, handler)
+	if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{}); err != nil {
+		t.Fatalf("invalid message returned error: %v", err)
+	}
+	if handler.calls != 0 {
+		t.Fatalf("invalid message called handler %d times", handler.calls)
+	}
+
+	message := model.BalanceRechargeTimeoutMessage{OutTradeNo: "BAL-1"}
+	if err := consumers.handleBalanceRechargeTimeout(context.Background(), message); err != nil {
+		t.Fatalf("valid message returned error: %v", err)
+	}
+	if handler.calls != 1 || handler.outTradeNo != message.OutTradeNo {
+		t.Fatalf("balance recharge message was not delegated exactly once: %+v", handler)
+	}
+}
+
+func TestHandleBatchExecute(t *testing.T) {
+	expectedErr := errors.New("batch failed")
+	expected := batch.Message{BatchNo: "BATCH-1", BatchType: batch.Type103}
+	handler := &batchHandlerStub{err: expectedErr}
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
+
+	if err := consumers.handleBatchExecute(context.Background(), expected); !errors.Is(err, expectedErr) {
+		t.Fatalf("expected delegated error, got %v", err)
+	}
+	if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
+		t.Fatalf("batch message was not delegated exactly once: %+v", handler.message)
+	}
+}
+
+func TestHandlePackageCustomerSync(t *testing.T) {
+	t.Run("invalid message is dropped", func(t *testing.T) {
+		handler := &packageCustomerSyncHandlerStub{}
+		consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
+		if err := consumers.handlePackageCustomerSync(context.Background(), model.PackageCustomerSyncMessage{}); err != nil {
+			t.Fatalf("invalid message returned error: %v", err)
+		}
+		if handler.calls != 0 {
+			t.Fatalf("invalid message called handler %d times", handler.calls)
+		}
+	})
+
+	t.Run("missing dept is dropped", func(t *testing.T) {
+		expected := model.PackageCustomerSyncMessage{TenantId: "000000", DeptId: 164, Trigger: "package_lifecycle", SourceBatchNo: "898604017"}
+		handler := &packageCustomerSyncHandlerStub{err: packagecustomerservice.ErrDeptNotFound}
+		consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
+		if err := consumers.handlePackageCustomerSync(context.Background(), expected); err != nil {
+			t.Fatalf("missing dept returned error: %v", err)
+		}
+		if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
+			t.Fatalf("package customer sync message was not delegated exactly once: %+v", handler.message)
+		}
+	})
+
+	t.Run("other errors are retried", func(t *testing.T) {
+		expectedErr := errors.New("database failed")
+		expected := model.PackageCustomerSyncMessage{TenantId: "000000", DeptId: 117}
+		handler := &packageCustomerSyncHandlerStub{err: expectedErr}
+		consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
+		if err := consumers.handlePackageCustomerSync(context.Background(), expected); !errors.Is(err, expectedErr) {
+			t.Fatalf("expected delegated error, got %v", err)
+		}
+	})
+}
+
+func TestHandleWorkerDiagnostic(t *testing.T) {
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+
+	if err := consumers.handleWorkerDiagnostic(context.Background(), model.WorkerDiagnosticMessage{}); err != nil {
+		t.Fatalf("invalid message returned error: %v", err)
+	}
+
+	message := model.WorkerDiagnosticMessage{
+		TraceID: "worker-test-1",
+		SentAt:  1780000000000,
+		Message: "worker ping",
+	}
+	if err := consumers.handleWorkerDiagnostic(context.Background(), message); err != nil {
+		t.Fatalf("valid message returned error: %v", err)
+	}
+}
+
+func TestWorkerDiagnosticMessageJSONContract(t *testing.T) {
+	message := model.WorkerDiagnosticMessage{
+		TraceID: "worker-test-1",
+		SentAt:  1780000000000,
+		Message: "worker ping",
+	}
+	body, err := json.Marshal(message)
+	if err != nil {
+		t.Fatalf("marshal diagnostic message: %v", err)
+	}
+	const expected = `{"traceId":"worker-test-1","sentAt":1780000000000,"message":"worker ping"}`
+	if string(body) != expected {
+		t.Fatalf("unexpected diagnostic JSON: %s", body)
+	}
+}
+
+func TestChannelRefreshMessageJSONContract(t *testing.T) {
+	message := model.ChannelRefreshMessage{
+		Type:       "channel_rename",
+		OldChanKey: "old_channel",
+		NewChanKey: "new_channel",
+	}
+	body, err := json.Marshal(message)
+	if err != nil {
+		t.Fatalf("marshal channel rename message: %v", err)
+	}
+	const expected = `{"type":"channel_rename","oldChanKey":"old_channel","newChanKey":"new_channel"}`
+	if string(body) != expected {
+		t.Fatalf("unexpected channel rename JSON: %s", body)
+	}
+}
+
+func TestHandleChannelRefreshDropsInvalidRename(t *testing.T) {
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+	for _, message := range []model.ChannelRefreshMessage{
+		{Type: "channel_rename"},
+		{Type: "channel_rename", OldChanKey: "old_channel"},
+		{Type: "channel_rename", NewChanKey: "new_channel"},
+		{Type: "channel_rename", OldChanKey: "same", NewChanKey: "same"},
+	} {
+		if err := consumers.handleChannelRefresh(context.Background(), message); err != nil {
+			t.Fatalf("invalid channel rename returned error: %v", err)
+		}
+	}
+}
+
+func TestHandleChannelRefreshRenameFailureReturnsError(t *testing.T) {
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	message := model.ChannelRefreshMessage{
+		Type:       "channel_rename",
+		OldChanKey: "old_channel",
+		NewChanKey: "new_channel",
+	}
+	// ctx 已取消时 renameChannelWithRetry 会在首次重试等待处返回 ctx.Err()，
+	// 消费者必须把错误向上抛出，让框架 Nack 重新入队而不是 Ack 丢弃。
+	if err := consumers.handleChannelRefresh(ctx, message); err == nil {
+		t.Fatalf("expected failed channel rename to return error for Nack+requeue")
+	}
+}
+
+func TestHandleTrafficUsageDailyStart(t *testing.T) {
+	t.Run("invalid message is dropped", func(t *testing.T) {
+		for _, message := range []model.TrafficUsageDailyStartMessage{
+			{},
+			{RunID: "run-1"},
+			{QueryDate: "20261009"},
+		} {
+			handler := &dailyStartHandlerStub{}
+			consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
+			if err := consumers.handleTrafficUsageDailyStart(context.Background(), message); err != nil {
+				t.Fatalf("invalid message returned error: %v", err)
+			}
+			if handler.calls != 0 {
+				t.Fatalf("invalid message called producer %d times", handler.calls)
+			}
+		}
+	})
+
+	t.Run("valid message is delegated", func(t *testing.T) {
+		expected := model.TrafficUsageDailyStartMessage{
+			RunID:       "daily-usage-20261009-abc",
+			Operation:   model.OperationPreviousDayUsageFullSync,
+			QueryDate:   "20261009",
+			TriggeredAt: "2026-10-10T06:00:00+08:00",
+			Scope:       &model.SyncScope{Channels: []string{"217"}},
+		}
+		handler := &dailyStartHandlerStub{}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
+		if err := consumers.handleTrafficUsageDailyStart(context.Background(), expected); err != nil {
+			t.Fatalf("valid message returned error: %v", err)
+		}
+		if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
+			t.Fatalf("daily start message was not delegated exactly once: %+v", handler.message)
+		}
+	})
+
+	t.Run("an already running producer is acked", func(t *testing.T) {
+		handler := &dailyStartHandlerStub{err: dailyusageproducer.ErrAlreadyRunning}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
+		if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); err != nil {
+			t.Fatalf("ErrAlreadyRunning must be acked, got %v", err)
+		}
+	})
+
+	t.Run("a finished run is acked instead of being resurrected", func(t *testing.T) {
+		handler := &dailyStartHandlerStub{err: dailyusageproducer.ErrRunFinished}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
+		if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); err != nil {
+			t.Fatalf("ErrRunFinished must be acked, got %v", err)
+		}
+	})
+
+	t.Run("other errors are surfaced for requeue", func(t *testing.T) {
+		expectedErr := errors.New("redis unavailable")
+		handler := &dailyStartHandlerStub{err: expectedErr}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
+		if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); !errors.Is(err, expectedErr) {
+			t.Fatalf("expected the producer error to propagate, got %v", err)
+		}
+	})
+}
+
+// TestDailyUsageStartConsumerIsOptIn pins the assembly contract: without an
+// explicit WithDailyUsageStart the initializer list must be unchanged, so a
+// disabled daily chain cannot alter existing deployments.
+func TestDailyUsageStartConsumerIsOptIn(t *testing.T) {
+	base := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+	withDaily := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(&dailyStartHandlerStub{})
+	if got, want := len(withDaily.initializers(false)), len(base.initializers(false))+1; got != want {
+		t.Fatalf("initializers = %d, want %d", got, want)
+	}
+}
+
+func TestHandleTrafficUsageMonthlyHistoryStart(t *testing.T) {
+	t.Run("invalid message is dropped", func(t *testing.T) {
+		for _, message := range []model.TrafficUsageMonthlyHistoryStartMessage{
+			{},
+			{RunID: "run-1"},
+			{QueryMonth: "202609"},
+		} {
+			handler := &monthlyHistoryStartHandlerStub{}
+			consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
+			if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), message); err != nil {
+				t.Fatalf("invalid message returned error: %v", err)
+			}
+			if handler.calls != 0 {
+				t.Fatalf("invalid message called producer %d times", handler.calls)
+			}
+		}
+	})
+
+	t.Run("valid message is delegated", func(t *testing.T) {
+		expected := model.TrafficUsageMonthlyHistoryStartMessage{
+			RunID:       "monthly-history-202609-abc",
+			Operation:   model.OperationPreviousMonthUsageFullSync,
+			QueryMonth:  "202609",
+			TriggeredAt: "2026-10-01T06:00:00+08:00",
+			Scope:       &model.SyncScope{Channels: []string{"217"}},
+		}
+		handler := &monthlyHistoryStartHandlerStub{}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
+		if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), expected); err != nil {
+			t.Fatalf("valid message returned error: %v", err)
+		}
+		if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
+			t.Fatalf("monthly history start message was not delegated exactly once: %+v", handler.message)
+		}
+	})
+
+	t.Run("an already running producer is acked", func(t *testing.T) {
+		handler := &monthlyHistoryStartHandlerStub{err: monthlyhistoryproducer.ErrAlreadyRunning}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
+		if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); err != nil {
+			t.Fatalf("ErrAlreadyRunning must be acked, got %v", err)
+		}
+	})
+
+	t.Run("a finished run is acked instead of being resurrected", func(t *testing.T) {
+		handler := &monthlyHistoryStartHandlerStub{err: monthlyhistoryproducer.ErrRunFinished}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
+		if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); err != nil {
+			t.Fatalf("ErrRunFinished must be acked, got %v", err)
+		}
+	})
+
+	t.Run("other errors are surfaced for requeue", func(t *testing.T) {
+		expectedErr := errors.New("redis unavailable")
+		handler := &monthlyHistoryStartHandlerStub{err: expectedErr}
+		consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
+		if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); !errors.Is(err, expectedErr) {
+			t.Fatalf("expected the producer error to propagate, got %v", err)
+		}
+	})
+}
+
+// TestMonthlyHistoryStartConsumerIsOptIn mirrors the daily chain: a disabled
+// monthly-history chain must not add an initializer.
+func TestMonthlyHistoryStartConsumerIsOptIn(t *testing.T) {
+	base := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+	withHistory := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(&monthlyHistoryStartHandlerStub{})
+	if got, want := len(withHistory.initializers(false)), len(base.initializers(false))+1; got != want {
+		t.Fatalf("initializers = %d, want %d", got, want)
+	}
+}
+
+func TestConsumerInitializersRespectWorkerDiagnosticSwitch(t *testing.T) {
+	consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
+	if got := len(consumers.initializers(false)); got != 6 {
+		t.Fatalf("diagnostic disabled: expected 6 business consumers, got %d", got)
+	}
+	if got := len(consumers.initializers(true)); got != 7 {
+		t.Fatalf("diagnostic enabled: expected 7 consumers, got %d", got)
+	}
+}

internal/consumer/rabbitmq/diagnostic.go

@@ -0,0 +1,43 @@
+package rabbitmq
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func (c *Consumers) initWorkerDiagnostic(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "worker diagnostic consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.WorkerDiagnosticMessage](
+		ctx,
+		consts.QueueWorkerDiagnostic,
+		"iotlcm-worker-diagnostic",
+		c.handleWorkerDiagnostic,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "worker diagnostic consumer started, queue:%s", consts.QueueWorkerDiagnostic)
+	return nil
+}
+
+func (c *Consumers) handleWorkerDiagnostic(ctx context.Context, message model.WorkerDiagnosticMessage) error {
+	if message.TraceID == "" {
+		g.Log().Errorf(ctx, "worker diagnostic message dropped, invalid traceId, message:%+v", message)
+		return nil
+	}
+	g.Log().Infof(
+		ctx,
+		"worker diagnostic message consumed, traceId:%s, sentAt:%d, message:%s",
+		message.TraceID,
+		message.SentAt,
+		message.Message,
+	)
+	return nil
+}

internal/consumer/rabbitmq/imei_queue_start.go

@@ -0,0 +1,44 @@
+package rabbitmq
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	imeiproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/imei"
+)
+
+func (c *Consumers) initImeiQueueStart(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "imei queue start consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.ImeiQueueStartMessage](ctx, consts.QueueImeiQueueStart, "iotlcm-imei-queue-start", c.handleImeiQueueStart); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "imei queue start consumer started, queue:%s", consts.QueueImeiQueueStart)
+	return nil
+}
+
+func (c *Consumers) handleImeiQueueStart(ctx context.Context, message model.ImeiQueueStartMessage) error {
+	if message.RunID == "" {
+		g.Log().Errorf(ctx, "imei queue start message dropped, invalid message:%+v", message)
+		return nil
+	}
+	err := c.imeiQueueStart.Produce(ctx, message)
+	if errors.Is(err, imeiproducer.ErrAlreadyRunning) {
+		g.Log().Warningf(ctx, "imei queue start skipped because a run is active, runId:%s", message.RunID)
+		return nil
+	}
+	// A run that already finished must not be resurrected by a redelivered
+	// message. Ack it instead of retrying forever.
+	if errors.Is(err, imeiproducer.ErrRunFinished) {
+		g.Log().Warningf(ctx, "imei queue start skipped because run already finished, runId:%s", message.RunID)
+		return nil
+	}
+	return err
+}

internal/consumer/rabbitmq/package_customer_sync.go

@@ -0,0 +1,53 @@
+package rabbitmq
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	packagecustomerservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/packagecustomer"
+)
+
+func (c *Consumers) initPackageCustomerSync(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "package customer sync consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if c.packageCustomerSync == nil {
+		return gerror.New("package customer sync handler is not configured")
+	}
+	if err := mq.ConsumeJSON[model.PackageCustomerSyncMessage](
+		ctx,
+		consts.QueuePackageCustomerSync,
+		"iotlcm-package-customer-sync",
+		c.handlePackageCustomerSync,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "package customer sync consumer started, queue:%s", consts.QueuePackageCustomerSync)
+	return nil
+}
+
+func (c *Consumers) handlePackageCustomerSync(ctx context.Context, message model.PackageCustomerSyncMessage) error {
+	if c.packageCustomerSync == nil {
+		return gerror.New("package customer sync handler is not configured")
+	}
+	if message.TenantId == "" || message.DeptId <= 0 {
+		g.Log().Errorf(ctx, "package customer sync message dropped, invalid message:%+v", message)
+		return nil
+	}
+	if err := c.packageCustomerSync.SyncDeptPackages(ctx, message); err != nil {
+		if errors.Is(err, packagecustomerservice.ErrDeptNotFound) {
+			g.Log().Warningf(ctx, "package customer sync message dropped, dept not found, tenantId:%s, deptId:%d, trigger:%s, sourceBatchNo:%s",
+				message.TenantId, message.DeptId, message.Trigger, message.SourceBatchNo)
+			return nil
+		}
+		return err
+	}
+	return nil
+}

internal/consumer/rabbitmq/sim_status_queue_start.go

@@ -0,0 +1,44 @@
+package rabbitmq
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	simstatusproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/simstatus"
+)
+
+func (c *Consumers) initSimStatusQueueStart(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "sim status queue start consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.SimStatusQueueStartMessage](ctx, consts.QueueSimStatusQueueStart, "iotlcm-sim-status-queue-start", c.handleSimStatusQueueStart); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "sim status queue start consumer started, queue:%s", consts.QueueSimStatusQueueStart)
+	return nil
+}
+
+func (c *Consumers) handleSimStatusQueueStart(ctx context.Context, message model.SimStatusQueueStartMessage) error {
+	if message.RunID == "" {
+		g.Log().Errorf(ctx, "sim status queue start message dropped, invalid message:%+v", message)
+		return nil
+	}
+	err := c.simStatusQueueStart.Produce(ctx, message)
+	if errors.Is(err, simstatusproducer.ErrAlreadyRunning) {
+		g.Log().Warningf(ctx, "sim status queue start skipped because a run is active, runId:%s", message.RunID)
+		return nil
+	}
+	// A run that already finished must not be resurrected by a redelivered
+	// message. Ack it instead of retrying forever.
+	if errors.Is(err, simstatusproducer.ErrRunFinished) {
+		g.Log().Warningf(ctx, "sim status queue start skipped because run already finished, runId:%s", message.RunID)
+		return nil
+	}
+	return err
+}

internal/consumer/rabbitmq/sim_status_queue_task.go

@@ -0,0 +1,31 @@
+package rabbitmq
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func (c *Consumers) initSimStatusQueueTask(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "sim status queue task consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.SimStatusQueueTaskMessage](ctx, consts.QueueSimStatusQueueTask, "iotlcm-sim-status-queue-task", c.handleSimStatusQueueTask); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "sim status queue task consumer started, queue:%s", consts.QueueSimStatusQueueTask)
+	return nil
+}
+
+func (c *Consumers) handleSimStatusQueueTask(ctx context.Context, message model.SimStatusQueueTaskMessage) error {
+	if message.CardID <= 0 || message.ICCID == "" || message.ChanKey == "" || message.TempKey == "" {
+		g.Log().Errorf(ctx, "sim status queue task dropped, invalid message:%+v", message)
+		return nil
+	}
+	return c.simStatusQueueTask.EnqueueCard(ctx, message)
+}

internal/consumer/rabbitmq/traffic_usage_daily.go

@@ -0,0 +1,42 @@
+package rabbitmq
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func (c *Consumers) initTrafficUsageDailyRetry(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "traffic usage daily retry consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.TrafficUsageDailyRetryMessage](
+		ctx,
+		consts.QueueTrafficUsageDailyRetry,
+		"iotlcm-traffic-usage-daily",
+		c.handleTrafficUsageDailyRetry,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "traffic usage daily retry consumer started, queue:%s", consts.QueueTrafficUsageDailyRetry)
+	return nil
+}
+
+func (c *Consumers) handleTrafficUsageDailyRetry(ctx context.Context, message model.TrafficUsageDailyRetryMessage) error {
+	err := validateTrafficUsageIdentity(message.SimKey, message.Card)
+	if err == nil {
+		err = c.trafficUsage.RetryDaily(ctx, message)
+	}
+	if err == nil {
+		return nil
+	}
+	message.Attempt++
+	message.LastError = err.Error()
+	permanent := trafficUsageFailureSnapshot(err, &message.UsageMb)
+	return publishTrafficUsageFailure(ctx, consts.QueueTrafficUsageDailyRetry, message, message.Attempt, permanent)
+}

internal/consumer/rabbitmq/traffic_usage_daily_start.go

@@ -0,0 +1,49 @@
+package rabbitmq
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	dailyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
+)
+
+func (c *Consumers) initTrafficUsageDailyStart(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "traffic usage daily start consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.TrafficUsageDailyStartMessage](
+		ctx,
+		consts.QueueTrafficUsageDailyStart,
+		"iotlcm-traffic-usage-daily-start",
+		c.handleTrafficUsageDailyStart,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "traffic usage daily start consumer started, queue:%s", consts.QueueTrafficUsageDailyStart)
+	return nil
+}
+
+func (c *Consumers) handleTrafficUsageDailyStart(ctx context.Context, message model.TrafficUsageDailyStartMessage) error {
+	if message.RunID == "" || message.QueryDate == "" {
+		g.Log().Errorf(ctx, "traffic usage daily start message dropped, invalid message:%+v", message)
+		return nil
+	}
+	err := c.trafficUsageDailyStart.Produce(ctx, message)
+	if errors.Is(err, dailyusageproducer.ErrAlreadyRunning) {
+		g.Log().Warningf(ctx, "traffic usage daily start skipped because previous run is active, runId:%s", message.RunID)
+		return nil
+	}
+	// A run that already finished must not be resurrected by a redelivered
+	// message. Ack it instead of retrying forever.
+	if errors.Is(err, dailyusageproducer.ErrRunFinished) {
+		g.Log().Warningf(ctx, "traffic usage daily start skipped because run already finished, runId:%s", message.RunID)
+		return nil
+	}
+	return err
+}

internal/consumer/rabbitmq/traffic_usage_monthly.go

@@ -0,0 +1,42 @@
+package rabbitmq
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func (c *Consumers) initTrafficUsageMonthlyRetry(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "traffic usage monthly retry consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.TrafficUsageMonthlyRetryMessage](
+		ctx,
+		consts.QueueTrafficUsageMonthlyRetry,
+		"iotlcm-traffic-usage-monthly",
+		c.handleTrafficUsageMonthlyRetry,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "traffic usage monthly retry consumer started, queue:%s", consts.QueueTrafficUsageMonthlyRetry)
+	return nil
+}
+
+func (c *Consumers) handleTrafficUsageMonthlyRetry(ctx context.Context, message model.TrafficUsageMonthlyRetryMessage) error {
+	err := validateTrafficUsageIdentity(message.SimKey, message.Card)
+	if err == nil {
+		err = c.trafficUsage.RetryMonthly(ctx, message)
+	}
+	if err == nil {
+		return nil
+	}
+	message.Attempt++
+	message.LastError = err.Error()
+	permanent := trafficUsageFailureSnapshot(err, &message.UsageMb)
+	return publishTrafficUsageFailure(ctx, consts.QueueTrafficUsageMonthlyRetry, message, message.Attempt, permanent)
+}

internal/consumer/rabbitmq/traffic_usage_monthly_history_start.go

@@ -0,0 +1,49 @@
+package rabbitmq
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	monthlyhistoryproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
+)
+
+func (c *Consumers) initTrafficUsageMonthlyHistoryStart(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "traffic usage monthly history start consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.TrafficUsageMonthlyHistoryStartMessage](
+		ctx,
+		consts.QueueTrafficUsageMonthlyHistoryStart,
+		"iotlcm-traffic-usage-monthly-history-start",
+		c.handleTrafficUsageMonthlyHistoryStart,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "traffic usage monthly history start consumer started, queue:%s", consts.QueueTrafficUsageMonthlyHistoryStart)
+	return nil
+}
+
+func (c *Consumers) handleTrafficUsageMonthlyHistoryStart(ctx context.Context, message model.TrafficUsageMonthlyHistoryStartMessage) error {
+	if message.RunID == "" || message.QueryMonth == "" {
+		g.Log().Errorf(ctx, "traffic usage monthly history start message dropped, invalid message:%+v", message)
+		return nil
+	}
+	err := c.trafficUsageMonthlyHistoryStart.Produce(ctx, message)
+	if errors.Is(err, monthlyhistoryproducer.ErrAlreadyRunning) {
+		g.Log().Warningf(ctx, "traffic usage monthly history start skipped because previous run is active, runId:%s", message.RunID)
+		return nil
+	}
+	// A run that already finished must not be resurrected by a redelivered
+	// message. Ack it instead of retrying forever.
+	if errors.Is(err, monthlyhistoryproducer.ErrRunFinished) {
+		g.Log().Warningf(ctx, "traffic usage monthly history start skipped because run already finished, runId:%s", message.RunID)
+		return nil
+	}
+	return err
+}

internal/consumer/rabbitmq/traffic_usage_monthly_start.go

@@ -0,0 +1,49 @@
+package rabbitmq
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	monthlyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusage"
+)
+
+func (c *Consumers) initTrafficUsageMonthlyStart(ctx context.Context) error {
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "traffic usage monthly start consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.TrafficUsageMonthlyStartMessage](
+		ctx,
+		consts.QueueTrafficUsageMonthlyStart,
+		"iotlcm-traffic-usage-monthly-start",
+		c.handleTrafficUsageMonthlyStart,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "traffic usage monthly start consumer started, queue:%s", consts.QueueTrafficUsageMonthlyStart)
+	return nil
+}
+
+func (c *Consumers) handleTrafficUsageMonthlyStart(ctx context.Context, message model.TrafficUsageMonthlyStartMessage) error {
+	if message.RunID == "" || message.QueryMonth == "" {
+		g.Log().Errorf(ctx, "traffic usage monthly start message dropped, invalid message:%+v", message)
+		return nil
+	}
+	err := c.trafficUsageMonthlyStart.Produce(ctx, message)
+	if errors.Is(err, monthlyusageproducer.ErrAlreadyRunning) {
+		g.Log().Warningf(ctx, "traffic usage monthly start skipped because previous run is active, runId:%s", message.RunID)
+		return nil
+	}
+	// A run that already finished must not be resurrected by a redelivered
+	// message. Ack it instead of retrying forever.
+	if errors.Is(err, monthlyusageproducer.ErrRunFinished) {
+		g.Log().Warningf(ctx, "traffic usage monthly start skipped because run already finished, runId:%s", message.RunID)
+		return nil
+	}
+	return err
+}

internal/consumer/rabbitmq/traffic_usage_retry.go

@@ -0,0 +1,68 @@
+package rabbitmq
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"time"
+
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficusage"
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+var trafficUsagePublish = mq.PublishBytesWithHeaders
+var trafficUsagePublishDelayed = mq.PublishDelayedBytesWithHeaders
+
+func validateTrafficUsageIdentity(simKey *model.SimKey, card model.TrafficUsageSimCard) error {
+	if simKey == nil || card.Iccid == "" || simKey.ICCID != card.Iccid {
+		return &trafficusage.Failure{Cause: gerror.New("invalid traffic usage card identity"), Permanent: true}
+	}
+	return nil
+}
+
+func trafficUsageFailureSnapshot(err error, usage **float64) bool {
+	var failure *trafficusage.Failure
+	if errors.As(err, &failure) {
+		if failure.UsageMb != nil {
+			*usage = failure.UsageMb
+		}
+		return failure.Permanent
+	}
+	return false
+}
+
+func publishTrafficUsageFailure(ctx context.Context, queue string, message any, attempt int, permanent bool) error {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	body, err := json.Marshal(message)
+	if err != nil {
+		return err
+	}
+	if permanent || attempt >= 10 {
+		g.Log().Errorf(ctx, "traffic usage retry exhausted or invalid, queue:%s, attempt:%d, payload:%s", queue, attempt, body)
+		err = trafficUsagePublish(ctx, queue+".dlq", body, nil)
+		return waitTrafficUsagePublishFailure(ctx, err)
+	}
+	delay := 30 * time.Second
+	if attempt >= 3 {
+		delay = 2 * time.Minute
+	}
+	if attempt >= 6 {
+		delay = 5 * time.Minute
+	}
+	err = trafficUsagePublishDelayed(ctx, queue+".delay."+delay.String(), queue, delay, body, nil)
+	return waitTrafficUsagePublishFailure(ctx, err)
+}
+
+func waitTrafficUsagePublishFailure(ctx context.Context, err error) error {
+	if err != nil {
+		select {
+		case <-ctx.Done():
+		case <-time.After(time.Second):
+		}
+	}
+	return err
+}

internal/consumer/rabbitmq/traffic_usage_retry_test.go

@@ -0,0 +1,60 @@
+package rabbitmq
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficusage"
+	amqp "github.com/rabbitmq/amqp091-go"
+	"testing"
+	"time"
+)
+
+func TestTrafficUsageRetryRetainsPayload(t *testing.T) {
+	stubTrafficPublish(t)
+	usage := 12.5
+	handler := &trafficUsageHandlerStub{dailyErr: &trafficusage.Failure{Cause: errors.New("database down"), UsageMb: &usage}}
+	consumer := New(handler, nil, nil)
+	message := model.TrafficUsageDailyRetryMessage{SimKey: &model.SimKey{ICCID: "card"}, Card: model.TrafficUsageSimCard{Iccid: "card"}, QueryDate: "20260831"}
+	calls := 0
+	trafficUsagePublishDelayed = func(_ context.Context, queue, target string, delay time.Duration, body []byte, _ amqp.Table) error {
+		calls++
+		var retry model.TrafficUsageDailyRetryMessage
+		if err := json.Unmarshal(body, &retry); err != nil {
+			t.Fatal(err)
+		}
+		if retry.Attempt != 1 || retry.UsageMb == nil || *retry.UsageMb != usage || retry.QueryDate != message.QueryDate || retry.LastError == "" || delay != 30*time.Second || target != consts.QueueTrafficUsageDailyRetry || queue == target {
+			t.Fatalf("invalid retry: %+v", retry)
+		}
+		return nil
+	}
+	if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); err != nil || calls != 1 {
+		t.Fatalf("calls=%d err=%v", calls, err)
+	}
+	message.Attempt = 9
+	dlqCalls := 0
+	trafficUsagePublish = func(_ context.Context, queue string, body []byte, _ amqp.Table) error {
+		dlqCalls++
+		if queue != consts.QueueTrafficUsageDailyRetry+".dlq" {
+			t.Fatal(queue)
+		}
+		var retry model.TrafficUsageDailyRetryMessage
+		if err := json.Unmarshal(body, &retry); err != nil {
+			t.Fatal(err)
+		}
+		if retry.Attempt != 10 || retry.UsageMb == nil {
+			t.Fatal("DLQ lost snapshot")
+		}
+		return nil
+	}
+	if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); err != nil || dlqCalls != 1 || calls != 1 {
+		t.Fatalf("DLQ not published: %v", err)
+	}
+	expected := errors.New("broker unavailable")
+	trafficUsagePublish = func(context.Context, string, []byte, amqp.Table) error { return expected }
+	if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); !errors.Is(err, expected) {
+		t.Fatal("failed publication must not ack original")
+	}
+}

internal/consumer/rabbitmq/work_order.go

@@ -0,0 +1,53 @@
+package rabbitmq
+
+import (
+	"context"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func (c *Consumers) initWorkOrderAutoClose(ctx context.Context) error {
+	if scanner, ok := c.workOrder.(interface{ CloseOverdue(context.Context) error }); ok {
+		go func() {
+			ticker := time.NewTicker(5 * time.Minute)
+			defer ticker.Stop()
+			for {
+				if err := scanner.CloseOverdue(ctx); err != nil {
+					g.Log().Errorf(ctx, "工单到期补偿失败: %v", err)
+				}
+				select {
+				case <-ctx.Done():
+					return
+				case <-ticker.C:
+				}
+			}
+		}()
+	}
+	if !mq.Enabled(ctx) {
+		g.Log().Info(ctx, "work order auto close consumer skipped, RabbitMQ disabled")
+		return nil
+	}
+	if err := mq.ConsumeJSON[model.WorkOrderAutoCloseMessage](
+		ctx,
+		consts.QueueWorkOrderAutoClose,
+		"iotlcm-work-order-auto-close",
+		c.handleWorkOrderAutoClose,
+	); err != nil {
+		return err
+	}
+	g.Log().Infof(ctx, "work order auto close consumer started, queue:%s", consts.QueueWorkOrderAutoClose)
+	return nil
+}
+
+func (c *Consumers) handleWorkOrderAutoClose(ctx context.Context, message model.WorkOrderAutoCloseMessage) error {
+	if message.WoNo == "" {
+		g.Log().Errorf(ctx, "work order auto close message dropped, invalid message:%+v", message)
+		return nil
+	}
+	return c.workOrder.AutoCloseResolved(ctx, message)
+}

internal/consumer/zset/consumer_test.go

@@ -0,0 +1,706 @@
+package zset
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type redisCall struct {
+	command string
+	args    []any
+}
+
+type fakeRedis struct {
+	mu        sync.Mutex
+	calls     []redisCall
+	responses []*gvar.Var
+}
+
+func (f *fakeRedis) Do(_ context.Context, command string, args ...any) (*gvar.Var, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.calls = append(f.calls, redisCall{command: command, args: args})
+	if len(f.responses) == 0 {
+		return gvar.New(int64(1)), nil
+	}
+	response := f.responses[0]
+	f.responses = f.responses[1:]
+	return response, nil
+}
+
+func (f *fakeRedis) Eval(_ context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
+	allArgs := []any{script, numKeys}
+	for _, key := range keys {
+		allArgs = append(allArgs, key)
+	}
+	allArgs = append(allArgs, args...)
+	return f.Do(context.Background(), "EVAL", allArgs...)
+}
+
+type testHandler struct {
+	query func(context.Context, queue.Task) Result
+	apply func(context.Context, []Result) error
+}
+
+func (h *testHandler) Query(ctx context.Context, task queue.Task) Result {
+	if h.query != nil {
+		return h.query(ctx, task)
+	}
+	return Result{Task: task, Success: true, UsageMB: float64(task.CardID)}
+}
+
+func (h *testHandler) Apply(ctx context.Context, results []Result) error {
+	if h.apply != nil {
+		return h.apply(ctx, results)
+	}
+	return nil
+}
+
+func taskMember(task queue.Task) string {
+	value, _ := json.Marshal(task)
+	return string(value)
+}
+
+func claimedTask(task queue.Task) queue.ClaimedTask {
+	return queue.ClaimedTask{Task: task, Member: taskMember(task), LeaseToken: "token-" + task.TaskID}
+}
+
+func newTestConsumer(t *testing.T, fake *fakeRedis, handler Handler, config Config) *Consumer {
+	t.Helper()
+	store, err := queue.New(fake, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	consumer, err := New(store, handler, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return consumer
+}
+
+func TestRoundSummaryAccumulatesAndFlushes(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{gvar.New([]any{"runId", "run-1"})}}
+	consumer := newTestConsumer(t, fake, &testHandler{}, Config{RoundRetryEnabled: true})
+	started := time.Date(2026, 9, 22, 3, 0, 0, 0, time.UTC)
+	consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
+		claimedTask(queue.Task{RunID: "run-1", TaskID: "a"}),
+		claimedTask(queue.Task{RunID: "run-1", TaskID: "b"}),
+	}, []queue.TaskOutcome{
+		{Success: true},
+		{Retry: true},
+	}, started)
+	consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
+		claimedTask(queue.Task{RunID: "run-1", TaskID: "c"}),
+	}, []queue.TaskOutcome{{Success: false}}, started.Add(2*time.Second))
+
+	key := "run-1\x001"
+	consumer.summaryMu.Lock()
+	summary := *consumer.roundSummaries[key]
+	consumer.summaryMu.Unlock()
+	if summary.inputCount != 3 || summary.successCount != 1 || summary.failedCount != 2 || summary.deferredCount != 1 || summary.terminalFailedCount != 1 {
+		t.Fatalf("unexpected summary: %+v", summary)
+	}
+	if !summary.startedAt.Equal(started) {
+		t.Fatalf("startedAt = %v, want %v", summary.startedAt, started)
+	}
+
+	consumer.flushRoundSummary(context.Background(), "run-1", 1)
+	consumer.summaryMu.Lock()
+	_, exists := consumer.roundSummaries[key]
+	consumer.summaryMu.Unlock()
+	if exists {
+		t.Fatal("round summary was not removed after flush")
+	}
+}
+
+func TestHandleClaimsAppliesSuccessOnceAndAcknowledgesBatch(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
+	applyCalls := 0
+	var applied []Result
+	handler := &testHandler{apply: func(_ context.Context, results []Result) error {
+		applyCalls++
+		applied = append(applied, results...)
+		return nil
+	}}
+	consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
+	claimed := []queue.ClaimedTask{
+		claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
+		claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
+	}
+	if err := consumer.handleClaims(context.Background(), claimed); err != nil {
+		t.Fatal(err)
+	}
+	if applyCalls != 1 || len(applied) != 2 {
+		t.Fatalf("apply calls = %d, results = %d", applyCalls, len(applied))
+	}
+	if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
+		t.Fatalf("Redis calls = %+v, want one batch ack", fake.calls)
+	}
+}
+
+func TestDatabaseFailureLeavesEntireBatchLeased(t *testing.T) {
+	fake := &fakeRedis{}
+	handler := &testHandler{
+		query: func(_ context.Context, task queue.Task) Result {
+			if task.CardID == 2 {
+				return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
+			}
+			return Result{Task: task, Success: true, UsageMB: 1.5}
+		},
+		apply: func(context.Context, []Result) error { return errors.New("database unavailable") },
+	}
+	consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
+	claimed := []queue.ClaimedTask{
+		claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
+		claimedTask(queue.Task{TaskID: "failed", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
+	}
+	err := consumer.handleClaims(context.Background(), claimed)
+	if err == nil || err.Error() != "apply monthly usage result batch: database unavailable" {
+		t.Fatalf("handleClaims() error = %v", err)
+	}
+	if len(fake.calls) != 0 {
+		t.Fatalf("Redis calls = %+v, want no ACK when database update fails", fake.calls)
+	}
+}
+
+func TestBusinessFailuresAreTerminalWithoutRetry(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
+	failureCount := 0
+	handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
+		return Result{Task: task, ErrorClass: "business", Code: 13000, Message: "busy"}
+	}}
+	consumer := newTestConsumer(t, fake, handler, Config{
+		MobileConcurrency:  2,
+		TelecomConcurrency: 1,
+		OnFailure: func(context.Context, Result) {
+			failureCount++
+		},
+	})
+	claimed := []queue.ClaimedTask{
+		claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
+		claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
+	}
+	if err := consumer.handleClaims(context.Background(), claimed); err != nil {
+		t.Fatal(err)
+	}
+	if failureCount != 2 {
+		t.Fatalf("failure count = %d, want 2", failureCount)
+	}
+	if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
+		t.Fatalf("Redis calls = %+v; business failures must only be acknowledged", fake.calls)
+	}
+}
+
+func TestRetryableFailuresRemainLeasedWhileSuccessesAreAcknowledged(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(1))}}
+	var applied []Result
+	handler := &testHandler{
+		query: func(_ context.Context, task queue.Task) Result {
+			if task.CardID == 1 {
+				return Result{Task: task, ErrorClass: "network", Retryable: true, Message: "temporary failure"}
+			}
+			return Result{Task: task, Success: true, UsageMB: 2}
+		},
+		apply: func(_ context.Context, results []Result) error {
+			applied = append(applied, results...)
+			return nil
+		},
+	}
+	consumer := newTestConsumer(t, fake, handler, Config{MobileConcurrency: 2, TelecomConcurrency: 1})
+	claimed := []queue.ClaimedTask{
+		claimedTask(queue.Task{TaskID: "retry", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 1}),
+		claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 2}),
+	}
+	if err := consumer.handleClaims(context.Background(), claimed); err != nil {
+		t.Fatal(err)
+	}
+	if len(applied) != 2 {
+		t.Fatalf("applied results = %d, want both result rows for handler", len(applied))
+	}
+	if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
+		t.Fatalf("Redis calls = %+v, want one ACK for successful/terminal result", fake.calls)
+	}
+	if len(fake.calls[0].args) == 0 {
+		t.Fatal("ACK call did not include arguments")
+	}
+}
+
+func TestRunRejectsAnotherActiveConsumer(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{gvar.New(nil)}}
+	consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
+	if err := consumer.Run(context.Background()); !errors.Is(err, ErrAlreadyRunning) {
+		t.Fatalf("Run error = %v, want ErrAlreadyRunning", err)
+	}
+}
+
+func TestProcessOnceStartsEveryRegisteredChannel(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{
+		gvar.New([]any{"mobile-a", "telecom-a", "mobile-b"}),
+		gvar.New([]any{}),
+		gvar.New([]any{}),
+		gvar.New([]any{}),
+	}}
+	consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
+	processed, err := consumer.processOnce(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !processed {
+		t.Fatal("processOnce() did not start registered channels")
+	}
+	deadline := time.Now().Add(time.Second)
+	for {
+		fake.mu.Lock()
+		callCount := len(fake.calls)
+		fake.mu.Unlock()
+		if callCount == 4 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("Redis calls = %d, want channels + 3 claims", callCount)
+		}
+		time.Sleep(time.Millisecond)
+	}
+	consumer.channelGroup.Wait()
+	if len(fake.calls) != 4 {
+		t.Fatalf("Redis calls = %d, want channels + 3 claims", len(fake.calls))
+	}
+}
+
+type channelRedis struct {
+	mu          sync.Mutex
+	claims      map[string][]queue.Task
+	claimCounts map[string]int
+}
+
+func (r *channelRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
+	return nil, errors.New("unexpected command " + command)
+}
+
+func (r *channelRedis) Eval(_ context.Context, script string, _ int64, keys []string, args []any) (*gvar.Var, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if strings.Contains(script, "ZRANGEBYSCORE") {
+		chanKey := strings.TrimPrefix(keys[0], "flow:")
+		r.claimCounts[chanKey]++
+		tasks := r.claims[chanKey]
+		if len(tasks) == 0 {
+			return gvar.New([]any{}), nil
+		}
+		task := tasks[0]
+		r.claims[chanKey] = tasks[1:]
+		return gvar.New([]any{taskMember(task), "token-" + task.TaskID}), nil
+	}
+	if len(keys) == 2 && len(args) == 2 {
+		return gvar.New(int64(1)), nil
+	}
+	return nil, errors.New("unexpected eval")
+}
+
+func TestSlowChannelDoesNotBlockAnotherChannelFromClaimingAgain(t *testing.T) {
+	redis := &channelRedis{
+		claims: map[string][]queue.Task{
+			"telecom-a": {{TaskID: "t1", RunID: "run", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}},
+			"mobile-a": {
+				{TaskID: "m1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
+				{TaskID: "m2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
+			},
+		},
+		claimCounts: make(map[string]int),
+	}
+	telecomStarted := make(chan struct{})
+	releaseTelecom := make(chan struct{})
+	handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
+		if task.ChanKey == "telecom-a" {
+			close(telecomStarted)
+			<-releaseTelecom
+		}
+		return Result{Task: task, Success: true}
+	}}
+	store, err := queue.New(redis, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	consumer, err := New(store, handler, Config{ClaimBatch: 1, MobileConcurrency: 1, TelecomConcurrency: 1, LeaseTTL: time.Minute})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	consumer.startChannel(ctx, "telecom-a")
+	consumer.startChannel(ctx, "mobile-a")
+	select {
+	case <-telecomStarted:
+	case <-time.After(time.Second):
+		t.Fatal("telecom channel did not start")
+	}
+	if consumer.startChannel(ctx, "telecom-a") {
+		t.Fatal("channel executor started twice while first executor is running")
+	}
+	deadline := time.Now().Add(time.Second)
+	for {
+		redis.mu.Lock()
+		mobileClaims := redis.claimCounts["mobile-a"]
+		redis.mu.Unlock()
+		if mobileClaims >= 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("mobile claim count = %d while telecom is blocked", mobileClaims)
+		}
+		time.Sleep(time.Millisecond)
+	}
+	close(releaseTelecom)
+	consumer.channelGroup.Wait()
+}
+
+func TestClaimedBatchRenewsLeaseUntilAcknowledged(t *testing.T) {
+	fake := &fakeRedis{}
+	handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
+		time.Sleep(45 * time.Millisecond)
+		return Result{Task: task, Success: true}
+	}}
+	consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
+	claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
+	if err := consumer.handleClaims(context.Background(), claimed); err != nil {
+		t.Fatal(err)
+	}
+	fake.mu.Lock()
+	defer fake.mu.Unlock()
+	if len(fake.calls) < 2 {
+		t.Fatalf("Redis calls = %d, want lease renewal and ACK", len(fake.calls))
+	}
+}
+
+func TestLostBatchLeasePreventsApplyAndAck(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(0))}}
+	applyCalls := 0
+	handler := &testHandler{
+		query: func(ctx context.Context, task queue.Task) Result {
+			<-ctx.Done()
+			return Result{Task: task, ErrorClass: "canceled"}
+		},
+		apply: func(context.Context, []Result) error {
+			applyCalls++
+			return nil
+		},
+	}
+	consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
+	claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
+	err := consumer.handleClaims(context.Background(), claimed)
+	if !errors.Is(err, ErrLeaseLost) {
+		t.Fatalf("handleClaims() error = %v, want ErrLeaseLost", err)
+	}
+	if applyCalls != 0 {
+		t.Fatalf("apply calls = %d, want 0 after lease loss", applyCalls)
+	}
+	fake.mu.Lock()
+	defer fake.mu.Unlock()
+	if len(fake.calls) != 1 {
+		t.Fatalf("Redis calls = %d, want only failed lease renewal", len(fake.calls))
+	}
+}
+
+func newTrackingHandler(release chan struct{}, current, maximum *atomic.Int32) *testHandler {
+	return &testHandler{query: func(_ context.Context, task queue.Task) Result {
+		active := current.Add(1)
+		for {
+			observed := maximum.Load()
+			if active <= observed || maximum.CompareAndSwap(observed, active) {
+				break
+			}
+		}
+		<-release
+		current.Add(-1)
+		return Result{Task: task, Success: true}
+	}}
+}
+
+func TestMobileChannelsHaveIndependentConcurrencyPools(t *testing.T) {
+	var current atomic.Int32
+	var maximum atomic.Int32
+	release := make(chan struct{})
+	consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, &current, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
+	var group sync.WaitGroup
+	for index := 0; index < 3; index++ {
+		group.Add(1)
+		go func(index int) {
+			defer group.Done()
+			task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-" + string(rune('a'+index)), TempKey: mobileTempKey, CardID: int64(index + 1)}
+			_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
+		}(index)
+	}
+	time.Sleep(100 * time.Millisecond)
+	if maximum.Load() != 3 {
+		t.Fatalf("maximum in-flight across channels = %d, want 3 (one pool per channel)", maximum.Load())
+	}
+	close(release)
+	group.Wait()
+}
+
+func TestChannelConcurrencyPoolCapsWithinChannel(t *testing.T) {
+	var current atomic.Int32
+	var maximum atomic.Int32
+	release := make(chan struct{})
+	consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, &current, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
+	var group sync.WaitGroup
+	for index := 0; index < 3; index++ {
+		group.Add(1)
+		go func(index int) {
+			defer group.Done()
+			task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-a", TempKey: mobileTempKey, CardID: int64(index + 1)}
+			_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
+		}(index)
+	}
+	time.Sleep(100 * time.Millisecond)
+	if maximum.Load() != 2 {
+		t.Fatalf("maximum in-flight within one channel = %d, want 2", maximum.Load())
+	}
+	close(release)
+	group.Wait()
+}
+
+func TestMobileAndTelecomUseIndependentChannelPools(t *testing.T) {
+	started := make(chan carrier, 2)
+	release := make(chan struct{})
+	handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
+		channelCarrier, _ := policyForTemplate(task.TempKey, Config{})
+		started <- channelCarrier
+		<-release
+		return Result{Task: task, Success: true}
+	}}
+	consumer := newTestConsumer(t, &fakeRedis{}, handler, Config{MobileConcurrency: 1, TelecomConcurrency: 1})
+	go func() {
+		task := queue.Task{TaskID: "mobile", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1}
+		_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
+	}()
+	go func() {
+		task := queue.Task{TaskID: "telecom", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}
+		_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierTelecom, telecomQPS)
+	}()
+
+	seen := map[carrier]bool{}
+	for len(seen) < 2 {
+		select {
+		case value := <-started:
+			seen[value] = true
+		case <-time.After(time.Second):
+			t.Fatalf("carrier pools did not progress independently: %+v", seen)
+		}
+	}
+	close(release)
+}
+
+func TestUnknownTemplateUsesGenericStrategy(t *testing.T) {
+	fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
+	queryCalls := 0
+	failures := 0
+	applyCalls := 0
+	handler := &testHandler{
+		query: func(_ context.Context, task queue.Task) Result {
+			queryCalls++
+			return Result{Task: task, Success: true, UsageMB: 1}
+		},
+		apply: func(_ context.Context, results []Result) error {
+			applyCalls++
+			for _, result := range results {
+				if !result.Success || result.BlockChannel {
+					t.Fatalf("unexpected result = %+v", result)
+				}
+			}
+			return nil
+		},
+	}
+	consumer := newTestConsumer(t, fake, handler, Config{OnFailure: func(context.Context, Result) { failures++ }})
+	claimed := []queue.ClaimedTask{
+		claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 1, CardID: 1}),
+		claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 2, CardID: 2}),
+	}
+	if err := consumer.handleClaims(context.Background(), claimed); err != nil {
+		t.Fatal(err)
+	}
+	if queryCalls != 2 || failures != 0 || applyCalls != 1 {
+		t.Fatalf("query calls = %d, failures = %d, apply calls = %d", queryCalls, failures, applyCalls)
+	}
+	if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
+		t.Fatalf("Redis calls = %+v, want one ack script", fake.calls)
+	}
+	if len(fake.responses) != 0 {
+		t.Fatalf("unexpected leftover scripted responses: %+v", fake.responses)
+	}
+}
+
+func TestChannelPacerLimitsEachChannelIndependently(t *testing.T) {
+	pacer := newChannelPacer()
+	now := time.Unix(100, 0)
+	pacedQPS := 30.0
+	if delay := pacer.reserve("mobile-a", pacedQPS, now); delay != 0 {
+		t.Fatalf("first reservation delay = %s", delay)
+	}
+	if delay := pacer.reserve("mobile-a", pacedQPS, now); delay < 33*time.Millisecond || delay > 34*time.Millisecond {
+		t.Fatalf("second reservation delay = %s", delay)
+	}
+	if delay := pacer.reserve("mobile-b", pacedQPS, now); delay != 0 {
+		t.Fatalf("independent channel delay = %s", delay)
+	}
+}
+
+func TestChannelPacerSkipsWhenQPSIsDisabled(t *testing.T) {
+	pacer := newChannelPacer()
+	now := time.Unix(100, 0)
+	if delay := pacer.reserve("mobile-a", 0, now); delay != 0 {
+		t.Fatalf("disabled qps delay = %s", delay)
+	}
+	if len(pacer.next) != 0 {
+		t.Fatalf("disabled qps should not reserve channel slots, got %+v", pacer.next)
+	}
+}
+
+func TestNewDefaultsTaskTimeout(t *testing.T) {
+	consumer := newTestConsumer(t, &fakeRedis{}, &testHandler{}, Config{})
+	if consumer.config.TaskTimeout != 30*time.Second {
+		t.Fatalf("TaskTimeout = %s, want 30s default", consumer.config.TaskTimeout)
+	}
+	if consumer.config.GenericQPS != defaultGenericQPS {
+		t.Fatalf("GenericQPS = %v, want %v default", consumer.config.GenericQPS, defaultGenericQPS)
+	}
+	if _, err := New(consumer.store, &testHandler{}, Config{LeaseTTL: 10 * time.Second}); err == nil {
+		t.Fatal("New() with LeaseTTL below default TaskTimeout should fail")
+	}
+}
+
+func TestPolicyForTemplateUsesTemplateValues(t *testing.T) {
+	tests := []struct {
+		name    string
+		tempKey string
+		carrier carrier
+		qps     float64
+	}{
+		{name: "mobile", tempKey: mobileTempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "mobile mock", tempKey: mobileMockKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Tengyu", tempKey: sharedTengyuTempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Xuzhou mobile", tempKey: sharedXuzhouMobileTempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Lelian", tempKey: sharedLelianTempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Qianxun", tempKey: sharedQianxunTempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Henan", tempKey: sharedHenanTempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Daihe", tempKey: sharedDaiheTempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Beixin", tempKey: sharedBeixinTempKey, carrier: carrierMobile, qps: mobileQPS},
+		// lanshang_8000/8003 are OneLink shared-token templates whose names
+		// omit the sim_onelink_ prefix. Before this case was added they fell
+		// through to carrierGeneric and were throttled to 1.5 QPS, which made a
+		// 6k-card status/traffic/IMEI run take over an hour per channel.
+		{name: "shared Lanshang 8000", tempKey: sharedLanshang8000TempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Lanshang 8003", tempKey: sharedLanshang8003TempKey, carrier: carrierMobile, qps: mobileQPS},
+		{name: "shared Lanshang case and padding", tempKey: " LANSHANG_8000 ", carrier: carrierMobile, qps: mobileQPS},
+		{name: "telecom", tempKey: telecomTempKey, carrier: carrierTelecom, qps: telecomQPS},
+		{name: "telecom v3", tempKey: " SIM_DX_V3 ", carrier: carrierTelecom, qps: telecomQPS},
+		{name: "unknown telecom version", tempKey: "sim_dx_v2_future", carrier: carrierGeneric, qps: defaultGenericQPS},
+		{name: "unknown template", tempKey: "unknown-template", carrier: carrierGeneric, qps: defaultGenericQPS},
+		{name: "missing template", tempKey: "", carrier: carrierGeneric, qps: defaultGenericQPS},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotCarrier, gotQPS := policyForTemplate(test.tempKey, Config{})
+			if gotCarrier != test.carrier || gotQPS != test.qps {
+				t.Fatalf("policyForTemplate(%q) = %q,%v", test.tempKey, gotCarrier, gotQPS)
+			}
+		})
+	}
+}
+
+func TestPolicyForTemplateConfigurationOverridesDefaults(t *testing.T) {
+	config := Config{
+		GenericQPS: 2.5,
+		TemplateQPS: map[string]float64{
+			" SIM_YD_V5 ":      12,
+			"unknown-template": 3,
+			"invalid":          0,
+		},
+	}
+	config.setDefaults()
+
+	if gotCarrier, gotQPS := policyForTemplate("sim_yd_v5", config); gotCarrier != carrierMobile || gotQPS != 12 {
+		t.Fatalf("mobile override = %q,%v, want mobile,12", gotCarrier, gotQPS)
+	}
+	if gotCarrier, gotQPS := policyForTemplate("unknown-template", config); gotCarrier != carrierGeneric || gotQPS != 3 {
+		t.Fatalf("generic override = %q,%v, want generic,3", gotCarrier, gotQPS)
+	}
+	if gotCarrier, gotQPS := policyForTemplate("other-template", config); gotCarrier != carrierGeneric || gotQPS != 2.5 {
+		t.Fatalf("generic default = %q,%v, want generic,2.5", gotCarrier, gotQPS)
+	}
+}
+
+// TestFailurePoolAlertThresholdIsObservabilityOnly 覆盖失败池告警：
+//   - 未配置阀值（0）时不发 HLEN 查询
+//   - 失败数低于阀值时不告警
+//   - 超过阀值时发一次 HLEN 并打 WARN，但绝不截断 Run
+//
+// 告警本身是纯观测行为，所以断言的是「有没有多打 Redis 命令」，
+// 而不是「Run 有没有被改坏」——后者必须永远为「没被改坏」。
+func TestFailurePoolAlertThresholdIsObservabilityOnly(t *testing.T) {
+	newMeta := func(failed int64) queue.RunMeta {
+		return queue.RunMeta{RunID: "run-1", Status: "completed_with_failures", FailedCount: failed, CurrentRound: 1}
+	}
+
+	cases := []struct {
+		name          string
+		threshold     int64
+		failed        int64
+		wantHLenCalls int
+	}{
+		{"未配置阀值不查询", 0, 1_000_000, 0},
+		{"低于阀值不告警", 50000, 49999, 0},
+		{"等于阀值不告警", 50000, 50000, 0},
+		{"超过阀值告警一次", 50000, 50001, 1},
+		{"远超阀值也只告警一次", 50000, 500_000, 1},
+	}
+	for _, tc := range cases {
+		fake := &fakeRedis{}
+		consumer := newTestConsumer(t, fake, &testHandler{}, Config{
+			FailurePoolAlertThreshold: tc.threshold,
+		})
+		consumer.reportFailurePoolSize(context.Background(), "run-1", newMeta(tc.failed))
+
+		hlen := 0
+		for _, call := range fake.calls {
+			if call.command == "HLEN" {
+				hlen++
+			}
+		}
+		if hlen != tc.wantHLenCalls {
+			t.Fatalf("%s: HLEN 调用 %d 次, want %d", tc.name, hlen, tc.wantHLenCalls)
+		}
+	}
+}
+
+// TestFailurePoolAlertReadsCorrectKey 告警必须查该 Run 自己的失败池键。
+func TestFailurePoolAlertReadsCorrectKey(t *testing.T) {
+	fake := &fakeRedis{}
+	consumer := newTestConsumer(t, fake, &testHandler{}, Config{FailurePoolAlertThreshold: 10})
+	consumer.reportFailurePoolSize(context.Background(), "run-abc", queue.RunMeta{
+		RunID: "run-abc", Status: "completed_with_failures", FailedCount: 11, CurrentRound: 1,
+	})
+
+	var key string
+	for _, call := range fake.calls {
+		if call.command == "HLEN" {
+			if len(call.args) > 0 {
+				key, _ = call.args[0].(string)
+			}
+		}
+	}
+	want, _ := consumer.store.FailedKey("run-abc")
+	if key != want {
+		t.Fatalf("HLEN key = %q, want %q", key, want)
+	}
+}

internal/consumer/zset/run_integration_test.go

@@ -0,0 +1,480 @@
+package zset
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+type runHandler struct {
+	query func(queue.Task) Result
+	apply func([]Result) error
+}
+
+func (h runHandler) Query(_ context.Context, task queue.Task) Result { return h.query(task) }
+func (h runHandler) Apply(_ context.Context, results []Result) error {
+	if h.apply != nil {
+		return h.apply(results)
+	}
+	return nil
+}
+
+func runTasks(t *testing.T, store *queue.Store, run string, count int) []queue.Task {
+	t.Helper()
+	ctx := context.Background()
+	if err := store.CreateRun(ctx, queue.RunMeta{RunID: run}); err != nil {
+		t.Fatal(err)
+	}
+	tasks := make([]queue.Task, count)
+	for i := range tasks {
+		tasks[i] = queue.Task{RunID: run, TaskID: fmt.Sprintf("%s:%d", run, i), CardID: int64(i + 1), ChanKey: "channel", TempKey: mobileTempKey, ICCID: fmt.Sprintf("card-%d", i)}
+	}
+	if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != count {
+		t.Fatalf("enqueue %d: %v", n, err)
+	}
+	return tasks
+}
+
+func runMeta(t *testing.T, store *queue.Store, run string) queue.RunMeta {
+	t.Helper()
+	meta, err := store.GetRunMeta(context.Background(), run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return meta
+}
+
+func runConsumer(t *testing.T, store *queue.Store, handler Handler, rounds bool) *Consumer {
+	t.Helper()
+	c, err := New(store, handler, Config{RunQueueEnabled: true, RoundRetryEnabled: rounds, BlockedTasksAsFailures: !rounds, EventPrefix: "sim_status_queue", TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c
+}
+
+func claimRun(t *testing.T, store *queue.Store, run string, now time.Time, count int) []queue.ClaimedTask {
+	t.Helper()
+	claims, err := store.ClaimRun(context.Background(), run, "channel", now, now.Add(time.Minute), count)
+	if err != nil || len(claims) != count {
+		t.Fatalf("claim got %d want %d: %v", len(claims), count, err)
+	}
+	return claims
+}
+
+func finishProducer(t *testing.T, store *queue.Store, run string) {
+	t.Helper()
+	if err := store.MarkProducerDone(context.Background(), run); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func checkTerminal(t *testing.T, store *queue.Store, run string, success, failed int64) {
+	t.Helper()
+	meta := runMeta(t, store, run)
+	want := "completed"
+	if failed > 0 {
+		want = "completed_with_failures"
+	}
+	if meta.Status != want || meta.CompletedAt.IsZero() || meta.SuccessCount != success || meta.FailedCount != failed || meta.ActiveCount != 0 || meta.InflightCount != 0 || meta.NextRoundCount != 0 {
+		t.Fatalf("unexpected terminal metadata: %+v", meta)
+	}
+}
+
+func TestRunConsumerCountsAndCompletes(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	for _, failure := range []bool{false, true} {
+		t.Run(fmt.Sprint(failure), func(t *testing.T) {
+			store, _ := queue.New(redis, "sim-status:")
+			run := fmt.Sprint("counts-", failure)
+			tasks := runTasks(t, store, run, 3)
+			if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != 0 {
+				t.Fatalf("duplicate enqueue: %d %v", n, err)
+			}
+			finishProducer(t, store, run)
+			handler := runHandler{query: func(task queue.Task) Result {
+				return Result{Task: task, Success: !failure || task.CardID != 2, ErrorClass: "business"}
+			}, apply: func(_ []Result) error {
+				if meta := runMeta(t, store, run); meta.SuccessCount != 0 || meta.FailedCount != 0 {
+					return errors.New("counted before DB apply")
+				}
+				return nil
+			}}
+			c := runConsumer(t, store, handler, false)
+			claims := claimRun(t, store, run, time.Now(), 3)
+			if err := c.handleRunClaims(ctx, claims); err != nil {
+				t.Fatal(err)
+			}
+			var failures int64
+			if failure {
+				failures = 1
+			}
+			checkTerminal(t, store, run, 3-failures, failures)
+			outcomes := []queue.TaskOutcome{{Success: true}, {Success: true}, {Success: true}}
+			if n, err := store.FinalizeRunTasks(ctx, claims, outcomes); err != nil || n != 0 {
+				t.Fatalf("duplicate finalize: %d %v", n, err)
+			}
+			checkTerminal(t, store, run, 3-failures, failures)
+			key, _ := store.RunMetaKey(run)
+			ttl, err := redis.Do(ctx, "TTL", key)
+			if err != nil || ttl.Int64() < 29*24*3600 {
+				t.Fatalf("retention=%v err=%v", ttl, err)
+			}
+		})
+	}
+}
+
+func TestRunConsumerLateProducerEmptyAndHistoricalRunCompletion(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	if err := store.CreateRun(ctx, queue.RunMeta{RunID: "historical", Status: "running", ProducerDone: true}); err != nil {
+		t.Fatal(err)
+	}
+	runTasks(t, store, "empty", 0)
+	runTasks(t, store, "late", 1)
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}, false)
+	claims := claimRun(t, store, "late", time.Now(), 1)
+	if err := c.handleRunClaims(ctx, claims); err != nil {
+		t.Fatal(err)
+	}
+	if meta := runMeta(t, store, "late"); !meta.CompletedAt.IsZero() || meta.SuccessCount != 1 {
+		t.Fatalf("premature completion: %+v", meta)
+	}
+	finishProducer(t, store, "late")
+	finishProducer(t, store, "empty")
+	if _, err := c.processOnce(ctx); err != nil {
+		t.Fatal(err)
+	}
+	c.channelGroup.Wait()
+	checkTerminal(t, store, "late", 1, 0)
+	checkTerminal(t, store, "empty", 0, 0)
+	checkTerminal(t, store, "historical", 0, 0)
+}
+
+func TestRunConsumerRecoveryAndFencing(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	for _, mode := range []string{"database", "network", "stale"} {
+		t.Run(mode, func(t *testing.T) {
+			runTasks(t, store, mode, 1)
+			finishProducer(t, store, mode)
+			now := time.Now()
+			claims := claimRun(t, store, mode, now, 1)
+			good := runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}
+			handler := good
+			if mode == "database" {
+				handler.apply = func([]Result) error { return errors.New("database unavailable") }
+			}
+			if mode == "network" {
+				handler.query = func(task queue.Task) Result { return Result{Task: task, Retryable: true, ErrorClass: "network"} }
+			}
+			c := runConsumer(t, store, handler, false)
+			if mode != "stale" {
+				err := c.handleRunClaims(ctx, claims)
+				if (err != nil) != (mode == "database") {
+					t.Fatalf("handle: %v", err)
+				}
+				if meta := runMeta(t, store, mode); meta.SuccessCount != 0 || meta.FailedCount != 0 || meta.ActiveCount != 1 || meta.InflightCount != 1 || meta.Status != "running" {
+					t.Fatalf("lost pending task: %+v", meta)
+				}
+			}
+			fresh := claimRun(t, store, mode, now.Add(2*time.Minute), 1)
+			if n, err := store.RenewRunBatch(ctx, claims, now.Add(3*time.Minute)); err != nil || n != 0 {
+				t.Fatalf("stale renewal: %d %v", n, err)
+			}
+			if n, err := store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil || n != 0 {
+				t.Fatalf("stale ack: %d %v", n, err)
+			}
+			if n, err := store.BlockRunScopedAsFailed(ctx, claims[0], queue.TaskOutcome{}); err != nil || n != -1 {
+				t.Fatalf("stale block: %d %v", n, err)
+			}
+			c.handler = good
+			if err := c.handleRunClaims(ctx, fresh); err != nil {
+				t.Fatal(err)
+			}
+			checkTerminal(t, store, mode, 1, 0)
+		})
+	}
+}
+
+func TestRunConsumerBlockedChannelCountsOnlyUnfinishedTasks(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	runTasks(t, store, "blocked", 5)
+	runTasks(t, store, "other", 1)
+	// Production continues while the channel is blocked.
+	claims := claimRun(t, store, "blocked", time.Now(), 2)
+	var applied int
+	successQueried := make(chan struct{})
+	handler := runHandler{query: func(task queue.Task) Result {
+		if task.CardID == 1 {
+			close(successQueried)
+			return Result{Task: task, Success: true}
+		}
+		<-successQueried
+		return Result{Task: task, BlockChannel: true, ErrorClass: "channel_limited", Code: 797, ErrCode: "S00004", Message: "limited"}
+	}, apply: func(results []Result) error { applied = countSuccessful(results); return nil }}
+	c := runConsumer(t, store, handler, false)
+	// Ensure the success response arrives before the channel-limit response.
+	if err := c.handleRunClaims(ctx, claims); err != nil {
+		t.Fatal(err)
+	}
+	meta := runMeta(t, store, "blocked")
+	if applied != 1 || meta.SuccessCount != 1 || meta.FailedCount != 4 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
+		t.Fatalf("applied=%d meta=%+v", applied, meta)
+	}
+	if n, err := store.BlockRunScopedAsFailed(ctx, claims[1], queue.TaskOutcome{}); err != nil || n != -1 {
+		t.Fatalf("duplicate block: %d %v", n, err)
+	}
+	late := queue.Task{TaskID: "late", RunID: "blocked", ChanKey: "channel", CardID: 99}
+	if n, err := store.EnqueueRunBatch(ctx, "blocked", "channel", []queue.Task{late}, time.Now()); err != nil || n != 0 {
+		t.Fatalf("blocked enqueue: %d %v", n, err)
+	}
+	key, _ := store.FailedKey("blocked")
+	failures, err := redis.Do(ctx, "HLEN", key)
+	if err != nil || failures.Int() != 4 {
+		t.Fatalf("failure records=%v err=%v", failures, err)
+	}
+	stored, err := redis.Do(ctx, "HGET", key, "blocked:4")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var record queue.FailureRecord
+	if err := json.Unmarshal([]byte(stored.String()), &record); err != nil {
+		t.Fatal(err)
+	}
+	if record.TaskID != "blocked:4" || record.CardID != 5 || record.ICCID != "card-4" || record.Attempts != 0 || record.FailureType != "channel_blocked" || record.ErrCode != "S00004" {
+		t.Fatalf("failure identity/reason: %+v", record)
+	}
+	stateKey, _ := store.TaskStateKey("blocked")
+	state, err := redis.Do(ctx, "HGET", stateKey, "blocked:4")
+	if err != nil || state.MapStrAny()["status"] != "channel_blocked" {
+		t.Fatalf("blocked state=%v err=%v", state, err)
+	}
+	if meta := runMeta(t, store, "other"); meta.ActiveCount != 1 || meta.FailedCount != 0 {
+		t.Fatalf("other run affected: %+v", meta)
+	}
+	finishProducer(t, store, "blocked")
+	if err := c.completeRun(ctx, "blocked"); err != nil {
+		t.Fatal(err)
+	}
+	checkTerminal(t, store, "blocked", 1, 4)
+}
+
+func TestRunConsumerMonthlyRoundsRemainEnabled(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "flow:")
+	ctx := context.Background()
+	runTasks(t, store, "monthly", 1)
+	finishProducer(t, store, "monthly")
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, true)
+	for round := 1; round <= 5; round++ {
+		if err := c.handleRunClaims(ctx, claimRun(t, store, "monthly", time.Now(), 1)); err != nil {
+			t.Fatal(err)
+		}
+		if round < 5 {
+			if meta := runMeta(t, store, "monthly"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
+				t.Fatalf("round %d: %+v", round, meta)
+			}
+			if ok, err := store.AdvanceRunRound(ctx, "monthly", 5); err != nil || !ok {
+				t.Fatalf("advance %v %v", ok, err)
+			}
+		}
+	}
+	checkTerminal(t, store, "monthly", 0, 1)
+}
+
+func TestRunConsumerStatusRetriesRetryableResultsForThreeRounds(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	runTasks(t, store, "status", 1)
+	finishProducer(t, store, "status")
+	c, err := New(store, runHandler{query: func(task queue.Task) Result {
+		return Result{Task: task, ErrorClass: "carrier_retry", Retryable: true, Code: 9999, ErrCode: "S00004"}
+	}}, Config{
+		RunQueueEnabled:   true,
+		RoundRetryEnabled: true,
+		MaxRounds:         3,
+		EventPrefix:       "sim_status_queue",
+		ShouldRetryNextRound: func(result Result) bool {
+			return result.Retryable
+		},
+		TemplateQPS: map[string]float64{mobileTempKey: 1000000},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for round := 1; round <= 3; round++ {
+		if err := c.handleRunClaims(ctx, claimRun(t, store, "status", time.Now(), 1)); err != nil {
+			t.Fatal(err)
+		}
+		if round < 3 {
+			if meta := runMeta(t, store, "status"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
+				t.Fatalf("round %d: %+v", round, meta)
+			}
+			if ok, err := store.AdvanceRunRound(ctx, "status", 3); err != nil || !ok {
+				t.Fatalf("advance %v %v", ok, err)
+			}
+		}
+	}
+	checkTerminal(t, store, "status", 0, 1)
+}
+
+func TestReplayMaxRoundOverridesConsumerDefault(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	if err := store.CreateRun(ctx, queue.RunMeta{RunID: "replay-one", Operation: "status", SyncType: "replay", MaxRound: 1}); err != nil {
+		t.Fatal(err)
+	}
+	task := queue.Task{RunID: "replay-one", TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "channel", TempKey: mobileTempKey}
+	if _, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []queue.Task{task}, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	finishProducer(t, store, "replay-one")
+	c, err := New(store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, Config{RunQueueEnabled: true, RoundRetryEnabled: true, MaxRounds: 3, TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := c.handleRunClaims(ctx, claimRun(t, store, "replay-one", time.Now(), 1)); err != nil {
+		t.Fatal(err)
+	}
+	checkTerminal(t, store, "replay-one", 0, 1)
+	meta := runMeta(t, store, "replay-one")
+	if meta.CurrentRound != 1 || meta.NextRoundCount != 0 {
+		t.Fatalf("replay advanced rounds: %+v", meta)
+	}
+}
+
+func TestRunQueueModeDoesNotEnableRoundRetry(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	runTasks(t, store, "single-round", 1)
+	finishProducer(t, store, "single-round")
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, false)
+	if err := c.handleRunClaims(context.Background(), claimRun(t, store, "single-round", time.Now(), 1)); err != nil {
+		t.Fatal(err)
+	}
+	checkTerminal(t, store, "single-round", 0, 1)
+	if meta := runMeta(t, store, "single-round"); meta.CurrentRound != 1 {
+		t.Fatalf("unexpected retry: %+v", meta)
+	}
+}
+
+func TestMonthlyChannelBlockKeepsExistingCountingPolicy(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "flow:")
+	ctx := context.Background()
+	runTasks(t, store, "monthly-block", 2)
+	claimRun(t, store, "monthly-block", time.Now(), 1)
+	for i := 0; i < 2; i++ {
+		want := int64(2)
+		if i == 1 {
+			want = 0
+		}
+		if n, err := store.BlockRunScoped(ctx, "monthly-block", "channel"); err != nil || n != want {
+			t.Fatalf("monthly block=%d err=%v", n, err)
+		}
+	}
+	if meta := runMeta(t, store, "monthly-block"); meta.FailedCount != 0 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
+		t.Fatalf("monthly counting changed: %+v", meta)
+	}
+}
+
+func TestRunConsumerConcurrentRuns(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	var mu sync.Mutex
+	applied := map[string]int{}
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }, apply: func(results []Result) error {
+		mu.Lock()
+		defer mu.Unlock()
+		for _, result := range results {
+			applied[result.Task.RunID]++
+		}
+		return nil
+	}}, false)
+	for _, run := range []string{"a", "b"} {
+		runTasks(t, store, run, 3)
+		finishProducer(t, store, run)
+	}
+	if started, err := c.processOnce(ctx); err != nil || !started {
+		t.Fatalf("process: %v %v", started, err)
+	}
+	c.channelGroup.Wait()
+	for _, run := range []string{"a", "b"} {
+		checkTerminal(t, store, run, 3, 0)
+		if applied[run] != 3 {
+			t.Fatalf("applied %s: %d", run, applied[run])
+		}
+	}
+}
+
+// TestRunConsumerCoveredResultsCompleteWithoutFailures drives the full
+// consumer path for a covered result: the round must close, the run must end
+// as "completed" rather than "completed_with_failures", and the failure pool
+// must stay empty. This is the IMEI 14000 "carrier has no record" case — on
+// 2026-10-20 it accounted for every failure recorded in production while the
+// run success rate read 21%.
+func TestRunConsumerCoveredResultsCompleteWithoutFailures(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "imei:")
+	ctx := context.Background()
+	runTasks(t, store, "covered", 4)
+	finishProducer(t, store, "covered")
+	queried := 0
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result {
+		queried++
+		switch task.CardID {
+		case 1:
+			return Result{Task: task, Success: true, Imei: "865466070817587", ImeiSynced: true}
+		case 2:
+			return Result{Task: task, ErrorClass: "no_record", Code: 14000, ImeiSynced: true, Covered: true, Message: "系统无记录，请检查输入参数是否正确"}
+		case 3:
+			return Result{Task: task, ErrorClass: "data_invalid", ImeiSynced: true, Covered: true, Message: "carrier returned no imei value"}
+		default:
+			return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
+		}
+	}}, false)
+	if err := c.handleRunClaims(ctx, claimRun(t, store, "covered", time.Now(), 4)); err != nil {
+		t.Fatal(err)
+	}
+	if queried != 4 {
+		t.Fatalf("queried = %d, want 4", queried)
+	}
+	meta := runMeta(t, store, "covered")
+	// Three cards produced a usable or definitive answer (success + 2 covered),
+	// and only the business failure is counted. Without the covered channel the
+	// two 14000-style answers would have made failedCount=3.
+	if meta.Status != "completed_with_failures" || meta.SuccessCount != 3 || meta.FailedCount != 1 || meta.NextRoundCount != 0 {
+		t.Fatalf("unexpected run accounting: %+v", meta)
+	}
+	failureKey, _ := store.FailedKey("covered")
+	size, err := redis.Do(ctx, "HLEN", failureKey)
+	if err != nil || size.Int64() != 1 {
+		t.Fatalf("failure pool entries = %d err=%v, want 1 (only the business failure)", size.Int64(), err)
+	}
+	for _, taskID := range []string{"covered:1", "covered:2"} {
+		exists, err := redis.Do(ctx, "HEXISTS", failureKey, taskID)
+		if err != nil || exists.Int() != 0 {
+			t.Fatalf("covered card %s leaked into the failure pool: %v %v", taskID, exists, err)
+		}
+	}
+	exists, err := redis.Do(ctx, "HEXISTS", failureKey, "covered:3")
+	if err != nil || exists.Int() != 1 {
+		t.Fatalf("the business failure is missing from the pool: %v %v", exists, err)
+	}
+}

internal/dao/batch_detail.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// batchDetailDao is the data access object for the table batch_detail.
+// You can define custom methods on it to extend its functionality as needed.
+type batchDetailDao struct {
+	*internal.BatchDetailDao
+}
+
+var (
+	// BatchDetail is a globally accessible object for table batch_detail operations.
+	BatchDetail = batchDetailDao{internal.NewBatchDetailDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/batch_main.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// batchMainDao is the data access object for the table batch_main.
+// You can define custom methods on it to extend its functionality as needed.
+type batchMainDao struct {
+	*internal.BatchMainDao
+}
+
+var (
+	// BatchMain is a globally accessible object for table batch_main operations.
+	BatchMain = batchMainDao{internal.NewBatchMainDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/internal/batch_detail.go

@@ -0,0 +1,109 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// BatchDetailDao is the data access object for the table batch_detail.
+type BatchDetailDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  BatchDetailColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// BatchDetailColumns defines and stores column names for the table batch_detail.
+type BatchDetailColumns struct {
+	Id             string // 主键ID
+	BatchNo        string // 批次编号
+	BatchType      string // 批次类型 枚举（与主表一致）
+	DetailStatus   string // 明细异步处理状态 0-待处理 1-处理中 2-成功 3-失败
+	BlankOutStatus string // 作废状态 0-是 1-否
+	RowNo          string // 行号
+	ItemId         string // 项ID：卡ICCID/设备编码
+	ErrorCode      string // 异常编码
+	ErrorReason    string // 异常原因
+	Snapshot       string // 信息快照
+	TenantId       string // 租户ID
+	CreatedDept    string // 创建部门ID
+	CreatedBy      string // 创建者ID
+	CreatedAt      string // 创建时间
+	UpdatedBy      string // 更新者ID
+	UpdatedAt      string // 更新时间
+}
+
+// batchDetailColumns holds the columns for the table batch_detail.
+var batchDetailColumns = BatchDetailColumns{
+	Id:             "id",
+	BatchNo:        "batch_no",
+	BatchType:      "batch_type",
+	DetailStatus:   "detail_status",
+	BlankOutStatus: "blank_out_status",
+	RowNo:          "row_no",
+	ItemId:         "item_id",
+	ErrorCode:      "error_code",
+	ErrorReason:    "error_reason",
+	Snapshot:       "snapshot",
+	TenantId:       "tenant_id",
+	CreatedDept:    "created_dept",
+	CreatedBy:      "created_by",
+	CreatedAt:      "created_at",
+	UpdatedBy:      "updated_by",
+	UpdatedAt:      "updated_at",
+}
+
+// NewBatchDetailDao creates and returns a new DAO object for table data access.
+func NewBatchDetailDao(handlers ...gdb.ModelHandler) *BatchDetailDao {
+	return &BatchDetailDao{
+		group:    "default",
+		table:    "batch_detail",
+		columns:  batchDetailColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *BatchDetailDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *BatchDetailDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *BatchDetailDao) Columns() BatchDetailColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *BatchDetailDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *BatchDetailDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *BatchDetailDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/batch_main.go

@@ -0,0 +1,139 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// BatchMainDao is the data access object for the table batch_main.
+type BatchMainDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  BatchMainColumns   // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// BatchMainColumns defines and stores column names for the table batch_main.
+type BatchMainColumns struct {
+	Id               string // 主键ID
+	BatchNo          string // 批次编号
+	OrderSn          string // 订单编号
+	BatchType        string // 批次类型 枚举
+	BatchStatus      string // 批次状态 枚举
+	BatchAsyncStatus string // 异步处理状态
+	SourceType       string // 来源方式 枚举
+	SourceBatchNo    string // 来源批次编号
+	SourceOssId      string // 源文件OSS ID
+	ResultOssId      string // 结果文件OSS ID
+	IsCrossDept      string // 是否跨级操作机构
+	TargetDeptId     string // 目标机构编号
+	DeptId           string // 机构编号
+	TotalCount       string // 总数量
+	ValidCount       string // 有效数量
+	SuccessCount     string // 成功数量
+	FailCount        string // 失败数量
+	RunResult        string // 运行结果
+	FailReason       string // 失败原因
+	BatchInfo        string // 批次信息：入库/出库展示字段
+	Remark           string // 备注扩展字段
+	StartTime        string // 开始时间
+	EndTime          string // 结束时间
+	TenantId         string // 租户ID
+	CreatedDept      string // 创建部门ID
+	CreatedBy        string // 创建者ID
+	CreatedAt        string // 创建时间
+	UpdatedBy        string // 更新者ID
+	UpdatedAt        string // 更新时间
+	SourceChannel    string // 发起渠道:console/mini_program/api/system/legacy
+	ExecutionMode    string // 操作规模:0未知1单卡2批量
+}
+
+// batchMainColumns holds the columns for the table batch_main.
+var batchMainColumns = BatchMainColumns{
+	Id:               "id",
+	BatchNo:          "batch_no",
+	OrderSn:          "order_sn",
+	BatchType:        "batch_type",
+	BatchStatus:      "batch_status",
+	BatchAsyncStatus: "batch_async_status",
+	SourceType:       "source_type",
+	SourceBatchNo:    "source_batch_no",
+	SourceOssId:      "source_oss_id",
+	ResultOssId:      "result_oss_id",
+	IsCrossDept:      "is_cross_dept",
+	TargetDeptId:     "target_dept_id",
+	DeptId:           "dept_id",
+	TotalCount:       "total_count",
+	ValidCount:       "valid_count",
+	SuccessCount:     "success_count",
+	FailCount:        "fail_count",
+	RunResult:        "run_result",
+	FailReason:       "fail_reason",
+	BatchInfo:        "batch_info",
+	Remark:           "remark",
+	StartTime:        "start_time",
+	EndTime:          "end_time",
+	TenantId:         "tenant_id",
+	CreatedDept:      "created_dept",
+	CreatedBy:        "created_by",
+	CreatedAt:        "created_at",
+	UpdatedBy:        "updated_by",
+	UpdatedAt:        "updated_at",
+	SourceChannel:    "source_channel",
+	ExecutionMode:    "execution_mode",
+}
+
+// NewBatchMainDao creates and returns a new DAO object for table data access.
+func NewBatchMainDao(handlers ...gdb.ModelHandler) *BatchMainDao {
+	return &BatchMainDao{
+		group:    "default",
+		table:    "batch_main",
+		columns:  batchMainColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *BatchMainDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *BatchMainDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *BatchMainDao) Columns() BatchMainColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *BatchMainDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *BatchMainDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *BatchMainDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/sys_dept.go

@@ -0,0 +1,125 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// SysDeptDao is the data access object for the table sys_dept.
+type SysDeptDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  SysDeptColumns     // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// SysDeptColumns defines and stores column names for the table sys_dept.
+type SysDeptColumns struct {
+	DeptId       string // 部门id
+	TenantId     string // 租户编号
+	ParentId     string // 父部门id
+	Ancestors    string // 祖级列表
+	DeptKey      string // 机构编号
+	DeptName     string // 部门名称
+	DeptType     string // 部门类型(0:部门 1:公司)
+	DeptCategory string // 部门类别编码
+	OrderNum     string // 显示顺序
+	Leader       string // 负责人
+	Phone        string // 联系电话
+	Email        string // 邮箱
+	Status       string // 部门状态（0正常 1停用）
+	CreditCode   string // 统一社会信用代码
+	AreaCode     string // 行政区域代码
+	Address      string // 地址
+	DeptRole     string // 机构默认角色ID
+	CreatedDept  string // 创建部门
+	CreatedBy    string // 创建者
+	CreatedAt    string // 创建时间
+	UpdatedBy    string // 更新者
+	UpdatedAt    string // 更新时间
+	DeletedBy    string // 删除人
+	DeletedAt    string // 删除时间
+}
+
+// sysDeptColumns holds the columns for the table sys_dept.
+var sysDeptColumns = SysDeptColumns{
+	DeptId:       "dept_id",
+	TenantId:     "tenant_id",
+	ParentId:     "parent_id",
+	Ancestors:    "ancestors",
+	DeptKey:      "dept_key",
+	DeptName:     "dept_name",
+	DeptType:     "dept_type",
+	DeptCategory: "dept_category",
+	OrderNum:     "order_num",
+	Leader:       "leader",
+	Phone:        "phone",
+	Email:        "email",
+	Status:       "status",
+	CreditCode:   "credit_code",
+	AreaCode:     "area_code",
+	Address:      "address",
+	DeptRole:     "dept_role",
+	CreatedDept:  "created_dept",
+	CreatedBy:    "created_by",
+	CreatedAt:    "created_at",
+	UpdatedBy:    "updated_by",
+	UpdatedAt:    "updated_at",
+	DeletedBy:    "deleted_by",
+	DeletedAt:    "deleted_at",
+}
+
+// NewSysDeptDao creates and returns a new DAO object for table data access.
+func NewSysDeptDao(handlers ...gdb.ModelHandler) *SysDeptDao {
+	return &SysDeptDao{
+		group:    "default",
+		table:    "sys_dept",
+		columns:  sysDeptColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *SysDeptDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *SysDeptDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *SysDeptDao) Columns() SysDeptColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *SysDeptDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *SysDeptDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *SysDeptDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/sys_oss.go

@@ -0,0 +1,111 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// SysOssDao is the data access object for the table sys_oss.
+type SysOssDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  SysOssColumns      // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// SysOssColumns defines and stores column names for the table sys_oss.
+type SysOssColumns struct {
+	OssId        string // 对象存储主键
+	TenantId     string // 租户编号
+	FileName     string // 文件名
+	OriginalName string // 原名
+	FileSuffix   string // 文件后缀名
+	Path         string // 存储路径
+	Url          string // URL地址
+	CreatedDept  string // 创建部门
+	CreatedAt    string // 创建时间
+	CreatedBy    string // 创建者
+	UpdatedAt    string // 更新时间
+	UpdatedBy    string // 更新者
+	Service      string // 服务商
+	Md5          string // 文件MD5
+	FileSize     string // 文件大小
+	FileCrc16    string // 文件Crc16
+	FileSum      string // 文件校验和
+}
+
+// sysOssColumns holds the columns for the table sys_oss.
+var sysOssColumns = SysOssColumns{
+	OssId:        "oss_id",
+	TenantId:     "tenant_id",
+	FileName:     "file_name",
+	OriginalName: "original_name",
+	FileSuffix:   "file_suffix",
+	Path:         "path",
+	Url:          "url",
+	CreatedDept:  "created_dept",
+	CreatedAt:    "created_at",
+	CreatedBy:    "created_by",
+	UpdatedAt:    "updated_at",
+	UpdatedBy:    "updated_by",
+	Service:      "service",
+	Md5:          "md5",
+	FileSize:     "file_size",
+	FileCrc16:    "file_crc16",
+	FileSum:      "file_sum",
+}
+
+// NewSysOssDao creates and returns a new DAO object for table data access.
+func NewSysOssDao(handlers ...gdb.ModelHandler) *SysOssDao {
+	return &SysOssDao{
+		group:    "default",
+		table:    "sys_oss",
+		columns:  sysOssColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *SysOssDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *SysOssDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *SysOssDao) Columns() SysOssColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *SysOssDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *SysOssDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *SysOssDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_message_log.go

@@ -0,0 +1,107 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TMessageLogDao is the data access object for the table t_message_log.
+type TMessageLogDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  TMessageLogColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// TMessageLogColumns defines and stores column names for the table t_message_log.
+type TMessageLogColumns struct {
+	MessageLogId   string // 主键ID
+	MessageType    string // 消息类型
+	SendUserType   string // 发送用户类型
+	UserIds        string // 接收用户ID集合(JSON数组)
+	MessageTitle   string // 消息标题
+	MessageContent string // 消息内容
+	MessageLink    string // 跳转链接地址
+	SendTime       string // 发送时间
+	IsRecall       string // 是否撤回
+	TenantId       string // 租户ID
+	CreatedDept    string // 创建部门
+	CreatedBy      string // 创建者
+	CreatedAt      string // 创建时间
+	UpdatedBy      string // 更新者
+	UpdatedAt      string // 更新时间
+}
+
+// tMessageLogColumns holds the columns for the table t_message_log.
+var tMessageLogColumns = TMessageLogColumns{
+	MessageLogId:   "message_log_id",
+	MessageType:    "message_type",
+	SendUserType:   "send_user_type",
+	UserIds:        "user_ids",
+	MessageTitle:   "message_title",
+	MessageContent: "message_content",
+	MessageLink:    "message_link",
+	SendTime:       "send_time",
+	IsRecall:       "is_recall",
+	TenantId:       "tenant_id",
+	CreatedDept:    "created_dept",
+	CreatedBy:      "created_by",
+	CreatedAt:      "created_at",
+	UpdatedBy:      "updated_by",
+	UpdatedAt:      "updated_at",
+}
+
+// NewTMessageLogDao creates and returns a new DAO object for table data access.
+func NewTMessageLogDao(handlers ...gdb.ModelHandler) *TMessageLogDao {
+	return &TMessageLogDao{
+		group:    "default",
+		table:    "t_message_log",
+		columns:  tMessageLogColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TMessageLogDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TMessageLogDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TMessageLogDao) Columns() TMessageLogColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TMessageLogDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TMessageLogDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TMessageLogDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_resource_batch_download.go

@@ -0,0 +1,141 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TResourceBatchDownloadDao is the data access object for the table t_resource_batch_download.
+type TResourceBatchDownloadDao struct {
+	table    string                        // table is the underlying table name of the DAO.
+	group    string                        // group is the database configuration group name of the current DAO.
+	columns  TResourceBatchDownloadColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler            // handlers for customized model modification.
+}
+
+// TResourceBatchDownloadColumns defines and stores column names for the table t_resource_batch_download.
+type TResourceBatchDownloadColumns struct {
+	Id               string // 主键ID
+	BatchNo          string // 批次编号（全局唯一）
+	TaskName         string // 任务名称
+	ResourceType     string // 资源类型 枚举
+	ExportScene      string // 导出场景 枚举
+	QueryParams      string // 导出查询条件快照
+	TotalCount       string // 预计导出总条数
+	SuccessCount     string // 已成功导出条数
+	FailCount        string // 失败条数
+	DownloadStatus   string // 下载状态 枚举
+	OssId            string // OSS文件ID
+	FileName         string // 导出文件名
+	FileExt          string // 文件扩展名
+	FileUrl          string // 文件访问地址
+	FilePath         string // 服务器本地路径
+	FileSize         string // 文件大小（字节）
+	FileMd5          string // 文件MD5
+	DownloadCount    string // 下载次数
+	LastDownloadTime string // 最后下载时间
+	MaxDownloadCount string // 最大下载次数限制
+	ErrorMessage     string // 失败原因
+	DeptId           string // 所属机构ID
+	StartTime        string // 任务开始时间
+	FinishTime       string // 任务完成时间
+	ExpireTime       string // 文件过期时间
+	Remark           string // 备注
+	TenantId         string // 租户ID
+	CreatedDept      string // 创建部门ID
+	CreatedBy        string // 创建者ID
+	CreatedAt        string // 创建时间
+	UpdatedBy        string // 更新者ID
+	UpdatedAt        string // 更新时间
+}
+
+// tResourceBatchDownloadColumns holds the columns for the table t_resource_batch_download.
+var tResourceBatchDownloadColumns = TResourceBatchDownloadColumns{
+	Id:               "id",
+	BatchNo:          "batch_no",
+	TaskName:         "task_name",
+	ResourceType:     "resource_type",
+	ExportScene:      "export_scene",
+	QueryParams:      "query_params",
+	TotalCount:       "total_count",
+	SuccessCount:     "success_count",
+	FailCount:        "fail_count",
+	DownloadStatus:   "download_status",
+	OssId:            "oss_id",
+	FileName:         "file_name",
+	FileExt:          "file_ext",
+	FileUrl:          "file_url",
+	FilePath:         "file_path",
+	FileSize:         "file_size",
+	FileMd5:          "file_md5",
+	DownloadCount:    "download_count",
+	LastDownloadTime: "last_download_time",
+	MaxDownloadCount: "max_download_count",
+	ErrorMessage:     "error_message",
+	DeptId:           "dept_id",
+	StartTime:        "start_time",
+	FinishTime:       "finish_time",
+	ExpireTime:       "expire_time",
+	Remark:           "remark",
+	TenantId:         "tenant_id",
+	CreatedDept:      "created_dept",
+	CreatedBy:        "created_by",
+	CreatedAt:        "created_at",
+	UpdatedBy:        "updated_by",
+	UpdatedAt:        "updated_at",
+}
+
+// NewTResourceBatchDownloadDao creates and returns a new DAO object for table data access.
+func NewTResourceBatchDownloadDao(handlers ...gdb.ModelHandler) *TResourceBatchDownloadDao {
+	return &TResourceBatchDownloadDao{
+		group:    "default",
+		table:    "t_resource_batch_download",
+		columns:  tResourceBatchDownloadColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TResourceBatchDownloadDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TResourceBatchDownloadDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TResourceBatchDownloadDao) Columns() TResourceBatchDownloadColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TResourceBatchDownloadDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TResourceBatchDownloadDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TResourceBatchDownloadDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_sim_card_cancellation.go

@@ -0,0 +1,191 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TSimCardCancellationDao is the data access object for the table t_sim_card_cancellation.
+type TSimCardCancellationDao struct {
+	table    string                      // table is the underlying table name of the DAO.
+	group    string                      // group is the database configuration group name of the current DAO.
+	columns  TSimCardCancellationColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler          // handlers for customized model modification.
+}
+
+// TSimCardCancellationColumns defines and stores column names for the table t_sim_card_cancellation.
+type TSimCardCancellationColumns struct {
+	Id                     string // 主键ID
+	Iccid                  string // ICCID
+	Msisdn                 string // MSISDN/卡号
+	Imsi                   string // IMSI
+	Imei                   string // IMEI
+	RegionCode             string // 归属地
+	DeptId                 string // 渠道客户ID
+	CurrentPackageCode     string // 当前套餐编号
+	PackageStatus          string // 套餐状态：0 无套餐，1待生效，2生效中，3已过期
+	AccessType             string // 接入方式：ipv4、ipv6、ipv4_ipv6
+	BossCardStatus         string // 运营商卡状态 枚举
+	CancelBatchNo          string // 系统销户批次号
+	CancelledAt            string // 系统销户完成时间
+	CardType               string // 卡类型 枚举
+	BossOpenAt             string // 运营商开户日期
+	BossActivateAt         string // 运营商激活日期
+	IsBossActivateEnabled  string // 是否按运营商激活减去赠送时长方：1是，0否
+	InboundBatchNo         string // 入库批次
+	OutboundBatchNo        string // 出库批次
+	AllocationBatchNo      string // 划拨批次
+	IsFlowPool             string // 是否为池卡：1是，0否
+	FlowPoolCode           string // 流量池编号
+	FlowPkgBeginAt         string // 当前套餐生效时间
+	FlowPkgPresentAt       string // 套餐赠送结束时间
+	FlowPkgRealPresentAt   string // 实际赠送结束时间
+	FlowPkgEndAt           string // 当前套餐预计到期时间
+	DataServiceStatus      string // 数据服务状态：0启用,1停用
+	PackageTotalFlowMb     string // 套餐总流量MB
+	UsedFlowMb             string // 已用流量MB
+	TrafficCutoffStatus    string // 达量断网配置状态：0未配置，1已开启，2已关闭
+	TrafficCutoffThreshold string // 达量断网阈值
+	SmsServiceStatus       string // 短信服务状态：0启用,1停用
+	PackageTotalSms        string // 套餐短信总条数
+	UsedSms                string // 已用短信条数
+	VoiceServiceStatus     string // 语音服务状态：0启用,1停用
+	PackageTotalVoice      string // 套餐语音总分钟数
+	UsedVoice              string // 已用语音分钟数
+	RoamServiceStatus      string // 国漫服务状态：0启用,1停用
+	PackageTotalRoamFlowMb string // 套餐国际流量MB
+	UsedRoamFlowMb         string // 已用国际流量MB
+	StatusSyncTime         string // 状态最后一次同步时间
+	FlowSyncTime           string // 流量最后一次同步时间
+	SmsSyncTime            string // 短信最后一次同步时间
+	VoiceSyncTime          string // 语音最后一次同步时间
+	RoamSyncTime           string // 国漫最后一次同步时间
+	ExtraFeeEnabled        string // 是否收取附加费：0是，1否
+	ExtraFeeMultiple       string // 附加费用倍数
+	ExtraFee               string // 附加金额
+	ChanKey                string // 通道编号
+	NetModel               string // 网络类型 枚举
+	Remark                 string // 备注
+	TenantId               string // 租户ID
+	CreatedDept            string // 创建部门
+	CreatedBy              string // 创建者
+	CreatedAt              string // 创建时间
+	UpdatedBy              string // 更新者
+	UpdatedAt              string // 更新时间
+}
+
+// tSimCardCancellationColumns holds the columns for the table t_sim_card_cancellation.
+var tSimCardCancellationColumns = TSimCardCancellationColumns{
+	Id:                     "id",
+	Iccid:                  "iccid",
+	Msisdn:                 "msisdn",
+	Imsi:                   "imsi",
+	Imei:                   "imei",
+	RegionCode:             "region_code",
+	DeptId:                 "dept_id",
+	CurrentPackageCode:     "current_package_code",
+	PackageStatus:          "package_status",
+	AccessType:             "access_type",
+	BossCardStatus:         "boss_card_status",
+	CancelBatchNo:          "cancel_batch_no",
+	CancelledAt:            "cancelled_at",
+	CardType:               "card_type",
+	BossOpenAt:             "boss_open_at",
+	BossActivateAt:         "boss_activate_at",
+	IsBossActivateEnabled:  "is_boss_activate_enabled",
+	InboundBatchNo:         "inbound_batch_no",
+	OutboundBatchNo:        "outbound_batch_no",
+	AllocationBatchNo:      "allocation_batch_no",
+	IsFlowPool:             "is_flow_pool",
+	FlowPoolCode:           "flow_pool_code",
+	FlowPkgBeginAt:         "flow_pkg_begin_at",
+	FlowPkgPresentAt:       "flow_pkg_present_at",
+	FlowPkgRealPresentAt:   "flow_pkg_real_present_at",
+	FlowPkgEndAt:           "flow_pkg_end_at",
+	DataServiceStatus:      "data_service_status",
+	PackageTotalFlowMb:     "package_total_flow_mb",
+	UsedFlowMb:             "used_flow_mb",
+	TrafficCutoffStatus:    "traffic_cutoff_status",
+	TrafficCutoffThreshold: "traffic_cutoff_threshold",
+	SmsServiceStatus:       "sms_service_status",
+	PackageTotalSms:        "package_total_sms",
+	UsedSms:                "used_sms",
+	VoiceServiceStatus:     "voice_service_status",
+	PackageTotalVoice:      "package_total_voice",
+	UsedVoice:              "used_voice",
+	RoamServiceStatus:      "roam_service_status",
+	PackageTotalRoamFlowMb: "package_total_roam_flow_mb",
+	UsedRoamFlowMb:         "used_roam_flow_mb",
+	StatusSyncTime:         "status_sync_time",
+	FlowSyncTime:           "flow_sync_time",
+	SmsSyncTime:            "sms_sync_time",
+	VoiceSyncTime:          "voice_sync_time",
+	RoamSyncTime:           "roam_sync_time",
+	ExtraFeeEnabled:        "extra_fee_enabled",
+	ExtraFeeMultiple:       "extra_fee_multiple",
+	ExtraFee:               "extra_fee",
+	ChanKey:                "chan_key",
+	NetModel:               "net_model",
+	Remark:                 "remark",
+	TenantId:               "tenant_id",
+	CreatedDept:            "created_dept",
+	CreatedBy:              "created_by",
+	CreatedAt:              "created_at",
+	UpdatedBy:              "updated_by",
+	UpdatedAt:              "updated_at",
+}
+
+// NewTSimCardCancellationDao creates and returns a new DAO object for table data access.
+func NewTSimCardCancellationDao(handlers ...gdb.ModelHandler) *TSimCardCancellationDao {
+	return &TSimCardCancellationDao{
+		group:    "default",
+		table:    "t_sim_card_cancellation",
+		columns:  tSimCardCancellationColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TSimCardCancellationDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TSimCardCancellationDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TSimCardCancellationDao) Columns() TSimCardCancellationColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TSimCardCancellationDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TSimCardCancellationDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TSimCardCancellationDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_sim_card_info.go

@@ -0,0 +1,193 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TSimCardInfoDao is the data access object for the table t_sim_card_info.
+type TSimCardInfoDao struct {
+	table    string              // table is the underlying table name of the DAO.
+	group    string              // group is the database configuration group name of the current DAO.
+	columns  TSimCardInfoColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler  // handlers for customized model modification.
+}
+
+// TSimCardInfoColumns defines and stores column names for the table t_sim_card_info.
+type TSimCardInfoColumns struct {
+	Id                     string // 主键ID
+	Iccid                  string // ICCID
+	Msisdn                 string // MSISDN/卡号
+	Imsi                   string // IMSI
+	Imei                   string // IMEI
+	RegionCode             string // 归属地
+	DeptId                 string // 渠道客户ID
+	CurrentPackageCode     string // 当前套餐编号
+	PackageStatus          string // 套餐状态：0 无套餐，1待生效，2生效中，3已过期
+	AccessType             string // 接入方式：ipv4、ipv6、ipv4_ipv6
+	BossCardStatus         string // 运营商卡状态 枚举
+	CardType               string // 卡类型 枚举
+	BossOpenAt             string // 运营商开户日期
+	BossActivateAt         string // 运营商激活日期
+	IsBossActivateEnabled  string // 是否按运营商激活减去赠送时长方：1是，0否
+	InboundBatchNo         string // 入库批次
+	OutboundBatchNo        string // 出库批次
+	AllocationBatchNo      string // 划拨批次
+	IsFlowPool             string // 是否为池卡：1是，0否
+	FlowPoolCode           string // 流量池编号
+	OutStockTime           string // 出库时间
+	FlowPkgBeginAt         string // 当前套餐生效时间
+	FlowPkgPresentAt       string // 套餐赠送结束时间
+	FlowPkgRealPresentAt   string // 实际赠送结束时间
+	FlowPkgEndAt           string // 当前套餐预计到期时间
+	DataServiceStatus      string // 数据服务状态：1启用、0禁用
+	PackageTotalFlowMb     string // 套餐总流量MB
+	UsedFlowMb             string // 已用流量MB
+	TrafficCutoffStatus    string // 达量断网配置状态：0未配置，1已开启，2已关闭
+	TrafficCutoffThreshold string // 达量断网阈值
+	SmsServiceStatus       string // 短信服务状态：1启用、0禁用
+	PackageTotalSms        string // 套餐短信总条数
+	UsedSms                string // 已用短信条数
+	VoiceServiceStatus     string // 语音服务状态：1启用、0禁用
+	PackageTotalVoice      string // 套餐语音总分钟数
+	UsedVoice              string // 已用语音分钟数
+	RoamServiceStatus      string // 国漫服务状态：1启用、0禁用
+	PackageTotalRoamFlowMb string // 套餐国际流量MB
+	UsedRoamFlowMb         string // 已用国际流量MB
+	StatusSyncTime         string // 状态最后一次同步时间
+	FlowSyncTime           string // 流量最后一次同步时间
+	SmsSyncTime            string // 短信最后一次同步时间
+	VoiceSyncTime          string // 语音最后一次同步时间
+	RoamSyncTime           string // 国漫最后一次同步时间
+	ImeiSyncTime           string // IMEI 最后一次轮询同步时间
+	ImeiUpdateTime         string // IMEI 值最后一次实际变更时间
+	ExtraFeeEnabled        string // 是否收取附加费：1是，0否
+	ExtraFeeMultiple       string // 附加费用倍数
+	ExtraFee               string // 附加金额
+	ChanKey                string // 通道编号
+	NetModel               string // 网络类型：4G、5G、NB-IoT
+	Remark                 string // 备注
+	TenantId               string // 租户ID
+	CreatedDept            string // 创建部门
+	CreatedBy              string // 创建者
+	CreatedAt              string // 创建时间
+	UpdatedBy              string // 更新者
+	UpdatedAt              string // 更新时间
+}
+
+// tSimCardInfoColumns holds the columns for the table t_sim_card_info.
+var tSimCardInfoColumns = TSimCardInfoColumns{
+	Id:                     "id",
+	Iccid:                  "iccid",
+	Msisdn:                 "msisdn",
+	Imsi:                   "imsi",
+	Imei:                   "imei",
+	RegionCode:             "region_code",
+	DeptId:                 "dept_id",
+	CurrentPackageCode:     "current_package_code",
+	PackageStatus:          "package_status",
+	AccessType:             "access_type",
+	BossCardStatus:         "boss_card_status",
+	CardType:               "card_type",
+	BossOpenAt:             "boss_open_at",
+	BossActivateAt:         "boss_activate_at",
+	IsBossActivateEnabled:  "is_boss_activate_enabled",
+	InboundBatchNo:         "inbound_batch_no",
+	OutboundBatchNo:        "outbound_batch_no",
+	AllocationBatchNo:      "allocation_batch_no",
+	IsFlowPool:             "is_flow_pool",
+	FlowPoolCode:           "flow_pool_code",
+	OutStockTime:           "out_stock_time",
+	FlowPkgBeginAt:         "flow_pkg_begin_at",
+	FlowPkgPresentAt:       "flow_pkg_present_at",
+	FlowPkgRealPresentAt:   "flow_pkg_real_present_at",
+	FlowPkgEndAt:           "flow_pkg_end_at",
+	DataServiceStatus:      "data_service_status",
+	PackageTotalFlowMb:     "package_total_flow_mb",
+	UsedFlowMb:             "used_flow_mb",
+	TrafficCutoffStatus:    "traffic_cutoff_status",
+	TrafficCutoffThreshold: "traffic_cutoff_threshold",
+	SmsServiceStatus:       "sms_service_status",
+	PackageTotalSms:        "package_total_sms",
+	UsedSms:                "used_sms",
+	VoiceServiceStatus:     "voice_service_status",
+	PackageTotalVoice:      "package_total_voice",
+	UsedVoice:              "used_voice",
+	RoamServiceStatus:      "roam_service_status",
+	PackageTotalRoamFlowMb: "package_total_roam_flow_mb",
+	UsedRoamFlowMb:         "used_roam_flow_mb",
+	StatusSyncTime:         "status_sync_time",
+	FlowSyncTime:           "flow_sync_time",
+	SmsSyncTime:            "sms_sync_time",
+	VoiceSyncTime:          "voice_sync_time",
+	RoamSyncTime:           "roam_sync_time",
+	ImeiSyncTime:           "imei_sync_time",
+	ImeiUpdateTime:         "imei_update_time",
+	ExtraFeeEnabled:        "extra_fee_enabled",
+	ExtraFeeMultiple:       "extra_fee_multiple",
+	ExtraFee:               "extra_fee",
+	ChanKey:                "chan_key",
+	NetModel:               "net_model",
+	Remark:                 "remark",
+	TenantId:               "tenant_id",
+	CreatedDept:            "created_dept",
+	CreatedBy:              "created_by",
+	CreatedAt:              "created_at",
+	UpdatedBy:              "updated_by",
+	UpdatedAt:              "updated_at",
+}
+
+// NewTSimCardInfoDao creates and returns a new DAO object for table data access.
+func NewTSimCardInfoDao(handlers ...gdb.ModelHandler) *TSimCardInfoDao {
+	return &TSimCardInfoDao{
+		group:    "default",
+		table:    "t_sim_card_info",
+		columns:  tSimCardInfoColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TSimCardInfoDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TSimCardInfoDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TSimCardInfoDao) Columns() TSimCardInfoColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TSimCardInfoDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TSimCardInfoDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TSimCardInfoDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_sim_flow_pool.go

@@ -0,0 +1,119 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TSimFlowPoolDao is the data access object for the table t_sim_flow_pool.
+type TSimFlowPoolDao struct {
+	table    string              // table is the underlying table name of the DAO.
+	group    string              // group is the database configuration group name of the current DAO.
+	columns  TSimFlowPoolColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler  // handlers for customized model modification.
+}
+
+// TSimFlowPoolColumns defines and stores column names for the table t_sim_flow_pool.
+type TSimFlowPoolColumns struct {
+	Id                string // 主键ID
+	FlowPoolCode      string // 流量池编号
+	DeptId            string // 客户编号
+	PackageCodes      string // 套餐编号列表（冗余）
+	PoolPackageCode   string // 池套餐编码
+	CardCount         string // 关联卡数量
+	CardFlowMb        string // 套餐卡流量MB
+	PoolPackageFlowMb string // 套内总流量MB
+	AddOnFlowMb       string // 加油包总流量MB
+	TotalFlowMb       string // 总流量MB(套内+加油包)
+	UsedFlowMb        string // 已使用流量MB
+	RemainFlowMb      string // 剩余流量MB
+	LastStatTime      string // 最后流量统计时间
+	PoolStatus        string // 状态：0 正常、1 停用、2 锁定
+	Remark            string // 备注
+	TenantId          string // 租户ID
+	CreatedDept       string // 创建部门
+	CreatedBy         string // 创建者
+	CreatedAt         string // 创建时间
+	UpdatedBy         string // 更新者
+	UpdatedAt         string // 更新时间
+}
+
+// tSimFlowPoolColumns holds the columns for the table t_sim_flow_pool.
+var tSimFlowPoolColumns = TSimFlowPoolColumns{
+	Id:                "id",
+	FlowPoolCode:      "flow_pool_code",
+	DeptId:            "dept_id",
+	PackageCodes:      "package_codes",
+	PoolPackageCode:   "pool_package_code",
+	CardCount:         "card_count",
+	CardFlowMb:        "card_flow_mb",
+	PoolPackageFlowMb: "pool_package_flow_mb",
+	AddOnFlowMb:       "add_on_flow_mb",
+	TotalFlowMb:       "total_flow_mb",
+	UsedFlowMb:        "used_flow_mb",
+	RemainFlowMb:      "remain_flow_mb",
+	LastStatTime:      "last_stat_time",
+	PoolStatus:        "pool_status",
+	Remark:            "remark",
+	TenantId:          "tenant_id",
+	CreatedDept:       "created_dept",
+	CreatedBy:         "created_by",
+	CreatedAt:         "created_at",
+	UpdatedBy:         "updated_by",
+	UpdatedAt:         "updated_at",
+}
+
+// NewTSimFlowPoolDao creates and returns a new DAO object for table data access.
+func NewTSimFlowPoolDao(handlers ...gdb.ModelHandler) *TSimFlowPoolDao {
+	return &TSimFlowPoolDao{
+		group:    "default",
+		table:    "t_sim_flow_pool",
+		columns:  tSimFlowPoolColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TSimFlowPoolDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TSimFlowPoolDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TSimFlowPoolDao) Columns() TSimFlowPoolColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TSimFlowPoolDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TSimFlowPoolDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TSimFlowPoolDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_user_message.go

@@ -0,0 +1,97 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TUserMessageDao is the data access object for the table t_user_message.
+type TUserMessageDao struct {
+	table    string              // table is the underlying table name of the DAO.
+	group    string              // group is the database configuration group name of the current DAO.
+	columns  TUserMessageColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler  // handlers for customized model modification.
+}
+
+// TUserMessageColumns defines and stores column names for the table t_user_message.
+type TUserMessageColumns struct {
+	MessageId    string // 主键ID
+	MessageLogId string // 消息日志ID
+	UserId       string // 用户ID
+	IsRead       string // 是否已读
+	TenantId     string // 租户ID
+	CreatedDept  string // 创建部门
+	CreatedBy    string // 创建者
+	CreatedAt    string // 创建时间
+	UpdatedBy    string // 更新者
+	UpdatedAt    string // 更新时间
+}
+
+// tUserMessageColumns holds the columns for the table t_user_message.
+var tUserMessageColumns = TUserMessageColumns{
+	MessageId:    "message_id",
+	MessageLogId: "message_log_id",
+	UserId:       "user_id",
+	IsRead:       "is_read",
+	TenantId:     "tenant_id",
+	CreatedDept:  "created_dept",
+	CreatedBy:    "created_by",
+	CreatedAt:    "created_at",
+	UpdatedBy:    "updated_by",
+	UpdatedAt:    "updated_at",
+}
+
+// NewTUserMessageDao creates and returns a new DAO object for table data access.
+func NewTUserMessageDao(handlers ...gdb.ModelHandler) *TUserMessageDao {
+	return &TUserMessageDao{
+		group:    "default",
+		table:    "t_user_message",
+		columns:  tUserMessageColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TUserMessageDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TUserMessageDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TUserMessageDao) Columns() TUserMessageColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TUserMessageDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TUserMessageDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TUserMessageDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_warn_config.go

@@ -0,0 +1,123 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TWarnConfigDao is the data access object for the table t_warn_config.
+type TWarnConfigDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  TWarnConfigColumns // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// TWarnConfigColumns defines and stores column names for the table t_warn_config.
+type TWarnConfigColumns struct {
+	Id               string // 主键ID
+	TenantId         string // 租户ID
+	BatchNo          string // 批次编号
+	ItemType         string // 对象类型:1卡,2流量池
+	ItemId           string // 对象ID:单卡为t_sim_card_info.id,流量池为t_sim_flow_pool.id
+	ItemCode         string // 对象编码快照:ICCID/MSISDN/流量池编码
+	WarnUnit         string // 预警单位:1MB,2百分比,3天数,4月数
+	FirstWarn        string // 第一档预警阈值,单位由warn_unit决定
+	SecondWarn       string // 第二档预警阈值,单位由warn_unit决定
+	ThirdWarn        string // 第三档预警阈值,单位由warn_unit决定
+	WarnMethods      string // 预警方式:0短信,1邮箱,2电话,3平台消息;多选逗号分隔
+	WarnPhones       string // 预警手机号,多个逗号分隔
+	WarnEmails       string // 预警邮箱,多个逗号分隔
+	AutoAction       string // 自动动作:0无,2断网;第一期只保存配置不自动执行
+	IsEnabled        string // 是否启用:1启用,0停用
+	WxTemplateId     string // 微信订阅消息模板ID
+	Remark           string // 备注
+	CreatedDept      string // 创建部门
+	CreatedBy        string // 创建者
+	CreatedAt        string // 创建时间
+	UpdatedBy        string // 更新者
+	UpdatedAt        string // 更新时间
+	AppNotifyEnabled string // 是否开启小程序预警通知:0关闭,1开启
+}
+
+// tWarnConfigColumns holds the columns for the table t_warn_config.
+var tWarnConfigColumns = TWarnConfigColumns{
+	Id:               "id",
+	TenantId:         "tenant_id",
+	BatchNo:          "batch_no",
+	ItemType:         "item_type",
+	ItemId:           "item_id",
+	ItemCode:         "item_code",
+	WarnUnit:         "warn_unit",
+	FirstWarn:        "first_warn",
+	SecondWarn:       "second_warn",
+	ThirdWarn:        "third_warn",
+	WarnMethods:      "warn_methods",
+	WarnPhones:       "warn_phones",
+	WarnEmails:       "warn_emails",
+	AutoAction:       "auto_action",
+	IsEnabled:        "is_enabled",
+	WxTemplateId:     "wx_template_id",
+	Remark:           "remark",
+	CreatedDept:      "created_dept",
+	CreatedBy:        "created_by",
+	CreatedAt:        "created_at",
+	UpdatedBy:        "updated_by",
+	UpdatedAt:        "updated_at",
+	AppNotifyEnabled: "app_notify_enabled",
+}
+
+// NewTWarnConfigDao creates and returns a new DAO object for table data access.
+func NewTWarnConfigDao(handlers ...gdb.ModelHandler) *TWarnConfigDao {
+	return &TWarnConfigDao{
+		group:    "default",
+		table:    "t_warn_config",
+		columns:  tWarnConfigColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TWarnConfigDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TWarnConfigDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TWarnConfigDao) Columns() TWarnConfigColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TWarnConfigDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TWarnConfigDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TWarnConfigDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_warn_log.go

@@ -0,0 +1,125 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TWarnLogDao is the data access object for the table t_warn_log.
+type TWarnLogDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  TWarnLogColumns    // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// TWarnLogColumns defines and stores column names for the table t_warn_log.
+type TWarnLogColumns struct {
+	Id             string // 主键ID
+	TenantId       string // 租户ID
+	ConfigId       string // 预警配置ID
+	ItemType       string // 对象类型:1单卡,2流量池
+	ItemId         string // 对象ID:单卡为t_sim_card_info.id,流量池为t_sim_flow_pool.id
+	ItemCode       string // 对象编码快照:ICCID/MSISDN/流量池编码
+	WarnStage      string // 触发档位:1第一档,2第二档,3第三档
+	WarnLevel      string // 预警级别:1提示,2注意,3警告,4危险,5紧急
+	WarnUnit       string // 预警单位/维度:1MB,2百分比,3天数,4月数
+	ThresholdValue string // 触发阈值,单位由warn_unit决定
+	ActualValue    string // 触发时实际值,单位由warn_unit决定
+	TotalFlowMb    string // 触发时总流量MB;流量类预警使用
+	RemainFlowMb   string // 触发时剩余流量MB;流量类预警使用
+	WarnMethods    string // 实际预警方式快照
+	WarnPhones     string // 实际预警手机号快照
+	WarnEmails     string // 实际预警邮箱快照
+	MessageLogId   string // 平台消息ID;聚合通知时可能为空
+	WarnTime       string // 预警时间
+	Remark         string // 备注
+	CreatedDept    string // 创建部门
+	CreatedBy      string // 创建者
+	CreatedAt      string // 创建时间
+	UpdatedBy      string // 更新者
+	UpdatedAt      string // 更新时间
+}
+
+// tWarnLogColumns holds the columns for the table t_warn_log.
+var tWarnLogColumns = TWarnLogColumns{
+	Id:             "id",
+	TenantId:       "tenant_id",
+	ConfigId:       "config_id",
+	ItemType:       "item_type",
+	ItemId:         "item_id",
+	ItemCode:       "item_code",
+	WarnStage:      "warn_stage",
+	WarnLevel:      "warn_level",
+	WarnUnit:       "warn_unit",
+	ThresholdValue: "threshold_value",
+	ActualValue:    "actual_value",
+	TotalFlowMb:    "total_flow_mb",
+	RemainFlowMb:   "remain_flow_mb",
+	WarnMethods:    "warn_methods",
+	WarnPhones:     "warn_phones",
+	WarnEmails:     "warn_emails",
+	MessageLogId:   "message_log_id",
+	WarnTime:       "warn_time",
+	Remark:         "remark",
+	CreatedDept:    "created_dept",
+	CreatedBy:      "created_by",
+	CreatedAt:      "created_at",
+	UpdatedBy:      "updated_by",
+	UpdatedAt:      "updated_at",
+}
+
+// NewTWarnLogDao creates and returns a new DAO object for table data access.
+func NewTWarnLogDao(handlers ...gdb.ModelHandler) *TWarnLogDao {
+	return &TWarnLogDao{
+		group:    "default",
+		table:    "t_warn_log",
+		columns:  tWarnLogColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TWarnLogDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TWarnLogDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TWarnLogDao) Columns() TWarnLogColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TWarnLogDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TWarnLogDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TWarnLogDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_wo_main.go

@@ -0,0 +1,137 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TWoMainDao is the data access object for the table t_wo_main.
+type TWoMainDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  TWoMainColumns     // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// TWoMainColumns defines and stores column names for the table t_wo_main.
+type TWoMainColumns struct {
+	Id                string // 主键ID
+	WoNo              string // 工单编号
+	WoType            string // 工单类型 0卡,1模组,2设备,3订单,4余额,5提现
+	RelatedId         string // 关联业务ID
+	Title             string // 工单标题
+	Content           string // 问题描述
+	Attachment        string // 附件
+	CreatorType       string // 创建人类型：1终端用户，2代理商
+	CreatorDeptId     string // 创建机构ID
+	CreatorUserId     string // 创建用户ID
+	CreatorName       string // 创建人姓名
+	CreatorPhone      string // 创建人手机号
+	AssignDeptId      string // 分派方机构ID
+	AssignUserId      string // 分派人用户ID
+	CurrentStatus     string // 当前处理状态：1已分派，2处理中，3已解决，4已完结_关单，5已取消
+	HandlerDeptId     string // 处理方机构ID
+	HandlerUserId     string // 处理人用户ID
+	FirstResponseTime string // 首次响应时间
+	ResolveTime       string // 确认时间
+	CloseTime         string // 关闭时间
+	SatisfactionLevel string // 满意度：1-5分
+	EvalContent       string // 评价内容
+	EvalTime          string // 评价时间
+	Remark            string // 备注
+	TenantId          string // 租户ID
+	CreatedDept       string // 创建部门
+	CreatedBy         string // 创建者
+	CreatedAt         string // 创建时间
+	UpdatedBy         string // 更新者
+	UpdatedAt         string // 更新时间
+}
+
+// tWoMainColumns holds the columns for the table t_wo_main.
+var tWoMainColumns = TWoMainColumns{
+	Id:                "id",
+	WoNo:              "wo_no",
+	WoType:            "wo_type",
+	RelatedId:         "related_id",
+	Title:             "title",
+	Content:           "content",
+	Attachment:        "attachment",
+	CreatorType:       "creator_type",
+	CreatorDeptId:     "creator_dept_id",
+	CreatorUserId:     "creator_user_id",
+	CreatorName:       "creator_name",
+	CreatorPhone:      "creator_phone",
+	AssignDeptId:      "assign_dept_id",
+	AssignUserId:      "assign_user_id",
+	CurrentStatus:     "current_status",
+	HandlerDeptId:     "handler_dept_id",
+	HandlerUserId:     "handler_user_id",
+	FirstResponseTime: "first_response_time",
+	ResolveTime:       "resolve_time",
+	CloseTime:         "close_time",
+	SatisfactionLevel: "satisfaction_level",
+	EvalContent:       "eval_content",
+	EvalTime:          "eval_time",
+	Remark:            "remark",
+	TenantId:          "tenant_id",
+	CreatedDept:       "created_dept",
+	CreatedBy:         "created_by",
+	CreatedAt:         "created_at",
+	UpdatedBy:         "updated_by",
+	UpdatedAt:         "updated_at",
+}
+
+// NewTWoMainDao creates and returns a new DAO object for table data access.
+func NewTWoMainDao(handlers ...gdb.ModelHandler) *TWoMainDao {
+	return &TWoMainDao{
+		group:    "default",
+		table:    "t_wo_main",
+		columns:  tWoMainColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TWoMainDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TWoMainDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TWoMainDao) Columns() TWoMainColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TWoMainDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TWoMainDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TWoMainDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/internal/t_wo_record.go

@@ -0,0 +1,107 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+package internal
+
+import (
+	"context"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// TWoRecordDao is the data access object for the table t_wo_record.
+type TWoRecordDao struct {
+	table    string             // table is the underlying table name of the DAO.
+	group    string             // group is the database configuration group name of the current DAO.
+	columns  TWoRecordColumns   // columns contains all the column names of Table for convenient usage.
+	handlers []gdb.ModelHandler // handlers for customized model modification.
+}
+
+// TWoRecordColumns defines and stores column names for the table t_wo_record.
+type TWoRecordColumns struct {
+	Id              string // 主键ID
+	WoNo            string // 工单编号
+	OperationType   string // 操作类型：0创建，1分派，2处理，3完成，4关单，5取消
+	OperationDeptId string // 操作机构ID
+	OperationUserId string // 操作用户ID
+	TargetDeptId    string // 目标部门ID
+	TargetUserId    string // 目标用户ID
+	BeforeStatus    string // 变更前状态
+	AfterStatus     string // 变更后状态
+	TenantId        string // 租户ID
+	CreatedDept     string // 创建部门
+	CreatedBy       string // 创建者
+	CreatedAt       string // 创建时间
+	UpdatedBy       string // 更新者
+	UpdatedAt       string // 更新时间
+}
+
+// tWoRecordColumns holds the columns for the table t_wo_record.
+var tWoRecordColumns = TWoRecordColumns{
+	Id:              "id",
+	WoNo:            "wo_no",
+	OperationType:   "operation_type",
+	OperationDeptId: "operation_dept_id",
+	OperationUserId: "operation_user_id",
+	TargetDeptId:    "target_dept_id",
+	TargetUserId:    "target_user_id",
+	BeforeStatus:    "before_status",
+	AfterStatus:     "after_status",
+	TenantId:        "tenant_id",
+	CreatedDept:     "created_dept",
+	CreatedBy:       "created_by",
+	CreatedAt:       "created_at",
+	UpdatedBy:       "updated_by",
+	UpdatedAt:       "updated_at",
+}
+
+// NewTWoRecordDao creates and returns a new DAO object for table data access.
+func NewTWoRecordDao(handlers ...gdb.ModelHandler) *TWoRecordDao {
+	return &TWoRecordDao{
+		group:    "default",
+		table:    "t_wo_record",
+		columns:  tWoRecordColumns,
+		handlers: handlers,
+	}
+}
+
+// DB retrieves and returns the underlying raw database management object of the current DAO.
+func (dao *TWoRecordDao) DB() gdb.DB {
+	return g.DB(dao.group)
+}
+
+// Table returns the table name of the current DAO.
+func (dao *TWoRecordDao) Table() string {
+	return dao.table
+}
+
+// Columns returns all column names of the current DAO.
+func (dao *TWoRecordDao) Columns() TWoRecordColumns {
+	return dao.columns
+}
+
+// Group returns the database configuration group name of the current DAO.
+func (dao *TWoRecordDao) Group() string {
+	return dao.group
+}
+
+// Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation.
+func (dao *TWoRecordDao) Ctx(ctx context.Context) *gdb.Model {
+	model := dao.DB().Model(dao.table)
+	for _, handler := range dao.handlers {
+		model = handler(model)
+	}
+	return model.Safe().Ctx(ctx)
+}
+
+// Transaction wraps the transaction logic using function f.
+// It rolls back the transaction and returns the error if function f returns a non-nil error.
+// It commits the transaction and returns nil if function f returns nil.
+//
+// Note: Do not commit or roll back the transaction in function f,
+// as it is automatically handled by this function.
+func (dao *TWoRecordDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) {
+	return dao.Ctx(ctx).Transaction(ctx, f)
+}

internal/dao/sys_dept.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// sysDeptDao is the data access object for the table sys_dept.
+// You can define custom methods on it to extend its functionality as needed.
+type sysDeptDao struct {
+	*internal.SysDeptDao
+}
+
+var (
+	// SysDept is a globally accessible object for table sys_dept operations.
+	SysDept = sysDeptDao{internal.NewSysDeptDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/sys_oss.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// sysOssDao is the data access object for the table sys_oss.
+// You can define custom methods on it to extend its functionality as needed.
+type sysOssDao struct {
+	*internal.SysOssDao
+}
+
+var (
+	// SysOss is a globally accessible object for table sys_oss operations.
+	SysOss = sysOssDao{internal.NewSysOssDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_message_log.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tMessageLogDao is the data access object for the table t_message_log.
+// You can define custom methods on it to extend its functionality as needed.
+type tMessageLogDao struct {
+	*internal.TMessageLogDao
+}
+
+var (
+	// TMessageLog is a globally accessible object for table t_message_log operations.
+	TMessageLog = tMessageLogDao{internal.NewTMessageLogDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_resource_batch_download.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tResourceBatchDownloadDao is the data access object for the table t_resource_batch_download.
+// You can define custom methods on it to extend its functionality as needed.
+type tResourceBatchDownloadDao struct {
+	*internal.TResourceBatchDownloadDao
+}
+
+var (
+	// TResourceBatchDownload is a globally accessible object for table t_resource_batch_download operations.
+	TResourceBatchDownload = tResourceBatchDownloadDao{internal.NewTResourceBatchDownloadDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_sim_card_cancellation.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tSimCardCancellationDao is the data access object for the table t_sim_card_cancellation.
+// You can define custom methods on it to extend its functionality as needed.
+type tSimCardCancellationDao struct {
+	*internal.TSimCardCancellationDao
+}
+
+var (
+	// TSimCardCancellation is a globally accessible object for table t_sim_card_cancellation operations.
+	TSimCardCancellation = tSimCardCancellationDao{internal.NewTSimCardCancellationDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_sim_card_info.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tSimCardInfoDao is the data access object for the table t_sim_card_info.
+// You can define custom methods on it to extend its functionality as needed.
+type tSimCardInfoDao struct {
+	*internal.TSimCardInfoDao
+}
+
+var (
+	// TSimCardInfo is a globally accessible object for table t_sim_card_info operations.
+	TSimCardInfo = tSimCardInfoDao{internal.NewTSimCardInfoDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_sim_flow_pool.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tSimFlowPoolDao is the data access object for the table t_sim_flow_pool.
+// You can define custom methods on it to extend its functionality as needed.
+type tSimFlowPoolDao struct {
+	*internal.TSimFlowPoolDao
+}
+
+var (
+	// TSimFlowPool is a globally accessible object for table t_sim_flow_pool operations.
+	TSimFlowPool = tSimFlowPoolDao{internal.NewTSimFlowPoolDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_user_message.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tUserMessageDao is the data access object for the table t_user_message.
+// You can define custom methods on it to extend its functionality as needed.
+type tUserMessageDao struct {
+	*internal.TUserMessageDao
+}
+
+var (
+	// TUserMessage is a globally accessible object for table t_user_message operations.
+	TUserMessage = tUserMessageDao{internal.NewTUserMessageDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_warn_config.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tWarnConfigDao is the data access object for the table t_warn_config.
+// You can define custom methods on it to extend its functionality as needed.
+type tWarnConfigDao struct {
+	*internal.TWarnConfigDao
+}
+
+var (
+	// TWarnConfig is a globally accessible object for table t_warn_config operations.
+	TWarnConfig = tWarnConfigDao{internal.NewTWarnConfigDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_warn_log.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tWarnLogDao is the data access object for the table t_warn_log.
+// You can define custom methods on it to extend its functionality as needed.
+type tWarnLogDao struct {
+	*internal.TWarnLogDao
+}
+
+var (
+	// TWarnLog is a globally accessible object for table t_warn_log operations.
+	TWarnLog = tWarnLogDao{internal.NewTWarnLogDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_wo_main.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tWoMainDao is the data access object for the table t_wo_main.
+// You can define custom methods on it to extend its functionality as needed.
+type tWoMainDao struct {
+	*internal.TWoMainDao
+}
+
+var (
+	// TWoMain is a globally accessible object for table t_wo_main operations.
+	TWoMain = tWoMainDao{internal.NewTWoMainDao()}
+)
+
+// Add your custom methods and functionality below.

internal/dao/t_wo_record.go

@@ -0,0 +1,22 @@
+// =================================================================================
+// This file is auto-generated by the GoFrame CLI tool. You may modify it as needed.
+// =================================================================================
+
+package dao
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao/internal"
+)
+
+// tWoRecordDao is the data access object for the table t_wo_record.
+// You can define custom methods on it to extend its functionality as needed.
+type tWoRecordDao struct {
+	*internal.TWoRecordDao
+}
+
+var (
+	// TWoRecord is a globally accessible object for table t_wo_record operations.
+	TWoRecord = tWoRecordDao{internal.NewTWoRecordDao()}
+)
+
+// Add your custom methods and functionality below.

internal/fullsync/archive/archive.go

@@ -0,0 +1,802 @@
+// Package archive durably stores summary-only receipts for completed full-sync
+// runs and replay actions. It depends only on the shared full-sync contracts;
+// repository implementations and service wiring live elsewhere.
+package archive
+
+import (
+	"bufio"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+const (
+	defaultInterval        = time.Minute
+	defaultBackoff         = time.Second
+	defaultMaxBackoff      = time.Minute
+	defaultMaxAttempts     = 4
+	defaultReplayLeaseTTL  = time.Minute
+	defaultReplayLeaseName = "fullsync-archive"
+	defaultReleaseTimeout  = 5 * time.Second
+	maxRounds              = 10000
+)
+
+var errChecksumMismatch = gerror.New("archive checksum verification failed")
+
+// Config controls the recovery scanner. Root is the archive directory. Zero
+// timing values select conservative defaults.
+type Config struct {
+	Root             string
+	Interval         time.Duration
+	InitialBackoff   time.Duration
+	MaxBackoff       time.Duration
+	MaxAttempts      int
+	ReplayLeaseOwner string
+	ReplayLeaseTTL   time.Duration
+	Now              func() time.Time
+}
+
+// ReplayRepositories are the contract boundaries used to recover replay
+// actions left in receipt_pending after a process restart.
+type ReplayRepositories struct {
+	Discovery contract.ReplayWorkDiscovery
+	Leases    contract.ReplayLeaseStore
+	Progress  contract.ReplayProgressStore
+}
+
+// Service publishes archive files and scans durable repository indexes at
+// startup and periodically. A Service may be configured for runs, actions, or
+// both.
+type Service struct {
+	root           string
+	runDiscovery   contract.ArchiveWorkDiscovery
+	runLoader      contract.ArchiveWorkLoader
+	runRegistrar   contract.ArchiveRegistrar
+	replay         ReplayRepositories
+	interval       time.Duration
+	initialBackoff time.Duration
+	maxBackoff     time.Duration
+	maxAttempts    int
+	leaseOwner     string
+	leaseTTL       time.Duration
+	now            func() time.Time
+
+	// Publications are serialized so repeat writes in this process observe the
+	// same final file timestamp and therefore produce the same receipt.
+	publishMu sync.Mutex
+
+	mu      sync.Mutex
+	running bool
+	cancel  context.CancelFunc
+	done    chan struct{}
+
+	// These hooks keep failure-path tests deterministic without weakening the
+	// production filesystem path.
+	replaceFile  func(string, string) error
+	afterReplace func(string) error
+}
+
+// New constructs a run-archive service with default scanner settings.
+func New(root string, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
+	return NewService(Config{Root: root}, discovery, loader, registrar)
+}
+
+// NewService constructs a run-archive service. Use NewServiceWithReplay when
+// replay receipt recovery is also required.
+func NewService(cfg Config, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
+	return NewServiceWithReplay(cfg, discovery, loader, registrar, ReplayRepositories{})
+}
+
+// NewServiceWithReplay constructs a lifecycle-enabled run and replay archive
+// service using only shared contract interfaces.
+func NewServiceWithReplay(
+	cfg Config,
+	discovery contract.ArchiveWorkDiscovery,
+	loader contract.ArchiveWorkLoader,
+	registrar contract.ArchiveRegistrar,
+	replay ReplayRepositories,
+) *Service {
+	if cfg.Interval <= 0 {
+		cfg.Interval = defaultInterval
+	}
+	if cfg.InitialBackoff <= 0 {
+		cfg.InitialBackoff = defaultBackoff
+	}
+	if cfg.MaxBackoff <= 0 {
+		cfg.MaxBackoff = defaultMaxBackoff
+	}
+	if cfg.MaxBackoff < cfg.InitialBackoff {
+		cfg.MaxBackoff = cfg.InitialBackoff
+	}
+	if cfg.MaxAttempts <= 0 {
+		cfg.MaxAttempts = defaultMaxAttempts
+	}
+	if strings.TrimSpace(cfg.ReplayLeaseOwner) == "" {
+		cfg.ReplayLeaseOwner = defaultReplayLeaseName
+	}
+	if cfg.ReplayLeaseTTL <= 0 {
+		cfg.ReplayLeaseTTL = defaultReplayLeaseTTL
+	}
+	if cfg.Now == nil {
+		cfg.Now = time.Now
+	}
+
+	return &Service{
+		root:           cfg.Root,
+		runDiscovery:   discovery,
+		runLoader:      loader,
+		runRegistrar:   registrar,
+		replay:         replay,
+		interval:       cfg.Interval,
+		initialBackoff: cfg.InitialBackoff,
+		maxBackoff:     cfg.MaxBackoff,
+		maxAttempts:    cfg.MaxAttempts,
+		leaseOwner:     cfg.ReplayLeaseOwner,
+		leaseTTL:       cfg.ReplayLeaseTTL,
+		now:            cfg.Now,
+		replaceFile:    atomicReplace,
+		afterReplace:   func(string) error { return nil },
+	}
+}
+
+// Start launches an immediate recovery scan followed by periodic scans. It is
+// idempotent while the service is running.
+func (s *Service) Start(parent context.Context) error {
+	if parent == nil {
+		return gerror.New("archive parent context is nil")
+	}
+	if err := s.validateConfiguration(); err != nil {
+		return err
+	}
+
+	s.mu.Lock()
+	if s.running {
+		s.mu.Unlock()
+		return nil
+	}
+	ctx, cancel := context.WithCancel(parent)
+	done := make(chan struct{})
+	s.cancel, s.done, s.running = cancel, done, true
+	s.mu.Unlock()
+
+	go func() {
+		defer func() {
+			s.mu.Lock()
+			if s.done == done {
+				s.running = false
+				s.cancel = nil
+			}
+			s.mu.Unlock()
+			close(done)
+		}()
+		s.scanLoop(ctx)
+	}()
+	return nil
+}
+
+// Stop cancels the scanner and waits for it to finish. It is safe to call more
+// than once.
+func (s *Service) Stop() error {
+	s.mu.Lock()
+	if !s.running {
+		s.mu.Unlock()
+		return nil
+	}
+	cancel, done := s.cancel, s.done
+	s.mu.Unlock()
+
+	cancel()
+	<-done
+	return nil
+}
+
+// Close implements the common service lifecycle convention.
+func (s *Service) Close() error { return s.Stop() }
+
+// Done is closed when the current scanner invocation exits. It returns nil
+// before the service has been started.
+func (s *Service) Done() <-chan struct{} {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.done
+}
+
+// RunOnce performs one recovery pass. It is useful for explicit lifecycle
+// wiring and deterministic integration tests.
+func (s *Service) RunOnce(ctx context.Context) error {
+	if ctx == nil {
+		return gerror.New("archive scan context is nil")
+	}
+	if err := s.validateConfiguration(); err != nil {
+		return err
+	}
+
+	var scanErrors []error
+	if s.runDiscovery != nil {
+		if err := s.scanRuns(ctx); err != nil {
+			scanErrors = append(scanErrors, gerror.Wrap(err, "scan run archives"))
+		}
+	}
+	if s.replay.Discovery != nil && ctx.Err() == nil {
+		if err := s.scanActions(ctx); err != nil {
+			scanErrors = append(scanErrors, gerror.Wrap(err, "scan replay action archives"))
+		}
+	}
+	return errors.Join(scanErrors...)
+}
+
+// PublishRun atomically writes a run summary and then conditionally registers
+// the durable receipt against the work version.
+func (s *Service) PublishRun(ctx context.Context, work contract.ArchiveWork, snapshot contract.RunArchive) (contract.ArchiveReceipt, error) {
+	if err := validateWork(work); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	normalized, err := normalizeRunArchive(work, snapshot)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+
+	pathTime := normalized.Run.CompletedAt
+	if pathTime.IsZero() {
+		pathTime = normalized.Run.CreatedAt
+	}
+	receipt, err := s.publishJSON(ctx, "runs", work.RunID, pathTime, normalized)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	if s.runRegistrar != nil {
+		if err := s.runRegistrar.RegisterArchive(ctx, work, receipt); err != nil {
+			return contract.ArchiveReceipt{}, gerror.Wrap(err, "register run archive")
+		}
+	}
+	return receipt, nil
+}
+
+// PublishAction atomically writes a receipt_pending replay action summary. The
+// repository update is performed by ArchiveAction because it requires a fenced
+// replay lease.
+func (s *Service) PublishAction(ctx context.Context, action contract.ReplayAction) (contract.ArchiveReceipt, error) {
+	normalized, err := normalizeAction(action)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	return s.publishJSON(ctx, "actions", normalized.ActionID, normalized.CreatedAt, normalized)
+}
+
+// ArchiveRun loads, publishes, and registers one durable run work item.
+func (s *Service) ArchiveRun(ctx context.Context, work contract.ArchiveWork) error {
+	if s.runLoader == nil {
+		return gerror.New("archive run loader is nil")
+	}
+	if s.runRegistrar == nil {
+		return gerror.New("archive run registrar is nil")
+	}
+
+	snapshot, err := s.runLoader.LoadArchiveWork(ctx, work)
+	if err != nil {
+		s.recordRunFailure(ctx, work, err)
+		return gerror.Wrap(err, "load run archive work")
+	}
+	if _, err = s.PublishRun(ctx, work, snapshot); err != nil {
+		s.recordRunFailure(ctx, work, err)
+		return err
+	}
+	return nil
+}
+
+// ArchiveAction acquires a replay lease, atomically publishes the action
+// summary, and fences the completed receipt update through ReplayProgressStore.
+func (s *Service) ArchiveAction(ctx context.Context, action contract.ReplayAction) error {
+	if s.replay.Leases == nil || s.replay.Progress == nil {
+		return gerror.New("replay archive repositories are incomplete")
+	}
+	if action.Status != contract.ActionReceiptPending {
+		return gerror.Wrap(contract.ErrInvalidArgument, "replay action is not receipt_pending")
+	}
+	if _, err := normalizeAction(action); err != nil {
+		return err
+	}
+
+	lease, err := s.replay.Leases.AcquireReplayLease(ctx, action.ActionID, s.leaseOwner, s.leaseTTL)
+	if err != nil {
+		return gerror.Wrap(err, "acquire replay archive lease")
+	}
+	defer func() {
+		releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultReleaseTimeout)
+		defer cancel()
+		_ = s.replay.Leases.ReleaseReplayLease(releaseCtx, lease)
+	}()
+
+	receipt, err := s.PublishAction(ctx, action)
+	if err != nil {
+		return err
+	}
+	lease, err = s.replay.Leases.RenewReplayLease(ctx, lease, s.leaseTTL)
+	if err != nil {
+		return gerror.Wrap(err, "renew replay archive lease")
+	}
+	completedAt := receipt.ArchivedAt
+	_, err = s.replay.Progress.UpdateReplayAction(ctx, contract.ReplayActionUpdate{
+		Lease:            lease,
+		ExpectedRevision: action.Revision,
+		Status:           contract.ActionCompleted,
+		Progress:         action.Progress,
+		Receipt:          &receipt,
+		CompletedAt:      &completedAt,
+	})
+	if err != nil {
+		return gerror.Wrap(err, "register replay action archive")
+	}
+	return nil
+}
+
+func (s *Service) scanLoop(ctx context.Context) {
+	_ = s.RunOnce(ctx)
+	ticker := time.NewTicker(s.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			_ = s.RunOnce(ctx)
+		}
+	}
+}
+
+func (s *Service) scanRuns(ctx context.Context) error {
+	page := contract.PageRequest{Limit: contract.DefaultPageLimit}
+	seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
+	var itemErrors []error
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		result, err := s.runDiscovery.ListArchiveWork(ctx, page)
+		if err != nil {
+			return err
+		}
+		for _, work := range result.Items {
+			err := s.retry(ctx, func() error { return s.ArchiveRun(ctx, work) })
+			if err != nil {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+				itemErrors = append(itemErrors, gerror.Wrapf(err, "archive run %q", work.RunID))
+			}
+		}
+		if result.NextCursor == "" {
+			return errors.Join(itemErrors...)
+		}
+		if _, repeated := seenCursors[result.NextCursor]; repeated {
+			return errors.Join(append(itemErrors, gerror.New("run archive pagination cursor did not advance"))...)
+		}
+		seenCursors[result.NextCursor] = struct{}{}
+		page.Cursor = result.NextCursor
+	}
+}
+
+func (s *Service) scanActions(ctx context.Context) error {
+	page := contract.PageRequest{Limit: contract.DefaultPageLimit}
+	seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
+	var itemErrors []error
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		result, err := s.replay.Discovery.ListReplayWork(ctx, page)
+		if err != nil {
+			return err
+		}
+		for _, action := range result.Items {
+			if action.Status != contract.ActionReceiptPending {
+				continue
+			}
+			err := s.retry(ctx, func() error { return s.ArchiveAction(ctx, action) })
+			if errors.Is(err, contract.ErrBusy) {
+				continue
+			}
+			if err != nil {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+				itemErrors = append(itemErrors, gerror.Wrapf(err, "archive replay action %q", action.ActionID))
+			}
+		}
+		if result.NextCursor == "" {
+			return errors.Join(itemErrors...)
+		}
+		if _, repeated := seenCursors[result.NextCursor]; repeated {
+			return errors.Join(append(itemErrors, gerror.New("replay archive pagination cursor did not advance"))...)
+		}
+		seenCursors[result.NextCursor] = struct{}{}
+		page.Cursor = result.NextCursor
+	}
+}
+
+func (s *Service) retry(ctx context.Context, fn func() error) error {
+	delay := s.initialBackoff
+	var lastErr error
+	for attempt := 0; attempt < s.maxAttempts; attempt++ {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		lastErr = fn()
+		if lastErr == nil || errors.Is(lastErr, contract.ErrBusy) {
+			return lastErr
+		}
+		if attempt+1 == s.maxAttempts {
+			break
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			return ctx.Err()
+		case <-timer.C:
+		}
+		if delay <= s.maxBackoff/2 {
+			delay *= 2
+		} else {
+			delay = s.maxBackoff
+		}
+	}
+	return lastErr
+}
+
+func (s *Service) recordRunFailure(ctx context.Context, work contract.ArchiveWork, archiveErr error) {
+	if s.runRegistrar == nil || ctx.Err() != nil {
+		return
+	}
+	_ = s.runRegistrar.RegisterArchiveFailure(ctx, work, contract.BoundedErrorMessage(archiveErr.Error()))
+}
+
+func (s *Service) publishJSON(ctx context.Context, kind, id string, pathTime time.Time, value any) (contract.ArchiveReceipt, error) {
+	if ctx == nil {
+		return contract.ArchiveReceipt{}, gerror.New("archive publication context is nil")
+	}
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	dir, final, relative, err := s.storagePath(kind, id, pathTime)
+	if err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+
+	s.publishMu.Lock()
+	defer s.publishMu.Unlock()
+
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive directory")
+	}
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+
+	temp, err := os.CreateTemp(dir, "."+id+"-*.tmp")
+	if err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive temporary file")
+	}
+	tempName := temp.Name()
+	defer func() { _ = os.Remove(tempName) }()
+
+	hasher := sha256.New()
+	buffer := bufio.NewWriter(io.MultiWriter(contextWriter{ctx: ctx, writer: temp}, hasher))
+	encoder := json.NewEncoder(buffer)
+	if err = encoder.Encode(value); err == nil {
+		err = buffer.Flush()
+	}
+	if err == nil {
+		err = temp.Chmod(0o644)
+	}
+	if err == nil {
+		err = temp.Sync()
+	}
+	closeErr := temp.Close()
+	if err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		_ = temp.Close()
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "encode archive temporary file")
+	}
+	expectedChecksum := hex.EncodeToString(hasher.Sum(nil))
+
+	if receipt, ok, existingErr := s.existingReceipt(final, relative, kind, id, expectedChecksum); existingErr != nil {
+		return contract.ArchiveReceipt{}, existingErr
+	} else if ok {
+		if err := syncDirectory(dir); err != nil {
+			return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
+		}
+		return receipt, nil
+	}
+
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	if err := s.replaceFile(tempName, final); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "publish archive file")
+	}
+	if err := syncDirectory(dir); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
+	}
+	if err := s.afterReplace(final); err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "archive post-publish verification hook")
+	}
+
+	stored, err := os.ReadFile(final)
+	if err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "read published archive file")
+	}
+	storedSum := sha256.Sum256(stored)
+	if hex.EncodeToString(storedSum[:]) != expectedChecksum {
+		return contract.ArchiveReceipt{}, errChecksumMismatch
+	}
+	info, err := os.Stat(final)
+	if err != nil {
+		return contract.ArchiveReceipt{}, gerror.Wrap(err, "stat published archive file")
+	}
+	receipt := contract.ArchiveReceipt{
+		Version:    contract.ArchiveSchemaVersion,
+		Path:       relative,
+		Checksum:   expectedChecksum,
+		ArchivedAt: archiveModTime(info, s.now),
+	}
+	if err := receipt.Validate(); err != nil {
+		return contract.ArchiveReceipt{}, err
+	}
+	return receipt, nil
+}
+
+func (s *Service) existingReceipt(final, relative, kind, id, expectedChecksum string) (contract.ArchiveReceipt, bool, error) {
+	info, err := os.Lstat(final)
+	if errors.Is(err, os.ErrNotExist) {
+		return contract.ArchiveReceipt{}, false, nil
+	}
+	if err != nil {
+		return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "inspect existing archive file")
+	}
+	if !info.Mode().IsRegular() {
+		return contract.ArchiveReceipt{}, false, nil
+	}
+	stored, err := os.ReadFile(final)
+	if err != nil {
+		return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "read existing archive file")
+	}
+	storedSum := sha256.Sum256(stored)
+	if hex.EncodeToString(storedSum[:]) != expectedChecksum {
+		if validExistingArchive(kind, id, stored) {
+			return contract.ArchiveReceipt{}, false, gerror.Wrap(contract.ErrConflict, "archive path already contains different valid content")
+		}
+		return contract.ArchiveReceipt{}, false, nil
+	}
+	receipt := contract.ArchiveReceipt{
+		Version:    contract.ArchiveSchemaVersion,
+		Path:       relative,
+		Checksum:   expectedChecksum,
+		ArchivedAt: archiveModTime(info, s.now),
+	}
+	if err := receipt.Validate(); err != nil {
+		return contract.ArchiveReceipt{}, false, err
+	}
+	return receipt, true, nil
+}
+
+func validExistingArchive(kind, id string, data []byte) bool {
+	switch kind {
+	case "runs":
+		var snapshot contract.RunArchive
+		if json.Unmarshal(data, &snapshot) != nil {
+			return false
+		}
+		_, err := normalizeRunArchive(contract.ArchiveWork{RunID: id, Version: "existing"}, snapshot)
+		return err == nil
+	case "actions":
+		var action contract.ReplayAction
+		if json.Unmarshal(data, &action) != nil || action.ActionID != id {
+			return false
+		}
+		_, err := normalizeAction(action)
+		return err == nil
+	default:
+		return false
+	}
+}
+
+func archiveModTime(info os.FileInfo, now func() time.Time) time.Time {
+	archivedAt := info.ModTime().UTC()
+	if archivedAt.IsZero() {
+		archivedAt = now().UTC()
+	}
+	return archivedAt
+}
+
+func (s *Service) storagePath(kind, id string, pathTime time.Time) (string, string, string, error) {
+	if err := validateStorageID(id); err != nil {
+		return "", "", "", err
+	}
+	if kind != "runs" && kind != "actions" {
+		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "unknown archive kind")
+	}
+	if strings.TrimSpace(s.root) == "" {
+		return "", "", "", gerror.New("archive root is empty")
+	}
+	if pathTime.IsZero() {
+		pathTime = s.now().UTC()
+	} else {
+		pathTime = pathTime.UTC()
+	}
+	if pathTime.Year() < 1 || pathTime.Year() > 9999 {
+		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive timestamp is outside the supported year range")
+	}
+
+	root := filepath.Clean(s.root)
+	prefix := "run-"
+	if kind == "actions" {
+		prefix = "action-"
+	}
+	relativeOS := prefix + id + ".json"
+	final := filepath.Join(root, relativeOS)
+	contained, err := filepath.Rel(root, final)
+	if err != nil || filepath.IsAbs(contained) || contained == ".." || strings.HasPrefix(contained, ".."+string(filepath.Separator)) {
+		return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive path escapes root")
+	}
+	return filepath.Dir(final), final, filepath.ToSlash(relativeOS), nil
+}
+
+func (s *Service) validateConfiguration() error {
+	if strings.TrimSpace(s.root) == "" {
+		return gerror.New("archive root is empty")
+	}
+	if s.runDiscovery != nil && (s.runLoader == nil || s.runRegistrar == nil) {
+		return gerror.New("run archive repositories are incomplete")
+	}
+	if s.replay.Discovery != nil && (s.replay.Leases == nil || s.replay.Progress == nil) {
+		return gerror.New("replay archive repositories are incomplete")
+	}
+	return nil
+}
+
+func validateWork(work contract.ArchiveWork) error {
+	if err := validateStorageID(work.RunID); err != nil {
+		return err
+	}
+	if strings.TrimSpace(work.Version) == "" {
+		return gerror.Wrap(contract.ErrInvalidArgument, "archive work version is required")
+	}
+	return nil
+}
+
+func validateStorageID(id string) error {
+	if err := contract.ValidateID(id); err != nil {
+		return err
+	}
+	if id == "." || id == ".." || filepath.VolumeName(id) != "" || strings.ContainsRune(id, ':') || filepath.Base(id) != id {
+		return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is not safe for archive storage")
+	}
+	name := strings.ToUpper(strings.TrimSuffix(id, filepath.Ext(id)))
+	switch name {
+	case "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9":
+		return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is reserved by the filesystem")
+	}
+	return nil
+}
+
+func normalizeRunArchive(work contract.ArchiveWork, snapshot contract.RunArchive) (contract.RunArchive, error) {
+	if snapshot.Version == 0 {
+		snapshot.Version = contract.ArchiveSchemaVersion
+	}
+	if snapshot.Version != contract.ArchiveSchemaVersion {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "invalid archive schema version")
+	}
+	if snapshot.Run.RunID != work.RunID {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archive run ID does not match work")
+	}
+	if !snapshot.Run.Status.Terminal() {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "only terminal runs can be archived")
+	}
+	if err := snapshot.Run.Validate(); err != nil {
+		return contract.RunArchive{}, err
+	}
+	if len(snapshot.Rounds) > maxRounds {
+		return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "too many archived rounds")
+	}
+
+	snapshot.Run = normalizeRunMeta(snapshot.Run)
+	snapshot.Rounds = append([]contract.RoundStats(nil), snapshot.Rounds...)
+	if snapshot.Rounds == nil {
+		snapshot.Rounds = make([]contract.RoundStats, 0)
+	}
+	seen := make(map[int]struct{}, len(snapshot.Rounds))
+	for i := range snapshot.Rounds {
+		round := &snapshot.Rounds[i]
+		if round.RunID != work.RunID {
+			return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archived round belongs to another run")
+		}
+		if _, duplicate := seen[round.Round]; duplicate {
+			return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "duplicate archived round")
+		}
+		seen[round.Round] = struct{}{}
+		if err := round.Validate(); err != nil {
+			return contract.RunArchive{}, err
+		}
+		normalizedCounts, err := round.ErrorCounts.Normalized()
+		if err != nil {
+			return contract.RunArchive{}, err
+		}
+		round.ErrorCounts = normalizedCounts
+		round.StartedAt = utcTimePtr(round.StartedAt)
+		round.CompletedAt = utcTimePtr(round.CompletedAt)
+	}
+	sort.Slice(snapshot.Rounds, func(i, j int) bool { return snapshot.Rounds[i].Round < snapshot.Rounds[j].Round })
+	return snapshot, nil
+}
+
+func normalizeRunMeta(meta contract.RunMeta) contract.RunMeta {
+	meta.CreatedAt = meta.CreatedAt.UTC()
+	meta.ProducerCompletedAt = meta.ProducerCompletedAt.UTC()
+	meta.CompletedAt = meta.CompletedAt.UTC()
+	meta.StartedAt = utcTimePtr(meta.StartedAt)
+	meta.ScanCompletedAt = utcTimePtr(meta.ScanCompletedAt)
+	// Archive state describes the repository registration and is intentionally
+	// excluded from the file whose receipt populates that state.
+	meta.ArchiveStatus = ""
+	meta.ArchiveVersion = 0
+	meta.ArchivePath = ""
+	meta.ArchiveChecksum = ""
+	meta.ArchivedAt = nil
+	return meta
+}
+
+func normalizeAction(action contract.ReplayAction) (contract.ReplayAction, error) {
+	if err := validateStorageID(action.ActionID); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	if action.Status != contract.ActionReceiptPending {
+		return contract.ReplayAction{}, gerror.Wrap(contract.ErrInvalidArgument, "only receipt_pending replay actions can be archived")
+	}
+	action.Error = contract.BoundedErrorMessage(action.Error)
+	action.CreatedAt = action.CreatedAt.UTC()
+	action.UpdatedAt = action.UpdatedAt.UTC()
+	action.CompletedAt = nil
+	action.Receipt = nil
+	if err := action.Validate(); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	return action, nil
+}
+
+func utcTimePtr(value *time.Time) *time.Time {
+	if value == nil {
+		return nil
+	}
+	utc := value.UTC()
+	return &utc
+}
+
+type contextWriter struct {
+	ctx    context.Context
+	writer io.Writer
+}
+
+func (w contextWriter) Write(p []byte) (int, error) {
+	if err := w.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return w.writer.Write(p)
+}

internal/fullsync/archive/archive_test.go

@@ -0,0 +1,588 @@
+package archive
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestPublishRunIsDurableDeterministicAndSummaryOnly(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-1"))
+	service := New(root, nil, nil, repository)
+	work := contract.ArchiveWork{RunID: "run-1", Version: "version-1"}
+
+	staleDir := root
+	if err := os.MkdirAll(staleDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	stalePath := filepath.Join(staleDir, ".run-1-stale.tmp")
+	if err := os.WriteFile(stalePath, []byte("incomplete"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("PublishRun() error = %v", err)
+	}
+	if receipt.Path != "run-run-1.json" {
+		t.Fatalf("receipt path = %q", receipt.Path)
+	}
+	if err := receipt.Validate(); err != nil {
+		t.Fatalf("receipt validation failed: %v", err)
+	}
+
+	archivePath := filepath.Join(root, filepath.FromSlash(receipt.Path))
+	info, err := os.Stat(archivePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runtime.GOOS != "windows" {
+		if got := info.Mode().Perm(); got != 0o644 {
+			t.Fatalf("archive mode = %o, want 644", got)
+		}
+	}
+	data, err := os.ReadFile(archivePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(data)
+	if got := hex.EncodeToString(sum[:]); got != receipt.Checksum {
+		t.Fatalf("checksum = %q, want %q", receipt.Checksum, got)
+	}
+	for _, forbidden := range [][]byte{[]byte(`"archiveStatus"`), []byte(`"archivePath"`), []byte(`"taskId"`), []byte(`"card"`), []byte(`"message"`)} {
+		if bytes.Contains(data, forbidden) {
+			t.Fatalf("archive contains forbidden detail %s: %s", forbidden, data)
+		}
+	}
+	var decoded contract.RunArchive
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded.Rounds) != 2 || decoded.Rounds[0].Round != 1 || decoded.Rounds[1].Round != 2 {
+		t.Fatalf("rounds were not deterministically sorted: %+v", decoded.Rounds)
+	}
+	if decoded.Rounds[1].ErrorCounts[contract.ErrorRateLimited] != 1 {
+		t.Fatalf("error categories were not normalized: %+v", decoded.Rounds[1].ErrorCounts)
+	}
+
+	repeated, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("repeated PublishRun() error = %v", err)
+	}
+	if !reflect.DeepEqual(repeated, receipt) {
+		t.Fatalf("repeated receipt changed:\nfirst:  %+v\nsecond: %+v", receipt, repeated)
+	}
+	if _, err := os.Stat(stalePath); err != nil {
+		t.Fatalf("stale temp residue should not block publication: %v", err)
+	}
+	repository.mu.Lock()
+	defer repository.mu.Unlock()
+	if len(repository.receipts) != 2 || !reflect.DeepEqual(repository.receipts[0], repository.receipts[1]) {
+		t.Fatalf("registered receipts = %+v", repository.receipts)
+	}
+}
+
+func TestArchiveActionPublishesAndRegistersFencedReceipt(t *testing.T) {
+	root := t.TempDir()
+	action := testReplayAction("action-1")
+	action.Error = strings.Repeat("x", contract.MaxErrorMessageBytes+50)
+	repository := newFakeReplayRepository(action)
+	service := NewServiceWithReplay(
+		Config{Root: root},
+		nil,
+		nil,
+		nil,
+		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
+	)
+
+	if err := service.ArchiveAction(context.Background(), action); err != nil {
+		t.Fatalf("ArchiveAction() error = %v", err)
+	}
+	repository.mu.Lock()
+	update := repository.update
+	released := repository.released
+	repository.mu.Unlock()
+	if update == nil || update.Status != contract.ActionCompleted || update.Receipt == nil || update.CompletedAt == nil {
+		t.Fatalf("registered update = %+v", update)
+	}
+	if update.ExpectedRevision != action.Revision || update.Receipt.ArchivedAt != *update.CompletedAt {
+		t.Fatalf("receipt registration lost fencing/time: %+v", update)
+	}
+	if update.Lease.Token != "lease-token-2" {
+		t.Fatalf("receipt registration used the unrenewed lease: %+v", update.Lease)
+	}
+	if released.Token != "lease-token-2" {
+		t.Fatalf("archive released the stale lease: %+v", released)
+	}
+	if update.Receipt.Path != "action-action-1.json" {
+		t.Fatalf("action receipt path = %q", update.Receipt.Path)
+	}
+
+	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(update.Receipt.Path)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, forbidden := range [][]byte{[]byte("idempotencyKey"), []byte("fingerprint"), []byte("leaseOwner"), []byte("leaseToken"), []byte("revision"), []byte(`"receipt"`), []byte("completedAt")} {
+		if bytes.Contains(data, forbidden) {
+			t.Fatalf("action archive contains internal field %q: %s", forbidden, data)
+		}
+	}
+	var decoded contract.ReplayAction
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.Status != contract.ActionReceiptPending || len(decoded.Error) != contract.MaxErrorMessageBytes {
+		t.Fatalf("archived action was not bounded receipt_pending summary: %+v", decoded)
+	}
+	repeated, err := service.PublishAction(context.Background(), action)
+	if err != nil {
+		t.Fatalf("repeated PublishAction() error = %v", err)
+	}
+	if !reflect.DeepEqual(repeated, *update.Receipt) {
+		t.Fatalf("repeated action receipt changed: %+v != %+v", repeated, *update.Receipt)
+	}
+}
+
+func TestPublishRunCleansTempAfterRenameFailure(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-rename"))
+	service := New(root, nil, nil, repository)
+	renameErr := gerror.New("injected rename failure")
+	service.replaceFile = func(string, string) error { return renameErr }
+
+	_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: "run-rename", Version: "v1"}, repository.snapshot)
+	if !errors.Is(err, renameErr) {
+		t.Fatalf("PublishRun() error = %v, want rename failure", err)
+	}
+	if matches, globErr := filepath.Glob(filepath.Join(root, ".run-rename-*.tmp")); globErr != nil || len(matches) != 0 {
+		t.Fatalf("temporary files after failure = %v, error = %v", matches, globErr)
+	}
+	if _, statErr := os.Stat(filepath.Join(root, "run-run-rename.json")); !errors.Is(statErr, os.ErrNotExist) {
+		t.Fatalf("final file exists after failed rename: %v", statErr)
+	}
+	if len(repository.receipts) != 0 {
+		t.Fatalf("registered receipt after failed rename: %+v", repository.receipts)
+	}
+}
+
+func TestChecksumFailureIsRecoveredByExactPathReplacement(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-checksum"))
+	service := New(root, nil, nil, repository)
+	service.afterReplace = func(path string) error {
+		return os.WriteFile(path, []byte("corrupt"), 0o600)
+	}
+	work := contract.ArchiveWork{RunID: "run-checksum", Version: "v1"}
+
+	if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, errChecksumMismatch) {
+		t.Fatalf("PublishRun() error = %v, want checksum mismatch", err)
+	}
+	if len(repository.receipts) != 0 {
+		t.Fatalf("registered receipt after checksum failure: %+v", repository.receipts)
+	}
+	service.afterReplace = func(string) error { return nil }
+	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("recovery PublishRun() error = %v", err)
+	}
+	if receipt.Path != "run-run-checksum.json" {
+		t.Fatalf("replacement used different path: %q", receipt.Path)
+	}
+	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(receipt.Path)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) == "corrupt" {
+		t.Fatal("corrupt final file was not replaced")
+	}
+}
+
+func TestRegistrationRetryReusesIdenticalReceipt(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-register"))
+	repository.registerErr = gerror.New("registration result was uncertain")
+	service := New(root, nil, nil, repository)
+	work := contract.ArchiveWork{RunID: "run-register", Version: "v1"}
+
+	if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, repository.registerErr) {
+		t.Fatalf("first PublishRun() error = %v", err)
+	}
+	repository.mu.Lock()
+	firstReceipt := repository.receipts[0]
+	repository.registerErr = nil
+	repository.mu.Unlock()
+
+	receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatalf("retry PublishRun() error = %v", err)
+	}
+	if !reflect.DeepEqual(receipt, firstReceipt) {
+		t.Fatalf("retry changed uncertain receipt:\nfirst: %+v\nretry: %+v", firstReceipt, receipt)
+	}
+}
+
+func TestPublishRunRejectsDifferentValidContentAtSamePath(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-conflict"))
+	service := New(root, nil, nil, repository)
+	work := contract.ArchiveWork{RunID: "run-conflict", Version: "v1"}
+
+	first, err := service.PublishRun(context.Background(), work, repository.snapshot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	changed := repository.snapshot
+	changed.Run.TriggerSource = "different-valid-summary"
+	if _, err = service.PublishRun(context.Background(), work, changed); !errors.Is(err, contract.ErrConflict) {
+		t.Fatalf("PublishRun() error = %v, want conflict", err)
+	}
+	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(first.Path)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(data)
+	if got := hex.EncodeToString(sum[:]); got != first.Checksum {
+		t.Fatalf("conflicting publication replaced the original file: %q != %q", got, first.Checksum)
+	}
+}
+
+func TestScannerRecoversFailedRunAfterRestart(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-restart"))
+	repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
+		return contract.RunArchive{}, gerror.New("source temporarily unavailable")
+	}
+	first := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
+	if err := first.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.failed, "initial archive failure")
+	if err := first.Stop(); err != nil {
+		t.Fatal(err)
+	}
+
+	repository.mu.Lock()
+	repository.load = nil
+	repository.mu.Unlock()
+	second := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
+	if err := second.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.succeeded, "restart archive success")
+	if err := second.Stop(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(root, "run-run-restart.json")); err != nil {
+		t.Fatalf("recovered archive file missing: %v", err)
+	}
+}
+
+func TestPeriodicScannerRetriesPendingRun(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-periodic"))
+	var calls atomic.Int32
+	repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
+		if calls.Add(1) == 1 {
+			return contract.RunArchive{}, gerror.New("first scan fails")
+		}
+		return repository.snapshot, nil
+	}
+	service := NewService(Config{Root: root, Interval: 10 * time.Millisecond, MaxAttempts: 1}, repository, repository, repository)
+	if err := service.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.succeeded, "periodic archive success")
+	if err := service.Stop(); err != nil {
+		t.Fatal(err)
+	}
+	if calls.Load() < 2 {
+		t.Fatalf("loader calls = %d, want at least 2", calls.Load())
+	}
+}
+
+func TestStartupScannerArchivesReceiptPendingAction(t *testing.T) {
+	action := testReplayAction("action-startup")
+	repository := newFakeReplayRepository(action)
+	service := NewServiceWithReplay(
+		Config{Root: t.TempDir(), Interval: time.Hour, MaxAttempts: 1},
+		nil,
+		nil,
+		nil,
+		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
+	)
+	if err := service.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, repository.completed, "startup replay archive success")
+	if err := service.Stop(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCancellationStopsBlockedScannerAndPublication(t *testing.T) {
+	root := t.TempDir()
+	repository := newFakeRunRepository(testRunArchive("run-cancel"))
+	started := make(chan struct{})
+	repository.load = func(ctx context.Context, _ contract.ArchiveWork) (contract.RunArchive, error) {
+		close(started)
+		<-ctx.Done()
+		return contract.RunArchive{}, ctx.Err()
+	}
+	service := NewService(Config{Root: root, Interval: time.Hour, InitialBackoff: time.Hour}, repository, repository, repository)
+	if err := service.Start(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	waitSignal(t, started, "blocked loader start")
+	stopped := make(chan struct{})
+	go func() {
+		_ = service.Stop()
+		close(stopped)
+	}()
+	waitSignal(t, stopped, "scanner cancellation")
+
+	canceled, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := service.PublishRun(canceled, contract.ArchiveWork{RunID: "run-cancel", Version: "v1"}, repository.snapshot)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("canceled PublishRun() error = %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(root, "run-run-cancel.json")); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("canceled publication wrote final file: %v", err)
+	}
+}
+
+func TestCancellationAfterRenameStillVerifiesPublicationAndReleasesLease(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	action := testReplayAction("action-cancel-after-rename")
+	repository := newFakeReplayRepository(action)
+	service := NewServiceWithReplay(
+		Config{Root: t.TempDir()},
+		nil,
+		nil,
+		nil,
+		ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
+	)
+	service.afterReplace = func(string) error {
+		cancel()
+		return nil
+	}
+
+	if err := service.ArchiveAction(ctx, action); err != nil {
+		t.Fatalf("ArchiveAction() error = %v", err)
+	}
+	repository.mu.Lock()
+	releaseCtxErr := repository.releaseCtxErr
+	repository.mu.Unlock()
+	if releaseCtxErr != nil {
+		t.Fatalf("lease release inherited canceled context: %v", releaseCtxErr)
+	}
+}
+
+func TestArchiveRejectsTraversalAndUnsafeFilesystemIDs(t *testing.T) {
+	root := t.TempDir()
+	service := New(root, nil, nil, newFakeRunRepository(contract.RunArchive{}))
+	for _, id := range []string{"../escape", `..\escape`, "/absolute", `C:\escape`, "drive:name", ".", "..", "{hash}", "CON"} {
+		t.Run(strings.ReplaceAll(id, "/", "_"), func(t *testing.T) {
+			snapshot := testRunArchive(id)
+			_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: id, Version: "v1"}, snapshot)
+			if !errors.Is(err, contract.ErrInvalidArgument) {
+				t.Fatalf("PublishRun(%q) error = %v, want invalid argument", id, err)
+			}
+		})
+	}
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("unsafe IDs created archive entries: %+v", entries)
+	}
+}
+
+func testRunArchive(runID string) contract.RunArchive {
+	created := time.Date(2026, time.August, 31, 23, 0, 0, 0, time.FixedZone("test", 8*60*60))
+	completed := time.Date(2026, time.September, 20, 16, 0, 0, 0, time.FixedZone("test", 8*60*60))
+	archived := completed.Add(time.Minute)
+	return contract.RunArchive{
+		Version: contract.ArchiveSchemaVersion,
+		Run: contract.RunMeta{
+			RunID: runID, Status: contract.RunCompletedWithFailures, ProducerDone: true,
+			CurrentRound: 2, MaxRound: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1, PendingFailureCount: 1,
+			CreatedAt: created, ProducerCompletedAt: completed, CompletedAt: completed,
+			ArchiveStatus: contract.ArchiveFailed, ArchiveVersion: 99, ArchivePath: "old.json",
+			ArchiveChecksum: strings.Repeat("a", 64), ArchivedAt: &archived,
+		},
+		Rounds: []contract.RoundStats{
+			{RunID: runID, Round: 2, EnqueuedCount: 1, FailedCount: 1, ErrorCounts: contract.ErrorCounts{"mobile_busy": 1}},
+			{RunID: runID, Round: 1, EnqueuedCount: 2, SuccessCount: 2, ErrorCounts: nil},
+		},
+	}
+}
+
+func testReplayAction(actionID string) contract.ReplayAction {
+	created := time.Date(2026, time.September, 20, 12, 0, 0, 0, time.FixedZone("test", 8*60*60))
+	return contract.ReplayAction{
+		ActionID: actionID, SourceRunID: "source-1", TargetRunID: "target-1", RequestedBy: "operator",
+		IdempotencyKey: "secret-key", Fingerprint: "secret-fingerprint", Status: contract.ActionReceiptPending,
+		ReplayCycle: 1, Progress: contract.ReplayProgress{DiscoveredCount: 2, EnqueuedCount: 2, CleanedCount: 2},
+		LeaseOwner: "engine", LeaseToken: "secret-token", LeaseUntil: created.Add(time.Hour), Revision: 7,
+		CreatedAt: created, UpdatedAt: created.Add(time.Minute),
+	}
+}
+
+type fakeRunRepository struct {
+	mu          sync.Mutex
+	work        contract.ArchiveWork
+	snapshot    contract.RunArchive
+	load        func(context.Context, contract.ArchiveWork) (contract.RunArchive, error)
+	registered  bool
+	receipts    []contract.ArchiveReceipt
+	failures    []string
+	registerErr error
+	succeeded   chan struct{}
+	failed      chan struct{}
+}
+
+func newFakeRunRepository(snapshot contract.RunArchive) *fakeRunRepository {
+	return &fakeRunRepository{
+		work:      contract.ArchiveWork{RunID: snapshot.Run.RunID, Version: "repository-version"},
+		snapshot:  snapshot,
+		succeeded: make(chan struct{}, 8),
+		failed:    make(chan struct{}, 8),
+	}
+}
+
+func (r *fakeRunRepository) ListArchiveWork(ctx context.Context, _ contract.PageRequest) (contract.ArchiveWorkPage, error) {
+	if err := ctx.Err(); err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.registered {
+		return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}, nil
+	}
+	return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{r.work}}, nil
+}
+
+func (r *fakeRunRepository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
+	r.mu.Lock()
+	load := r.load
+	snapshot := r.snapshot
+	r.mu.Unlock()
+	if load != nil {
+		return load(ctx, work)
+	}
+	return snapshot, nil
+}
+
+func (r *fakeRunRepository) RegisterArchive(_ context.Context, _ contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
+	r.mu.Lock()
+	r.receipts = append(r.receipts, receipt)
+	err := r.registerErr
+	if err == nil {
+		r.registered = true
+	}
+	r.mu.Unlock()
+	if err != nil {
+		return err
+	}
+	notify(r.succeeded)
+	return nil
+}
+
+func (r *fakeRunRepository) RegisterArchiveFailure(_ context.Context, _ contract.ArchiveWork, message string) error {
+	r.mu.Lock()
+	r.failures = append(r.failures, message)
+	r.mu.Unlock()
+	notify(r.failed)
+	return nil
+}
+
+type fakeReplayRepository struct {
+	mu            sync.Mutex
+	action        contract.ReplayAction
+	update        *contract.ReplayActionUpdate
+	released      contract.ReplayLease
+	releaseCtxErr error
+	completed     chan struct{}
+}
+
+func newFakeReplayRepository(action contract.ReplayAction) *fakeReplayRepository {
+	return &fakeReplayRepository{action: action, completed: make(chan struct{}, 4)}
+}
+
+func (r *fakeReplayRepository) ListReplayWork(ctx context.Context, _ contract.PageRequest) (contract.ReplayActionPage, error) {
+	if err := ctx.Err(); err != nil {
+		return contract.ReplayActionPage{}, err
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.update != nil {
+		return contract.ReplayActionPage{Items: []contract.ReplayAction{}}, nil
+	}
+	return contract.ReplayActionPage{Items: []contract.ReplayAction{r.action}}, nil
+}
+
+func (r *fakeReplayRepository) AcquireReplayLease(_ context.Context, actionID, owner string, ttl time.Duration) (contract.ReplayLease, error) {
+	return contract.ReplayLease{ActionID: actionID, Owner: owner, Token: "lease-token", Until: time.Now().Add(ttl)}, nil
+}
+
+func (r *fakeReplayRepository) RenewReplayLease(_ context.Context, lease contract.ReplayLease, ttl time.Duration) (contract.ReplayLease, error) {
+	lease.Token = "lease-token-2"
+	lease.Until = time.Now().Add(ttl)
+	return lease, nil
+}
+
+func (r *fakeReplayRepository) ReleaseReplayLease(ctx context.Context, lease contract.ReplayLease) error {
+	r.mu.Lock()
+	r.released = lease
+	r.releaseCtxErr = ctx.Err()
+	r.mu.Unlock()
+	return nil
+}
+
+func (r *fakeReplayRepository) UpdateReplayAction(_ context.Context, update contract.ReplayActionUpdate) (contract.ReplayAction, error) {
+	r.mu.Lock()
+	copyOfUpdate := update
+	r.update = &copyOfUpdate
+	action := r.action
+	action.Status = update.Status
+	action.Receipt = update.Receipt
+	action.CompletedAt = update.CompletedAt
+	r.mu.Unlock()
+	notify(r.completed)
+	return action, nil
+}
+
+func notify(ch chan struct{}) {
+	select {
+	case ch <- struct{}{}:
+	default:
+	}
+}
+
+func waitSignal(t *testing.T, ch <-chan struct{}, operation string) {
+	t.Helper()
+	select {
+	case <-ch:
+	case <-time.After(3 * time.Second):
+		t.Fatalf("timed out waiting for %s", operation)
+	}
+}

internal/fullsync/archive/replace_other.go

@@ -0,0 +1,21 @@
+//go:build !windows
+
+package archive
+
+import "os"
+
+func atomicReplace(from, to string) error {
+	return os.Rename(from, to)
+}
+
+func syncDirectory(path string) error {
+	dir, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	if err := dir.Sync(); err != nil {
+		_ = dir.Close()
+		return err
+	}
+	return dir.Close()
+}

internal/fullsync/archive/replace_windows.go

@@ -0,0 +1,21 @@
+//go:build windows
+
+package archive
+
+import "golang.org/x/sys/windows"
+
+func atomicReplace(from, to string) error {
+	fromUTF16, err := windows.UTF16PtrFromString(from)
+	if err != nil {
+		return err
+	}
+	toUTF16, err := windows.UTF16PtrFromString(to)
+	if err != nil {
+		return err
+	}
+	return windows.MoveFileEx(fromUTF16, toUTF16, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH)
+}
+
+// Windows has no portable directory fsync operation. MoveFileEx with
+// MOVEFILE_WRITE_THROUGH provides the strongest supported replacement flush.
+func syncDirectory(string) error { return nil }

internal/fullsync/contract/README.md

@@ -0,0 +1,124 @@
+# Full-sync shared contracts
+
+This package establishes the boundary for later queue accounting, archive,
+replay, and HTTP work. It does not change running service behavior. Production
+code in this package does not import the queue package; compatibility tests
+compare its models and keys with the existing queue implementation.
+
+## Compatibility and counters
+
+- `RunMeta` retains all existing JSON fields and adds optional fields. Existing
+  `zset.RunMeta`, `zset.Task`, and `zset.FailureRecord` remain unchanged; adapters
+  are a later integration step. The new `RunStatus` is a named string type.
+- `Validate` accepts missing extension fields on old records.
+  `ValidateAccounting` is opt-in for runs with known extended counters and
+  enforces `enqueuedCount = successCount + failedCount + activeCount + nextRoundCount`.
+  `inflightCount` is a subset of `activeCount`.
+- Source cleanup changes `pendingFailureCount`, never historical `failedCount`.
+- Round statistics have no task/card/channel dimensions. `ErrorCounts`
+  normalizes both JSON input and output to eight canonical categories, merging
+  unknown names into `other` and rejecting negative/overflowing totals.
+- Existing failure payloads retain their free-form classification and message.
+  The HTTP `Failure` projection normalizes categories and bounds messages to
+  2,048 UTF-8 bytes. Historical blocked tasks may have round zero.
+
+## Status domains
+
+| Domain | Wire values |
+| --- | --- |
+| Run | `producing`, `running`, `completed`, `completed_with_failures` |
+| Sync type | `normal`, `replay` |
+| Archive | `disabled`, `pending`, `processing`, `succeeded`, `failed` |
+| Source replay disposition | `none`, `pending`, `processing`, `completed`, `needs_attention` |
+| Replay action | `accepted`, `processing`, `cleanup_pending`, `receipt_pending`, `completed`, `needs_attention` |
+| Error category | `timeout`, `rate_limited`, `authentication`, `validation`, `upstream`, `transport`, `internal`, `other` |
+
+The source field is `replayDisposition`, distinct from the replay action's
+`status`. Lease tokens, record versions, action revision, idempotency key, and
+request fingerprint are internal and excluded from HTTP JSON.
+
+## Interfaces and persistence obligations
+
+HTTP consumers depend on `RunReader`, `FailurePager`, `ReplayRequester`, and
+`ActionReader`. Separate live/archive readers support fallback only after a live
+`ErrNotFound`; storage failures must remain visible. An archived summary cannot
+provide per-card details: use `ErrFailureDetailsUnavailable` when live details
+are gone. Repositories and HTTP both apply the default page limit 100 and
+maximum 500. Cursors remain opaque and are bound to the original query. An empty
+scan page with a next cursor does not signal completion.
+
+Archive discovery, scheduling, snapshot loading, and result registration use
+separate narrow interfaces. Registration is conditional on an opaque work
+version. Archives contain run and round aggregates only. Archive receipts use
+a schema version, storage-owned path, lowercase SHA-256 checksum of stored
+bytes, and publication time. Archive schema version starts at 1.
+
+Replay acceptance atomically reserves `(sourceRunId, idempotencyKey)` with a
+SHA-256 fingerprint of the source and exact requestedBy body value. Repeating
+the request returns its original action/target; changing the request returns
+`ErrConflict`. The action fixes its target ID before target creation/enqueue.
+
+Replay work discovery, lease management, progress updates, source pagination,
+target creation/enqueue, and source cleanup are separate interfaces. Progress
+updates are fenced and use a revision check. Target enqueue distinguishes
+durably present duplicates from rejected writes; a zero insert count by itself
+does not authorize source cleanup. Cleanup checks exact source record versions
+and target acceptance evidence. Implementations must persist enough evidence
+to recover partial batches; a scan cursor by itself is insufficient. Source
+pagination must remain correct while already-enqueued entries are deleted.
+
+`ErrNotFound`, `ErrConflict`, `ErrInvalidArgument`,
+`ErrFailureDetailsUnavailable`, `ErrLeaseLost`, and `ErrBusy` support
+`errors.Is` through wrapping. HTTP error envelopes and status mappings remain
+the HTTP implementation's responsibility.
+
+## Redis naming
+
+All names are prefixed with the configured namespace (`flow:` by default;
+`sim-status:` for status sync). Existing run/round/task-state/failure names are
+preserved. New keys include:
+
+| Suffix after namespace | Scope |
+| --- | --- |
+| `{runId}:round:N:stats` | Run slot |
+| `{runId}:archive` | Run slot |
+| `runs:all` | Global index |
+| `archive:pending` | Global index |
+| `{replay}:action:actionId:meta` | Replay slot |
+| `{replay}:action:actionId:lease` | Replay slot |
+| `{replay}:idempotency:SHA256(sourceRunId,key)` | Replay slot; tuple encoded as a JSON array before hashing |
+| `{replay}:pending` | Replay slot |
+
+Replay action creation, idempotency, and discovery can be atomic within their
+shared slot. Source/target runs and global indexes can occupy different slots;
+their coordination requires recoverable registrations/receipts rather than a
+single cross-slot Lua transaction. New identifier validation rejects hash-tag
+or path delimiters without rewriting legacy JSON on decode.
+
+## HTTP surface
+
+| Method | Route | Response |
+| --- | --- | --- |
+| GET | `/internal/v1/full-sync-runs?syncKind=&limit=` | `RunList` merged from active Redis Runs and `run-<runId>.json` archives |
+| GET | `/internal/v1/full-sync-runs/{runId}` | `RunView` with `source`, `run`, optional `rounds` |
+| GET | `/internal/v1/full-sync-runs/{runId}/failures?cursor=&limit=` | `FailurePage` with `items`, `nextCursor` |
+| POST | `/internal/v1/full-sync-runs/{runId}/replay` | HTTP 202 `ReplayAccepted` |
+| GET | `/internal/v1/full-sync-actions/{actionId}` | `ReplayAction` |
+
+POST requires `Idempotency-Key` and JSON `{"requestedBy":"..."}`. Authentication,
+HTTP handlers, storage, replay execution, archive files, and service wiring are
+intentionally left to subsequent implementations.
+
+## Verification
+
+```text
+gofmt -l internal/fullsync/contract
+go test ./internal/fullsync/contract
+go test ./internal/queue/zset ./internal/consumer/zset ./internal/producer/simstatus ./internal/producer/monthlyusage ./internal/service/simstatus ./internal/service/monthlyusage
+```
+
+Tests cover legacy schema and JSON compatibility, existing key compatibility,
+cluster key boundaries, bounded categories/messages, overflow, run accounting,
+pagination limits, status domains, replay identity, internal field exclusion,
+and receipt/progress validation. Regression tests use the repository's isolated
+Redis helper and require `redis-server` or `REDIS_SERVER_BINARY`.

internal/fullsync/contract/archive.go

@@ -0,0 +1,46 @@
+package contract
+
+import "time"
+
+const ArchiveSchemaVersion = 1
+
+// ArchiveReceipt identifies a durably published archive object. Path is a
+// storage-owned reference, never a client-supplied destination. Checksum is
+// lowercase hex SHA-256 of the final stored bytes; version is the schema.
+type ArchiveReceipt struct {
+	Version    int       `json:"version"`
+	Path       string    `json:"path"`
+	Checksum   string    `json:"checksum"`
+	ArchivedAt time.Time `json:"archivedAt"`
+}
+
+func (r ArchiveReceipt) Validate() error {
+	if r.Version < 1 || r.Path == "" || r.ArchivedAt.IsZero() || len(r.Checksum) != 64 {
+		return invalid("incomplete archive receipt")
+	}
+	for _, c := range r.Checksum {
+		if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
+			return invalid("archive checksum must be lowercase SHA-256 hex")
+		}
+	}
+	return nil
+}
+
+// RunArchive is summary-only: never embed per-card task/failure records.
+type RunArchive struct {
+	Version int          `json:"version"`
+	Run     RunMeta      `json:"run"`
+	Rounds  []RoundStats `json:"rounds"`
+}
+
+// ArchiveWork.Version is an opaque concurrency token for registration; it is
+// distinct from the archive schema version. Work discovery must survive restarts.
+type ArchiveWork struct {
+	RunID   string `json:"runId"`
+	Version string `json:"-"`
+}
+
+type ArchiveWorkPage struct {
+	Items      []ArchiveWork `json:"items"`
+	NextCursor Cursor        `json:"nextCursor"`
+}

internal/fullsync/contract/compatibility_test.go

@@ -0,0 +1,107 @@
+package contract_test
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+// Check every old field, including zero/omitempty behavior, so future adapters
+// can adopt the contract without silently changing the persisted wire format.
+func TestLegacyFieldSchemas(t *testing.T) {
+	for _, pair := range []struct{ old, next any }{
+		{zset.RunMeta{}, contract.RunMeta{}},
+		{zset.FailureRecord{}, contract.FailureRecord{}},
+		{zset.Task{}, contract.Task{}},
+	} {
+		old, next := reflect.TypeOf(pair.old), reflect.TypeOf(pair.next)
+		for i := 0; i < old.NumField(); i++ {
+			field := old.Field(i)
+			replacement, ok := next.FieldByName(field.Name)
+			if !ok || replacement.Tag.Get("json") != field.Tag.Get("json") || replacement.Type.Kind() != field.Type.Kind() {
+				t.Errorf("%s.%s lost its legacy schema: old=%+v new=%+v", old.Name(), field.Name, field, replacement)
+			}
+		}
+	}
+}
+
+func TestLegacyRunJSONRoundTrip(t *testing.T) {
+	now := time.Date(2026, 9, 20, 1, 2, 3, 456, time.UTC)
+	legacy := zset.RunMeta{
+		RunID: "legacy", Operation: "current_month_usage_full_sync", QueryMonth: "202609",
+		Status: "completed_with_failures", ProducerDone: true, CurrentRound: 5,
+		SuccessCount: 9, FailedCount: 1, CreatedAt: now, ProducerCompletedAt: now, CompletedAt: now,
+	}
+	data := mustJSON(t, legacy)
+	var decoded contract.RunMeta
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if err := decoded.Validate(); err != nil {
+		t.Fatalf("legacy run rejected: %v", err)
+	}
+	if got := string(mustJSON(t, decoded)); got != string(data) {
+		t.Fatalf("legacy wire format changed:\n%s\n%s", data, got)
+	}
+	// New fields do not prevent old readers from consuming the same metadata.
+	decoded.SyncType = contract.SyncReplay
+	decoded.SourceRunID = "source"
+	decoded.ReplayCycle = 1
+	decoded.EnqueuedCount = 10
+	decoded.PendingFailureCount = 1
+	var oldReader zset.RunMeta
+	if err := json.Unmarshal(mustJSON(t, decoded), &oldReader); err != nil {
+		t.Fatal(err)
+	}
+	oldReader.SyncType = ""
+	oldReader.SourceRunID = ""
+	oldReader.ReplayCycle = 0
+	oldReader.EnqueuedCount = 0
+	oldReader.PendingFailureCount = 0
+	if !reflect.DeepEqual(oldReader, legacy) {
+		t.Fatalf("old reader lost fields: %+v", oldReader)
+	}
+}
+
+func TestLegacyFailureAndTaskJSON(t *testing.T) {
+	legacy := zset.FailureRecord{
+		Version: 1, TaskID: "source:op:42", OriginalRunID: "source", LatestRunID: "latest",
+		QueryMonth: "202609", Operation: "op", CardID: 42, ICCID: "iccid", MSISDN: "msisdn", IMSI: "imsi",
+		ChanKey: "chan", TempKey: "template", Carrier: "mobile", FailureType: "mobile_busy_exhausted",
+		Round: 5, Attempts: 7, Code: 13000, ErrCode: "BUSY", Message: "try again",
+		ReviewStatus: "pending", ReviewRemark: "remark", ReviewedBy: "operator", ReviewedAt: "2026-09-20",
+		RetryCycle: 2, ReplayRunID: "replayed", FirstFailedAt: time.Unix(1, 0).UTC(), LastFailedAt: time.Unix(2, 0).UTC(),
+	}
+	var record contract.FailureRecord
+	if err := json.Unmarshal(mustJSON(t, legacy), &record); err != nil {
+		t.Fatal(err)
+	}
+	if string(mustJSON(t, record)) != string(mustJSON(t, legacy)) {
+		t.Fatal("failure record wire format changed")
+	}
+	if summary := record.Summary(); summary.ErrorCategory != contract.ErrorRateLimited || summary.TaskID != legacy.TaskID || summary.FailedAt != legacy.LastFailedAt {
+		t.Fatalf("incorrect failure projection: %+v", summary)
+	}
+	task := zset.Task{TaskID: "r:op:42", RunID: "r", Operation: "op", QueryMonth: "202609", ChanKey: "c", TempKey: "t", Operator: 2, CardID: 42, ICCID: "i", MSISDN: "m", IMSI: "s", NextExecuteAt: 1234}
+	var newTask contract.Task
+	if err := json.Unmarshal(mustJSON(t, task), &newTask); err != nil {
+		t.Fatal(err)
+	}
+	newTask.NextExecuteAt = 9999
+	if string(mustJSON(t, newTask)) != string(mustJSON(t, task)) {
+		t.Fatal("task wire format or scheduling exclusion changed")
+	}
+}
+
+func mustJSON(t *testing.T, value any) []byte {
+	t.Helper()
+	data, err := json.Marshal(value)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}

internal/fullsync/contract/doc.go

@@ -0,0 +1,14 @@
+// Package contract defines the shared full-sync domain and service boundaries.
+// It contains no Redis, filesystem, replay execution, or HTTP handler code.
+//
+// Legacy queue models remain in internal/queue/zset until adapters adopt this
+// package. Their existing JSON fields are retained here; new run fields are
+// optional so historical records still decode. Missing extended counters mean
+// unknown, not a reconstructed total. FailedCount is the historical terminal
+// count for a run; cleaning source failures changes PendingFailureCount only.
+//
+// Implementations must preserve these boundaries: run-local Redis keys share
+// one hash tag, global indexes do not; archive summaries never contain card
+// details; cursors are opaque and scoped to the original query; replay writes
+// are idempotent and fenced by a lease before source cleanup is allowed.
+package contract

internal/fullsync/contract/errors.go

@@ -0,0 +1,112 @@
+package contract
+
+import (
+	"encoding/json"
+	"math"
+	"strings"
+	"unicode/utf8"
+)
+
+type ErrorCategory string
+
+const (
+	ErrorTimeout         ErrorCategory = "timeout"
+	ErrorRateLimited     ErrorCategory = "rate_limited"
+	ErrorAuthentication  ErrorCategory = "authentication"
+	ErrorValidation      ErrorCategory = "validation"
+	ErrorUpstream        ErrorCategory = "upstream"
+	ErrorTransport       ErrorCategory = "transport"
+	ErrorInternal        ErrorCategory = "internal"
+	ErrorOther           ErrorCategory = "other"
+	MaxErrorCategories                 = 8
+	MaxErrorMessageBytes               = 2048
+)
+
+// NormalizeErrorCategory maps the existing consumer classifications into the
+// approved, bounded vocabulary. Never use an upstream message/code as a key.
+func NormalizeErrorCategory(value string) ErrorCategory {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "timeout":
+		return ErrorTimeout
+	case "rate_limited", "mobile_busy", "mobile_busy_exhausted", "channel_limited", "channel_blocked":
+		return ErrorRateLimited
+	case "authentication":
+		return ErrorAuthentication
+	case "validation":
+		return ErrorValidation
+	case "upstream", "business", "carrier_retry":
+		return ErrorUpstream
+	case "transport", "network":
+		return ErrorTransport
+	case "internal", "canceled":
+		return ErrorInternal
+	default:
+		return ErrorOther
+	}
+}
+
+func (c ErrorCategory) MarshalJSON() ([]byte, error) {
+	return json.Marshal(string(NormalizeErrorCategory(string(c))))
+}
+
+func (c *ErrorCategory) UnmarshalJSON(data []byte) error {
+	var value string
+	if err := json.Unmarshal(data, &value); err != nil {
+		return invalid("error category must be a string: %v", err)
+	}
+	*c = NormalizeErrorCategory(value)
+	return nil
+}
+
+// ErrorCounts is normalized on both JSON input and output, including map keys
+// (encoding/json does not invoke ErrorCategory.MarshalJSON for map keys).
+// Counts are per category, not card IDs or individual error messages.
+type ErrorCounts map[ErrorCategory]int64
+
+func (counts ErrorCounts) Normalized() (ErrorCounts, error) {
+	normalized := make(ErrorCounts)
+	for category, count := range counts {
+		key := NormalizeErrorCategory(string(category))
+		if count < 0 || count > math.MaxInt64-normalized[key] {
+			return nil, invalid("error counts must be nonnegative and must not overflow")
+		}
+		normalized[key] += count
+	}
+	return normalized, nil
+}
+
+func (counts ErrorCounts) MarshalJSON() ([]byte, error) {
+	normalized, err := counts.Normalized()
+	if err != nil {
+		return nil, err
+	}
+	type wireCounts map[ErrorCategory]int64
+	return json.Marshal(wireCounts(normalized))
+}
+
+func (counts *ErrorCounts) UnmarshalJSON(data []byte) error {
+	var decoded map[ErrorCategory]int64
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return invalid("invalid error counts: %v", err)
+	}
+	normalized, err := ErrorCounts(decoded).Normalized()
+	if err != nil {
+		return err
+	}
+	*counts = normalized
+	return nil
+}
+
+// BoundedErrorMessage makes new summary/action error messages valid UTF-8 and
+// caps their byte length. It does not modify legacy FailureRecord.Message.
+func BoundedErrorMessage(message string) string {
+	message = strings.ToValidUTF8(message, "\uFFFD")
+	if len(message) <= MaxErrorMessageBytes {
+		return message
+	}
+	end := MaxErrorMessageBytes
+	for !utf8.RuneStart(message[end]) {
+		end--
+	}
+	return message[:end]
+}

internal/fullsync/contract/errors_test.go

@@ -0,0 +1,91 @@
+package contract_test
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestBoundedErrorCategories(t *testing.T) {
+	for raw, want := range map[string]contract.ErrorCategory{
+		"timeout": contract.ErrorTimeout, " RATE_LIMITED ": contract.ErrorRateLimited,
+		"authentication": contract.ErrorAuthentication, "validation": contract.ErrorValidation,
+		"upstream": contract.ErrorUpstream, "transport": contract.ErrorTransport,
+		"internal": contract.ErrorInternal, "other": contract.ErrorOther,
+		"network": contract.ErrorTransport, "mobile_busy": contract.ErrorRateLimited,
+		"mobile_busy_exhausted": contract.ErrorRateLimited, "channel_blocked": contract.ErrorRateLimited,
+		"channel_limited": contract.ErrorRateLimited, "business": contract.ErrorUpstream,
+		"carrier_retry": contract.ErrorUpstream, "canceled": contract.ErrorInternal,
+		"": contract.ErrorOther, "card-898600-unique-error": contract.ErrorOther,
+	} {
+		if got := contract.NormalizeErrorCategory(raw); got != want {
+			t.Errorf("normalize %q = %q, want %q", raw, got, want)
+		}
+		var decoded contract.ErrorCategory
+		if err := json.Unmarshal(mustJSON(t, raw), &decoded); err != nil || decoded != want {
+			t.Errorf("decode %q = %q, err %v", raw, decoded, err)
+		}
+		if got := string(mustJSON(t, contract.ErrorCategory(raw))); got != string(mustJSON(t, string(want))) {
+			t.Errorf("encode %q = %s", raw, got)
+		}
+	}
+}
+
+func TestErrorCountsCannotGrowWithCardinality(t *testing.T) {
+	counts := contract.ErrorCounts{contract.ErrorTimeout: 2, contract.ErrorOther: 3, "network": 4}
+	for i := 0; i < 1000; i++ {
+		counts[contract.ErrorCategory(fmt.Sprintf("card-error-%d", i))] = 1
+	}
+	var decoded contract.ErrorCounts
+	if err := json.Unmarshal(mustJSON(t, counts), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded) != 3 || len(decoded) > contract.MaxErrorCategories || decoded[contract.ErrorOther] != 1003 || decoded[contract.ErrorTransport] != 4 {
+		t.Fatalf("unbounded or lost counts: %+v", decoded)
+	}
+	if err := json.Unmarshal([]byte(`{"unknown":2,"different":3,"other":4,"network":5}`), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded) != 2 || decoded[contract.ErrorOther] != 9 || decoded[contract.ErrorTransport] != 5 {
+		t.Fatalf("input normalization failed: %+v", decoded)
+	}
+}
+
+func TestErrorCountsRejectInvalidNumbers(t *testing.T) {
+	for _, raw := range []string{`{"timeout":-1}`, `{"one":9223372036854775807,"two":1}`, `{"other":1.5}`, `{"other":"1"}`} {
+		var counts contract.ErrorCounts
+		if err := json.Unmarshal([]byte(raw), &counts); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("invalid input %s returned %v", raw, err)
+		}
+	}
+	for _, counts := range []contract.ErrorCounts{{"timeout": -1}, {"one": math.MaxInt64, "two": 1}} {
+		if _, err := json.Marshal(counts); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("invalid output counts returned %v", err)
+		}
+	}
+}
+
+func TestBoundedFailureMessage(t *testing.T) {
+	for _, message := range []string{"short", strings.Repeat("x", 3000), strings.Repeat("卡", 1000), strings.Repeat("x", 2047) + "卡", "invalid\xfftext"} {
+		got := contract.BoundedErrorMessage(message)
+		if len(got) > contract.MaxErrorMessageBytes || !utf8.ValidString(got) {
+			t.Fatalf("invalid bounded output: bytes=%d", len(got))
+		}
+		if utf8.ValidString(message) && len(message) <= contract.MaxErrorMessageBytes && got != message {
+			t.Fatal("short message changed")
+		}
+	}
+	record := contract.FailureRecord{TaskID: "task", Message: strings.Repeat("卡", 1000)}
+	if err := record.Summary().Validate(); err != nil {
+		t.Fatal(err)
+	}
+	if len(record.Message) != 3000 {
+		t.Fatal("summary mutated legacy failure message")
+	}
+}

internal/fullsync/contract/failure.go

@@ -0,0 +1,129 @@
+package contract
+
+import (
+	"time"
+	"unicode/utf8"
+)
+
+// Cursor is an opaque repository-issued continuation token. Clients must not
+// parse, increment, or reuse it for another run/filter. Empty starts a scan;
+// an empty nextCursor ends it. An empty page with a cursor is not end-of-scan.
+// Repositories validate token contents and query binding with ErrInvalidArgument.
+type Cursor string
+
+const (
+	DefaultPageLimit = 100
+	MaxPageLimit     = 500
+)
+
+type PageRequest struct {
+	Cursor Cursor `json:"cursor,omitempty"`
+	Limit  int    `json:"limit,omitempty"`
+}
+
+// Normalized must be called at both HTTP and repository boundaries. Limits
+// outside the bounds are rejected, not silently truncated.
+func (p PageRequest) Normalized() (PageRequest, error) {
+	if p.Limit == 0 {
+		p.Limit = DefaultPageLimit
+	}
+	if p.Limit < 1 || p.Limit > MaxPageLimit {
+		return PageRequest{}, invalid("limit must be between 1 and %d", MaxPageLimit)
+	}
+	return p, nil
+}
+
+// Failure is the bounded public projection, not the replay payload. Legacy
+// free-form failureType/message fields remain available in FailureRecord.
+type Failure struct {
+	TaskID        string        `json:"taskId"`
+	CardID        int64         `json:"cardId"`
+	ErrorCategory ErrorCategory `json:"errorCategory"`
+	ErrorMessage  string        `json:"errorMessage"`
+	FailedAt      time.Time     `json:"failedAt"`
+	Round         int           `json:"round"`
+}
+
+func (f Failure) Validate() error {
+	if f.TaskID == "" || f.Round < 0 || len(f.ErrorMessage) > MaxErrorMessageBytes || !utf8.ValidString(f.ErrorMessage) {
+		return invalid("invalid failure summary")
+	}
+	return nil
+}
+
+type FailurePage struct {
+	Items      []Failure `json:"items"`
+	NextCursor Cursor    `json:"nextCursor"`
+}
+
+// FailureRecord preserves the existing persisted payload without changing
+// review semantics or discarding fields needed to reconstruct replay tasks.
+type FailureRecord struct {
+	Version       int    `json:"version"`
+	TaskID        string `json:"taskId"`
+	OriginalRunID string `json:"originalRunId"`
+	LatestRunID   string `json:"latestRunId"`
+	QueryMonth    string `json:"queryMonth"`
+	Operation     string `json:"operation"`
+	CardID        int64  `json:"cardId"`
+	ICCID         string `json:"iccid"`
+	MSISDN        string `json:"msisdn"`
+	IMSI          string `json:"imsi"`
+	ChanKey       string `json:"chanKey"`
+	TempKey       string `json:"tempKey"`
+	// PackageCode is additive; older failure records decode it as empty.
+	PackageCode   string    `json:"packageCode,omitempty"`
+	Carrier       string    `json:"carrier"`
+	FailureType   string    `json:"failureType"`
+	Round         int       `json:"round"`
+	Attempts      int       `json:"attempts"`
+	Code          int       `json:"code"`
+	ErrCode       string    `json:"errCode"`
+	Message       string    `json:"message"`
+	ReviewStatus  string    `json:"reviewStatus"`
+	ReviewRemark  string    `json:"reviewRemark"`
+	ReviewedBy    string    `json:"reviewedBy"`
+	ReviewedAt    string    `json:"reviewedAt"`
+	RetryCycle    int       `json:"retryCycle"`
+	ReplayRunID   string    `json:"replayRunId"`
+	FirstFailedAt time.Time `json:"firstFailedAt"`
+	LastFailedAt  time.Time `json:"lastFailedAt"`
+}
+
+func (f FailureRecord) Summary() Failure {
+	return Failure{
+		TaskID: f.TaskID, CardID: f.CardID, ErrorCategory: NormalizeErrorCategory(f.FailureType),
+		ErrorMessage: BoundedErrorMessage(f.Message), FailedAt: f.LastFailedAt, Round: f.Round,
+	}
+}
+
+// FailureEntry carries an opaque version for conditional source cleanup. It
+// must identify the precise record read, not just its reusable task ID.
+type FailureEntry struct {
+	Record  FailureRecord `json:"record"`
+	Version string        `json:"-"`
+}
+
+type FailureRecordPage struct {
+	Items      []FailureEntry `json:"items"`
+	NextCursor Cursor         `json:"nextCursor"`
+}
+
+// Task keeps the existing queue member serialization. The scheduling score
+// must never be part of the JSON member or producer retries lose idempotency.
+type Task struct {
+	TaskID     string `json:"taskId"`
+	RunID      string `json:"runId"`
+	Operation  string `json:"operation"`
+	QueryMonth string `json:"queryMonth"`
+	ChanKey    string `json:"chanKey"`
+	TempKey    string `json:"tempKey"`
+	Operator   int    `json:"operator,omitempty"`
+	CardID     int64  `json:"cardId"`
+	ICCID      string `json:"iccid"`
+	MSISDN     string `json:"msisdn"`
+	IMSI       string `json:"imsi"`
+	// PackageCode is additive; older members decode it as empty.
+	PackageCode   string `json:"packageCode,omitempty"`
+	NextExecuteAt int64  `json:"-"`
+}

internal/fullsync/contract/http.go

@@ -0,0 +1,28 @@
+package contract
+
+import "net/url"
+
+const (
+	ListRunsRoute     = "/internal/v1/full-sync-runs"
+	GetRunRoute       = "/internal/v1/full-sync-runs/{runId}"
+	GetFailuresRoute  = "/internal/v1/full-sync-runs/{runId}/failures"
+	PostReplayRoute   = "/internal/v1/full-sync-runs/{runId}/replay"
+	GetActionRoute    = "/internal/v1/full-sync-actions/{actionId}"
+	IdempotencyHeader = "Idempotency-Key"
+)
+
+// ReplayAccepted is the 202 response from a successful ReplayRequester call.
+// A repeated request returns the original action and its current status.
+type ReplayAccepted struct {
+	ActionID    string       `json:"actionId"`
+	TargetRunID string       `json:"targetRunId"`
+	Status      ActionStatus `json:"status"`
+	StatusURL   string       `json:"statusUrl"`
+}
+
+func ActionStatusURL(actionID string) (string, error) {
+	if err := ValidateID(actionID); err != nil {
+		return "", err
+	}
+	return "/internal/v1/full-sync-actions/" + url.PathEscape(actionID), nil
+}

internal/fullsync/contract/interfaces.go

@@ -0,0 +1,145 @@
+package contract
+
+import (
+	"context"
+	"time"
+)
+
+// RunReader is the HTTP lookup boundary. Fall back to archive only when the
+// live reader returns ErrNotFound, not on transient live storage failures.
+type RunReader interface {
+	GetRun(ctx context.Context, runID string) (RunView, error)
+}
+
+type RunListFilter struct {
+	SyncKind SyncKind
+	Limit    int
+}
+
+type RunList struct {
+	Items []RunView `json:"items"`
+	Total int       `json:"total"`
+}
+
+type RunLister interface {
+	ListRuns(ctx context.Context, filter RunListFilter) (RunList, error)
+}
+
+type LiveRunReader interface {
+	GetRunMeta(ctx context.Context, runID string) (RunMeta, error)
+}
+
+type ArchivedRunReader interface {
+	GetArchivedRun(ctx context.Context, runID string) (RunArchive, error)
+}
+
+type RoundStatsReader interface {
+	GetRoundStats(ctx context.Context, runID string) ([]RoundStats, error)
+}
+
+// FailurePager returns live details only. An archived summary without live
+// details returns ErrFailureDetailsUnavailable, never a misleading empty page.
+// Items must serialize as [] on empty pages; NextCursor alone signals EOF.
+type FailurePager interface {
+	ListFailures(ctx context.Context, runID string, page PageRequest) (FailurePage, error)
+}
+
+type ReplayRequester interface {
+	RequestReplay(ctx context.Context, request ReplayRequest) (ReplayAccepted, error)
+}
+
+type ActionReader interface {
+	GetReplayAction(ctx context.Context, actionID string) (ReplayAction, error)
+}
+
+// ArchiveWorkDiscovery returns pending/retryable work from a durable index.
+// Terminal run registration must not depend on an in-memory notification.
+type ArchiveWorkDiscovery interface {
+	ListArchiveWork(ctx context.Context, page PageRequest) (ArchiveWorkPage, error)
+}
+
+// RegisterArchiveWork idempotently schedules a terminal run/version. Since the
+// index can occupy another Redis slot, callers must reconcile registration
+// after a crash between completing the run and updating the index.
+type ArchiveWorkRegistrar interface {
+	RegisterArchiveWork(ctx context.Context, work ArchiveWork) error
+}
+
+type ArchiveWorkLoader interface {
+	LoadArchiveWork(ctx context.Context, work ArchiveWork) (RunArchive, error)
+}
+
+// ArchiveRegistrar conditionally records a durable publication against the
+// work version. Repeating the same receipt is a no-op; a different version or
+// conflicting receipt returns ErrConflict. Failure retains discoverable work.
+type ArchiveRegistrar interface {
+	RegisterArchive(ctx context.Context, work ArchiveWork, receipt ArchiveReceipt) error
+	RegisterArchiveFailure(ctx context.Context, work ArchiveWork, message string) error
+}
+
+// ReplayActionCreator atomically reserves (sourceRunId, idempotencyKey) with
+// its fingerprint and action. Same key/request returns the original action
+// regardless of candidate IDs; different request returns ErrConflict. The
+// candidate starts accepted and fixes the target ID before any target writes.
+type ReplayActionCreator interface {
+	GetOrCreateReplayAction(ctx context.Context, request ReplayRequest, candidate ReplayAction) (ReplayActionCreation, error)
+}
+
+// Discovery includes accepted, processing, cleanup_pending and receipt_pending
+// actions after crashes. Completed/needs_attention actions require no automatic
+// work. Lease acquisition, not listing, chooses an exclusive owner.
+type ReplayWorkDiscovery interface {
+	ListReplayWork(ctx context.Context, page PageRequest) (ReplayActionPage, error)
+}
+
+type ReplayLeaseStore interface {
+	AcquireReplayLease(ctx context.Context, actionID, owner string, ttl time.Duration) (ReplayLease, error)
+	RenewReplayLease(ctx context.Context, lease ReplayLease, ttl time.Duration) (ReplayLease, error)
+	ReleaseReplayLease(ctx context.Context, lease ReplayLease) error
+}
+
+// All updates are fenced and compare ExpectedRevision. Expired/replaced leases
+// return ErrLeaseLost; revision conflicts return ErrConflict. Implementations
+// reject progress regression and preserve immutable action identity.
+type ReplayProgressStore interface {
+	UpdateReplayAction(ctx context.Context, update ReplayActionUpdate) (ReplayAction, error)
+}
+
+// The implementation must preserve scan continuity while source entries are
+// cleaned. This can use a durable selection/snapshot; deleting records must not
+// make a later cursor skip an unprocessed failure. Tokens are bound to actionID.
+type ReplayFailurePager interface {
+	ListReplayFailures(ctx context.Context, actionID string, page PageRequest) (FailureRecordPage, error)
+}
+
+// TargetRunCreator is idempotent: retrying the same target preserves counters;
+// incompatible source/operation/month/cycle returns ErrConflict. Replay targets
+// are distinct runs and must not mutate the source's historical FailedCount.
+type TargetRunCreator interface {
+	EnsureTargetRun(ctx context.Context, meta RunMeta) (RunMeta, error)
+}
+
+// TargetEnqueueResult distinguishes a durable duplicate from a rejected write.
+// A zero NewlyEnqueued count alone is never sufficient evidence for cleanup.
+// Every input TaskID must be durably present on nil error; otherwise return an
+// error and let the caller safely retry the identical batch. Both counts are
+// unique task counts, and their sum equals the number of distinct input IDs.
+type TargetEnqueueResult struct {
+	NewlyEnqueued  int64
+	AlreadyPresent int64
+}
+
+type TargetRunEnqueuer interface {
+	// Tasks must match the target run and channel and use stable logical IDs.
+	EnqueueTargetBatch(ctx context.Context, lease ReplayLease, runID, chanKey string, tasks []Task, nextExecuteAt time.Time) (TargetEnqueueResult, error)
+	MarkTargetProducerDone(ctx context.Context, lease ReplayLease, runID string) error
+}
+
+// Cleanup is idempotent and conditional on the lease, exact source versions,
+// and durable evidence of target acceptance. It changes pending failures only,
+// never historical failedCount. Conflicts must remain visible for attention.
+// Source keys and action/target keys may occupy different Redis Cluster slots;
+// implementations need recoverable receipts, not cross-slot Lua assumptions.
+type ReplayFailureCleaner interface {
+	CleanupReplayFailures(ctx context.Context, cleanup ReplayCleanup) (ReplayCleanupResult, error)
+}

internal/fullsync/contract/keys.go

@@ -0,0 +1,116 @@
+package contract
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const (
+	DefaultFlowPrefix      = "flow:"
+	DefaultSimStatusPrefix = "sim-status:"
+)
+
+// Keys is a naming contract only. It does not create indexes or storage.
+// Existing run names are byte-for-byte compatible with zset.Store. Global
+// run/archive indexes are outside run hash slots and cannot join run-local Lua.
+type Keys struct{ prefix string }
+
+func NewKeys(prefix string) (Keys, error) {
+	if strings.TrimSpace(prefix) == "" {
+		prefix = DefaultFlowPrefix
+	}
+	if strings.ContainsAny(prefix, "{}") {
+		return Keys{}, invalid("prefix must not contain Redis hash tags")
+	}
+	return Keys{prefix: prefix}, nil
+}
+
+// The zero value uses the legacy flow prefix, just like NewKeys("").
+func (k Keys) Prefix() string {
+	if k.prefix == "" {
+		return DefaultFlowPrefix
+	}
+	return k.prefix
+}
+
+func (k Keys) runKey(runID, suffix string) (string, error) {
+	if err := ValidateID(runID); err != nil {
+		return "", err
+	}
+	return k.Prefix() + "{" + runID + "}:" + suffix, nil
+}
+
+func (k Keys) RunMetaKey(runID string) (string, error)      { return k.runKey(runID, "meta") }
+func (k Keys) RunChannelsKey(runID string) (string, error)  { return k.runKey(runID, "channels") }
+func (k Keys) TaskStateKey(runID string) (string, error)    { return k.runKey(runID, "task-state") }
+func (k Keys) FailedKey(runID string) (string, error)       { return k.runKey(runID, "failed") }
+func (k Keys) ArchiveStateKey(runID string) (string, error) { return k.runKey(runID, "archive") }
+
+func (k Keys) roundKey(runID string, round int, suffix string) (string, error) {
+	if round < 1 {
+		return "", invalid("round must be positive")
+	}
+	return k.runKey(runID, fmt.Sprintf("round:%d:%s", round, suffix))
+}
+
+func (k Keys) RoundStatsKey(runID string, round int) (string, error) {
+	return k.roundKey(runID, round, "stats")
+}
+
+func (k Keys) roundChannelKey(runID string, round int, kind, chanKey string) (string, error) {
+	if strings.TrimSpace(chanKey) == "" {
+		return "", invalid("chanKey is empty")
+	}
+	return k.roundKey(runID, round, kind+":"+chanKey)
+}
+
+func (k Keys) RoundActiveKey(runID string, round int, chanKey string) (string, error) {
+	return k.roundChannelKey(runID, round, "active", chanKey)
+}
+
+func (k Keys) RoundWaitingKey(runID string, round int, chanKey string) (string, error) {
+	return k.roundChannelKey(runID, round, "waiting", chanKey)
+}
+
+func (k Keys) RoundLeaseKey(runID string, round int, chanKey string) (string, error) {
+	return k.roundChannelKey(runID, round, "lease", chanKey)
+}
+
+func (k Keys) FailedRunsKey() string      { return k.Prefix() + "failed:runs" }
+func (k Keys) ActiveRunsKey() string      { return k.Prefix() + "runs:active" }
+func (k Keys) AllRunsKey() string         { return k.Prefix() + "runs:all" }
+func (k Keys) PendingArchivesKey() string { return k.Prefix() + "archive:pending" }
+
+// Replay action state, leases, idempotency reservations and pending index share
+// a dedicated slot, allowing atomic acceptance/discovery. Source/target run
+// writes remain in their own slots and require a recoverable replay protocol.
+func (k Keys) replayKey(actionID, suffix string) (string, error) {
+	if err := ValidateID(actionID); err != nil {
+		return "", err
+	}
+	return k.Prefix() + "{replay}:action:" + actionID + ":" + suffix, nil
+}
+
+func (k Keys) ReplayActionKey(actionID string) (string, error) { return k.replayKey(actionID, "meta") }
+func (k Keys) ReplayLeaseKey(actionID string) (string, error)  { return k.replayKey(actionID, "lease") }
+func (k Keys) PendingReplayActionsKey() string                 { return k.Prefix() + "{replay}:pending" }
+
+// Hash the complete tuple so neither delimiter collisions nor arbitrary
+// Idempotency-Key characters can change the Redis key or hash slot.
+func (k Keys) ReplayIdempotencyKey(sourceRunID, idempotencyKey string) (string, error) {
+	if err := ValidateID(sourceRunID); err != nil {
+		return "", err
+	}
+	if err := ValidateIdempotencyKey(idempotencyKey); err != nil {
+		return "", err
+	}
+	data, err := json.Marshal([2]string{sourceRunID, idempotencyKey})
+	if err != nil {
+		return "", invalid("cannot encode idempotency scope: %v", err)
+	}
+	digest := sha256.Sum256(data)
+	return k.Prefix() + "{replay}:idempotency:" + hex.EncodeToString(digest[:]), nil
+}

internal/fullsync/contract/keys_test.go

@@ -0,0 +1,116 @@
+package contract_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+
+	"github.com/gogf/gf/v2/container/gvar"
+)
+
+type unusedRedis struct{}
+
+func (unusedRedis) Do(context.Context, string, ...any) (*gvar.Var, error) {
+	panic("naming contracts must not access Redis")
+}
+
+func (unusedRedis) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	panic("naming contracts must not access Redis")
+}
+
+func TestKeysMatchExistingQueue(t *testing.T) {
+	for _, prefix := range []string{"", "flow:", "sim-status:", "custom-prefix"} {
+		keys, err := contract.NewKeys(prefix)
+		if err != nil {
+			t.Fatal(err)
+		}
+		store, err := zset.New(unusedRedis{}, prefix)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, pair := range []struct {
+			old, next func(string) (string, error)
+		}{
+			{store.RunMetaKey, keys.RunMetaKey}, {store.RunChannelsKey, keys.RunChannelsKey},
+			{store.TaskStateKey, keys.TaskStateKey}, {store.FailedKey, keys.FailedKey},
+		} {
+			old, oldErr := pair.old("run:1")
+			next, nextErr := pair.next("run:1")
+			if oldErr != nil || nextErr != nil || old != next || !strings.Contains(next, "{run:1}") {
+				t.Errorf("run naming changed: old=%q new=%q errors=%v/%v", old, next, oldErr, nextErr)
+			}
+		}
+		for _, pair := range []struct {
+			old, next func(string, int, string) (string, error)
+		}{
+			{store.RoundActiveKey, keys.RoundActiveKey}, {store.RoundWaitingKey, keys.RoundWaitingKey},
+			{store.RoundLeaseKey, keys.RoundLeaseKey},
+		} {
+			old, oldErr := pair.old("run:1", 3, "channel")
+			next, nextErr := pair.next("run:1", 3, "channel")
+			if oldErr != nil || nextErr != nil || old != next {
+				t.Errorf("round naming changed: old=%q new=%q errors=%v/%v", old, next, oldErr, nextErr)
+			}
+		}
+		if keys.ActiveRunsKey() != store.ActiveRunsKey() || keys.FailedRunsKey() != store.FailedRunsKey() {
+			t.Fatal("existing global index names changed")
+		}
+	}
+}
+
+func TestNewKeysAndClusterBoundaries(t *testing.T) {
+	keys, _ := contract.NewKeys("flow:")
+	stats, _ := keys.RoundStatsKey("run", 2)
+	archive, _ := keys.ArchiveStateKey("run")
+	if stats != "flow:{run}:round:2:stats" || archive != "flow:{run}:archive" {
+		t.Fatalf("new run keys changed: %s, %s", stats, archive)
+	}
+	action, _ := keys.ReplayActionKey("a")
+	lease, _ := keys.ReplayLeaseKey("a")
+	idempotency, _ := keys.ReplayIdempotencyKey("source", "a{different-slot}:key")
+	for _, key := range []string{action, lease, idempotency, keys.PendingReplayActionsKey()} {
+		if !strings.HasPrefix(key, "flow:{replay}:") || strings.Count(key, "{") != 1 {
+			t.Fatalf("replay key escaped action slot: %s", key)
+		}
+	}
+	if action != "flow:{replay}:action:a:meta" || lease != "flow:{replay}:action:a:lease" {
+		t.Fatal("action naming changed")
+	}
+	for _, key := range []string{keys.ActiveRunsKey(), keys.FailedRunsKey(), keys.AllRunsKey(), keys.PendingArchivesKey()} {
+		if strings.ContainsAny(key, "{}") {
+			t.Fatalf("global index unexpectedly shares run slot: %s", key)
+		}
+	}
+	first, _ := keys.ReplayIdempotencyKey("a:b", "c")
+	second, _ := keys.ReplayIdempotencyKey("a", "b:c")
+	repeat, _ := keys.ReplayIdempotencyKey("a:b", "c")
+	otherSource, _ := keys.ReplayIdempotencyKey("other", "c")
+	if first == second || first == otherSource || first != repeat {
+		t.Fatal("idempotency scope is ambiguous or unstable")
+	}
+}
+
+func TestInvalidKeyParts(t *testing.T) {
+	keys := contract.Keys{}
+	for _, id := range []string{"", " ", "trim ", "{run}", "run}", "run/path", "run\\path", "run\n", "invalid\xff"} {
+		if _, err := keys.RunMetaKey(id); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("invalid ID %q returned %v", id, err)
+		}
+	}
+	if _, err := contract.NewKeys("prefix:{wrong}:"); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("invalid prefix returned %v", err)
+	}
+	if _, err := keys.RoundStatsKey("run", 0); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("invalid round returned %v", err)
+	}
+	if _, err := keys.RoundActiveKey("run", 1, " "); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("invalid channel returned %v", err)
+	}
+	if key, err := keys.RunMetaKey("run"); err != nil || key != "flow:{run}:meta" {
+		t.Fatalf("zero-value keys differ from legacy default: %s %v", key, err)
+	}
+}

internal/fullsync/contract/replay.go

@@ -0,0 +1,172 @@
+package contract
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+const MaxIdempotencyKeyBytes = 256
+
+// ReplayRequest combines the path, header and body. Only requestedBy is in the
+// HTTP body. Idempotency keys are scoped to the source run; fingerprints cover
+// the source and exact request body, not the key or a generated target run ID.
+type ReplayRequest struct {
+	SourceRunID    string `json:"-"`
+	IdempotencyKey string `json:"-"`
+	RequestedBy    string `json:"requestedBy"`
+}
+
+func ValidateIdempotencyKey(key string) error {
+	if strings.TrimSpace(key) == "" || len(key) > MaxIdempotencyKeyBytes || !utf8.ValidString(key) || strings.ContainsFunc(key, unicode.IsControl) {
+		return invalid("invalid Idempotency-Key")
+	}
+	return nil
+}
+
+func (r ReplayRequest) Validate() error {
+	if err := ValidateID(r.SourceRunID); err != nil {
+		return err
+	}
+	if err := ValidateIdempotencyKey(r.IdempotencyKey); err != nil {
+		return err
+	}
+	if strings.TrimSpace(r.RequestedBy) == "" || !utf8.ValidString(r.RequestedBy) {
+		return invalid("requestedBy is required")
+	}
+	return nil
+}
+
+func (r ReplayRequest) Fingerprint() (string, error) {
+	if err := r.Validate(); err != nil {
+		return "", err
+	}
+	// JSON avoids ambiguous delimiter concatenation and fixes field ordering.
+	data, err := json.Marshal(struct {
+		SourceRunID string `json:"sourceRunId"`
+		RequestedBy string `json:"requestedBy"`
+	}{r.SourceRunID, r.RequestedBy})
+	if err != nil {
+		return "", invalid("cannot encode replay request: %v", err)
+	}
+	digest := sha256.Sum256(data)
+	return hex.EncodeToString(digest[:]), nil
+}
+
+// Counters describe distinct source failures, not scan visits or retry calls.
+// ScanCursor alone is not an enqueue/cleanup checkpoint: repositories must
+// retain sufficient durable per-batch evidence to resume partial writes.
+type ReplayProgress struct {
+	ScanCursor      Cursor `json:"scanCursor"`
+	DiscoveredCount int64  `json:"discoveredCount"`
+	EnqueuedCount   int64  `json:"enqueuedCount"`
+	CleanedCount    int64  `json:"cleanedCount"`
+	FailureCount    int64  `json:"failureCount"`
+}
+
+func (p ReplayProgress) Validate() error {
+	if err := nonnegative(p.DiscoveredCount, p.EnqueuedCount, p.CleanedCount, p.FailureCount); err != nil {
+		return err
+	}
+	if p.EnqueuedCount > p.DiscoveredCount || p.CleanedCount > p.EnqueuedCount {
+		return invalid("replay progress must satisfy cleaned <= enqueued <= discovered")
+	}
+	return nil
+}
+
+type ReplayAction struct {
+	ActionID       string          `json:"actionId"`
+	SourceRunID    string          `json:"sourceRunId"`
+	TargetRunID    string          `json:"targetRunId"`
+	RequestedBy    string          `json:"requestedBy"`
+	IdempotencyKey string          `json:"-"`
+	Fingerprint    string          `json:"-"`
+	Status         ActionStatus    `json:"status"`
+	ReplayCycle    int             `json:"replayCycle"`
+	Progress       ReplayProgress  `json:"progress"`
+	LeaseOwner     string          `json:"-"`
+	LeaseToken     string          `json:"-"`
+	LeaseUntil     time.Time       `json:"-"`
+	Revision       uint64          `json:"-"`
+	Error          string          `json:"error,omitempty"`
+	CreatedAt      time.Time       `json:"createdAt"`
+	UpdatedAt      time.Time       `json:"updatedAt"`
+	CompletedAt    *time.Time      `json:"completedAt,omitempty"`
+	Receipt        *ArchiveReceipt `json:"receipt,omitempty"`
+}
+
+func (a ReplayAction) Validate() error {
+	for _, id := range []string{a.ActionID, a.SourceRunID, a.TargetRunID} {
+		if err := ValidateID(id); err != nil {
+			return err
+		}
+	}
+	if a.SourceRunID == a.TargetRunID || a.ReplayCycle < 1 || !a.Status.Valid() || strings.TrimSpace(a.RequestedBy) == "" {
+		return invalid("invalid replay action identity or status")
+	}
+	if len(a.Error) > MaxErrorMessageBytes || !utf8.ValidString(a.Error) {
+		return invalid("action error exceeds message bounds")
+	}
+	if err := a.Progress.Validate(); err != nil {
+		return err
+	}
+	if a.Receipt != nil {
+		if err := a.Receipt.Validate(); err != nil {
+			return err
+		}
+	}
+	if a.Status == ActionCompleted && (a.Receipt == nil || a.CompletedAt == nil || a.CompletedAt.IsZero()) {
+		return invalid("completed action requires a durable receipt and completedAt")
+	}
+	return nil
+}
+
+type ReplayActionPage struct {
+	Items      []ReplayAction `json:"items"`
+	NextCursor Cursor         `json:"nextCursor"`
+}
+
+type ReplayActionCreation struct {
+	Action  ReplayAction
+	Created bool
+}
+
+// Lease token fences a particular acquisition, even if the same owner name is
+// reused. Repository methods check expiry against their authoritative clock.
+type ReplayLease struct {
+	ActionID string
+	Owner    string
+	Token    string
+	Until    time.Time
+}
+
+// ReplayActionUpdate is a compare-and-swap of mutable action state. Identity,
+// provenance, request fingerprint, and target run must never be changed.
+type ReplayActionUpdate struct {
+	Lease            ReplayLease
+	ExpectedRevision uint64
+	Status           ActionStatus
+	Progress         ReplayProgress
+	Error            string
+	Receipt          *ArchiveReceipt
+	CompletedAt      *time.Time
+}
+
+// Cleanup contains the exact source versions durably accepted by the target.
+// The engine must not request cleanup until target enqueue is confirmed.
+type ReplayCleanup struct {
+	Lease       ReplayLease
+	SourceRunID string
+	TargetRunID string
+	Entries     []FailureEntry
+}
+
+type ReplayCleanupResult struct {
+	CleanedCount        int64
+	AlreadyCleanedCount int64
+	ConflictCount       int64
+}

internal/fullsync/contract/replay_test.go

@@ -0,0 +1,103 @@
+package contract_test
+
+import (
+	"encoding/json"
+	"errors"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestReplayRequestIdentity(t *testing.T) {
+	request := contract.ReplayRequest{SourceRunID: "source", IdempotencyKey: "key", RequestedBy: "operator"}
+	fingerprint, err := request.Fingerprint()
+	if err != nil || len(fingerprint) != 64 {
+		t.Fatalf("fingerprint=%q, err=%v", fingerprint, err)
+	}
+	for name, mutate := range map[string]func(*contract.ReplayRequest){
+		"same":   func(*contract.ReplayRequest) {},
+		"key":    func(r *contract.ReplayRequest) { r.IdempotencyKey = "another-key" },
+		"source": func(r *contract.ReplayRequest) { r.SourceRunID = "another-source" },
+		"body":   func(r *contract.ReplayRequest) { r.RequestedBy = "another-operator" },
+	} {
+		copy := request
+		mutate(&copy)
+		got, err := copy.Fingerprint()
+		if err != nil || (got == fingerprint) != (name == "same" || name == "key") {
+			t.Errorf("%s fingerprint=%s err=%v", name, got, err)
+		}
+	}
+	if got := string(mustJSON(t, request)); got != `{"requestedBy":"operator"}` {
+		t.Fatalf("path/header leaked into body: %s", got)
+	}
+	for _, bad := range []contract.ReplayRequest{
+		{SourceRunID: "source", RequestedBy: "operator"},
+		{SourceRunID: "source", IdempotencyKey: "key", RequestedBy: " "},
+		{SourceRunID: "{source}", IdempotencyKey: "key", RequestedBy: "operator"},
+		{SourceRunID: "source", IdempotencyKey: "key\r\ninjected", RequestedBy: "operator"},
+		{SourceRunID: "source", IdempotencyKey: strings.Repeat("x", 257), RequestedBy: "operator"},
+	} {
+		if _, err := bad.Fingerprint(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Errorf("bad request returned %v", err)
+		}
+	}
+}
+
+func TestActionProgressAndReceiptValidation(t *testing.T) {
+	for _, progress := range []contract.ReplayProgress{
+		{DiscoveredCount: -1}, {DiscoveredCount: 1, EnqueuedCount: 2},
+		{DiscoveredCount: 2, EnqueuedCount: 1, CleanedCount: 2}, {FailureCount: -1},
+	} {
+		if err := progress.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Fatalf("bad progress %+v returned %v", progress, err)
+		}
+	}
+	now := time.Now().UTC()
+	action := contract.ReplayAction{ActionID: "action", SourceRunID: "source", TargetRunID: "target", RequestedBy: "operator", ReplayCycle: 1, Status: contract.ActionAccepted}
+	if err := action.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	action.Status = contract.ActionCompleted
+	if err := action.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("completed action without receipt returned %v", err)
+	}
+	action.Receipt = &contract.ArchiveReceipt{Version: 1, Path: "receipt.json", Checksum: strings.Repeat("a", 64), ArchivedAt: now}
+	action.CompletedAt = &now
+	if err := action.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	action.Receipt.Checksum = "not-a-checksum"
+	if err := action.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+		t.Fatalf("bad receipt returned %v", err)
+	}
+}
+
+func TestActionWireFormatHidesInternalState(t *testing.T) {
+	action := contract.ReplayAction{
+		ActionID: "action", SourceRunID: "source", TargetRunID: "target", RequestedBy: "operator", ReplayCycle: 1,
+		Status: contract.ActionCleanupPending, IdempotencyKey: "secret", Fingerprint: "internal-fingerprint",
+		LeaseOwner: "worker", LeaseToken: "token", LeaseUntil: time.Now(), Revision: 9,
+		Progress: contract.ReplayProgress{ScanCursor: "opaque", DiscoveredCount: 10, EnqueuedCount: 8, CleanedCount: 7, FailureCount: 2},
+	}
+	data := mustJSON(t, action)
+	for _, hidden := range []string{"idempotency", "fingerprint", "lease", "revision", "secret", "token"} {
+		if strings.Contains(strings.ToLower(string(data)), hidden) {
+			t.Errorf("internal %s leaked: %s", hidden, data)
+		}
+	}
+	var decoded contract.ReplayAction
+	if err := json.Unmarshal(data, &decoded); err != nil || !reflect.DeepEqual(decoded.Progress, action.Progress) || decoded.Status != contract.ActionCleanupPending {
+		t.Fatalf("action roundtrip lost state: %+v, %v", decoded, err)
+	}
+	statusURL, err := contract.ActionStatusURL("action%1")
+	if err != nil || statusURL != "/internal/v1/full-sync-actions/action%251" {
+		t.Fatalf("status URL=%s, %v", statusURL, err)
+	}
+	accepted := contract.ReplayAccepted{ActionID: "action", TargetRunID: "target", Status: contract.ActionAccepted, StatusURL: "/internal/v1/full-sync-actions/action"}
+	if got := string(mustJSON(t, accepted)); got != `{"actionId":"action","targetRunId":"target","status":"accepted","statusUrl":"/internal/v1/full-sync-actions/action"}` {
+		t.Fatalf("202 contract changed: %s", got)
+	}
+}

internal/fullsync/contract/run.go

@@ -0,0 +1,146 @@
+package contract
+
+import (
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
+	"time"
+)
+
+// RunMeta retains every existing queue metadata field and JSON tag. The
+// optional fields are additive; zero extension fields are legal on old runs.
+type RunMeta struct {
+	SupplyObservation   *supply.Snapshot `json:"supplyObservation,omitempty"`
+	RunID               string           `json:"runId"`
+	Operation           string           `json:"operation"`
+	QueryMonth          string           `json:"queryMonth"`
+	Status              RunStatus        `json:"status"`
+	ProducerDone        bool             `json:"producerDone"`
+	CurrentRound        int              `json:"currentRound"`
+	ActiveCount         int64            `json:"activeCount"`
+	InflightCount       int64            `json:"inflightCount"`
+	NextRoundCount      int64            `json:"nextRoundCount"`
+	SuccessCount        int64            `json:"successCount"`
+	FailedCount         int64            `json:"failedCount"`
+	CreatedAt           time.Time        `json:"createdAt"`
+	ProducerCompletedAt time.Time        `json:"producerCompletedAt"`
+	CompletedAt         time.Time        `json:"completedAt"`
+
+	SyncType             SyncType          `json:"syncType,omitempty"`
+	SourceRunID          string            `json:"sourceRunId,omitempty"`
+	ReplayCycle          int               `json:"replayCycle,omitempty"`
+	TriggerSource        string            `json:"triggerSource,omitempty"`
+	ScannedCount         int64             `json:"scannedCount,omitempty"`
+	EnqueuedCount        int64             `json:"enqueuedCount,omitempty"`
+	UpstreamAttemptCount int64             `json:"upstreamAttemptCount,omitempty"`
+	BatchCount           int64             `json:"batchCount,omitempty"`
+	MaxRound             int               `json:"maxRound,omitempty"`
+	PendingFailureCount  int64             `json:"pendingFailureCount,omitempty"`
+	StartedAt            *time.Time        `json:"startedAt,omitempty"`
+	ScanCompletedAt      *time.Time        `json:"scanCompletedAt,omitempty"`
+	ArchiveStatus        ArchiveStatus     `json:"archiveStatus,omitempty"`
+	ArchiveVersion       int               `json:"archiveVersion,omitempty"`
+	ArchivePath          string            `json:"archivePath,omitempty"`
+	ArchiveChecksum      string            `json:"archiveChecksum,omitempty"`
+	ArchivedAt           *time.Time        `json:"archivedAt,omitempty"`
+	ReplayDisposition    ReplayDisposition `json:"replayDisposition,omitempty"`
+	// Scope fields mirror zset.RunMeta. ScopeChannels == 0 means a full sync.
+	ScopeChannels       int          `json:"scopeChannels,omitempty"`
+	ScopeDigest         string       `json:"scopeDigest,omitempty"`
+	ScopeChannelsSample string       `json:"scopeChannelsSample,omitempty"`
+	RoundStats          []RoundStats `json:"rounds,omitempty"`
+}
+
+func (m RunMeta) Validate() error {
+	if err := ValidateID(m.RunID); err != nil {
+		return err
+	}
+	if m.Status != "" && !m.Status.Valid() {
+		return invalid("unknown run status")
+	}
+	if m.SyncType != "" && !m.SyncType.Valid() {
+		return invalid("unknown sync type")
+	}
+	if m.ArchiveStatus != "" && !m.ArchiveStatus.Valid() {
+		return invalid("unknown archive status")
+	}
+	if m.ReplayDisposition != "" && !m.ReplayDisposition.Valid() {
+		return invalid("unknown replay disposition")
+	}
+	if m.SourceRunID != "" {
+		if err := ValidateID(m.SourceRunID); err != nil {
+			return err
+		}
+		if m.SourceRunID == m.RunID {
+			return invalid("a run cannot replay itself")
+		}
+	}
+	if m.SyncType == SyncReplay && (m.SourceRunID == "" || m.ReplayCycle < 1) {
+		return invalid("replay requires sourceRunId and a positive replayCycle")
+	}
+	if m.SyncType == SyncNormal && (m.SourceRunID != "" || m.ReplayCycle != 0) {
+		return invalid("normal runs cannot have replay provenance")
+	}
+	if m.MaxRound > 0 && m.CurrentRound > m.MaxRound {
+		return invalid("currentRound exceeds maxRound")
+	}
+	return nonnegative(int64(m.CurrentRound), int64(m.MaxRound), int64(m.ReplayCycle), int64(m.ArchiveVersion), m.ActiveCount, m.InflightCount, m.NextRoundCount, m.SuccessCount, m.FailedCount, m.ScannedCount, m.EnqueuedCount, m.UpstreamAttemptCount, m.BatchCount, m.PendingFailureCount, int64(m.ScopeChannels))
+}
+
+// ValidateAccounting is opt-in for runs whose extended counters are known.
+// Do not apply it to legacy metadata with a missing enqueuedCount. Inflight is
+// a subset of active and is never added to the completion equation.
+func (m RunMeta) ValidateAccounting() error {
+	if err := m.Validate(); err != nil {
+		return err
+	}
+	remaining := m.EnqueuedCount
+	for _, count := range []int64{m.SuccessCount, m.FailedCount, m.ActiveCount, m.NextRoundCount} {
+		if count > remaining {
+			return invalid("enqueuedCount does not balance run outcomes")
+		}
+		remaining -= count
+	}
+	if remaining != 0 || m.InflightCount > m.ActiveCount || m.PendingFailureCount > m.FailedCount {
+		return invalid("inconsistent run counters")
+	}
+	if m.Status.Terminal() && (!m.ProducerDone || m.ActiveCount != 0 || m.NextRoundCount != 0) {
+		return invalid("terminal run still has producer or queue work")
+	}
+	return nil
+}
+
+// RoundStats contains only bounded aggregate fields. It must never grow with
+// the number of cards, channels, task IDs, upstream codes, or error messages.
+type RoundStats struct {
+	RunID                string      `json:"runId"`
+	Round                int         `json:"round"`
+	EnqueuedCount        int64       `json:"enqueuedCount"`
+	ActiveCount          int64       `json:"activeCount"`
+	NextRoundCount       int64       `json:"nextRoundCount"`
+	SuccessCount         int64       `json:"successCount"`
+	FailedCount          int64       `json:"failedCount"`
+	UpstreamAttemptCount int64       `json:"upstreamAttemptCount"`
+	BatchCount           int64       `json:"batchCount"`
+	StartedAt            *time.Time  `json:"startedAt,omitempty"`
+	CompletedAt          *time.Time  `json:"completedAt,omitempty"`
+	ErrorCounts          ErrorCounts `json:"errorCounts"`
+}
+
+func (s RoundStats) Validate() error {
+	if err := ValidateID(s.RunID); err != nil {
+		return err
+	}
+	if s.Round < 1 {
+		return invalid("round must be positive")
+	}
+	if err := nonnegative(s.EnqueuedCount, s.ActiveCount, s.NextRoundCount, s.SuccessCount, s.FailedCount, s.UpstreamAttemptCount, s.BatchCount); err != nil {
+		return err
+	}
+	_, err := s.ErrorCounts.Normalized()
+	return err
+}
+
+type RunView struct {
+	Source RunSource    `json:"source"`
+	Run    RunMeta      `json:"run"`
+	Rounds []RoundStats `json:"rounds,omitempty"`
+}

internal/fullsync/contract/status.go

@@ -0,0 +1,137 @@
+package contract
+
+type RunStatus string
+
+const (
+	RunProducing             RunStatus = "producing"
+	RunRunning               RunStatus = "running"
+	RunCompleted             RunStatus = "completed"
+	RunCompletedWithFailures RunStatus = "completed_with_failures"
+)
+
+func (s RunStatus) Valid() bool {
+	return s == RunProducing || s == RunRunning || s.Terminal()
+}
+
+func (s RunStatus) Terminal() bool { return s == RunCompleted || s == RunCompletedWithFailures }
+
+type SyncType string
+
+const (
+	SyncNormal SyncType = "normal"
+	SyncReplay SyncType = "replay"
+)
+
+func (s SyncType) Valid() bool { return s == SyncNormal || s == SyncReplay }
+
+type SyncKind string
+
+const (
+	SyncKindTrafficUsage        SyncKind = "traffic_usage"
+	SyncKindSimStatus           SyncKind = "sim_status"
+	SyncKindTrafficUsageDaily   SyncKind = "traffic_usage_daily"
+	SyncKindTrafficUsageMonthly SyncKind = "traffic_usage_monthly"
+	SyncKindImei                SyncKind = "imei"
+)
+
+func (s SyncKind) Valid() bool {
+	return s == SyncKindTrafficUsage || s == SyncKindSimStatus || s == SyncKindTrafficUsageDaily || s == SyncKindTrafficUsageMonthly || s == SyncKindImei
+}
+
+func SyncKindForOperation(operation string) SyncKind {
+	switch operation {
+	case "current_month_usage_full_sync":
+		return SyncKindTrafficUsage
+	case "sim_status_queue_full_sync":
+		return SyncKindSimStatus
+	case "previous_day_usage_full_sync":
+		return SyncKindTrafficUsageDaily
+	case "previous_month_usage_full_sync":
+		return SyncKindTrafficUsageMonthly
+	case "imei_full_sync":
+		return SyncKindImei
+	default:
+		return ""
+	}
+}
+
+type ArchiveStatus string
+
+const (
+	ArchiveDisabled   ArchiveStatus = "disabled"
+	ArchivePending    ArchiveStatus = "pending"
+	ArchiveProcessing ArchiveStatus = "processing"
+	ArchiveSucceeded  ArchiveStatus = "succeeded"
+	ArchiveFailed     ArchiveStatus = "failed"
+)
+
+func (s ArchiveStatus) Valid() bool {
+	switch s {
+	case ArchiveDisabled, ArchivePending, ArchiveProcessing, ArchiveSucceeded, ArchiveFailed:
+		return true
+	}
+	return false
+}
+
+// ReplayDisposition describes a source run, not an individual replay action.
+type ReplayDisposition string
+
+const (
+	ReplayNone           ReplayDisposition = "none"
+	ReplayPending        ReplayDisposition = "pending"
+	ReplayProcessing     ReplayDisposition = "processing"
+	ReplayCompleted      ReplayDisposition = "completed"
+	ReplayNeedsAttention ReplayDisposition = "needs_attention"
+)
+
+func (s ReplayDisposition) Valid() bool {
+	switch s {
+	case ReplayNone, ReplayPending, ReplayProcessing, ReplayCompleted, ReplayNeedsAttention:
+		return true
+	}
+	return false
+}
+
+type ActionStatus string
+
+const (
+	ActionAccepted       ActionStatus = "accepted"
+	ActionProcessing     ActionStatus = "processing"
+	ActionCleanupPending ActionStatus = "cleanup_pending"
+	ActionReceiptPending ActionStatus = "receipt_pending"
+	ActionCompleted      ActionStatus = "completed"
+	ActionNeedsAttention ActionStatus = "needs_attention"
+)
+
+func (s ActionStatus) Valid() bool {
+	switch s {
+	case ActionAccepted, ActionProcessing, ActionCleanupPending, ActionReceiptPending, ActionCompleted, ActionNeedsAttention:
+		return true
+	}
+	return false
+}
+
+// Terminal means automatic work has stopped; needs_attention may be resumed
+// only by an explicit recovery policy, never by assuming it succeeded.
+func (s ActionStatus) Terminal() bool { return s == ActionCompleted || s == ActionNeedsAttention }
+
+type RunSource string
+
+const (
+	SourceLive    RunSource = "live"
+	SourceArchive RunSource = "archive"
+)
+
+func (s RunSource) Valid() bool { return s == SourceLive || s == SourceArchive }
+
+// These values are already persisted by the queue protocol.
+const (
+	TaskQueued              = "queued"
+	TaskClaimed             = "claimed"
+	TaskWaitingNextRound    = "waiting_next_round"
+	TaskSucceeded           = "succeeded"
+	TaskCovered             = "covered"
+	TaskFailedPendingReview = "failed_pending_review"
+	TaskChannelBlocked      = "channel_blocked"
+	ReviewPending           = "pending"
+)

internal/fullsync/contract/status_test.go

@@ -0,0 +1,57 @@
+package contract_test
+
+import (
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+// TestSyncKindForDailyUsageOperation guards the mapping that makes a daily run
+// discoverable. Catalog.ListRuns drops any run whose operation maps to an empty
+// kind, so losing this case would make daily runs invisible in the monitor
+// without any error.
+func TestSyncKindForDailyUsageOperation(t *testing.T) {
+	if got := contract.SyncKindForOperation(model.OperationPreviousDayUsageFullSync); got != contract.SyncKindTrafficUsageDaily {
+		t.Fatalf("SyncKindForOperation(%q) = %q, want %q",
+			model.OperationPreviousDayUsageFullSync, got, contract.SyncKindTrafficUsageDaily)
+	}
+	if !contract.SyncKindTrafficUsageDaily.Valid() {
+		t.Fatal("SyncKindTrafficUsageDaily must be a valid kind")
+	}
+	if contract.SyncKindForOperation("some_unknown_operation") != "" {
+		t.Fatal("an unknown operation must not be assigned a sync kind")
+	}
+}
+
+// TestSyncKindForMonthlyHistoryOperation guards the mapping that makes a monthly
+// history run discoverable. Catalog.ListRuns drops any run whose operation maps
+// to an empty kind, so losing this case would make monthly history runs
+// invisible in the monitor without any error.
+func TestSyncKindForMonthlyHistoryOperation(t *testing.T) {
+	if got := contract.SyncKindForOperation(model.OperationPreviousMonthUsageFullSync); got != contract.SyncKindTrafficUsageMonthly {
+		t.Fatalf("SyncKindForOperation(%q) = %q, want %q",
+			model.OperationPreviousMonthUsageFullSync, got, contract.SyncKindTrafficUsageMonthly)
+	}
+}
+
+// TestDailyAndMonthlyUsageHaveDistinctSyncKinds keeps the usage runs separately
+// filterable. There are three distinct usage runs now: the MySQL current-month
+// snapshot, the TDengine daily history, and the TDengine monthly history.
+func TestDailyAndMonthlyUsageHaveDistinctSyncKinds(t *testing.T) {
+	monthly := contract.SyncKindForOperation("current_month_usage_full_sync")
+	daily := contract.SyncKindForOperation(model.OperationPreviousDayUsageFullSync)
+	monthlyHistory := contract.SyncKindForOperation(model.OperationPreviousMonthUsageFullSync)
+	if monthly == daily {
+		t.Fatalf("current-month snapshot and daily history share the sync kind %q", daily)
+	}
+	if monthly == monthlyHistory {
+		t.Fatalf("current-month snapshot and monthly history share the sync kind %q", monthlyHistory)
+	}
+	if daily == monthlyHistory {
+		t.Fatalf("daily and monthly history share the sync kind %q", daily)
+	}
+	if !monthlyHistory.Valid() {
+		t.Fatal("SyncKindTrafficUsageMonthly must be a valid kind")
+	}
+}

internal/fullsync/contract/sync_kind_test.go

@@ -0,0 +1,85 @@
+package contract_test
+
+import (
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+// TestSyncKindForOperationCoversEveryChain pins the operation→kind mapping for
+// all five full-sync chains. A missing case makes the run invisible in the
+// admin monitor, because the catalog filters runs by SyncKind.
+func TestSyncKindForOperationCoversEveryChain(t *testing.T) {
+	cases := []struct {
+		operation string
+		want      contract.SyncKind
+	}{
+		{"current_month_usage_full_sync", contract.SyncKindTrafficUsage},
+		{"sim_status_queue_full_sync", contract.SyncKindSimStatus},
+		{"previous_day_usage_full_sync", contract.SyncKindTrafficUsageDaily},
+		{"previous_month_usage_full_sync", contract.SyncKindTrafficUsageMonthly},
+		{"imei_full_sync", contract.SyncKindImei},
+	}
+	for _, tc := range cases {
+		got := contract.SyncKindForOperation(tc.operation)
+		if got != tc.want {
+			t.Fatalf("SyncKindForOperation(%q) = %q, want %q", tc.operation, got, tc.want)
+		}
+		if !got.Valid() {
+			t.Fatalf("%q 应通过 Valid()", got)
+		}
+	}
+}
+
+// TestSyncKindForOperationRejectsUnknown 未知 operation 返回空 kind，
+// 由调用方决定是跳过还是报错。
+func TestSyncKindForOperationRejectsUnknown(t *testing.T) {
+	for _, operation := range []string{"", "unknown", "IMEI_FULL_SYNC"} {
+		got := contract.SyncKindForOperation(operation)
+		if got != "" {
+			t.Fatalf("SyncKindForOperation(%q) = %q, want empty", operation, got)
+		}
+		if got.Valid() {
+			t.Fatalf("空 kind 不应通过 Valid()")
+		}
+	}
+}
+
+// TestSyncKindValidRejectsUnknown 只有五个已注册的 kind 有效。
+func TestSyncKindValidRejectsUnknown(t *testing.T) {
+	valid := []contract.SyncKind{
+		contract.SyncKindTrafficUsage,
+		contract.SyncKindSimStatus,
+		contract.SyncKindTrafficUsageDaily,
+		contract.SyncKindTrafficUsageMonthly,
+		contract.SyncKindImei,
+	}
+	for _, kind := range valid {
+		if !kind.Valid() {
+			t.Fatalf("%q 应有效", kind)
+		}
+	}
+	invalid := []contract.SyncKind{"", "imei_full_sync", "IMEI", "traffic", "sim-status"}
+	for _, kind := range invalid {
+		if kind.Valid() {
+			t.Fatalf("%q 不应有效", kind)
+		}
+	}
+}
+
+// TestSyncKindValuesAreStable 枚举字符串是持久化契约（写入 Run 元数据、
+// 归档 JSON 与 admin 查询参数），不得随意改动。
+func TestSyncKindValuesAreStable(t *testing.T) {
+	cases := map[contract.SyncKind]string{
+		contract.SyncKindTrafficUsage:        "traffic_usage",
+		contract.SyncKindSimStatus:           "sim_status",
+		contract.SyncKindTrafficUsageDaily:   "traffic_usage_daily",
+		contract.SyncKindTrafficUsageMonthly: "traffic_usage_monthly",
+		contract.SyncKindImei:                "imei",
+	}
+	for kind, want := range cases {
+		if string(kind) != want {
+			t.Fatalf("kind = %q, want %q（持久化契约不得改动）", kind, want)
+		}
+	}
+}

internal/fullsync/contract/validation.go

@@ -0,0 +1,42 @@
+package contract
+
+import (
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+// Implementations wrap these errors so callers can use errors.Is without
+// inspecting error text. HTTP adapters choose the response envelope themselves.
+var (
+	ErrNotFound                  = gerror.New("full-sync resource not found")
+	ErrConflict                  = gerror.New("full-sync conflict")
+	ErrInvalidArgument           = gerror.New("invalid full-sync argument")
+	ErrFailureDetailsUnavailable = gerror.New("live failure details unavailable")
+	ErrLeaseLost                 = gerror.New("full-sync lease lost")
+	ErrBusy                      = gerror.New("full-sync resource busy")
+)
+
+func invalid(format string, args ...any) error {
+	return gerror.Wrapf(ErrInvalidArgument, format, args...)
+}
+
+// ValidateID checks new resource identifiers used both as Redis hash tags and
+// HTTP path segments. It does not rewrite historical identifiers during decode.
+func ValidateID(id string) error {
+	if id == "" || strings.TrimSpace(id) != id || !utf8.ValidString(id) || strings.ContainsAny(id, "{}/\\") || strings.ContainsFunc(id, unicode.IsControl) {
+		return invalid("invalid resource identifier")
+	}
+	return nil
+}
+
+func nonnegative(values ...int64) error {
+	for _, value := range values {
+		if value < 0 {
+			return invalid("counts must be nonnegative")
+		}
+	}
+	return nil
+}

internal/fullsync/contract/validation_test.go

@@ -0,0 +1,121 @@
+package contract_test
+
+import (
+	"errors"
+	"math"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func TestRunAccounting(t *testing.T) {
+	valid := contract.RunMeta{
+		RunID: "run", Status: contract.RunRunning, CurrentRound: 1, MaxRound: 5,
+		EnqueuedCount: 10, ActiveCount: 3, InflightCount: 2, NextRoundCount: 2,
+		SuccessCount: 4, FailedCount: 1, PendingFailureCount: 1,
+	}
+	if err := valid.ValidateAccounting(); err != nil {
+		t.Fatal(err)
+	}
+	for name, mutate := range map[string]func(*contract.RunMeta){
+		"unbalanced": func(m *contract.RunMeta) { m.EnqueuedCount++ },
+		"negative":   func(m *contract.RunMeta) { m.BatchCount = -1 },
+		"overflow": func(m *contract.RunMeta) {
+			m.EnqueuedCount = math.MaxInt64
+			m.SuccessCount = math.MaxInt64
+			m.FailedCount = math.MaxInt64
+		},
+		"inflight double count":        func(m *contract.RunMeta) { m.InflightCount = m.ActiveCount + 1 },
+		"pending exceeds history":      func(m *contract.RunMeta) { m.PendingFailureCount = 2 },
+		"terminal with work":           func(m *contract.RunMeta) { m.Status = contract.RunCompletedWithFailures; m.ProducerDone = true },
+		"unknown status":               func(m *contract.RunMeta) { m.Status = "succeeded" },
+		"unknown sync type":            func(m *contract.RunMeta) { m.SyncType = "manual" },
+		"action status in archive":     func(m *contract.RunMeta) { m.ArchiveStatus = "completed" },
+		"action status in disposition": func(m *contract.RunMeta) { m.ReplayDisposition = "cleanup_pending" },
+		"round out of range":           func(m *contract.RunMeta) { m.CurrentRound = 6 },
+		"replay without source":        func(m *contract.RunMeta) { m.SyncType = contract.SyncReplay },
+		"self replay": func(m *contract.RunMeta) {
+			m.SyncType = contract.SyncReplay
+			m.SourceRunID = m.RunID
+			m.ReplayCycle = 1
+		},
+		"normal with provenance": func(m *contract.RunMeta) { m.SyncType = contract.SyncNormal; m.SourceRunID = "source" },
+	} {
+		t.Run(name, func(t *testing.T) {
+			meta := valid
+			mutate(&meta)
+			if err := meta.ValidateAccounting(); !errors.Is(err, contract.ErrInvalidArgument) {
+				t.Fatalf("invalid accounting returned %v", err)
+			}
+		})
+	}
+	// Cleaned source failures never subtract from the historical failed total.
+	completed := contract.RunMeta{RunID: "source", Status: contract.RunCompletedWithFailures, ProducerDone: true, EnqueuedCount: 5, SuccessCount: 3, FailedCount: 2, PendingFailureCount: 0}
+	if err := completed.ValidateAccounting(); err != nil {
+		t.Fatal(err)
+	}
+	completed.ProducerDone = false
+	if err := completed.ValidateAccounting(); err == nil {
+		t.Fatal("accepted terminal run with unfinished producer")
+	}
+	if err := (contract.RunMeta{RunID: "old", Status: contract.RunRunning, SuccessCount: 42}).Validate(); err != nil {
+		t.Fatalf("historical run without new counters rejected: %v", err)
+	}
+}
+
+func TestRoundStatsValidation(t *testing.T) {
+	valid := contract.RoundStats{RunID: "run", Round: 1, ErrorCounts: contract.ErrorCounts{"unrecognized": 1}}
+	if err := valid.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	for _, stats := range []contract.RoundStats{
+		{RunID: "run", Round: 0}, {RunID: "run", Round: 1, BatchCount: -1},
+		{RunID: "run", Round: 1, ErrorCounts: contract.ErrorCounts{contract.ErrorOther: -1}},
+	} {
+		if err := stats.Validate(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Fatalf("invalid round stats returned %v", err)
+		}
+	}
+}
+
+func TestPageLimitsAndOpaqueCursor(t *testing.T) {
+	for _, limit := range []int{0, 1, 100, 500} {
+		request, err := (contract.PageRequest{Limit: limit, Cursor: "opaque+/=:%?next"}).Normalized()
+		if err != nil || request.Cursor != "opaque+/=:%?next" || request.Limit < 1 || request.Limit > contract.MaxPageLimit {
+			t.Fatalf("unexpected page: %+v, %v", request, err)
+		}
+		if limit == 0 && request.Limit != 100 {
+			t.Fatal("default limit changed")
+		}
+	}
+	for _, limit := range []int{-1, 501, math.MaxInt} {
+		if _, err := (contract.PageRequest{Limit: limit}).Normalized(); !errors.Is(err, contract.ErrInvalidArgument) {
+			t.Fatalf("limit %d returned %v", limit, err)
+		}
+	}
+	page := contract.FailurePage{Items: []contract.Failure{}, NextCursor: "continue"}
+	if got := string(mustJSON(t, page)); got != `{"items":[],"nextCursor":"continue"}` {
+		t.Fatalf("empty scan page lost its continuation: %s", got)
+	}
+}
+
+func TestStatusVocabulary(t *testing.T) {
+	for _, status := range []contract.ActionStatus{"accepted", "processing", "cleanup_pending", "receipt_pending", "completed", "needs_attention"} {
+		if !status.Valid() || status.Terminal() != (status == "completed" || status == "needs_attention") {
+			t.Errorf("invalid action status %q", status)
+		}
+	}
+	for _, status := range []contract.ArchiveStatus{"disabled", "pending", "processing", "succeeded", "failed"} {
+		if !status.Valid() {
+			t.Errorf("invalid archive status %q", status)
+		}
+	}
+	for _, status := range []contract.ReplayDisposition{"none", "pending", "processing", "completed", "needs_attention"} {
+		if !status.Valid() {
+			t.Errorf("invalid disposition %q", status)
+		}
+	}
+	if contract.ActionStatus("succeeded").Valid() || contract.ArchiveStatus("completed").Valid() || contract.ReplayDisposition("receipt_pending").Valid() {
+		t.Fatal("status domains were conflated")
+	}
+}

internal/fullsync/httpapi/README.md

@@ -0,0 +1,99 @@
+# Full-sync HTTP control API
+
+The handler imports no queue or storage implementations. Inject the five
+contract interfaces in `Dependencies`: `RunLister`, `RunReader`,
+`FailurePager`, `ReplayRequester`, and `ActionReader`. The replay service owns durable
+idempotency and execution; the HTTP layer does not cache requests or launch work.
+
+Use `NewRunReader(live, archive, rounds)` to compose `LiveRunReader`,
+`ArchivedRunReader`, and `RoundStatsReader` into a `RunReader`. Live metadata
+wins and returns `source: "live"`; only `ErrNotFound` from live metadata permits
+archive fallback, returning `source: "archive"`. Live storage errors and round
+lookup errors propagate. Archive and round readers may be nil to disable those
+lookups. A failure listing always calls `FailurePager` directly, never archives.
+
+## Routes and responses
+
+All routes require `Authorization: Bearer <configured token>`. Missing, wrong,
+or ambiguous credentials return 401 with a `WWW-Authenticate` challenge. An
+empty configured token is a startup configuration error. Responses use JSON and
+`Cache-Control: no-store`.
+
+| Method | Route | Success body |
+| --- | --- | --- |
+| GET | `/internal/v1/full-sync-runs?syncKind=&limit=` | 200 `contract.RunList` |
+| GET | `/internal/v1/full-sync-runs/{runId}` | 200 `contract.RunView` |
+| GET | `/internal/v1/full-sync-runs/{runId}/failures?cursor=&limit=` | 200 `contract.FailurePage` |
+| POST | `/internal/v1/full-sync-runs/{runId}/replay` | 202 `contract.ReplayAccepted` |
+| GET | `/internal/v1/full-sync-actions/{actionId}` | 200 `contract.ReplayAction` |
+
+Run list results merge active Redis metadata with flat `run-<runId>.json`
+archives under the configured archive root. Active Redis metadata wins when an
+ID exists in both sources. `syncKind` accepts `traffic_usage` or `sim_status`;
+`limit` defaults to 50 and is capped at 100.
+
+Failure limits use the contract rules: absent, empty, or zero selects 100;
+positive values up to 500 are accepted; other values are rejected. Repeated
+cursor/limit parameters and malformed query encoding are rejected. Cursors are
+opaque and passed unchanged after URL decoding. The pager validates cursor
+contents and run binding. Empty pages serialize `items: []` and preserve
+`nextCursor`; only an empty next cursor signals completion.
+
+Replay requires a single `Idempotency-Key` header and a JSON body such as
+`{"requestedBy":"operator"}`. The body is limited to 64 KiB, must contain one
+JSON object with no unknown fields, and must satisfy `ReplayRequest.Validate`.
+If supplied, Content-Type must be `application/json` (parameters are allowed).
+The exact `requestedBy` value is preserved for fingerprinting. Repeated accepted
+requests return the service's existing action/target and current status with
+202. The adapter constructs `statusUrl` and the `Location` header from actionId.
+
+All application errors use this envelope, without storage error details:
+
+```json
+{"error":{"code":"invalid_argument","message":"Invalid request"}}
+```
+
+| Condition | HTTP status | Error code |
+| --- | --- | --- |
+| Invalid path, query, header, body, or `ErrInvalidArgument` | 400 | `invalid_argument` |
+| Missing/wrong bearer credentials | 401 | `unauthorized` |
+| Unknown route/resource or `ErrNotFound` | 404 | `not_found` |
+| Unsupported route method | 405 | `method_not_allowed` |
+| `ErrConflict` (including changed idempotent request), `ErrLeaseLost` | 409 | `conflict` |
+| `ErrFailureDetailsUnavailable` | 410 | `failure_details_unavailable` |
+| Oversized replay body | 413 | `body_too_large` |
+| `ErrBusy` | 503 | `busy` |
+| Other service errors or unencodable service output | 500 | `internal_error` |
+
+## Lifecycle
+
+`NewHandler(token, deps)` can be mounted by an existing HTTP server. For an
+independent listener, construct `NewServer(Config{Addr: address, BearerToken:
+token}, deps)` and call `Start(workerCtx)`. `Start` binds synchronously and
+returns address-in-use and other startup errors directly. `Addr()` exposes the
+bound address, including an OS-assigned port when configured with port zero.
+
+Call `Wait()` after Start, normally in the worker's supervision goroutine. It
+returns serving or shutdown failures; it also returns startup failures if Start
+failed. The server is single-use and rejects a second Start. Wait can have
+multiple callers but must not be called before Start.
+
+Canceling `workerCtx` closes the listener and drains active HTTP requests.
+Active request contexts stay usable during the drain. `ShutdownTimeout` defaults
+to ten seconds; if it expires, active connections are closed and Wait returns
+an error wrapping `context.DeadlineExceeded`. Normal cancellation with a
+successful drain returns nil. No worker wiring or configuration-file changes
+are part of this package.
+
+## Verification
+
+```text
+gofmt -l internal/fullsync/httpapi
+go test ./internal/fullsync/...
+go test ./internal/fullsync/httpapi -cover
+go vet ./internal/fullsync/...
+```
+
+Tests use contract mocks and local HTTP listeners, requiring no Redis or archive
+files. `go test -race ./internal/fullsync/...` additionally requires a supported
+C compiler and CGO enabled.

internal/fullsync/httpapi/SIMPLE_REPLAY.md

@@ -0,0 +1,10 @@
+# Simple full-sync replay
+
+The Worker control server listens on `:18081` inside the private Docker network. Do not publish port 18081 to the host or the internet. Admin already enforces user JWT authentication and replay permissions and calls `http://iotlcm-worker-svr:18081` directly; no additional Worker URL or bearer token environment variables are needed.
+
+- `POST /api/v1/iot/fullSyncRuns/replay` with `{ "sourceRunId": "..." }` returns `replayRunId` and `accepted`. Admin requires `cpm:iot:fullSyncRuns:replay`.
+- `GET /api/v1/iot/fullSyncRuns/replay/{replayRunId}` reports admission (`accepted`, `processing`, `needs_attention`, `completed`), remaining source records, enqueued count, and the independent target Run status. Requires `cpm:iot:fullSyncRuns:replay:view`.
+
+All failures in a completed `flow:` or `sim-status:` Run are selected. No retryability filter; the target Run has exactly one business round. Source failure entries are deleted only after target task-state confirms enqueue; original outcome counts and historical JSON remain unchanged. Repeat requests to the same source return the same target. After repairing a `needs_attention` condition, repeat the request to resume it. To retry again after completion, use the new Replay Run as the source.
+
+No deployment or live Replay request is part of this change.

internal/fullsync/httpapi/handler.go

@@ -0,0 +1,321 @@
+// Package httpapi exposes the internal full-sync control API through contract
+// interfaces. Persistence and replay execution belong to the injected services.
+package httpapi
+
+import (
+	"bytes"
+	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"io"
+	"mime"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	fullsyncruntime "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/runtime"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+const maxReplayBodyBytes = 64 << 10
+
+// Dependencies are the only application services used by the HTTP handlers.
+type Dependencies struct {
+	Runs     contract.RunReader
+	List     contract.RunLister
+	Failures contract.FailurePager
+	Replays  contract.ReplayRequester
+	Actions  contract.ActionReader
+	Simple   *fullsyncruntime.SimpleReplay
+}
+
+// ErrorResponse is shared by authentication, routing, validation and service
+// failures. Messages deliberately exclude underlying storage error details.
+type ErrorResponse struct {
+	Error APIError `json:"error"`
+}
+
+type APIError struct {
+	Code    string `json:"code"`
+	Message string `json:"message"`
+}
+
+// NewHandler exposes the control routes on the private Docker network. Admin
+// performs the user JWT and permission checks before calling this listener.
+func NewHandler(bearerToken string, deps Dependencies) (http.Handler, error) {
+	if strings.ContainsFunc(bearerToken, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) {
+		return nil, gerror.New("invalid full-sync HTTP token")
+	}
+	if deps.Runs == nil || deps.List == nil || deps.Failures == nil || deps.Replays == nil || deps.Actions == nil {
+		return nil, gerror.New("full-sync HTTP requires run list, run, failure, replay and action services")
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc(contract.ListRunsRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		query, err := url.ParseQuery(r.URL.RawQuery)
+		if err != nil || len(query["limit"]) > 1 || len(query["syncKind"]) > 1 {
+			writeError(w, contract.ErrInvalidArgument)
+			return
+		}
+		filter := contract.RunListFilter{SyncKind: contract.SyncKind(query.Get("syncKind"))}
+		if filter.SyncKind != "" && !filter.SyncKind.Valid() {
+			writeError(w, contract.ErrInvalidArgument)
+			return
+		}
+		if query.Get("limit") != "" {
+			filter.Limit, err = strconv.Atoi(query.Get("limit"))
+			if err != nil || filter.Limit < 1 || filter.Limit > 100 {
+				writeError(w, contract.ErrInvalidArgument)
+				return
+			}
+		}
+		result, err := deps.List.ListRuns(r.Context(), filter)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, result)
+	}))
+	mux.HandleFunc(contract.GetRunRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		id := r.PathValue("runId")
+		if err := contract.ValidateID(id); err != nil {
+			writeError(w, err)
+			return
+		}
+		view, err := deps.Runs.GetRun(r.Context(), id)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, view)
+	}))
+	mux.HandleFunc(contract.GetFailuresRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		id := r.PathValue("runId")
+		if err := contract.ValidateID(id); err != nil {
+			writeError(w, err)
+			return
+		}
+		page, err := parsePage(r.URL.RawQuery)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		result, err := deps.Failures.ListFailures(r.Context(), id, page)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		if result.Items == nil {
+			result.Items = []contract.Failure{}
+		}
+		writeJSON(w, http.StatusOK, result)
+	}))
+	mux.HandleFunc(contract.PostReplayRoute, method(http.MethodPost, func(w http.ResponseWriter, r *http.Request) {
+		if deps.Simple != nil {
+			id := r.PathValue("runId")
+			if err := contract.ValidateID(id); err != nil {
+				writeError(w, err)
+				return
+			}
+			var request struct {
+				RequestedBy string `json:"requestedBy"`
+				MaxRounds   int    `json:"maxRounds"`
+			}
+			decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxReplayBodyBytes))
+			decoder.DisallowUnknownFields()
+			if err := decoder.Decode(&request); err != nil {
+				writeError(w, contract.ErrInvalidArgument)
+				return
+			}
+			if err := decoder.Decode(new(any)); err != io.EOF {
+				writeError(w, contract.ErrInvalidArgument)
+				return
+			}
+			accepted, err := deps.Simple.Request(r.Context(), id, request.RequestedBy, request.MaxRounds)
+			if errors.Is(err, fullsyncruntime.ErrNoFailures) {
+				writeAPIError(w, http.StatusUnprocessableEntity, "no_failures", "Run has no failures to replay")
+				return
+			}
+			if err != nil {
+				writeError(w, err)
+				return
+			}
+			writeJSON(w, http.StatusAccepted, accepted)
+			return
+		}
+		request, err := parseReplay(w, r)
+		if err != nil {
+			var tooLarge *http.MaxBytesError
+			if errors.As(err, &tooLarge) {
+				writeAPIError(w, http.StatusRequestEntityTooLarge, "body_too_large", "Replay request body is too large")
+			} else {
+				writeError(w, err)
+			}
+			return
+		}
+		accepted, err := deps.Replays.RequestReplay(r.Context(), request)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		// Construct a local status URL from the returned identity, never an
+		// externally supplied redirect. Invalid service output is a server error.
+		accepted.StatusURL, err = contract.ActionStatusURL(accepted.ActionID)
+		if err != nil || contract.ValidateID(accepted.TargetRunID) != nil || !accepted.Status.Valid() {
+			writeAPIError(w, http.StatusInternalServerError, "internal_error", "Internal server error")
+			return
+		}
+		w.Header().Set("Location", accepted.StatusURL)
+		writeJSON(w, http.StatusAccepted, accepted)
+	}))
+	mux.HandleFunc("/internal/v1/full-sync-replays/{replayRunId}", method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		if deps.Simple == nil {
+			writeError(w, contract.ErrNotFound)
+			return
+		}
+		status, err := deps.Simple.Status(r.Context(), r.PathValue("replayRunId"))
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, status)
+	}))
+	mux.HandleFunc(contract.GetActionRoute, method(http.MethodGet, func(w http.ResponseWriter, r *http.Request) {
+		if deps.Simple != nil {
+			writeError(w, contract.ErrNotFound)
+			return
+		}
+		id := r.PathValue("actionId")
+		if err := contract.ValidateID(id); err != nil {
+			writeError(w, err)
+			return
+		}
+		action, err := deps.Actions.GetReplayAction(r.Context(), id)
+		if err != nil {
+			writeError(w, err)
+			return
+		}
+		writeJSON(w, http.StatusOK, action)
+	}))
+	mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
+		writeError(w, contract.ErrNotFound)
+	})
+	if bearerToken == "" {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Cache-Control", "no-store")
+			mux.ServeHTTP(w, r)
+		}), nil
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Cache-Control", "no-store")
+		values := r.Header.Values("Authorization")
+		if len(values) == 1 {
+			parts := strings.Fields(values[0])
+			if len(parts) == 2 && strings.EqualFold(parts[0], "Bearer") && subtle.ConstantTimeCompare([]byte(parts[1]), []byte(bearerToken)) == 1 {
+				mux.ServeHTTP(w, r)
+				return
+			}
+		}
+		w.Header().Set("WWW-Authenticate", `Bearer realm="full-sync"`)
+		writeAPIError(w, http.StatusUnauthorized, "unauthorized", "Valid bearer authentication is required")
+	}), nil
+}
+
+func method(expected string, next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != expected {
+			w.Header().Set("Allow", expected)
+			writeAPIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "Method not allowed")
+			return
+		}
+		next(w, r)
+	}
+}
+
+func parsePage(rawQuery string) (contract.PageRequest, error) {
+	query, err := url.ParseQuery(rawQuery)
+	if err != nil || len(query["cursor"]) > 1 || len(query["limit"]) > 1 {
+		return contract.PageRequest{}, gerror.Wrap(contract.ErrInvalidArgument, "invalid pagination query")
+	}
+	page := contract.PageRequest{Cursor: contract.Cursor(query.Get("cursor"))}
+	if value := query.Get("limit"); value != "" {
+		page.Limit, err = strconv.Atoi(value)
+		if err != nil {
+			return contract.PageRequest{}, gerror.Wrap(contract.ErrInvalidArgument, "limit must be an integer")
+		}
+	}
+	return page.Normalized()
+}
+
+func parseReplay(w http.ResponseWriter, r *http.Request) (contract.ReplayRequest, error) {
+	var request contract.ReplayRequest
+	if value := r.Header.Get("Content-Type"); value != "" {
+		mediaType, _, err := mime.ParseMediaType(value)
+		if err != nil || mediaType != "application/json" {
+			return request, gerror.Wrap(contract.ErrInvalidArgument, "replay body must be JSON")
+		}
+	}
+	data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxReplayBodyBytes))
+	if err != nil {
+		var tooLarge *http.MaxBytesError
+		if errors.As(err, &tooLarge) {
+			return request, err
+		}
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "cannot read replay body")
+	}
+	if !utf8.Valid(data) {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "replay body must be UTF-8")
+	}
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&request); err != nil {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "invalid replay JSON")
+	}
+	if err := decoder.Decode(new(any)); err != io.EOF {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "replay body must contain exactly one JSON object")
+	}
+	if len(r.Header.Values(contract.IdempotencyHeader)) != 1 {
+		return request, gerror.Wrap(contract.ErrInvalidArgument, "exactly one Idempotency-Key is required")
+	}
+	request.SourceRunID = r.PathValue("runId")
+	request.IdempotencyKey = r.Header.Get(contract.IdempotencyHeader)
+	return request, request.Validate()
+}
+
+func writeError(w http.ResponseWriter, err error) {
+	switch {
+	case errors.Is(err, contract.ErrInvalidArgument):
+		writeAPIError(w, http.StatusBadRequest, "invalid_argument", "Invalid request")
+	case errors.Is(err, contract.ErrNotFound):
+		writeAPIError(w, http.StatusNotFound, "not_found", "Resource not found")
+	case errors.Is(err, contract.ErrFailureDetailsUnavailable):
+		writeAPIError(w, http.StatusGone, "failure_details_unavailable", "Live failure details are no longer available")
+	case errors.Is(err, contract.ErrConflict), errors.Is(err, contract.ErrLeaseLost):
+		writeAPIError(w, http.StatusConflict, "conflict", "Request conflicts with the current resource")
+	case errors.Is(err, contract.ErrBusy):
+		writeAPIError(w, http.StatusServiceUnavailable, "busy", "Resource is busy; retry later")
+	default:
+		writeAPIError(w, http.StatusInternalServerError, "internal_error", "Internal server error")
+	}
+}
+
+func writeAPIError(w http.ResponseWriter, status int, code, message string) {
+	writeJSON(w, status, ErrorResponse{Error: APIError{Code: code, Message: message}})
+}
+
+func writeJSON(w http.ResponseWriter, status int, value any) {
+	// Marshal before writing headers so an invalid service payload cannot
+	// produce a successful status with truncated JSON.
+	data, err := json.Marshal(value)
+	if err != nil {
+		status = http.StatusInternalServerError
+		data = []byte(`{"error":{"code":"internal_error","message":"Internal server error"}}`)
+	}
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(status)
+	_, _ = w.Write(append(data, '\n'))
+}

internal/fullsync/httpapi/handler_test.go

@@ -0,0 +1,437 @@
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+const (
+	testToken   = "test-secret"
+	runPath     = "/internal/v1/full-sync-runs/run-1"
+	failurePath = runPath + "/failures"
+	replayPath  = runPath + "/replay"
+	actionPath  = "/internal/v1/full-sync-actions/action-1"
+)
+
+type runFunc func(context.Context, string) (contract.RunView, error)
+
+func (f runFunc) GetRun(ctx context.Context, id string) (contract.RunView, error) {
+	return f(ctx, id)
+}
+
+type runListFunc func(context.Context, contract.RunListFilter) (contract.RunList, error)
+
+func (f runListFunc) ListRuns(ctx context.Context, filter contract.RunListFilter) (contract.RunList, error) {
+	return f(ctx, filter)
+}
+
+type failureFunc func(context.Context, string, contract.PageRequest) (contract.FailurePage, error)
+
+func (f failureFunc) ListFailures(ctx context.Context, id string, page contract.PageRequest) (contract.FailurePage, error) {
+	return f(ctx, id, page)
+}
+
+type replayFunc func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error)
+
+func (f replayFunc) RequestReplay(ctx context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
+	return f(ctx, req)
+}
+
+type actionFunc func(context.Context, string) (contract.ReplayAction, error)
+
+func (f actionFunc) GetReplayAction(ctx context.Context, id string) (contract.ReplayAction, error) {
+	return f(ctx, id)
+}
+
+func testDependencies() Dependencies {
+	return Dependencies{
+		Runs: runFunc(func(context.Context, string) (contract.RunView, error) {
+			return contract.RunView{}, contract.ErrNotFound
+		}),
+		List: runListFunc(func(context.Context, contract.RunListFilter) (contract.RunList, error) {
+			return contract.RunList{}, contract.ErrNotFound
+		}),
+		Failures: failureFunc(func(context.Context, string, contract.PageRequest) (contract.FailurePage, error) {
+			return contract.FailurePage{}, contract.ErrNotFound
+		}),
+		Replays: replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+			return contract.ReplayAccepted{}, contract.ErrNotFound
+		}),
+		Actions: actionFunc(func(context.Context, string) (contract.ReplayAction, error) {
+			return contract.ReplayAction{}, contract.ErrNotFound
+		}),
+	}
+}
+
+func testHandler(t *testing.T, deps Dependencies) http.Handler {
+	t.Helper()
+	handler, err := NewHandler(testToken, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return handler
+}
+
+func request(handler http.Handler, method, path, body string, headers ...string) *httptest.ResponseRecorder {
+	req := httptest.NewRequest(method, path, strings.NewReader(body))
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	for i := 0; i < len(headers); i += 2 {
+		req.Header.Set(headers[i], headers[i+1])
+	}
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, req)
+	return recorder
+}
+
+func assertError(t *testing.T, response *httptest.ResponseRecorder, status int, code string) {
+	t.Helper()
+	if response.Code != status {
+		t.Fatalf("status = %d, want %d: %s", response.Code, status, response.Body)
+	}
+	if !strings.HasPrefix(response.Header().Get("Content-Type"), "application/json") {
+		t.Fatalf("missing JSON content type: %v", response.Header())
+	}
+	var body ErrorResponse
+	if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+		t.Fatal(err)
+	}
+	if body.Error.Code != code || body.Error.Message == "" {
+		t.Fatalf("error = %+v, want code %q and message", body, code)
+	}
+}
+
+func TestAuthenticationProtectsEveryRoute(t *testing.T) {
+	deps := testDependencies()
+	deps.Runs = runFunc(func(context.Context, string) (contract.RunView, error) {
+		t.Fatal("unauthenticated request reached service")
+		return contract.RunView{}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, path := range []string{runPath, failurePath, replayPath, actionPath, "/unknown"} {
+		for _, auth := range []string{"", "Bearer", "Bearer wrong", "Basic " + testToken, "Bearer " + testToken + " extra"} {
+			t.Run(path+"/"+auth, func(t *testing.T) {
+				response := request(handler, http.MethodGet, path, "", "Authorization", auth)
+				assertError(t, response, 401, "unauthorized")
+				if response.Header().Get("WWW-Authenticate") == "" {
+					t.Fatal("missing authentication challenge")
+				}
+			})
+		}
+	}
+	req := httptest.NewRequest(http.MethodGet, runPath, nil)
+	req.Header.Add("Authorization", "Bearer "+testToken)
+	req.Header.Add("Authorization", "Bearer "+testToken)
+	response := httptest.NewRecorder()
+	handler.ServeHTTP(response, req)
+	assertError(t, response, 401, "unauthorized")
+}
+
+func TestRunListQuery(t *testing.T) {
+	deps := testDependencies()
+	deps.List = runListFunc(func(_ context.Context, filter contract.RunListFilter) (contract.RunList, error) {
+		if filter.SyncKind != contract.SyncKindTrafficUsage || filter.Limit != 7 {
+			t.Fatalf("filter = %+v", filter)
+		}
+		return contract.RunList{Items: []contract.RunView{{Source: contract.SourceLive, Run: contract.RunMeta{RunID: "run-1", Operation: "current_month_usage_full_sync"}}}, Total: 1}, nil
+	})
+	response := request(testHandler(t, deps), http.MethodGet, "/internal/v1/full-sync-runs?syncKind=traffic_usage&limit=7", "")
+	if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), "run-1") {
+		t.Fatalf("response = %d %s", response.Code, response.Body)
+	}
+	for _, query := range []string{"syncKind=invalid", "limit=101", "limit=abc", "limit=1&limit=2"} {
+		assertError(t, request(testHandler(t, deps), http.MethodGet, "/internal/v1/full-sync-runs?"+query, ""), 400, "invalid_argument")
+	}
+}
+
+func TestRoutesAndValidation(t *testing.T) {
+	handler := testHandler(t, testDependencies())
+	for _, route := range []struct{ method, path string }{
+		{http.MethodGet, runPath}, {http.MethodGet, failurePath},
+		{http.MethodPost, replayPath}, {http.MethodGet, actionPath},
+	} {
+		t.Run(route.path, func(t *testing.T) {
+			response := request(handler, route.method, route.path, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key")
+			assertError(t, response, 404, "not_found")
+			wrongMethod := http.MethodPost
+			if route.method == http.MethodPost {
+				wrongMethod = http.MethodGet
+			}
+			response = request(handler, wrongMethod, route.path, "")
+			assertError(t, response, 405, "method_not_allowed")
+			if response.Header().Get("Allow") != route.method {
+				t.Fatalf("Allow = %q", response.Header().Get("Allow"))
+			}
+		})
+	}
+	for _, path := range []string{"/unknown", "/internal/v1/full-sync-runs", runPath + "/other", runPath + "/"} {
+		assertError(t, request(handler, http.MethodGet, path, ""), 404, "not_found")
+	}
+	for _, path := range []string{
+		"/internal/v1/full-sync-runs/%20run",
+		"/internal/v1/full-sync-runs/a%2Fb/failures",
+		"/internal/v1/full-sync-actions/%7Baction%7D",
+	} {
+		assertError(t, request(handler, http.MethodGet, path, ""), 400, "invalid_argument")
+	}
+	assertError(t, request(handler, http.MethodPost, "/internal/v1/full-sync-runs/a%5Cb/replay", `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key"), 400, "invalid_argument")
+}
+
+func TestRunAndActionResponsesPropagateContext(t *testing.T) {
+	type contextKey struct{}
+	deps := testDependencies()
+	deps.Runs = runFunc(func(ctx context.Context, id string) (contract.RunView, error) {
+		if id != "run-1" || ctx.Value(contextKey{}) != "request" {
+			t.Fatal("run identity or request context lost")
+		}
+		return contract.RunView{Source: contract.SourceLive, Run: contract.RunMeta{RunID: id}}, nil
+	})
+	deps.Actions = actionFunc(func(ctx context.Context, id string) (contract.ReplayAction, error) {
+		if id != "action-1" || ctx.Value(contextKey{}) != "request" {
+			t.Fatal("action identity or request context lost")
+		}
+		return contract.ReplayAction{ActionID: id, Status: contract.ActionProcessing, IdempotencyKey: "private-key", Fingerprint: "private-fingerprint", LeaseToken: "private-lease", Revision: 99}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, path := range []string{runPath, actionPath} {
+		req := httptest.NewRequest(http.MethodGet, path, nil).WithContext(context.WithValue(context.Background(), contextKey{}, "request"))
+		req.Header.Set("Authorization", "bearer "+testToken)
+		response := httptest.NewRecorder()
+		handler.ServeHTTP(response, req)
+		if response.Code != 200 || response.Header().Get("Cache-Control") != "no-store" {
+			t.Fatalf("response = %d %s", response.Code, response.Body)
+		}
+		if strings.Contains(response.Body.String(), "private-") || strings.Contains(response.Body.String(), "revision") {
+			t.Fatalf("internal fields leaked: %s", response.Body)
+		}
+	}
+}
+
+func TestFailurePagination(t *testing.T) {
+	for _, test := range []struct {
+		query  string
+		limit  int
+		cursor contract.Cursor
+	}{
+		{"", 100, ""}, {"?limit=", 100, ""}, {"?limit=0", 100, ""},
+		{"?limit=1", 1, ""}, {"?limit=500", 500, ""},
+		{"?limit=17&cursor=" + url.QueryEscape("opaque/+ token==:雪"), 17, "opaque/+ token==:雪"},
+	} {
+		t.Run(test.query, func(t *testing.T) {
+			deps := testDependencies()
+			calls := 0
+			deps.Failures = failureFunc(func(_ context.Context, id string, page contract.PageRequest) (contract.FailurePage, error) {
+				calls++
+				if id != "run-1" || page.Limit != test.limit || page.Cursor != test.cursor {
+					t.Fatalf("query = %q %+v", id, page)
+				}
+				return contract.FailurePage{NextCursor: "next:opaque/+="}, nil
+			})
+			response := request(testHandler(t, deps), http.MethodGet, failurePath+test.query, "")
+			if response.Code != 200 || calls != 1 {
+				t.Fatalf("response = %d, calls = %d", response.Code, calls)
+			}
+			var page contract.FailurePage
+			if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil {
+				t.Fatal(err)
+			}
+			if page.Items == nil || len(page.Items) != 0 || page.NextCursor != "next:opaque/+=" {
+				t.Fatalf("empty scan page lost array or continuation: %s", response.Body)
+			}
+		})
+	}
+	deps := testDependencies()
+	deps.Failures = failureFunc(func(context.Context, string, contract.PageRequest) (contract.FailurePage, error) {
+		t.Fatal("invalid pagination reached service")
+		return contract.FailurePage{}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, query := range []string{"limit=-1", "limit=501", "limit=abc", "limit=1.5", "limit=999999999999999999999999", "limit=1&limit=2", "cursor=a&cursor=b", "cursor=%ZZ", "cursor=a;b"} {
+		assertError(t, request(handler, http.MethodGet, failurePath+"?"+query, ""), 400, "invalid_argument")
+	}
+}
+
+func TestFailureCursorValidationAndLiveOnly(t *testing.T) {
+	deps := testDependencies()
+	deps.Runs = runFunc(func(context.Context, string) (contract.RunView, error) {
+		t.Fatal("failure route must not query run summaries or fall back to archives")
+		return contract.RunView{}, nil
+	})
+	deps.Failures = failureFunc(func(_ context.Context, _ string, page contract.PageRequest) (contract.FailurePage, error) {
+		if page.Cursor == "another-run-token" {
+			return contract.FailurePage{}, fmt.Errorf("wrong query binding: %w", contract.ErrInvalidArgument)
+		}
+		return contract.FailurePage{}, fmt.Errorf("expired details: %w", contract.ErrFailureDetailsUnavailable)
+	})
+	handler := testHandler(t, deps)
+	assertError(t, request(handler, http.MethodGet, failurePath+"?cursor=another-run-token", ""), 400, "invalid_argument")
+	assertError(t, request(handler, http.MethodGet, failurePath, ""), 410, "failure_details_unavailable")
+}
+
+func TestReplayValidation(t *testing.T) {
+	deps := testDependencies()
+	deps.Replays = replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+		t.Fatal("invalid replay reached service")
+		return contract.ReplayAccepted{}, nil
+	})
+	handler := testHandler(t, deps)
+	for _, body := range []string{"", "{", "null", "[]", `{}`, `{"requestedBy":""}`, `{"requestedBy":"  "}`, `{"requestedBy":12}`, `{"requestedBy":"a","targetRunId":"injected"}`, `{"requestedBy":"a"} {}`, `{"requestedBy":"a"} trailing`, "{\"requestedBy\":\"\xff\"}"} {
+		t.Run(body, func(t *testing.T) {
+			assertError(t, request(handler, http.MethodPost, replayPath, body, contract.IdempotencyHeader, "key"), 400, "invalid_argument")
+		})
+	}
+	for _, key := range []string{"", " ", strings.Repeat("x", contract.MaxIdempotencyKeyBytes+1), "bad\tkey"} {
+		assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, key), 400, "invalid_argument")
+	}
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`), 400, "invalid_argument")
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key", "Content-Type", "text/plain"), 400, "invalid_argument")
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"`+strings.Repeat("x", maxReplayBodyBytes)+`"}`, contract.IdempotencyHeader, "key"), 413, "body_too_large")
+	req := httptest.NewRequest(http.MethodPost, replayPath, strings.NewReader(`{"requestedBy":"operator"}`))
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	req.Header.Add(contract.IdempotencyHeader, "key")
+	req.Header.Add(contract.IdempotencyHeader, "key")
+	response := httptest.NewRecorder()
+	handler.ServeHTTP(response, req)
+	assertError(t, response, 400, "invalid_argument")
+}
+
+// The mock owns idempotency; the HTTP adapter must pass exact request identity
+// and return the service's existing action/target even across handler instances.
+func TestReplayAcceptedIdempotencyAndConflict(t *testing.T) {
+	deps := testDependencies()
+	var (
+		stored contract.ReplayRequest
+		calls  int
+	)
+	deps.Replays = replayFunc(func(_ context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
+		calls++
+		if calls == 1 {
+			stored = req
+		}
+		if req != stored {
+			return contract.ReplayAccepted{}, fmt.Errorf("changed fingerprint: %w", contract.ErrConflict)
+		}
+		return contract.ReplayAccepted{ActionID: "action-1", TargetRunID: "replay-1", Status: contract.ActionAccepted, StatusURL: "https://untrusted.invalid"}, nil
+	})
+	var first contract.ReplayAccepted
+	for i := 0; i < 2; i++ {
+		response := request(testHandler(t, deps), http.MethodPost, replayPath, "{\"requestedBy\":\" operator \"}\n", contract.IdempotencyHeader, "key", "Content-Type", "application/json; charset=utf-8")
+		if response.Code != 202 {
+			t.Fatalf("response = %d %s", response.Code, response.Body)
+		}
+		var accepted contract.ReplayAccepted
+		if err := json.Unmarshal(response.Body.Bytes(), &accepted); err != nil {
+			t.Fatal(err)
+		}
+		if accepted.ActionID != "action-1" || accepted.TargetRunID != "replay-1" || accepted.StatusURL != actionPath || response.Header().Get("Location") != actionPath {
+			t.Fatalf("acceptance = %+v, headers = %v", accepted, response.Header())
+		}
+		if i == 0 {
+			first = accepted
+		} else if !reflect.DeepEqual(first, accepted) {
+			t.Fatalf("repeat changed identity: %+v vs %+v", first, accepted)
+		}
+	}
+	if stored.SourceRunID != "run-1" || stored.RequestedBy != " operator " || stored.IdempotencyKey != "key" {
+		t.Fatalf("request identity changed: %+v", stored)
+	}
+	assertError(t, request(testHandler(t, deps), http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key"), 409, "conflict")
+	if calls != 3 {
+		t.Fatalf("replay calls = %d", calls)
+	}
+}
+
+func TestWrappedServiceErrorsUseConsistentEnvelope(t *testing.T) {
+	for _, test := range []struct {
+		err    error
+		status int
+		code   string
+	}{
+		{contract.ErrInvalidArgument, 400, "invalid_argument"},
+		{contract.ErrNotFound, 404, "not_found"},
+		{contract.ErrConflict, 409, "conflict"},
+		{contract.ErrLeaseLost, 409, "conflict"},
+		{contract.ErrFailureDetailsUnavailable, 410, "failure_details_unavailable"},
+		{contract.ErrBusy, 503, "busy"},
+		{fmt.Errorf("private database password"), 500, "internal_error"},
+	} {
+		t.Run(test.code, func(t *testing.T) {
+			err := fmt.Errorf("private database detail: %w", test.err)
+			deps := Dependencies{
+				Runs: runFunc(func(context.Context, string) (contract.RunView, error) { return contract.RunView{}, err }),
+				List: runListFunc(func(context.Context, contract.RunListFilter) (contract.RunList, error) {
+					return contract.RunList{}, err
+				}),
+				Failures: failureFunc(func(context.Context, string, contract.PageRequest) (contract.FailurePage, error) {
+					return contract.FailurePage{}, err
+				}),
+				Replays: replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+					return contract.ReplayAccepted{}, err
+				}),
+				Actions: actionFunc(func(context.Context, string) (contract.ReplayAction, error) { return contract.ReplayAction{}, err }),
+			}
+			handler := testHandler(t, deps)
+			for _, path := range []string{runPath, failurePath, actionPath, replayPath} {
+				method := http.MethodGet
+				if path == replayPath {
+					method = http.MethodPost
+				}
+				response := request(handler, method, path, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key")
+				assertError(t, response, test.status, test.code)
+				if strings.Contains(response.Body.String(), "private") {
+					t.Fatal("service details leaked")
+				}
+			}
+		})
+	}
+}
+
+func TestInvalidServiceOutputReturns500(t *testing.T) {
+	deps := testDependencies()
+	deps.Replays = replayFunc(func(context.Context, contract.ReplayRequest) (contract.ReplayAccepted, error) {
+		return contract.ReplayAccepted{ActionID: "bad/id"}, nil
+	})
+	deps.Runs = runFunc(func(context.Context, string) (contract.RunView, error) {
+		return contract.RunView{Rounds: []contract.RoundStats{{ErrorCounts: contract.ErrorCounts{contract.ErrorOther: -1}}}}, nil
+	})
+	handler := testHandler(t, deps)
+	assertError(t, request(handler, http.MethodPost, replayPath, `{"requestedBy":"operator"}`, contract.IdempotencyHeader, "key"), 500, "internal_error")
+	assertError(t, request(handler, http.MethodGet, runPath, ""), 500, "internal_error")
+}
+
+func TestNewHandlerRejectsInvalidConfiguration(t *testing.T) {
+	if _, err := NewHandler("", testDependencies()); err != nil {
+		t.Fatalf("empty token should use Docker-network-only access: %v", err)
+	}
+	for _, token := range []string{" ", "a b", "token\n"} {
+		if _, err := NewHandler(token, testDependencies()); err == nil {
+			t.Errorf("accepted token %q", token)
+		}
+	}
+	for i := 0; i < 5; i++ {
+		deps := testDependencies()
+		switch i {
+		case 0:
+			deps.Runs = nil
+		case 1:
+			deps.Failures = nil
+		case 2:
+			deps.Replays = nil
+		case 3:
+			deps.Actions = nil
+		case 4:
+			deps.List = nil
+		}
+		if _, err := NewHandler(testToken, deps); err == nil {
+			t.Errorf("accepted missing dependency %d", i)
+		}
+	}
+}

internal/fullsync/httpapi/run_reader.go

@@ -0,0 +1,48 @@
+package httpapi
+
+import (
+	"context"
+	"errors"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+type runReader struct {
+	live    contract.LiveRunReader
+	archive contract.ArchivedRunReader
+	rounds  contract.RoundStatsReader
+}
+
+// NewRunReader composes live-first run queries without knowing either storage
+// implementation. Archive and rounds are optional (nil disables that lookup).
+// Only absent live metadata permits archive fallback; live round errors do not.
+func NewRunReader(live contract.LiveRunReader, archive contract.ArchivedRunReader, rounds contract.RoundStatsReader) (contract.RunReader, error) {
+	if live == nil {
+		return nil, gerror.New("full-sync run reader requires a live reader")
+	}
+	return &runReader{live: live, archive: archive, rounds: rounds}, nil
+}
+
+func (r *runReader) GetRun(ctx context.Context, runID string) (contract.RunView, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunView{}, err
+	}
+	meta, err := r.live.GetRunMeta(ctx, runID)
+	if err == nil {
+		view := contract.RunView{Source: contract.SourceLive, Run: meta}
+		if r.rounds != nil {
+			view.Rounds, err = r.rounds.GetRoundStats(ctx, runID)
+		}
+		return view, err
+	}
+	if !errors.Is(err, contract.ErrNotFound) || r.archive == nil {
+		return contract.RunView{}, err
+	}
+	archive, err := r.archive.GetArchivedRun(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	return contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}, nil
+}

internal/fullsync/httpapi/run_reader_test.go

@@ -0,0 +1,125 @@
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+type liveFunc func(context.Context, string) (contract.RunMeta, error)
+
+func (f liveFunc) GetRunMeta(ctx context.Context, id string) (contract.RunMeta, error) {
+	return f(ctx, id)
+}
+
+type archiveFunc func(context.Context, string) (contract.RunArchive, error)
+
+func (f archiveFunc) GetArchivedRun(ctx context.Context, id string) (contract.RunArchive, error) {
+	return f(ctx, id)
+}
+
+type roundsFunc func(context.Context, string) ([]contract.RoundStats, error)
+
+func (f roundsFunc) GetRoundStats(ctx context.Context, id string) ([]contract.RoundStats, error) {
+	return f(ctx, id)
+}
+
+func TestRunLookupLiveFirstAndArchiveFallback(t *testing.T) {
+	for _, test := range []struct {
+		name         string
+		liveErr      error
+		roundsErr    error
+		archiveErr   error
+		wantStatus   int
+		wantSource   contract.RunSource
+		wantArchives int
+		wantRounds   int
+	}{
+		{name: "live", wantStatus: 200, wantSource: contract.SourceLive, wantRounds: 1},
+		{name: "archive", liveErr: fmt.Errorf("expired: %w", contract.ErrNotFound), wantStatus: 200, wantSource: contract.SourceArchive, wantArchives: 1},
+		{name: "unknown", liveErr: contract.ErrNotFound, archiveErr: contract.ErrNotFound, wantStatus: 404, wantArchives: 1},
+		{name: "live failure never falls back", liveErr: fmt.Errorf("live offline"), wantStatus: 500},
+		{name: "round error never falls back", roundsErr: fmt.Errorf("rounds offline"), wantStatus: 500, wantRounds: 1},
+		{name: "missing rounds never fall back", roundsErr: contract.ErrNotFound, wantStatus: 404, wantRounds: 1},
+		{name: "archive error", liveErr: contract.ErrNotFound, archiveErr: fmt.Errorf("archive offline"), wantStatus: 500, wantArchives: 1},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var liveCalls, archiveCalls, roundsCalls int
+			reader, err := NewRunReader(
+				liveFunc(func(_ context.Context, id string) (contract.RunMeta, error) {
+					liveCalls++
+					if id != "run-1" {
+						t.Fatalf("live ID = %q", id)
+					}
+					return contract.RunMeta{RunID: id, CurrentRound: 2}, test.liveErr
+				}),
+				archiveFunc(func(_ context.Context, id string) (contract.RunArchive, error) {
+					archiveCalls++
+					if id != "run-1" {
+						t.Fatalf("archive ID = %q", id)
+					}
+					return contract.RunArchive{Run: contract.RunMeta{RunID: id, CurrentRound: 3}, Rounds: []contract.RoundStats{{RunID: id, Round: 3}}}, test.archiveErr
+				}),
+				roundsFunc(func(_ context.Context, id string) ([]contract.RoundStats, error) {
+					roundsCalls++
+					if id != "run-1" {
+						t.Fatalf("rounds ID = %q", id)
+					}
+					return []contract.RoundStats{{RunID: id, Round: 2}}, test.roundsErr
+				}),
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+			deps := testDependencies()
+			deps.Runs = reader
+			response := request(testHandler(t, deps), http.MethodGet, runPath, "")
+			if response.Code != test.wantStatus {
+				t.Fatalf("response = %d %s", response.Code, response.Body)
+			}
+			if liveCalls != 1 || archiveCalls != test.wantArchives || roundsCalls != test.wantRounds {
+				t.Fatalf("calls: live=%d archive=%d rounds=%d", liveCalls, archiveCalls, roundsCalls)
+			}
+			if response.Code == 200 {
+				var view contract.RunView
+				if err := json.Unmarshal(response.Body.Bytes(), &view); err != nil {
+					t.Fatal(err)
+				}
+				if view.Source != test.wantSource || view.Run.RunID != "run-1" || len(view.Rounds) != 1 || view.Rounds[0].Round != view.Run.CurrentRound {
+					t.Fatalf("view = %+v", view)
+				}
+			}
+		})
+	}
+}
+
+func TestRunReaderOptionalServicesAndValidation(t *testing.T) {
+	if _, err := NewRunReader(nil, nil, nil); err == nil {
+		t.Fatal("accepted nil live reader")
+	}
+	calls := 0
+	reader, err := NewRunReader(liveFunc(func(_ context.Context, id string) (contract.RunMeta, error) {
+		calls++
+		if id == "missing" {
+			return contract.RunMeta{}, contract.ErrNotFound
+		}
+		return contract.RunMeta{RunID: id}, nil
+	}), nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := reader.GetRun(context.Background(), "bad/id"); err == nil || calls != 0 {
+		t.Fatal("invalid ID reached live reader")
+	}
+	view, err := reader.GetRun(context.Background(), "run-1")
+	if err != nil || view.Source != contract.SourceLive || view.Rounds != nil {
+		t.Fatalf("view = %+v, err = %v", view, err)
+	}
+	if _, err := reader.GetRun(context.Background(), "missing"); err != contract.ErrNotFound {
+		t.Fatalf("error = %v", err)
+	}
+}

internal/fullsync/httpapi/server.go

@@ -0,0 +1,119 @@
+package httpapi
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+// Config is deliberately independent of worker/config-file wiring. Addr must
+// be explicit; use 127.0.0.1:0 to let the OS allocate a local test port.
+type Config struct {
+	Addr            string
+	BearerToken     string // Optional for standalone callers; production Worker leaves it empty.
+	ShutdownTimeout time.Duration
+}
+
+// Server is single-use. Start reports bind errors synchronously; after a
+// successful Start, Wait reports serve or shutdown errors. Canceling the Start
+// context stops accepting requests and drains active requests up to the timeout.
+type Server struct {
+	server          *http.Server
+	shutdownTimeout time.Duration
+	mu              sync.Mutex
+	started         bool
+	addr            net.Addr
+	done            chan struct{}
+	err             error
+}
+
+func NewServer(config Config, deps Dependencies) (*Server, error) {
+	if config.Addr == "" || config.ShutdownTimeout < 0 {
+		return nil, gerror.New("full-sync HTTP requires an address and a nonnegative shutdown timeout")
+	}
+	handler, err := NewHandler(config.BearerToken, deps)
+	if err != nil {
+		return nil, err
+	}
+	if config.ShutdownTimeout == 0 {
+		config.ShutdownTimeout = 10 * time.Second
+	}
+	return &Server{
+		server: &http.Server{
+			Addr: config.Addr, Handler: handler,
+			ReadHeaderTimeout: 5 * time.Second,
+			ReadTimeout:       30 * time.Second,
+			WriteTimeout:      30 * time.Second,
+			IdleTimeout:       60 * time.Second,
+		},
+		shutdownTimeout: config.ShutdownTimeout,
+		done:            make(chan struct{}),
+	}, nil
+}
+
+func (s *Server) Start(ctx context.Context) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.started {
+		return gerror.New("full-sync HTTP server has already been started")
+	}
+	s.started = true
+	if err := ctx.Err(); err != nil {
+		s.err = err
+		close(s.done)
+		return err
+	}
+	listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", s.server.Addr)
+	if err != nil {
+		s.err = gerror.Wrap(err, "listen for full-sync HTTP")
+		close(s.done)
+		return s.err
+	}
+	s.addr = listener.Addr()
+	go s.run(ctx, listener)
+	return nil
+}
+
+// Addr returns the bound address, or nil before a successful Start.
+func (s *Server) Addr() net.Addr {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.addr
+}
+
+// Wait may be called concurrently after Start (including a failed Start).
+func (s *Server) Wait() error {
+	<-s.done
+	return s.err
+}
+
+func (s *Server) run(ctx context.Context, listener net.Listener) {
+	serveDone := make(chan error, 1)
+	go func() { serveDone <- s.server.Serve(listener) }()
+	var err error
+	select {
+	case err = <-serveDone:
+		// An unexpected listener failure also needs to drain active requests.
+	case <-ctx.Done():
+	}
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), s.shutdownTimeout)
+	shutdownErr := s.server.Shutdown(shutdownCtx)
+	cancel()
+	if shutdownErr != nil {
+		// Shutdown does not close active connections when its deadline expires.
+		_ = s.server.Close()
+	}
+	if err == nil {
+		err = <-serveDone
+	}
+	if errors.Is(err, http.ErrServerClosed) {
+		err = nil
+	}
+	s.err = errors.Join(err, shutdownErr)
+	close(s.done)
+}

internal/fullsync/httpapi/server_test.go

@@ -0,0 +1,226 @@
+package httpapi
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+func awaitServer(t *testing.T, done <-chan error) error {
+	t.Helper()
+	select {
+	case err := <-done:
+		return err
+	case <-time.After(5 * time.Second):
+		t.Fatal("server did not stop")
+		return nil
+	}
+}
+
+func startTestServer(t *testing.T, deps Dependencies, timeout time.Duration) (*Server, context.CancelFunc, <-chan error) {
+	t.Helper()
+	server, err := NewServer(Config{Addr: "127.0.0.1:0", BearerToken: testToken, ShutdownTimeout: timeout}, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if server.Addr() != nil {
+		t.Fatal("address before Start")
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	if err := server.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+	done := make(chan error, 1)
+	go func() { done <- server.Wait() }()
+	return server, cancel, done
+}
+
+func serverRequest(server *Server) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, "http://"+server.Addr().String()+runPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	client := &http.Client{Timeout: 4 * time.Second}
+	return client.Do(req)
+}
+
+func TestServerStartServeAndCancel(t *testing.T) {
+	server, cancel, done := startTestServer(t, testDependencies(), 0)
+	response, err := serverRequest(server)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = response.Body.Close()
+	if response.StatusCode != 404 {
+		t.Fatalf("status = %d", response.StatusCode)
+	}
+	if err := server.Start(context.Background()); err == nil {
+		t.Fatal("duplicate Start succeeded")
+	}
+	cancel()
+	if err := awaitServer(t, done); err != nil {
+		t.Fatal(err)
+	}
+	if err := server.Wait(); err != nil {
+		t.Fatal(err)
+	}
+	connection, err := net.DialTimeout("tcp", server.Addr().String(), time.Second)
+	if err == nil {
+		_ = connection.Close()
+		t.Fatal("listener still accepts after shutdown")
+	}
+}
+
+func TestServerGracefulShutdownDrainsActiveRequest(t *testing.T) {
+	var (
+		entered = make(chan struct{})
+		release = make(chan struct{})
+		once    sync.Once
+	)
+	t.Cleanup(func() { once.Do(func() { close(release) }) })
+	deps := testDependencies()
+	deps.Runs = runFunc(func(ctx context.Context, id string) (contract.RunView, error) {
+		close(entered)
+		select {
+		case <-release:
+			return contract.RunView{Source: contract.SourceLive, Run: contract.RunMeta{RunID: id}}, nil
+		case <-ctx.Done():
+			return contract.RunView{}, ctx.Err()
+		}
+	})
+	server, cancel, done := startTestServer(t, deps, 2*time.Second)
+	requestDone := make(chan error, 1)
+	go func() {
+		response, err := serverRequest(server)
+		if err == nil {
+			_, err = io.ReadAll(response.Body)
+			_ = response.Body.Close()
+			if response.StatusCode != 200 {
+				err = errors.New("active request did not complete successfully")
+			}
+		}
+		requestDone <- err
+	}()
+	select {
+	case <-entered:
+	case <-time.After(3 * time.Second):
+		t.Fatal("request did not enter handler")
+	}
+	cancel()
+	select {
+	case err := <-done:
+		t.Fatalf("shutdown returned before draining: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+	once.Do(func() { close(release) })
+	if err := awaitServer(t, requestDone); err != nil {
+		t.Fatal(err)
+	}
+	if err := awaitServer(t, done); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestServerShutdownDeadlineClosesActiveConnections(t *testing.T) {
+	var (
+		entered = make(chan struct{})
+		exited  = make(chan struct{})
+	)
+	deps := testDependencies()
+	deps.Runs = runFunc(func(ctx context.Context, _ string) (contract.RunView, error) {
+		close(entered)
+		<-ctx.Done()
+		close(exited)
+		return contract.RunView{}, ctx.Err()
+	})
+	server, cancel, done := startTestServer(t, deps, 50*time.Millisecond)
+	requestDone := make(chan error, 1)
+	go func() {
+		response, err := serverRequest(server)
+		if response != nil {
+			_ = response.Body.Close()
+		}
+		requestDone <- err
+	}()
+	select {
+	case <-entered:
+	case <-time.After(3 * time.Second):
+		t.Fatal("request did not enter handler")
+	}
+	cancel()
+	if err := awaitServer(t, done); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("shutdown error = %v", err)
+	}
+	select {
+	case <-exited:
+	case <-time.After(3 * time.Second):
+		t.Fatal("forced close did not cancel request context")
+	}
+	_ = awaitServer(t, requestDone)
+}
+
+func TestServerStartupFailureIsSynchronous(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = listener.Close() })
+	server, err := NewServer(Config{Addr: listener.Addr().String(), BearerToken: testToken}, testDependencies())
+	if err != nil {
+		t.Fatal(err)
+	}
+	startErr := server.Start(context.Background())
+	if startErr == nil {
+		t.Fatal("Start did not report occupied port")
+	}
+	if server.Wait() != startErr || server.Addr() != nil {
+		t.Fatal("failed startup state was lost")
+	}
+	var netErr *net.OpError
+	if !errors.As(startErr, &netErr) {
+		t.Fatalf("network error cause lost: %v", startErr)
+	}
+}
+
+func TestServerCanceledBeforeStartAndImmediateCancellation(t *testing.T) {
+	server, err := NewServer(Config{Addr: "127.0.0.1:0", BearerToken: testToken}, testDependencies())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := server.Start(ctx); !errors.Is(err, context.Canceled) {
+		t.Fatalf("Start = %v", err)
+	}
+	if err := server.Wait(); !errors.Is(err, context.Canceled) {
+		t.Fatalf("Wait = %v", err)
+	}
+	// Cancellation can win the race before Serve starts accepting requests.
+	for i := 0; i < 10; i++ {
+		_, cancel, done := startTestServer(t, testDependencies(), time.Second)
+		cancel()
+		if err := awaitServer(t, done); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestNewServerRejectsInvalidConfiguration(t *testing.T) {
+	for _, config := range []Config{
+		{BearerToken: testToken},
+		{Addr: "127.0.0.1:0", BearerToken: testToken, ShutdownTimeout: -time.Second},
+	} {
+		if _, err := NewServer(config, testDependencies()); err == nil {
+			t.Fatalf("accepted config %+v", config)
+		}
+	}
+}

internal/fullsync/httpapi/simple_replay_test.go

@@ -0,0 +1,59 @@
+package httpapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/runtime"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayHTTPPrivateNetworkAcceptance(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := zset.New(redis, "sim-status:")
+	service, _ := runtime.NewSimpleReplay(store)
+	ctx := context.Background()
+	key, _ := store.RunMetaKey("source")
+	failed, _ := store.FailedKey("source")
+	_, _ = redis.Do(ctx, "HSET", key, "runId", "source", "status", "completed_with_failures", "producerDone", "1", "currentRound", "1")
+	record := zset.FailureRecord{TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "ch", TempKey: "template"}
+	value, _ := json.Marshal(record)
+	_, _ = redis.Do(ctx, "HSET", failed, "task", string(value))
+	deps := testDependencies()
+	deps.Simple = service
+	handler, err := NewHandler("", deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := httptest.NewRequest(http.MethodPost, "/internal/v1/full-sync-runs/source/replay", bytes.NewBufferString(`{"requestedBy":"42"}`))
+	w := httptest.NewRecorder()
+	handler.ServeHTTP(w, req)
+	if w.Code != 202 {
+		t.Fatalf("accepted: %d %s", w.Code, w.Body.String())
+	}
+	var receipt runtime.ReplayReceipt
+	if err := json.Unmarshal(w.Body.Bytes(), &receipt); err != nil || receipt.ReplayRunID == "" {
+		t.Fatalf("receipt: %+v %v", receipt, err)
+	}
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		w := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodGet, "/internal/v1/full-sync-replays/"+receipt.ReplayRunID, nil)
+		handler.ServeHTTP(w, req)
+		if w.Code == 200 {
+			var status runtime.ReplayStatus
+			_ = json.Unmarshal(w.Body.Bytes(), &status)
+			if status.Status == "completed" && status.Enqueued == 1 {
+				return
+			}
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("replay HTTP never completed")
+}

internal/fullsync/runtime/catalog.go

@@ -0,0 +1,211 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+)
+
+const (
+	defaultRunListLimit = 50
+	maxRunListLimit     = 100
+)
+
+// Catalog aggregates the flow and sim-status Redis namespaces while keeping
+// each run's storage protocol inside its existing Repository.
+type Catalog struct {
+	repositories []*Repository
+}
+
+func NewCatalog(repositories ...*Repository) (*Catalog, error) {
+	if len(repositories) == 0 {
+		return nil, errors.New("full-sync catalog requires repositories")
+	}
+	for _, repository := range repositories {
+		if repository == nil {
+			return nil, errors.New("full-sync catalog repository is nil")
+		}
+	}
+	return &Catalog{repositories: repositories}, nil
+}
+
+func (c *Catalog) GetRun(ctx context.Context, runID string) (contract.RunView, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunView{}, err
+	}
+	for _, repository := range c.repositories {
+		meta, err := repository.GetRunMeta(ctx, runID)
+		if err == nil {
+			rounds, roundErr := repository.GetRoundStats(ctx, runID)
+			if roundErr != nil {
+				return contract.RunView{}, roundErr
+			}
+			return contract.RunView{Source: contract.SourceLive, Run: meta, Rounds: rounds}, nil
+		}
+		if !errors.Is(err, contract.ErrNotFound) {
+			return contract.RunView{}, err
+		}
+	}
+	for _, repository := range c.repositories {
+		archive, err := repository.GetArchivedRun(ctx, runID)
+		if err == nil {
+			return contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}, nil
+		}
+		if !errors.Is(err, contract.ErrNotFound) {
+			return contract.RunView{}, err
+		}
+	}
+	return contract.RunView{}, contract.ErrNotFound
+}
+
+func (c *Catalog) ListRuns(ctx context.Context, filter contract.RunListFilter) (contract.RunList, error) {
+	if filter.Limit == 0 {
+		filter.Limit = defaultRunListLimit
+	}
+	if filter.Limit < 1 || filter.Limit > maxRunListLimit {
+		return contract.RunList{}, contract.ErrInvalidArgument
+	}
+	byID := make(map[string]contract.RunView)
+	for _, repository := range c.repositories {
+		active, err := repository.store.ActiveRuns(ctx)
+		if err != nil {
+			return contract.RunList{}, err
+		}
+		for _, runID := range active {
+			view, getErr := repository.liveRun(ctx, runID)
+			if errors.Is(getErr, contract.ErrNotFound) {
+				continue
+			}
+			if getErr != nil {
+				return contract.RunList{}, getErr
+			}
+			byID[runID] = view
+		}
+	}
+	if err := c.listArchives(ctx, byID); err != nil {
+		return contract.RunList{}, err
+	}
+	items := make([]contract.RunView, 0, len(byID))
+	for _, item := range byID {
+		kind := contract.SyncKindForOperation(item.Run.Operation)
+		if kind == "" || (filter.SyncKind != "" && kind != filter.SyncKind) {
+			continue
+		}
+		items = append(items, item)
+	}
+	sort.SliceStable(items, func(i, j int) bool {
+		return runSortTime(items[i].Run).After(runSortTime(items[j].Run))
+	})
+	total := len(items)
+	if len(items) > filter.Limit {
+		items = items[:filter.Limit]
+	}
+	if items == nil {
+		items = []contract.RunView{}
+	}
+	return contract.RunList{Items: items, Total: total}, nil
+}
+
+func (c *Catalog) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.FailurePage{}, err
+	}
+	for _, repository := range c.repositories {
+		meta, err := repository.GetRunMeta(ctx, runID)
+		if err == nil {
+			result, listErr := repository.ListFailures(ctx, runID, page)
+			if listErr != nil {
+				return contract.FailurePage{}, listErr
+			}
+			if meta.FailedCount > 0 && len(result.Items) == 0 && result.NextCursor == "" {
+				failedKey, _ := repository.store.FailedKey(runID)
+				size, sizeErr := repository.redis.Do(ctx, "HLEN", failedKey)
+				if sizeErr != nil {
+					return contract.FailurePage{}, sizeErr
+				}
+				if size.Int64() == 0 {
+					return contract.FailurePage{}, contract.ErrFailureDetailsUnavailable
+				}
+			}
+			return result, nil
+		} else if !errors.Is(err, contract.ErrNotFound) {
+			return contract.FailurePage{}, err
+		}
+	}
+	view, err := c.GetRun(ctx, runID)
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	if view.Run.FailedCount == 0 {
+		return contract.FailurePage{Items: []contract.Failure{}}, nil
+	}
+	for _, repository := range c.repositories {
+		failedKey, _ := repository.store.FailedKey(runID)
+		size, sizeErr := repository.redis.Do(ctx, "HLEN", failedKey)
+		if sizeErr != nil {
+			return contract.FailurePage{}, sizeErr
+		}
+		if size.Int64() > 0 {
+			return repository.ListFailures(ctx, runID, page)
+		}
+	}
+	return contract.FailurePage{}, contract.ErrFailureDetailsUnavailable
+}
+
+func (c *Catalog) listArchives(ctx context.Context, byID map[string]contract.RunView) error {
+	root := c.repositories[0].root
+	if strings.TrimSpace(root) == "" {
+		return nil
+	}
+	matches, err := filepath.Glob(filepath.Join(root, "run-*.json"))
+	if err != nil {
+		return err
+	}
+	for _, path := range matches {
+		data, readErr := os.ReadFile(path)
+		if readErr != nil {
+			g.Log().Warningf(ctx, "event=full_sync_list_archive status=skip path=%s error=%q", path, readErr.Error())
+			continue
+		}
+		var archive contract.RunArchive
+		if decodeErr := json.Unmarshal(data, &archive); decodeErr != nil || archive.Run.RunID == "" || filepath.Base(path) != "run-"+archive.Run.RunID+".json" {
+			g.Log().Warningf(ctx, "event=full_sync_list_archive status=skip path=%s error=%v", path, decodeErr)
+			continue
+		}
+		if _, exists := byID[archive.Run.RunID]; !exists {
+			byID[archive.Run.RunID] = contract.RunView{Source: contract.SourceArchive, Run: archive.Run, Rounds: archive.Rounds}
+		}
+	}
+	return nil
+}
+
+func (r *Repository) liveRun(ctx context.Context, runID string) (contract.RunView, error) {
+	meta, err := r.GetRunMeta(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	rounds, err := r.GetRoundStats(ctx, runID)
+	if err != nil {
+		return contract.RunView{}, err
+	}
+	return contract.RunView{Source: contract.SourceLive, Run: meta, Rounds: rounds}, nil
+}
+
+func runSortTime(run contract.RunMeta) (value time.Time) {
+	if run.StartedAt != nil && !run.StartedAt.IsZero() {
+		return *run.StartedAt
+	}
+	if !run.CompletedAt.IsZero() {
+		return run.CompletedAt
+	}
+	return run.CreatedAt
+}

internal/fullsync/runtime/catalog_test.go

@@ -0,0 +1,123 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type catalogRedisStub struct {
+	responses []*gvar.Var
+	calls     []string
+}
+
+func (s *catalogRedisStub) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
+	s.calls = append(s.calls, command)
+	if len(s.responses) == 0 {
+		return gvar.New(nil), nil
+	}
+	response := s.responses[0]
+	s.responses = s.responses[1:]
+	return response, nil
+}
+
+func (s *catalogRedisStub) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	return gvar.New(int64(1)), nil
+}
+
+func TestCatalogListRunsMergesLiveAndArchive(t *testing.T) {
+	root := t.TempDir()
+	now := time.Date(2026, 9, 28, 1, 0, 0, 0, time.UTC)
+	archive := contract.RunArchive{Version: 1, Run: contract.RunMeta{
+		RunID: "archived", Operation: "sim_status_queue_full_sync", Status: contract.RunCompleted,
+		CreatedAt: now.Add(-time.Hour), CompletedAt: now.Add(-time.Minute),
+	}}
+	data, err := json.Marshal(archive)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "run-archived.json"), data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	metaResponse := gvar.New([]any{"runId", "live", "operation", "current_month_usage_full_sync", "status", "running", "currentRound", "1", "createdAt", now.Format(time.RFC3339Nano), "roundStats", "[]"})
+	redis := &catalogRedisStub{responses: []*gvar.Var{
+		gvar.New([]any{"live"}), metaResponse, metaResponse,
+		gvar.New([]any{"live"}), metaResponse, metaResponse,
+	}}
+	store, err := zset.New(redis, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	catalog, err := NewCatalog(repository)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := catalog.ListRuns(context.Background(), contract.RunListFilter{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Total != 2 || len(result.Items) != 2 {
+		t.Fatalf("result = %+v", result)
+	}
+	if result.Items[0].Run.RunID != "live" || result.Items[1].Run.RunID != "archived" {
+		t.Fatalf("items = %+v", result.Items)
+	}
+	filtered, err := catalog.ListRuns(context.Background(), contract.RunListFilter{SyncKind: contract.SyncKindTrafficUsage, Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if filtered.Total != 1 || filtered.Items[0].Run.RunID != "live" {
+		t.Fatalf("filtered = %+v", filtered)
+	}
+	view, err := catalog.GetRun(context.Background(), "archived")
+	if err != nil || view.Source != contract.SourceArchive || view.Run.RunID != "archived" {
+		t.Fatalf("archived view = %+v, err = %v", view, err)
+	}
+}
+
+func TestCatalogPrefersLiveAndSkipsBrokenArchive(t *testing.T) {
+	root := t.TempDir()
+	for name, payload := range map[string]string{
+		"run-live.json":   `{"version":1,"run":{"runId":"live","operation":"sim_status_queue_full_sync","status":"completed"}}`,
+		"run-broken.json": `{not-json}`,
+	} {
+		if err := os.WriteFile(filepath.Join(root, name), []byte(payload), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	redis := &catalogRedisStub{responses: []*gvar.Var{
+		gvar.New([]any{"live"}),
+		gvar.New([]any{"runId", "live", "operation", "sim_status_queue_full_sync", "status", "running"}),
+		gvar.New([]any{"runId", "live", "operation", "sim_status_queue_full_sync", "status", "running"}),
+	}}
+	store, err := zset.New(redis, "sim-status:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	catalog, err := NewCatalog(repository)
+	if err != nil {
+		t.Fatal(err)
+	}
+	result, err := catalog.ListRuns(context.Background(), contract.RunListFilter{Limit: 10})
+	if err != nil || result.Total != 1 || result.Items[0].Source != contract.SourceLive || result.Items[0].Run.Status != contract.RunRunning {
+		t.Fatalf("result = %+v, err = %v", result, err)
+	}
+}

internal/fullsync/runtime/runtime.go

@@ -0,0 +1,497 @@
+// Package runtime provides the small concrete storage adapters used by the
+// full-sync HTTP and archive services. It intentionally keeps queue protocol
+// transitions in queue/zset and only reads or records durable metadata here.
+package runtime
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+	"github.com/gogf/gf/v2/frame/g"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type Repository struct {
+	store *zset.Store
+	redis zset.RedisClient
+	keys  contract.Keys
+	root  string
+}
+
+func (r *Repository) GetRoundStats(ctx context.Context, runID string) ([]contract.RoundStats, error) {
+	stats, err := r.store.GetRoundStats(ctx, runID)
+	if err != nil {
+		return nil, wrapNotFound(err)
+	}
+	out := make([]contract.RoundStats, 0, len(stats))
+	for _, stat := range stats {
+		out = append(out, contract.RoundStats{
+			RunID:                stat.RunID,
+			Round:                stat.Round,
+			EnqueuedCount:        stat.EnqueuedCount,
+			ActiveCount:          stat.ActiveCount,
+			NextRoundCount:       stat.NextRoundCount,
+			SuccessCount:         stat.SuccessCount,
+			FailedCount:          stat.FailedCount,
+			UpstreamAttemptCount: stat.UpstreamAttemptCount,
+			BatchCount:           stat.BatchCount,
+			StartedAt:            stat.StartedAt,
+			CompletedAt:          stat.CompletedAt,
+		})
+	}
+	return out, nil
+}
+
+func (r *Repository) RequestReplay(ctx context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
+	if err := req.Validate(); err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	fp, err := req.Fingerprint()
+	if err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	key, err := r.keys.ReplayIdempotencyKey(req.SourceRunID, req.IdempotencyKey)
+	if err != nil {
+		return contract.ReplayAccepted{}, err
+	}
+	actionID := "replay-" + fp[:24]
+	actionKey, _ := r.keys.ReplayActionKey(actionID)
+	targetID := "replay-run-" + fp[:24]
+	now := time.Now().UTC()
+	action := contract.ReplayAction{ActionID: actionID, SourceRunID: req.SourceRunID, TargetRunID: targetID, RequestedBy: req.RequestedBy, IdempotencyKey: req.IdempotencyKey, Fingerprint: fp, Status: contract.ActionAccepted, ReplayCycle: 1, Progress: contract.ReplayProgress{}, CreatedAt: now, UpdatedAt: now}
+	b, _ := json.Marshal(action)
+	// Reserve idempotency first. A repeat reads the original action; a changed
+	// request under the same key is rejected by fingerprint comparison.
+	set, e := r.redis.Do(ctx, "SET", key, actionID, "NX", "EX", 604800)
+	if e != nil {
+		return contract.ReplayAccepted{}, e
+	}
+	if set.String() == "" || strings.EqualFold(set.String(), "nil") {
+		reserved, e := r.redis.Do(ctx, "GET", key)
+		if e != nil {
+			return contract.ReplayAccepted{}, e
+		}
+		reservedID := reserved.String()
+		old, e := r.GetReplayAction(ctx, reservedID)
+		if e != nil {
+			return contract.ReplayAccepted{}, e
+		}
+		if old.Fingerprint != fp {
+			return contract.ReplayAccepted{}, contract.ErrConflict
+		}
+		u, _ := contract.ActionStatusURL(old.ActionID)
+		return contract.ReplayAccepted{ActionID: old.ActionID, TargetRunID: old.TargetRunID, Status: old.Status, StatusURL: u}, nil
+	}
+	if _, e = r.redis.Do(ctx, "SET", actionKey, string(b), "EX", 604800); e != nil {
+		return contract.ReplayAccepted{}, e
+	}
+	_, _ = r.redis.Do(ctx, "SADD", r.keys.PendingReplayActionsKey(), actionID)
+	go r.executeReplay(actionID)
+	u, _ := contract.ActionStatusURL(actionID)
+	return contract.ReplayAccepted{ActionID: actionID, TargetRunID: targetID, Status: action.Status, StatusURL: u}, nil
+}
+
+// executeReplay is deliberately small: it snapshots the source failure hash,
+// enqueues equivalent tasks into one target run, then removes only entries
+// that were accepted. Redis action JSON is the durable retry checkpoint.
+func (r *Repository) executeReplay(actionID string) {
+	ctx := context.Background()
+	action, err := r.GetReplayAction(ctx, actionID)
+	if err != nil {
+		return
+	}
+	action.Status = contract.ActionProcessing
+	action.UpdatedAt = time.Now().UTC()
+	_ = r.saveAction(ctx, action)
+	source, err := r.GetRunMeta(ctx, action.SourceRunID)
+	if err != nil {
+		action.Error = err.Error()
+		action.Status = contract.ActionNeedsAttention
+		action.UpdatedAt = time.Now().UTC()
+		_ = r.saveAction(ctx, action)
+		return
+	}
+	target := zset.RunMeta{RunID: action.TargetRunID, Operation: source.Operation, QueryMonth: source.QueryMonth, Status: "producing", CurrentRound: 1, CreatedAt: time.Now().UTC(), SyncType: string(contract.SyncReplay), SourceRunID: action.SourceRunID, ReplayCycle: action.ReplayCycle, TriggerSource: "replay"}
+	if err := r.store.CreateRun(ctx, target); err != nil {
+		if _, getErr := r.store.GetRunMeta(ctx, target.RunID); getErr != nil {
+			action.Error = err.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+	}
+	fkey, _ := r.store.FailedKey(action.SourceRunID)
+	var records []struct {
+		field string
+		rec   zset.FailureRecord
+	}
+	var cursor uint64
+	for {
+		v, e := r.redis.Do(ctx, "HSCAN", fkey, cursor, "COUNT", 500)
+		if e != nil {
+			action.Error = e.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+		cursor, pairs, e := parseScanResult(v)
+		if e != nil {
+			action.Error = e.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+		for i := 0; i < len(pairs); i += 2 {
+			var rec zset.FailureRecord
+			if json.Unmarshal([]byte(pairs[i+1].String()), &rec) == nil {
+				records = append(records, struct {
+					field string
+					rec   zset.FailureRecord
+				}{pairs[i].String(), rec})
+			}
+		}
+		if cursor == 0 {
+			break
+		}
+	}
+	groups := map[string][]zset.Task{}
+	for _, item := range records {
+		rec := item.rec
+		chanKey := rec.ChanKey
+		if chanKey == "" {
+			chanKey = "default"
+		}
+		groups[chanKey] = append(groups[chanKey], zset.Task{TaskID: action.TargetRunID + ":" + rec.TaskID, RunID: action.TargetRunID, Operation: rec.Operation, QueryMonth: rec.QueryMonth, ChanKey: chanKey, TempKey: rec.TempKey, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI})
+	}
+	for chanKey, tasks := range groups {
+		if _, err := r.store.EnqueueRunBatch(ctx, action.TargetRunID, chanKey, tasks, time.Now()); err != nil {
+			action.Error = err.Error()
+			action.Status = contract.ActionNeedsAttention
+			action.UpdatedAt = time.Now().UTC()
+			_ = r.saveAction(ctx, action)
+			return
+		}
+	}
+	for _, item := range records {
+		_, _ = r.redis.Do(ctx, "HDEL", fkey, item.field)
+	}
+	_ = r.store.MarkProducerDone(ctx, action.TargetRunID)
+	action.Progress = contract.ReplayProgress{DiscoveredCount: int64(len(records)), EnqueuedCount: int64(len(records)), CleanedCount: int64(len(records)), FailureCount: int64(len(records))}
+	action.Status = contract.ActionCompleted
+	action.UpdatedAt = time.Now().UTC()
+	completed := action.UpdatedAt
+	action.CompletedAt = &completed
+	digest := sha256.Sum256([]byte(action.ActionID + action.TargetRunID))
+	action.Receipt = &contract.ArchiveReceipt{Version: 1, Path: "replay://" + action.ActionID, Checksum: hex.EncodeToString(digest[:]), ArchivedAt: completed}
+	_ = r.saveAction(ctx, action)
+}
+
+func (r *Repository) saveAction(ctx context.Context, action contract.ReplayAction) error {
+	b, err := json.Marshal(action)
+	if err != nil {
+		return err
+	}
+	key, err := r.keys.ReplayActionKey(action.ActionID)
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "SET", key, string(b), "EX", 604800)
+	return err
+}
+
+func (r *Repository) GetReplayAction(ctx context.Context, actionID string) (contract.ReplayAction, error) {
+	if err := contract.ValidateID(actionID); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	key, _ := r.keys.ReplayActionKey(actionID)
+	v, err := r.redis.Do(ctx, "GET", key)
+	if err != nil {
+		return contract.ReplayAction{}, err
+	}
+	if v == nil || v.IsNil() || strings.TrimSpace(v.String()) == "" {
+		return contract.ReplayAction{}, contract.ErrNotFound
+	}
+	var action contract.ReplayAction
+	if err := json.Unmarshal([]byte(v.String()), &action); err != nil {
+		return contract.ReplayAction{}, err
+	}
+	return action, nil
+}
+
+func New(store *zset.Store, archiveRoot string) (*Repository, error) {
+	if store == nil {
+		return nil, errors.New("full-sync runtime store is nil")
+	}
+	keys, err := contract.NewKeys(store.Prefix())
+	if err != nil {
+		return nil, err
+	}
+	return &Repository{store: store, redis: store.RedisClient(), keys: keys, root: archiveRoot}, nil
+}
+
+func (r *Repository) GetRunMeta(ctx context.Context, runID string) (contract.RunMeta, error) {
+	m, err := r.store.GetRunMeta(ctx, runID)
+	if err != nil {
+		return contract.RunMeta{}, wrapNotFound(err)
+	}
+	return toContractMeta(m), nil
+}
+
+func (r *Repository) GetArchivedRun(ctx context.Context, runID string) (contract.RunArchive, error) {
+	if err := contract.ValidateID(runID); err != nil {
+		return contract.RunArchive{}, err
+	}
+	if strings.TrimSpace(r.root) == "" {
+		return contract.RunArchive{}, contract.ErrNotFound
+	}
+	b, err := os.ReadFile(filepath.Join(r.root, "run-"+runID+".json"))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return contract.RunArchive{}, contract.ErrNotFound
+		}
+		return contract.RunArchive{}, err
+	}
+	var archive contract.RunArchive
+	if err := json.Unmarshal(b, &archive); err != nil {
+		return contract.RunArchive{}, fmt.Errorf("decode run archive: %w", err)
+	}
+	if archive.Run.RunID != runID {
+		return contract.RunArchive{}, fmt.Errorf("archive run ID does not match filename")
+	}
+	return archive, nil
+}
+
+func (r *Repository) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) {
+	p, err := page.Normalized()
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	key, err := r.store.FailedKey(runID)
+	if err != nil {
+		return contract.FailurePage{}, err
+	}
+	start := uint64(0)
+	if p.Cursor != "" {
+		start, err = decodeCursor(p.Cursor, runID)
+		if err != nil {
+			return contract.FailurePage{}, err
+		}
+	}
+	var items []contract.Failure
+	var next uint64
+	for {
+		v, e := r.redis.Do(ctx, "HSCAN", key, start, "COUNT", p.Limit)
+		if e != nil {
+			return contract.FailurePage{}, e
+		}
+		next, pairs, e := parseScanResult(v)
+		if e != nil {
+			return contract.FailurePage{}, e
+		}
+		for i := 1; i < len(pairs); i += 2 {
+			var rec zset.FailureRecord
+			if json.Unmarshal([]byte(pairs[i].String()), &rec) == nil {
+				items = append(items, toFailure(rec))
+				if len(items) >= p.Limit {
+					break
+				}
+			}
+		}
+		if len(items) >= p.Limit || next == 0 {
+			break
+		}
+		start = next
+	}
+	result := contract.FailurePage{Items: items}
+	if next != 0 {
+		result.NextCursor = encodeCursor(runID, next)
+	}
+	if result.Items == nil {
+		result.Items = []contract.Failure{}
+	}
+	return result, nil
+}
+
+func (r *Repository) ListArchiveWork(ctx context.Context, page contract.PageRequest) (contract.ArchiveWorkPage, error) {
+	p, err := page.Normalized()
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	start := uint64(0)
+	if p.Cursor != "" {
+		start, err = decodeCursor(p.Cursor, "archive")
+		if err != nil {
+			return contract.ArchiveWorkPage{}, err
+		}
+	}
+	v, err := r.redis.Do(ctx, "ZSCAN", r.keys.PendingArchivesKey(), start, "COUNT", p.Limit)
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	out := contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}
+	next, pairs, err := parseScanResult(v)
+	if err != nil {
+		return contract.ArchiveWorkPage{}, err
+	}
+	for i := 0; i < len(pairs); i += 2 {
+		out.Items = append(out.Items, contract.ArchiveWork{RunID: pairs[i].String(), Version: pairs[i+1].String()})
+	}
+	if next != 0 {
+		out.NextCursor = encodeCursor("archive", next)
+	}
+	return out, nil
+}
+
+func parseScanResult(value *gvar.Var) (uint64, []*gvar.Var, error) {
+	if value == nil {
+		return 0, nil, errors.New("redis scan returned nil response")
+	}
+	values := value.Vars()
+	if len(values) != 2 {
+		return 0, nil, fmt.Errorf("unexpected Redis scan response length: %d", len(values))
+	}
+	pairs := values[1].Vars()
+	if len(pairs)%2 != 0 {
+		return 0, nil, fmt.Errorf("redis scan returned odd number of entry values: %d", len(pairs))
+	}
+	return uint64(values[0].Int64()), pairs, nil
+}
+
+func (r *Repository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
+	if archived, err := r.GetArchivedRun(ctx, work.RunID); err == nil {
+		return archived, nil
+	}
+	meta, err := r.GetRunMeta(ctx, work.RunID)
+	if err != nil {
+		return contract.RunArchive{}, err
+	}
+	rounds, err := r.GetRoundStats(ctx, work.RunID)
+	if err != nil {
+		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=read error=%q", work.RunID, err.Error())
+		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
+	}
+	if err := validateRoundAccounting(meta, rounds); err != nil {
+		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=validate error=%q", work.RunID, err.Error())
+		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
+	}
+	return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: rounds}, nil
+}
+
+func validateRoundAccounting(meta contract.RunMeta, rounds []contract.RoundStats) error {
+	if len(rounds) == 0 {
+		return nil
+	}
+	var success, failed int64
+	seen := make(map[int]struct{}, len(rounds))
+	for _, round := range rounds {
+		if round.RunID != meta.RunID {
+			return fmt.Errorf("round %d belongs to run %q", round.Round, round.RunID)
+		}
+		if _, ok := seen[round.Round]; ok {
+			return fmt.Errorf("duplicate round %d", round.Round)
+		}
+		seen[round.Round] = struct{}{}
+		if round.EnqueuedCount != round.SuccessCount+round.NextRoundCount+round.FailedCount {
+			return fmt.Errorf("round %d counters do not close", round.Round)
+		}
+		success += round.SuccessCount
+		failed += round.FailedCount
+	}
+	if success != meta.SuccessCount || failed != meta.FailedCount {
+		return fmt.Errorf("run counters do not match round totals: success=%d/%d failed=%d/%d", success, meta.SuccessCount, failed, meta.FailedCount)
+	}
+	return nil
+}
+
+func (r *Repository) RegisterArchiveWork(ctx context.Context, work contract.ArchiveWork) error {
+	if err := contract.ValidateID(work.RunID); err != nil {
+		return err
+	}
+	if strings.TrimSpace(work.Version) == "" {
+		return contract.ErrInvalidArgument
+	}
+	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), work.Version, work.RunID)
+	return err
+}
+
+func (r *Repository) RegisterArchive(ctx context.Context, work contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
+	if err := receipt.Validate(); err != nil {
+		return err
+	}
+	key, err := r.keys.ArchiveStateKey(work.RunID)
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "HSET", key, "version", receipt.Version, "path", receipt.Path, "checksum", receipt.Checksum, "archivedAt", receipt.ArchivedAt.UTC().Format(time.RFC3339Nano))
+	if err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "ZREM", r.keys.PendingArchivesKey(), work.RunID)
+	return err
+}
+
+func (r *Repository) RegisterArchiveFailure(ctx context.Context, work contract.ArchiveWork, message string) error {
+	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), time.Now().UnixMilli(), work.RunID)
+	return err
+}
+
+func toContractMeta(m zset.RunMeta) contract.RunMeta {
+	syncType := contract.SyncType(m.SyncType)
+	if syncType == "" {
+		syncType = contract.SyncNormal
+	}
+	return contract.RunMeta{SupplyObservation: m.SupplyObservation, RunID: m.RunID, Operation: m.Operation, QueryMonth: m.QueryMonth, Status: contract.RunStatus(m.Status), ProducerDone: m.ProducerDone, CurrentRound: m.CurrentRound, ActiveCount: m.ActiveCount, InflightCount: m.InflightCount, NextRoundCount: m.NextRoundCount, SuccessCount: m.SuccessCount, FailedCount: m.FailedCount, CreatedAt: m.CreatedAt, ProducerCompletedAt: m.ProducerCompletedAt, CompletedAt: m.CompletedAt, SyncType: syncType, SourceRunID: m.SourceRunID, ReplayCycle: m.ReplayCycle, TriggerSource: m.TriggerSource, ScannedCount: m.ScannedCount, EnqueuedCount: m.EnqueuedCount, UpstreamAttemptCount: m.UpstreamAttemptCount, BatchCount: m.BatchCount, MaxRound: m.MaxRound, PendingFailureCount: m.PendingFailureCount, ArchiveStatus: contract.ArchiveStatus(m.ArchiveStatus), ArchiveVersion: m.ArchiveVersion, ArchivePath: m.ArchivePath, ArchiveChecksum: m.ArchiveChecksum, ReplayDisposition: contract.ReplayDisposition(m.ReplayDisposition), ScopeChannels: m.ScopeChannels, ScopeDigest: m.ScopeDigest, ScopeChannelsSample: m.ScopeChannelsSample, StartedAt: m.StartedAt, ScanCompletedAt: m.ScanCompletedAt, ArchivedAt: m.ArchivedAt}
+}
+func toFailure(f zset.FailureRecord) contract.Failure {
+	return contract.Failure{TaskID: f.TaskID, CardID: f.CardID, ErrorCategory: contract.NormalizeErrorCategory(f.FailureType), ErrorMessage: contract.BoundedErrorMessage(f.Message), FailedAt: f.LastFailedAt, Round: f.Round}
+}
+func wrapNotFound(err error) error {
+	if strings.Contains(strings.ToLower(err.Error()), "not found") {
+		return contract.ErrNotFound
+	}
+	return err
+}
+func encodeCursor(scope string, n uint64) contract.Cursor {
+	return contract.Cursor(base64.RawURLEncoding.EncodeToString([]byte(scope + ":" + strconv.FormatUint(n, 10))))
+}
+func decodeCursor(c contract.Cursor, scope string) (uint64, error) {
+	b, e := base64.RawURLEncoding.DecodeString(string(c))
+	if e != nil {
+		return 0, contract.ErrInvalidArgument
+	}
+	p := strings.Split(string(b), ":")
+	if len(p) != 2 || p[0] != scope {
+		return 0, contract.ErrInvalidArgument
+	}
+	n, e := strconv.ParseUint(p[1], 10, 64)
+	if e != nil {
+		return 0, contract.ErrInvalidArgument
+	}
+	return n, nil
+}
+
+var _ contract.LiveRunReader = (*Repository)(nil)
+var _ contract.ArchivedRunReader = (*Repository)(nil)
+var _ contract.FailurePager = (*Repository)(nil)
+var _ contract.ArchiveWorkDiscovery = (*Repository)(nil)
+var _ contract.ArchiveWorkLoader = (*Repository)(nil)
+var _ contract.ArchiveWorkRegistrar = (*Repository)(nil)
+var _ contract.ArchiveRegistrar = (*Repository)(nil)
+var _ contract.ReplayRequester = (*Repository)(nil)
+var _ contract.ActionReader = (*Repository)(nil)

internal/fullsync/runtime/runtime_test.go

@@ -0,0 +1,143 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+type runtimeRedisStub struct {
+	responses []*gvar.Var
+}
+
+func (s *runtimeRedisStub) Do(context.Context, string, ...any) (*gvar.Var, error) {
+	if len(s.responses) == 0 {
+		return gvar.New(nil), nil
+	}
+	response := s.responses[0]
+	s.responses = s.responses[1:]
+	return response, nil
+}
+
+func (s *runtimeRedisStub) Eval(context.Context, string, int64, []string, []any) (*gvar.Var, error) {
+	return gvar.New(int64(1)), nil
+}
+
+func newRuntimeRepository(t *testing.T, responses ...*gvar.Var) *Repository {
+	t.Helper()
+	store, err := zset.New(&runtimeRedisStub{responses: responses}, "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository, err := New(store, t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return repository
+}
+
+func TestListArchiveWorkParsesNestedRedisScanResult(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"run-1", "17", "run-2", "18"},
+	}))
+
+	page, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 2 {
+		t.Fatalf("archive work count = %d, want 2", len(page.Items))
+	}
+	if page.Items[0].RunID != "run-1" || page.Items[0].Version != "17" {
+		t.Fatalf("first archive work = %+v", page.Items[0])
+	}
+	if page.Items[1].RunID != "run-2" || page.Items[1].Version != "18" {
+		t.Fatalf("second archive work = %+v", page.Items[1])
+	}
+}
+
+func TestGetRoundStatsReadsRunHashJSON(t *testing.T) {
+	raw, err := json.Marshal([]zset.RoundStats{{RunID: "run-1", Round: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"runId", "run-1", "status", "completed", "roundStats", string(raw),
+	}))
+
+	got, err := repository.GetRoundStats(context.Background(), "run-1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Round != 2 || got[0].SuccessCount != 2 || got[0].FailedCount != 1 {
+		t.Fatalf("unexpected round stats: %+v", got)
+	}
+}
+
+func TestValidateRoundAccountingRejectsUnbalancedRound(t *testing.T) {
+	err := validateRoundAccounting(contract.RunMeta{RunID: "run-1", SuccessCount: 1, FailedCount: 0}, []contract.RoundStats{{RunID: "run-1", Round: 1, EnqueuedCount: 2, SuccessCount: 1}})
+	if err == nil {
+		t.Fatal("expected unbalanced round error")
+	}
+}
+
+func TestListArchiveWorkHandlesEmptyRedisScanResult(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{"0", []any{}}))
+
+	page, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 0 {
+		t.Fatalf("archive work = %+v, want empty", page.Items)
+	}
+}
+
+func TestListFailuresParsesNestedRedisScanResult(t *testing.T) {
+	failedAt := time.Date(2026, 9, 21, 6, 0, 0, 0, time.UTC)
+	record := zset.FailureRecord{
+		TaskID:       "task-1",
+		CardID:       42,
+		FailureType:  "business",
+		Message:      "upstream rejected request",
+		Round:        3,
+		LastFailedAt: failedAt,
+	}
+	payload, err := json.Marshal(record)
+	if err != nil {
+		t.Fatal(err)
+	}
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"task-1", string(payload)},
+	}))
+
+	page, err := repository.ListFailures(context.Background(), "run-1", contract.PageRequest{Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(page.Items) != 1 {
+		t.Fatalf("failure count = %d, want 1", len(page.Items))
+	}
+	if page.Items[0].TaskID != "task-1" || page.Items[0].CardID != 42 || page.Items[0].Round != 3 {
+		t.Fatalf("failure = %+v", page.Items[0])
+	}
+}
+
+func TestListArchiveWorkRejectsOddRedisScanEntries(t *testing.T) {
+	repository := newRuntimeRepository(t, gvar.New([]any{
+		"0",
+		[]any{"run-without-score"},
+	}))
+
+	if _, err := repository.ListArchiveWork(context.Background(), contract.PageRequest{Limit: 10}); err == nil {
+		t.Fatal("ListArchiveWork() error = nil, want malformed response error")
+	}
+}

internal/fullsync/runtime/simple_replay.go

@@ -0,0 +1,474 @@
+package runtime
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/container/gvar"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+)
+
+// SimpleReplay hands every source failure to a separate Run. The source hash
+// is the checkpoint: a record is removed only after the target's task-state
+// confirms that EnqueueRunBatch accepted it. No cursor is retained across
+// mutations of the source hash.
+//
+// The target's retry-round budget is taken from the caller when supplied and
+// falls back to the configured default otherwise. It is persisted on the
+// source reservation because a worker restart re-enters through reconcile(),
+// which no longer has the caller's arguments.
+type SimpleReplay struct {
+	stores []*zset.Store
+	redis  zset.RedisClient
+	// defaultMaxRounds applies when a replay request does not specify one.
+	defaultMaxRounds int
+}
+
+// Replay round bounds. MaxReplayRounds is a server-side guard, not a capacity
+// limit: the observed chain of 10-09 needed 74 rounds to reach 98% recovery,
+// so a low ceiling would cut off a legitimate convergence path.
+//
+// DefaultReplayMaxRounds is only the fallback for a request that omits a
+// budget; callers may ask for more, up to MaxReplayRounds.
+const (
+	DefaultReplayMaxRounds = 3
+	MaxReplayRounds        = 100
+	legacyReplayMaxRounds  = 1
+)
+
+type ReplayReceipt struct {
+	ReplayRunID string `json:"replayRunId"`
+	Status      string `json:"status"`
+}
+
+type ReplayStatus struct {
+	ReplayRunID string `json:"replayRunId"`
+	SourceRunID string `json:"sourceRunId"`
+	Status      string `json:"status"`
+	Remaining   int64  `json:"remaining"`
+	Enqueued    int64  `json:"enqueued"`
+	RunStatus   string `json:"runStatus,omitempty"`
+	Error       string `json:"error,omitempty"`
+}
+
+var ErrNoFailures = errors.New("run has no failures to replay")
+
+func NewSimpleReplay(stores ...*zset.Store) (*SimpleReplay, error) {
+	return NewSimpleReplayWithDefault(DefaultReplayMaxRounds, stores...)
+}
+
+// NewSimpleReplayWithDefault builds a replay service whose unspecified replay
+// requests inherit defaultMaxRounds. A non-positive value falls back to
+// DefaultReplayMaxRounds so a missing config key cannot disable the budget.
+func NewSimpleReplayWithDefault(defaultMaxRounds int, stores ...*zset.Store) (*SimpleReplay, error) {
+	if len(stores) == 0 || stores[0] == nil {
+		return nil, fmt.Errorf("replay requires Redis stores")
+	}
+	for _, store := range stores {
+		if store == nil {
+			return nil, fmt.Errorf("replay store is nil")
+		}
+	}
+	if defaultMaxRounds <= 0 {
+		defaultMaxRounds = DefaultReplayMaxRounds
+	}
+	if defaultMaxRounds > MaxReplayRounds {
+		defaultMaxRounds = MaxReplayRounds
+	}
+	return &SimpleReplay{stores: stores, redis: stores[0].RedisClient(), defaultMaxRounds: defaultMaxRounds}, nil
+}
+
+func (r *SimpleReplay) find(ctx context.Context, id string) (*zset.Store, zset.RunMeta, error) {
+	if err := contract.ValidateID(id); err != nil || len(id) > 128 {
+		return nil, zset.RunMeta{}, contract.ErrInvalidArgument
+	}
+	var found *zset.Store
+	var meta zset.RunMeta
+	for _, store := range r.stores {
+		key, _ := store.RunMetaKey(id)
+		v, err := r.redis.Do(ctx, "EXISTS", key)
+		if err != nil {
+			return nil, meta, err
+		}
+		if v.Int() == 0 {
+			continue
+		}
+		if found != nil {
+			return nil, meta, contract.ErrConflict
+		}
+		found = store
+		meta, err = store.GetRunMeta(ctx, id)
+		if err != nil {
+			return nil, meta, err
+		}
+	}
+	if found == nil {
+		return nil, meta, contract.ErrNotFound
+	}
+	return found, meta, nil
+}
+
+// Request accepts a replay of the source run's failures. maxRounds == 0 means
+// "use the configured default"; requests above MaxReplayRounds are rejected.
+func (r *SimpleReplay) Request(ctx context.Context, sourceID, requestedBy string, maxRounds int) (ReplayReceipt, error) {
+	if len(requestedBy) == 0 || len(requestedBy) > 64 {
+		return ReplayReceipt{}, contract.ErrInvalidArgument
+	}
+	if maxRounds < 0 || maxRounds > MaxReplayRounds {
+		return ReplayReceipt{}, contract.ErrInvalidArgument
+	}
+	if maxRounds == 0 {
+		maxRounds = r.defaultMaxRounds
+	}
+	store, _, err := r.find(ctx, sourceID)
+	if err != nil {
+		return ReplayReceipt{}, err
+	}
+	metaKey, _ := store.RunMetaKey(sourceID)
+	failedKey, _ := store.FailedKey(sourceID)
+	var random [16]byte
+	if _, err := rand.Read(random[:]); err != nil {
+		return ReplayReceipt{}, err
+	}
+	targetID := "replay-" + hex.EncodeToString(random[:])
+	// An index written before the reservation makes the pending source
+	// discoverable even if the process stops immediately after the Lua call.
+	if _, err = r.redis.Do(ctx, "SADD", store.Prefix()+"replay:pending", sourceID); err != nil {
+		return ReplayReceipt{}, err
+	}
+	v, err := r.redis.Eval(ctx, reserveReplay, 2, []string{metaKey, failedKey}, []any{targetID, requestedBy, maxRounds})
+	if err != nil {
+		return ReplayReceipt{}, err
+	}
+	result := v.String()
+	// An unsuccessful reservation leaves an inert discovery entry. Never
+	// remove it here: a concurrent caller might have reserved the source
+	// after this script returned, and that reservation needs recovery.
+	switch result {
+	case "missing":
+		return ReplayReceipt{}, contract.ErrNotFound
+	case "running":
+		return ReplayReceipt{}, contract.ErrConflict
+	case "empty":
+		return ReplayReceipt{}, ErrNoFailures
+	}
+	if !strings.HasPrefix(result, "replay-") {
+		return ReplayReceipt{}, fmt.Errorf("unexpected replay reservation result")
+	}
+	go r.process(store, sourceID)
+	return ReplayReceipt{ReplayRunID: result, Status: "accepted"}, nil
+}
+
+// Start reconciles pending sources after restarts and missed goroutine wakes.
+// Every process may run it: the per-source Redis lease elects one writer.
+func (r *SimpleReplay) Start(ctx context.Context) {
+	go func() {
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		for {
+			r.reconcile(ctx)
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+}
+
+func (r *SimpleReplay) reconcile(ctx context.Context) {
+	for _, store := range r.stores {
+		v, err := r.redis.Do(ctx, "SMEMBERS", store.Prefix()+"replay:pending")
+		if err != nil {
+			continue
+		}
+		for _, id := range v.Strings() {
+			if ctx.Err() != nil {
+				return
+			}
+			key, keyErr := store.RunMetaKey(id)
+			if keyErr != nil {
+				continue
+			}
+			reservation, getErr := r.redis.Do(ctx, "HGET", key, "replayTargetId")
+			// SADD happens before the reservation so even a crash during
+			// acceptance cannot lose work. Unreserved entries are inert.
+			if getErr != nil || reservation == nil || reservation.String() == "" {
+				continue
+			}
+			state, stateErr := r.redis.Do(ctx, "HGET", key, "replayState")
+			if stateErr != nil || state == nil || state.String() == "needs_attention" {
+				continue
+			}
+			go r.process(store, id)
+		}
+	}
+}
+
+func (r *SimpleReplay) Status(ctx context.Context, targetID string) (ReplayStatus, error) {
+	store, target, err := r.find(ctx, targetID)
+	if err != nil {
+		// The target may not exist yet: locate its durable source reservation.
+		if !errors.Is(err, contract.ErrNotFound) {
+			return ReplayStatus{}, err
+		}
+		for _, candidate := range r.stores {
+			v, e := r.redis.Do(ctx, "SMEMBERS", candidate.Prefix()+"replay:pending")
+			if e != nil {
+				return ReplayStatus{}, e
+			}
+			for _, sourceID := range v.Strings() {
+				key, _ := candidate.RunMetaKey(sourceID)
+				value, e := r.redis.Do(ctx, "HGET", key, "replayTargetId")
+				if e == nil && value != nil && value.String() == targetID {
+					store = candidate
+					target.SourceRunID = sourceID
+					break
+				}
+			}
+		}
+		if store == nil {
+			return ReplayStatus{}, contract.ErrNotFound
+		}
+	}
+	if target.SourceRunID == "" {
+		return ReplayStatus{}, contract.ErrNotFound
+	}
+	sourceKey, _ := store.RunMetaKey(target.SourceRunID)
+	failureKey, _ := store.FailedKey(target.SourceRunID)
+	values, err := r.redis.Do(ctx, "HMGET", sourceKey, "replayTargetId", "replayState", "replayError", "replayEnqueued")
+	if err != nil {
+		return ReplayStatus{}, err
+	}
+	fields := values.Vars()
+	if len(fields) != 4 || fields[0].String() != targetID {
+		return ReplayStatus{}, contract.ErrNotFound
+	}
+	remaining, err := r.redis.Do(ctx, "HLEN", failureKey)
+	if err != nil {
+		return ReplayStatus{}, err
+	}
+	return ReplayStatus{ReplayRunID: targetID, SourceRunID: target.SourceRunID, Status: fields[1].String(), Error: fields[2].String(), Enqueued: fields[3].Int64(), Remaining: remaining.Int64(), RunStatus: target.Status}, nil
+}
+
+func (r *SimpleReplay) process(store *zset.Store, sourceID string) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
+	defer cancel()
+	// Keep one process per source; batches are bounded so a short lease is
+	// renewed before it expires. A lost lease stops cleanup at the next batch.
+	lockKey := store.Prefix() + "{" + sourceID + "}:replay-lock"
+	var token [16]byte
+	if _, err := rand.Read(token[:]); err != nil {
+		return
+	}
+	owner := hex.EncodeToString(token[:])
+	locked, err := store.AcquireLock(ctx, lockKey, owner, time.Minute)
+	if err != nil || !locked {
+		return
+	}
+	defer func() { _, _ = store.ReleaseLock(context.Background(), lockKey, owner) }()
+	if err := r.handoff(ctx, store, sourceID, lockKey, owner); err != nil {
+		// The time slice ended, not the replay: the reconciler will resume
+		// from remaining source records under a new lease.
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+			return
+		}
+		sourceKey, _ := store.RunMetaKey(sourceID)
+		// A stale owner must not overwrite a newer worker's status.
+		_, _ = r.redis.Eval(context.Background(), replayAttention, 2, []string{sourceKey, lockKey}, []any{owner})
+	}
+}
+
+func (r *SimpleReplay) handoff(ctx context.Context, store *zset.Store, sourceID, lockKey, owner string) error {
+	sourceKey, _ := store.RunMetaKey(sourceID)
+	failedKey, _ := store.FailedKey(sourceID)
+	meta, err := store.GetRunMeta(ctx, sourceID)
+	if err != nil {
+		return err
+	}
+	v, err := r.redis.Do(ctx, "HGET", sourceKey, "replayTargetId")
+	if err != nil {
+		return err
+	}
+	if v == nil || v.String() == "" {
+		return fmt.Errorf("replay reservation missing")
+	}
+	targetID := v.String()
+	// New reservations persist their budget. Reservations made before this
+	// field existed used one round; a restart must not apply today's default
+	// to those already accepted runs.
+	storedMaxRounds, err := r.redis.Do(ctx, "HGET", sourceKey, "replayMaxRound")
+	if err != nil {
+		return err
+	}
+	maxRounds := legacyReplayMaxRounds
+	if storedMaxRounds != nil && storedMaxRounds.String() != "" {
+		maxRounds = storedMaxRounds.Int()
+	}
+	if maxRounds <= 0 {
+		return fmt.Errorf("replay reservation has invalid round budget")
+	}
+	if maxRounds > MaxReplayRounds {
+		return fmt.Errorf("replay reservation exceeds round budget limit")
+	}
+	// CreateRun is idempotent for the main hash but writes extension fields
+	// again. Read and validate an existing target BEFORE CreateRun, otherwise
+	// recovery could silently replace its accepted maxRound.
+	existing, err := store.GetRunMeta(ctx, targetID)
+	if err != nil && !errors.Is(err, zset.ErrRunNotFound) {
+		return err
+	}
+	if err == nil {
+		if storedMaxRounds == nil || storedMaxRounds.String() == "" {
+			maxRounds = existing.MaxRound
+		}
+		if maxRounds <= 0 || maxRounds > MaxReplayRounds || existing.SourceRunID != sourceID || existing.SyncType != "replay" || existing.MaxRound != maxRounds || existing.Operation != meta.Operation || existing.QueryMonth != meta.QueryMonth {
+			return fmt.Errorf("replay target metadata is inconsistent")
+		}
+	}
+	target := zset.RunMeta{RunID: targetID, Operation: meta.Operation, QueryMonth: meta.QueryMonth, SyncType: "replay", SourceRunID: sourceID, ReplayCycle: meta.ReplayCycle + 1, TriggerSource: "replay", MaxRound: maxRounds}
+	if err := store.CreateRun(ctx, target); err != nil {
+		return err
+	}
+	actual, err := store.GetRunMeta(ctx, targetID)
+	if err != nil || actual.SourceRunID != sourceID || actual.SyncType != "replay" || actual.MaxRound != maxRounds || actual.Operation != meta.Operation || actual.QueryMonth != meta.QueryMonth {
+		return fmt.Errorf("replay target metadata is inconsistent")
+	}
+	// The target task-state is the admission receipt during recovery. It may
+	// not expire while source failures are still being transferred.
+	targetKey, _ := store.RunMetaKey(targetID)
+	stateKey, _ := store.TaskStateKey(targetID)
+	if !actual.ProducerDone {
+		if _, err := r.redis.Do(ctx, "PERSIST", targetKey); err != nil {
+			return err
+		}
+		if _, err := r.redis.Do(ctx, "PERSIST", stateKey); err != nil {
+			return err
+		}
+	}
+	_, err = r.redis.Do(ctx, "HSET", sourceKey, "replayState", "processing", "replayError", "")
+	if err != nil {
+		return err
+	}
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		valid, err := store.RenewLock(ctx, lockKey, owner, time.Minute)
+		if err != nil || !valid {
+			return fmt.Errorf("replay lease lost: %w", err)
+		}
+		// HSCAN is not a snapshot. Complete a bounded cursor pass, then
+		// recheck HLEN and repeat from zero if deletion changed the scan.
+		var cursor uint64
+		for {
+			v, e := r.redis.Do(ctx, "HSCAN", failedKey, cursor, "COUNT", 100)
+			if e != nil {
+				return e
+			}
+			var scanPairs []*gvar.Var
+			cursor, scanPairs, e = parseScanResult(v)
+			if e != nil {
+				return e
+			}
+			if len(scanPairs) > 0 {
+				for i := 0; i < len(scanPairs); i += 2 {
+					valid, renewErr := store.RenewLock(ctx, lockKey, owner, time.Minute)
+					if renewErr != nil || !valid {
+						return fmt.Errorf("replay lease lost")
+					}
+					field, raw := scanPairs[i].String(), scanPairs[i+1].String()
+					var rec zset.FailureRecord
+					if json.Unmarshal([]byte(raw), &rec) != nil || rec.TaskID == "" || rec.CardID <= 0 || rec.ICCID == "" || rec.ChanKey == "" || rec.TempKey == "" || field != rec.TaskID || (rec.LatestRunID != "" && rec.LatestRunID != sourceID) {
+						return fmt.Errorf("source failure record cannot be replayed")
+					}
+					operation := rec.Operation
+					if operation == "" {
+						operation = meta.Operation
+					}
+					// Source task identity (not just card ID) prevents collisions
+					// between distinct failures for the same card.
+					taskID := targetID + ":" + rec.TaskID
+					queryMonth := rec.QueryMonth
+					if queryMonth == "" {
+						queryMonth = meta.QueryMonth
+					}
+					task := zset.Task{RunID: targetID, TaskID: taskID, Operation: operation, QueryMonth: queryMonth, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI, ChanKey: rec.ChanKey, TempKey: rec.TempKey, PackageCode: rec.PackageCode}
+					if _, e = store.EnqueueRunBatch(ctx, targetID, rec.ChanKey, []zset.Task{task}, time.Now()); e != nil {
+						return e
+					}
+					confirmed, e := r.redis.Do(ctx, "HEXISTS", stateKey, taskID)
+					if e != nil || confirmed == nil || confirmed.Int() != 1 {
+						return fmt.Errorf("target enqueue unconfirmed: %w", e)
+					}
+					removed, e := r.redis.Eval(ctx, cleanupReplayFailure, 3, []string{sourceKey, failedKey, lockKey}, []any{field, raw, targetID, owner})
+					if e != nil {
+						return e
+					}
+					if removed.Int() == -1 {
+						return fmt.Errorf("source failure changed during replay")
+					}
+				}
+			}
+			if cursor == 0 {
+				break
+			}
+		}
+		remaining, err := r.redis.Do(ctx, "HLEN", failedKey)
+		if err != nil {
+			return err
+		}
+		if remaining.Int64() == 0 {
+			break
+		}
+	}
+	// Persist producerDone only after every source record was confirmed.
+	if err := store.MarkProducerDone(ctx, targetID); err != nil {
+		return err
+	}
+	if _, err := r.redis.Do(ctx, "HSET", sourceKey, "replayState", "completed", "replayError", ""); err != nil {
+		return err
+	}
+	// The source summary has already been archived. Keep the small source
+	// metadata for lookup but restore a bounded retention after handoff.
+	if _, err := r.redis.Do(ctx, "EXPIRE", sourceKey, 30*24*3600); err != nil {
+		return err
+	}
+	_, err = r.redis.Do(ctx, "SREM", store.Prefix()+"replay:pending", sourceID)
+	return err
+}
+
+const replayAttention = `
+if redis.call('GET',KEYS[2])~=ARGV[1] then return 0 end
+redis.call('HSET',KEYS[1],'replayState','needs_attention','replayError','Replay handoff stopped; check Worker logs')
+return 1
+`
+
+const reserveReplay = `
+local id=redis.call('HGET',KEYS[1],'replayTargetId')
+if id then return id end
+local status=redis.call('HGET',KEYS[1],'status')
+if not status then return 'missing' end
+if status~='completed' and status~='completed_with_failures' then return 'running' end
+if redis.call('HLEN',KEYS[2])==0 then return 'empty' end
+redis.call('HSET',KEYS[1],'replayTargetId',ARGV[1],'replayState','accepted','replayRequestedBy',ARGV[2],'replayEnqueued',0,'replayMaxRound',ARGV[3])
+redis.call('PERSIST',KEYS[1]); redis.call('PERSIST',KEYS[2])
+return ARGV[1]
+`
+const cleanupReplayFailure = `
+if redis.call('HGET',KEYS[1],'replayTargetId')~=ARGV[3] or redis.call('GET',KEYS[3])~=ARGV[4] then return -1 end
+local value=redis.call('HGET',KEYS[2],ARGV[1])
+if not value then return 0 end
+if value~=ARGV[2] then return -1 end
+redis.call('HDEL',KEYS[2],ARGV[1]); redis.call('HINCRBY',KEYS[1],'replayEnqueued',1)
+redis.call('HSET',KEYS[1],'pendingFailureCount',redis.call('HLEN',KEYS[2]))
+return 1
+`

internal/fullsync/runtime/simple_replay_invalid_test.go

@@ -0,0 +1,59 @@
+package runtime
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func TestSimpleReplayRetainsInvalidFailureAndResumesSameTarget(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	store, _ := zset.New(redis, "sim-status:")
+	key, _ := store.RunMetaKey("source")
+	failed, _ := store.FailedKey("source")
+	_, _ = redis.Do(ctx, "HSET", key, "runId", "source", "status", "completed_with_failures", "producerDone", "1", "currentRound", "1", "failedCount", "2")
+	valid := zset.FailureRecord{TaskID: "a", CardID: 1, ICCID: "card", ChanKey: "ch", TempKey: "template"}
+	payload, _ := json.Marshal(valid)
+	_, _ = redis.Do(ctx, "HSET", failed, "a", string(payload), "bad", `{"taskId":"bad"}`)
+	svc, _ := NewSimpleReplay(store)
+	accepted, err := svc.Request(ctx, "source", "42", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		state, err := svc.Status(ctx, accepted.ReplayRunID)
+		if err == nil && state.Status == "needs_attention" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	state, _ := svc.Status(ctx, accepted.ReplayRunID)
+	if state.Status != "needs_attention" || state.Remaining < 1 {
+		t.Fatalf("unexpected status: %+v", state)
+	}
+	// No unconfirmed record may be deleted. Retry the SAME target after repair.
+	repaired := zset.FailureRecord{TaskID: "bad", CardID: 2, ICCID: "card2", ChanKey: "ch", TempKey: "template"}
+	payload, _ = json.Marshal(repaired)
+	_, _ = redis.Do(ctx, "HSET", failed, "bad", string(payload))
+	repeat, err := svc.Request(ctx, "source", "42", 0)
+	if err != nil || repeat.ReplayRunID != accepted.ReplayRunID {
+		t.Fatalf("repeat: %+v %v", repeat, err)
+	}
+	deadline = time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		state, _ = svc.Status(ctx, accepted.ReplayRunID)
+		if state.Status == "completed" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if state.Status != "completed" || state.Enqueued != 2 || state.Remaining != 0 {
+		t.Fatalf("resume: %+v", state)
+	}
+}