chore: 初始开源发布

iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。 - Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件） - 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异，不破坏「校验 MSISDN 格式」等测试断言） - 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23），移除写死的 git 凭据 - README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 已知：3 个测试包在源仓同一提交上同样失败（client/upstream、 service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。
2026-10-22 16:37:56 +08:00
commit 2bec257289
361 changed files with 71524 additions and 0 deletions
@@ -0,0 +1,480 @@
+package zset
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+type runHandler struct {
+	query func(queue.Task) Result
+	apply func([]Result) error
+}
+
+func (h runHandler) Query(_ context.Context, task queue.Task) Result { return h.query(task) }
+func (h runHandler) Apply(_ context.Context, results []Result) error {
+	if h.apply != nil {
+		return h.apply(results)
+	}
+	return nil
+}
+
+func runTasks(t *testing.T, store *queue.Store, run string, count int) []queue.Task {
+	t.Helper()
+	ctx := context.Background()
+	if err := store.CreateRun(ctx, queue.RunMeta{RunID: run}); err != nil {
+		t.Fatal(err)
+	}
+	tasks := make([]queue.Task, count)
+	for i := range tasks {
+		tasks[i] = queue.Task{RunID: run, TaskID: fmt.Sprintf("%s:%d", run, i), CardID: int64(i + 1), ChanKey: "channel", TempKey: mobileTempKey, ICCID: fmt.Sprintf("card-%d", i)}
+	}
+	if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != count {
+		t.Fatalf("enqueue %d: %v", n, err)
+	}
+	return tasks
+}
+
+func runMeta(t *testing.T, store *queue.Store, run string) queue.RunMeta {
+	t.Helper()
+	meta, err := store.GetRunMeta(context.Background(), run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return meta
+}
+
+func runConsumer(t *testing.T, store *queue.Store, handler Handler, rounds bool) *Consumer {
+	t.Helper()
+	c, err := New(store, handler, Config{RunQueueEnabled: true, RoundRetryEnabled: rounds, BlockedTasksAsFailures: !rounds, EventPrefix: "sim_status_queue", TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c
+}
+
+func claimRun(t *testing.T, store *queue.Store, run string, now time.Time, count int) []queue.ClaimedTask {
+	t.Helper()
+	claims, err := store.ClaimRun(context.Background(), run, "channel", now, now.Add(time.Minute), count)
+	if err != nil || len(claims) != count {
+		t.Fatalf("claim got %d want %d: %v", len(claims), count, err)
+	}
+	return claims
+}
+
+func finishProducer(t *testing.T, store *queue.Store, run string) {
+	t.Helper()
+	if err := store.MarkProducerDone(context.Background(), run); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func checkTerminal(t *testing.T, store *queue.Store, run string, success, failed int64) {
+	t.Helper()
+	meta := runMeta(t, store, run)
+	want := "completed"
+	if failed > 0 {
+		want = "completed_with_failures"
+	}
+	if meta.Status != want || meta.CompletedAt.IsZero() || meta.SuccessCount != success || meta.FailedCount != failed || meta.ActiveCount != 0 || meta.InflightCount != 0 || meta.NextRoundCount != 0 {
+		t.Fatalf("unexpected terminal metadata: %+v", meta)
+	}
+}
+
+func TestRunConsumerCountsAndCompletes(t *testing.T) {
+	redis := testutil.Redis(t)
+	ctx := context.Background()
+	for _, failure := range []bool{false, true} {
+		t.Run(fmt.Sprint(failure), func(t *testing.T) {
+			store, _ := queue.New(redis, "sim-status:")
+			run := fmt.Sprint("counts-", failure)
+			tasks := runTasks(t, store, run, 3)
+			if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != 0 {
+				t.Fatalf("duplicate enqueue: %d %v", n, err)
+			}
+			finishProducer(t, store, run)
+			handler := runHandler{query: func(task queue.Task) Result {
+				return Result{Task: task, Success: !failure || task.CardID != 2, ErrorClass: "business"}
+			}, apply: func(_ []Result) error {
+				if meta := runMeta(t, store, run); meta.SuccessCount != 0 || meta.FailedCount != 0 {
+					return errors.New("counted before DB apply")
+				}
+				return nil
+			}}
+			c := runConsumer(t, store, handler, false)
+			claims := claimRun(t, store, run, time.Now(), 3)
+			if err := c.handleRunClaims(ctx, claims); err != nil {
+				t.Fatal(err)
+			}
+			var failures int64
+			if failure {
+				failures = 1
+			}
+			checkTerminal(t, store, run, 3-failures, failures)
+			outcomes := []queue.TaskOutcome{{Success: true}, {Success: true}, {Success: true}}
+			if n, err := store.FinalizeRunTasks(ctx, claims, outcomes); err != nil || n != 0 {
+				t.Fatalf("duplicate finalize: %d %v", n, err)
+			}
+			checkTerminal(t, store, run, 3-failures, failures)
+			key, _ := store.RunMetaKey(run)
+			ttl, err := redis.Do(ctx, "TTL", key)
+			if err != nil || ttl.Int64() < 29*24*3600 {
+				t.Fatalf("retention=%v err=%v", ttl, err)
+			}
+		})
+	}
+}
+
+func TestRunConsumerLateProducerEmptyAndHistoricalRunCompletion(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	if err := store.CreateRun(ctx, queue.RunMeta{RunID: "historical", Status: "running", ProducerDone: true}); err != nil {
+		t.Fatal(err)
+	}
+	runTasks(t, store, "empty", 0)
+	runTasks(t, store, "late", 1)
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}, false)
+	claims := claimRun(t, store, "late", time.Now(), 1)
+	if err := c.handleRunClaims(ctx, claims); err != nil {
+		t.Fatal(err)
+	}
+	if meta := runMeta(t, store, "late"); !meta.CompletedAt.IsZero() || meta.SuccessCount != 1 {
+		t.Fatalf("premature completion: %+v", meta)
+	}
+	finishProducer(t, store, "late")
+	finishProducer(t, store, "empty")
+	if _, err := c.processOnce(ctx); err != nil {
+		t.Fatal(err)
+	}
+	c.channelGroup.Wait()
+	checkTerminal(t, store, "late", 1, 0)
+	checkTerminal(t, store, "empty", 0, 0)
+	checkTerminal(t, store, "historical", 0, 0)
+}
+
+func TestRunConsumerRecoveryAndFencing(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	for _, mode := range []string{"database", "network", "stale"} {
+		t.Run(mode, func(t *testing.T) {
+			runTasks(t, store, mode, 1)
+			finishProducer(t, store, mode)
+			now := time.Now()
+			claims := claimRun(t, store, mode, now, 1)
+			good := runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}
+			handler := good
+			if mode == "database" {
+				handler.apply = func([]Result) error { return errors.New("database unavailable") }
+			}
+			if mode == "network" {
+				handler.query = func(task queue.Task) Result { return Result{Task: task, Retryable: true, ErrorClass: "network"} }
+			}
+			c := runConsumer(t, store, handler, false)
+			if mode != "stale" {
+				err := c.handleRunClaims(ctx, claims)
+				if (err != nil) != (mode == "database") {
+					t.Fatalf("handle: %v", err)
+				}
+				if meta := runMeta(t, store, mode); meta.SuccessCount != 0 || meta.FailedCount != 0 || meta.ActiveCount != 1 || meta.InflightCount != 1 || meta.Status != "running" {
+					t.Fatalf("lost pending task: %+v", meta)
+				}
+			}
+			fresh := claimRun(t, store, mode, now.Add(2*time.Minute), 1)
+			if n, err := store.RenewRunBatch(ctx, claims, now.Add(3*time.Minute)); err != nil || n != 0 {
+				t.Fatalf("stale renewal: %d %v", n, err)
+			}
+			if n, err := store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil || n != 0 {
+				t.Fatalf("stale ack: %d %v", n, err)
+			}
+			if n, err := store.BlockRunScopedAsFailed(ctx, claims[0], queue.TaskOutcome{}); err != nil || n != -1 {
+				t.Fatalf("stale block: %d %v", n, err)
+			}
+			c.handler = good
+			if err := c.handleRunClaims(ctx, fresh); err != nil {
+				t.Fatal(err)
+			}
+			checkTerminal(t, store, mode, 1, 0)
+		})
+	}
+}
+
+func TestRunConsumerBlockedChannelCountsOnlyUnfinishedTasks(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	runTasks(t, store, "blocked", 5)
+	runTasks(t, store, "other", 1)
+	// Production continues while the channel is blocked.
+	claims := claimRun(t, store, "blocked", time.Now(), 2)
+	var applied int
+	successQueried := make(chan struct{})
+	handler := runHandler{query: func(task queue.Task) Result {
+		if task.CardID == 1 {
+			close(successQueried)
+			return Result{Task: task, Success: true}
+		}
+		<-successQueried
+		return Result{Task: task, BlockChannel: true, ErrorClass: "channel_limited", Code: 797, ErrCode: "S00004", Message: "limited"}
+	}, apply: func(results []Result) error { applied = countSuccessful(results); return nil }}
+	c := runConsumer(t, store, handler, false)
+	// Ensure the success response arrives before the channel-limit response.
+	if err := c.handleRunClaims(ctx, claims); err != nil {
+		t.Fatal(err)
+	}
+	meta := runMeta(t, store, "blocked")
+	if applied != 1 || meta.SuccessCount != 1 || meta.FailedCount != 4 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
+		t.Fatalf("applied=%d meta=%+v", applied, meta)
+	}
+	if n, err := store.BlockRunScopedAsFailed(ctx, claims[1], queue.TaskOutcome{}); err != nil || n != -1 {
+		t.Fatalf("duplicate block: %d %v", n, err)
+	}
+	late := queue.Task{TaskID: "late", RunID: "blocked", ChanKey: "channel", CardID: 99}
+	if n, err := store.EnqueueRunBatch(ctx, "blocked", "channel", []queue.Task{late}, time.Now()); err != nil || n != 0 {
+		t.Fatalf("blocked enqueue: %d %v", n, err)
+	}
+	key, _ := store.FailedKey("blocked")
+	failures, err := redis.Do(ctx, "HLEN", key)
+	if err != nil || failures.Int() != 4 {
+		t.Fatalf("failure records=%v err=%v", failures, err)
+	}
+	stored, err := redis.Do(ctx, "HGET", key, "blocked:4")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var record queue.FailureRecord
+	if err := json.Unmarshal([]byte(stored.String()), &record); err != nil {
+		t.Fatal(err)
+	}
+	if record.TaskID != "blocked:4" || record.CardID != 5 || record.ICCID != "card-4" || record.Attempts != 0 || record.FailureType != "channel_blocked" || record.ErrCode != "S00004" {
+		t.Fatalf("failure identity/reason: %+v", record)
+	}
+	stateKey, _ := store.TaskStateKey("blocked")
+	state, err := redis.Do(ctx, "HGET", stateKey, "blocked:4")
+	if err != nil || state.MapStrAny()["status"] != "channel_blocked" {
+		t.Fatalf("blocked state=%v err=%v", state, err)
+	}
+	if meta := runMeta(t, store, "other"); meta.ActiveCount != 1 || meta.FailedCount != 0 {
+		t.Fatalf("other run affected: %+v", meta)
+	}
+	finishProducer(t, store, "blocked")
+	if err := c.completeRun(ctx, "blocked"); err != nil {
+		t.Fatal(err)
+	}
+	checkTerminal(t, store, "blocked", 1, 4)
+}
+
+func TestRunConsumerMonthlyRoundsRemainEnabled(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "flow:")
+	ctx := context.Background()
+	runTasks(t, store, "monthly", 1)
+	finishProducer(t, store, "monthly")
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, true)
+	for round := 1; round <= 5; round++ {
+		if err := c.handleRunClaims(ctx, claimRun(t, store, "monthly", time.Now(), 1)); err != nil {
+			t.Fatal(err)
+		}
+		if round < 5 {
+			if meta := runMeta(t, store, "monthly"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
+				t.Fatalf("round %d: %+v", round, meta)
+			}
+			if ok, err := store.AdvanceRunRound(ctx, "monthly", 5); err != nil || !ok {
+				t.Fatalf("advance %v %v", ok, err)
+			}
+		}
+	}
+	checkTerminal(t, store, "monthly", 0, 1)
+}
+
+func TestRunConsumerStatusRetriesRetryableResultsForThreeRounds(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	runTasks(t, store, "status", 1)
+	finishProducer(t, store, "status")
+	c, err := New(store, runHandler{query: func(task queue.Task) Result {
+		return Result{Task: task, ErrorClass: "carrier_retry", Retryable: true, Code: 9999, ErrCode: "S00004"}
+	}}, Config{
+		RunQueueEnabled:   true,
+		RoundRetryEnabled: true,
+		MaxRounds:         3,
+		EventPrefix:       "sim_status_queue",
+		ShouldRetryNextRound: func(result Result) bool {
+			return result.Retryable
+		},
+		TemplateQPS: map[string]float64{mobileTempKey: 1000000},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for round := 1; round <= 3; round++ {
+		if err := c.handleRunClaims(ctx, claimRun(t, store, "status", time.Now(), 1)); err != nil {
+			t.Fatal(err)
+		}
+		if round < 3 {
+			if meta := runMeta(t, store, "status"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
+				t.Fatalf("round %d: %+v", round, meta)
+			}
+			if ok, err := store.AdvanceRunRound(ctx, "status", 3); err != nil || !ok {
+				t.Fatalf("advance %v %v", ok, err)
+			}
+		}
+	}
+	checkTerminal(t, store, "status", 0, 1)
+}
+
+func TestReplayMaxRoundOverridesConsumerDefault(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	if err := store.CreateRun(ctx, queue.RunMeta{RunID: "replay-one", Operation: "status", SyncType: "replay", MaxRound: 1}); err != nil {
+		t.Fatal(err)
+	}
+	task := queue.Task{RunID: "replay-one", TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "channel", TempKey: mobileTempKey}
+	if _, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []queue.Task{task}, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	finishProducer(t, store, "replay-one")
+	c, err := New(store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, Config{RunQueueEnabled: true, RoundRetryEnabled: true, MaxRounds: 3, TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := c.handleRunClaims(ctx, claimRun(t, store, "replay-one", time.Now(), 1)); err != nil {
+		t.Fatal(err)
+	}
+	checkTerminal(t, store, "replay-one", 0, 1)
+	meta := runMeta(t, store, "replay-one")
+	if meta.CurrentRound != 1 || meta.NextRoundCount != 0 {
+		t.Fatalf("replay advanced rounds: %+v", meta)
+	}
+}
+
+func TestRunQueueModeDoesNotEnableRoundRetry(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	runTasks(t, store, "single-round", 1)
+	finishProducer(t, store, "single-round")
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, false)
+	if err := c.handleRunClaims(context.Background(), claimRun(t, store, "single-round", time.Now(), 1)); err != nil {
+		t.Fatal(err)
+	}
+	checkTerminal(t, store, "single-round", 0, 1)
+	if meta := runMeta(t, store, "single-round"); meta.CurrentRound != 1 {
+		t.Fatalf("unexpected retry: %+v", meta)
+	}
+}
+
+func TestMonthlyChannelBlockKeepsExistingCountingPolicy(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "flow:")
+	ctx := context.Background()
+	runTasks(t, store, "monthly-block", 2)
+	claimRun(t, store, "monthly-block", time.Now(), 1)
+	for i := 0; i < 2; i++ {
+		want := int64(2)
+		if i == 1 {
+			want = 0
+		}
+		if n, err := store.BlockRunScoped(ctx, "monthly-block", "channel"); err != nil || n != want {
+			t.Fatalf("monthly block=%d err=%v", n, err)
+		}
+	}
+	if meta := runMeta(t, store, "monthly-block"); meta.FailedCount != 0 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
+		t.Fatalf("monthly counting changed: %+v", meta)
+	}
+}
+
+func TestRunConsumerConcurrentRuns(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "sim-status:")
+	ctx := context.Background()
+	var mu sync.Mutex
+	applied := map[string]int{}
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }, apply: func(results []Result) error {
+		mu.Lock()
+		defer mu.Unlock()
+		for _, result := range results {
+			applied[result.Task.RunID]++
+		}
+		return nil
+	}}, false)
+	for _, run := range []string{"a", "b"} {
+		runTasks(t, store, run, 3)
+		finishProducer(t, store, run)
+	}
+	if started, err := c.processOnce(ctx); err != nil || !started {
+		t.Fatalf("process: %v %v", started, err)
+	}
+	c.channelGroup.Wait()
+	for _, run := range []string{"a", "b"} {
+		checkTerminal(t, store, run, 3, 0)
+		if applied[run] != 3 {
+			t.Fatalf("applied %s: %d", run, applied[run])
+		}
+	}
+}
+
+// TestRunConsumerCoveredResultsCompleteWithoutFailures drives the full
+// consumer path for a covered result: the round must close, the run must end
+// as "completed" rather than "completed_with_failures", and the failure pool
+// must stay empty. This is the IMEI 14000 "carrier has no record" case — on
+// 2026-10-20 it accounted for every failure recorded in production while the
+// run success rate read 21%.
+func TestRunConsumerCoveredResultsCompleteWithoutFailures(t *testing.T) {
+	redis := testutil.Redis(t)
+	store, _ := queue.New(redis, "imei:")
+	ctx := context.Background()
+	runTasks(t, store, "covered", 4)
+	finishProducer(t, store, "covered")
+	queried := 0
+	c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result {
+		queried++
+		switch task.CardID {
+		case 1:
+			return Result{Task: task, Success: true, Imei: "865466070817587", ImeiSynced: true}
+		case 2:
+			return Result{Task: task, ErrorClass: "no_record", Code: 14000, ImeiSynced: true, Covered: true, Message: "系统无记录，请检查输入参数是否正确"}
+		case 3:
+			return Result{Task: task, ErrorClass: "data_invalid", ImeiSynced: true, Covered: true, Message: "carrier returned no imei value"}
+		default:
+			return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
+		}
+	}}, false)
+	if err := c.handleRunClaims(ctx, claimRun(t, store, "covered", time.Now(), 4)); err != nil {
+		t.Fatal(err)
+	}
+	if queried != 4 {
+		t.Fatalf("queried = %d, want 4", queried)
+	}
+	meta := runMeta(t, store, "covered")
+	// Three cards produced a usable or definitive answer (success + 2 covered),
+	// and only the business failure is counted. Without the covered channel the
+	// two 14000-style answers would have made failedCount=3.
+	if meta.Status != "completed_with_failures" || meta.SuccessCount != 3 || meta.FailedCount != 1 || meta.NextRoundCount != 0 {
+		t.Fatalf("unexpected run accounting: %+v", meta)
+	}
+	failureKey, _ := store.FailedKey("covered")
+	size, err := redis.Do(ctx, "HLEN", failureKey)
+	if err != nil || size.Int64() != 1 {
+		t.Fatalf("failure pool entries = %d err=%v, want 1 (only the business failure)", size.Int64(), err)
+	}
+	for _, taskID := range []string{"covered:1", "covered:2"} {
+		exists, err := redis.Do(ctx, "HEXISTS", failureKey, taskID)
+		if err != nil || exists.Int() != 0 {
+			t.Fatalf("covered card %s leaked into the failure pool: %v %v", taskID, exists, err)
+		}
+	}
+	exists, err := redis.Do(ctx, "HEXISTS", failureKey, "covered:3")
+	if err != nil || exists.Int() != 1 {
+		t.Fatalf("the business failure is missing from the pool: %v %v", exists, err)
+	}
+}