package zset import ( "context" "encoding/json" "errors" "fmt" "sync" "testing" "time" queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil" ) type runHandler struct { query func(queue.Task) Result apply func([]Result) error } func (h runHandler) Query(_ context.Context, task queue.Task) Result { return h.query(task) } func (h runHandler) Apply(_ context.Context, results []Result) error { if h.apply != nil { return h.apply(results) } return nil } func runTasks(t *testing.T, store *queue.Store, run string, count int) []queue.Task { t.Helper() ctx := context.Background() if err := store.CreateRun(ctx, queue.RunMeta{RunID: run}); err != nil { t.Fatal(err) } tasks := make([]queue.Task, count) for i := range tasks { tasks[i] = queue.Task{RunID: run, TaskID: fmt.Sprintf("%s:%d", run, i), CardID: int64(i + 1), ChanKey: "channel", TempKey: mobileTempKey, ICCID: fmt.Sprintf("card-%d", i)} } if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != count { t.Fatalf("enqueue %d: %v", n, err) } return tasks } func runMeta(t *testing.T, store *queue.Store, run string) queue.RunMeta { t.Helper() meta, err := store.GetRunMeta(context.Background(), run) if err != nil { t.Fatal(err) } return meta } func runConsumer(t *testing.T, store *queue.Store, handler Handler, rounds bool) *Consumer { t.Helper() c, err := New(store, handler, Config{RunQueueEnabled: true, RoundRetryEnabled: rounds, BlockedTasksAsFailures: !rounds, EventPrefix: "sim_status_queue", TemplateQPS: map[string]float64{mobileTempKey: 1000000}}) if err != nil { t.Fatal(err) } return c } func claimRun(t *testing.T, store *queue.Store, run string, now time.Time, count int) []queue.ClaimedTask { t.Helper() claims, err := store.ClaimRun(context.Background(), run, "channel", now, now.Add(time.Minute), count) if err != nil || len(claims) != count { t.Fatalf("claim got %d want %d: %v", len(claims), count, err) } return claims } func finishProducer(t *testing.T, store *queue.Store, run string) { t.Helper() if err := store.MarkProducerDone(context.Background(), run); err != nil { t.Fatal(err) } } func checkTerminal(t *testing.T, store *queue.Store, run string, success, failed int64) { t.Helper() meta := runMeta(t, store, run) want := "completed" if failed > 0 { want = "completed_with_failures" } if meta.Status != want || meta.CompletedAt.IsZero() || meta.SuccessCount != success || meta.FailedCount != failed || meta.ActiveCount != 0 || meta.InflightCount != 0 || meta.NextRoundCount != 0 { t.Fatalf("unexpected terminal metadata: %+v", meta) } } func TestRunConsumerCountsAndCompletes(t *testing.T) { redis := testutil.Redis(t) ctx := context.Background() for _, failure := range []bool{false, true} { t.Run(fmt.Sprint(failure), func(t *testing.T) { store, _ := queue.New(redis, "sim-status:") run := fmt.Sprint("counts-", failure) tasks := runTasks(t, store, run, 3) if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != 0 { t.Fatalf("duplicate enqueue: %d %v", n, err) } finishProducer(t, store, run) handler := runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: !failure || task.CardID != 2, ErrorClass: "business"} }, apply: func(_ []Result) error { if meta := runMeta(t, store, run); meta.SuccessCount != 0 || meta.FailedCount != 0 { return errors.New("counted before DB apply") } return nil }} c := runConsumer(t, store, handler, false) claims := claimRun(t, store, run, time.Now(), 3) if err := c.handleRunClaims(ctx, claims); err != nil { t.Fatal(err) } var failures int64 if failure { failures = 1 } checkTerminal(t, store, run, 3-failures, failures) outcomes := []queue.TaskOutcome{{Success: true}, {Success: true}, {Success: true}} if n, err := store.FinalizeRunTasks(ctx, claims, outcomes); err != nil || n != 0 { t.Fatalf("duplicate finalize: %d %v", n, err) } checkTerminal(t, store, run, 3-failures, failures) key, _ := store.RunMetaKey(run) ttl, err := redis.Do(ctx, "TTL", key) if err != nil || ttl.Int64() < 29*24*3600 { t.Fatalf("retention=%v err=%v", ttl, err) } }) } } func TestRunConsumerLateProducerEmptyAndHistoricalRunCompletion(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "sim-status:") ctx := context.Background() if err := store.CreateRun(ctx, queue.RunMeta{RunID: "historical", Status: "running", ProducerDone: true}); err != nil { t.Fatal(err) } runTasks(t, store, "empty", 0) runTasks(t, store, "late", 1) c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}, false) claims := claimRun(t, store, "late", time.Now(), 1) if err := c.handleRunClaims(ctx, claims); err != nil { t.Fatal(err) } if meta := runMeta(t, store, "late"); !meta.CompletedAt.IsZero() || meta.SuccessCount != 1 { t.Fatalf("premature completion: %+v", meta) } finishProducer(t, store, "late") finishProducer(t, store, "empty") if _, err := c.processOnce(ctx); err != nil { t.Fatal(err) } c.channelGroup.Wait() checkTerminal(t, store, "late", 1, 0) checkTerminal(t, store, "empty", 0, 0) checkTerminal(t, store, "historical", 0, 0) } func TestRunConsumerRecoveryAndFencing(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "sim-status:") ctx := context.Background() for _, mode := range []string{"database", "network", "stale"} { t.Run(mode, func(t *testing.T) { runTasks(t, store, mode, 1) finishProducer(t, store, mode) now := time.Now() claims := claimRun(t, store, mode, now, 1) good := runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }} handler := good if mode == "database" { handler.apply = func([]Result) error { return errors.New("database unavailable") } } if mode == "network" { handler.query = func(task queue.Task) Result { return Result{Task: task, Retryable: true, ErrorClass: "network"} } } c := runConsumer(t, store, handler, false) if mode != "stale" { err := c.handleRunClaims(ctx, claims) if (err != nil) != (mode == "database") { t.Fatalf("handle: %v", err) } if meta := runMeta(t, store, mode); meta.SuccessCount != 0 || meta.FailedCount != 0 || meta.ActiveCount != 1 || meta.InflightCount != 1 || meta.Status != "running" { t.Fatalf("lost pending task: %+v", meta) } } fresh := claimRun(t, store, mode, now.Add(2*time.Minute), 1) if n, err := store.RenewRunBatch(ctx, claims, now.Add(3*time.Minute)); err != nil || n != 0 { t.Fatalf("stale renewal: %d %v", n, err) } if n, err := store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil || n != 0 { t.Fatalf("stale ack: %d %v", n, err) } if n, err := store.BlockRunScopedAsFailed(ctx, claims[0], queue.TaskOutcome{}); err != nil || n != -1 { t.Fatalf("stale block: %d %v", n, err) } c.handler = good if err := c.handleRunClaims(ctx, fresh); err != nil { t.Fatal(err) } checkTerminal(t, store, mode, 1, 0) }) } } func TestRunConsumerBlockedChannelCountsOnlyUnfinishedTasks(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "sim-status:") ctx := context.Background() runTasks(t, store, "blocked", 5) runTasks(t, store, "other", 1) // Production continues while the channel is blocked. claims := claimRun(t, store, "blocked", time.Now(), 2) var applied int successQueried := make(chan struct{}) handler := runHandler{query: func(task queue.Task) Result { if task.CardID == 1 { close(successQueried) return Result{Task: task, Success: true} } <-successQueried return Result{Task: task, BlockChannel: true, ErrorClass: "channel_limited", Code: 797, ErrCode: "S00004", Message: "limited"} }, apply: func(results []Result) error { applied = countSuccessful(results); return nil }} c := runConsumer(t, store, handler, false) // Ensure the success response arrives before the channel-limit response. if err := c.handleRunClaims(ctx, claims); err != nil { t.Fatal(err) } meta := runMeta(t, store, "blocked") if applied != 1 || meta.SuccessCount != 1 || meta.FailedCount != 4 || meta.ActiveCount != 0 || meta.InflightCount != 0 { t.Fatalf("applied=%d meta=%+v", applied, meta) } if n, err := store.BlockRunScopedAsFailed(ctx, claims[1], queue.TaskOutcome{}); err != nil || n != -1 { t.Fatalf("duplicate block: %d %v", n, err) } late := queue.Task{TaskID: "late", RunID: "blocked", ChanKey: "channel", CardID: 99} if n, err := store.EnqueueRunBatch(ctx, "blocked", "channel", []queue.Task{late}, time.Now()); err != nil || n != 0 { t.Fatalf("blocked enqueue: %d %v", n, err) } key, _ := store.FailedKey("blocked") failures, err := redis.Do(ctx, "HLEN", key) if err != nil || failures.Int() != 4 { t.Fatalf("failure records=%v err=%v", failures, err) } stored, err := redis.Do(ctx, "HGET", key, "blocked:4") if err != nil { t.Fatal(err) } var record queue.FailureRecord if err := json.Unmarshal([]byte(stored.String()), &record); err != nil { t.Fatal(err) } if record.TaskID != "blocked:4" || record.CardID != 5 || record.ICCID != "card-4" || record.Attempts != 0 || record.FailureType != "channel_blocked" || record.ErrCode != "S00004" { t.Fatalf("failure identity/reason: %+v", record) } stateKey, _ := store.TaskStateKey("blocked") state, err := redis.Do(ctx, "HGET", stateKey, "blocked:4") if err != nil || state.MapStrAny()["status"] != "channel_blocked" { t.Fatalf("blocked state=%v err=%v", state, err) } if meta := runMeta(t, store, "other"); meta.ActiveCount != 1 || meta.FailedCount != 0 { t.Fatalf("other run affected: %+v", meta) } finishProducer(t, store, "blocked") if err := c.completeRun(ctx, "blocked"); err != nil { t.Fatal(err) } checkTerminal(t, store, "blocked", 1, 4) } func TestRunConsumerMonthlyRoundsRemainEnabled(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "flow:") ctx := context.Background() runTasks(t, store, "monthly", 1) finishProducer(t, store, "monthly") c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, true) for round := 1; round <= 5; round++ { if err := c.handleRunClaims(ctx, claimRun(t, store, "monthly", time.Now(), 1)); err != nil { t.Fatal(err) } if round < 5 { if meta := runMeta(t, store, "monthly"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" { t.Fatalf("round %d: %+v", round, meta) } if ok, err := store.AdvanceRunRound(ctx, "monthly", 5); err != nil || !ok { t.Fatalf("advance %v %v", ok, err) } } } checkTerminal(t, store, "monthly", 0, 1) } func TestRunConsumerStatusRetriesRetryableResultsForThreeRounds(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "sim-status:") ctx := context.Background() runTasks(t, store, "status", 1) finishProducer(t, store, "status") c, err := New(store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "carrier_retry", Retryable: true, Code: 9999, ErrCode: "S00004"} }}, Config{ RunQueueEnabled: true, RoundRetryEnabled: true, MaxRounds: 3, EventPrefix: "sim_status_queue", ShouldRetryNextRound: func(result Result) bool { return result.Retryable }, TemplateQPS: map[string]float64{mobileTempKey: 1000000}, }) if err != nil { t.Fatal(err) } for round := 1; round <= 3; round++ { if err := c.handleRunClaims(ctx, claimRun(t, store, "status", time.Now(), 1)); err != nil { t.Fatal(err) } if round < 3 { if meta := runMeta(t, store, "status"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" { t.Fatalf("round %d: %+v", round, meta) } if ok, err := store.AdvanceRunRound(ctx, "status", 3); err != nil || !ok { t.Fatalf("advance %v %v", ok, err) } } } checkTerminal(t, store, "status", 0, 1) } func TestReplayMaxRoundOverridesConsumerDefault(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "sim-status:") ctx := context.Background() if err := store.CreateRun(ctx, queue.RunMeta{RunID: "replay-one", Operation: "status", SyncType: "replay", MaxRound: 1}); err != nil { t.Fatal(err) } task := queue.Task{RunID: "replay-one", TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "channel", TempKey: mobileTempKey} if _, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []queue.Task{task}, time.Now()); err != nil { t.Fatal(err) } finishProducer(t, store, "replay-one") c, err := New(store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, Config{RunQueueEnabled: true, RoundRetryEnabled: true, MaxRounds: 3, TemplateQPS: map[string]float64{mobileTempKey: 1000000}}) if err != nil { t.Fatal(err) } if err := c.handleRunClaims(ctx, claimRun(t, store, "replay-one", time.Now(), 1)); err != nil { t.Fatal(err) } checkTerminal(t, store, "replay-one", 0, 1) meta := runMeta(t, store, "replay-one") if meta.CurrentRound != 1 || meta.NextRoundCount != 0 { t.Fatalf("replay advanced rounds: %+v", meta) } } func TestRunQueueModeDoesNotEnableRoundRetry(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "sim-status:") runTasks(t, store, "single-round", 1) finishProducer(t, store, "single-round") c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, false) if err := c.handleRunClaims(context.Background(), claimRun(t, store, "single-round", time.Now(), 1)); err != nil { t.Fatal(err) } checkTerminal(t, store, "single-round", 0, 1) if meta := runMeta(t, store, "single-round"); meta.CurrentRound != 1 { t.Fatalf("unexpected retry: %+v", meta) } } func TestMonthlyChannelBlockKeepsExistingCountingPolicy(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "flow:") ctx := context.Background() runTasks(t, store, "monthly-block", 2) claimRun(t, store, "monthly-block", time.Now(), 1) for i := 0; i < 2; i++ { want := int64(2) if i == 1 { want = 0 } if n, err := store.BlockRunScoped(ctx, "monthly-block", "channel"); err != nil || n != want { t.Fatalf("monthly block=%d err=%v", n, err) } } if meta := runMeta(t, store, "monthly-block"); meta.FailedCount != 0 || meta.ActiveCount != 0 || meta.InflightCount != 0 { t.Fatalf("monthly counting changed: %+v", meta) } } func TestRunConsumerConcurrentRuns(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "sim-status:") ctx := context.Background() var mu sync.Mutex applied := map[string]int{} c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }, apply: func(results []Result) error { mu.Lock() defer mu.Unlock() for _, result := range results { applied[result.Task.RunID]++ } return nil }}, false) for _, run := range []string{"a", "b"} { runTasks(t, store, run, 3) finishProducer(t, store, run) } if started, err := c.processOnce(ctx); err != nil || !started { t.Fatalf("process: %v %v", started, err) } c.channelGroup.Wait() for _, run := range []string{"a", "b"} { checkTerminal(t, store, run, 3, 0) if applied[run] != 3 { t.Fatalf("applied %s: %d", run, applied[run]) } } } // TestRunConsumerCoveredResultsCompleteWithoutFailures drives the full // consumer path for a covered result: the round must close, the run must end // as "completed" rather than "completed_with_failures", and the failure pool // must stay empty. This is the IMEI 14000 "carrier has no record" case — on // 2026-10-20 it accounted for every failure recorded in production while the // run success rate read 21%. func TestRunConsumerCoveredResultsCompleteWithoutFailures(t *testing.T) { redis := testutil.Redis(t) store, _ := queue.New(redis, "imei:") ctx := context.Background() runTasks(t, store, "covered", 4) finishProducer(t, store, "covered") queried := 0 c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { queried++ switch task.CardID { case 1: return Result{Task: task, Success: true, Imei: "865466070817587", ImeiSynced: true} case 2: return Result{Task: task, ErrorClass: "no_record", Code: 14000, ImeiSynced: true, Covered: true, Message: "系统无记录,请检查输入参数是否正确"} case 3: return Result{Task: task, ErrorClass: "data_invalid", ImeiSynced: true, Covered: true, Message: "carrier returned no imei value"} default: return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"} } }}, false) if err := c.handleRunClaims(ctx, claimRun(t, store, "covered", time.Now(), 4)); err != nil { t.Fatal(err) } if queried != 4 { t.Fatalf("queried = %d, want 4", queried) } meta := runMeta(t, store, "covered") // Three cards produced a usable or definitive answer (success + 2 covered), // and only the business failure is counted. Without the covered channel the // two 14000-style answers would have made failedCount=3. if meta.Status != "completed_with_failures" || meta.SuccessCount != 3 || meta.FailedCount != 1 || meta.NextRoundCount != 0 { t.Fatalf("unexpected run accounting: %+v", meta) } failureKey, _ := store.FailedKey("covered") size, err := redis.Do(ctx, "HLEN", failureKey) if err != nil || size.Int64() != 1 { t.Fatalf("failure pool entries = %d err=%v, want 1 (only the business failure)", size.Int64(), err) } for _, taskID := range []string{"covered:1", "covered:2"} { exists, err := redis.Do(ctx, "HEXISTS", failureKey, taskID) if err != nil || exists.Int() != 0 { t.Fatalf("covered card %s leaked into the failure pool: %v %v", taskID, exists, err) } } exists, err := redis.Do(ctx, "HEXISTS", failureKey, "covered:3") if err != nil || exists.Int() != 1 { t.Fatalf("the business failure is missing from the pool: %v %v", exists, err) } }