package zset import ( "context" "encoding/json" "errors" "strings" "sync" "sync/atomic" "testing" "time" "github.com/gogf/gf/v2/container/gvar" queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset" ) type redisCall struct { command string args []any } type fakeRedis struct { mu sync.Mutex calls []redisCall responses []*gvar.Var } func (f *fakeRedis) Do(_ context.Context, command string, args ...any) (*gvar.Var, error) { f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, redisCall{command: command, args: args}) if len(f.responses) == 0 { return gvar.New(int64(1)), nil } response := f.responses[0] f.responses = f.responses[1:] return response, nil } func (f *fakeRedis) Eval(_ context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) { allArgs := []any{script, numKeys} for _, key := range keys { allArgs = append(allArgs, key) } allArgs = append(allArgs, args...) return f.Do(context.Background(), "EVAL", allArgs...) } type testHandler struct { query func(context.Context, queue.Task) Result apply func(context.Context, []Result) error } func (h *testHandler) Query(ctx context.Context, task queue.Task) Result { if h.query != nil { return h.query(ctx, task) } return Result{Task: task, Success: true, UsageMB: float64(task.CardID)} } func (h *testHandler) Apply(ctx context.Context, results []Result) error { if h.apply != nil { return h.apply(ctx, results) } return nil } func taskMember(task queue.Task) string { value, _ := json.Marshal(task) return string(value) } func claimedTask(task queue.Task) queue.ClaimedTask { return queue.ClaimedTask{Task: task, Member: taskMember(task), LeaseToken: "token-" + task.TaskID} } func newTestConsumer(t *testing.T, fake *fakeRedis, handler Handler, config Config) *Consumer { t.Helper() store, err := queue.New(fake, "flow:") if err != nil { t.Fatal(err) } consumer, err := New(store, handler, config) if err != nil { t.Fatal(err) } return consumer } func TestRoundSummaryAccumulatesAndFlushes(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{gvar.New([]any{"runId", "run-1"})}} consumer := newTestConsumer(t, fake, &testHandler{}, Config{RoundRetryEnabled: true}) started := time.Date(2026, 9, 22, 3, 0, 0, 0, time.UTC) consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{ claimedTask(queue.Task{RunID: "run-1", TaskID: "a"}), claimedTask(queue.Task{RunID: "run-1", TaskID: "b"}), }, []queue.TaskOutcome{ {Success: true}, {Retry: true}, }, started) consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{ claimedTask(queue.Task{RunID: "run-1", TaskID: "c"}), }, []queue.TaskOutcome{{Success: false}}, started.Add(2*time.Second)) key := "run-1\x001" consumer.summaryMu.Lock() summary := *consumer.roundSummaries[key] consumer.summaryMu.Unlock() if summary.inputCount != 3 || summary.successCount != 1 || summary.failedCount != 2 || summary.deferredCount != 1 || summary.terminalFailedCount != 1 { t.Fatalf("unexpected summary: %+v", summary) } if !summary.startedAt.Equal(started) { t.Fatalf("startedAt = %v, want %v", summary.startedAt, started) } consumer.flushRoundSummary(context.Background(), "run-1", 1) consumer.summaryMu.Lock() _, exists := consumer.roundSummaries[key] consumer.summaryMu.Unlock() if exists { t.Fatal("round summary was not removed after flush") } } func TestHandleClaimsAppliesSuccessOnceAndAcknowledgesBatch(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}} applyCalls := 0 var applied []Result handler := &testHandler{apply: func(_ context.Context, results []Result) error { applyCalls++ applied = append(applied, results...) return nil }} consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1}) claimed := []queue.ClaimedTask{ claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}), claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}), } if err := consumer.handleClaims(context.Background(), claimed); err != nil { t.Fatal(err) } if applyCalls != 1 || len(applied) != 2 { t.Fatalf("apply calls = %d, results = %d", applyCalls, len(applied)) } if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" { t.Fatalf("Redis calls = %+v, want one batch ack", fake.calls) } } func TestDatabaseFailureLeavesEntireBatchLeased(t *testing.T) { fake := &fakeRedis{} handler := &testHandler{ query: func(_ context.Context, task queue.Task) Result { if task.CardID == 2 { return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"} } return Result{Task: task, Success: true, UsageMB: 1.5} }, apply: func(context.Context, []Result) error { return errors.New("database unavailable") }, } consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1}) claimed := []queue.ClaimedTask{ claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}), claimedTask(queue.Task{TaskID: "failed", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}), } err := consumer.handleClaims(context.Background(), claimed) if err == nil || err.Error() != "apply monthly usage result batch: database unavailable" { t.Fatalf("handleClaims() error = %v", err) } if len(fake.calls) != 0 { t.Fatalf("Redis calls = %+v, want no ACK when database update fails", fake.calls) } } func TestBusinessFailuresAreTerminalWithoutRetry(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}} failureCount := 0 handler := &testHandler{query: func(_ context.Context, task queue.Task) Result { return Result{Task: task, ErrorClass: "business", Code: 13000, Message: "busy"} }} consumer := newTestConsumer(t, fake, handler, Config{ MobileConcurrency: 2, TelecomConcurrency: 1, OnFailure: func(context.Context, Result) { failureCount++ }, }) claimed := []queue.ClaimedTask{ claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}), claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}), } if err := consumer.handleClaims(context.Background(), claimed); err != nil { t.Fatal(err) } if failureCount != 2 { t.Fatalf("failure count = %d, want 2", failureCount) } if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" { t.Fatalf("Redis calls = %+v; business failures must only be acknowledged", fake.calls) } } func TestRetryableFailuresRemainLeasedWhileSuccessesAreAcknowledged(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(1))}} var applied []Result handler := &testHandler{ query: func(_ context.Context, task queue.Task) Result { if task.CardID == 1 { return Result{Task: task, ErrorClass: "network", Retryable: true, Message: "temporary failure"} } return Result{Task: task, Success: true, UsageMB: 2} }, apply: func(_ context.Context, results []Result) error { applied = append(applied, results...) return nil }, } consumer := newTestConsumer(t, fake, handler, Config{MobileConcurrency: 2, TelecomConcurrency: 1}) claimed := []queue.ClaimedTask{ claimedTask(queue.Task{TaskID: "retry", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 1}), claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 2}), } if err := consumer.handleClaims(context.Background(), claimed); err != nil { t.Fatal(err) } if len(applied) != 2 { t.Fatalf("applied results = %d, want both result rows for handler", len(applied)) } if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" { t.Fatalf("Redis calls = %+v, want one ACK for successful/terminal result", fake.calls) } if len(fake.calls[0].args) == 0 { t.Fatal("ACK call did not include arguments") } } func TestRunRejectsAnotherActiveConsumer(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{gvar.New(nil)}} consumer := newTestConsumer(t, fake, &testHandler{}, Config{}) if err := consumer.Run(context.Background()); !errors.Is(err, ErrAlreadyRunning) { t.Fatalf("Run error = %v, want ErrAlreadyRunning", err) } } func TestProcessOnceStartsEveryRegisteredChannel(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{ gvar.New([]any{"mobile-a", "telecom-a", "mobile-b"}), gvar.New([]any{}), gvar.New([]any{}), gvar.New([]any{}), }} consumer := newTestConsumer(t, fake, &testHandler{}, Config{}) processed, err := consumer.processOnce(context.Background()) if err != nil { t.Fatal(err) } if !processed { t.Fatal("processOnce() did not start registered channels") } deadline := time.Now().Add(time.Second) for { fake.mu.Lock() callCount := len(fake.calls) fake.mu.Unlock() if callCount == 4 { break } if time.Now().After(deadline) { t.Fatalf("Redis calls = %d, want channels + 3 claims", callCount) } time.Sleep(time.Millisecond) } consumer.channelGroup.Wait() if len(fake.calls) != 4 { t.Fatalf("Redis calls = %d, want channels + 3 claims", len(fake.calls)) } } type channelRedis struct { mu sync.Mutex claims map[string][]queue.Task claimCounts map[string]int } func (r *channelRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) { return nil, errors.New("unexpected command " + command) } func (r *channelRedis) Eval(_ context.Context, script string, _ int64, keys []string, args []any) (*gvar.Var, error) { r.mu.Lock() defer r.mu.Unlock() if strings.Contains(script, "ZRANGEBYSCORE") { chanKey := strings.TrimPrefix(keys[0], "flow:") r.claimCounts[chanKey]++ tasks := r.claims[chanKey] if len(tasks) == 0 { return gvar.New([]any{}), nil } task := tasks[0] r.claims[chanKey] = tasks[1:] return gvar.New([]any{taskMember(task), "token-" + task.TaskID}), nil } if len(keys) == 2 && len(args) == 2 { return gvar.New(int64(1)), nil } return nil, errors.New("unexpected eval") } func TestSlowChannelDoesNotBlockAnotherChannelFromClaimingAgain(t *testing.T) { redis := &channelRedis{ claims: map[string][]queue.Task{ "telecom-a": {{TaskID: "t1", RunID: "run", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}}, "mobile-a": { {TaskID: "m1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1}, {TaskID: "m2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1}, }, }, claimCounts: make(map[string]int), } telecomStarted := make(chan struct{}) releaseTelecom := make(chan struct{}) handler := &testHandler{query: func(_ context.Context, task queue.Task) Result { if task.ChanKey == "telecom-a" { close(telecomStarted) <-releaseTelecom } return Result{Task: task, Success: true} }} store, err := queue.New(redis, "flow:") if err != nil { t.Fatal(err) } consumer, err := New(store, handler, Config{ClaimBatch: 1, MobileConcurrency: 1, TelecomConcurrency: 1, LeaseTTL: time.Minute}) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() consumer.startChannel(ctx, "telecom-a") consumer.startChannel(ctx, "mobile-a") select { case <-telecomStarted: case <-time.After(time.Second): t.Fatal("telecom channel did not start") } if consumer.startChannel(ctx, "telecom-a") { t.Fatal("channel executor started twice while first executor is running") } deadline := time.Now().Add(time.Second) for { redis.mu.Lock() mobileClaims := redis.claimCounts["mobile-a"] redis.mu.Unlock() if mobileClaims >= 2 { break } if time.Now().After(deadline) { t.Fatalf("mobile claim count = %d while telecom is blocked", mobileClaims) } time.Sleep(time.Millisecond) } close(releaseTelecom) consumer.channelGroup.Wait() } func TestClaimedBatchRenewsLeaseUntilAcknowledged(t *testing.T) { fake := &fakeRedis{} handler := &testHandler{query: func(_ context.Context, task queue.Task) Result { time.Sleep(45 * time.Millisecond) return Result{Task: task, Success: true} }} consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1}) claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})} if err := consumer.handleClaims(context.Background(), claimed); err != nil { t.Fatal(err) } fake.mu.Lock() defer fake.mu.Unlock() if len(fake.calls) < 2 { t.Fatalf("Redis calls = %d, want lease renewal and ACK", len(fake.calls)) } } func TestLostBatchLeasePreventsApplyAndAck(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(0))}} applyCalls := 0 handler := &testHandler{ query: func(ctx context.Context, task queue.Task) Result { <-ctx.Done() return Result{Task: task, ErrorClass: "canceled"} }, apply: func(context.Context, []Result) error { applyCalls++ return nil }, } consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1}) claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})} err := consumer.handleClaims(context.Background(), claimed) if !errors.Is(err, ErrLeaseLost) { t.Fatalf("handleClaims() error = %v, want ErrLeaseLost", err) } if applyCalls != 0 { t.Fatalf("apply calls = %d, want 0 after lease loss", applyCalls) } fake.mu.Lock() defer fake.mu.Unlock() if len(fake.calls) != 1 { t.Fatalf("Redis calls = %d, want only failed lease renewal", len(fake.calls)) } } func newTrackingHandler(release chan struct{}, current, maximum *atomic.Int32) *testHandler { return &testHandler{query: func(_ context.Context, task queue.Task) Result { active := current.Add(1) for { observed := maximum.Load() if active <= observed || maximum.CompareAndSwap(observed, active) { break } } <-release current.Add(-1) return Result{Task: task, Success: true} }} } func TestMobileChannelsHaveIndependentConcurrencyPools(t *testing.T) { var current atomic.Int32 var maximum atomic.Int32 release := make(chan struct{}) consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, ¤t, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1}) var group sync.WaitGroup for index := 0; index < 3; index++ { group.Add(1) go func(index int) { defer group.Done() task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-" + string(rune('a'+index)), TempKey: mobileTempKey, CardID: int64(index + 1)} _ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS) }(index) } time.Sleep(100 * time.Millisecond) if maximum.Load() != 3 { t.Fatalf("maximum in-flight across channels = %d, want 3 (one pool per channel)", maximum.Load()) } close(release) group.Wait() } func TestChannelConcurrencyPoolCapsWithinChannel(t *testing.T) { var current atomic.Int32 var maximum atomic.Int32 release := make(chan struct{}) consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, ¤t, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1}) var group sync.WaitGroup for index := 0; index < 3; index++ { group.Add(1) go func(index int) { defer group.Done() task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-a", TempKey: mobileTempKey, CardID: int64(index + 1)} _ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS) }(index) } time.Sleep(100 * time.Millisecond) if maximum.Load() != 2 { t.Fatalf("maximum in-flight within one channel = %d, want 2", maximum.Load()) } close(release) group.Wait() } func TestMobileAndTelecomUseIndependentChannelPools(t *testing.T) { started := make(chan carrier, 2) release := make(chan struct{}) handler := &testHandler{query: func(_ context.Context, task queue.Task) Result { channelCarrier, _ := policyForTemplate(task.TempKey, Config{}) started <- channelCarrier <-release return Result{Task: task, Success: true} }} consumer := newTestConsumer(t, &fakeRedis{}, handler, Config{MobileConcurrency: 1, TelecomConcurrency: 1}) go func() { task := queue.Task{TaskID: "mobile", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1} _ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS) }() go func() { task := queue.Task{TaskID: "telecom", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2} _ = consumer.queryClaim(context.Background(), claimedTask(task), carrierTelecom, telecomQPS) }() seen := map[carrier]bool{} for len(seen) < 2 { select { case value := <-started: seen[value] = true case <-time.After(time.Second): t.Fatalf("carrier pools did not progress independently: %+v", seen) } } close(release) } func TestUnknownTemplateUsesGenericStrategy(t *testing.T) { fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}} queryCalls := 0 failures := 0 applyCalls := 0 handler := &testHandler{ query: func(_ context.Context, task queue.Task) Result { queryCalls++ return Result{Task: task, Success: true, UsageMB: 1} }, apply: func(_ context.Context, results []Result) error { applyCalls++ for _, result := range results { if !result.Success || result.BlockChannel { t.Fatalf("unexpected result = %+v", result) } } return nil }, } consumer := newTestConsumer(t, fake, handler, Config{OnFailure: func(context.Context, Result) { failures++ }}) claimed := []queue.ClaimedTask{ claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 1, CardID: 1}), claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 2, CardID: 2}), } if err := consumer.handleClaims(context.Background(), claimed); err != nil { t.Fatal(err) } if queryCalls != 2 || failures != 0 || applyCalls != 1 { t.Fatalf("query calls = %d, failures = %d, apply calls = %d", queryCalls, failures, applyCalls) } if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" { t.Fatalf("Redis calls = %+v, want one ack script", fake.calls) } if len(fake.responses) != 0 { t.Fatalf("unexpected leftover scripted responses: %+v", fake.responses) } } func TestChannelPacerLimitsEachChannelIndependently(t *testing.T) { pacer := newChannelPacer() now := time.Unix(100, 0) pacedQPS := 30.0 if delay := pacer.reserve("mobile-a", pacedQPS, now); delay != 0 { t.Fatalf("first reservation delay = %s", delay) } if delay := pacer.reserve("mobile-a", pacedQPS, now); delay < 33*time.Millisecond || delay > 34*time.Millisecond { t.Fatalf("second reservation delay = %s", delay) } if delay := pacer.reserve("mobile-b", pacedQPS, now); delay != 0 { t.Fatalf("independent channel delay = %s", delay) } } func TestChannelPacerSkipsWhenQPSIsDisabled(t *testing.T) { pacer := newChannelPacer() now := time.Unix(100, 0) if delay := pacer.reserve("mobile-a", 0, now); delay != 0 { t.Fatalf("disabled qps delay = %s", delay) } if len(pacer.next) != 0 { t.Fatalf("disabled qps should not reserve channel slots, got %+v", pacer.next) } } func TestNewDefaultsTaskTimeout(t *testing.T) { consumer := newTestConsumer(t, &fakeRedis{}, &testHandler{}, Config{}) if consumer.config.TaskTimeout != 30*time.Second { t.Fatalf("TaskTimeout = %s, want 30s default", consumer.config.TaskTimeout) } if consumer.config.GenericQPS != defaultGenericQPS { t.Fatalf("GenericQPS = %v, want %v default", consumer.config.GenericQPS, defaultGenericQPS) } if _, err := New(consumer.store, &testHandler{}, Config{LeaseTTL: 10 * time.Second}); err == nil { t.Fatal("New() with LeaseTTL below default TaskTimeout should fail") } } func TestPolicyForTemplateUsesTemplateValues(t *testing.T) { tests := []struct { name string tempKey string carrier carrier qps float64 }{ {name: "mobile", tempKey: mobileTempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "mobile mock", tempKey: mobileMockKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Tengyu", tempKey: sharedTengyuTempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Xuzhou mobile", tempKey: sharedXuzhouMobileTempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Lelian", tempKey: sharedLelianTempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Qianxun", tempKey: sharedQianxunTempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Henan", tempKey: sharedHenanTempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Daihe", tempKey: sharedDaiheTempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Beixin", tempKey: sharedBeixinTempKey, carrier: carrierMobile, qps: mobileQPS}, // lanshang_8000/8003 are OneLink shared-token templates whose names // omit the sim_onelink_ prefix. Before this case was added they fell // through to carrierGeneric and were throttled to 1.5 QPS, which made a // 6k-card status/traffic/IMEI run take over an hour per channel. {name: "shared Lanshang 8000", tempKey: sharedLanshang8000TempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Lanshang 8003", tempKey: sharedLanshang8003TempKey, carrier: carrierMobile, qps: mobileQPS}, {name: "shared Lanshang case and padding", tempKey: " LANSHANG_8000 ", carrier: carrierMobile, qps: mobileQPS}, {name: "telecom", tempKey: telecomTempKey, carrier: carrierTelecom, qps: telecomQPS}, {name: "telecom v3", tempKey: " SIM_DX_V3 ", carrier: carrierTelecom, qps: telecomQPS}, {name: "unknown telecom version", tempKey: "sim_dx_v2_future", carrier: carrierGeneric, qps: defaultGenericQPS}, {name: "unknown template", tempKey: "unknown-template", carrier: carrierGeneric, qps: defaultGenericQPS}, {name: "missing template", tempKey: "", carrier: carrierGeneric, qps: defaultGenericQPS}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { gotCarrier, gotQPS := policyForTemplate(test.tempKey, Config{}) if gotCarrier != test.carrier || gotQPS != test.qps { t.Fatalf("policyForTemplate(%q) = %q,%v", test.tempKey, gotCarrier, gotQPS) } }) } } func TestPolicyForTemplateConfigurationOverridesDefaults(t *testing.T) { config := Config{ GenericQPS: 2.5, TemplateQPS: map[string]float64{ " SIM_YD_V5 ": 12, "unknown-template": 3, "invalid": 0, }, } config.setDefaults() if gotCarrier, gotQPS := policyForTemplate("sim_yd_v5", config); gotCarrier != carrierMobile || gotQPS != 12 { t.Fatalf("mobile override = %q,%v, want mobile,12", gotCarrier, gotQPS) } if gotCarrier, gotQPS := policyForTemplate("unknown-template", config); gotCarrier != carrierGeneric || gotQPS != 3 { t.Fatalf("generic override = %q,%v, want generic,3", gotCarrier, gotQPS) } if gotCarrier, gotQPS := policyForTemplate("other-template", config); gotCarrier != carrierGeneric || gotQPS != 2.5 { t.Fatalf("generic default = %q,%v, want generic,2.5", gotCarrier, gotQPS) } } // TestFailurePoolAlertThresholdIsObservabilityOnly 覆盖失败池告警: // - 未配置阀值(0)时不发 HLEN 查询 // - 失败数低于阀值时不告警 // - 超过阀值时发一次 HLEN 并打 WARN,但绝不截断 Run // // 告警本身是纯观测行为,所以断言的是「有没有多打 Redis 命令」, // 而不是「Run 有没有被改坏」——后者必须永远为「没被改坏」。 func TestFailurePoolAlertThresholdIsObservabilityOnly(t *testing.T) { newMeta := func(failed int64) queue.RunMeta { return queue.RunMeta{RunID: "run-1", Status: "completed_with_failures", FailedCount: failed, CurrentRound: 1} } cases := []struct { name string threshold int64 failed int64 wantHLenCalls int }{ {"未配置阀值不查询", 0, 1_000_000, 0}, {"低于阀值不告警", 50000, 49999, 0}, {"等于阀值不告警", 50000, 50000, 0}, {"超过阀值告警一次", 50000, 50001, 1}, {"远超阀值也只告警一次", 50000, 500_000, 1}, } for _, tc := range cases { fake := &fakeRedis{} consumer := newTestConsumer(t, fake, &testHandler{}, Config{ FailurePoolAlertThreshold: tc.threshold, }) consumer.reportFailurePoolSize(context.Background(), "run-1", newMeta(tc.failed)) hlen := 0 for _, call := range fake.calls { if call.command == "HLEN" { hlen++ } } if hlen != tc.wantHLenCalls { t.Fatalf("%s: HLEN 调用 %d 次, want %d", tc.name, hlen, tc.wantHLenCalls) } } } // TestFailurePoolAlertReadsCorrectKey 告警必须查该 Run 自己的失败池键。 func TestFailurePoolAlertReadsCorrectKey(t *testing.T) { fake := &fakeRedis{} consumer := newTestConsumer(t, fake, &testHandler{}, Config{FailurePoolAlertThreshold: 10}) consumer.reportFailurePoolSize(context.Background(), "run-abc", queue.RunMeta{ RunID: "run-abc", Status: "completed_with_failures", FailedCount: 11, CurrentRound: 1, }) var key string for _, call := range fake.calls { if call.command == "HLEN" { if len(call.args) > 0 { key, _ = call.args[0].(string) } } } want, _ := consumer.store.FailedKey("run-abc") if key != want { t.Fatalf("HLEN key = %q, want %q", key, want) } }