chore: 初始开源发布

iotlcm-worker-svr:异步执行中枢(RabbitMQ 消费者 + 全量同步编排)。

- Go module:iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr(214 文件)
- 凭据与内部标识脱敏:MySQL/Redis 凭据、私网与生产 IP、内网域名、
  库名 xiujie_admin → iotlcm、哨兵名占位
- 159 处真实手机号 → 合法 11 位编造号(保留号段与后 4 位,维持互异,
  不破坏「校验 MSISDN 格式」等测试断言)
- 移除内部资产:CI 触发器、Jenkins 触发测试文档、预演门禁与镜像
  digest 记录
- Dockerfile 改用公共基础镜像(golang:1.25-alpine → alpine:3.23),
  移除写死的 git 凭据
- README 重写(架构/全量同步模型/配置/已知失败),补 Apache-2.0
  LICENSE,LF 行尾 + .gitattributes

已知:3 个测试包在源仓同一提交上同样失败(client/upstream、
service/allocate、service/inbound),属既有缺陷,本仓未引入亦未修复。
This commit is contained in:
MiFaZhan
2026-10-22 16:37:56 +08:00
commit 2bec257289
361 changed files with 71524 additions and 0 deletions
File diff suppressed because it is too large Load Diff
+706
View File
@@ -0,0 +1,706 @@
package zset
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/gogf/gf/v2/container/gvar"
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
)
type redisCall struct {
command string
args []any
}
type fakeRedis struct {
mu sync.Mutex
calls []redisCall
responses []*gvar.Var
}
func (f *fakeRedis) Do(_ context.Context, command string, args ...any) (*gvar.Var, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, redisCall{command: command, args: args})
if len(f.responses) == 0 {
return gvar.New(int64(1)), nil
}
response := f.responses[0]
f.responses = f.responses[1:]
return response, nil
}
func (f *fakeRedis) Eval(_ context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
allArgs := []any{script, numKeys}
for _, key := range keys {
allArgs = append(allArgs, key)
}
allArgs = append(allArgs, args...)
return f.Do(context.Background(), "EVAL", allArgs...)
}
type testHandler struct {
query func(context.Context, queue.Task) Result
apply func(context.Context, []Result) error
}
func (h *testHandler) Query(ctx context.Context, task queue.Task) Result {
if h.query != nil {
return h.query(ctx, task)
}
return Result{Task: task, Success: true, UsageMB: float64(task.CardID)}
}
func (h *testHandler) Apply(ctx context.Context, results []Result) error {
if h.apply != nil {
return h.apply(ctx, results)
}
return nil
}
func taskMember(task queue.Task) string {
value, _ := json.Marshal(task)
return string(value)
}
func claimedTask(task queue.Task) queue.ClaimedTask {
return queue.ClaimedTask{Task: task, Member: taskMember(task), LeaseToken: "token-" + task.TaskID}
}
func newTestConsumer(t *testing.T, fake *fakeRedis, handler Handler, config Config) *Consumer {
t.Helper()
store, err := queue.New(fake, "flow:")
if err != nil {
t.Fatal(err)
}
consumer, err := New(store, handler, config)
if err != nil {
t.Fatal(err)
}
return consumer
}
func TestRoundSummaryAccumulatesAndFlushes(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{gvar.New([]any{"runId", "run-1"})}}
consumer := newTestConsumer(t, fake, &testHandler{}, Config{RoundRetryEnabled: true})
started := time.Date(2026, 9, 22, 3, 0, 0, 0, time.UTC)
consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
claimedTask(queue.Task{RunID: "run-1", TaskID: "a"}),
claimedTask(queue.Task{RunID: "run-1", TaskID: "b"}),
}, []queue.TaskOutcome{
{Success: true},
{Retry: true},
}, started)
consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
claimedTask(queue.Task{RunID: "run-1", TaskID: "c"}),
}, []queue.TaskOutcome{{Success: false}}, started.Add(2*time.Second))
key := "run-1\x001"
consumer.summaryMu.Lock()
summary := *consumer.roundSummaries[key]
consumer.summaryMu.Unlock()
if summary.inputCount != 3 || summary.successCount != 1 || summary.failedCount != 2 || summary.deferredCount != 1 || summary.terminalFailedCount != 1 {
t.Fatalf("unexpected summary: %+v", summary)
}
if !summary.startedAt.Equal(started) {
t.Fatalf("startedAt = %v, want %v", summary.startedAt, started)
}
consumer.flushRoundSummary(context.Background(), "run-1", 1)
consumer.summaryMu.Lock()
_, exists := consumer.roundSummaries[key]
consumer.summaryMu.Unlock()
if exists {
t.Fatal("round summary was not removed after flush")
}
}
func TestHandleClaimsAppliesSuccessOnceAndAcknowledgesBatch(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
applyCalls := 0
var applied []Result
handler := &testHandler{apply: func(_ context.Context, results []Result) error {
applyCalls++
applied = append(applied, results...)
return nil
}}
consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
claimed := []queue.ClaimedTask{
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
}
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
t.Fatal(err)
}
if applyCalls != 1 || len(applied) != 2 {
t.Fatalf("apply calls = %d, results = %d", applyCalls, len(applied))
}
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
t.Fatalf("Redis calls = %+v, want one batch ack", fake.calls)
}
}
func TestDatabaseFailureLeavesEntireBatchLeased(t *testing.T) {
fake := &fakeRedis{}
handler := &testHandler{
query: func(_ context.Context, task queue.Task) Result {
if task.CardID == 2 {
return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
}
return Result{Task: task, Success: true, UsageMB: 1.5}
},
apply: func(context.Context, []Result) error { return errors.New("database unavailable") },
}
consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
claimed := []queue.ClaimedTask{
claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
claimedTask(queue.Task{TaskID: "failed", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
}
err := consumer.handleClaims(context.Background(), claimed)
if err == nil || err.Error() != "apply monthly usage result batch: database unavailable" {
t.Fatalf("handleClaims() error = %v", err)
}
if len(fake.calls) != 0 {
t.Fatalf("Redis calls = %+v, want no ACK when database update fails", fake.calls)
}
}
func TestBusinessFailuresAreTerminalWithoutRetry(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
failureCount := 0
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
return Result{Task: task, ErrorClass: "business", Code: 13000, Message: "busy"}
}}
consumer := newTestConsumer(t, fake, handler, Config{
MobileConcurrency: 2,
TelecomConcurrency: 1,
OnFailure: func(context.Context, Result) {
failureCount++
},
})
claimed := []queue.ClaimedTask{
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
}
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
t.Fatal(err)
}
if failureCount != 2 {
t.Fatalf("failure count = %d, want 2", failureCount)
}
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
t.Fatalf("Redis calls = %+v; business failures must only be acknowledged", fake.calls)
}
}
func TestRetryableFailuresRemainLeasedWhileSuccessesAreAcknowledged(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(1))}}
var applied []Result
handler := &testHandler{
query: func(_ context.Context, task queue.Task) Result {
if task.CardID == 1 {
return Result{Task: task, ErrorClass: "network", Retryable: true, Message: "temporary failure"}
}
return Result{Task: task, Success: true, UsageMB: 2}
},
apply: func(_ context.Context, results []Result) error {
applied = append(applied, results...)
return nil
},
}
consumer := newTestConsumer(t, fake, handler, Config{MobileConcurrency: 2, TelecomConcurrency: 1})
claimed := []queue.ClaimedTask{
claimedTask(queue.Task{TaskID: "retry", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 1}),
claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 2}),
}
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
t.Fatal(err)
}
if len(applied) != 2 {
t.Fatalf("applied results = %d, want both result rows for handler", len(applied))
}
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
t.Fatalf("Redis calls = %+v, want one ACK for successful/terminal result", fake.calls)
}
if len(fake.calls[0].args) == 0 {
t.Fatal("ACK call did not include arguments")
}
}
func TestRunRejectsAnotherActiveConsumer(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(nil)}}
consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
if err := consumer.Run(context.Background()); !errors.Is(err, ErrAlreadyRunning) {
t.Fatalf("Run error = %v, want ErrAlreadyRunning", err)
}
}
func TestProcessOnceStartsEveryRegisteredChannel(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{
gvar.New([]any{"mobile-a", "telecom-a", "mobile-b"}),
gvar.New([]any{}),
gvar.New([]any{}),
gvar.New([]any{}),
}}
consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
processed, err := consumer.processOnce(context.Background())
if err != nil {
t.Fatal(err)
}
if !processed {
t.Fatal("processOnce() did not start registered channels")
}
deadline := time.Now().Add(time.Second)
for {
fake.mu.Lock()
callCount := len(fake.calls)
fake.mu.Unlock()
if callCount == 4 {
break
}
if time.Now().After(deadline) {
t.Fatalf("Redis calls = %d, want channels + 3 claims", callCount)
}
time.Sleep(time.Millisecond)
}
consumer.channelGroup.Wait()
if len(fake.calls) != 4 {
t.Fatalf("Redis calls = %d, want channels + 3 claims", len(fake.calls))
}
}
type channelRedis struct {
mu sync.Mutex
claims map[string][]queue.Task
claimCounts map[string]int
}
func (r *channelRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
return nil, errors.New("unexpected command " + command)
}
func (r *channelRedis) Eval(_ context.Context, script string, _ int64, keys []string, args []any) (*gvar.Var, error) {
r.mu.Lock()
defer r.mu.Unlock()
if strings.Contains(script, "ZRANGEBYSCORE") {
chanKey := strings.TrimPrefix(keys[0], "flow:")
r.claimCounts[chanKey]++
tasks := r.claims[chanKey]
if len(tasks) == 0 {
return gvar.New([]any{}), nil
}
task := tasks[0]
r.claims[chanKey] = tasks[1:]
return gvar.New([]any{taskMember(task), "token-" + task.TaskID}), nil
}
if len(keys) == 2 && len(args) == 2 {
return gvar.New(int64(1)), nil
}
return nil, errors.New("unexpected eval")
}
func TestSlowChannelDoesNotBlockAnotherChannelFromClaimingAgain(t *testing.T) {
redis := &channelRedis{
claims: map[string][]queue.Task{
"telecom-a": {{TaskID: "t1", RunID: "run", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}},
"mobile-a": {
{TaskID: "m1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
{TaskID: "m2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
},
},
claimCounts: make(map[string]int),
}
telecomStarted := make(chan struct{})
releaseTelecom := make(chan struct{})
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
if task.ChanKey == "telecom-a" {
close(telecomStarted)
<-releaseTelecom
}
return Result{Task: task, Success: true}
}}
store, err := queue.New(redis, "flow:")
if err != nil {
t.Fatal(err)
}
consumer, err := New(store, handler, Config{ClaimBatch: 1, MobileConcurrency: 1, TelecomConcurrency: 1, LeaseTTL: time.Minute})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
consumer.startChannel(ctx, "telecom-a")
consumer.startChannel(ctx, "mobile-a")
select {
case <-telecomStarted:
case <-time.After(time.Second):
t.Fatal("telecom channel did not start")
}
if consumer.startChannel(ctx, "telecom-a") {
t.Fatal("channel executor started twice while first executor is running")
}
deadline := time.Now().Add(time.Second)
for {
redis.mu.Lock()
mobileClaims := redis.claimCounts["mobile-a"]
redis.mu.Unlock()
if mobileClaims >= 2 {
break
}
if time.Now().After(deadline) {
t.Fatalf("mobile claim count = %d while telecom is blocked", mobileClaims)
}
time.Sleep(time.Millisecond)
}
close(releaseTelecom)
consumer.channelGroup.Wait()
}
func TestClaimedBatchRenewsLeaseUntilAcknowledged(t *testing.T) {
fake := &fakeRedis{}
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
time.Sleep(45 * time.Millisecond)
return Result{Task: task, Success: true}
}}
consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
t.Fatal(err)
}
fake.mu.Lock()
defer fake.mu.Unlock()
if len(fake.calls) < 2 {
t.Fatalf("Redis calls = %d, want lease renewal and ACK", len(fake.calls))
}
}
func TestLostBatchLeasePreventsApplyAndAck(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(0))}}
applyCalls := 0
handler := &testHandler{
query: func(ctx context.Context, task queue.Task) Result {
<-ctx.Done()
return Result{Task: task, ErrorClass: "canceled"}
},
apply: func(context.Context, []Result) error {
applyCalls++
return nil
},
}
consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
err := consumer.handleClaims(context.Background(), claimed)
if !errors.Is(err, ErrLeaseLost) {
t.Fatalf("handleClaims() error = %v, want ErrLeaseLost", err)
}
if applyCalls != 0 {
t.Fatalf("apply calls = %d, want 0 after lease loss", applyCalls)
}
fake.mu.Lock()
defer fake.mu.Unlock()
if len(fake.calls) != 1 {
t.Fatalf("Redis calls = %d, want only failed lease renewal", len(fake.calls))
}
}
func newTrackingHandler(release chan struct{}, current, maximum *atomic.Int32) *testHandler {
return &testHandler{query: func(_ context.Context, task queue.Task) Result {
active := current.Add(1)
for {
observed := maximum.Load()
if active <= observed || maximum.CompareAndSwap(observed, active) {
break
}
}
<-release
current.Add(-1)
return Result{Task: task, Success: true}
}}
}
func TestMobileChannelsHaveIndependentConcurrencyPools(t *testing.T) {
var current atomic.Int32
var maximum atomic.Int32
release := make(chan struct{})
consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, &current, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
var group sync.WaitGroup
for index := 0; index < 3; index++ {
group.Add(1)
go func(index int) {
defer group.Done()
task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-" + string(rune('a'+index)), TempKey: mobileTempKey, CardID: int64(index + 1)}
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
}(index)
}
time.Sleep(100 * time.Millisecond)
if maximum.Load() != 3 {
t.Fatalf("maximum in-flight across channels = %d, want 3 (one pool per channel)", maximum.Load())
}
close(release)
group.Wait()
}
func TestChannelConcurrencyPoolCapsWithinChannel(t *testing.T) {
var current atomic.Int32
var maximum atomic.Int32
release := make(chan struct{})
consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, &current, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
var group sync.WaitGroup
for index := 0; index < 3; index++ {
group.Add(1)
go func(index int) {
defer group.Done()
task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-a", TempKey: mobileTempKey, CardID: int64(index + 1)}
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
}(index)
}
time.Sleep(100 * time.Millisecond)
if maximum.Load() != 2 {
t.Fatalf("maximum in-flight within one channel = %d, want 2", maximum.Load())
}
close(release)
group.Wait()
}
func TestMobileAndTelecomUseIndependentChannelPools(t *testing.T) {
started := make(chan carrier, 2)
release := make(chan struct{})
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
channelCarrier, _ := policyForTemplate(task.TempKey, Config{})
started <- channelCarrier
<-release
return Result{Task: task, Success: true}
}}
consumer := newTestConsumer(t, &fakeRedis{}, handler, Config{MobileConcurrency: 1, TelecomConcurrency: 1})
go func() {
task := queue.Task{TaskID: "mobile", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1}
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
}()
go func() {
task := queue.Task{TaskID: "telecom", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierTelecom, telecomQPS)
}()
seen := map[carrier]bool{}
for len(seen) < 2 {
select {
case value := <-started:
seen[value] = true
case <-time.After(time.Second):
t.Fatalf("carrier pools did not progress independently: %+v", seen)
}
}
close(release)
}
func TestUnknownTemplateUsesGenericStrategy(t *testing.T) {
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
queryCalls := 0
failures := 0
applyCalls := 0
handler := &testHandler{
query: func(_ context.Context, task queue.Task) Result {
queryCalls++
return Result{Task: task, Success: true, UsageMB: 1}
},
apply: func(_ context.Context, results []Result) error {
applyCalls++
for _, result := range results {
if !result.Success || result.BlockChannel {
t.Fatalf("unexpected result = %+v", result)
}
}
return nil
},
}
consumer := newTestConsumer(t, fake, handler, Config{OnFailure: func(context.Context, Result) { failures++ }})
claimed := []queue.ClaimedTask{
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 1, CardID: 1}),
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 2, CardID: 2}),
}
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
t.Fatal(err)
}
if queryCalls != 2 || failures != 0 || applyCalls != 1 {
t.Fatalf("query calls = %d, failures = %d, apply calls = %d", queryCalls, failures, applyCalls)
}
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
t.Fatalf("Redis calls = %+v, want one ack script", fake.calls)
}
if len(fake.responses) != 0 {
t.Fatalf("unexpected leftover scripted responses: %+v", fake.responses)
}
}
func TestChannelPacerLimitsEachChannelIndependently(t *testing.T) {
pacer := newChannelPacer()
now := time.Unix(100, 0)
pacedQPS := 30.0
if delay := pacer.reserve("mobile-a", pacedQPS, now); delay != 0 {
t.Fatalf("first reservation delay = %s", delay)
}
if delay := pacer.reserve("mobile-a", pacedQPS, now); delay < 33*time.Millisecond || delay > 34*time.Millisecond {
t.Fatalf("second reservation delay = %s", delay)
}
if delay := pacer.reserve("mobile-b", pacedQPS, now); delay != 0 {
t.Fatalf("independent channel delay = %s", delay)
}
}
func TestChannelPacerSkipsWhenQPSIsDisabled(t *testing.T) {
pacer := newChannelPacer()
now := time.Unix(100, 0)
if delay := pacer.reserve("mobile-a", 0, now); delay != 0 {
t.Fatalf("disabled qps delay = %s", delay)
}
if len(pacer.next) != 0 {
t.Fatalf("disabled qps should not reserve channel slots, got %+v", pacer.next)
}
}
func TestNewDefaultsTaskTimeout(t *testing.T) {
consumer := newTestConsumer(t, &fakeRedis{}, &testHandler{}, Config{})
if consumer.config.TaskTimeout != 30*time.Second {
t.Fatalf("TaskTimeout = %s, want 30s default", consumer.config.TaskTimeout)
}
if consumer.config.GenericQPS != defaultGenericQPS {
t.Fatalf("GenericQPS = %v, want %v default", consumer.config.GenericQPS, defaultGenericQPS)
}
if _, err := New(consumer.store, &testHandler{}, Config{LeaseTTL: 10 * time.Second}); err == nil {
t.Fatal("New() with LeaseTTL below default TaskTimeout should fail")
}
}
func TestPolicyForTemplateUsesTemplateValues(t *testing.T) {
tests := []struct {
name string
tempKey string
carrier carrier
qps float64
}{
{name: "mobile", tempKey: mobileTempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "mobile mock", tempKey: mobileMockKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Tengyu", tempKey: sharedTengyuTempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Xuzhou mobile", tempKey: sharedXuzhouMobileTempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Lelian", tempKey: sharedLelianTempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Qianxun", tempKey: sharedQianxunTempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Henan", tempKey: sharedHenanTempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Daihe", tempKey: sharedDaiheTempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Beixin", tempKey: sharedBeixinTempKey, carrier: carrierMobile, qps: mobileQPS},
// lanshang_8000/8003 are OneLink shared-token templates whose names
// omit the sim_onelink_ prefix. Before this case was added they fell
// through to carrierGeneric and were throttled to 1.5 QPS, which made a
// 6k-card status/traffic/IMEI run take over an hour per channel.
{name: "shared Lanshang 8000", tempKey: sharedLanshang8000TempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Lanshang 8003", tempKey: sharedLanshang8003TempKey, carrier: carrierMobile, qps: mobileQPS},
{name: "shared Lanshang case and padding", tempKey: " LANSHANG_8000 ", carrier: carrierMobile, qps: mobileQPS},
{name: "telecom", tempKey: telecomTempKey, carrier: carrierTelecom, qps: telecomQPS},
{name: "telecom v3", tempKey: " SIM_DX_V3 ", carrier: carrierTelecom, qps: telecomQPS},
{name: "unknown telecom version", tempKey: "sim_dx_v2_future", carrier: carrierGeneric, qps: defaultGenericQPS},
{name: "unknown template", tempKey: "unknown-template", carrier: carrierGeneric, qps: defaultGenericQPS},
{name: "missing template", tempKey: "", carrier: carrierGeneric, qps: defaultGenericQPS},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
gotCarrier, gotQPS := policyForTemplate(test.tempKey, Config{})
if gotCarrier != test.carrier || gotQPS != test.qps {
t.Fatalf("policyForTemplate(%q) = %q,%v", test.tempKey, gotCarrier, gotQPS)
}
})
}
}
func TestPolicyForTemplateConfigurationOverridesDefaults(t *testing.T) {
config := Config{
GenericQPS: 2.5,
TemplateQPS: map[string]float64{
" SIM_YD_V5 ": 12,
"unknown-template": 3,
"invalid": 0,
},
}
config.setDefaults()
if gotCarrier, gotQPS := policyForTemplate("sim_yd_v5", config); gotCarrier != carrierMobile || gotQPS != 12 {
t.Fatalf("mobile override = %q,%v, want mobile,12", gotCarrier, gotQPS)
}
if gotCarrier, gotQPS := policyForTemplate("unknown-template", config); gotCarrier != carrierGeneric || gotQPS != 3 {
t.Fatalf("generic override = %q,%v, want generic,3", gotCarrier, gotQPS)
}
if gotCarrier, gotQPS := policyForTemplate("other-template", config); gotCarrier != carrierGeneric || gotQPS != 2.5 {
t.Fatalf("generic default = %q,%v, want generic,2.5", gotCarrier, gotQPS)
}
}
// TestFailurePoolAlertThresholdIsObservabilityOnly 覆盖失败池告警:
// - 未配置阀值(0)时不发 HLEN 查询
// - 失败数低于阀值时不告警
// - 超过阀值时发一次 HLEN 并打 WARN,但绝不截断 Run
//
// 告警本身是纯观测行为,所以断言的是「有没有多打 Redis 命令」,
// 而不是「Run 有没有被改坏」——后者必须永远为「没被改坏」。
func TestFailurePoolAlertThresholdIsObservabilityOnly(t *testing.T) {
newMeta := func(failed int64) queue.RunMeta {
return queue.RunMeta{RunID: "run-1", Status: "completed_with_failures", FailedCount: failed, CurrentRound: 1}
}
cases := []struct {
name string
threshold int64
failed int64
wantHLenCalls int
}{
{"未配置阀值不查询", 0, 1_000_000, 0},
{"低于阀值不告警", 50000, 49999, 0},
{"等于阀值不告警", 50000, 50000, 0},
{"超过阀值告警一次", 50000, 50001, 1},
{"远超阀值也只告警一次", 50000, 500_000, 1},
}
for _, tc := range cases {
fake := &fakeRedis{}
consumer := newTestConsumer(t, fake, &testHandler{}, Config{
FailurePoolAlertThreshold: tc.threshold,
})
consumer.reportFailurePoolSize(context.Background(), "run-1", newMeta(tc.failed))
hlen := 0
for _, call := range fake.calls {
if call.command == "HLEN" {
hlen++
}
}
if hlen != tc.wantHLenCalls {
t.Fatalf("%s: HLEN 调用 %d 次, want %d", tc.name, hlen, tc.wantHLenCalls)
}
}
}
// TestFailurePoolAlertReadsCorrectKey 告警必须查该 Run 自己的失败池键。
func TestFailurePoolAlertReadsCorrectKey(t *testing.T) {
fake := &fakeRedis{}
consumer := newTestConsumer(t, fake, &testHandler{}, Config{FailurePoolAlertThreshold: 10})
consumer.reportFailurePoolSize(context.Background(), "run-abc", queue.RunMeta{
RunID: "run-abc", Status: "completed_with_failures", FailedCount: 11, CurrentRound: 1,
})
var key string
for _, call := range fake.calls {
if call.command == "HLEN" {
if len(call.args) > 0 {
key, _ = call.args[0].(string)
}
}
}
want, _ := consumer.store.FailedKey("run-abc")
if key != want {
t.Fatalf("HLEN key = %q, want %q", key, want)
}
}
@@ -0,0 +1,480 @@
package zset
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"testing"
"time"
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
)
type runHandler struct {
query func(queue.Task) Result
apply func([]Result) error
}
func (h runHandler) Query(_ context.Context, task queue.Task) Result { return h.query(task) }
func (h runHandler) Apply(_ context.Context, results []Result) error {
if h.apply != nil {
return h.apply(results)
}
return nil
}
func runTasks(t *testing.T, store *queue.Store, run string, count int) []queue.Task {
t.Helper()
ctx := context.Background()
if err := store.CreateRun(ctx, queue.RunMeta{RunID: run}); err != nil {
t.Fatal(err)
}
tasks := make([]queue.Task, count)
for i := range tasks {
tasks[i] = queue.Task{RunID: run, TaskID: fmt.Sprintf("%s:%d", run, i), CardID: int64(i + 1), ChanKey: "channel", TempKey: mobileTempKey, ICCID: fmt.Sprintf("card-%d", i)}
}
if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != count {
t.Fatalf("enqueue %d: %v", n, err)
}
return tasks
}
func runMeta(t *testing.T, store *queue.Store, run string) queue.RunMeta {
t.Helper()
meta, err := store.GetRunMeta(context.Background(), run)
if err != nil {
t.Fatal(err)
}
return meta
}
func runConsumer(t *testing.T, store *queue.Store, handler Handler, rounds bool) *Consumer {
t.Helper()
c, err := New(store, handler, Config{RunQueueEnabled: true, RoundRetryEnabled: rounds, BlockedTasksAsFailures: !rounds, EventPrefix: "sim_status_queue", TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
if err != nil {
t.Fatal(err)
}
return c
}
func claimRun(t *testing.T, store *queue.Store, run string, now time.Time, count int) []queue.ClaimedTask {
t.Helper()
claims, err := store.ClaimRun(context.Background(), run, "channel", now, now.Add(time.Minute), count)
if err != nil || len(claims) != count {
t.Fatalf("claim got %d want %d: %v", len(claims), count, err)
}
return claims
}
func finishProducer(t *testing.T, store *queue.Store, run string) {
t.Helper()
if err := store.MarkProducerDone(context.Background(), run); err != nil {
t.Fatal(err)
}
}
func checkTerminal(t *testing.T, store *queue.Store, run string, success, failed int64) {
t.Helper()
meta := runMeta(t, store, run)
want := "completed"
if failed > 0 {
want = "completed_with_failures"
}
if meta.Status != want || meta.CompletedAt.IsZero() || meta.SuccessCount != success || meta.FailedCount != failed || meta.ActiveCount != 0 || meta.InflightCount != 0 || meta.NextRoundCount != 0 {
t.Fatalf("unexpected terminal metadata: %+v", meta)
}
}
func TestRunConsumerCountsAndCompletes(t *testing.T) {
redis := testutil.Redis(t)
ctx := context.Background()
for _, failure := range []bool{false, true} {
t.Run(fmt.Sprint(failure), func(t *testing.T) {
store, _ := queue.New(redis, "sim-status:")
run := fmt.Sprint("counts-", failure)
tasks := runTasks(t, store, run, 3)
if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != 0 {
t.Fatalf("duplicate enqueue: %d %v", n, err)
}
finishProducer(t, store, run)
handler := runHandler{query: func(task queue.Task) Result {
return Result{Task: task, Success: !failure || task.CardID != 2, ErrorClass: "business"}
}, apply: func(_ []Result) error {
if meta := runMeta(t, store, run); meta.SuccessCount != 0 || meta.FailedCount != 0 {
return errors.New("counted before DB apply")
}
return nil
}}
c := runConsumer(t, store, handler, false)
claims := claimRun(t, store, run, time.Now(), 3)
if err := c.handleRunClaims(ctx, claims); err != nil {
t.Fatal(err)
}
var failures int64
if failure {
failures = 1
}
checkTerminal(t, store, run, 3-failures, failures)
outcomes := []queue.TaskOutcome{{Success: true}, {Success: true}, {Success: true}}
if n, err := store.FinalizeRunTasks(ctx, claims, outcomes); err != nil || n != 0 {
t.Fatalf("duplicate finalize: %d %v", n, err)
}
checkTerminal(t, store, run, 3-failures, failures)
key, _ := store.RunMetaKey(run)
ttl, err := redis.Do(ctx, "TTL", key)
if err != nil || ttl.Int64() < 29*24*3600 {
t.Fatalf("retention=%v err=%v", ttl, err)
}
})
}
}
func TestRunConsumerLateProducerEmptyAndHistoricalRunCompletion(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "sim-status:")
ctx := context.Background()
if err := store.CreateRun(ctx, queue.RunMeta{RunID: "historical", Status: "running", ProducerDone: true}); err != nil {
t.Fatal(err)
}
runTasks(t, store, "empty", 0)
runTasks(t, store, "late", 1)
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}, false)
claims := claimRun(t, store, "late", time.Now(), 1)
if err := c.handleRunClaims(ctx, claims); err != nil {
t.Fatal(err)
}
if meta := runMeta(t, store, "late"); !meta.CompletedAt.IsZero() || meta.SuccessCount != 1 {
t.Fatalf("premature completion: %+v", meta)
}
finishProducer(t, store, "late")
finishProducer(t, store, "empty")
if _, err := c.processOnce(ctx); err != nil {
t.Fatal(err)
}
c.channelGroup.Wait()
checkTerminal(t, store, "late", 1, 0)
checkTerminal(t, store, "empty", 0, 0)
checkTerminal(t, store, "historical", 0, 0)
}
func TestRunConsumerRecoveryAndFencing(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "sim-status:")
ctx := context.Background()
for _, mode := range []string{"database", "network", "stale"} {
t.Run(mode, func(t *testing.T) {
runTasks(t, store, mode, 1)
finishProducer(t, store, mode)
now := time.Now()
claims := claimRun(t, store, mode, now, 1)
good := runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}
handler := good
if mode == "database" {
handler.apply = func([]Result) error { return errors.New("database unavailable") }
}
if mode == "network" {
handler.query = func(task queue.Task) Result { return Result{Task: task, Retryable: true, ErrorClass: "network"} }
}
c := runConsumer(t, store, handler, false)
if mode != "stale" {
err := c.handleRunClaims(ctx, claims)
if (err != nil) != (mode == "database") {
t.Fatalf("handle: %v", err)
}
if meta := runMeta(t, store, mode); meta.SuccessCount != 0 || meta.FailedCount != 0 || meta.ActiveCount != 1 || meta.InflightCount != 1 || meta.Status != "running" {
t.Fatalf("lost pending task: %+v", meta)
}
}
fresh := claimRun(t, store, mode, now.Add(2*time.Minute), 1)
if n, err := store.RenewRunBatch(ctx, claims, now.Add(3*time.Minute)); err != nil || n != 0 {
t.Fatalf("stale renewal: %d %v", n, err)
}
if n, err := store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil || n != 0 {
t.Fatalf("stale ack: %d %v", n, err)
}
if n, err := store.BlockRunScopedAsFailed(ctx, claims[0], queue.TaskOutcome{}); err != nil || n != -1 {
t.Fatalf("stale block: %d %v", n, err)
}
c.handler = good
if err := c.handleRunClaims(ctx, fresh); err != nil {
t.Fatal(err)
}
checkTerminal(t, store, mode, 1, 0)
})
}
}
func TestRunConsumerBlockedChannelCountsOnlyUnfinishedTasks(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "sim-status:")
ctx := context.Background()
runTasks(t, store, "blocked", 5)
runTasks(t, store, "other", 1)
// Production continues while the channel is blocked.
claims := claimRun(t, store, "blocked", time.Now(), 2)
var applied int
successQueried := make(chan struct{})
handler := runHandler{query: func(task queue.Task) Result {
if task.CardID == 1 {
close(successQueried)
return Result{Task: task, Success: true}
}
<-successQueried
return Result{Task: task, BlockChannel: true, ErrorClass: "channel_limited", Code: 797, ErrCode: "S00004", Message: "limited"}
}, apply: func(results []Result) error { applied = countSuccessful(results); return nil }}
c := runConsumer(t, store, handler, false)
// Ensure the success response arrives before the channel-limit response.
if err := c.handleRunClaims(ctx, claims); err != nil {
t.Fatal(err)
}
meta := runMeta(t, store, "blocked")
if applied != 1 || meta.SuccessCount != 1 || meta.FailedCount != 4 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
t.Fatalf("applied=%d meta=%+v", applied, meta)
}
if n, err := store.BlockRunScopedAsFailed(ctx, claims[1], queue.TaskOutcome{}); err != nil || n != -1 {
t.Fatalf("duplicate block: %d %v", n, err)
}
late := queue.Task{TaskID: "late", RunID: "blocked", ChanKey: "channel", CardID: 99}
if n, err := store.EnqueueRunBatch(ctx, "blocked", "channel", []queue.Task{late}, time.Now()); err != nil || n != 0 {
t.Fatalf("blocked enqueue: %d %v", n, err)
}
key, _ := store.FailedKey("blocked")
failures, err := redis.Do(ctx, "HLEN", key)
if err != nil || failures.Int() != 4 {
t.Fatalf("failure records=%v err=%v", failures, err)
}
stored, err := redis.Do(ctx, "HGET", key, "blocked:4")
if err != nil {
t.Fatal(err)
}
var record queue.FailureRecord
if err := json.Unmarshal([]byte(stored.String()), &record); err != nil {
t.Fatal(err)
}
if record.TaskID != "blocked:4" || record.CardID != 5 || record.ICCID != "card-4" || record.Attempts != 0 || record.FailureType != "channel_blocked" || record.ErrCode != "S00004" {
t.Fatalf("failure identity/reason: %+v", record)
}
stateKey, _ := store.TaskStateKey("blocked")
state, err := redis.Do(ctx, "HGET", stateKey, "blocked:4")
if err != nil || state.MapStrAny()["status"] != "channel_blocked" {
t.Fatalf("blocked state=%v err=%v", state, err)
}
if meta := runMeta(t, store, "other"); meta.ActiveCount != 1 || meta.FailedCount != 0 {
t.Fatalf("other run affected: %+v", meta)
}
finishProducer(t, store, "blocked")
if err := c.completeRun(ctx, "blocked"); err != nil {
t.Fatal(err)
}
checkTerminal(t, store, "blocked", 1, 4)
}
func TestRunConsumerMonthlyRoundsRemainEnabled(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "flow:")
ctx := context.Background()
runTasks(t, store, "monthly", 1)
finishProducer(t, store, "monthly")
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, true)
for round := 1; round <= 5; round++ {
if err := c.handleRunClaims(ctx, claimRun(t, store, "monthly", time.Now(), 1)); err != nil {
t.Fatal(err)
}
if round < 5 {
if meta := runMeta(t, store, "monthly"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
t.Fatalf("round %d: %+v", round, meta)
}
if ok, err := store.AdvanceRunRound(ctx, "monthly", 5); err != nil || !ok {
t.Fatalf("advance %v %v", ok, err)
}
}
}
checkTerminal(t, store, "monthly", 0, 1)
}
func TestRunConsumerStatusRetriesRetryableResultsForThreeRounds(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "sim-status:")
ctx := context.Background()
runTasks(t, store, "status", 1)
finishProducer(t, store, "status")
c, err := New(store, runHandler{query: func(task queue.Task) Result {
return Result{Task: task, ErrorClass: "carrier_retry", Retryable: true, Code: 9999, ErrCode: "S00004"}
}}, Config{
RunQueueEnabled: true,
RoundRetryEnabled: true,
MaxRounds: 3,
EventPrefix: "sim_status_queue",
ShouldRetryNextRound: func(result Result) bool {
return result.Retryable
},
TemplateQPS: map[string]float64{mobileTempKey: 1000000},
})
if err != nil {
t.Fatal(err)
}
for round := 1; round <= 3; round++ {
if err := c.handleRunClaims(ctx, claimRun(t, store, "status", time.Now(), 1)); err != nil {
t.Fatal(err)
}
if round < 3 {
if meta := runMeta(t, store, "status"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
t.Fatalf("round %d: %+v", round, meta)
}
if ok, err := store.AdvanceRunRound(ctx, "status", 3); err != nil || !ok {
t.Fatalf("advance %v %v", ok, err)
}
}
}
checkTerminal(t, store, "status", 0, 1)
}
func TestReplayMaxRoundOverridesConsumerDefault(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "sim-status:")
ctx := context.Background()
if err := store.CreateRun(ctx, queue.RunMeta{RunID: "replay-one", Operation: "status", SyncType: "replay", MaxRound: 1}); err != nil {
t.Fatal(err)
}
task := queue.Task{RunID: "replay-one", TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "channel", TempKey: mobileTempKey}
if _, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []queue.Task{task}, time.Now()); err != nil {
t.Fatal(err)
}
finishProducer(t, store, "replay-one")
c, err := New(store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, Config{RunQueueEnabled: true, RoundRetryEnabled: true, MaxRounds: 3, TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
if err != nil {
t.Fatal(err)
}
if err := c.handleRunClaims(ctx, claimRun(t, store, "replay-one", time.Now(), 1)); err != nil {
t.Fatal(err)
}
checkTerminal(t, store, "replay-one", 0, 1)
meta := runMeta(t, store, "replay-one")
if meta.CurrentRound != 1 || meta.NextRoundCount != 0 {
t.Fatalf("replay advanced rounds: %+v", meta)
}
}
func TestRunQueueModeDoesNotEnableRoundRetry(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "sim-status:")
runTasks(t, store, "single-round", 1)
finishProducer(t, store, "single-round")
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, false)
if err := c.handleRunClaims(context.Background(), claimRun(t, store, "single-round", time.Now(), 1)); err != nil {
t.Fatal(err)
}
checkTerminal(t, store, "single-round", 0, 1)
if meta := runMeta(t, store, "single-round"); meta.CurrentRound != 1 {
t.Fatalf("unexpected retry: %+v", meta)
}
}
func TestMonthlyChannelBlockKeepsExistingCountingPolicy(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "flow:")
ctx := context.Background()
runTasks(t, store, "monthly-block", 2)
claimRun(t, store, "monthly-block", time.Now(), 1)
for i := 0; i < 2; i++ {
want := int64(2)
if i == 1 {
want = 0
}
if n, err := store.BlockRunScoped(ctx, "monthly-block", "channel"); err != nil || n != want {
t.Fatalf("monthly block=%d err=%v", n, err)
}
}
if meta := runMeta(t, store, "monthly-block"); meta.FailedCount != 0 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
t.Fatalf("monthly counting changed: %+v", meta)
}
}
func TestRunConsumerConcurrentRuns(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "sim-status:")
ctx := context.Background()
var mu sync.Mutex
applied := map[string]int{}
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }, apply: func(results []Result) error {
mu.Lock()
defer mu.Unlock()
for _, result := range results {
applied[result.Task.RunID]++
}
return nil
}}, false)
for _, run := range []string{"a", "b"} {
runTasks(t, store, run, 3)
finishProducer(t, store, run)
}
if started, err := c.processOnce(ctx); err != nil || !started {
t.Fatalf("process: %v %v", started, err)
}
c.channelGroup.Wait()
for _, run := range []string{"a", "b"} {
checkTerminal(t, store, run, 3, 0)
if applied[run] != 3 {
t.Fatalf("applied %s: %d", run, applied[run])
}
}
}
// TestRunConsumerCoveredResultsCompleteWithoutFailures drives the full
// consumer path for a covered result: the round must close, the run must end
// as "completed" rather than "completed_with_failures", and the failure pool
// must stay empty. This is the IMEI 14000 "carrier has no record" case — on
// 2026-10-20 it accounted for every failure recorded in production while the
// run success rate read 21%.
func TestRunConsumerCoveredResultsCompleteWithoutFailures(t *testing.T) {
redis := testutil.Redis(t)
store, _ := queue.New(redis, "imei:")
ctx := context.Background()
runTasks(t, store, "covered", 4)
finishProducer(t, store, "covered")
queried := 0
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result {
queried++
switch task.CardID {
case 1:
return Result{Task: task, Success: true, Imei: "865466070817587", ImeiSynced: true}
case 2:
return Result{Task: task, ErrorClass: "no_record", Code: 14000, ImeiSynced: true, Covered: true, Message: "系统无记录,请检查输入参数是否正确"}
case 3:
return Result{Task: task, ErrorClass: "data_invalid", ImeiSynced: true, Covered: true, Message: "carrier returned no imei value"}
default:
return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
}
}}, false)
if err := c.handleRunClaims(ctx, claimRun(t, store, "covered", time.Now(), 4)); err != nil {
t.Fatal(err)
}
if queried != 4 {
t.Fatalf("queried = %d, want 4", queried)
}
meta := runMeta(t, store, "covered")
// Three cards produced a usable or definitive answer (success + 2 covered),
// and only the business failure is counted. Without the covered channel the
// two 14000-style answers would have made failedCount=3.
if meta.Status != "completed_with_failures" || meta.SuccessCount != 3 || meta.FailedCount != 1 || meta.NextRoundCount != 0 {
t.Fatalf("unexpected run accounting: %+v", meta)
}
failureKey, _ := store.FailedKey("covered")
size, err := redis.Do(ctx, "HLEN", failureKey)
if err != nil || size.Int64() != 1 {
t.Fatalf("failure pool entries = %d err=%v, want 1 (only the business failure)", size.Int64(), err)
}
for _, taskID := range []string{"covered:1", "covered:2"} {
exists, err := redis.Do(ctx, "HEXISTS", failureKey, taskID)
if err != nil || exists.Int() != 0 {
t.Fatalf("covered card %s leaked into the failure pool: %v %v", taskID, exists, err)
}
}
exists, err := redis.Do(ctx, "HEXISTS", failureKey, "covered:3")
if err != nil || exists.Int() != 1 {
t.Fatalf("the business failure is missing from the pool: %v %v", exists, err)
}
}