chore: 初始开源发布
iotlcm-worker-svr:异步执行中枢(RabbitMQ 消费者 + 全量同步编排)。 - Go module:iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr(214 文件) - 凭据与内部标识脱敏:MySQL/Redis 凭据、私网与生产 IP、内网域名、 库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号(保留号段与后 4 位,维持互异, 不破坏「校验 MSISDN 格式」等测试断言) - 移除内部资产:CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像(golang:1.25-alpine → alpine:3.23), 移除写死的 git 凭据 - README 重写(架构/全量同步模型/配置/已知失败),补 Apache-2.0 LICENSE,LF 行尾 + .gitattributes 已知:3 个测试包在源仓同一提交上同样失败(client/upstream、 service/allocate、service/inbound),属既有缺陷,本仓未引入亦未修复。
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,706 @@
|
||||
package zset
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
type redisCall struct {
|
||||
command string
|
||||
args []any
|
||||
}
|
||||
|
||||
type fakeRedis struct {
|
||||
mu sync.Mutex
|
||||
calls []redisCall
|
||||
responses []*gvar.Var
|
||||
}
|
||||
|
||||
func (f *fakeRedis) Do(_ context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, redisCall{command: command, args: args})
|
||||
if len(f.responses) == 0 {
|
||||
return gvar.New(int64(1)), nil
|
||||
}
|
||||
response := f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (f *fakeRedis) Eval(_ context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
allArgs := []any{script, numKeys}
|
||||
for _, key := range keys {
|
||||
allArgs = append(allArgs, key)
|
||||
}
|
||||
allArgs = append(allArgs, args...)
|
||||
return f.Do(context.Background(), "EVAL", allArgs...)
|
||||
}
|
||||
|
||||
type testHandler struct {
|
||||
query func(context.Context, queue.Task) Result
|
||||
apply func(context.Context, []Result) error
|
||||
}
|
||||
|
||||
func (h *testHandler) Query(ctx context.Context, task queue.Task) Result {
|
||||
if h.query != nil {
|
||||
return h.query(ctx, task)
|
||||
}
|
||||
return Result{Task: task, Success: true, UsageMB: float64(task.CardID)}
|
||||
}
|
||||
|
||||
func (h *testHandler) Apply(ctx context.Context, results []Result) error {
|
||||
if h.apply != nil {
|
||||
return h.apply(ctx, results)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func taskMember(task queue.Task) string {
|
||||
value, _ := json.Marshal(task)
|
||||
return string(value)
|
||||
}
|
||||
|
||||
func claimedTask(task queue.Task) queue.ClaimedTask {
|
||||
return queue.ClaimedTask{Task: task, Member: taskMember(task), LeaseToken: "token-" + task.TaskID}
|
||||
}
|
||||
|
||||
func newTestConsumer(t *testing.T, fake *fakeRedis, handler Handler, config Config) *Consumer {
|
||||
t.Helper()
|
||||
store, err := queue.New(fake, "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer, err := New(store, handler, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return consumer
|
||||
}
|
||||
|
||||
func TestRoundSummaryAccumulatesAndFlushes(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New([]any{"runId", "run-1"})}}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{RoundRetryEnabled: true})
|
||||
started := time.Date(2026, 9, 22, 3, 0, 0, 0, time.UTC)
|
||||
consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{RunID: "run-1", TaskID: "a"}),
|
||||
claimedTask(queue.Task{RunID: "run-1", TaskID: "b"}),
|
||||
}, []queue.TaskOutcome{
|
||||
{Success: true},
|
||||
{Retry: true},
|
||||
}, started)
|
||||
consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{RunID: "run-1", TaskID: "c"}),
|
||||
}, []queue.TaskOutcome{{Success: false}}, started.Add(2*time.Second))
|
||||
|
||||
key := "run-1\x001"
|
||||
consumer.summaryMu.Lock()
|
||||
summary := *consumer.roundSummaries[key]
|
||||
consumer.summaryMu.Unlock()
|
||||
if summary.inputCount != 3 || summary.successCount != 1 || summary.failedCount != 2 || summary.deferredCount != 1 || summary.terminalFailedCount != 1 {
|
||||
t.Fatalf("unexpected summary: %+v", summary)
|
||||
}
|
||||
if !summary.startedAt.Equal(started) {
|
||||
t.Fatalf("startedAt = %v, want %v", summary.startedAt, started)
|
||||
}
|
||||
|
||||
consumer.flushRoundSummary(context.Background(), "run-1", 1)
|
||||
consumer.summaryMu.Lock()
|
||||
_, exists := consumer.roundSummaries[key]
|
||||
consumer.summaryMu.Unlock()
|
||||
if exists {
|
||||
t.Fatal("round summary was not removed after flush")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleClaimsAppliesSuccessOnceAndAcknowledgesBatch(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
|
||||
applyCalls := 0
|
||||
var applied []Result
|
||||
handler := &testHandler{apply: func(_ context.Context, results []Result) error {
|
||||
applyCalls++
|
||||
applied = append(applied, results...)
|
||||
return nil
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if applyCalls != 1 || len(applied) != 2 {
|
||||
t.Fatalf("apply calls = %d, results = %d", applyCalls, len(applied))
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v, want one batch ack", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseFailureLeavesEntireBatchLeased(t *testing.T) {
|
||||
fake := &fakeRedis{}
|
||||
handler := &testHandler{
|
||||
query: func(_ context.Context, task queue.Task) Result {
|
||||
if task.CardID == 2 {
|
||||
return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
|
||||
}
|
||||
return Result{Task: task, Success: true, UsageMB: 1.5}
|
||||
},
|
||||
apply: func(context.Context, []Result) error { return errors.New("database unavailable") },
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "failed", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
|
||||
}
|
||||
err := consumer.handleClaims(context.Background(), claimed)
|
||||
if err == nil || err.Error() != "apply monthly usage result batch: database unavailable" {
|
||||
t.Fatalf("handleClaims() error = %v", err)
|
||||
}
|
||||
if len(fake.calls) != 0 {
|
||||
t.Fatalf("Redis calls = %+v, want no ACK when database update fails", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBusinessFailuresAreTerminalWithoutRetry(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
|
||||
failureCount := 0
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
return Result{Task: task, ErrorClass: "business", Code: 13000, Message: "busy"}
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{
|
||||
MobileConcurrency: 2,
|
||||
TelecomConcurrency: 1,
|
||||
OnFailure: func(context.Context, Result) {
|
||||
failureCount++
|
||||
},
|
||||
})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failureCount != 2 {
|
||||
t.Fatalf("failure count = %d, want 2", failureCount)
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v; business failures must only be acknowledged", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryableFailuresRemainLeasedWhileSuccessesAreAcknowledged(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(1))}}
|
||||
var applied []Result
|
||||
handler := &testHandler{
|
||||
query: func(_ context.Context, task queue.Task) Result {
|
||||
if task.CardID == 1 {
|
||||
return Result{Task: task, ErrorClass: "network", Retryable: true, Message: "temporary failure"}
|
||||
}
|
||||
return Result{Task: task, Success: true, UsageMB: 2}
|
||||
},
|
||||
apply: func(_ context.Context, results []Result) error {
|
||||
applied = append(applied, results...)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "retry", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(applied) != 2 {
|
||||
t.Fatalf("applied results = %d, want both result rows for handler", len(applied))
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v, want one ACK for successful/terminal result", fake.calls)
|
||||
}
|
||||
if len(fake.calls[0].args) == 0 {
|
||||
t.Fatal("ACK call did not include arguments")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsAnotherActiveConsumer(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(nil)}}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
|
||||
if err := consumer.Run(context.Background()); !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("Run error = %v, want ErrAlreadyRunning", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessOnceStartsEveryRegisteredChannel(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{
|
||||
gvar.New([]any{"mobile-a", "telecom-a", "mobile-b"}),
|
||||
gvar.New([]any{}),
|
||||
gvar.New([]any{}),
|
||||
gvar.New([]any{}),
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
|
||||
processed, err := consumer.processOnce(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !processed {
|
||||
t.Fatal("processOnce() did not start registered channels")
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
fake.mu.Lock()
|
||||
callCount := len(fake.calls)
|
||||
fake.mu.Unlock()
|
||||
if callCount == 4 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("Redis calls = %d, want channels + 3 claims", callCount)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
consumer.channelGroup.Wait()
|
||||
if len(fake.calls) != 4 {
|
||||
t.Fatalf("Redis calls = %d, want channels + 3 claims", len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
type channelRedis struct {
|
||||
mu sync.Mutex
|
||||
claims map[string][]queue.Task
|
||||
claimCounts map[string]int
|
||||
}
|
||||
|
||||
func (r *channelRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
|
||||
return nil, errors.New("unexpected command " + command)
|
||||
}
|
||||
|
||||
func (r *channelRedis) Eval(_ context.Context, script string, _ int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if strings.Contains(script, "ZRANGEBYSCORE") {
|
||||
chanKey := strings.TrimPrefix(keys[0], "flow:")
|
||||
r.claimCounts[chanKey]++
|
||||
tasks := r.claims[chanKey]
|
||||
if len(tasks) == 0 {
|
||||
return gvar.New([]any{}), nil
|
||||
}
|
||||
task := tasks[0]
|
||||
r.claims[chanKey] = tasks[1:]
|
||||
return gvar.New([]any{taskMember(task), "token-" + task.TaskID}), nil
|
||||
}
|
||||
if len(keys) == 2 && len(args) == 2 {
|
||||
return gvar.New(int64(1)), nil
|
||||
}
|
||||
return nil, errors.New("unexpected eval")
|
||||
}
|
||||
|
||||
func TestSlowChannelDoesNotBlockAnotherChannelFromClaimingAgain(t *testing.T) {
|
||||
redis := &channelRedis{
|
||||
claims: map[string][]queue.Task{
|
||||
"telecom-a": {{TaskID: "t1", RunID: "run", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}},
|
||||
"mobile-a": {
|
||||
{TaskID: "m1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
|
||||
{TaskID: "m2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
|
||||
},
|
||||
},
|
||||
claimCounts: make(map[string]int),
|
||||
}
|
||||
telecomStarted := make(chan struct{})
|
||||
releaseTelecom := make(chan struct{})
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
if task.ChanKey == "telecom-a" {
|
||||
close(telecomStarted)
|
||||
<-releaseTelecom
|
||||
}
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
store, err := queue.New(redis, "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer, err := New(store, handler, Config{ClaimBatch: 1, MobileConcurrency: 1, TelecomConcurrency: 1, LeaseTTL: time.Minute})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
consumer.startChannel(ctx, "telecom-a")
|
||||
consumer.startChannel(ctx, "mobile-a")
|
||||
select {
|
||||
case <-telecomStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("telecom channel did not start")
|
||||
}
|
||||
if consumer.startChannel(ctx, "telecom-a") {
|
||||
t.Fatal("channel executor started twice while first executor is running")
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
redis.mu.Lock()
|
||||
mobileClaims := redis.claimCounts["mobile-a"]
|
||||
redis.mu.Unlock()
|
||||
if mobileClaims >= 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("mobile claim count = %d while telecom is blocked", mobileClaims)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(releaseTelecom)
|
||||
consumer.channelGroup.Wait()
|
||||
}
|
||||
|
||||
func TestClaimedBatchRenewsLeaseUntilAcknowledged(t *testing.T) {
|
||||
fake := &fakeRedis{}
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
time.Sleep(45 * time.Millisecond)
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
if len(fake.calls) < 2 {
|
||||
t.Fatalf("Redis calls = %d, want lease renewal and ACK", len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLostBatchLeasePreventsApplyAndAck(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(0))}}
|
||||
applyCalls := 0
|
||||
handler := &testHandler{
|
||||
query: func(ctx context.Context, task queue.Task) Result {
|
||||
<-ctx.Done()
|
||||
return Result{Task: task, ErrorClass: "canceled"}
|
||||
},
|
||||
apply: func(context.Context, []Result) error {
|
||||
applyCalls++
|
||||
return nil
|
||||
},
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
|
||||
err := consumer.handleClaims(context.Background(), claimed)
|
||||
if !errors.Is(err, ErrLeaseLost) {
|
||||
t.Fatalf("handleClaims() error = %v, want ErrLeaseLost", err)
|
||||
}
|
||||
if applyCalls != 0 {
|
||||
t.Fatalf("apply calls = %d, want 0 after lease loss", applyCalls)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
if len(fake.calls) != 1 {
|
||||
t.Fatalf("Redis calls = %d, want only failed lease renewal", len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func newTrackingHandler(release chan struct{}, current, maximum *atomic.Int32) *testHandler {
|
||||
return &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
active := current.Add(1)
|
||||
for {
|
||||
observed := maximum.Load()
|
||||
if active <= observed || maximum.CompareAndSwap(observed, active) {
|
||||
break
|
||||
}
|
||||
}
|
||||
<-release
|
||||
current.Add(-1)
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
}
|
||||
|
||||
func TestMobileChannelsHaveIndependentConcurrencyPools(t *testing.T) {
|
||||
var current atomic.Int32
|
||||
var maximum atomic.Int32
|
||||
release := make(chan struct{})
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, ¤t, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
var group sync.WaitGroup
|
||||
for index := 0; index < 3; index++ {
|
||||
group.Add(1)
|
||||
go func(index int) {
|
||||
defer group.Done()
|
||||
task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-" + string(rune('a'+index)), TempKey: mobileTempKey, CardID: int64(index + 1)}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
|
||||
}(index)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if maximum.Load() != 3 {
|
||||
t.Fatalf("maximum in-flight across channels = %d, want 3 (one pool per channel)", maximum.Load())
|
||||
}
|
||||
close(release)
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func TestChannelConcurrencyPoolCapsWithinChannel(t *testing.T) {
|
||||
var current atomic.Int32
|
||||
var maximum atomic.Int32
|
||||
release := make(chan struct{})
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, ¤t, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
var group sync.WaitGroup
|
||||
for index := 0; index < 3; index++ {
|
||||
group.Add(1)
|
||||
go func(index int) {
|
||||
defer group.Done()
|
||||
task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-a", TempKey: mobileTempKey, CardID: int64(index + 1)}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
|
||||
}(index)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if maximum.Load() != 2 {
|
||||
t.Fatalf("maximum in-flight within one channel = %d, want 2", maximum.Load())
|
||||
}
|
||||
close(release)
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func TestMobileAndTelecomUseIndependentChannelPools(t *testing.T) {
|
||||
started := make(chan carrier, 2)
|
||||
release := make(chan struct{})
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
channelCarrier, _ := policyForTemplate(task.TempKey, Config{})
|
||||
started <- channelCarrier
|
||||
<-release
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, handler, Config{MobileConcurrency: 1, TelecomConcurrency: 1})
|
||||
go func() {
|
||||
task := queue.Task{TaskID: "mobile", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
|
||||
}()
|
||||
go func() {
|
||||
task := queue.Task{TaskID: "telecom", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierTelecom, telecomQPS)
|
||||
}()
|
||||
|
||||
seen := map[carrier]bool{}
|
||||
for len(seen) < 2 {
|
||||
select {
|
||||
case value := <-started:
|
||||
seen[value] = true
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("carrier pools did not progress independently: %+v", seen)
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
func TestUnknownTemplateUsesGenericStrategy(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
|
||||
queryCalls := 0
|
||||
failures := 0
|
||||
applyCalls := 0
|
||||
handler := &testHandler{
|
||||
query: func(_ context.Context, task queue.Task) Result {
|
||||
queryCalls++
|
||||
return Result{Task: task, Success: true, UsageMB: 1}
|
||||
},
|
||||
apply: func(_ context.Context, results []Result) error {
|
||||
applyCalls++
|
||||
for _, result := range results {
|
||||
if !result.Success || result.BlockChannel {
|
||||
t.Fatalf("unexpected result = %+v", result)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{OnFailure: func(context.Context, Result) { failures++ }})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 2, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queryCalls != 2 || failures != 0 || applyCalls != 1 {
|
||||
t.Fatalf("query calls = %d, failures = %d, apply calls = %d", queryCalls, failures, applyCalls)
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v, want one ack script", fake.calls)
|
||||
}
|
||||
if len(fake.responses) != 0 {
|
||||
t.Fatalf("unexpected leftover scripted responses: %+v", fake.responses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelPacerLimitsEachChannelIndependently(t *testing.T) {
|
||||
pacer := newChannelPacer()
|
||||
now := time.Unix(100, 0)
|
||||
pacedQPS := 30.0
|
||||
if delay := pacer.reserve("mobile-a", pacedQPS, now); delay != 0 {
|
||||
t.Fatalf("first reservation delay = %s", delay)
|
||||
}
|
||||
if delay := pacer.reserve("mobile-a", pacedQPS, now); delay < 33*time.Millisecond || delay > 34*time.Millisecond {
|
||||
t.Fatalf("second reservation delay = %s", delay)
|
||||
}
|
||||
if delay := pacer.reserve("mobile-b", pacedQPS, now); delay != 0 {
|
||||
t.Fatalf("independent channel delay = %s", delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelPacerSkipsWhenQPSIsDisabled(t *testing.T) {
|
||||
pacer := newChannelPacer()
|
||||
now := time.Unix(100, 0)
|
||||
if delay := pacer.reserve("mobile-a", 0, now); delay != 0 {
|
||||
t.Fatalf("disabled qps delay = %s", delay)
|
||||
}
|
||||
if len(pacer.next) != 0 {
|
||||
t.Fatalf("disabled qps should not reserve channel slots, got %+v", pacer.next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewDefaultsTaskTimeout(t *testing.T) {
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, &testHandler{}, Config{})
|
||||
if consumer.config.TaskTimeout != 30*time.Second {
|
||||
t.Fatalf("TaskTimeout = %s, want 30s default", consumer.config.TaskTimeout)
|
||||
}
|
||||
if consumer.config.GenericQPS != defaultGenericQPS {
|
||||
t.Fatalf("GenericQPS = %v, want %v default", consumer.config.GenericQPS, defaultGenericQPS)
|
||||
}
|
||||
if _, err := New(consumer.store, &testHandler{}, Config{LeaseTTL: 10 * time.Second}); err == nil {
|
||||
t.Fatal("New() with LeaseTTL below default TaskTimeout should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyForTemplateUsesTemplateValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
tempKey string
|
||||
carrier carrier
|
||||
qps float64
|
||||
}{
|
||||
{name: "mobile", tempKey: mobileTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "mobile mock", tempKey: mobileMockKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Tengyu", tempKey: sharedTengyuTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Xuzhou mobile", tempKey: sharedXuzhouMobileTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Lelian", tempKey: sharedLelianTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Qianxun", tempKey: sharedQianxunTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Henan", tempKey: sharedHenanTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Daihe", tempKey: sharedDaiheTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Beixin", tempKey: sharedBeixinTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
// lanshang_8000/8003 are OneLink shared-token templates whose names
|
||||
// omit the sim_onelink_ prefix. Before this case was added they fell
|
||||
// through to carrierGeneric and were throttled to 1.5 QPS, which made a
|
||||
// 6k-card status/traffic/IMEI run take over an hour per channel.
|
||||
{name: "shared Lanshang 8000", tempKey: sharedLanshang8000TempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Lanshang 8003", tempKey: sharedLanshang8003TempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Lanshang case and padding", tempKey: " LANSHANG_8000 ", carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "telecom", tempKey: telecomTempKey, carrier: carrierTelecom, qps: telecomQPS},
|
||||
{name: "telecom v3", tempKey: " SIM_DX_V3 ", carrier: carrierTelecom, qps: telecomQPS},
|
||||
{name: "unknown telecom version", tempKey: "sim_dx_v2_future", carrier: carrierGeneric, qps: defaultGenericQPS},
|
||||
{name: "unknown template", tempKey: "unknown-template", carrier: carrierGeneric, qps: defaultGenericQPS},
|
||||
{name: "missing template", tempKey: "", carrier: carrierGeneric, qps: defaultGenericQPS},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
gotCarrier, gotQPS := policyForTemplate(test.tempKey, Config{})
|
||||
if gotCarrier != test.carrier || gotQPS != test.qps {
|
||||
t.Fatalf("policyForTemplate(%q) = %q,%v", test.tempKey, gotCarrier, gotQPS)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyForTemplateConfigurationOverridesDefaults(t *testing.T) {
|
||||
config := Config{
|
||||
GenericQPS: 2.5,
|
||||
TemplateQPS: map[string]float64{
|
||||
" SIM_YD_V5 ": 12,
|
||||
"unknown-template": 3,
|
||||
"invalid": 0,
|
||||
},
|
||||
}
|
||||
config.setDefaults()
|
||||
|
||||
if gotCarrier, gotQPS := policyForTemplate("sim_yd_v5", config); gotCarrier != carrierMobile || gotQPS != 12 {
|
||||
t.Fatalf("mobile override = %q,%v, want mobile,12", gotCarrier, gotQPS)
|
||||
}
|
||||
if gotCarrier, gotQPS := policyForTemplate("unknown-template", config); gotCarrier != carrierGeneric || gotQPS != 3 {
|
||||
t.Fatalf("generic override = %q,%v, want generic,3", gotCarrier, gotQPS)
|
||||
}
|
||||
if gotCarrier, gotQPS := policyForTemplate("other-template", config); gotCarrier != carrierGeneric || gotQPS != 2.5 {
|
||||
t.Fatalf("generic default = %q,%v, want generic,2.5", gotCarrier, gotQPS)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailurePoolAlertThresholdIsObservabilityOnly 覆盖失败池告警:
|
||||
// - 未配置阀值(0)时不发 HLEN 查询
|
||||
// - 失败数低于阀值时不告警
|
||||
// - 超过阀值时发一次 HLEN 并打 WARN,但绝不截断 Run
|
||||
//
|
||||
// 告警本身是纯观测行为,所以断言的是「有没有多打 Redis 命令」,
|
||||
// 而不是「Run 有没有被改坏」——后者必须永远为「没被改坏」。
|
||||
func TestFailurePoolAlertThresholdIsObservabilityOnly(t *testing.T) {
|
||||
newMeta := func(failed int64) queue.RunMeta {
|
||||
return queue.RunMeta{RunID: "run-1", Status: "completed_with_failures", FailedCount: failed, CurrentRound: 1}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
threshold int64
|
||||
failed int64
|
||||
wantHLenCalls int
|
||||
}{
|
||||
{"未配置阀值不查询", 0, 1_000_000, 0},
|
||||
{"低于阀值不告警", 50000, 49999, 0},
|
||||
{"等于阀值不告警", 50000, 50000, 0},
|
||||
{"超过阀值告警一次", 50000, 50001, 1},
|
||||
{"远超阀值也只告警一次", 50000, 500_000, 1},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
fake := &fakeRedis{}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{
|
||||
FailurePoolAlertThreshold: tc.threshold,
|
||||
})
|
||||
consumer.reportFailurePoolSize(context.Background(), "run-1", newMeta(tc.failed))
|
||||
|
||||
hlen := 0
|
||||
for _, call := range fake.calls {
|
||||
if call.command == "HLEN" {
|
||||
hlen++
|
||||
}
|
||||
}
|
||||
if hlen != tc.wantHLenCalls {
|
||||
t.Fatalf("%s: HLEN 调用 %d 次, want %d", tc.name, hlen, tc.wantHLenCalls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailurePoolAlertReadsCorrectKey 告警必须查该 Run 自己的失败池键。
|
||||
func TestFailurePoolAlertReadsCorrectKey(t *testing.T) {
|
||||
fake := &fakeRedis{}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{FailurePoolAlertThreshold: 10})
|
||||
consumer.reportFailurePoolSize(context.Background(), "run-abc", queue.RunMeta{
|
||||
RunID: "run-abc", Status: "completed_with_failures", FailedCount: 11, CurrentRound: 1,
|
||||
})
|
||||
|
||||
var key string
|
||||
for _, call := range fake.calls {
|
||||
if call.command == "HLEN" {
|
||||
if len(call.args) > 0 {
|
||||
key, _ = call.args[0].(string)
|
||||
}
|
||||
}
|
||||
}
|
||||
want, _ := consumer.store.FailedKey("run-abc")
|
||||
if key != want {
|
||||
t.Fatalf("HLEN key = %q, want %q", key, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
package zset
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type runHandler struct {
|
||||
query func(queue.Task) Result
|
||||
apply func([]Result) error
|
||||
}
|
||||
|
||||
func (h runHandler) Query(_ context.Context, task queue.Task) Result { return h.query(task) }
|
||||
func (h runHandler) Apply(_ context.Context, results []Result) error {
|
||||
if h.apply != nil {
|
||||
return h.apply(results)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runTasks(t *testing.T, store *queue.Store, run string, count int) []queue.Task {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if err := store.CreateRun(ctx, queue.RunMeta{RunID: run}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks := make([]queue.Task, count)
|
||||
for i := range tasks {
|
||||
tasks[i] = queue.Task{RunID: run, TaskID: fmt.Sprintf("%s:%d", run, i), CardID: int64(i + 1), ChanKey: "channel", TempKey: mobileTempKey, ICCID: fmt.Sprintf("card-%d", i)}
|
||||
}
|
||||
if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != count {
|
||||
t.Fatalf("enqueue %d: %v", n, err)
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
func runMeta(t *testing.T, store *queue.Store, run string) queue.RunMeta {
|
||||
t.Helper()
|
||||
meta, err := store.GetRunMeta(context.Background(), run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return meta
|
||||
}
|
||||
|
||||
func runConsumer(t *testing.T, store *queue.Store, handler Handler, rounds bool) *Consumer {
|
||||
t.Helper()
|
||||
c, err := New(store, handler, Config{RunQueueEnabled: true, RoundRetryEnabled: rounds, BlockedTasksAsFailures: !rounds, EventPrefix: "sim_status_queue", TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func claimRun(t *testing.T, store *queue.Store, run string, now time.Time, count int) []queue.ClaimedTask {
|
||||
t.Helper()
|
||||
claims, err := store.ClaimRun(context.Background(), run, "channel", now, now.Add(time.Minute), count)
|
||||
if err != nil || len(claims) != count {
|
||||
t.Fatalf("claim got %d want %d: %v", len(claims), count, err)
|
||||
}
|
||||
return claims
|
||||
}
|
||||
|
||||
func finishProducer(t *testing.T, store *queue.Store, run string) {
|
||||
t.Helper()
|
||||
if err := store.MarkProducerDone(context.Background(), run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkTerminal(t *testing.T, store *queue.Store, run string, success, failed int64) {
|
||||
t.Helper()
|
||||
meta := runMeta(t, store, run)
|
||||
want := "completed"
|
||||
if failed > 0 {
|
||||
want = "completed_with_failures"
|
||||
}
|
||||
if meta.Status != want || meta.CompletedAt.IsZero() || meta.SuccessCount != success || meta.FailedCount != failed || meta.ActiveCount != 0 || meta.InflightCount != 0 || meta.NextRoundCount != 0 {
|
||||
t.Fatalf("unexpected terminal metadata: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerCountsAndCompletes(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
ctx := context.Background()
|
||||
for _, failure := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(failure), func(t *testing.T) {
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
run := fmt.Sprint("counts-", failure)
|
||||
tasks := runTasks(t, store, run, 3)
|
||||
if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != 0 {
|
||||
t.Fatalf("duplicate enqueue: %d %v", n, err)
|
||||
}
|
||||
finishProducer(t, store, run)
|
||||
handler := runHandler{query: func(task queue.Task) Result {
|
||||
return Result{Task: task, Success: !failure || task.CardID != 2, ErrorClass: "business"}
|
||||
}, apply: func(_ []Result) error {
|
||||
if meta := runMeta(t, store, run); meta.SuccessCount != 0 || meta.FailedCount != 0 {
|
||||
return errors.New("counted before DB apply")
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
c := runConsumer(t, store, handler, false)
|
||||
claims := claimRun(t, store, run, time.Now(), 3)
|
||||
if err := c.handleRunClaims(ctx, claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var failures int64
|
||||
if failure {
|
||||
failures = 1
|
||||
}
|
||||
checkTerminal(t, store, run, 3-failures, failures)
|
||||
outcomes := []queue.TaskOutcome{{Success: true}, {Success: true}, {Success: true}}
|
||||
if n, err := store.FinalizeRunTasks(ctx, claims, outcomes); err != nil || n != 0 {
|
||||
t.Fatalf("duplicate finalize: %d %v", n, err)
|
||||
}
|
||||
checkTerminal(t, store, run, 3-failures, failures)
|
||||
key, _ := store.RunMetaKey(run)
|
||||
ttl, err := redis.Do(ctx, "TTL", key)
|
||||
if err != nil || ttl.Int64() < 29*24*3600 {
|
||||
t.Fatalf("retention=%v err=%v", ttl, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerLateProducerEmptyAndHistoricalRunCompletion(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
if err := store.CreateRun(ctx, queue.RunMeta{RunID: "historical", Status: "running", ProducerDone: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runTasks(t, store, "empty", 0)
|
||||
runTasks(t, store, "late", 1)
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}, false)
|
||||
claims := claimRun(t, store, "late", time.Now(), 1)
|
||||
if err := c.handleRunClaims(ctx, claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta := runMeta(t, store, "late"); !meta.CompletedAt.IsZero() || meta.SuccessCount != 1 {
|
||||
t.Fatalf("premature completion: %+v", meta)
|
||||
}
|
||||
finishProducer(t, store, "late")
|
||||
finishProducer(t, store, "empty")
|
||||
if _, err := c.processOnce(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.channelGroup.Wait()
|
||||
checkTerminal(t, store, "late", 1, 0)
|
||||
checkTerminal(t, store, "empty", 0, 0)
|
||||
checkTerminal(t, store, "historical", 0, 0)
|
||||
}
|
||||
|
||||
func TestRunConsumerRecoveryAndFencing(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
for _, mode := range []string{"database", "network", "stale"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
runTasks(t, store, mode, 1)
|
||||
finishProducer(t, store, mode)
|
||||
now := time.Now()
|
||||
claims := claimRun(t, store, mode, now, 1)
|
||||
good := runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}
|
||||
handler := good
|
||||
if mode == "database" {
|
||||
handler.apply = func([]Result) error { return errors.New("database unavailable") }
|
||||
}
|
||||
if mode == "network" {
|
||||
handler.query = func(task queue.Task) Result { return Result{Task: task, Retryable: true, ErrorClass: "network"} }
|
||||
}
|
||||
c := runConsumer(t, store, handler, false)
|
||||
if mode != "stale" {
|
||||
err := c.handleRunClaims(ctx, claims)
|
||||
if (err != nil) != (mode == "database") {
|
||||
t.Fatalf("handle: %v", err)
|
||||
}
|
||||
if meta := runMeta(t, store, mode); meta.SuccessCount != 0 || meta.FailedCount != 0 || meta.ActiveCount != 1 || meta.InflightCount != 1 || meta.Status != "running" {
|
||||
t.Fatalf("lost pending task: %+v", meta)
|
||||
}
|
||||
}
|
||||
fresh := claimRun(t, store, mode, now.Add(2*time.Minute), 1)
|
||||
if n, err := store.RenewRunBatch(ctx, claims, now.Add(3*time.Minute)); err != nil || n != 0 {
|
||||
t.Fatalf("stale renewal: %d %v", n, err)
|
||||
}
|
||||
if n, err := store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil || n != 0 {
|
||||
t.Fatalf("stale ack: %d %v", n, err)
|
||||
}
|
||||
if n, err := store.BlockRunScopedAsFailed(ctx, claims[0], queue.TaskOutcome{}); err != nil || n != -1 {
|
||||
t.Fatalf("stale block: %d %v", n, err)
|
||||
}
|
||||
c.handler = good
|
||||
if err := c.handleRunClaims(ctx, fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, mode, 1, 0)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerBlockedChannelCountsOnlyUnfinishedTasks(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "blocked", 5)
|
||||
runTasks(t, store, "other", 1)
|
||||
// Production continues while the channel is blocked.
|
||||
claims := claimRun(t, store, "blocked", time.Now(), 2)
|
||||
var applied int
|
||||
successQueried := make(chan struct{})
|
||||
handler := runHandler{query: func(task queue.Task) Result {
|
||||
if task.CardID == 1 {
|
||||
close(successQueried)
|
||||
return Result{Task: task, Success: true}
|
||||
}
|
||||
<-successQueried
|
||||
return Result{Task: task, BlockChannel: true, ErrorClass: "channel_limited", Code: 797, ErrCode: "S00004", Message: "limited"}
|
||||
}, apply: func(results []Result) error { applied = countSuccessful(results); return nil }}
|
||||
c := runConsumer(t, store, handler, false)
|
||||
// Ensure the success response arrives before the channel-limit response.
|
||||
if err := c.handleRunClaims(ctx, claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := runMeta(t, store, "blocked")
|
||||
if applied != 1 || meta.SuccessCount != 1 || meta.FailedCount != 4 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
|
||||
t.Fatalf("applied=%d meta=%+v", applied, meta)
|
||||
}
|
||||
if n, err := store.BlockRunScopedAsFailed(ctx, claims[1], queue.TaskOutcome{}); err != nil || n != -1 {
|
||||
t.Fatalf("duplicate block: %d %v", n, err)
|
||||
}
|
||||
late := queue.Task{TaskID: "late", RunID: "blocked", ChanKey: "channel", CardID: 99}
|
||||
if n, err := store.EnqueueRunBatch(ctx, "blocked", "channel", []queue.Task{late}, time.Now()); err != nil || n != 0 {
|
||||
t.Fatalf("blocked enqueue: %d %v", n, err)
|
||||
}
|
||||
key, _ := store.FailedKey("blocked")
|
||||
failures, err := redis.Do(ctx, "HLEN", key)
|
||||
if err != nil || failures.Int() != 4 {
|
||||
t.Fatalf("failure records=%v err=%v", failures, err)
|
||||
}
|
||||
stored, err := redis.Do(ctx, "HGET", key, "blocked:4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var record queue.FailureRecord
|
||||
if err := json.Unmarshal([]byte(stored.String()), &record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.TaskID != "blocked:4" || record.CardID != 5 || record.ICCID != "card-4" || record.Attempts != 0 || record.FailureType != "channel_blocked" || record.ErrCode != "S00004" {
|
||||
t.Fatalf("failure identity/reason: %+v", record)
|
||||
}
|
||||
stateKey, _ := store.TaskStateKey("blocked")
|
||||
state, err := redis.Do(ctx, "HGET", stateKey, "blocked:4")
|
||||
if err != nil || state.MapStrAny()["status"] != "channel_blocked" {
|
||||
t.Fatalf("blocked state=%v err=%v", state, err)
|
||||
}
|
||||
if meta := runMeta(t, store, "other"); meta.ActiveCount != 1 || meta.FailedCount != 0 {
|
||||
t.Fatalf("other run affected: %+v", meta)
|
||||
}
|
||||
finishProducer(t, store, "blocked")
|
||||
if err := c.completeRun(ctx, "blocked"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, "blocked", 1, 4)
|
||||
}
|
||||
|
||||
func TestRunConsumerMonthlyRoundsRemainEnabled(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "flow:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "monthly", 1)
|
||||
finishProducer(t, store, "monthly")
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, true)
|
||||
for round := 1; round <= 5; round++ {
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "monthly", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if round < 5 {
|
||||
if meta := runMeta(t, store, "monthly"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
|
||||
t.Fatalf("round %d: %+v", round, meta)
|
||||
}
|
||||
if ok, err := store.AdvanceRunRound(ctx, "monthly", 5); err != nil || !ok {
|
||||
t.Fatalf("advance %v %v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkTerminal(t, store, "monthly", 0, 1)
|
||||
}
|
||||
|
||||
func TestRunConsumerStatusRetriesRetryableResultsForThreeRounds(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "status", 1)
|
||||
finishProducer(t, store, "status")
|
||||
c, err := New(store, runHandler{query: func(task queue.Task) Result {
|
||||
return Result{Task: task, ErrorClass: "carrier_retry", Retryable: true, Code: 9999, ErrCode: "S00004"}
|
||||
}}, Config{
|
||||
RunQueueEnabled: true,
|
||||
RoundRetryEnabled: true,
|
||||
MaxRounds: 3,
|
||||
EventPrefix: "sim_status_queue",
|
||||
ShouldRetryNextRound: func(result Result) bool {
|
||||
return result.Retryable
|
||||
},
|
||||
TemplateQPS: map[string]float64{mobileTempKey: 1000000},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for round := 1; round <= 3; round++ {
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "status", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if round < 3 {
|
||||
if meta := runMeta(t, store, "status"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
|
||||
t.Fatalf("round %d: %+v", round, meta)
|
||||
}
|
||||
if ok, err := store.AdvanceRunRound(ctx, "status", 3); err != nil || !ok {
|
||||
t.Fatalf("advance %v %v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkTerminal(t, store, "status", 0, 1)
|
||||
}
|
||||
|
||||
func TestReplayMaxRoundOverridesConsumerDefault(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
if err := store.CreateRun(ctx, queue.RunMeta{RunID: "replay-one", Operation: "status", SyncType: "replay", MaxRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := queue.Task{RunID: "replay-one", TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "channel", TempKey: mobileTempKey}
|
||||
if _, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []queue.Task{task}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finishProducer(t, store, "replay-one")
|
||||
c, err := New(store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, Config{RunQueueEnabled: true, RoundRetryEnabled: true, MaxRounds: 3, TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "replay-one", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, "replay-one", 0, 1)
|
||||
meta := runMeta(t, store, "replay-one")
|
||||
if meta.CurrentRound != 1 || meta.NextRoundCount != 0 {
|
||||
t.Fatalf("replay advanced rounds: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunQueueModeDoesNotEnableRoundRetry(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
runTasks(t, store, "single-round", 1)
|
||||
finishProducer(t, store, "single-round")
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, false)
|
||||
if err := c.handleRunClaims(context.Background(), claimRun(t, store, "single-round", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, "single-round", 0, 1)
|
||||
if meta := runMeta(t, store, "single-round"); meta.CurrentRound != 1 {
|
||||
t.Fatalf("unexpected retry: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonthlyChannelBlockKeepsExistingCountingPolicy(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "flow:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "monthly-block", 2)
|
||||
claimRun(t, store, "monthly-block", time.Now(), 1)
|
||||
for i := 0; i < 2; i++ {
|
||||
want := int64(2)
|
||||
if i == 1 {
|
||||
want = 0
|
||||
}
|
||||
if n, err := store.BlockRunScoped(ctx, "monthly-block", "channel"); err != nil || n != want {
|
||||
t.Fatalf("monthly block=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
if meta := runMeta(t, store, "monthly-block"); meta.FailedCount != 0 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
|
||||
t.Fatalf("monthly counting changed: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerConcurrentRuns(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
var mu sync.Mutex
|
||||
applied := map[string]int{}
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }, apply: func(results []Result) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, result := range results {
|
||||
applied[result.Task.RunID]++
|
||||
}
|
||||
return nil
|
||||
}}, false)
|
||||
for _, run := range []string{"a", "b"} {
|
||||
runTasks(t, store, run, 3)
|
||||
finishProducer(t, store, run)
|
||||
}
|
||||
if started, err := c.processOnce(ctx); err != nil || !started {
|
||||
t.Fatalf("process: %v %v", started, err)
|
||||
}
|
||||
c.channelGroup.Wait()
|
||||
for _, run := range []string{"a", "b"} {
|
||||
checkTerminal(t, store, run, 3, 0)
|
||||
if applied[run] != 3 {
|
||||
t.Fatalf("applied %s: %d", run, applied[run])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunConsumerCoveredResultsCompleteWithoutFailures drives the full
|
||||
// consumer path for a covered result: the round must close, the run must end
|
||||
// as "completed" rather than "completed_with_failures", and the failure pool
|
||||
// must stay empty. This is the IMEI 14000 "carrier has no record" case — on
|
||||
// 2026-10-20 it accounted for every failure recorded in production while the
|
||||
// run success rate read 21%.
|
||||
func TestRunConsumerCoveredResultsCompleteWithoutFailures(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "imei:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "covered", 4)
|
||||
finishProducer(t, store, "covered")
|
||||
queried := 0
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result {
|
||||
queried++
|
||||
switch task.CardID {
|
||||
case 1:
|
||||
return Result{Task: task, Success: true, Imei: "865466070817587", ImeiSynced: true}
|
||||
case 2:
|
||||
return Result{Task: task, ErrorClass: "no_record", Code: 14000, ImeiSynced: true, Covered: true, Message: "系统无记录,请检查输入参数是否正确"}
|
||||
case 3:
|
||||
return Result{Task: task, ErrorClass: "data_invalid", ImeiSynced: true, Covered: true, Message: "carrier returned no imei value"}
|
||||
default:
|
||||
return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
|
||||
}
|
||||
}}, false)
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "covered", time.Now(), 4)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queried != 4 {
|
||||
t.Fatalf("queried = %d, want 4", queried)
|
||||
}
|
||||
meta := runMeta(t, store, "covered")
|
||||
// Three cards produced a usable or definitive answer (success + 2 covered),
|
||||
// and only the business failure is counted. Without the covered channel the
|
||||
// two 14000-style answers would have made failedCount=3.
|
||||
if meta.Status != "completed_with_failures" || meta.SuccessCount != 3 || meta.FailedCount != 1 || meta.NextRoundCount != 0 {
|
||||
t.Fatalf("unexpected run accounting: %+v", meta)
|
||||
}
|
||||
failureKey, _ := store.FailedKey("covered")
|
||||
size, err := redis.Do(ctx, "HLEN", failureKey)
|
||||
if err != nil || size.Int64() != 1 {
|
||||
t.Fatalf("failure pool entries = %d err=%v, want 1 (only the business failure)", size.Int64(), err)
|
||||
}
|
||||
for _, taskID := range []string{"covered:1", "covered:2"} {
|
||||
exists, err := redis.Do(ctx, "HEXISTS", failureKey, taskID)
|
||||
if err != nil || exists.Int() != 0 {
|
||||
t.Fatalf("covered card %s leaked into the failure pool: %v %v", taskID, exists, err)
|
||||
}
|
||||
}
|
||||
exists, err := redis.Do(ctx, "HEXISTS", failureKey, "covered:3")
|
||||
if err != nil || exists.Int() != 1 {
|
||||
t.Fatalf("the business failure is missing from the pool: %v %v", exists, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user