chore: 初始开源发布
iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。 - Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件） - 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、 库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异， 不破坏「校验 MSISDN 格式」等测试断言） - 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23）， 移除写死的 git 凭据 - README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 已知：3 个测试包在源仓同一提交上同样失败（client/upstream、 service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。
This commit is contained in:
MiFaZhan
@@ -0,0 +1,206 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// Pause the real producer after its first enqueue, before MarkProducerDone.
|
||||
type interruptedScanner struct {
|
||||
*datasetScanner
|
||||
paused chan struct{}
|
||||
}
|
||||
|
||||
func TestInterruptedRunReclamationGuards(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
for _, status := range []string{"producing", "running"} {
|
||||
for _, guard := range []string{"no_lock", "foreign_lock", "no_current_run", "same_run", "producer_done", "replay"} {
|
||||
t.Run(mode+"/"+status+"/"+guard, func(t *testing.T) {
|
||||
ctx := contextForTest(t)
|
||||
store := testStore(t)
|
||||
p := testProducer(t, scannerFor(nil, 0), store, Config{ScanMode: mode})
|
||||
meta := zset.RunMeta{RunID: "guarded", Status: status, ProducerDone: guard == "producer_done"}
|
||||
if guard == "replay" {
|
||||
meta.SyncType = "replay"
|
||||
}
|
||||
if err := store.CreateRun(ctx, meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := "fresh"
|
||||
owner := p.owner
|
||||
if guard == "same_run" {
|
||||
current = meta.RunID
|
||||
}
|
||||
if guard == "no_current_run" {
|
||||
current = ""
|
||||
}
|
||||
if guard == "foreign_lock" {
|
||||
owner = "other-owner"
|
||||
}
|
||||
if guard != "no_lock" {
|
||||
if ok, err := store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, owner, current, p.config.RunLockTTL); err != nil || !ok {
|
||||
t.Fatalf("acquire=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
key, _ := store.RunMetaKey(meta.RunID)
|
||||
before, err := store.RedisClient().Do(ctx, "HGETALL", key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := p.hasPendingTasks(ctx, current)
|
||||
if err != nil || pending != (guard != "same_run") {
|
||||
t.Fatalf("guard=%s pending=%t err=%v", guard, pending, err)
|
||||
}
|
||||
after, err := store.RedisClient().Do(ctx, "HGETALL", key)
|
||||
if err != nil || !reflect.DeepEqual(before.MapStrStr(), after.MapStrStr()) {
|
||||
t.Fatalf("guarded metadata changed: before=%v after=%v err=%v", before, after, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *interruptedScanner) Scan(ctx context.Context, last, upper int64, size int) ([]Card, error) {
|
||||
if last > 0 {
|
||||
close(s.paused)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return s.datasetScanner.Scan(ctx, last, upper, size)
|
||||
}
|
||||
|
||||
func (s *interruptedScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
if last > 0 {
|
||||
close(s.paused)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return s.datasetScanner.ScanChannel(ctx, key, last, size)
|
||||
}
|
||||
|
||||
func TestBlockedInterruptedProducerRecovery(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
for _, backlog := range []string{"none", "queued", "leased", "waiting"} {
|
||||
t.Run(mode+"/"+backlog, func(t *testing.T) {
|
||||
ctx := contextForTest(t)
|
||||
store := testStore(t)
|
||||
scanner := &interruptedScanner{datasetScanner: scannerFor([]Card{{ID: 1, ChanKey: "blocked"}, {ID: 2, ChanKey: "blocked"}}, 2), paused: make(chan struct{})}
|
||||
old := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 1})
|
||||
oldCtx, interrupt := context.WithCancel(ctx)
|
||||
defer interrupt()
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- old.Produce(oldCtx, scopedMessage("interrupted")) }()
|
||||
select {
|
||||
case <-scanner.paused:
|
||||
case err := <-result:
|
||||
t.Fatalf("producer exited before pause: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
claim := func(channel string) zset.ClaimedTask {
|
||||
t.Helper()
|
||||
tasks, err := store.ClaimRun(ctx, "interrupted", channel, time.Now().Add(time.Second), time.Now().Add(time.Minute), 1)
|
||||
if err != nil || len(tasks) != 1 {
|
||||
t.Fatalf("claim %s=%v err=%v", channel, tasks, err)
|
||||
}
|
||||
return tasks[0]
|
||||
}
|
||||
if n, err := store.BlockRunScopedAsFailed(ctx, claim("blocked"), zset.TaskOutcome{Code: 797, ErrCode: "S00004"}); err != nil || n != 1 {
|
||||
t.Fatalf("blocked=%d err=%v", n, err)
|
||||
}
|
||||
var remaining zset.ClaimedTask
|
||||
if backlog != "none" {
|
||||
tasks := []zset.Task{{TaskID: "remaining", RunID: "interrupted", ChanKey: "other"}}
|
||||
if n, err := store.EnqueueRunBatch(ctx, "interrupted", "other", tasks, time.Now()); err != nil || n != 1 {
|
||||
t.Fatalf("enqueue=%d err=%v", n, err)
|
||||
}
|
||||
if backlog != "queued" {
|
||||
remaining = claim("other")
|
||||
}
|
||||
if backlog == "waiting" {
|
||||
if ok, err := store.FinalizeRunTask(ctx, remaining, zset.TaskOutcome{Retry: true}); err != nil || !ok {
|
||||
t.Fatalf("defer=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
before, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || before.Status != "running" || before.ProducerDone || before.FailedCount != 1 {
|
||||
t.Fatalf("blocked metadata=%+v err=%v", before, err)
|
||||
}
|
||||
fresh := testProducer(t, scannerFor([]Card{{ID: 3, ChanKey: "fresh"}}, 3), store, Config{ScanMode: mode})
|
||||
if err := fresh.Produce(ctx, scopedMessage("fresh")); !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("live producer lock must prevent reclamation: %v", err)
|
||||
}
|
||||
requireMeta(t, store, "interrupted", false, before.ActiveCount)
|
||||
interrupt()
|
||||
if err := <-result; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("interruption=%v", err)
|
||||
}
|
||||
lock, err := store.RedisClient().Do(ctx, "EXISTS", old.config.RunLockKey)
|
||||
if err != nil || lock.Int() != 0 {
|
||||
t.Fatalf("old lock=%v err=%v", lock, err)
|
||||
}
|
||||
err = fresh.Produce(ctx, scopedMessage("fresh"))
|
||||
if backlog == "none" && err != nil || backlog != "none" && !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("fresh start with %s backlog: %v", backlog, err)
|
||||
}
|
||||
after, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || !after.ProducerDone || after.Status != "running" || after.ActiveCount != before.ActiveCount || after.InflightCount != before.InflightCount || after.NextRoundCount != before.NextRoundCount || after.FailedCount != before.FailedCount {
|
||||
t.Fatalf("reclaimed metadata=%+v before=%+v err=%v", after, before, err)
|
||||
}
|
||||
if backlog != "none" {
|
||||
if _, err := store.GetRunMeta(ctx, "fresh"); !errors.Is(err, zset.ErrRunNotFound) {
|
||||
t.Fatalf("backlog must prevent creating fresh run: %v", err)
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "interrupted"); err != nil || complete {
|
||||
t.Fatalf("premature completion=%t err=%v", complete, err)
|
||||
}
|
||||
if backlog == "waiting" {
|
||||
if advanced, err := store.AdvanceRunRound(ctx, "interrupted"); err != nil || !advanced {
|
||||
t.Fatalf("advance=%t err=%v", advanced, err)
|
||||
}
|
||||
}
|
||||
if backlog != "leased" {
|
||||
remaining = claim("other")
|
||||
}
|
||||
if ok, err := store.FinalizeRunTask(ctx, remaining, zset.TaskOutcome{Success: true}); err != nil || !ok {
|
||||
t.Fatalf("drain=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "interrupted", time.Hour); err != nil || !complete {
|
||||
t.Fatalf("normal completion=%t err=%v", complete, err)
|
||||
}
|
||||
closed, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || closed.Status != "completed_with_failures" || closed.FailedCount != 1 || closed.ActiveCount != 0 || closed.InflightCount != 0 || closed.NextRoundCount != 0 {
|
||||
t.Fatalf("closed metadata=%+v err=%v", closed, err)
|
||||
}
|
||||
metaKey, _ := store.RunMetaKey("interrupted")
|
||||
failedKey, _ := store.FailedKey("interrupted")
|
||||
stateKey, _ := store.TaskStateKey("interrupted")
|
||||
channelsKey, _ := store.RunChannelsKey("interrupted")
|
||||
for _, key := range []string{metaKey, failedKey, stateKey, channelsKey} {
|
||||
ttl, err := store.RedisClient().Do(ctx, "PTTL", key)
|
||||
if err != nil || ttl.Int64() <= 0 || ttl.Int64() > time.Hour.Milliseconds() {
|
||||
t.Fatalf("retention %s=%v err=%v", key, ttl, err)
|
||||
}
|
||||
}
|
||||
if backlog != "none" {
|
||||
if err := fresh.Produce(ctx, scopedMessage("fresh")); err != nil {
|
||||
t.Fatalf("fresh start after draining: %v", err)
|
||||
}
|
||||
}
|
||||
requireMeta(t, store, "fresh", true, 1)
|
||||
active, err := store.ActiveRuns(ctx)
|
||||
if err != nil || len(active) != 1 || active[0] != "fresh" {
|
||||
t.Fatalf("only closed run must leave active index: %v err=%v", active, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,491 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
var errProducerDone = gerror.New("run production already closed")
|
||||
|
||||
func producerStateError(state zset.ProducerState) error {
|
||||
if state.Status == "completed" || state.Status == "completed_with_failures" {
|
||||
return ErrRunFinished
|
||||
}
|
||||
if state.ProducerDone {
|
||||
return errProducerDone
|
||||
}
|
||||
if state.Status != "producing" && state.Status != "running" {
|
||||
return gerror.Newf("unexpected producer run status %q", state.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// produceChannels owns all per-attempt coordination. There is one worker per
|
||||
// target, NOT a pool of four long-lived channel workers. SQL tokens and the
|
||||
// cancellable admission gate never span sleeps or goroutine joins.
|
||||
func (p *Producer) produceChannels(ctx context.Context, message model.TrafficUsageDailyStartMessage, scope *model.SyncScope, scanUpperID int64, startedAt time.Time, total *productionStats) error {
|
||||
scanner := p.scanner.(channelScanner)
|
||||
metadata, err := scanner.ListChannels(ctx)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "initialize channel scan")
|
||||
}
|
||||
state, err := p.store.ReadProducerState(ctx, message.RunID, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return err
|
||||
}
|
||||
targets := make(map[string]channelMetadata, len(metadata))
|
||||
known := make(map[string]bool, len(metadata))
|
||||
for _, channel := range metadata {
|
||||
if channel.ChanKey == "" || !scope.Allows(channel.ChanKey) {
|
||||
continue
|
||||
}
|
||||
if known[channel.ChanKey] {
|
||||
return gerror.Newf("duplicate channel metadata %q", channel.ChanKey)
|
||||
}
|
||||
known[channel.ChanKey] = true
|
||||
reason := ""
|
||||
if channel.SyncEnabled {
|
||||
reason = "sync_disabled"
|
||||
} else if channel.TempKey == "" {
|
||||
reason = "missing_channel_template"
|
||||
}
|
||||
if reason != "" {
|
||||
total.skippedChannels++
|
||||
g.Log().Warningf(ctx, "event=daily_usage_channel status=skipped runId=%s chanKey=%s reason=%s skipUnit=channel", message.RunID, channel.ChanKey, reason)
|
||||
continue
|
||||
}
|
||||
targets[channel.ChanKey] = channel
|
||||
}
|
||||
if scope != nil {
|
||||
for _, chanKey := range scope.Channels {
|
||||
if !known[chanKey] {
|
||||
total.skippedChannels++
|
||||
g.Log().Warningf(ctx, "event=daily_usage_channel status=skipped runId=%s chanKey=%s reason=unknown_channel skipUnit=channel", message.RunID, chanKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
g.Log().Infof(ctx, "event=daily_usage_channel_scan status=started runId=%s targetChannels=%d scanUpperID=%d scanConcurrency=%d pageSize=%d skipUnit=channel", message.RunID, len(targets), scanUpperID, p.config.ScanConcurrency, p.config.PageSize)
|
||||
keys := make([]string, 0, len(targets))
|
||||
for key := range targets {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
finishObservation := p.startObservation(ctx, message.RunID, keys, startedAt)
|
||||
defer finishObservation()
|
||||
var (
|
||||
workers sync.WaitGroup
|
||||
slots = make(chan struct{}, p.config.ScanConcurrency)
|
||||
gate = make(chan struct{}, 1)
|
||||
metrics = newScanActivity()
|
||||
results = make([]*channelScanStats, len(keys))
|
||||
)
|
||||
pool, hasPool := scanner.(interface {
|
||||
poolWait(context.Context) (int64, int64)
|
||||
})
|
||||
var poolCount, poolNanos int64
|
||||
if hasPool {
|
||||
poolCount, poolNanos = pool.poolWait(ctx)
|
||||
}
|
||||
workCtx, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
for i, key := range keys {
|
||||
stats := &channelScanStats{chanKey: key, lastSummary: time.Now()}
|
||||
results[i] = stats
|
||||
workers.Add(1)
|
||||
go func(channel channelMetadata, stats *channelScanStats) {
|
||||
defer workers.Done()
|
||||
scanErr := p.scanOneChannel(workCtx, scanner, message, channel, scanUpperID, startedAt, slots, gate, metrics, stats)
|
||||
if scanErr != nil {
|
||||
cancel(scanErr)
|
||||
}
|
||||
reason := stats.stopReason
|
||||
if reason == "" {
|
||||
reason = "scan_complete"
|
||||
}
|
||||
if scanErr != nil {
|
||||
reason = "cancelled"
|
||||
}
|
||||
p.observe(channel.ChanKey, reason, 0, false, true)
|
||||
stats.log(workCtx, message.RunID, "ended", startedAt, scanErr)
|
||||
}(targets[key], stats)
|
||||
}
|
||||
workers.Wait()
|
||||
var summary channelScanStats
|
||||
for _, stats := range results {
|
||||
total.pages += stats.scanPages
|
||||
total.scanned += stats.eligible
|
||||
total.eligible += stats.eligible
|
||||
total.enqueued += stats.added
|
||||
total.stoppedTasks += stats.rejected
|
||||
if stats.eligible > 0 {
|
||||
total.channels[stats.chanKey] = struct{}{}
|
||||
}
|
||||
summary.merge(stats)
|
||||
}
|
||||
metrics.log(ctx, message.RunID, &summary)
|
||||
if hasPool {
|
||||
count, nanos := pool.poolWait(ctx)
|
||||
g.Log().Infof(ctx, "event=daily_usage_scan_pool runId=%s sharedPoolWaitCountDelta=%d sharedPoolWaitMsDelta=%d attribution=shared_pool", message.RunID, count-poolCount, (nanos-poolNanos)/int64(time.Millisecond))
|
||||
}
|
||||
return context.Cause(workCtx)
|
||||
}
|
||||
|
||||
func acquireContext(ctx context.Context, token chan struct{}) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case token <- struct{}{}:
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-token
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) scanOneChannel(ctx context.Context, scanner channelScanner, message model.TrafficUsageDailyStartMessage, channel channelMetadata, scanUpperID int64, startedAt time.Time, slots, gate chan struct{}, metrics *scanActivity, stats *channelScanStats) error {
|
||||
for stats.cursor < scanUpperID {
|
||||
// Detect a blocked/closed run before querying, including empty channels.
|
||||
p.observe(channel.ChanKey, "redis_read", 0, false, false)
|
||||
state, err := p.store.ReadProducerState(ctx, message.RunID, channel.ChanKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Blocked {
|
||||
// No page was read: the unknown unscanned remainder has no card
|
||||
// count. Never turn it into a fabricated success/failure total.
|
||||
stats.stopReason = "blocked_unscanned_remainder"
|
||||
return nil
|
||||
}
|
||||
p.observe(channel.ChanKey, "sql_slot", 0, false, false)
|
||||
waitStart := time.Now()
|
||||
if err = acquireContext(ctx, slots); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.slotWait += time.Since(waitStart)
|
||||
metrics.begin()
|
||||
p.observe(channel.ChanKey, "sql_query", 0, false, false)
|
||||
queryStart := time.Now()
|
||||
cards, scanErr := scanner.ScanChannel(ctx, channel.ChanKey, stats.cursor, p.config.PageSize)
|
||||
elapsed := time.Since(queryStart)
|
||||
metrics.end()
|
||||
<-slots // release immediately after results have been materialized
|
||||
stats.sqlTime += elapsed
|
||||
stats.sqlLatency.observe(elapsed)
|
||||
stats.scanPages++
|
||||
if scanErr != nil {
|
||||
return gerror.Wrapf(scanErr, "scan channel %s after %d", channel.ChanKey, stats.cursor)
|
||||
}
|
||||
if len(cards) > p.config.PageSize {
|
||||
return gerror.New("scanner exceeded page size")
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
stats.stopReason = "empty_result"
|
||||
return nil
|
||||
}
|
||||
var (
|
||||
pageCursor = stats.cursor
|
||||
overUpper bool
|
||||
tasks = make([]zset.Task, 0, len(cards))
|
||||
)
|
||||
for i, card := range cards {
|
||||
if card.ChanKey != channel.ChanKey || card.ID <= pageCursor {
|
||||
return gerror.New("channel scanner returned unordered or foreign cards")
|
||||
}
|
||||
if card.ID > scanUpperID {
|
||||
stats.overUpper += int64(len(cards) - i)
|
||||
overUpper = true
|
||||
break
|
||||
}
|
||||
pageCursor = card.ID
|
||||
tasks = append(tasks, zset.Task{TaskID: zset.StableTaskID(message.RunID, message.Operation, card.ID), RunID: message.RunID, Operation: message.Operation, QueryMonth: message.QueryDate, ChanKey: channel.ChanKey, TempKey: channel.TempKey, Operator: channel.Operator, CardID: card.ID, ICCID: card.ICCID, MSISDN: card.MSISDN, IMSI: card.IMSI, PackageCode: card.PackageCode})
|
||||
}
|
||||
p.observe(channel.ChanKey, "page_ready", len(tasks), false, false)
|
||||
stats.eligible += int64(len(tasks))
|
||||
blocked, err := p.admitChannelPage(ctx, message.RunID, tasks, startedAt, gate, stats)
|
||||
if err != nil {
|
||||
return err // page cursor is not committed, even after a partial write
|
||||
}
|
||||
if blocked {
|
||||
stats.stopReason = "blocked"
|
||||
return nil
|
||||
}
|
||||
stats.cursor = pageCursor
|
||||
stats.progress(ctx, message.RunID, startedAt)
|
||||
if overUpper || stats.cursor >= scanUpperID {
|
||||
stats.stopReason = "upper_bound"
|
||||
return nil
|
||||
}
|
||||
// Do not use a short page as EOF: query-time eligibility may change.
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// admitChannelPage admits only new TaskIDs, in capacity-sized slices. Each
|
||||
// worker retains at most this page. No capacity reservation survives the gate.
|
||||
func (p *Producer) admitChannelPage(ctx context.Context, runID string, tasks []zset.Task, startedAt time.Time, gate chan struct{}, stats *channelScanStats) (bool, error) {
|
||||
for len(tasks) > 0 {
|
||||
p.observe(stats.chanKey, "admission_gate", len(tasks), false, false)
|
||||
gateStart := time.Now()
|
||||
if err := acquireContext(ctx, gate); err != nil {
|
||||
return false, err
|
||||
}
|
||||
stats.gateWait += time.Since(gateStart)
|
||||
blocked, waitKind, remaining, err := p.admitChannelSlice(ctx, runID, tasks, startedAt, stats)
|
||||
<-gate
|
||||
if err != nil || blocked {
|
||||
return blocked, err
|
||||
}
|
||||
tasks = remaining
|
||||
p.observe(stats.chanKey, "page_ready", len(tasks), false, false)
|
||||
if waitKind != "" {
|
||||
p.observe(stats.chanKey, waitKind+"_capacity", len(tasks), false, false)
|
||||
waitStart := time.Now()
|
||||
err = sleepContext(ctx, p.config.PollInterval)
|
||||
if waitKind == "global" {
|
||||
stats.globalWait += time.Since(waitStart)
|
||||
} else {
|
||||
stats.channelWait += time.Since(waitStart)
|
||||
}
|
||||
stats.progress(ctx, runID, startedAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, ctx.Err()
|
||||
}
|
||||
|
||||
// Caller holds the short admission gate; there is no sleep in this method.
|
||||
func (p *Producer) admitChannelSlice(ctx context.Context, runID string, tasks []zset.Task, startedAt time.Time, stats *channelScanStats) (blocked bool, waitKind string, remaining []zset.Task, err error) {
|
||||
p.observe(stats.chanKey, "redis_read", len(tasks), false, false)
|
||||
readStart := time.Now()
|
||||
state, err := p.store.ReadProducerState(ctx, runID, stats.chanKey)
|
||||
if err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
existing, err := p.store.ExistingRunTasks(ctx, runID, tasks)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
pending := tasks[:0]
|
||||
for i, exists := range existing {
|
||||
if exists {
|
||||
stats.duplicates++
|
||||
} else {
|
||||
pending = append(pending, tasks[i])
|
||||
}
|
||||
}
|
||||
tasks = pending
|
||||
p.observe(stats.chanKey, "redis_read", len(tasks), false, false)
|
||||
if state.Blocked {
|
||||
stats.rejected += int64(len(tasks))
|
||||
return true, "", nil, nil
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return false, "", nil, nil
|
||||
}
|
||||
space := int64(len(tasks))
|
||||
if limit := p.config.GlobalHighWaterMark; limit > 0 {
|
||||
space = min(space, limit-state.ActiveCount) // inflight is already included
|
||||
if space <= 0 {
|
||||
return false, "global", tasks, nil
|
||||
}
|
||||
}
|
||||
if limit := p.config.ChannelHighWaterMark; limit > 0 {
|
||||
key, keyErr := p.store.RoundActiveKey(runID, state.CurrentRound, stats.chanKey)
|
||||
if keyErr != nil {
|
||||
return false, "", tasks, keyErr
|
||||
}
|
||||
readStart = time.Now()
|
||||
size, sizeErr := p.store.SizeKey(ctx, key)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if sizeErr != nil {
|
||||
return false, "", tasks, sizeErr
|
||||
}
|
||||
space = min(space, limit-size)
|
||||
if space <= 0 {
|
||||
return false, "channel", tasks, nil
|
||||
}
|
||||
}
|
||||
batch := tasks[:int(space)]
|
||||
p.observe(stats.chanKey, "redis_write", len(tasks), false, false)
|
||||
enqueueStart := time.Now()
|
||||
added, err := p.store.EnqueueRunBatch(ctx, runID, stats.chanKey, batch, time.Now())
|
||||
stats.enqueueTime += time.Since(enqueueStart)
|
||||
stats.enqueueBatches++
|
||||
if err != nil {
|
||||
return false, "", tasks, err // uncertain responses require idempotent rescan
|
||||
}
|
||||
if added < 0 || added > len(batch) {
|
||||
return false, "", tasks, gerror.New("invalid enqueue count")
|
||||
}
|
||||
stats.added += int64(added)
|
||||
p.observe(stats.chanKey, "redis_write", len(tasks)-len(batch), added > 0, false)
|
||||
if added > 0 && !stats.firstEnqueueRecorded {
|
||||
stats.firstEnqueueRecorded = true
|
||||
stats.firstEnqueue = time.Since(startedAt)
|
||||
stats.log(ctx, runID, "first_enqueue", startedAt, nil)
|
||||
}
|
||||
if added < len(batch) {
|
||||
readStart = time.Now()
|
||||
confirmed, checkErr := p.store.ExistingRunTasks(ctx, runID, batch)
|
||||
if checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
state, checkErr = p.store.ReadProducerState(ctx, runID, stats.chanKey)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
if checkErr = producerStateError(state); checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
present := 0
|
||||
for _, exists := range confirmed {
|
||||
if exists {
|
||||
present++
|
||||
}
|
||||
}
|
||||
if present < added {
|
||||
return false, "", tasks, gerror.New("enqueue count exceeds confirmed task states")
|
||||
}
|
||||
stats.duplicates += int64(present - added)
|
||||
if state.Blocked {
|
||||
stats.rejected += int64(len(tasks) - present)
|
||||
return true, "", nil, nil
|
||||
}
|
||||
if present != len(batch) {
|
||||
return false, "", tasks, gerror.Newf("unexplained enqueue rejection: added=%d submitted=%d confirmed=%d", added, len(batch), present)
|
||||
}
|
||||
}
|
||||
return false, "", tasks[len(batch):], nil
|
||||
}
|
||||
|
||||
// Retained only to keep errors.Is semantics visible at the call site.
|
||||
func isProductionClosed(err error) bool { return errors.Is(err, errProducerDone) }
|
||||
|
||||
func durationMS(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) }
|
||||
|
||||
// latencyHistogram is fixed-size. Reported percentiles are bucket upper bounds,
|
||||
// never exact samples or a server-side SLA; the last bucket is unbounded.
|
||||
var latencyBounds = [...]time.Duration{time.Millisecond, 2 * time.Millisecond, 5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond, 50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond, 500 * time.Millisecond, time.Second, 2 * time.Second, 5 * time.Second, 10 * time.Second, 30 * time.Second, time.Minute}
|
||||
|
||||
type latencyHistogram struct{ counts [16]int64 }
|
||||
|
||||
func (h *latencyHistogram) observe(d time.Duration) {
|
||||
i := 0
|
||||
for i < len(latencyBounds) && d > latencyBounds[i] {
|
||||
i++
|
||||
}
|
||||
h.counts[i]++
|
||||
}
|
||||
func (h *latencyHistogram) percentile(percent int64) string {
|
||||
var total, cumulative int64
|
||||
for _, count := range h.counts {
|
||||
total += count
|
||||
}
|
||||
if total == 0 {
|
||||
return "n/a"
|
||||
}
|
||||
target := (total*percent + 99) / 100
|
||||
for i, count := range h.counts {
|
||||
cumulative += count
|
||||
if cumulative >= target {
|
||||
if i == len(latencyBounds) {
|
||||
return ">60000"
|
||||
}
|
||||
return fmt.Sprintf("%.0f", durationMS(latencyBounds[i]))
|
||||
}
|
||||
}
|
||||
return "n/a"
|
||||
}
|
||||
|
||||
type channelScanStats struct {
|
||||
firstEnqueueRecorded bool
|
||||
chanKey, stopReason string
|
||||
cursor, scanPages, eligible, overUpper, enqueueBatches, added, duplicates, rejected int64
|
||||
sqlTime, slotWait, gateWait, globalWait, channelWait, stateTime, enqueueTime, firstEnqueue time.Duration
|
||||
sqlLatency latencyHistogram
|
||||
lastSummary time.Time
|
||||
}
|
||||
|
||||
func (s *channelScanStats) log(ctx context.Context, runID, status string, startedAt time.Time, err error) {
|
||||
errorText := ""
|
||||
if err != nil {
|
||||
errorText = err.Error()
|
||||
}
|
||||
g.Log().Infof(ctx, "event=daily_usage_channel_scan status=%s runId=%s chanKey=%s cursor=%d scanPages=%d eligible=%d overUpper=%d enqueueBatches=%d added=%d duplicates=%d rejected=%d sqlMs=%.3f sqlP50UpperMs=%s sqlP95UpperMs=%s slotWaitMs=%.3f admissionWaitMs=%.3f globalCapacityWaitMs=%.3f channelCapacityWaitMs=%.3f stateReadMs=%.3f enqueueMs=%.3f firstEnqueueMs=%.3f wallMs=%d stopReason=%s error=%q", status, runID, s.chanKey, s.cursor, s.scanPages, s.eligible, s.overUpper, s.enqueueBatches, s.added, s.duplicates, s.rejected, durationMS(s.sqlTime), s.sqlLatency.percentile(50), s.sqlLatency.percentile(95), durationMS(s.slotWait), durationMS(s.gateWait), durationMS(s.globalWait), durationMS(s.channelWait), durationMS(s.stateTime), durationMS(s.enqueueTime), durationMS(s.firstEnqueue), time.Since(startedAt).Milliseconds(), s.stopReason, errorText)
|
||||
}
|
||||
func (s *channelScanStats) progress(ctx context.Context, runID string, startedAt time.Time) {
|
||||
if time.Since(s.lastSummary) >= time.Minute {
|
||||
s.lastSummary = time.Now()
|
||||
s.log(ctx, runID, "progress", startedAt, nil)
|
||||
}
|
||||
}
|
||||
func (s *channelScanStats) merge(other *channelScanStats) {
|
||||
s.scanPages += other.scanPages
|
||||
s.eligible += other.eligible
|
||||
s.overUpper += other.overUpper
|
||||
s.enqueueBatches += other.enqueueBatches
|
||||
s.added += other.added
|
||||
s.duplicates += other.duplicates
|
||||
s.rejected += other.rejected
|
||||
s.sqlTime += other.sqlTime
|
||||
s.slotWait += other.slotWait
|
||||
s.gateWait += other.gateWait
|
||||
s.globalWait += other.globalWait
|
||||
s.channelWait += other.channelWait
|
||||
s.stateTime += other.stateTime
|
||||
s.enqueueTime += other.enqueueTime
|
||||
for i, count := range other.sqlLatency.counts {
|
||||
s.sqlLatency.counts[i] += count
|
||||
}
|
||||
}
|
||||
|
||||
type scanActivity struct {
|
||||
mu sync.Mutex
|
||||
started, changed time.Time
|
||||
running, peak int
|
||||
area time.Duration
|
||||
}
|
||||
|
||||
func newScanActivity() *scanActivity {
|
||||
now := time.Now()
|
||||
return &scanActivity{started: now, changed: now}
|
||||
}
|
||||
func (s *scanActivity) change(delta int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := time.Now()
|
||||
s.area += time.Duration(s.running) * now.Sub(s.changed)
|
||||
s.changed = now
|
||||
s.running += delta
|
||||
s.peak = max(s.peak, s.running)
|
||||
}
|
||||
func (s *scanActivity) begin() { s.change(1) }
|
||||
func (s *scanActivity) end() { s.change(-1) }
|
||||
func (s *scanActivity) log(ctx context.Context, runID string, stats *channelScanStats) {
|
||||
wall := time.Since(s.started)
|
||||
g.Log().Infof(ctx, "event=daily_usage_scan_summary runId=%s wallMs=%d sqlInFlightPeak=%d sqlInFlightAvg=%.4f scanPages=%d enqueueBatches=%d eligible=%d overUpper=%d added=%d duplicates=%d rejected=%d sqlCumulativeMs=%.3f sqlP50UpperMs=%s sqlP95UpperMs=%s slotWaitCumulativeMs=%.3f admissionWaitCumulativeMs=%.3f globalCapacityWaitCumulativeMs=%.3f channelCapacityWaitCumulativeMs=%.3f stateReadCumulativeMs=%.3f enqueueCumulativeMs=%.3f", runID, wall.Milliseconds(), s.peak, float64(s.area)/float64(wall), stats.scanPages, stats.enqueueBatches, stats.eligible, stats.overUpper, stats.added, stats.duplicates, stats.rejected, durationMS(stats.sqlTime), stats.sqlLatency.percentile(50), stats.sqlLatency.percentile(95), durationMS(stats.slotWait), durationMS(stats.gateWait), durationMS(stats.globalWait), durationMS(stats.channelWait), durationMS(stats.stateTime), durationMS(stats.enqueueTime))
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
)
|
||||
|
||||
// datasetScanner applies the SQL eligibility predicate before LIMIT; metadata
|
||||
// eligibility is applied by the producer. Optional hooks simulate SQL latency,
|
||||
// live additions and failures without accessing an application database.
|
||||
type datasetScanner struct {
|
||||
cards []Card
|
||||
channels []channelMetadata
|
||||
upper int64
|
||||
hook func(context.Context, string, int64) error
|
||||
queries atomic.Int32
|
||||
lists atomic.Int32
|
||||
inflight atomic.Int32
|
||||
peak atomic.Int32
|
||||
}
|
||||
|
||||
func (s *datasetScanner) SnapshotMaxID(context.Context) (int64, error) { return s.upper, nil }
|
||||
func (s *datasetScanner) ListChannels(context.Context) ([]channelMetadata, error) {
|
||||
s.lists.Add(1)
|
||||
return s.channels, nil
|
||||
}
|
||||
func (s *datasetScanner) Scan(ctx context.Context, last, upper int64, size int) ([]Card, error) {
|
||||
var rows []Card
|
||||
for _, card := range s.cards {
|
||||
if card.ID > last && card.ID <= upper {
|
||||
for _, channel := range s.channels {
|
||||
if channel.ChanKey == card.ChanKey {
|
||||
card.TempKey = channel.TempKey
|
||||
card.SyncEnabled = channel.SyncEnabled
|
||||
card.Operator = channel.Operator
|
||||
}
|
||||
}
|
||||
rows = append(rows, card)
|
||||
if len(rows) == size {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
func (s *datasetScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
s.queries.Add(1)
|
||||
running := s.inflight.Add(1)
|
||||
defer s.inflight.Add(-1)
|
||||
for {
|
||||
old := s.peak.Load()
|
||||
if running <= old || s.peak.CompareAndSwap(old, running) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if s.hook != nil {
|
||||
if err := s.hook(ctx, key, last); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var rows []Card
|
||||
for _, card := range s.cards {
|
||||
if card.ChanKey == key && card.ID > last {
|
||||
rows = append(rows, card)
|
||||
if len(rows) == size {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
func scannerFor(cards []Card, upper int64) *datasetScanner {
|
||||
keys := map[string]bool{}
|
||||
s := &datasetScanner{cards: cards, upper: upper}
|
||||
for _, card := range cards {
|
||||
if card.ChanKey != "" && !keys[card.ChanKey] {
|
||||
keys[card.ChanKey] = true
|
||||
s.channels = append(s.channels, channelMetadata{ChanKey: card.ChanKey, TempKey: "sim_yd_v5", Operator: 1})
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
func testStore(t *testing.T) *zset.Store {
|
||||
t.Helper()
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
func testProducer(t *testing.T, scanner CardScanner, store *zset.Store, config Config) *Producer {
|
||||
t.Helper()
|
||||
config.RoundRetryEnabled = true
|
||||
if config.ScanMode == "" {
|
||||
config.ScanMode = "channel"
|
||||
}
|
||||
p, err := New(scanner, store, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
func contextForTest(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
return ctx
|
||||
}
|
||||
func taskIDs(t *testing.T, store *zset.Store, run string) []string {
|
||||
t.Helper()
|
||||
key, _ := store.TaskStateKey(run)
|
||||
result, err := store.RedisClient().Do(context.Background(), "HKEYS", key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := result.Strings()
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
func requireMeta(t *testing.T, store *zset.Store, run string, done bool, active int64) {
|
||||
t.Helper()
|
||||
meta, err := store.GetRunMeta(context.Background(), run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ProducerDone != done || meta.ActiveCount != active {
|
||||
t.Fatalf("meta=%+v, want done=%t active=%d", meta, done, active)
|
||||
}
|
||||
}
|
||||
func waitUntil(t *testing.T, ctx context.Context, condition func() bool) {
|
||||
t.Helper()
|
||||
for !condition() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelModeSettings(t *testing.T) {
|
||||
store, _ := zset.New(&producerRedis{}, "flow:")
|
||||
p, err := New(&fakeScanner{}, store, Config{})
|
||||
if err != nil || p.config.ScanMode != "serial" || p.config.ScanConcurrency != 4 {
|
||||
t.Fatalf("defaults: p=%+v err=%v", p, err)
|
||||
}
|
||||
for _, config := range []Config{{ScanMode: "parallel"}, {ScanConcurrency: -1}, {ScanMode: "channel"}} {
|
||||
if _, err = New(&fakeScanner{}, store, config); err == nil {
|
||||
t.Fatalf("accepted invalid config/capability %+v", config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanFiniteMemoryBoundary(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
upper int64
|
||||
cards []Card
|
||||
ids []int64
|
||||
queries int32
|
||||
}{
|
||||
{"zero upper", 0, []Card{{ID: 1, ChanKey: "a"}}, nil, 0},
|
||||
{"exact page", 4, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}}, []int64{1, 2, 3, 4}, 2},
|
||||
{"tail then empty", 9, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}}, []int64{1, 2, 3}, 3},
|
||||
{"first outside", 3, []Card{{ID: 4, ChanKey: "a"}, {ID: 5, ChanKey: "a"}}, nil, 1},
|
||||
{"mixed final page", 3, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}, {ID: 5, ChanKey: "a"}}, []int64{1, 2, 3}, 2},
|
||||
{"empty channel", 9, nil, nil, 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor(test.cards, test.upper)
|
||||
if len(scanner.channels) == 0 {
|
||||
scanner.channels = []channelMetadata{{ChanKey: "a", TempKey: "sim_yd_v5"}}
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("boundary")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []string
|
||||
for _, id := range test.ids {
|
||||
want = append(want, zset.StableTaskID("boundary", operation, id))
|
||||
}
|
||||
sort.Strings(want)
|
||||
got := taskIDs(t, store, "boundary")
|
||||
if len(got) != len(want) || (len(want) > 0 && !reflect.DeepEqual(got, want)) {
|
||||
t.Fatalf("got=%v want=%v", got, want)
|
||||
}
|
||||
if scanner.queries.Load() != test.queries {
|
||||
t.Fatalf("queries=%d want=%d", scanner.queries.Load(), test.queries)
|
||||
}
|
||||
requireMeta(t, store, "boundary", true, int64(len(want)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanLiveHigherIDsDoNotExtendRun(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}}, 3)
|
||||
scanner.hook = func(_ context.Context, _ string, last int64) error {
|
||||
if last > 0 {
|
||||
scanner.cards = append(scanner.cards, Card{ID: 100 + last, ChanKey: "a"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("live")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(taskIDs(t, store, "live")) != 3 || scanner.queries.Load() != 2 {
|
||||
t.Fatal("higher IDs extended finite range")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanSerialTaskIDSetParity(t *testing.T) {
|
||||
for _, scope := range [][]string{nil, {"a", "b"}, {"disabled", "unknown"}} {
|
||||
t.Run(fmt.Sprint(scope), func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
cards := []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "b"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "disabled"}, {ID: 5, ChanKey: "no-template"}, {ID: 6, ChanKey: "unknown"}, {ID: 7, ChanKey: "b"}, {ID: 8, ChanKey: "a"}}
|
||||
var sets [][]string
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
scanner := scannerFor(cards, 7)
|
||||
scanner.channels = []channelMetadata{{ChanKey: "a", TempKey: "sim_yd_v5", Operator: 1}, {ChanKey: "b", TempKey: "sim_dx_v1", Operator: 2}, {ChanKey: "disabled", TempKey: "sim_yd_v5", SyncEnabled: true}, {ChanKey: "no-template"}}
|
||||
p := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 2})
|
||||
run := "parity-" + mode
|
||||
if err := p.Produce(contextForTest(t), scopedMessage(run, scope...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := taskIDs(t, store, run)
|
||||
for i, id := range ids {
|
||||
ids[i] = strings.TrimPrefix(id, run+":")
|
||||
}
|
||||
sets = append(sets, ids)
|
||||
if mode == "channel" && scanner.lists.Load() != 1 {
|
||||
t.Fatal("metadata not loaded once")
|
||||
}
|
||||
_, _ = store.RedisClient().Do(context.Background(), "SREM", store.ActiveRunsKey(), run)
|
||||
}
|
||||
if !reflect.DeepEqual(sets[0], sets[1]) {
|
||||
t.Fatalf("serial=%v channel=%v", sets[0], sets[1])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelSQLSlotsReleasedWhileChannelsWait(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
var cards []Card
|
||||
for i := 0; i < 9; i++ {
|
||||
cards = append(cards, Card{ID: int64(i + 1), ChanKey: fmt.Sprint(i)}, Card{ID: int64(i + 10), ChanKey: fmt.Sprint(i)})
|
||||
}
|
||||
sort.Slice(cards, func(i, j int) bool { return cards[i].ID < cards[j].ID })
|
||||
scanner := scannerFor(cards, 100)
|
||||
scanner.hook = func(ctx context.Context, _ string, _ int64) error { return sleepContext(ctx, 5*time.Millisecond) }
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2, ScanConcurrency: 4, ChannelHighWaterMark: 1, GlobalHighWaterMark: 100, PollInterval: time.Millisecond})
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- p.Produce(ctx, scopedMessage("isolation")) }()
|
||||
waitUntil(t, ctx, func() bool { return len(taskIDs(t, store, "isolation")) == 9 })
|
||||
if peak := scanner.peak.Load(); peak > 4 || peak < 2 {
|
||||
t.Fatalf("SQL peak=%d", peak)
|
||||
}
|
||||
// All nine channels got a first task despite the first four waiting at
|
||||
// channel capacity. SQL slots are free while every channel is paused.
|
||||
if scanner.inflight.Load() != 0 {
|
||||
t.Fatal("SQL tokens held during channel backpressure")
|
||||
}
|
||||
for _, channel := range scanner.channels {
|
||||
if _, err := store.BlockRunScoped(ctx, "isolation", channel.ChanKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "isolation")
|
||||
if err != nil || !meta.ProducerDone {
|
||||
t.Fatalf("meta=%+v err=%v", meta, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelRemainingCapacityAndDuplicateAdmission(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
scanner := scannerFor(nil, 0)
|
||||
p := testProducer(t, scanner, store, Config{GlobalHighWaterMark: 3, ChannelHighWaterMark: 2, PollInterval: time.Millisecond})
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "capacity", Status: "producing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Existing success and waiting states still deduplicate and consume no new
|
||||
// capacity. Active includes leases: inflight must not be added again.
|
||||
stateKey, _ := store.TaskStateKey("capacity")
|
||||
metaKey, _ := store.RunMetaKey("capacity")
|
||||
activeKey, _ := store.RoundActiveKey("capacity", 1, "a")
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", stateKey, "one", `{"status":"success"}`, "two", `{"status":"waiting"}`)
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", metaKey, "activeCount", 2, "inflightCount", 2)
|
||||
stats := &channelScanStats{chanKey: "a", lastSummary: time.Now()}
|
||||
tasks := []zset.Task{{TaskID: "one", RunID: "capacity", ChanKey: "a"}, {TaskID: "two", RunID: "capacity", ChanKey: "a"}, {TaskID: "three", RunID: "capacity", ChanKey: "a"}, {TaskID: "four", RunID: "capacity", ChanKey: "a"}}
|
||||
gate := make(chan struct{}, 1)
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := p.admitChannelPage(ctx, "capacity", tasks, time.Now(), gate, stats); result <- err }()
|
||||
waitUntil(t, ctx, func() bool { meta, e := store.GetRunMeta(ctx, "capacity"); return e == nil && meta.ActiveCount == 3 })
|
||||
if size, err := store.SizeKey(ctx, activeKey); err != nil || size != 1 {
|
||||
t.Fatalf("initial slice size=%d err=%v", size, err)
|
||||
}
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", metaKey, "activeCount", 0, "inflightCount", 0)
|
||||
if err := <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats.duplicates != 2 || stats.added != 2 || stats.enqueueBatches != 2 || stats.globalWait == 0 {
|
||||
t.Fatalf("stats=%+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
// faultRedis forwards to a real isolated Redis, optionally returning a lost
|
||||
// response after Lua succeeded. Never mocks the Lua semantics under test.
|
||||
type faultRedis struct {
|
||||
zset.RedisClient
|
||||
failEnqueue atomic.Int32
|
||||
failRead atomic.Bool
|
||||
failRenew atomic.Bool
|
||||
lostResponse bool
|
||||
reject bool
|
||||
}
|
||||
|
||||
func (r *faultRedis) Do(ctx context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
if r.failRead.Load() && command == "HMGET" {
|
||||
return nil, errors.New("Redis read failure")
|
||||
}
|
||||
return r.RedisClient.Do(ctx, command, args...)
|
||||
}
|
||||
func (r *faultRedis) Eval(ctx context.Context, script string, n int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if r.failRenew.Load() && strings.Contains(script, "PEXPIRE") && strings.Contains(script, "owner ~=") {
|
||||
return gvar.New(0), nil
|
||||
}
|
||||
if n == 4 && strings.Contains(script, "HEXISTS") {
|
||||
if r.reject {
|
||||
return gvar.New(0), nil
|
||||
}
|
||||
if r.failEnqueue.CompareAndSwap(1, 0) {
|
||||
if r.lostResponse {
|
||||
if _, err := r.RedisClient.Eval(ctx, script, n, keys, args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, errors.New("enqueue response unavailable")
|
||||
}
|
||||
}
|
||||
return r.RedisClient.Eval(ctx, script, n, keys, args)
|
||||
}
|
||||
|
||||
func TestChannelRescanPartialAndUncertainWrites(t *testing.T) {
|
||||
for _, uncertain := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(uncertain), func(t *testing.T) {
|
||||
client := &faultRedis{RedisClient: testutil.Redis(t), lostResponse: uncertain}
|
||||
store, _ := zset.New(client, "flow:")
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}}, 4)
|
||||
// Fail the second page: first page must survive; uncertain success is
|
||||
// confirmed only on a same-Run rescan, with stable IDs and no double count.
|
||||
scanner.hook = func(_ context.Context, _ string, last int64) error {
|
||||
if last == 2 {
|
||||
client.failEnqueue.Store(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err == nil {
|
||||
t.Fatal("expected enqueue failure")
|
||||
}
|
||||
active := int64(2)
|
||||
if uncertain {
|
||||
active = 4
|
||||
}
|
||||
requireMeta(t, store, "rescan", false, active)
|
||||
scanner.hook = nil
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMeta(t, store, "rescan", true, 4)
|
||||
if len(taskIDs(t, store, "rescan")) != 4 {
|
||||
t.Fatal("wrong stable task set")
|
||||
}
|
||||
before := scanner.queries.Load()
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if before != scanner.queries.Load() {
|
||||
t.Fatal("producerDone redelivery rescanned cards")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelFailureCancelsAllWorkersWithoutProducerDone(t *testing.T) {
|
||||
for _, kind := range []string{"sql", "redis", "unexplained_rejection", "lease"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
client := &faultRedis{RedisClient: testutil.Redis(t)}
|
||||
store, _ := zset.New(client, "flow:")
|
||||
cards := []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "b"}, {ID: 3, ChanKey: "c"}}
|
||||
scanner := scannerFor(cards, 10)
|
||||
allStarted := make(chan struct{})
|
||||
var started atomic.Int32
|
||||
var once sync.Once
|
||||
scanner.hook = func(ctx context.Context, key string, _ int64) error {
|
||||
if started.Add(1) >= 3 {
|
||||
once.Do(func() { close(allStarted) })
|
||||
}
|
||||
select {
|
||||
case <-allStarted:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if kind == "lease" {
|
||||
client.failRenew.Store(true)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
if key == "a" {
|
||||
switch kind {
|
||||
case "sql":
|
||||
return errors.New("SQL failed")
|
||||
case "redis":
|
||||
client.failRead.Store(true)
|
||||
case "unexplained_rejection":
|
||||
client.reject = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2, RunLockTTL: 90 * time.Millisecond})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("failure")); err == nil {
|
||||
t.Fatal("expected fatal failure")
|
||||
}
|
||||
if scanner.inflight.Load() != 0 {
|
||||
t.Fatal("scanner goroutines leaked")
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "failure")
|
||||
if err != nil || meta.ProducerDone {
|
||||
t.Fatalf("partial scan marked done: %+v err=%v", meta, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanStateStopsAndNoTarget(t *testing.T) {
|
||||
for _, kind := range []string{"terminal", "producerDone", "blocked", "missing", "no-target", "no-cards"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 2)
|
||||
if kind == "no-target" {
|
||||
scanner.channels[0].SyncEnabled = true
|
||||
}
|
||||
if kind == "no-cards" {
|
||||
scanner.cards = nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{})
|
||||
if kind != "no-target" && kind != "no-cards" {
|
||||
if err := store.CreateRun(context.Background(), zset.RunMeta{RunID: "stop", Status: "producing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := store.RunMetaKey("stop")
|
||||
switch kind {
|
||||
case "terminal":
|
||||
_, _ = store.RedisClient().Do(context.Background(), "HSET", key, "status", "completed")
|
||||
case "producerDone":
|
||||
if err := store.MarkProducerDone(context.Background(), "stop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "blocked":
|
||||
_, _ = store.BlockRunScoped(context.Background(), "stop", "a")
|
||||
case "missing":
|
||||
scanner.hook = func(ctx context.Context, _ string, _ int64) error {
|
||||
_, err := store.RedisClient().Do(ctx, "DEL", key)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
err := p.Produce(contextForTest(t), scopedMessage("stop"))
|
||||
if kind == "terminal" {
|
||||
if !errors.Is(err, ErrRunFinished) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
} else if kind == "missing" {
|
||||
if !errors.Is(err, zset.ErrRunNotFound) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(taskIDs(t, store, "stop")) != 0 {
|
||||
t.Fatal("stopped/empty run enqueued tasks")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelOldNonRunPathRemainsSerial(t *testing.T) {
|
||||
redis := &producerRedis{batches: make(map[string]int)}
|
||||
store, _ := zset.New(redis, "flow:")
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 1)
|
||||
p, err := New(scanner, store, Config{ScanMode: "channel", RoundRetryEnabled: false})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = p.Produce(context.Background(), scopedMessage("legacy")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scanner.queries.Load() != 0 || scanner.lists.Load() != 0 {
|
||||
t.Fatal("non-Run path invoked channel scanner")
|
||||
}
|
||||
}
|
||||
|
||||
const operation = FullSyncOperation
|
||||
|
||||
type fakeScanner struct{}
|
||||
|
||||
func (*fakeScanner) SnapshotMaxID(context.Context) (int64, error) { return 0, nil }
|
||||
func (*fakeScanner) Scan(context.Context, int64, int64, int) ([]Card, error) { return nil, nil }
|
||||
func scopedMessage(run string, channels ...string) model.TrafficUsageDailyStartMessage {
|
||||
m := model.TrafficUsageDailyStartMessage{RunID: run, QueryDate: "20261012", Operation: FullSyncOperation}
|
||||
if len(channels) > 0 {
|
||||
m.Scope = &model.SyncScope{Channels: channels}
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
// channelScanner augments, rather than replaces, the legacy scanner contract.
|
||||
// ListChannels is called once before any workers start. ScanChannel must not
|
||||
// mutate the legacy DatabaseScanner.channelMetadata cache.
|
||||
type channelScanner interface {
|
||||
ListChannels(context.Context) ([]channelMetadata, error)
|
||||
ScanChannel(context.Context, string, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
// This exact MySQL query shape is a release gate. Do not silently remove hints,
|
||||
// add an SQL upper bound, or move eligibility filtering after LIMIT.
|
||||
const channelPageSQL = `SELECT c.id, c.iccid, c.msisdn, c.imsi, c.chan_key, c.current_package_code
|
||||
FROM t_sim_card_info AS c FORCE INDEX (idx_card_chan_key)
|
||||
STRAIGHT_JOIN sys_dept AS d
|
||||
ON c.dept_id = d.dept_id
|
||||
AND c.tenant_id = d.tenant_id
|
||||
WHERE c.chan_key = ?
|
||||
AND c.id > ?
|
||||
AND c.iccid <> ''
|
||||
AND c.current_package_code <> ''
|
||||
AND d.deleted_at IS NULL
|
||||
ORDER BY c.id ASC
|
||||
LIMIT ?`
|
||||
|
||||
func (s *DatabaseScanner) ListChannels(ctx context.Context) ([]channelMetadata, error) {
|
||||
var metadata []channelMetadata
|
||||
if err := dao.TSimCardInfo.DB().Model("t_sim_channels").Ctx(ctx).Fields("chan_key,temp_key,sync_enabled,operator").Scan(&metadata); err != nil {
|
||||
return nil, gerror.Wrap(err, "list channel metadata")
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) ScanChannel(ctx context.Context, chanKey string, lastID int64, pageSize int) ([]Card, error) {
|
||||
if chanKey == "" || lastID < 0 || pageSize <= 0 {
|
||||
return nil, gerror.New("invalid channel pagination parameters")
|
||||
}
|
||||
var rows []entity.TSimCardInfo
|
||||
// Use the card DAO's database group, not a separately configured connection.
|
||||
if err := dao.TSimCardInfo.DB().Raw(channelPageSQL, chanKey, lastID, pageSize).Ctx(ctx).Scan(&rows); err != nil {
|
||||
return nil, gerror.Wrap(err, "scan channel cards")
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
cards = append(cards, Card{ID: row.Id, ICCID: row.Iccid, MSISDN: row.Msisdn, IMSI: row.Imsi, ChanKey: row.ChanKey, PackageCode: row.CurrentPackageCode})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
// Pool counters are shared with other worker traffic. Their deltas are direct
|
||||
// observations, not an attribution of all connection waits to this producer.
|
||||
func (*DatabaseScanner) poolWait(ctx context.Context) (int64, int64) {
|
||||
var count, nanos int64
|
||||
for _, item := range dao.TSimCardInfo.DB().Stats(ctx) {
|
||||
stats := item.Stats()
|
||||
count += stats.WaitCount
|
||||
nanos += int64(stats.WaitDuration)
|
||||
}
|
||||
return count, nanos
|
||||
}
|
||||
|
||||
// ValidateScanSettings is strict for values read from configuration. The Go
|
||||
// constructor first fills omitted zero values; explicit YAML zero is rejected.
|
||||
func ValidateScanSettings(mode string, concurrency int) error {
|
||||
if mode != "serial" && mode != "channel" {
|
||||
return gerror.Newf("invalid trafficUsageDailySync.producer.scanMode %q (serial or channel required)", mode)
|
||||
}
|
||||
if concurrency <= 0 {
|
||||
return gerror.New("trafficUsageDailySync.producer.scanConcurrency must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
)
|
||||
|
||||
// SQLite runs the SAME predicate and ordering with only the two MySQL hints
|
||||
// removed. This checks full eligibility/TaskID parity, not MySQL access plans.
|
||||
// Production EXPLAIN must still validate the unmodified channelPageSQL.
|
||||
type sqliteChannelScanner struct{ DatabaseScanner }
|
||||
|
||||
func (s *sqliteChannelScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
query := strings.ReplaceAll(channelPageSQL, "FORCE INDEX (idx_card_chan_key)", "")
|
||||
query = strings.ReplaceAll(query, "STRAIGHT_JOIN", "INNER JOIN")
|
||||
var rows []struct {
|
||||
ID int64 `orm:"id"`
|
||||
ICCID string `orm:"iccid"`
|
||||
MSISDN string `orm:"msisdn"`
|
||||
IMSI string `orm:"imsi"`
|
||||
ChanKey string `orm:"chan_key"`
|
||||
PackageCode string `orm:"current_package_code"`
|
||||
}
|
||||
if err := dao.TSimCardInfo.DB().Raw(query, key, last, size).Ctx(ctx).Scan(&rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
cards = append(cards, Card{ID: row.ID, ICCID: row.ICCID, MSISDN: row.MSISDN, IMSI: row.IMSI, ChanKey: row.ChanKey, PackageCode: row.PackageCode})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
func TestDatabaseQualificationAndCompleteTaskIDParity(t *testing.T) {
|
||||
// No application DB/config: isolated SQLite file and loopback Redis only.
|
||||
if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "sqlite", Link: "sqlite::@file(" + t.TempDir() + "/cards.db)"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := contextForTest(t)
|
||||
db := dao.TSimCardInfo.DB()
|
||||
t.Cleanup(func() { _ = db.Close(context.Background()) })
|
||||
for _, sql := range []string{
|
||||
`CREATE TABLE sys_dept (dept_id INTEGER PRIMARY KEY, tenant_id TEXT, deleted_at DATETIME)`,
|
||||
`CREATE TABLE t_sim_card_info (id INTEGER PRIMARY KEY, iccid TEXT, msisdn TEXT, imsi TEXT, chan_key TEXT, current_package_code TEXT, dept_id INTEGER, tenant_id TEXT)`,
|
||||
`CREATE INDEX idx_card_chan_key ON t_sim_card_info(chan_key)`,
|
||||
`CREATE TABLE t_sim_channels (chan_key TEXT, temp_key TEXT, sync_enabled INTEGER, operator INTEGER)`,
|
||||
`INSERT INTO sys_dept VALUES (1,'t1',NULL),(2,'t2',NULL),(3,'t1','2026-01-01')`,
|
||||
`INSERT INTO t_sim_channels VALUES ('a','sim_yd_v5',0,1),('b','sim_dx_v1',0,2),('disabled','sim_yd_v5',1,1),('no-template','',0,1)`,
|
||||
`INSERT INTO t_sim_card_info VALUES
|
||||
(1,'i1','m1','s1','a','pkg',1,'t1'),
|
||||
(2,'i2','m2','s2','b','pkg',2,'t2'),
|
||||
(3,'i3','','','a','pkg',1,'t1'),
|
||||
(4,'','','','a','pkg',1,'t1'),
|
||||
(5,'i5','','','a','',1,'t1'),
|
||||
(6,'i6','','','a','pkg',1,'wrong-tenant'),
|
||||
(7,'i7','','','a','pkg',3,'t1'),
|
||||
(8,'i8','','','b','pkg',99,'t2'),
|
||||
(9,'i9','','','disabled','pkg',1,'t1'),
|
||||
(10,'i10','','','no-template','pkg',1,'t1'),
|
||||
(11,'i11','','','unknown','pkg',1,'t1'),
|
||||
(12,'i12','','','','pkg',1,'t1'),
|
||||
(13,'i13','','','b','pkg',2,'t2')`,
|
||||
} {
|
||||
if _, err := db.Exec(ctx, sql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store := testStore(t)
|
||||
for _, scope := range [][]string{nil, {"a"}, {"b", "disabled", "no-template", "unknown"}, {"unknown"}} {
|
||||
var sets [][]string
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
scanner := &sqliteChannelScanner{}
|
||||
p := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 2})
|
||||
run := "db-" + mode
|
||||
if err := p.Produce(ctx, scopedMessage(run, scope...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := taskIDs(t, store, run)
|
||||
for i, id := range ids {
|
||||
ids[i] = strings.TrimPrefix(id, run+":")
|
||||
}
|
||||
sort.Strings(ids)
|
||||
sets = append(sets, ids)
|
||||
stateKey, _ := store.TaskStateKey(run)
|
||||
metaKey, _ := store.RunMetaKey(run)
|
||||
chanKey, _ := store.RunChannelsKey(run)
|
||||
_, _ = store.RedisClient().Do(ctx, "DEL", stateKey, metaKey, chanKey)
|
||||
for _, key := range []string{"a", "b"} {
|
||||
activeKey, _ := store.RoundActiveKey(run, 1, key)
|
||||
_, _ = store.RedisClient().Do(ctx, "DEL", activeKey)
|
||||
}
|
||||
_, _ = store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), run)
|
||||
}
|
||||
if !reflect.DeepEqual(sets[0], sets[1]) {
|
||||
t.Fatalf("scope=%v serial=%v channel=%v", scope, sets[0], sets[1])
|
||||
}
|
||||
if scope == nil {
|
||||
want := []string{operation + ":1", operation + ":13", operation + ":2", operation + ":3"}
|
||||
sort.Strings(want)
|
||||
if !reflect.DeepEqual(sets[0], want) {
|
||||
t.Fatalf("qualification set=%v want=%v", sets[0], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelSQLShapeAndParameters(t *testing.T) {
|
||||
for _, term := range []string{"FORCE INDEX (idx_card_chan_key)", "STRAIGHT_JOIN sys_dept", "c.chan_key = ?", "c.id > ?", "c.iccid <> ''", "c.current_package_code", "c.current_package_code <> ''", "d.deleted_at IS NULL", "c.tenant_id = d.tenant_id", "ORDER BY c.id ASC", "LIMIT ?"} {
|
||||
if !strings.Contains(channelPageSQL, term) {
|
||||
t.Fatalf("missing %q", term)
|
||||
}
|
||||
}
|
||||
if strings.Contains(channelPageSQL, "<=") || strings.Count(channelPageSQL, "?") != 3 {
|
||||
t.Fatal("SQL contains upper bound or changed bind parameter count")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
func TestChannelBackpressureDoesNotStarveHighIDChannel(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx, cancel := context.WithCancel(contextForTest(t))
|
||||
defer cancel()
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "slow"}, {ID: 2, ChanKey: "slow"}, {ID: 3, ChanKey: "slow"}, {ID: 1000, ChanKey: "fast"}}, 1000)
|
||||
p := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 1, ScanConcurrency: 1, ChannelHighWaterMark: 1, GlobalHighWaterMark: 10, PollInterval: time.Millisecond})
|
||||
result := make(chan error, 1)
|
||||
started := time.Now()
|
||||
go func() { result <- p.Produce(ctx, scopedMessage("starvation")) }()
|
||||
slowKey, _ := store.RoundActiveKey("starvation", 1, "slow")
|
||||
fastKey, _ := store.RoundActiveKey("starvation", 1, "fast")
|
||||
waitUntil(t, ctx, func() bool { n, e := store.SizeKey(ctx, slowKey); return e == nil && n == 1 })
|
||||
if mode == "channel" {
|
||||
waitUntil(t, ctx, func() bool { n, e := store.SizeKey(ctx, fastKey); return e == nil && n == 1 })
|
||||
} else {
|
||||
// No consumer drains slow. The serial cursor cannot reach card 1000.
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
if n, err := store.SizeKey(ctx, fastKey); err != nil || n != 0 {
|
||||
t.Fatalf("serial fast size=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
slow, _ := store.SizeKey(ctx, slowKey)
|
||||
fast, _ := store.SizeKey(ctx, fastKey)
|
||||
t.Logf("mode=%s slowQueued=%d fastQueued=%d scanConcurrency=1 channelHWM=1 elapsedMs=%d", mode, slow, fast, time.Since(started).Milliseconds())
|
||||
cancel()
|
||||
if err := <-result; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation err=%v", err)
|
||||
}
|
||||
requireMeta(t, store, "starvation", false, slow+fast)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelPayloadRetainsPeriodPackageAndIdentifiers(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
card := Card{ID: 10, ICCID: "iccid", MSISDN: "msisdn", IMSI: "imsi", ChanKey: "a", PackageCode: "pkg-history"}
|
||||
p := testProducer(t, scannerFor([]Card{card}, card.ID), store, Config{PageSize: 1})
|
||||
message := scopedMessage("payload", "a")
|
||||
if err := p.Produce(ctx, message); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := store.ClaimRun(ctx, message.RunID, "a", time.Now().Add(time.Second), time.Now().Add(time.Minute), 1)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
task := claimed[0].Task
|
||||
if task.TaskID != zset.StableTaskID(message.RunID, FullSyncOperation, card.ID) || task.Operation != FullSyncOperation || task.QueryMonth != message.QueryDate || task.PackageCode != card.PackageCode || task.ICCID != card.ICCID || task.MSISDN != card.MSISDN || task.IMSI != card.IMSI || task.TempKey != "sim_yd_v5" || task.Operator != 1 {
|
||||
t.Fatalf("task=%+v", task)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZombieWithBacklogDrainsBeforeCompletion(t *testing.T) {
|
||||
p, store := newZombieTestProducer(t)
|
||||
ctx := contextForTest(t)
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-backlog"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks := []zset.Task{{TaskID: "zombie-card", RunID: "zombie-backlog", ChanKey: "a"}}
|
||||
if _, err := store.EnqueueRunBatch(ctx, "zombie-backlog", "a", tasks, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending, err := p.hasPendingTasks(ctx, "new"); err != nil || !pending {
|
||||
t.Fatalf("backlog must retain backpressure: pending=%t err=%v", pending, err)
|
||||
}
|
||||
requireMeta(t, store, "zombie-backlog", true, 1)
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "zombie-backlog"); err != nil || complete {
|
||||
t.Fatalf("premature complete=%t err=%v", complete, err)
|
||||
}
|
||||
claimed, err := store.ClaimRun(ctx, "zombie-backlog", "a", time.Now().Add(time.Second), time.Now().Add(time.Minute), 1)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
if ok, err := store.FinalizeRunTask(ctx, claimed[0], zset.TaskOutcome{Success: true}); err != nil || !ok {
|
||||
t.Fatalf("finalize=%t err=%v", ok, err)
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "zombie-backlog", time.Hour); err != nil || !complete {
|
||||
t.Fatalf("complete=%t err=%v", complete, err)
|
||||
}
|
||||
meta, _ := store.RunMetaKey("zombie-backlog")
|
||||
ttl, err := store.RedisClient().Do(ctx, "TTL", meta)
|
||||
if err != nil || ttl.Int() <= 0 {
|
||||
t.Fatalf("retention ttl=%v err=%v", ttl, err)
|
||||
}
|
||||
active, err := store.ActiveRuns(ctx)
|
||||
if err != nil || len(active) != 0 {
|
||||
t.Fatalf("active=%v err=%v", active, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,739 @@
|
||||
// Package dailyusage produces the previous-day usage full-sync run.
|
||||
//
|
||||
// It is the daily-usage twin of producer/monthlyusage and producer/simstatus:
|
||||
// it scans the eligible cards once, groups them by channel, and enqueues one
|
||||
// Redis ZSet task per card. The zset consumer owns pacing, retries, failure
|
||||
// pooling, archiving and replay.
|
||||
//
|
||||
// The only structural difference from the monthly producer is that the run
|
||||
// period is a day (yyyyMMdd) rather than a month (yyyyMM). That value travels
|
||||
// in zset.Task.QueryMonth, which is a period identifier rather than a month
|
||||
// name; see that field's documentation.
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// FullSyncOperation identifies a daily usage run in run metadata, archives and
|
||||
// replay. contract.SyncKindForOperation maps it to traffic_usage_daily.
|
||||
const FullSyncOperation = model.OperationPreviousDayUsageFullSync
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
// The run hash and its archive JSON must stay bounded regardless of how many
|
||||
// channels a scope selects.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels. It lets
|
||||
// operators confirm two runs covered the same scope without storing the list.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ScanMode string
|
||||
ScanConcurrency int
|
||||
PageSize int
|
||||
GlobalHighWaterMark int64
|
||||
ChannelHighWaterMark int64
|
||||
PollInterval time.Duration
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
RoundRetryEnabled bool
|
||||
MaxRounds int
|
||||
FailureRetention time.Duration
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.ScanMode == "" {
|
||||
c.ScanMode = "serial"
|
||||
}
|
||||
if c.ScanConcurrency == 0 {
|
||||
c.ScanConcurrency = 4
|
||||
}
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 100
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = 2 * time.Second
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "daily:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.MaxRounds <= 0 {
|
||||
c.MaxRounds = 3
|
||||
}
|
||||
if c.FailureRetention <= 0 {
|
||||
c.FailureRetention = 30 * 24 * time.Hour
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type CardScanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
Scan(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
PackageCode string
|
||||
SyncEnabled bool
|
||||
Operator int
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
Operator int `orm:"operator"`
|
||||
}
|
||||
|
||||
type DatabaseScanner struct {
|
||||
channelMetadata map[string]channelMetadata
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := dailyUsageEligibleCardModel(ctx).
|
||||
Fields(cardTable + "." + columns.Id + " AS " + columns.Id).
|
||||
OrderDesc(cardTable + "." + columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read card snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) Scan(ctx context.Context, lastID, maxID int64, pageSize int) ([]Card, error) {
|
||||
if maxID <= lastID {
|
||||
return nil, nil
|
||||
}
|
||||
if lastID == 0 || s.channelMetadata == nil {
|
||||
var metadata []channelMetadata
|
||||
if err := g.DB().Model("t_sim_channels").Fields("chan_key,temp_key,sync_enabled,operator").Scan(&metadata); err != nil {
|
||||
return nil, fmt.Errorf("scan channel metadata: %w", err)
|
||||
}
|
||||
s.channelMetadata = make(map[string]channelMetadata, len(metadata))
|
||||
for _, item := range metadata {
|
||||
s.channelMetadata[item.ChanKey] = item
|
||||
}
|
||||
}
|
||||
var rows []entity.TSimCardInfo
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
query := dailyUsageEligibleCardModel(ctx).
|
||||
Fields(
|
||||
cardTable+"."+columns.Id+" AS "+columns.Id,
|
||||
cardTable+"."+columns.Iccid+" AS "+columns.Iccid,
|
||||
cardTable+"."+columns.Msisdn+" AS "+columns.Msisdn,
|
||||
cardTable+"."+columns.Imsi+" AS "+columns.Imsi,
|
||||
cardTable+"."+columns.ChanKey+" AS "+columns.ChanKey,
|
||||
// The daily usage child table stores the package code and the
|
||||
// diagnosis page reads it back, so it must be carried on the task.
|
||||
cardTable+"."+columns.CurrentPackageCode+" AS "+columns.CurrentPackageCode,
|
||||
).
|
||||
WhereGT(cardTable+"."+columns.Id, lastID).
|
||||
WhereLTE(cardTable+"."+columns.Id, maxID).
|
||||
OrderAsc(cardTable + "." + columns.Id).
|
||||
Limit(pageSize)
|
||||
if err := query.Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan card information: %w", err)
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
channel := s.channelMetadata[row.ChanKey]
|
||||
cards = append(cards, Card{
|
||||
ID: row.Id,
|
||||
ICCID: row.Iccid,
|
||||
MSISDN: row.Msisdn,
|
||||
IMSI: row.Imsi,
|
||||
ChanKey: row.ChanKey,
|
||||
TempKey: channel.TempKey,
|
||||
PackageCode: row.CurrentPackageCode,
|
||||
SyncEnabled: channel.SyncEnabled,
|
||||
Operator: channel.Operator,
|
||||
})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
// dailyUsageEligibleCardModel mirrors the monthly usage eligibility rule and the
|
||||
// legacy admin SyncDaily scan: the card must belong to a live department, carry
|
||||
// an ICCID and channel, and have a current package code.
|
||||
func dailyUsageEligibleCardModel(ctx context.Context) *gdb.Model {
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
deptTable := dao.SysDept.Table()
|
||||
cardColumns := dao.TSimCardInfo.Columns()
|
||||
deptColumns := dao.SysDept.Columns()
|
||||
|
||||
return dao.TSimCardInfo.Ctx(ctx).
|
||||
InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId).
|
||||
Where(cardTable+"."+cardColumns.Iccid+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.ChanKey+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", "").
|
||||
WhereNull(deptTable + "." + deptColumns.DeletedAt)
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner CardScanner
|
||||
store *zset.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
observation *supplyObservation
|
||||
}
|
||||
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
eligible int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
stoppedTasks int64
|
||||
channels map[string]struct{}
|
||||
scopeChannels int
|
||||
skippedChannels int
|
||||
}
|
||||
|
||||
func (s *productionStats) channelCount() int {
|
||||
return len(s.channels)
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, queryDate, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=daily_usage_producer status=%s runId=%s queryDate=%s snapshotMaxId=%d pages=%d scanned=%d eligible=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d stoppedTasks=%d channelCount=%d scopeChannels=%d skippedChannels=%d durationMs=%d", status, runID, queryDate, maxID, s.pages, s.scanned, s.eligible, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.stoppedTasks, s.channelCount(), s.scopeChannels, s.skippedChannels, time.Since(startedAt).Milliseconds())
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
if status == "failed" {
|
||||
g.Log().Error(ctx, message)
|
||||
return
|
||||
}
|
||||
if status == "skipped" {
|
||||
g.Log().Warning(ctx, message)
|
||||
return
|
||||
}
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
|
||||
func New(scanner CardScanner, store *zset.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("daily usage producer dependencies are incomplete")
|
||||
}
|
||||
config = config.withDefaults()
|
||||
if err := ValidateScanSettings(config.ScanMode, config.ScanConcurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.ScanMode == "channel" {
|
||||
if _, ok := scanner.(channelScanner); !ok {
|
||||
return nil, fmt.Errorf("channel scan requires ListChannels and ScanChannel")
|
||||
}
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config, owner: newOwner()}, nil
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("daily usage producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected; callers
|
||||
// should log and ack instead of retrying.
|
||||
var ErrRunFinished = fmt.Errorf("daily usage run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if errors.Is(err, zset.ErrRunNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
// Produce scans the eligible cards once and enqueues one task per card.
|
||||
//
|
||||
// message.QueryDate (yyyyMMdd) is stored in zset.Task.QueryMonth, the queue's
|
||||
// period-identifier field, so the zset store, run metadata, archive, replay and
|
||||
// Lua scripts need no change for a daily run.
|
||||
func (p *Producer) Produce(ctx context.Context, message model.TrafficUsageDailyStartMessage) (err error) {
|
||||
startedAt := time.Now()
|
||||
stats := productionStats{channels: make(map[string]struct{})}
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryDate, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
|
||||
if message.RunID == "" {
|
||||
err = fmt.Errorf("runId is empty")
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "validate", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if message.QueryDate == "" {
|
||||
err = fmt.Errorf("queryDate is empty")
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "validate", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats.scopeChannels = scope.ChannelCount()
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if p.config.RoundRetryEnabled {
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryDate, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
}
|
||||
acquired, err := p.store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryDate, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer releaseCancel()
|
||||
_, _ = p.store.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
pending, err := p.hasPendingTasks(ctx, message.RunID)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "pending_check", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if pending {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryDate, "pending_check", "pending_tasks", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
renewErr := make(chan error, 1)
|
||||
go p.renewLock(runCtx, cancel, renewErr)
|
||||
ctx = runCtx
|
||||
if message.Operation == "" {
|
||||
message.Operation = FullSyncOperation
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
if err = p.store.CreateRun(ctx, zset.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: message.Operation,
|
||||
QueryMonth: message.QueryDate,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "run_init", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
maxID, err := p.scanner.SnapshotMaxID(ctx)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, "snapshot", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=daily_usage_producer status=started runId=%s queryDate=%s snapshotMaxId=%d scopeChannels=%d scanMode=%s scanConcurrency=%d pageSize=%d", message.RunID, message.QueryDate, maxID, scope.ChannelCount(), p.config.ScanMode, p.config.ScanConcurrency, p.config.PageSize)
|
||||
stage := "scan"
|
||||
defer func() {
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryDate, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, message.QueryDate, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
if p.config.RoundRetryEnabled && p.config.ScanMode == "channel" {
|
||||
stage = "channel_scan"
|
||||
if err = p.produceChannels(ctx, message, scope, maxID, startedAt, &stats); err != nil {
|
||||
if isProductionClosed(err) {
|
||||
// A same-Run redelivery after producerDone leaves consumption alone.
|
||||
err = nil
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Verify ownership immediately before closing production. The periodic
|
||||
// renewer still cancels scanners and capacity waits on lease loss.
|
||||
var owned bool
|
||||
owned, err = p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owned {
|
||||
return fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "producer_done"
|
||||
err = p.store.MarkProducerDone(ctx, message.RunID)
|
||||
return err
|
||||
}
|
||||
lastID := int64(0)
|
||||
for lastID < maxID {
|
||||
stage = "capacity"
|
||||
if err = p.waitForCapacity(ctx, message.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "scan"
|
||||
cards, scanErr := p.scanner.Scan(ctx, lastID, maxID, p.config.PageSize)
|
||||
if scanErr != nil {
|
||||
err = scanErr
|
||||
return err
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
stats.pages++
|
||||
stats.scanned += int64(len(cards))
|
||||
groupedTasks := make(map[string][]zset.Task)
|
||||
for _, card := range cards {
|
||||
lastID = card.ID
|
||||
if card.ChanKey == "" {
|
||||
stats.skippedEmptyChan++
|
||||
g.Log().Warningf(ctx, "event=daily_usage_card status=skipped stage=validate cardId=%d reason=empty_chan_key", card.ID)
|
||||
continue
|
||||
}
|
||||
// Scope is applied before the sync_enabled and template checks so a
|
||||
// partial run only ever reports on the channels it was asked for.
|
||||
if !scope.Allows(card.ChanKey) {
|
||||
stats.skippedOutOfScope++
|
||||
continue
|
||||
}
|
||||
if card.SyncEnabled {
|
||||
stats.skippedSyncDisabled++
|
||||
g.Log().Warningf(ctx, "event=daily_usage_card status=skipped stage=validate cardId=%d chanKey=%s reason=sync_disabled", card.ID, card.ChanKey)
|
||||
continue
|
||||
}
|
||||
if card.TempKey == "" {
|
||||
stats.skippedNoTemplate++
|
||||
g.Log().Warningf(ctx, "event=daily_usage_card status=skipped stage=validate cardId=%d chanKey=%s reason=missing_channel_template", card.ID, card.ChanKey)
|
||||
continue
|
||||
}
|
||||
stats.eligible++
|
||||
stats.channels[card.ChanKey] = struct{}{}
|
||||
task := zset.Task{
|
||||
TaskID: zset.StableTaskID(message.RunID, message.Operation, card.ID),
|
||||
RunID: message.RunID,
|
||||
Operation: message.Operation,
|
||||
QueryMonth: message.QueryDate,
|
||||
ChanKey: card.ChanKey,
|
||||
TempKey: card.TempKey,
|
||||
Operator: card.Operator,
|
||||
CardID: card.ID,
|
||||
ICCID: card.ICCID,
|
||||
MSISDN: card.MSISDN,
|
||||
IMSI: card.IMSI,
|
||||
PackageCode: card.PackageCode,
|
||||
}
|
||||
groupedTasks[card.ChanKey] = append(groupedTasks[card.ChanKey], task)
|
||||
}
|
||||
now := time.Now()
|
||||
for chanKey, tasks := range groupedTasks {
|
||||
stage = "enqueue"
|
||||
if err = p.waitForChannelCapacity(ctx, message.RunID, chanKey); err != nil {
|
||||
return err
|
||||
}
|
||||
var added int
|
||||
var enqueueErr error
|
||||
if p.config.RoundRetryEnabled {
|
||||
added, enqueueErr = p.store.EnqueueRunBatch(ctx, message.RunID, chanKey, tasks, now)
|
||||
} else {
|
||||
added, enqueueErr = p.store.EnqueueBatch(ctx, chanKey, tasks, now)
|
||||
}
|
||||
if enqueueErr != nil {
|
||||
err = enqueueErr
|
||||
return err
|
||||
}
|
||||
stats.enqueued += int64(added)
|
||||
if added < len(tasks) {
|
||||
stats.stoppedTasks += int64(len(tasks) - added)
|
||||
}
|
||||
if added == 0 {
|
||||
g.Log().Warningf(ctx, "event=daily_usage_channel status=skipped runId=%s queryDate=%s chanKey=%s count=%d reason=stopped", message.RunID, message.QueryDate, chanKey, len(tasks))
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
if err = p.store.MarkProducerDone(ctx, message.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Producer) hasPendingTasks(ctx context.Context, currentRunID ...string) (bool, error) {
|
||||
currentID := ""
|
||||
if len(currentRunID) > 0 {
|
||||
currentID = currentRunID[0]
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, runID := range runs {
|
||||
if runID == currentID {
|
||||
// A redelivered message for the same run may resume a producer
|
||||
// that stopped before producerDone. Stable task IDs make the
|
||||
// rescan idempotent; other runs still provide backpressure.
|
||||
continue
|
||||
}
|
||||
meta, metaErr := p.store.GetRunMeta(ctx, runID)
|
||||
if metaErr != nil {
|
||||
// The active index is a global key written outside the run-scoped
|
||||
// Lua transaction, so it can list a run whose hash is already gone
|
||||
// (expired, evicted, or cleaned by hand). Returning an error here
|
||||
// would surface as a non-retryable failure and make the consumer
|
||||
// redeliver this request forever. Treat it as a stale index entry
|
||||
// and drop it.
|
||||
if errors.Is(metaErr, zset.ErrRunNotFound) {
|
||||
if _, removeErr := p.store.RedisClient().Do(ctx, "SREM", p.store.ActiveRunsKey(), runID); removeErr != nil {
|
||||
return false, removeErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=daily_usage_producer status=skipped stage=pending_check reason=stale_index runId=%s", runID)
|
||||
continue
|
||||
}
|
||||
return false, metaErr
|
||||
}
|
||||
if meta.Status != "producing" && meta.Status != "running" {
|
||||
continue
|
||||
}
|
||||
// Channel blocking can set running before production ends. Only
|
||||
// producerDone identifies consumer-owned work; replay has its own lock.
|
||||
if meta.ProducerDone || meta.SyncType == "replay" {
|
||||
return true, nil
|
||||
}
|
||||
// Recheck the unique producer lease immediately before reclaiming
|
||||
// another run. A lost lease or an unspecified current run is not proof
|
||||
// that the old producer has stopped.
|
||||
if currentID == "" {
|
||||
return true, nil
|
||||
}
|
||||
owned, lockErr := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if lockErr != nil {
|
||||
return false, lockErr
|
||||
}
|
||||
if !owned {
|
||||
return true, nil
|
||||
}
|
||||
// Close production, not the run: consumers retain queued, leased and
|
||||
// next-round work and apply retention/index cleanup on normal completion.
|
||||
reclaimed, reclaimErr := p.store.MarkProducerAbandoned(ctx, runID)
|
||||
if reclaimErr != nil {
|
||||
return false, reclaimErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=daily_usage_producer status=skipped stage=pending_check reason=zombie_run runId=%s reclaimed=%t activeCount=%d inflightCount=%d nextRoundCount=%d", runID, reclaimed, meta.ActiveCount, meta.InflightCount, meta.NextRoundCount)
|
||||
if !reclaimed || meta.ActiveCount > 0 || meta.InflightCount > 0 || meta.NextRoundCount > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
channels, err := p.store.ActiveChannels(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, chanKey := range channels {
|
||||
size, sizeErr := p.store.Size(ctx, chanKey)
|
||||
if sizeErr != nil {
|
||||
return false, sizeErr
|
||||
}
|
||||
if size > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelFunc, errors chan<- error) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ok, err := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil || !ok {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
select {
|
||||
case errors <- fmt.Errorf("renew producer lock: %w", err):
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var bytes [12]byte
|
||||
if _, err := rand.Read(bytes[:]); err == nil {
|
||||
return "producer-" + hex.EncodeToString(bytes[:])
|
||||
}
|
||||
return fmt.Sprintf("producer-%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func (p *Producer) waitForCapacity(ctx context.Context, runID ...string) error {
|
||||
if p.config.GlobalHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
if p.config.RoundRetryEnabled && len(runID) > 0 && runID[0] != "" {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
channels, err := p.store.RunChannels(ctx, runID[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, chanKey := range channels {
|
||||
key, keyErr := p.store.RoundActiveKey(runID[0], meta.CurrentRound, chanKey)
|
||||
if keyErr != nil {
|
||||
return keyErr
|
||||
}
|
||||
size, sizeErr := p.store.SizeKey(ctx, key)
|
||||
if sizeErr != nil {
|
||||
return sizeErr
|
||||
}
|
||||
total += size
|
||||
}
|
||||
if total < p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
if err = sleepContext(ctx, p.config.PollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
channels, err := p.store.ActiveChannels(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, chanKey := range channels {
|
||||
size, sizeErr := p.store.Size(ctx, chanKey)
|
||||
if sizeErr != nil {
|
||||
return sizeErr
|
||||
}
|
||||
total += size
|
||||
}
|
||||
if total < p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
if err = sleepContext(ctx, p.config.PollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) waitForChannelCapacity(ctx context.Context, runIDAndChan ...string) error {
|
||||
if p.config.ChannelHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
chanKey := ""
|
||||
runID := ""
|
||||
if len(runIDAndChan) == 1 {
|
||||
chanKey = runIDAndChan[0]
|
||||
} else if len(runIDAndChan) >= 2 {
|
||||
runID, chanKey = runIDAndChan[0], runIDAndChan[1]
|
||||
}
|
||||
for {
|
||||
var size int64
|
||||
var err error
|
||||
if p.config.RoundRetryEnabled && runID != "" {
|
||||
meta, metaErr := p.store.GetRunMeta(ctx, runID)
|
||||
if metaErr != nil {
|
||||
return metaErr
|
||||
}
|
||||
key, keyErr := p.store.RoundActiveKey(runID, meta.CurrentRound, chanKey)
|
||||
if keyErr != nil {
|
||||
return keyErr
|
||||
}
|
||||
size, err = p.store.SizeKey(ctx, key)
|
||||
} else {
|
||||
size, err = p.store.Size(ctx, chanKey)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size < p.config.ChannelHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
if err = sleepContext(ctx, p.config.PollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, duration time.Duration) error {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
type pageScanner struct {
|
||||
cards []Card
|
||||
}
|
||||
|
||||
func (s *pageScanner) SnapshotMaxID(context.Context) (int64, error) {
|
||||
if len(s.cards) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return s.cards[len(s.cards)-1].ID, nil
|
||||
}
|
||||
|
||||
func (s *pageScanner) Scan(_ context.Context, lastID, _ int64, _ int) ([]Card, error) {
|
||||
if lastID != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.cards, nil
|
||||
}
|
||||
|
||||
// producerRedis records the enqueued members so a test can assert both the
|
||||
// batch shape and the exact task payload (period and package code).
|
||||
type producerRedis struct {
|
||||
mu sync.Mutex
|
||||
members []string
|
||||
batches map[string]int
|
||||
}
|
||||
|
||||
func (r *producerRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
|
||||
switch strings.ToUpper(command) {
|
||||
case "SET":
|
||||
return gvar.New("OK"), nil
|
||||
case "SMEMBERS":
|
||||
return gvar.New([]string{}), nil
|
||||
case "HGETALL":
|
||||
// No stored run metadata: the producer treats this as a first delivery.
|
||||
return gvar.New(map[string]string{}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected command %s", command)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *producerRedis) Eval(_ context.Context, _ string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if numKeys == 1 {
|
||||
return gvar.New(int64(1)), nil
|
||||
}
|
||||
if numKeys != 3 {
|
||||
return nil, fmt.Errorf("unexpected key count %d", numKeys)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.batches == nil {
|
||||
r.batches = make(map[string]int)
|
||||
}
|
||||
count := len(args) - 3
|
||||
r.batches[keys[0]] += count
|
||||
for _, arg := range args[3:] {
|
||||
if member, ok := arg.(string); ok {
|
||||
r.members = append(r.members, member)
|
||||
}
|
||||
}
|
||||
return gvar.New(count), nil
|
||||
}
|
||||
|
||||
func newProducer(t *testing.T, cards []Card) (*Producer, *producerRedis) {
|
||||
t.Helper()
|
||||
redis := &producerRedis{batches: make(map[string]int)}
|
||||
store, err := zset.New(redis, "daily:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(&pageScanner{cards: cards}, store, Config{PageSize: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return producer, redis
|
||||
}
|
||||
|
||||
func dailyMessage(runID string, channels ...string) model.TrafficUsageDailyStartMessage {
|
||||
message := model.TrafficUsageDailyStartMessage{
|
||||
RunID: runID,
|
||||
Operation: FullSyncOperation,
|
||||
QueryDate: "20261009",
|
||||
TriggeredAt: "2026-10-10T06:00:00+08:00",
|
||||
}
|
||||
if len(channels) > 0 {
|
||||
message.Scope = &model.SyncScope{Channels: channels}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func TestProduceCarriesQueryDateAndPackageCode(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "89860001", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "PKG-A", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), dailyMessage("daily-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.members) != 1 {
|
||||
t.Fatalf("members = %+v, want exactly one task", redis.members)
|
||||
}
|
||||
member := redis.members[0]
|
||||
// The period rides in queryMonth because the queue only copies that field.
|
||||
if !strings.Contains(member, `"queryMonth":"20261009"`) {
|
||||
t.Fatalf("task member lost the daily period: %s", member)
|
||||
}
|
||||
// The package code must reach the TDengine child table via the task.
|
||||
if !strings.Contains(member, `"packageCode":"PKG-A"`) {
|
||||
t.Fatalf("task member lost the package code: %s", member)
|
||||
}
|
||||
if !strings.Contains(member, `"operation":"`+FullSyncOperation+`"`) {
|
||||
t.Fatalf("task member lost the operation: %s", member)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceUsesStableTaskIDForIdempotency(t *testing.T) {
|
||||
producer, _ := newProducer(t, []Card{
|
||||
{ID: 42, ICCID: "89860042", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "PKG-A"},
|
||||
})
|
||||
want := zset.StableTaskID("daily-1", FullSyncOperation, 42)
|
||||
if got := zset.StableTaskID("daily-1", FullSyncOperation, 42); got != want {
|
||||
t.Fatalf("StableTaskID is not deterministic: %s vs %s", got, want)
|
||||
}
|
||||
if err := producer.Produce(context.Background(), dailyMessage("daily-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeEnqueuesOnlySelectedChannels(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 2, ICCID: "2", ChanKey: "173", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 3, ICCID: "3", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 4, ICCID: "4", ChanKey: "189", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), dailyMessage("daily-scoped", "217")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 1 {
|
||||
t.Fatalf("enqueue batches = %+v, want only the 217 channel", redis.batches)
|
||||
}
|
||||
if redis.batches["daily:217"] != 2 {
|
||||
t.Fatalf("enqueue batches = %+v, want 2 tasks on daily:217", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceFullScopeKeepsEveryChannel(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 2, ICCID: "2", ChanKey: "173", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), dailyMessage("daily-full")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 2 {
|
||||
t.Fatalf("enqueue batches = %+v, want both channels for a full sync", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeDoesNotOverrideSyncDisabled(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 2, ICCID: "2", ChanKey: "173", TempKey: "sim_yd_v5", PackageCode: "P", SyncEnabled: true},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), dailyMessage("daily-scoped", "173")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 0 {
|
||||
t.Fatalf("enqueue batches = %+v, want none (selected channel is sync-disabled)", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceRejectsMissingPeriod(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
message := dailyMessage("daily-bad")
|
||||
message.QueryDate = ""
|
||||
if err := producer.Produce(context.Background(), message); err == nil {
|
||||
t.Fatal("expected a missing queryDate to be rejected")
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 0 {
|
||||
t.Fatalf("rejected message must not enqueue: %+v", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceRejectsInvalidScope(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
if err := producer.Produce(context.Background(), dailyMessage("daily-bad", "invalid/chan")); err == nil {
|
||||
t.Fatal("expected invalid chanKey to be rejected")
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 0 {
|
||||
t.Fatalf("rejected scope must not enqueue: %+v", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeDigestIsOrderIndependent(t *testing.T) {
|
||||
first, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"173", "217"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"217", "173"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scopeDigest(first) != scopeDigest(second) {
|
||||
t.Fatalf("digest depends on channel order: %s vs %s", scopeDigest(first), scopeDigest(second))
|
||||
}
|
||||
if scopeDigest(nil) != "" {
|
||||
t.Fatal("a full sync must have an empty digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAlreadyFinishedTreatsMissingRunAsFirstDelivery(t *testing.T) {
|
||||
producer, _ := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
finished, err := producer.runAlreadyFinished(context.Background(), "fresh")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if finished {
|
||||
t.Fatal("a run without stored metadata must not be reported as finished")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
// Producer-local telemetry: no run/archive schema changes and no assertion of
|
||||
// consumer readiness. Empty queues alone do not prove consumer starvation.
|
||||
type supplyChannel struct {
|
||||
stage string
|
||||
pending int
|
||||
version uint64
|
||||
ended bool
|
||||
firstQuery, firstEnqueue *int64
|
||||
changed time.Time
|
||||
stageDuration map[string]time.Duration
|
||||
validSamples, missingSamples int64
|
||||
}
|
||||
|
||||
type supplyObservation struct {
|
||||
mu sync.Mutex
|
||||
started time.Time
|
||||
channels map[string]*supplyChannel
|
||||
}
|
||||
|
||||
func (o *supplyObservation) event(key, stage string, pending int, added, ended bool, now time.Time) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
s := o.channels[key]
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.stageDuration[s.stage] += now.Sub(s.changed)
|
||||
s.changed, s.stage, s.pending, s.ended = now, stage, pending, ended
|
||||
s.version++
|
||||
if stage == "sql_query" && s.firstQuery == nil {
|
||||
v := now.Sub(o.started).Milliseconds()
|
||||
s.firstQuery = &v
|
||||
}
|
||||
if added && s.firstEnqueue == nil {
|
||||
v := now.Sub(o.started).Milliseconds()
|
||||
s.firstEnqueue = &v
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve fractional milliseconds across transitions; round only for logs.
|
||||
func (s *supplyChannel) stageMilliseconds(now time.Time) map[string]int64 {
|
||||
if !s.ended {
|
||||
s.stageDuration[s.stage] += now.Sub(s.changed)
|
||||
s.changed = now
|
||||
}
|
||||
ms := make(map[string]int64, len(s.stageDuration))
|
||||
for stage, duration := range s.stageDuration {
|
||||
ms[stage] = duration.Milliseconds()
|
||||
}
|
||||
return ms
|
||||
}
|
||||
|
||||
func (p *Producer) observe(key, stage string, pending int, added, ended bool) {
|
||||
if p.observation != nil {
|
||||
p.observation.event(key, stage, pending, added, ended, time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
// Queue-only samples mirror e6ec5b8: cardinalities, never task bodies. All keys
|
||||
// share the run hash tag, and round/producer state is checked in the same Lua.
|
||||
// Leases are token HASHes, not ZSets; future scores include leased members.
|
||||
const supplySampleScript = `
|
||||
local run=redis.call('HMGET',KEYS[1],'runId','status','currentRound','producerDone','syncType')
|
||||
local known=run[1] and run[2] and run[3] and run[4]
|
||||
local eligible=known and (run[2]=='producing' or run[2]=='running') and run[3]=='1' and run[4]=='0' and run[5]~='replay'
|
||||
local blocked=redis.call('HGET',KEYS[1],'blocked:'..ARGV[2])
|
||||
return {known and 1 or 0,(eligible and blocked~='1') and 1 or 0,redis.call('ZCOUNT',KEYS[2],'-inf',ARGV[1]),redis.call('HLEN',KEYS[3]),redis.call('ZCOUNT',KEYS[2],'('..ARGV[1],'+inf')}
|
||||
`
|
||||
|
||||
func (p *Producer) sampleSupply(ctx context.Context, run, key string) ([]int64, error) {
|
||||
meta, err := p.store.RunMetaKey(run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
active, err := p.store.RoundActiveKey(run, 1, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lease, err := p.store.RoundLeaseKey(run, 1, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := p.store.RedisClient().Eval(ctx, supplySampleScript, 3, []string{meta, active, lease}, []any{time.Now().UnixMilli(), key})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.Int64s(), nil
|
||||
}
|
||||
|
||||
func (p *Producer) startObservation(ctx context.Context, run string, keys []string, started time.Time) func() {
|
||||
o := &supplyObservation{started: started, channels: make(map[string]*supplyChannel, len(keys))}
|
||||
for _, key := range keys {
|
||||
o.channels[key] = &supplyChannel{stage: "initializing", changed: started, stageDuration: make(map[string]time.Duration)}
|
||||
}
|
||||
p.observation = o
|
||||
stop, done := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
for _, key := range keys {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
o.mu.Lock()
|
||||
s := o.channels[key]
|
||||
version, ended := s.version, s.ended
|
||||
o.mu.Unlock()
|
||||
if ended {
|
||||
continue
|
||||
}
|
||||
sampleCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
|
||||
values, err := p.sampleSupply(sampleCtx, run, key)
|
||||
cancel()
|
||||
o.mu.Lock()
|
||||
known := err == nil && len(values) == 5 && values[0] == 1 && s.version == version && !s.ended
|
||||
stage, pending := s.stage, s.pending
|
||||
if known {
|
||||
s.validSamples++
|
||||
} else {
|
||||
s.missingSamples++
|
||||
}
|
||||
o.mu.Unlock()
|
||||
// Logging must not hold the mutex used by producer progress.
|
||||
if known {
|
||||
g.Log().Infof(ctx, "event=daily_usage_supply status=sample runId=%s chanKey=%s stage=%s pending=%d firstRoundEligible=%t ready=%d leases=%d futureIncludingLeases=%d consumerEligibility=unknown", run, key, stage, pending, values[1] == 1, values[2], values[3], values[4])
|
||||
} else {
|
||||
g.Log().Infof(ctx, "event=daily_usage_supply status=unknown runId=%s chanKey=%s stage=%s pending=%d consumerEligibility=unknown", run, key, stage, pending)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
close(stop)
|
||||
<-done
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
for _, key := range keys {
|
||||
s := o.channels[key]
|
||||
stageMS := s.stageMilliseconds(time.Now())
|
||||
query, enqueue := int64(-1), int64(-1)
|
||||
if s.firstQuery != nil {
|
||||
query = *s.firstQuery
|
||||
}
|
||||
if s.firstEnqueue != nil {
|
||||
enqueue = *s.firstEnqueue
|
||||
}
|
||||
g.Log().Infof(ctx, "event=daily_usage_supply status=ended runId=%s chanKey=%s firstQueryMs=%d firstEnqueueMs=%d stageMs=%v validSamples=%d missingSamples=%d terminalStage=%s consumerEligibility=unknown", run, key, query, enqueue, stageMS, s.validSamples, s.missingSamples, s.stage)
|
||||
}
|
||||
p.observation = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
func TestLocalSupplyObservationFirstEventsAndStages(t *testing.T) {
|
||||
started := time.Now()
|
||||
o := &supplyObservation{started: started, channels: map[string]*supplyChannel{"a": {stage: "initializing", changed: started, stageDuration: map[string]time.Duration{}}}}
|
||||
o.event("a", "sql_query", 0, false, false, started.Add(10*time.Millisecond))
|
||||
o.event("a", "page_ready", 2, false, false, started.Add(20*time.Millisecond))
|
||||
o.event("a", "redis_write", 1, true, false, started.Add(30*time.Millisecond))
|
||||
o.event("a", "channel_capacity", 1, false, false, started.Add(40*time.Millisecond))
|
||||
o.event("a", "sql_query", 0, false, false, started.Add(50*time.Millisecond))
|
||||
o.event("a", "redis_write", 0, true, false, started.Add(60*time.Millisecond))
|
||||
o.event("a", "upper_bound", 0, false, true, started.Add(70*time.Millisecond))
|
||||
o.event("unknown", "ignored", 0, false, false, started)
|
||||
s := o.channels["a"]
|
||||
if s.firstQuery == nil || *s.firstQuery != 10 || s.firstEnqueue == nil || *s.firstEnqueue != 30 || !s.ended || s.pending != 0 || s.version != 7 || s.stageDuration["channel_capacity"] != 10*time.Millisecond || s.stageDuration["sql_query"] != 20*time.Millisecond {
|
||||
t.Fatalf("observation=%+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplyObservationAccumulatesSubMillisecondStages(t *testing.T) {
|
||||
started := time.Unix(0, 0)
|
||||
o := &supplyObservation{started: started, channels: map[string]*supplyChannel{"a": {stage: "redis_read", changed: started, stageDuration: map[string]time.Duration{}}}}
|
||||
for i := 1; i <= 100000; i++ {
|
||||
stage := "redis_read"
|
||||
if i%2 == 1 {
|
||||
stage = "redis_write"
|
||||
}
|
||||
o.event("a", stage, 0, false, i == 100000, started.Add(time.Duration(i)*500*time.Microsecond))
|
||||
}
|
||||
if got := o.channels["a"].stageMilliseconds(started.Add(time.Hour)); got["redis_read"] != 25000 || got["redis_write"] != 25000 {
|
||||
t.Fatalf("stageMs=%v, want 25000ms per stage", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplyObservationFinalFlushPreservesFraction(t *testing.T) {
|
||||
started := time.Unix(0, 0)
|
||||
o := &supplyObservation{started: started, channels: map[string]*supplyChannel{"a": {stage: "redis_write", changed: started, stageDuration: map[string]time.Duration{}}}}
|
||||
o.event("a", "redis_read", 0, false, false, started.Add(600*time.Microsecond))
|
||||
o.event("a", "redis_write", 0, false, false, started.Add(1200*time.Microsecond))
|
||||
s := o.channels["a"]
|
||||
if got := s.stageMilliseconds(started.Add(1800 * time.Microsecond)); got["redis_write"] != 1 || got["redis_read"] != 0 {
|
||||
t.Fatalf("final stageMs=%v", got)
|
||||
}
|
||||
if s.stageDuration["redis_write"] != 1200*time.Microsecond || s.stageDuration["redis_read"] != 600*time.Microsecond {
|
||||
t.Fatalf("fractional duration lost: %v", s.stageDuration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplySamplesDistinguishReadyLeaseFutureAndClosedRound(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
p := testProducer(t, scannerFor(nil, 0), store, Config{})
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "supply"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
active, _ := store.RoundActiveKey("supply", 1, "a")
|
||||
lease, _ := store.RoundLeaseKey("supply", 1, "a")
|
||||
_, _ = store.RedisClient().Do(ctx, "ZADD", active, time.Now().Add(-time.Minute).UnixMilli(), "ready", time.Now().Add(time.Hour).UnixMilli(), "future")
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", lease, "leased", "token")
|
||||
values, err := p.sampleSupply(ctx, "supply", "a")
|
||||
if err != nil || !reflect.DeepEqual(values, []int64{1, 1, 1, 1, 1}) {
|
||||
t.Fatalf("sample=%v err=%v", values, err)
|
||||
}
|
||||
meta, _ := store.RunMetaKey("supply")
|
||||
for _, args := range [][]any{{"blocked:a", 1}, {"blocked:a", 0, "producerDone", 1}, {"producerDone", 0, "currentRound", 2}, {"currentRound", 1, "syncType", "replay"}} {
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", append([]any{meta}, args...)...)
|
||||
values, err = p.sampleSupply(ctx, "supply", "a")
|
||||
if err != nil || values[0] != 1 || values[1] != 0 {
|
||||
t.Fatalf("ineligible sample=%v err=%v", values, err)
|
||||
}
|
||||
}
|
||||
values, err = p.sampleSupply(ctx, "missing", "a")
|
||||
if err != nil || values[0] != 0 || values[1] != 0 {
|
||||
t.Fatalf("missing sample=%v err=%v", values, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplySamplerTracksWaitingChannelsAndStops(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
p := testProducer(t, scannerFor(nil, 0), store, Config{})
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "sampling"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finish := p.startObservation(ctx, "sampling", []string{"a", "b"}, time.Now())
|
||||
defer finish()
|
||||
o := p.observation
|
||||
p.observe("a", "channel_capacity", 1, false, false)
|
||||
p.observe("b", "sql_slot", 0, false, false)
|
||||
waitUntil(t, ctx, func() bool {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.channels["a"].validSamples > 0 && o.channels["b"].validSamples > 0
|
||||
})
|
||||
p.observe("a", "cancelled", 0, false, true)
|
||||
p.observe("b", "cancelled", 0, false, true)
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.channels["a"].stageDuration["channel_capacity"] < 900*time.Millisecond || o.channels["b"].stageDuration["sql_slot"] < 900*time.Millisecond {
|
||||
t.Fatalf("missing wait attribution: a=%+v b=%+v", o.channels["a"], o.channels["b"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
func newZombieTestProducer(t *testing.T) (*Producer, *zset.Store) {
|
||||
t.Helper()
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(&pageScanner{cards: nil}, store, Config{PageSize: 100, RoundRetryEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if acquired, err := store.AcquireLockOrTakeover(ctx, producer.config.RunLockKey, producer.owner, "pending-check", producer.config.RunLockTTL); err != nil || !acquired {
|
||||
t.Fatalf("acquire producer lock=%t err=%v", acquired, err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = store.ReleaseLock(ctx, producer.config.RunLockKey, producer.owner) })
|
||||
return producer, store
|
||||
}
|
||||
|
||||
// TestHasPendingTasksReclaimsZombieRun guards the "zombie run" regression.
|
||||
//
|
||||
// A producer that dies before MarkProducerDone (for example the container is
|
||||
// replaced mid-run) leaves its run in status=producing with producerDone=0.
|
||||
// completeRunIfIdle and advanceRunRound both require producerDone=1, so such a
|
||||
// run can never be closed, never leaves the active index, and never gets the
|
||||
// retention TTL. Treating it as backpressure rejected every later request
|
||||
// forever; merely warning about it leaked keys and an index entry indefinitely.
|
||||
func TestHasPendingTasksReclaimsZombieRun(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// A genuinely in-flight run (producerDone set, consumer working) must keep
|
||||
// providing backpressure.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "running-run", Status: "running", ProducerDone: true, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := producer.hasPendingTasks(ctx, "new-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !pending {
|
||||
t.Fatal("a running run must still provide backpressure")
|
||||
}
|
||||
if _, err := store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), "running-run"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A zombie run must not block a new request, and must be promoted so the
|
||||
// normal lifecycle can drain and close it.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err = producer.hasPendingTasks(ctx, "new-run-2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending {
|
||||
t.Fatal("a producing run with producerDone=0 must not block new requests (zombie run)")
|
||||
}
|
||||
|
||||
meta, err := store.GetRunMeta(ctx, "zombie-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.ProducerDone {
|
||||
t.Fatal("the zombie run must be promoted (producerDone=1) so it can be closed")
|
||||
}
|
||||
if meta.Status != "running" {
|
||||
t.Fatalf("promoted run status = %q, want running", meta.Status)
|
||||
}
|
||||
// Promotion makes the run closable, which is what applies the retention TTL
|
||||
// and releases the active index entry.
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "zombie-run"); err != nil || !completed {
|
||||
t.Fatalf("promoted run must be closable: completed=%v err=%v", completed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPendingTasksNeverReclaimsReplayRun guards the boundary: a replay target
|
||||
// is also created as "producing", but it is driven by the replay engine under
|
||||
// its own lock, so its producerDone is not the producer's to set.
|
||||
func TestHasPendingTasksNeverReclaimsReplayRun(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{
|
||||
RunID: "replay-run", Status: "producing", ProducerDone: false, CurrentRound: 1,
|
||||
SyncType: "replay", SourceRunID: "source-run", ReplayCycle: 1,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending, err := producer.hasPendingTasks(ctx, "new-run"); err != nil || !pending {
|
||||
t.Fatalf("replay must retain backpressure: pending=%t err=%v", pending, err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "replay-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ProducerDone {
|
||||
t.Fatal("a replay run must not be promoted by the full-sync producer")
|
||||
}
|
||||
if meta.Status != "producing" {
|
||||
t.Fatalf("replay run status = %q, want producing", meta.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPendingTasksDropsStaleIndexEntry covers a second unrecoverable path:
|
||||
// the active index can list a run whose hash is already gone. Returning that as
|
||||
// an error made the request fail permanently (the consumer redelivers it
|
||||
// forever), which is the same "only fixable by hand-cleaning Redis" failure.
|
||||
func TestHasPendingTasksDropsStaleIndexEntry(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Index a run that has no metadata hash.
|
||||
if _, err := store.RedisClient().Do(ctx, "SADD", store.ActiveRunsKey(), "ghost-run"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := producer.hasPendingTasks(ctx, "new-run")
|
||||
if err != nil {
|
||||
t.Fatalf("a stale index entry must not fail the request: %v", err)
|
||||
}
|
||||
if pending {
|
||||
t.Fatal("a stale index entry must not provide backpressure")
|
||||
}
|
||||
// The stale entry is removed so it does not have to be handled again.
|
||||
members, err := store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, member := range members {
|
||||
if member == "ghost-run" {
|
||||
t.Fatal("the stale index entry must be removed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceCreatesRunAfterZombie covers the end-to-end path through Produce:
|
||||
// a zombie run must not stop a new run from being created.
|
||||
func TestProduceCreatesRunAfterZombie(t *testing.T) {
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
producer, err := New(&pageScanner{cards: []Card{
|
||||
{ID: 1, ChanKey: "channel-a", TempKey: "sim_yd_v5", Operator: 1},
|
||||
}}, store, Config{PageSize: 100, RoundRetryEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A zombie run from a crashed producer.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A new request must still produce its own run.
|
||||
if err := producer.Produce(ctx, scopedMessage("fresh-run", "channel-a")); err != nil {
|
||||
t.Fatalf("a zombie run must not stop a new run: %v", err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "fresh-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The producer enqueues into the active ZSet; ActiveCount is the count it
|
||||
// maintains. EnqueuedCount is an accounting field the producer does not set.
|
||||
if !meta.ProducerDone || meta.ActiveCount != 1 {
|
||||
t.Fatalf("fresh run metadata = %+v, want producerDone with 1 active task", meta)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
goredis "github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
type leaseClientFactory func() (queue.RedisClient, func() error, error)
|
||||
|
||||
// A custom client must explicitly supply the same bounded-I/O contract. There
|
||||
// is no fallback to context cancellation on an unknown Redis implementation.
|
||||
type boundedLeaseProvider interface {
|
||||
newBoundedLeaseClient(time.Duration) (queue.RedisClient, func() error, error)
|
||||
}
|
||||
|
||||
func newLeaseClientFactory(client queue.RedisClient, config *gredis.Config, budget time.Duration) (leaseClientFactory, error) {
|
||||
if budget <= 0 {
|
||||
return nil, gerror.New("IMEI lease I/O budget must be positive")
|
||||
}
|
||||
if provider, ok := client.(boundedLeaseProvider); ok {
|
||||
return func() (queue.RedisClient, func() error, error) { return provider.newBoundedLeaseClient(budget) }, nil
|
||||
}
|
||||
getter, ok := client.(interface{ GetAdapter() gredis.Adapter })
|
||||
if !ok || getter.GetAdapter() == nil {
|
||||
return nil, gerror.New("IMEI producer requires a bounded GoFrame Redis adapter")
|
||||
}
|
||||
raw, ok := getter.GetAdapter().Client().(*goredis.Client)
|
||||
if !ok {
|
||||
return nil, gerror.New("IMEI producer does not support Redis Cluster or an unknown Redis adapter")
|
||||
}
|
||||
opts := *raw.Options()
|
||||
if opts.Addr == "FailoverClient" {
|
||||
if config == nil || config.MasterName == "" || config.Cluster || config.SlaveOnly {
|
||||
return nil, gerror.New("IMEI Sentinel lease client requires its master configuration")
|
||||
}
|
||||
// Rebuild Sentinel discovery too: sharing the old dialer would retain
|
||||
// its unbounded Sentinel socket reads even if master I/O were bounded.
|
||||
cfg := *config
|
||||
return func() (queue.RedisClient, func() error, error) {
|
||||
client := goredis.NewFailoverClient(&goredis.FailoverOptions{
|
||||
MasterName: cfg.MasterName, SentinelAddrs: sentinelAddresses(cfg.Address),
|
||||
SentinelUsername: cfg.SentinelUser, SentinelPassword: cfg.SentinelPass,
|
||||
Username: cfg.User, Password: cfg.Pass, DB: cfg.Db, Protocol: cfg.Protocol,
|
||||
TLSConfig: opts.TLSConfig,
|
||||
DialTimeout: budget, ReadTimeout: budget, WriteTimeout: budget, PoolTimeout: budget,
|
||||
ContextTimeoutEnabled: true, MaxRetries: -1, DialerRetries: 1,
|
||||
PoolSize: 1, MaxActiveConns: 1,
|
||||
})
|
||||
return &leaseRedis{client}, client.Close, nil
|
||||
}, nil
|
||||
}
|
||||
// Own a separate pool so a timeout/close never closes a consumer's socket.
|
||||
opts.DialTimeout, opts.ReadTimeout, opts.WriteTimeout, opts.PoolTimeout = budget, budget, budget, budget
|
||||
opts.ContextTimeoutEnabled = true
|
||||
// Maintenance handoffs can relax socket deadlines; leases must never do so.
|
||||
opts.MaintNotificationsConfig = &maintnotifications.Config{Mode: maintnotifications.ModeDisabled}
|
||||
opts.MaxRetries, opts.DialerRetries = -1, 1
|
||||
opts.PoolSize, opts.MaxActiveConns, opts.MinIdleConns, opts.MaxIdleConns = 1, 1, 0, 1
|
||||
opts.Dialer = goredis.NewDialer(&opts)
|
||||
return func() (queue.RedisClient, func() error, error) {
|
||||
client := goredis.NewClient(&opts)
|
||||
return &leaseRedis{client}, client.Close, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sentinelAddresses(address string) []string {
|
||||
var addresses []string
|
||||
for _, addr := range strings.Split(address, ",") {
|
||||
if addr = strings.TrimSpace(addr); addr != "" {
|
||||
addresses = append(addresses, addr)
|
||||
}
|
||||
}
|
||||
return addresses
|
||||
}
|
||||
|
||||
type leaseRedis struct{ client *goredis.Client }
|
||||
|
||||
func (r *leaseRedis) Do(ctx context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
result, err := r.client.Do(ctx, append([]any{command}, args...)...).Result()
|
||||
return gvar.New(result), err
|
||||
}
|
||||
|
||||
func (r *leaseRedis) Eval(ctx context.Context, script string, _ int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
result, err := r.client.Eval(ctx, script, keys, args...).Result()
|
||||
return gvar.New(result), err
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
goredis "github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// This proxy withholds a real Redis request/reply without closing its TCP socket.
|
||||
func readLeaseRESP(r *bufio.Reader) ([]byte, error) {
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch line[0] {
|
||||
case '$':
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(line[1:])))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n >= 0 {
|
||||
payload := make([]byte, n+2)
|
||||
if _, err := io.ReadFull(r, payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
line = append(line, payload...)
|
||||
}
|
||||
case '*', '%', '~', '>', '|':
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(line[1:])))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if line[0] == '%' || line[0] == '|' {
|
||||
n *= 2
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
payload, err := readLeaseRESP(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
line = append(line, payload...)
|
||||
}
|
||||
}
|
||||
return line, nil
|
||||
}
|
||||
|
||||
func TestProduceBoundsRealRedisRenewalIO(t *testing.T) {
|
||||
for _, mode := range []string{"standalone", "sentinel"} {
|
||||
t.Run(mode, func(t *testing.T) { testProduceBoundsRealRedisRenewalIO(t, mode) })
|
||||
}
|
||||
}
|
||||
|
||||
func testProduceBoundsRealRedisRenewalIO(t *testing.T, mode string) {
|
||||
for _, callerCancel := range []bool{false, true} {
|
||||
t.Run(strconv.FormatBool(callerCancel), func(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
real, held, release := leaseRedisProxy(t, backend, "redis.call('PEXPIRE'", true)
|
||||
// Release the gate even on a red-test failure so no worker leaks.
|
||||
defer release()
|
||||
opts := real.GetAdapter().Client().(*goredis.Client).Options()
|
||||
if opts.ReadTimeout != 0 || opts.WriteTimeout != 0 || opts.ContextTimeoutEnabled {
|
||||
t.Fatal("test must start with production-default unbounded socket I/O")
|
||||
}
|
||||
cause := errors.New("snapshot failed independently")
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
defer cancel(nil)
|
||||
scanner := &lockTestScanner{snapshot: func(workCtx context.Context) (int64, error) {
|
||||
select {
|
||||
case <-held:
|
||||
if callerCancel {
|
||||
cancel(cause)
|
||||
<-workCtx.Done()
|
||||
return 0, workCtx.Err()
|
||||
}
|
||||
return 0, cause
|
||||
case <-workCtx.Done():
|
||||
return 0, workCtx.Err()
|
||||
}
|
||||
}}
|
||||
store, err := queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := Config{RunLockTTL: time.Second}
|
||||
if mode == "sentinel" {
|
||||
real, config.LeaseRedisConfig = sentinelLeaseRedis(t, real)
|
||||
store, err = queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
producer, err := New(scanner, store, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "lock-test"}) }()
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("proxy did not hold renewal response")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, cause) {
|
||||
t.Fatalf("lost original cause: %v", err)
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("canceled renewal socket I/O blocked cleanup beyond its lease budget")
|
||||
}
|
||||
if producer.running.Load() != 0 {
|
||||
t.Fatal("running flag was not reset")
|
||||
}
|
||||
lock, err := backend.Do(context.Background(), "GET", producer.config.RunLockKey)
|
||||
if err != nil || !lock.IsNil() {
|
||||
t.Fatalf("lock not released: %v, %v", lock, err)
|
||||
}
|
||||
if opts.ReadTimeout != 0 || opts.WriteTimeout != 0 {
|
||||
t.Fatal("lease client mutated the shared production client")
|
||||
}
|
||||
// A fresh run must start without opening the stalled gate.
|
||||
producer.scanner = cardScanner{}
|
||||
if err := producer.Produce(context.Background(), model.ImeiQueueStartMessage{RunID: "next-run"}); err != nil {
|
||||
t.Fatalf("producer cannot restart: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceFencesDelayedRealRedisWrites(t *testing.T) {
|
||||
for _, mode := range []string{"standalone", "sentinel"} {
|
||||
t.Run(mode, func(t *testing.T) { testProduceFencesDelayedRealRedisWrites(t, mode) })
|
||||
}
|
||||
}
|
||||
|
||||
func testProduceFencesDelayedRealRedisWrites(t *testing.T, mode string) {
|
||||
for _, stage := range []string{"enqueue", "producer_done"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
match := "local taskID=ARGV[i]"
|
||||
cards := []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}}
|
||||
if stage == "producer_done" {
|
||||
match = "'producerCompletedAt'"
|
||||
cards = nil
|
||||
}
|
||||
real, held, release := leaseRedisProxy(t, backend, match, false)
|
||||
defer release()
|
||||
workContexts := make(chan context.Context, 1)
|
||||
scanner := &lockTestScanner{cardScanner: cardScanner{cards: cards}}
|
||||
scanner.snapshot = func(ctx context.Context) (int64, error) { workContexts <- ctx; return int64(len(cards)), nil }
|
||||
store, err := queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := Config{RunLockTTL: time.Second}
|
||||
if mode == "sentinel" {
|
||||
real, config.LeaseRedisConfig = sentinelLeaseRedis(t, real)
|
||||
store, err = queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
producer, err := New(scanner, store, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("proxy did not hold old-owner EVAL")
|
||||
}
|
||||
workCtx := <-workContexts
|
||||
observer, err := queue.New(backend, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ok, err := observer.AcquireLockOrTakeover(context.Background(), producer.config.RunLockKey, "new-owner", "lock-test", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("takeover: %v, %v", ok, err)
|
||||
}
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("real renewal did not detect takeover")
|
||||
}
|
||||
if !errors.Is(context.Cause(workCtx), ErrAlreadyRunning) {
|
||||
t.Fatalf("wrong loss cause: %v", context.Cause(workCtx))
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("wrong result: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not finish")
|
||||
}
|
||||
meta, err := observer.GetRunMeta(context.Background(), "lock-test")
|
||||
if err != nil || meta.ActiveCount != 0 || meta.ProducerDone {
|
||||
t.Fatalf("stale owner changed run state: %+v, %v", meta, err)
|
||||
}
|
||||
channels, err := observer.RunChannels(context.Background(), "lock-test")
|
||||
if err != nil || len(channels) != 0 {
|
||||
t.Fatalf("stale owner indexed channel: %v, %v", channels, err)
|
||||
}
|
||||
ok, err = observer.RenewLock(context.Background(), producer.config.RunLockKey, "new-owner", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("old cleanup removed new lease: %v, %v", ok, err)
|
||||
}
|
||||
added, err := observer.EnqueueRunBatch(context.Background(), "lock-test", "ch", []queue.Task{{TaskID: "next", RunID: "lock-test", ChanKey: "ch", CardID: 2}}, time.Now())
|
||||
if err != nil || added != 1 {
|
||||
t.Fatalf("new work rejected: added=%d, %v", added, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func leaseRedisProxy(t *testing.T, upstream *gredis.Redis, match string, holdReply bool) (*gredis.Redis, <-chan struct{}, func()) {
|
||||
t.Helper()
|
||||
raw := upstream.GetAdapter().Client().(*goredis.Client)
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, gate := make(chan struct{}), make(chan struct{})
|
||||
var heldOnce, releaseOnce sync.Once
|
||||
release := func() { releaseOnce.Do(func() { close(gate) }) }
|
||||
var connections sync.Map
|
||||
go func() {
|
||||
for {
|
||||
down, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
connections.Store(down, struct{}{})
|
||||
go func() {
|
||||
defer down.Close()
|
||||
defer connections.Delete(down)
|
||||
up, err := net.Dial("tcp", raw.Options().Addr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer up.Close()
|
||||
dr, ur := bufio.NewReader(down), bufio.NewReader(up)
|
||||
for {
|
||||
request, err := readLeaseRESP(dr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
holding := strings.Contains(string(request), match)
|
||||
if holding && !holdReply {
|
||||
heldOnce.Do(func() { close(held) })
|
||||
<-gate
|
||||
}
|
||||
if _, err := up.Write(request); err != nil {
|
||||
return
|
||||
}
|
||||
response, err := readLeaseRESP(ur)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if holding && holdReply {
|
||||
heldOnce.Do(func() { close(held) })
|
||||
<-gate
|
||||
}
|
||||
if _, err := down.Write(response); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
release()
|
||||
_ = listener.Close()
|
||||
connections.Range(func(key, _ any) bool { _ = key.(net.Conn).Close(); return true })
|
||||
})
|
||||
client, err := gredis.New(&gredis.Config{Address: listener.Addr().String(), DialTimeout: 100 * time.Millisecond, Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = client.Close(context.Background()) })
|
||||
if _, err := client.Do(context.Background(), "PING"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := client.GetAdapter().Client().(*goredis.Client).Options()
|
||||
t.Logf("real driver options: readTimeout=%s writeTimeout=%s contextTimeoutEnabled=%v", opts.ReadTimeout, opts.WriteTimeout, opts.ContextTimeoutEnabled)
|
||||
return client, held, release
|
||||
}
|
||||
|
||||
// A real disposable Sentinel discovers the proxy as its master; neither client
|
||||
// configuration nor server state comes from the application's Redis settings.
|
||||
func sentinelLeaseRedis(t *testing.T, master *gredis.Redis) (*gredis.Redis, *gredis.Config) {
|
||||
t.Helper()
|
||||
binary := os.Getenv("REDIS_SERVER_BINARY")
|
||||
if binary == "" {
|
||||
var err error
|
||||
binary, err = exec.LookPath("redis-server")
|
||||
if err != nil {
|
||||
t.Skip("Sentinel tests require redis-server")
|
||||
}
|
||||
}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address := listener.Addr().String()
|
||||
_, port, _ := net.SplitHostPort(address)
|
||||
_ = listener.Close()
|
||||
masterAddress := master.GetAdapter().Client().(*goredis.Client).Options().Addr
|
||||
host, masterPort, err := net.SplitHostPort(masterAddress)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "sentinel.conf")
|
||||
contents := "bind 127.0.0.1\nport " + port + "\nsentinel monitor lease-test " + host + " " + masterPort + " 1\nsentinel down-after-milliseconds lease-test 60000\n"
|
||||
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
command := exec.Command(binary, filepath.Base(path), "--sentinel")
|
||||
command.Dir = dir
|
||||
logPath := filepath.Join(dir, "sentinel.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
command.Stdout, command.Stderr = logFile, logFile
|
||||
if err := command.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = command.Process.Kill(); _ = command.Wait() })
|
||||
probe, err := gredis.New(&gredis.Config{Address: address, ReadTimeout: 100 * time.Millisecond, WriteTimeout: 100 * time.Millisecond, DialTimeout: 100 * time.Millisecond, Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = probe.Close(context.Background()) })
|
||||
for deadline := time.Now().Add(5 * time.Second); ; {
|
||||
if _, err = probe.Do(context.Background(), "SENTINEL", "get-master-addr-by-name", "lease-test"); err == nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
log, _ := os.ReadFile(logPath)
|
||||
t.Fatalf("isolated Sentinel did not start: %v; %s", err, log)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
config := &gredis.Config{Address: address, MasterName: "lease-test", DialTimeout: 100 * time.Millisecond, Protocol: 2}
|
||||
client, err := gredis.New(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = client.Close(context.Background()) })
|
||||
return client, config
|
||||
}
|
||||
|
||||
func TestLeaseClientBoundsSentinelDiscoveryIO(t *testing.T) {
|
||||
master := testutil.Redis(t)
|
||||
_, config := sentinelLeaseRedis(t, master)
|
||||
sentinel, err := gredis.New(&gredis.Config{Address: config.Address, ReadTimeout: time.Second, WriteTimeout: time.Second, Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sentinel.Close(context.Background()) })
|
||||
proxy, held, release := leaseRedisProxy(t, sentinel, "get-master-addr-by-name", true)
|
||||
defer release()
|
||||
config.Address = proxy.GetAdapter().Client().(*goredis.Client).Options().Addr
|
||||
shared, err := gredis.New(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = shared.Close(context.Background()) })
|
||||
factory, err := newLeaseClientFactory(shared, config, 100*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, closeClient, err := factory()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeClient()
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := client.Do(context.Background(), "PING"); result <- err }()
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Sentinel discovery request was not held")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if err == nil {
|
||||
t.Fatal("stalled Sentinel discovery unexpectedly succeeded")
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("Sentinel discovery has no socket I/O upper bound")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaseClientRejectsClusterAndUnknownAdapter(t *testing.T) {
|
||||
cluster, err := gredis.New(&gredis.Config{Address: "127.0.0.1:1", Cluster: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cluster.Close(context.Background())
|
||||
store, err := queue.New(cluster, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := New(cardScanner{}, store, Config{}); err == nil || !strings.Contains(err.Error(), "Cluster") {
|
||||
t.Fatalf("Cluster must be rejected at initialization before any write: %v", err)
|
||||
}
|
||||
if _, err := newLeaseClientFactory(&leaseRedis{}, nil, time.Second); err == nil {
|
||||
t.Fatal("unknown adapter must not fall back to unbounded context-only I/O")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceBoundsReleaseIO(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
real, held, release := leaseRedisProxy(t, backend, "redis.call('DEL', KEYS[1])", true)
|
||||
defer release()
|
||||
store, err := queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(cardScanner{}, store, Config{RunLockTTL: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("release response was not held")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("release socket I/O blocked cleanup")
|
||||
}
|
||||
if producer.running.Load() != 0 {
|
||||
t.Fatal("cleanup did not reset running")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaseClientBoundsSocketWriteIO(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
upstream := backend.GetAdapter().Client().(*goredis.Client).Options().Addr
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stopReading atomic.Bool
|
||||
held, gate := make(chan struct{}), make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
release := func() { releaseOnce.Do(func() { close(gate) }) }
|
||||
defer release()
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
go func() {
|
||||
down, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer down.Close()
|
||||
up, err := net.Dial("tcp", upstream)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer up.Close()
|
||||
dr, ur := bufio.NewReader(down), bufio.NewReader(up)
|
||||
for {
|
||||
// Read only the array header once armed: the large EVAL payload
|
||||
// cannot enter the upstream or fit in TCP buffers, so Write blocks.
|
||||
header, err := dr.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if stopReading.Load() {
|
||||
close(held)
|
||||
<-gate
|
||||
return
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(header[1:])))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
part, err := readLeaseRESP(dr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
header = append(header, part...)
|
||||
}
|
||||
if _, err := up.Write(header); err != nil {
|
||||
return
|
||||
}
|
||||
response, err := readLeaseRESP(ur)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := down.Write(response); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
shared, err := gredis.New(&gredis.Config{Address: listener.Addr().String(), Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer shared.Close(context.Background())
|
||||
factory, err := newLeaseClientFactory(shared, nil, 100*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, closeClient, err := factory()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeClient()
|
||||
if _, err := client.Do(context.Background(), "PING"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := client.(*leaseRedis).client.Options()
|
||||
if opts.ReadTimeout != 100*time.Millisecond || opts.WriteTimeout != 100*time.Millisecond || !opts.ContextTimeoutEnabled || opts.MaxRetries != 0 || opts.DialerRetries != 1 {
|
||||
t.Fatalf("lease client has unsafe timeout/retry options")
|
||||
}
|
||||
stopReading.Store(true)
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.Eval(context.Background(), "return 1 --"+strings.Repeat("x", 32<<20), 0, nil, nil)
|
||||
result <- err
|
||||
}()
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("write proxy did not stop reading")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
var timeout net.Error
|
||||
if !errors.As(err, &timeout) || !timeout.Timeout() || !strings.Contains(err.Error(), "write") {
|
||||
t.Fatalf("expected real socket write timeout, got %v", err)
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("socket write has no upper bound")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type lockTestRedis struct {
|
||||
queue.RedisClient
|
||||
do func(context.Context, string, []any) (*gvar.Var, error)
|
||||
eval func(context.Context, string, int64, []string, []any) (*gvar.Var, error)
|
||||
}
|
||||
|
||||
func (r *lockTestRedis) newBoundedLeaseClient(budget time.Duration) (queue.RedisClient, func() error, error) {
|
||||
factory, err := newLeaseClientFactory(r.RedisClient, nil, budget)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client, closeClient, err := factory()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return &lockTestRedis{RedisClient: client,
|
||||
do: func(ctx context.Context, command string, args []any) (*gvar.Var, error) {
|
||||
if r.do != nil {
|
||||
return r.do(ctx, command, args)
|
||||
}
|
||||
return client.Do(ctx, command, args...)
|
||||
},
|
||||
eval: func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if r.eval != nil {
|
||||
return r.eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
return client.Eval(ctx, script, numKeys, keys, args)
|
||||
},
|
||||
}, closeClient, nil
|
||||
}
|
||||
|
||||
func (r *lockTestRedis) Do(ctx context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
if r.do != nil {
|
||||
return r.do(ctx, command, args)
|
||||
}
|
||||
return r.RedisClient.Do(ctx, command, args...)
|
||||
}
|
||||
|
||||
func (r *lockTestRedis) Eval(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if r.eval != nil {
|
||||
return r.eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
return r.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
|
||||
type lockTestScanner struct {
|
||||
cardScanner
|
||||
scanCalls atomic.Int32
|
||||
snapshot func(context.Context) (int64, error)
|
||||
scan func(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
func (s *lockTestScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
if s.snapshot != nil {
|
||||
return s.snapshot(ctx)
|
||||
}
|
||||
return s.cardScanner.SnapshotMaxID(ctx)
|
||||
}
|
||||
|
||||
func (s *lockTestScanner) Scan(ctx context.Context, last, max int64, limit int) ([]Card, error) {
|
||||
s.scanCalls.Add(1)
|
||||
if s.scan != nil {
|
||||
return s.scan(ctx, last, max, limit)
|
||||
}
|
||||
return s.cardScanner.Scan(ctx, last, max, limit)
|
||||
}
|
||||
|
||||
func startLockTestProducer(t *testing.T, producer *Producer) <-chan error {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 1)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
result <- producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "lock-test"})
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("producer goroutine did not stop after cleanup cancellation")
|
||||
}
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func TestProduceStopsOnLockLossWhileWaitingForCapacity(t *testing.T) {
|
||||
backendErr := errors.New("renewal backend unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{name: "taken_over", err: ErrAlreadyRunning},
|
||||
{name: "redis_error", err: backendErr},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
client = &lockTestRedis{RedisClient: testutil.Redis(t)}
|
||||
waiting = make(chan struct{})
|
||||
failRenew = make(chan struct{})
|
||||
waitingOnce sync.Once
|
||||
capacityRead atomic.Bool
|
||||
scanner = &lockTestScanner{cardScanner: cardScanner{cards: []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
{ID: 3, ICCID: "c", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
}}}
|
||||
)
|
||||
store, err := queue.New(client, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(scanner, store, Config{
|
||||
PageSize: 2, GlobalHighWaterMark: 1,
|
||||
RunLockTTL: 300 * time.Millisecond, PollInterval: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.do = func(ctx context.Context, command string, args []any) (*gvar.Var, error) {
|
||||
result, err := client.RedisClient.Do(ctx, command, args...)
|
||||
if err == nil && command == "SMEMBERS" && args[0] == store.ActiveRunsKey() {
|
||||
capacityRead.Store(true)
|
||||
}
|
||||
if err == nil && command == "HGETALL" && capacityRead.Load() && result.MapStrStr()["activeCount"] == "2" {
|
||||
waitingOnce.Do(func() { close(waiting) })
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
client.eval = func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if strings.Contains(script, "redis.call('PEXPIRE'") {
|
||||
select {
|
||||
case <-failRenew:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if test.err == backendErr {
|
||||
return nil, backendErr
|
||||
}
|
||||
}
|
||||
return client.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
select {
|
||||
case <-waiting:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not reach backpressure")
|
||||
}
|
||||
// The long poll cannot finish naturally; renewal failure must wake it.
|
||||
select {
|
||||
case err := <-result:
|
||||
t.Fatalf("producer exited before lock loss: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if test.err == ErrAlreadyRunning {
|
||||
ok, err := store.AcquireLockOrTakeover(context.Background(), producer.config.RunLockKey, "new-owner", "lock-test", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("take over lock: acquired=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
close(failRenew)
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, test.err) {
|
||||
t.Fatalf("producer error = %v, want renewal cause %v", err, test.err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("lock loss did not interrupt backpressure before the next capacity poll")
|
||||
}
|
||||
if calls := scanner.scanCalls.Load(); calls != 1 {
|
||||
t.Fatalf("scan calls = %d, want only the first page", calls)
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "lock-test")
|
||||
if err != nil || meta.ProducerDone || meta.ActiveCount != 2 {
|
||||
t.Fatalf("run must remain incomplete without additional tasks: meta=%+v err=%v", meta, err)
|
||||
}
|
||||
if producer.running.Load() != 0 {
|
||||
t.Fatal("producer running flag was not reset")
|
||||
}
|
||||
if test.err == ErrAlreadyRunning {
|
||||
ok, err := store.RenewLock(context.Background(), producer.config.RunLockKey, "new-owner", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("old producer released the new owner's lock: renewed=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceCancelsLockedOperationsWithRenewalCause(t *testing.T) {
|
||||
for _, stage := range []string{"run_init", "snapshot", "scan", "enqueue", "producer_done"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
var (
|
||||
client = &lockTestRedis{RedisClient: testutil.Redis(t)}
|
||||
entered = make(chan context.Context, 1)
|
||||
failRenew = make(chan struct{})
|
||||
cause = errors.New("renewal failed during " + stage)
|
||||
scanner = &lockTestScanner{cardScanner: cardScanner{cards: []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
}}}
|
||||
)
|
||||
block := func(ctx context.Context) error {
|
||||
entered <- ctx
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
if stage == "snapshot" {
|
||||
scanner.snapshot = func(ctx context.Context) (int64, error) { return 0, block(ctx) }
|
||||
}
|
||||
if stage == "scan" {
|
||||
scanner.scan = func(ctx context.Context, _, _ int64, _ int) ([]Card, error) { return nil, block(ctx) }
|
||||
}
|
||||
store, err := queue.New(client, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(scanner, store, Config{RunLockTTL: 300 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.eval = func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if strings.Contains(script, "redis.call('PEXPIRE'") {
|
||||
select {
|
||||
case <-failRenew:
|
||||
return nil, cause
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if stage == "run_init" && strings.Contains(script, "'createdAt', ARGV[12]") ||
|
||||
stage == "enqueue" && strings.Contains(script, "local taskID=ARGV[i]") ||
|
||||
stage == "producer_done" && strings.Contains(script, "'producerCompletedAt'") {
|
||||
return nil, block(ctx)
|
||||
}
|
||||
return client.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
var workCtx context.Context
|
||||
select {
|
||||
case workCtx = <-entered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not enter " + stage)
|
||||
}
|
||||
close(failRenew)
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, cause) || !errors.Is(context.Cause(workCtx), cause) {
|
||||
t.Fatalf("renewal cause lost: result=%v context cause=%v", err, context.Cause(workCtx))
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("lock loss did not interrupt " + stage)
|
||||
}
|
||||
if stage == "run_init" {
|
||||
if scanner.scanCalls.Load() != 0 {
|
||||
t.Fatal("producer scanned after run initialization failed")
|
||||
}
|
||||
return
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "lock-test")
|
||||
wantActive := int64(0)
|
||||
if stage == "producer_done" {
|
||||
wantActive = 1
|
||||
}
|
||||
if err != nil || meta.ProducerDone || meta.ActiveCount != wantActive {
|
||||
t.Fatalf("unexpected work after lock loss: meta=%+v err=%v", meta, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceJoinsRenewalBeforeReleasingLock(t *testing.T) {
|
||||
scanErr := errors.New("snapshot failed")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{name: "success"},
|
||||
{name: "scanner_error", err: scanErr},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
client = &lockTestRedis{RedisClient: testutil.Redis(t)}
|
||||
renewEntered = make(chan struct{})
|
||||
renewCanceled = make(chan struct{})
|
||||
allowRenewReturn = make(chan struct{})
|
||||
resumeSnapshot = make(chan struct{})
|
||||
released = make(chan struct{}, 1)
|
||||
allowReturnOnce sync.Once
|
||||
scanner = &lockTestScanner{}
|
||||
)
|
||||
scanner.snapshot = func(ctx context.Context) (int64, error) {
|
||||
select {
|
||||
case <-resumeSnapshot:
|
||||
return 0, test.err
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
}
|
||||
}
|
||||
store, err := queue.New(client, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(scanner, store, Config{RunLockTTL: 300 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.eval = func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if strings.Contains(script, "redis.call('PEXPIRE'") {
|
||||
close(renewEntered)
|
||||
<-ctx.Done()
|
||||
close(renewCanceled)
|
||||
<-allowRenewReturn
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if len(keys) == 1 && keys[0] == producer.config.RunLockKey && len(args) == 1 {
|
||||
released <- struct{}{}
|
||||
}
|
||||
return client.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
t.Cleanup(func() { allowReturnOnce.Do(func() { close(allowRenewReturn) }) })
|
||||
select {
|
||||
case <-renewEntered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("renewal did not start")
|
||||
}
|
||||
close(resumeSnapshot)
|
||||
select {
|
||||
case <-renewCanceled:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not cancel renewal on exit")
|
||||
}
|
||||
// Keep the in-flight renewal alive to verify cleanup actually joins it.
|
||||
select {
|
||||
case err := <-result:
|
||||
t.Fatalf("producer returned before renewal exited: %v", err)
|
||||
case <-released:
|
||||
t.Fatal("producer released the lock before renewal exited")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
allowReturnOnce.Do(func() { close(allowRenewReturn) })
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, test.err) {
|
||||
t.Fatalf("cleanup changed the production error: got %v, want %v", err, test.err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not finish after renewal exited")
|
||||
}
|
||||
select {
|
||||
case <-released:
|
||||
default:
|
||||
t.Fatal("producer did not release the lock after joining renewal")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducePreservesCallerCancellationCause(t *testing.T) {
|
||||
producer, store := newProducer(t, nil)
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
defer cancel(nil)
|
||||
cause := errors.New("caller stopped the run")
|
||||
producer.scanner = &lockTestScanner{snapshot: func(ctx context.Context) (int64, error) {
|
||||
cancel(cause)
|
||||
<-ctx.Done()
|
||||
return 0, ctx.Err()
|
||||
}}
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "caller-canceled"}); !errors.Is(err, cause) {
|
||||
t.Fatalf("caller cancellation cause lost: %v", err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "caller-canceled")
|
||||
if err != nil || meta.ProducerDone || producer.running.Load() != 0 {
|
||||
t.Fatalf("canceled producer did not clean up: meta=%+v err=%v running=%d", meta, err, producer.running.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForCapacityHonorsCancellationWithoutWatermark(t *testing.T) {
|
||||
producer := &Producer{}
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
cause := errors.New("lock lost")
|
||||
cancel(cause)
|
||||
if err := producer.waitForCapacity(ctx); !errors.Is(err, cause) {
|
||||
t.Fatalf("disabled watermark ignored cancellation: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// FullSyncOperation is recorded on the run and used by SyncKindForOperation to
|
||||
// classify the run for the admin monitor.
|
||||
const FullSyncOperation = "imei_full_sync"
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// productionStats mirrors the status producer so every full-sync path reports a
|
||||
// single comparable summary line.
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
skippedNoImei int64
|
||||
scopeChannels int
|
||||
channels map[string]struct{}
|
||||
}
|
||||
|
||||
func newProductionStats(scopeChannels int) productionStats {
|
||||
return productionStats{channels: make(map[string]struct{}), scopeChannels: scopeChannels}
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=imei_queue_producer status=%s runId=%s snapshotMaxId=%d pages=%d scanned=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d skippedNoImei=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, maxID, s.pages, s.scanned, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.skippedNoImei, len(s.channels), s.scopeChannels, time.Since(startedAt).Milliseconds())
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
switch status {
|
||||
case "failed":
|
||||
g.Log().Error(ctx, message)
|
||||
case "skipped":
|
||||
g.Log().Warning(ctx, message)
|
||||
default:
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
PageSize int
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
PollInterval time.Duration
|
||||
GlobalHighWaterMark int64
|
||||
// LeaseRedisConfig is required only for Sentinel discovery. Standalone
|
||||
// settings come from the actual adapter; Cluster is rejected in New.
|
||||
LeaseRedisConfig *gredis.Config
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 1000
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "imei:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = time.Second
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
Operator int
|
||||
SyncEnabled bool
|
||||
Imei string
|
||||
}
|
||||
|
||||
type Scanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
Scan(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
Operator int `orm:"operator"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
}
|
||||
|
||||
// DatabaseScanner uses keyset pagination so a large IMEI run does not hold the
|
||||
// entire card table in memory and does not degrade through OFFSET scans.
|
||||
type DatabaseScanner struct {
|
||||
metadata map[string]channelMetadata
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id).
|
||||
OrderDesc(columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read imei snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) Scan(ctx context.Context, lastID, maxID int64, pageSize int) ([]Card, error) {
|
||||
if maxID <= lastID {
|
||||
return nil, nil
|
||||
}
|
||||
if s.metadata == nil {
|
||||
var rows []channelMetadata
|
||||
if err := g.DB().Model("t_sim_channels").Ctx(ctx).Fields("chan_key,temp_key,sync_enabled,operator").Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan imei channel metadata: %w", err)
|
||||
}
|
||||
s.metadata = make(map[string]channelMetadata, len(rows))
|
||||
for _, row := range rows {
|
||||
s.metadata[row.ChanKey] = row
|
||||
}
|
||||
}
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
var rows []entity.TSimCardInfo
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id, columns.Iccid, columns.Msisdn, columns.Imsi, columns.ChanKey, columns.Imei).
|
||||
WhereGT(columns.Id, lastID).
|
||||
WhereLTE(columns.Id, maxID).
|
||||
OrderAsc(columns.Id).
|
||||
Limit(pageSize).
|
||||
Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan imei cards: %w", err)
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.Id <= 0 || row.Iccid == "" || row.ChanKey == "" {
|
||||
// Keep the page cursor moving even when a row cannot become a
|
||||
// runnable task. The producer advances by the highest returned ID.
|
||||
if row.Id > 0 {
|
||||
cards = append(cards, Card{ID: row.Id})
|
||||
}
|
||||
continue
|
||||
}
|
||||
metadata := s.metadata[row.ChanKey]
|
||||
cards = append(cards, Card{ID: row.Id, ICCID: row.Iccid, MSISDN: row.Msisdn, IMSI: row.Imsi, ChanKey: row.ChanKey, TempKey: metadata.TempKey, Operator: metadata.Operator, SyncEnabled: metadata.SyncEnabled, Imei: row.Imei})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner Scanner
|
||||
store *queue.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
leaseFactory leaseClientFactory
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("imei producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected.
|
||||
var ErrRunFinished = fmt.Errorf("imei run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
func New(scanner Scanner, store *queue.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("imei producer dependencies are incomplete")
|
||||
}
|
||||
config = config.withDefaults()
|
||||
factory, err := newLeaseClientFactory(store.RedisClient(), config.LeaseRedisConfig, leaseIOBudget(config.RunLockTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config, owner: newOwner(), leaseFactory: factory}, nil
|
||||
}
|
||||
|
||||
func leaseIOBudget(ttl time.Duration) time.Duration {
|
||||
return min(time.Second, ttl/6)
|
||||
}
|
||||
|
||||
func (p *Producer) Produce(ctx context.Context, message model.ImeiQueueStartMessage) (err error) {
|
||||
startedAt := time.Now()
|
||||
if message.RunID == "" {
|
||||
return fmt.Errorf("imei sync runId is empty")
|
||||
}
|
||||
operation := message.Operation
|
||||
if operation == "" {
|
||||
operation = FullSyncOperation
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
failedStats := newProductionStats(0)
|
||||
failedStats.log(ctx, "failed", message.RunID, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats := newProductionStats(scope.ChannelCount())
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
leaseClient, closeLeaseClient, err := p.leaseFactory()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeLeaseClient()
|
||||
leaseStore, err := queue.New(leaseClient, p.store.Prefix())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
// Tokens are per lease, not per Producer: a delayed command from an earlier
|
||||
// attempt must never match a later lease on this same Producer instance.
|
||||
p.owner = newOwner()
|
||||
acquireCtx, cancelAcquire := context.WithTimeout(ctx, leaseIOBudget(p.config.RunLockTTL))
|
||||
acquired, err := leaseStore.AcquireLockOrTakeover(acquireCtx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
cancelAcquire()
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), leaseIOBudget(p.config.RunLockTTL))
|
||||
defer cancel()
|
||||
_, _ = leaseStore.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
workStore, err := p.store.WithIMEIProducerLease(p.config.RunLockKey, p.owner, message.RunID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockCtx, cancelLock := context.WithCancelCause(ctx)
|
||||
// Stop renewal separately so normal cleanup does not become a lock-loss cause.
|
||||
renewCtx, stopRenew := context.WithCancel(lockCtx)
|
||||
renewDone := make(chan struct{})
|
||||
var (
|
||||
maxID int64
|
||||
stage = "run_init"
|
||||
)
|
||||
go func() {
|
||||
defer close(renewDone)
|
||||
p.renewLock(renewCtx, cancelLock, leaseStore)
|
||||
}()
|
||||
defer func() {
|
||||
stopRenew()
|
||||
<-renewDone
|
||||
// Join before reading the cause and releasing the lease: a failed renewal
|
||||
// must not be masked by a scanner/Redis context-cancellation error.
|
||||
if cause := context.Cause(lockCtx); cause != nil {
|
||||
err = cause
|
||||
}
|
||||
if errors.Is(err, queue.ErrProducerLeaseLost) {
|
||||
err = ErrAlreadyRunning
|
||||
}
|
||||
cancelLock(nil)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
if err = workStore.CreateRun(lockCtx, queue.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: operation,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "snapshot"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
maxID, err = p.scanner.SnapshotMaxID(lockCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=imei_queue_producer status=started runId=%s snapshotMaxId=%d scopeChannels=%d", message.RunID, maxID, scope.ChannelCount())
|
||||
lastID := int64(0)
|
||||
for lastID < maxID {
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "scan"
|
||||
cards, scanErr := p.scanner.Scan(lockCtx, lastID, maxID, p.config.PageSize)
|
||||
if scanErr != nil {
|
||||
return scanErr
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
stats.pages++
|
||||
stats.scanned += int64(len(cards))
|
||||
groups := make(map[string][]queue.Task)
|
||||
for _, card := range cards {
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if card.ID > lastID {
|
||||
lastID = card.ID
|
||||
}
|
||||
if card.ChanKey == "" {
|
||||
stats.skippedEmptyChan++
|
||||
continue
|
||||
}
|
||||
// Scope is applied before the template and sync_enabled checks so a
|
||||
// partial run only reports on the channels it was asked for.
|
||||
if !scope.Allows(card.ChanKey) {
|
||||
stats.skippedOutOfScope++
|
||||
continue
|
||||
}
|
||||
if card.TempKey == "" {
|
||||
stats.skippedNoTemplate++
|
||||
continue
|
||||
}
|
||||
if card.SyncEnabled {
|
||||
// sync_enabled=1(否/不参与) → skip
|
||||
stats.skippedSyncDisabled++
|
||||
continue
|
||||
}
|
||||
// skippedNoImei counts cards that already have a value. They are
|
||||
// still enqueued (this is a full re-poll, so a device swap is
|
||||
// detected), but the count makes the "how much of this pass is
|
||||
// actually filling gaps" question answerable from one log line.
|
||||
if strings.TrimSpace(card.Imei) != "" {
|
||||
stats.skippedNoImei++
|
||||
}
|
||||
stats.channels[card.ChanKey] = struct{}{}
|
||||
task := queue.Task{TaskID: queue.StableTaskID(message.RunID, operation, card.ID), RunID: message.RunID, Operation: operation, ChanKey: card.ChanKey, TempKey: card.TempKey, Operator: card.Operator, CardID: card.ID, ICCID: card.ICCID, MSISDN: card.MSISDN, IMSI: card.IMSI}
|
||||
groups[card.ChanKey] = append(groups[card.ChanKey], task)
|
||||
}
|
||||
for chanKey, tasks := range groups {
|
||||
stage = "enqueue"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
added, enqueueErr := workStore.EnqueueRunBatch(lockCtx, message.RunID, chanKey, tasks, time.Now())
|
||||
if enqueueErr != nil {
|
||||
return fmt.Errorf("enqueue imei channel %s: %w", chanKey, enqueueErr)
|
||||
}
|
||||
stats.enqueued += int64(added)
|
||||
}
|
||||
stage = "capacity"
|
||||
if err = p.waitForCapacity(lockCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
stage = "producer_done"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return workStore.MarkProducerDone(lockCtx, message.RunID)
|
||||
}
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelCauseFunc, leaseStore *queue.Store) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
renewCtx, stop := context.WithTimeout(ctx, leaseIOBudget(p.config.RunLockTTL))
|
||||
ok, err := leaseStore.RenewLock(renewCtx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err == nil && !ok {
|
||||
err = ErrAlreadyRunning
|
||||
}
|
||||
if err != nil {
|
||||
cancel(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) waitForCapacity(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
if p.config.GlobalHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, runID := range runs {
|
||||
meta, sizeErr := p.store.GetRunMeta(ctx, runID)
|
||||
if sizeErr != nil {
|
||||
return sizeErr
|
||||
}
|
||||
// ActiveCount includes leased tasks; do not count InflightCount twice.
|
||||
total += meta.ActiveCount + meta.NextRoundCount
|
||||
}
|
||||
if total <= p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return context.Cause(ctx)
|
||||
case <-time.After(p.config.PollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var b [12]byte
|
||||
if _, err := rand.Read(b[:]); err == nil {
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
return fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type cardScanner struct{ cards []Card }
|
||||
|
||||
func (s cardScanner) SnapshotMaxID(context.Context) (int64, error) { return int64(len(s.cards)), nil }
|
||||
func (s cardScanner) Scan(_ context.Context, last, max int64, limit int) ([]Card, error) {
|
||||
end := last + int64(limit)
|
||||
if end > max {
|
||||
end = max
|
||||
}
|
||||
return s.cards[last:end], nil
|
||||
}
|
||||
|
||||
func newProducer(t *testing.T, cards []Card, config ...Config) (*Producer, *queue.Store) {
|
||||
t.Helper()
|
||||
redis := testutil.Redis(t)
|
||||
store, err := queue.New(redis, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := Config{PageSize: 1}
|
||||
if len(config) > 0 {
|
||||
cfg = config[0]
|
||||
}
|
||||
producer, err := New(cardScanner{cards: cards}, store, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return producer, store
|
||||
}
|
||||
|
||||
// TestProduceEnqueuesEligibleCards 覆盖 §7 T1：全量入队，
|
||||
// 跳过无通道、无模板、不参与轮询的卡。
|
||||
func TestProduceEnqueuesEligibleCards(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}, // 合格
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5", SyncEnabled: true}, // sync_enabled=1 跳过
|
||||
{ID: 3, ICCID: "c", ChanKey: "ch"}, // 无模板
|
||||
{ID: 4, ICCID: "d", ChanKey: "ch", TempKey: "sim_yd_v5"}, // 合格
|
||||
})
|
||||
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "full"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "full")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.ProducerDone {
|
||||
t.Fatal("producerDone 应为 true")
|
||||
}
|
||||
// 用 ActiveCount 而非 EnqueuedCount：后者是 RunMeta 上的既有死字段
|
||||
// （只有 RoundStats.EnqueuedCount 会被真实写入），状态链路同样如此。
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("activeCount = %d, want 2", meta.ActiveCount)
|
||||
}
|
||||
claims, err := store.ClaimRun(ctx, "full", "ch", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(claims) != 2 {
|
||||
t.Fatalf("claims = %d, want 2", len(claims))
|
||||
}
|
||||
for _, claim := range claims {
|
||||
if claim.Task.CardID != 1 && claim.Task.CardID != 4 {
|
||||
t.Fatalf("不应入队 card %d", claim.Task.CardID)
|
||||
}
|
||||
if claim.Task.Operation != FullSyncOperation {
|
||||
t.Fatalf("operation = %q, want %q", claim.Task.Operation, FullSyncOperation)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceRePollIncludesCardsThatAlreadyHaveImei 是用户决策的直接回归测试：
|
||||
// 全量重查（而非只查空值卡），已有 IMEI 的卡也必须入队，否则无法发现换机。
|
||||
func TestProduceRePollIncludesCardsThatAlreadyHaveImei(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5", Imei: "865466070817587"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5", Imei: ""},
|
||||
})
|
||||
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "repoll"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "repoll")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("全量重查应入队全部合格卡（含已有值），activeCount = %d, want 2", meta.ActiveCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceAppliesChannelScope 覆盖 §7 T2：按通道裁剪。
|
||||
func TestProduceAppliesChannelScope(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "173", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "189", TempKey: "sim_yd_v5"},
|
||||
{ID: 3, ICCID: "c", ChanKey: "173", TempKey: "sim_yd_v5"},
|
||||
})
|
||||
|
||||
scope := &model.SyncScope{Channels: []string{"173"}, TriggerSource: model.TriggerSourceManual}
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "scoped", Scope: scope}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("activeCount = %d, want 2（只入 173 通道）", meta.ActiveCount)
|
||||
}
|
||||
if meta.ScopeChannels != 1 {
|
||||
t.Fatalf("scopeChannels = %d, want 1", meta.ScopeChannels)
|
||||
}
|
||||
if meta.ScopeDigest == "" {
|
||||
t.Fatal("scopeDigest 不应为空")
|
||||
}
|
||||
|
||||
claims, err := store.ClaimRun(ctx, "scoped", "173", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(claims) != 2 {
|
||||
t.Fatalf("173 通道应有 2 张卡，实际 %d", len(claims))
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceRejectsInvalidScope 非法 scope 应在建 Run 前失败。
|
||||
func TestProduceRejectsInvalidScope(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}})
|
||||
|
||||
// 超过 200 个通道
|
||||
tooMany := make([]string, 0, model.MaxScopeChannels+1)
|
||||
for i := 0; i <= model.MaxScopeChannels; i++ {
|
||||
tooMany = append(tooMany, "ch"+string(rune('a'+i%26)))
|
||||
}
|
||||
// 通道名重复会被去重，故用不重复的
|
||||
tooMany = tooMany[:0]
|
||||
for i := 0; i <= model.MaxScopeChannels; i++ {
|
||||
tooMany = append(tooMany, "ch"+padInt(i))
|
||||
}
|
||||
err := producer.Produce(ctx, model.ImeiQueueStartMessage{
|
||||
RunID: "bad-scope",
|
||||
Scope: &model.SyncScope{Channels: tooMany},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("超限 scope 应被拒绝")
|
||||
}
|
||||
if _, runErr := store.GetRunMeta(ctx, "bad-scope"); runErr == nil {
|
||||
t.Fatal("非法 scope 不应创建 Run")
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceIsIdempotentForSameRun 同一 RunID 重复投递不重复入队。
|
||||
func TestProduceIsIdempotentForSameRun(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
})
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "dup"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "dup")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("activeCount = %d, want 2（重复投递不应翻倍）", meta.ActiveCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceSkipsFinishedRun 终态 Run 不被重投消息复活。
|
||||
func TestProduceSkipsFinishedRun(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}})
|
||||
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "done"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 排空队列让 Run 进入终态：先领取并终结任务，再收尾。
|
||||
claims, err := store.ClaimRun(ctx, "done", "ch", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil || len(claims) != 1 {
|
||||
t.Fatalf("claims=%v err=%v", claims, err)
|
||||
}
|
||||
if _, err = store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "done"); err != nil || !completed {
|
||||
t.Fatalf("complete run: completed=%v err=%v", completed, err)
|
||||
}
|
||||
|
||||
err = producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "done"})
|
||||
if !errors.Is(err, ErrRunFinished) {
|
||||
t.Fatalf("终态 Run 重投应返回 ErrRunFinished，实际 %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceRejectsEmptyRunID 空 RunID 直接失败。
|
||||
func TestProduceRejectsEmptyRunID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, _ := newProducer(t, []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}})
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{}); err == nil {
|
||||
t.Fatal("空 RunID 应被拒绝")
|
||||
}
|
||||
}
|
||||
|
||||
// TestScopeDigestIsStableAcrossOrdering 摘要对通道顺序不敏感，
|
||||
// 使「同一批通道以不同顺序请求」得到同一个 digest。
|
||||
//
|
||||
// 排序由 NormalizeScope 完成，所以必须走归一化后再算 digest
|
||||
// （直接对未排序输入调 scopeDigest 得到不同结果属预期行为）。
|
||||
func TestScopeDigestIsStableAcrossOrdering(t *testing.T) {
|
||||
a, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"173", "189", "216"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"216", "173", "189"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scopeDigest(a) != scopeDigest(b) {
|
||||
t.Fatalf("digest 应对顺序不敏感: %q vs %q", scopeDigest(a), scopeDigest(b))
|
||||
}
|
||||
if scopeDigest(a) == "" {
|
||||
t.Fatal("非空 scope 的 digest 不应为空")
|
||||
}
|
||||
if scopeDigest(nil) != "" {
|
||||
t.Fatal("全量同步的 digest 应为空")
|
||||
}
|
||||
}
|
||||
|
||||
func padInt(value int) string {
|
||||
if value == 0 {
|
||||
return "0"
|
||||
}
|
||||
digits := make([]byte, 0, 8)
|
||||
for value > 0 {
|
||||
digits = append([]byte{byte('0' + value%10)}, digits...)
|
||||
value /= 10
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
@@ -0,0 +1,493 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
var errProducerDone = gerror.New("run production already closed")
|
||||
|
||||
func producerStateError(state zset.ProducerState) error {
|
||||
if state.Status == "completed" || state.Status == "completed_with_failures" {
|
||||
return ErrRunFinished
|
||||
}
|
||||
if state.ProducerDone {
|
||||
return errProducerDone
|
||||
}
|
||||
if state.Status != "producing" && state.Status != "running" {
|
||||
return gerror.Newf("unexpected producer run status %q", state.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// produceChannels owns all per-attempt coordination. There is one worker per
|
||||
// target, NOT a pool of four long-lived channel workers. SQL tokens and the
|
||||
// cancellable admission gate never span sleeps or goroutine joins.
|
||||
func (p *Producer) produceChannels(ctx context.Context, message model.TrafficUsageMonthlyStartMessage, scope *model.SyncScope, scanUpperID int64, startedAt time.Time, total *productionStats) error {
|
||||
scanner := p.scanner.(channelScanner)
|
||||
metadata, err := scanner.ListChannels(ctx)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "initialize channel scan")
|
||||
}
|
||||
state, err := p.store.ReadProducerState(ctx, message.RunID, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return err
|
||||
}
|
||||
targets := make(map[string]channelMetadata, len(metadata))
|
||||
known := make(map[string]bool, len(metadata))
|
||||
for _, channel := range metadata {
|
||||
if channel.ChanKey == "" || !scope.Allows(channel.ChanKey) {
|
||||
continue
|
||||
}
|
||||
if known[channel.ChanKey] {
|
||||
return gerror.Newf("duplicate channel metadata %q", channel.ChanKey)
|
||||
}
|
||||
known[channel.ChanKey] = true
|
||||
reason := ""
|
||||
if channel.SyncEnabled {
|
||||
reason = "sync_disabled"
|
||||
} else if channel.TempKey == "" {
|
||||
reason = "missing_channel_template"
|
||||
}
|
||||
if reason != "" {
|
||||
total.skippedChannels++
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_channel status=skipped runId=%s chanKey=%s reason=%s skipUnit=channel", message.RunID, channel.ChanKey, reason)
|
||||
continue
|
||||
}
|
||||
targets[channel.ChanKey] = channel
|
||||
}
|
||||
if scope != nil {
|
||||
for _, chanKey := range scope.Channels {
|
||||
if !known[chanKey] {
|
||||
total.skippedChannels++
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_channel status=skipped runId=%s chanKey=%s reason=unknown_channel skipUnit=channel", message.RunID, chanKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_channel_scan status=started runId=%s targetChannels=%d scanUpperID=%d scanConcurrency=%d pageSize=%d skipUnit=channel", message.RunID, len(targets), scanUpperID, p.config.ScanConcurrency, p.config.PageSize)
|
||||
keys := make([]string, 0, len(targets))
|
||||
for key := range targets {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
finishObservation := p.startObservation(ctx, message.RunID, keys, startedAt)
|
||||
observationSuccess := false
|
||||
defer func() { finishObservation(observationSuccess) }()
|
||||
var (
|
||||
workers sync.WaitGroup
|
||||
slots = make(chan struct{}, p.config.ScanConcurrency)
|
||||
gate = make(chan struct{}, 1)
|
||||
metrics = newScanActivity()
|
||||
results = make([]*channelScanStats, len(keys))
|
||||
)
|
||||
pool, hasPool := scanner.(interface {
|
||||
poolWait(context.Context) (int64, int64)
|
||||
})
|
||||
var poolCount, poolNanos int64
|
||||
if hasPool {
|
||||
poolCount, poolNanos = pool.poolWait(ctx)
|
||||
}
|
||||
workCtx, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
for i, key := range keys {
|
||||
stats := &channelScanStats{chanKey: key, lastSummary: time.Now()}
|
||||
results[i] = stats
|
||||
workers.Add(1)
|
||||
go func(channel channelMetadata, stats *channelScanStats) {
|
||||
defer workers.Done()
|
||||
scanErr := p.scanOneChannel(workCtx, scanner, message, channel, scanUpperID, startedAt, slots, gate, metrics, stats)
|
||||
if scanErr != nil {
|
||||
cancel(scanErr)
|
||||
}
|
||||
reason := stats.stopReason
|
||||
if reason == "" {
|
||||
reason = "scan_complete"
|
||||
}
|
||||
if scanErr != nil {
|
||||
reason = "cancelled"
|
||||
}
|
||||
p.observe(channel.ChanKey, reason, 0, false, true)
|
||||
stats.log(workCtx, message.RunID, "ended", startedAt, scanErr)
|
||||
}(targets[key], stats)
|
||||
}
|
||||
workers.Wait()
|
||||
var summary channelScanStats
|
||||
for _, stats := range results {
|
||||
total.pages += stats.scanPages
|
||||
total.scanned += stats.eligible
|
||||
total.eligible += stats.eligible
|
||||
total.enqueued += stats.added
|
||||
total.stoppedTasks += stats.rejected
|
||||
if stats.eligible > 0 {
|
||||
total.channels[stats.chanKey] = struct{}{}
|
||||
}
|
||||
summary.merge(stats)
|
||||
}
|
||||
metrics.log(ctx, message.RunID, &summary)
|
||||
if hasPool {
|
||||
count, nanos := pool.poolWait(ctx)
|
||||
g.Log().Infof(ctx, "event=monthly_usage_scan_pool runId=%s sharedPoolWaitCountDelta=%d sharedPoolWaitMsDelta=%d attribution=shared_pool", message.RunID, count-poolCount, (nanos-poolNanos)/int64(time.Millisecond))
|
||||
}
|
||||
observationSuccess = context.Cause(workCtx) == nil
|
||||
return context.Cause(workCtx)
|
||||
}
|
||||
|
||||
func acquireContext(ctx context.Context, token chan struct{}) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case token <- struct{}{}:
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-token
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) scanOneChannel(ctx context.Context, scanner channelScanner, message model.TrafficUsageMonthlyStartMessage, channel channelMetadata, scanUpperID int64, startedAt time.Time, slots, gate chan struct{}, metrics *scanActivity, stats *channelScanStats) error {
|
||||
for stats.cursor < scanUpperID {
|
||||
// Detect a blocked/closed run before querying, including empty channels.
|
||||
p.observe(channel.ChanKey, "redis_read", 0, false, false)
|
||||
state, err := p.store.ReadProducerState(ctx, message.RunID, channel.ChanKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Blocked {
|
||||
// No page was read: the unknown unscanned remainder has no card
|
||||
// count. Never turn it into a fabricated success/failure total.
|
||||
stats.stopReason = "blocked_unscanned_remainder"
|
||||
return nil
|
||||
}
|
||||
p.observe(channel.ChanKey, "sql_slot", 0, false, false)
|
||||
waitStart := time.Now()
|
||||
if err = acquireContext(ctx, slots); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.slotWait += time.Since(waitStart)
|
||||
metrics.begin()
|
||||
p.observe(channel.ChanKey, "sql_query", 0, false, false)
|
||||
queryStart := time.Now()
|
||||
cards, scanErr := scanner.ScanChannel(ctx, channel.ChanKey, stats.cursor, p.config.PageSize)
|
||||
elapsed := time.Since(queryStart)
|
||||
metrics.end()
|
||||
<-slots // release immediately after results have been materialized
|
||||
stats.sqlTime += elapsed
|
||||
stats.sqlLatency.observe(elapsed)
|
||||
stats.scanPages++
|
||||
if scanErr != nil {
|
||||
return gerror.Wrapf(scanErr, "scan channel %s after %d", channel.ChanKey, stats.cursor)
|
||||
}
|
||||
if len(cards) > p.config.PageSize {
|
||||
return gerror.New("scanner exceeded page size")
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
stats.stopReason = "empty_result"
|
||||
return nil
|
||||
}
|
||||
var (
|
||||
pageCursor = stats.cursor
|
||||
overUpper bool
|
||||
tasks = make([]zset.Task, 0, len(cards))
|
||||
)
|
||||
for i, card := range cards {
|
||||
if card.ChanKey != channel.ChanKey || card.ID <= pageCursor {
|
||||
return gerror.New("channel scanner returned unordered or foreign cards")
|
||||
}
|
||||
if card.ID > scanUpperID {
|
||||
stats.overUpper += int64(len(cards) - i)
|
||||
overUpper = true
|
||||
break
|
||||
}
|
||||
pageCursor = card.ID
|
||||
tasks = append(tasks, zset.Task{TaskID: zset.StableTaskID(message.RunID, message.Operation, card.ID), RunID: message.RunID, Operation: message.Operation, QueryMonth: message.QueryMonth, ChanKey: channel.ChanKey, TempKey: channel.TempKey, Operator: channel.Operator, CardID: card.ID, ICCID: card.ICCID, MSISDN: card.MSISDN, IMSI: card.IMSI})
|
||||
}
|
||||
p.observe(channel.ChanKey, "page_ready", len(tasks), false, false)
|
||||
stats.eligible += int64(len(tasks))
|
||||
blocked, err := p.admitChannelPage(ctx, message.RunID, tasks, startedAt, gate, stats)
|
||||
if err != nil {
|
||||
return err // page cursor is not committed, even after a partial write
|
||||
}
|
||||
if blocked {
|
||||
stats.stopReason = "blocked"
|
||||
return nil
|
||||
}
|
||||
stats.cursor = pageCursor
|
||||
stats.progress(ctx, message.RunID, startedAt)
|
||||
if overUpper || stats.cursor >= scanUpperID {
|
||||
stats.stopReason = "upper_bound"
|
||||
return nil
|
||||
}
|
||||
// Do not use a short page as EOF: query-time eligibility may change.
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// admitChannelPage admits only new TaskIDs, in capacity-sized slices. Each
|
||||
// worker retains at most this page. No capacity reservation survives the gate.
|
||||
func (p *Producer) admitChannelPage(ctx context.Context, runID string, tasks []zset.Task, startedAt time.Time, gate chan struct{}, stats *channelScanStats) (bool, error) {
|
||||
for len(tasks) > 0 {
|
||||
p.observe(stats.chanKey, "admission_gate", len(tasks), false, false)
|
||||
gateStart := time.Now()
|
||||
if err := acquireContext(ctx, gate); err != nil {
|
||||
return false, err
|
||||
}
|
||||
stats.gateWait += time.Since(gateStart)
|
||||
blocked, waitKind, remaining, err := p.admitChannelSlice(ctx, runID, tasks, startedAt, stats)
|
||||
<-gate
|
||||
if err != nil || blocked {
|
||||
return blocked, err
|
||||
}
|
||||
tasks = remaining
|
||||
p.observe(stats.chanKey, "page_ready", len(tasks), false, false)
|
||||
if waitKind != "" {
|
||||
p.observe(stats.chanKey, waitKind+"_capacity", len(tasks), false, false)
|
||||
waitStart := time.Now()
|
||||
err = sleepContext(ctx, p.config.PollInterval)
|
||||
if waitKind == "global" {
|
||||
stats.globalWait += time.Since(waitStart)
|
||||
} else {
|
||||
stats.channelWait += time.Since(waitStart)
|
||||
}
|
||||
stats.progress(ctx, runID, startedAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, ctx.Err()
|
||||
}
|
||||
|
||||
// Caller holds the short admission gate; there is no sleep in this method.
|
||||
func (p *Producer) admitChannelSlice(ctx context.Context, runID string, tasks []zset.Task, startedAt time.Time, stats *channelScanStats) (blocked bool, waitKind string, remaining []zset.Task, err error) {
|
||||
p.observe(stats.chanKey, "redis_read", len(tasks), false, false)
|
||||
readStart := time.Now()
|
||||
state, err := p.store.ReadProducerState(ctx, runID, stats.chanKey)
|
||||
if err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
existing, err := p.store.ExistingRunTasks(ctx, runID, tasks)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
pending := tasks[:0]
|
||||
for i, exists := range existing {
|
||||
if exists {
|
||||
stats.duplicates++
|
||||
} else {
|
||||
pending = append(pending, tasks[i])
|
||||
}
|
||||
}
|
||||
tasks = pending
|
||||
p.observe(stats.chanKey, "redis_read", len(tasks), false, false)
|
||||
if state.Blocked {
|
||||
stats.rejected += int64(len(tasks))
|
||||
return true, "", nil, nil
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return false, "", nil, nil
|
||||
}
|
||||
space := int64(len(tasks))
|
||||
if limit := p.config.GlobalHighWaterMark; limit > 0 {
|
||||
space = min(space, limit-state.ActiveCount) // inflight is already included
|
||||
if space <= 0 {
|
||||
return false, "global", tasks, nil
|
||||
}
|
||||
}
|
||||
if limit := p.config.ChannelHighWaterMark; limit > 0 {
|
||||
key, keyErr := p.store.RoundActiveKey(runID, state.CurrentRound, stats.chanKey)
|
||||
if keyErr != nil {
|
||||
return false, "", tasks, keyErr
|
||||
}
|
||||
readStart = time.Now()
|
||||
size, sizeErr := p.store.SizeKey(ctx, key)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if sizeErr != nil {
|
||||
return false, "", tasks, sizeErr
|
||||
}
|
||||
space = min(space, limit-size)
|
||||
if space <= 0 {
|
||||
return false, "channel", tasks, nil
|
||||
}
|
||||
}
|
||||
batch := tasks[:int(space)]
|
||||
p.observe(stats.chanKey, "redis_write", len(tasks), false, false)
|
||||
enqueueStart := time.Now()
|
||||
added, err := p.store.EnqueueRunBatch(ctx, runID, stats.chanKey, batch, time.Now())
|
||||
stats.enqueueTime += time.Since(enqueueStart)
|
||||
stats.enqueueBatches++
|
||||
if err != nil {
|
||||
return false, "", tasks, err // uncertain responses require idempotent rescan
|
||||
}
|
||||
if added < 0 || added > len(batch) {
|
||||
return false, "", tasks, gerror.New("invalid enqueue count")
|
||||
}
|
||||
stats.added += int64(added)
|
||||
p.observe(stats.chanKey, "redis_write", len(tasks)-len(batch), added > 0, false)
|
||||
if added > 0 && !stats.firstEnqueueRecorded {
|
||||
stats.firstEnqueueRecorded = true
|
||||
stats.firstEnqueue = time.Since(startedAt)
|
||||
stats.log(ctx, runID, "first_enqueue", startedAt, nil)
|
||||
}
|
||||
if added < len(batch) {
|
||||
readStart = time.Now()
|
||||
confirmed, checkErr := p.store.ExistingRunTasks(ctx, runID, batch)
|
||||
if checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
state, checkErr = p.store.ReadProducerState(ctx, runID, stats.chanKey)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
if checkErr = producerStateError(state); checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
present := 0
|
||||
for _, exists := range confirmed {
|
||||
if exists {
|
||||
present++
|
||||
}
|
||||
}
|
||||
if present < added {
|
||||
return false, "", tasks, gerror.New("enqueue count exceeds confirmed task states")
|
||||
}
|
||||
stats.duplicates += int64(present - added)
|
||||
if state.Blocked {
|
||||
stats.rejected += int64(len(tasks) - present)
|
||||
return true, "", nil, nil
|
||||
}
|
||||
if present != len(batch) {
|
||||
return false, "", tasks, gerror.Newf("unexplained enqueue rejection: added=%d submitted=%d confirmed=%d", added, len(batch), present)
|
||||
}
|
||||
}
|
||||
return false, "", tasks[len(batch):], nil
|
||||
}
|
||||
|
||||
// Retained only to keep errors.Is semantics visible at the call site.
|
||||
func isProductionClosed(err error) bool { return errors.Is(err, errProducerDone) }
|
||||
|
||||
func durationMS(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) }
|
||||
|
||||
// latencyHistogram is fixed-size. Reported percentiles are bucket upper bounds,
|
||||
// never exact samples or a server-side SLA; the last bucket is unbounded.
|
||||
var latencyBounds = [...]time.Duration{time.Millisecond, 2 * time.Millisecond, 5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond, 50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond, 500 * time.Millisecond, time.Second, 2 * time.Second, 5 * time.Second, 10 * time.Second, 30 * time.Second, time.Minute}
|
||||
|
||||
type latencyHistogram struct{ counts [16]int64 }
|
||||
|
||||
func (h *latencyHistogram) observe(d time.Duration) {
|
||||
i := 0
|
||||
for i < len(latencyBounds) && d > latencyBounds[i] {
|
||||
i++
|
||||
}
|
||||
h.counts[i]++
|
||||
}
|
||||
func (h *latencyHistogram) percentile(percent int64) string {
|
||||
var total, cumulative int64
|
||||
for _, count := range h.counts {
|
||||
total += count
|
||||
}
|
||||
if total == 0 {
|
||||
return "n/a"
|
||||
}
|
||||
target := (total*percent + 99) / 100
|
||||
for i, count := range h.counts {
|
||||
cumulative += count
|
||||
if cumulative >= target {
|
||||
if i == len(latencyBounds) {
|
||||
return ">60000"
|
||||
}
|
||||
return fmt.Sprintf("%.0f", durationMS(latencyBounds[i]))
|
||||
}
|
||||
}
|
||||
return "n/a"
|
||||
}
|
||||
|
||||
type channelScanStats struct {
|
||||
firstEnqueueRecorded bool
|
||||
chanKey, stopReason string
|
||||
cursor, scanPages, eligible, overUpper, enqueueBatches, added, duplicates, rejected int64
|
||||
sqlTime, slotWait, gateWait, globalWait, channelWait, stateTime, enqueueTime, firstEnqueue time.Duration
|
||||
sqlLatency latencyHistogram
|
||||
lastSummary time.Time
|
||||
}
|
||||
|
||||
func (s *channelScanStats) log(ctx context.Context, runID, status string, startedAt time.Time, err error) {
|
||||
errorText := ""
|
||||
if err != nil {
|
||||
errorText = err.Error()
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_channel_scan status=%s runId=%s chanKey=%s cursor=%d scanPages=%d eligible=%d overUpper=%d enqueueBatches=%d added=%d duplicates=%d rejected=%d sqlMs=%.3f sqlP50UpperMs=%s sqlP95UpperMs=%s slotWaitMs=%.3f admissionWaitMs=%.3f globalCapacityWaitMs=%.3f channelCapacityWaitMs=%.3f stateReadMs=%.3f enqueueMs=%.3f firstEnqueueMs=%.3f wallMs=%d stopReason=%s error=%q", status, runID, s.chanKey, s.cursor, s.scanPages, s.eligible, s.overUpper, s.enqueueBatches, s.added, s.duplicates, s.rejected, durationMS(s.sqlTime), s.sqlLatency.percentile(50), s.sqlLatency.percentile(95), durationMS(s.slotWait), durationMS(s.gateWait), durationMS(s.globalWait), durationMS(s.channelWait), durationMS(s.stateTime), durationMS(s.enqueueTime), durationMS(s.firstEnqueue), time.Since(startedAt).Milliseconds(), s.stopReason, errorText)
|
||||
}
|
||||
func (s *channelScanStats) progress(ctx context.Context, runID string, startedAt time.Time) {
|
||||
if time.Since(s.lastSummary) >= time.Minute {
|
||||
s.lastSummary = time.Now()
|
||||
s.log(ctx, runID, "progress", startedAt, nil)
|
||||
}
|
||||
}
|
||||
func (s *channelScanStats) merge(other *channelScanStats) {
|
||||
s.scanPages += other.scanPages
|
||||
s.eligible += other.eligible
|
||||
s.overUpper += other.overUpper
|
||||
s.enqueueBatches += other.enqueueBatches
|
||||
s.added += other.added
|
||||
s.duplicates += other.duplicates
|
||||
s.rejected += other.rejected
|
||||
s.sqlTime += other.sqlTime
|
||||
s.slotWait += other.slotWait
|
||||
s.gateWait += other.gateWait
|
||||
s.globalWait += other.globalWait
|
||||
s.channelWait += other.channelWait
|
||||
s.stateTime += other.stateTime
|
||||
s.enqueueTime += other.enqueueTime
|
||||
for i, count := range other.sqlLatency.counts {
|
||||
s.sqlLatency.counts[i] += count
|
||||
}
|
||||
}
|
||||
|
||||
type scanActivity struct {
|
||||
mu sync.Mutex
|
||||
started, changed time.Time
|
||||
running, peak int
|
||||
area time.Duration
|
||||
}
|
||||
|
||||
func newScanActivity() *scanActivity {
|
||||
now := time.Now()
|
||||
return &scanActivity{started: now, changed: now}
|
||||
}
|
||||
func (s *scanActivity) change(delta int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := time.Now()
|
||||
s.area += time.Duration(s.running) * now.Sub(s.changed)
|
||||
s.changed = now
|
||||
s.running += delta
|
||||
s.peak = max(s.peak, s.running)
|
||||
}
|
||||
func (s *scanActivity) begin() { s.change(1) }
|
||||
func (s *scanActivity) end() { s.change(-1) }
|
||||
func (s *scanActivity) log(ctx context.Context, runID string, stats *channelScanStats) {
|
||||
wall := time.Since(s.started)
|
||||
g.Log().Infof(ctx, "event=monthly_usage_scan_summary runId=%s wallMs=%d sqlInFlightPeak=%d sqlInFlightAvg=%.4f scanPages=%d enqueueBatches=%d eligible=%d overUpper=%d added=%d duplicates=%d rejected=%d sqlCumulativeMs=%.3f sqlP50UpperMs=%s sqlP95UpperMs=%s slotWaitCumulativeMs=%.3f admissionWaitCumulativeMs=%.3f globalCapacityWaitCumulativeMs=%.3f channelCapacityWaitCumulativeMs=%.3f stateReadCumulativeMs=%.3f enqueueCumulativeMs=%.3f", runID, wall.Milliseconds(), s.peak, float64(s.area)/float64(wall), stats.scanPages, stats.enqueueBatches, stats.eligible, stats.overUpper, stats.added, stats.duplicates, stats.rejected, durationMS(stats.sqlTime), stats.sqlLatency.percentile(50), stats.sqlLatency.percentile(95), durationMS(stats.slotWait), durationMS(stats.gateWait), durationMS(stats.globalWait), durationMS(stats.channelWait), durationMS(stats.stateTime), durationMS(stats.enqueueTime))
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
)
|
||||
|
||||
// datasetScanner applies the SQL eligibility predicate before LIMIT; metadata
|
||||
// eligibility is applied by the producer. Optional hooks simulate SQL latency,
|
||||
// live additions and failures without accessing an application database.
|
||||
type datasetScanner struct {
|
||||
cards []Card
|
||||
channels []channelMetadata
|
||||
upper int64
|
||||
hook func(context.Context, string, int64) error
|
||||
queries atomic.Int32
|
||||
lists atomic.Int32
|
||||
inflight atomic.Int32
|
||||
peak atomic.Int32
|
||||
}
|
||||
|
||||
func (s *datasetScanner) SnapshotMaxID(context.Context) (int64, error) { return s.upper, nil }
|
||||
func (s *datasetScanner) ListChannels(context.Context) ([]channelMetadata, error) {
|
||||
s.lists.Add(1)
|
||||
return s.channels, nil
|
||||
}
|
||||
func (s *datasetScanner) Scan(ctx context.Context, last, upper int64, size int) ([]Card, error) {
|
||||
var rows []Card
|
||||
for _, card := range s.cards {
|
||||
if card.ID > last && card.ID <= upper {
|
||||
for _, channel := range s.channels {
|
||||
if channel.ChanKey == card.ChanKey {
|
||||
card.TempKey = channel.TempKey
|
||||
card.SyncEnabled = channel.SyncEnabled
|
||||
card.Operator = channel.Operator
|
||||
}
|
||||
}
|
||||
rows = append(rows, card)
|
||||
if len(rows) == size {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
func (s *datasetScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
s.queries.Add(1)
|
||||
running := s.inflight.Add(1)
|
||||
defer s.inflight.Add(-1)
|
||||
for {
|
||||
old := s.peak.Load()
|
||||
if running <= old || s.peak.CompareAndSwap(old, running) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if s.hook != nil {
|
||||
if err := s.hook(ctx, key, last); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var rows []Card
|
||||
for _, card := range s.cards {
|
||||
if card.ChanKey == key && card.ID > last {
|
||||
rows = append(rows, card)
|
||||
if len(rows) == size {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
func scannerFor(cards []Card, upper int64) *datasetScanner {
|
||||
keys := map[string]bool{}
|
||||
s := &datasetScanner{cards: cards, upper: upper}
|
||||
for _, card := range cards {
|
||||
if card.ChanKey != "" && !keys[card.ChanKey] {
|
||||
keys[card.ChanKey] = true
|
||||
s.channels = append(s.channels, channelMetadata{ChanKey: card.ChanKey, TempKey: "sim_yd_v5", Operator: 1})
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
func testStore(t *testing.T) *zset.Store {
|
||||
t.Helper()
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
func testProducer(t *testing.T, scanner CardScanner, store *zset.Store, config Config) *Producer {
|
||||
t.Helper()
|
||||
config.RoundRetryEnabled = true
|
||||
p, err := New(scanner, store, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
func contextForTest(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
return ctx
|
||||
}
|
||||
func taskIDs(t *testing.T, store *zset.Store, run string) []string {
|
||||
t.Helper()
|
||||
key, _ := store.TaskStateKey(run)
|
||||
result, err := store.RedisClient().Do(context.Background(), "HKEYS", key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := result.Strings()
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
func requireMeta(t *testing.T, store *zset.Store, run string, done bool, active int64) {
|
||||
t.Helper()
|
||||
meta, err := store.GetRunMeta(context.Background(), run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ProducerDone != done || meta.ActiveCount != active {
|
||||
t.Fatalf("meta=%+v, want done=%t active=%d", meta, done, active)
|
||||
}
|
||||
}
|
||||
func waitUntil(t *testing.T, ctx context.Context, condition func() bool) {
|
||||
t.Helper()
|
||||
for !condition() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelModeSettings(t *testing.T) {
|
||||
store, _ := zset.New(&producerRedis{}, "flow:")
|
||||
p, err := New(&fakeScanner{}, store, Config{})
|
||||
if err != nil || p.config.ScanConcurrency != 4 {
|
||||
t.Fatalf("defaults: p=%+v err=%v", p, err)
|
||||
}
|
||||
for _, config := range []Config{{ScanConcurrency: -1}} {
|
||||
if _, err = New(&fakeScanner{}, store, config); err == nil {
|
||||
t.Fatalf("accepted invalid config/capability %+v", config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanFiniteMemoryBoundary(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
upper int64
|
||||
cards []Card
|
||||
ids []int64
|
||||
queries int32
|
||||
}{
|
||||
{"zero upper", 0, []Card{{ID: 1, ChanKey: "a"}}, nil, 0},
|
||||
{"exact page", 4, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}}, []int64{1, 2, 3, 4}, 2},
|
||||
{"tail then empty", 9, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}}, []int64{1, 2, 3}, 3},
|
||||
{"first outside", 3, []Card{{ID: 4, ChanKey: "a"}, {ID: 5, ChanKey: "a"}}, nil, 1},
|
||||
{"mixed final page", 3, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}, {ID: 5, ChanKey: "a"}}, []int64{1, 2, 3}, 2},
|
||||
{"empty channel", 9, nil, nil, 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor(test.cards, test.upper)
|
||||
if len(scanner.channels) == 0 {
|
||||
scanner.channels = []channelMetadata{{ChanKey: "a", TempKey: "sim_yd_v5"}}
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("boundary")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []string
|
||||
for _, id := range test.ids {
|
||||
want = append(want, zset.StableTaskID("boundary", operation, id))
|
||||
}
|
||||
sort.Strings(want)
|
||||
got := taskIDs(t, store, "boundary")
|
||||
if len(got) != len(want) || (len(want) > 0 && !reflect.DeepEqual(got, want)) {
|
||||
t.Fatalf("got=%v want=%v", got, want)
|
||||
}
|
||||
if scanner.queries.Load() != test.queries {
|
||||
t.Fatalf("queries=%d want=%d", scanner.queries.Load(), test.queries)
|
||||
}
|
||||
requireMeta(t, store, "boundary", true, int64(len(want)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanLiveHigherIDsDoNotExtendRun(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}}, 3)
|
||||
scanner.hook = func(_ context.Context, _ string, last int64) error {
|
||||
if last > 0 {
|
||||
scanner.cards = append(scanner.cards, Card{ID: 100 + last, ChanKey: "a"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("live")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(taskIDs(t, store, "live")) != 3 || scanner.queries.Load() != 2 {
|
||||
t.Fatal("higher IDs extended finite range")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanSerialTaskIDSetParity(t *testing.T) {
|
||||
for _, scope := range [][]string{nil, {"a", "b"}, {"disabled", "unknown"}} {
|
||||
t.Run(fmt.Sprint(scope), func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
cards := []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "b"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "disabled"}, {ID: 5, ChanKey: "no-template"}, {ID: 6, ChanKey: "unknown"}, {ID: 7, ChanKey: "b"}, {ID: 8, ChanKey: "a"}}
|
||||
var sets [][]string
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
scanner := scannerFor(cards, 7)
|
||||
scanner.channels = []channelMetadata{{ChanKey: "a", TempKey: "sim_yd_v5", Operator: 1}, {ChanKey: "b", TempKey: "sim_dx_v1", Operator: 2}, {ChanKey: "disabled", TempKey: "sim_yd_v5", SyncEnabled: true}, {ChanKey: "no-template"}}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
run := "parity-" + mode
|
||||
if err := p.Produce(contextForTest(t), scopedMessage(run, scope...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := taskIDs(t, store, run)
|
||||
for i, id := range ids {
|
||||
ids[i] = strings.TrimPrefix(id, run+":")
|
||||
}
|
||||
sets = append(sets, ids)
|
||||
if mode == "channel" && scanner.lists.Load() != 1 {
|
||||
t.Fatal("metadata not loaded once")
|
||||
}
|
||||
_, _ = store.RedisClient().Do(context.Background(), "SREM", store.ActiveRunsKey(), run)
|
||||
}
|
||||
if !reflect.DeepEqual(sets[0], sets[1]) {
|
||||
t.Fatalf("serial=%v channel=%v", sets[0], sets[1])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelSQLSlotsReleasedWhileChannelsWait(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
var cards []Card
|
||||
for i := 0; i < 9; i++ {
|
||||
cards = append(cards, Card{ID: int64(i + 1), ChanKey: fmt.Sprint(i)}, Card{ID: int64(i + 10), ChanKey: fmt.Sprint(i)})
|
||||
}
|
||||
sort.Slice(cards, func(i, j int) bool { return cards[i].ID < cards[j].ID })
|
||||
scanner := scannerFor(cards, 100)
|
||||
scanner.hook = func(ctx context.Context, _ string, _ int64) error { return sleepContext(ctx, 5*time.Millisecond) }
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2, ScanConcurrency: 4, ChannelHighWaterMark: 1, GlobalHighWaterMark: 100, PollInterval: time.Millisecond})
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- p.Produce(ctx, scopedMessage("isolation")) }()
|
||||
waitUntil(t, ctx, func() bool { return len(taskIDs(t, store, "isolation")) == 9 })
|
||||
if peak := scanner.peak.Load(); peak > 4 || peak < 2 {
|
||||
t.Fatalf("SQL peak=%d", peak)
|
||||
}
|
||||
// All nine channels got a first task despite the first four waiting at
|
||||
// channel capacity. SQL slots are free while every channel is paused.
|
||||
if scanner.inflight.Load() != 0 {
|
||||
t.Fatal("SQL tokens held during channel backpressure")
|
||||
}
|
||||
for _, channel := range scanner.channels {
|
||||
if _, err := store.BlockRunScoped(ctx, "isolation", channel.ChanKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "isolation")
|
||||
if err != nil || !meta.ProducerDone {
|
||||
t.Fatalf("meta=%+v err=%v", meta, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelRemainingCapacityAndDuplicateAdmission(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
scanner := scannerFor(nil, 0)
|
||||
p := testProducer(t, scanner, store, Config{GlobalHighWaterMark: 3, ChannelHighWaterMark: 2, PollInterval: time.Millisecond})
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "capacity", Status: "producing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Existing success and waiting states still deduplicate and consume no new
|
||||
// capacity. Active includes leases: inflight must not be added again.
|
||||
stateKey, _ := store.TaskStateKey("capacity")
|
||||
metaKey, _ := store.RunMetaKey("capacity")
|
||||
activeKey, _ := store.RoundActiveKey("capacity", 1, "a")
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", stateKey, "one", `{"status":"success"}`, "two", `{"status":"waiting"}`)
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", metaKey, "activeCount", 2, "inflightCount", 2)
|
||||
stats := &channelScanStats{chanKey: "a", lastSummary: time.Now()}
|
||||
tasks := []zset.Task{{TaskID: "one", RunID: "capacity", ChanKey: "a"}, {TaskID: "two", RunID: "capacity", ChanKey: "a"}, {TaskID: "three", RunID: "capacity", ChanKey: "a"}, {TaskID: "four", RunID: "capacity", ChanKey: "a"}}
|
||||
gate := make(chan struct{}, 1)
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := p.admitChannelPage(ctx, "capacity", tasks, time.Now(), gate, stats); result <- err }()
|
||||
waitUntil(t, ctx, func() bool { meta, e := store.GetRunMeta(ctx, "capacity"); return e == nil && meta.ActiveCount == 3 })
|
||||
if size, err := store.SizeKey(ctx, activeKey); err != nil || size != 1 {
|
||||
t.Fatalf("initial slice size=%d err=%v", size, err)
|
||||
}
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", metaKey, "activeCount", 0, "inflightCount", 0)
|
||||
if err := <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats.duplicates != 2 || stats.added != 2 || stats.enqueueBatches != 2 || stats.globalWait == 0 {
|
||||
t.Fatalf("stats=%+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
// faultRedis forwards to a real isolated Redis, optionally returning a lost
|
||||
// response after Lua succeeded. Never mocks the Lua semantics under test.
|
||||
type faultRedis struct {
|
||||
zset.RedisClient
|
||||
failEnqueue atomic.Int32
|
||||
failRead atomic.Bool
|
||||
failRenew atomic.Bool
|
||||
lostResponse bool
|
||||
reject bool
|
||||
}
|
||||
|
||||
func (r *faultRedis) Do(ctx context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
if r.failRead.Load() && command == "HMGET" {
|
||||
return nil, errors.New("Redis read failure")
|
||||
}
|
||||
return r.RedisClient.Do(ctx, command, args...)
|
||||
}
|
||||
func (r *faultRedis) Eval(ctx context.Context, script string, n int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if r.failRenew.Load() && strings.Contains(script, "PEXPIRE") && strings.Contains(script, "owner ~=") {
|
||||
return gvar.New(0), nil
|
||||
}
|
||||
if n == 4 && strings.Contains(script, "HEXISTS") {
|
||||
if r.reject {
|
||||
return gvar.New(0), nil
|
||||
}
|
||||
if r.failEnqueue.CompareAndSwap(1, 0) {
|
||||
if r.lostResponse {
|
||||
if _, err := r.RedisClient.Eval(ctx, script, n, keys, args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, errors.New("enqueue response unavailable")
|
||||
}
|
||||
}
|
||||
return r.RedisClient.Eval(ctx, script, n, keys, args)
|
||||
}
|
||||
|
||||
func TestChannelRescanPartialAndUncertainWrites(t *testing.T) {
|
||||
for _, uncertain := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(uncertain), func(t *testing.T) {
|
||||
client := &faultRedis{RedisClient: testutil.Redis(t), lostResponse: uncertain}
|
||||
store, _ := zset.New(client, "flow:")
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}}, 4)
|
||||
// Fail the second page: first page must survive; uncertain success is
|
||||
// confirmed only on a same-Run rescan, with stable IDs and no double count.
|
||||
scanner.hook = func(_ context.Context, _ string, last int64) error {
|
||||
if last == 2 {
|
||||
client.failEnqueue.Store(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err == nil {
|
||||
t.Fatal("expected enqueue failure")
|
||||
}
|
||||
active := int64(2)
|
||||
if uncertain {
|
||||
active = 4
|
||||
}
|
||||
requireMeta(t, store, "rescan", false, active)
|
||||
scanner.hook = nil
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMeta(t, store, "rescan", true, 4)
|
||||
if len(taskIDs(t, store, "rescan")) != 4 {
|
||||
t.Fatal("wrong stable task set")
|
||||
}
|
||||
before := scanner.queries.Load()
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if before != scanner.queries.Load() {
|
||||
t.Fatal("producerDone redelivery rescanned cards")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelFailureCancelsAllWorkersWithoutProducerDone(t *testing.T) {
|
||||
for _, kind := range []string{"sql", "redis", "unexplained_rejection", "lease"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
client := &faultRedis{RedisClient: testutil.Redis(t)}
|
||||
store, _ := zset.New(client, "flow:")
|
||||
cards := []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "b"}, {ID: 3, ChanKey: "c"}}
|
||||
scanner := scannerFor(cards, 10)
|
||||
allStarted := make(chan struct{})
|
||||
var started atomic.Int32
|
||||
var once sync.Once
|
||||
scanner.hook = func(ctx context.Context, key string, _ int64) error {
|
||||
if started.Add(1) >= 3 {
|
||||
once.Do(func() { close(allStarted) })
|
||||
}
|
||||
select {
|
||||
case <-allStarted:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if kind == "lease" {
|
||||
client.failRenew.Store(true)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
if key == "a" {
|
||||
switch kind {
|
||||
case "sql":
|
||||
return errors.New("SQL failed")
|
||||
case "redis":
|
||||
client.failRead.Store(true)
|
||||
case "unexplained_rejection":
|
||||
client.reject = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2, RunLockTTL: 90 * time.Millisecond})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("failure")); err == nil {
|
||||
t.Fatal("expected fatal failure")
|
||||
}
|
||||
if scanner.inflight.Load() != 0 {
|
||||
t.Fatal("scanner goroutines leaked")
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "failure")
|
||||
if err != nil || meta.ProducerDone {
|
||||
t.Fatalf("partial scan marked done: %+v err=%v", meta, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanStateStopsAndNoTarget(t *testing.T) {
|
||||
for _, kind := range []string{"terminal", "producerDone", "blocked", "missing", "no-target", "no-cards"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 2)
|
||||
if kind == "no-target" {
|
||||
scanner.channels[0].SyncEnabled = true
|
||||
}
|
||||
if kind == "no-cards" {
|
||||
scanner.cards = nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{})
|
||||
if kind != "no-target" && kind != "no-cards" {
|
||||
if err := store.CreateRun(context.Background(), zset.RunMeta{RunID: "stop", Status: "producing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := store.RunMetaKey("stop")
|
||||
switch kind {
|
||||
case "terminal":
|
||||
_, _ = store.RedisClient().Do(context.Background(), "HSET", key, "status", "completed")
|
||||
case "producerDone":
|
||||
if err := store.MarkProducerDone(context.Background(), "stop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "blocked":
|
||||
_, _ = store.BlockRunScoped(context.Background(), "stop", "a")
|
||||
case "missing":
|
||||
scanner.hook = func(ctx context.Context, _ string, _ int64) error {
|
||||
_, err := store.RedisClient().Do(ctx, "DEL", key)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
err := p.Produce(contextForTest(t), scopedMessage("stop"))
|
||||
if kind == "terminal" {
|
||||
if !errors.Is(err, ErrRunFinished) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
} else if kind == "missing" {
|
||||
if !errors.Is(err, zset.ErrRunNotFound) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(taskIDs(t, store, "stop")) != 0 {
|
||||
t.Fatal("stopped/empty run enqueued tasks")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Omitted retry/scan-mode settings must still use the channel Run producer.
|
||||
func TestDefaultProducerUsesChannelRun(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 1)
|
||||
p, err := New(scanner, store, Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = p.Produce(contextForTest(t), scopedMessage("default-channel")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scanner.queries.Load() == 0 || scanner.lists.Load() != 1 {
|
||||
t.Fatal("default did not use channel scan")
|
||||
}
|
||||
requireMeta(t, store, "default-channel", true, 1)
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
// channelScanner augments, rather than replaces, the legacy scanner contract.
|
||||
// ListChannels is called once before any workers start. ScanChannel must not
|
||||
// mutate the legacy DatabaseScanner.channelMetadata cache.
|
||||
type channelScanner interface {
|
||||
ListChannels(context.Context) ([]channelMetadata, error)
|
||||
ScanChannel(context.Context, string, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
// This exact MySQL query shape is a release gate. Do not silently remove hints,
|
||||
// add an SQL upper bound, or move eligibility filtering after LIMIT.
|
||||
const channelPageSQL = `SELECT c.id, c.iccid, c.msisdn, c.imsi, c.chan_key
|
||||
FROM t_sim_card_info AS c FORCE INDEX (idx_card_chan_key)
|
||||
STRAIGHT_JOIN sys_dept AS d
|
||||
ON c.dept_id = d.dept_id
|
||||
AND c.tenant_id = d.tenant_id
|
||||
WHERE c.chan_key = ?
|
||||
AND c.id > ?
|
||||
AND c.iccid <> ''
|
||||
AND c.current_package_code <> ''
|
||||
AND d.deleted_at IS NULL
|
||||
ORDER BY c.id ASC
|
||||
LIMIT ?`
|
||||
|
||||
func (s *DatabaseScanner) ListChannels(ctx context.Context) ([]channelMetadata, error) {
|
||||
var metadata []channelMetadata
|
||||
if err := dao.TSimCardInfo.DB().Model("t_sim_channels").Ctx(ctx).Fields("chan_key,temp_key,sync_enabled,operator").Scan(&metadata); err != nil {
|
||||
return nil, gerror.Wrap(err, "list channel metadata")
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) ScanChannel(ctx context.Context, chanKey string, lastID int64, pageSize int) ([]Card, error) {
|
||||
if chanKey == "" || lastID < 0 || pageSize <= 0 {
|
||||
return nil, gerror.New("invalid channel pagination parameters")
|
||||
}
|
||||
var rows []entity.TSimCardInfo
|
||||
// Use the card DAO's database group, not a separately configured connection.
|
||||
if err := dao.TSimCardInfo.DB().Raw(channelPageSQL, chanKey, lastID, pageSize).Ctx(ctx).Scan(&rows); err != nil {
|
||||
return nil, gerror.Wrap(err, "scan channel cards")
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
cards = append(cards, Card{ID: row.Id, ICCID: row.Iccid, MSISDN: row.Msisdn, IMSI: row.Imsi, ChanKey: row.ChanKey})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
// Pool counters are shared with other worker traffic. Their deltas are direct
|
||||
// observations, not an attribution of all connection waits to this producer.
|
||||
func (*DatabaseScanner) poolWait(ctx context.Context) (int64, int64) {
|
||||
var count, nanos int64
|
||||
for _, item := range dao.TSimCardInfo.DB().Stats(ctx) {
|
||||
stats := item.Stats()
|
||||
count += stats.WaitCount
|
||||
nanos += int64(stats.WaitDuration)
|
||||
}
|
||||
return count, nanos
|
||||
}
|
||||
|
||||
// ValidateScanSettings is strict for values read from configuration. The Go
|
||||
// constructor first fills omitted zero values; explicit YAML zero is rejected.
|
||||
func ValidateScanSettings(concurrency int) error {
|
||||
if concurrency <= 0 {
|
||||
return gerror.New("trafficUsageFullSync.producer.scanConcurrency must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
)
|
||||
|
||||
// SQLite runs the SAME predicate and ordering with only the two MySQL hints
|
||||
// removed. This checks full eligibility/TaskID parity, not MySQL access plans.
|
||||
// Production EXPLAIN must still validate the unmodified channelPageSQL.
|
||||
type sqliteChannelScanner struct{ DatabaseScanner }
|
||||
|
||||
func (s *sqliteChannelScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
query := strings.ReplaceAll(channelPageSQL, "FORCE INDEX (idx_card_chan_key)", "")
|
||||
query = strings.ReplaceAll(query, "STRAIGHT_JOIN", "INNER JOIN")
|
||||
var rows []struct {
|
||||
ID int64 `orm:"id"`
|
||||
ICCID string `orm:"iccid"`
|
||||
MSISDN string `orm:"msisdn"`
|
||||
IMSI string `orm:"imsi"`
|
||||
ChanKey string `orm:"chan_key"`
|
||||
}
|
||||
if err := dao.TSimCardInfo.DB().Raw(query, key, last, size).Ctx(ctx).Scan(&rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
cards = append(cards, Card{ID: row.ID, ICCID: row.ICCID, MSISDN: row.MSISDN, IMSI: row.IMSI, ChanKey: row.ChanKey})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
func TestDatabaseQualificationAndCompleteTaskIDParity(t *testing.T) {
|
||||
// No application DB/config: isolated SQLite file and loopback Redis only.
|
||||
if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "sqlite", Link: "sqlite::@file(" + t.TempDir() + "/cards.db)"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := contextForTest(t)
|
||||
db := dao.TSimCardInfo.DB()
|
||||
t.Cleanup(func() { _ = db.Close(context.Background()) })
|
||||
for _, sql := range []string{
|
||||
`CREATE TABLE sys_dept (dept_id INTEGER PRIMARY KEY, tenant_id TEXT, deleted_at DATETIME)`,
|
||||
`CREATE TABLE t_sim_card_info (id INTEGER PRIMARY KEY, iccid TEXT, msisdn TEXT, imsi TEXT, chan_key TEXT, current_package_code TEXT, dept_id INTEGER, tenant_id TEXT)`,
|
||||
`CREATE INDEX idx_card_chan_key ON t_sim_card_info(chan_key)`,
|
||||
`CREATE TABLE t_sim_channels (chan_key TEXT, temp_key TEXT, sync_enabled INTEGER, operator INTEGER)`,
|
||||
`INSERT INTO sys_dept VALUES (1,'t1',NULL),(2,'t2',NULL),(3,'t1','2026-01-01')`,
|
||||
`INSERT INTO t_sim_channels VALUES ('a','sim_yd_v5',0,1),('b','sim_dx_v1',0,2),('disabled','sim_yd_v5',1,1),('no-template','',0,1)`,
|
||||
`INSERT INTO t_sim_card_info VALUES
|
||||
(1,'i1','m1','s1','a','pkg',1,'t1'),
|
||||
(2,'i2','m2','s2','b','pkg',2,'t2'),
|
||||
(3,'i3','','','a','pkg',1,'t1'),
|
||||
(4,'','','','a','pkg',1,'t1'),
|
||||
(5,'i5','','','a','',1,'t1'),
|
||||
(6,'i6','','','a','pkg',1,'wrong-tenant'),
|
||||
(7,'i7','','','a','pkg',3,'t1'),
|
||||
(8,'i8','','','b','pkg',99,'t2'),
|
||||
(9,'i9','','','disabled','pkg',1,'t1'),
|
||||
(10,'i10','','','no-template','pkg',1,'t1'),
|
||||
(11,'i11','','','unknown','pkg',1,'t1'),
|
||||
(12,'i12','','','','pkg',1,'t1'),
|
||||
(13,'i13','','','b','pkg',2,'t2')`,
|
||||
} {
|
||||
if _, err := db.Exec(ctx, sql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store := testStore(t)
|
||||
for _, scope := range [][]string{nil, {"a"}, {"b", "disabled", "no-template", "unknown"}, {"unknown"}} {
|
||||
var sets [][]string
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
scanner := &sqliteChannelScanner{}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
run := "db-" + mode
|
||||
if err := p.Produce(ctx, scopedMessage(run, scope...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := taskIDs(t, store, run)
|
||||
for i, id := range ids {
|
||||
ids[i] = strings.TrimPrefix(id, run+":")
|
||||
}
|
||||
sort.Strings(ids)
|
||||
sets = append(sets, ids)
|
||||
stateKey, _ := store.TaskStateKey(run)
|
||||
metaKey, _ := store.RunMetaKey(run)
|
||||
chanKey, _ := store.RunChannelsKey(run)
|
||||
_, _ = store.RedisClient().Do(ctx, "DEL", stateKey, metaKey, chanKey)
|
||||
for _, key := range []string{"a", "b"} {
|
||||
activeKey, _ := store.RoundActiveKey(run, 1, key)
|
||||
_, _ = store.RedisClient().Do(ctx, "DEL", activeKey)
|
||||
}
|
||||
_, _ = store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), run)
|
||||
}
|
||||
if !reflect.DeepEqual(sets[0], sets[1]) {
|
||||
t.Fatalf("scope=%v serial=%v channel=%v", scope, sets[0], sets[1])
|
||||
}
|
||||
if scope == nil {
|
||||
want := []string{operation + ":1", operation + ":13", operation + ":2", operation + ":3"}
|
||||
sort.Strings(want)
|
||||
if !reflect.DeepEqual(sets[0], want) {
|
||||
t.Fatalf("qualification set=%v want=%v", sets[0], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelSQLShapeAndParameters(t *testing.T) {
|
||||
for _, term := range []string{"FORCE INDEX (idx_card_chan_key)", "STRAIGHT_JOIN sys_dept", "c.chan_key = ?", "c.id > ?", "c.iccid <> ''", "c.current_package_code <> ''", "d.deleted_at IS NULL", "c.tenant_id = d.tenant_id", "ORDER BY c.id ASC", "LIMIT ?"} {
|
||||
if !strings.Contains(channelPageSQL, term) {
|
||||
t.Fatalf("missing %q", term)
|
||||
}
|
||||
}
|
||||
if strings.Contains(channelPageSQL, "<=") || strings.Count(channelPageSQL, "?") != 3 {
|
||||
t.Fatal("SQL contains upper bound or changed bind parameter count")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,485 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
const operation = "current_month_usage_full_sync"
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
// The run hash and its archive JSON must stay bounded regardless of how many
|
||||
// channels a scope selects.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels. It lets
|
||||
// operators confirm two runs covered the same scope without storing the list.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
SupplyObservation *supply.Tracker
|
||||
// Empty/zero mean omitted for Go callers; startup config validates explicit values.
|
||||
ScanConcurrency int
|
||||
PageSize int
|
||||
GlobalHighWaterMark int64
|
||||
ChannelHighWaterMark int64
|
||||
PollInterval time.Duration
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
RoundRetryEnabled bool
|
||||
MaxRounds int
|
||||
FailureRetention time.Duration
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
c.RoundRetryEnabled = true
|
||||
if c.ScanConcurrency == 0 {
|
||||
c.ScanConcurrency = 4
|
||||
}
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 100
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = 2 * time.Second
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "flow:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.MaxRounds <= 0 {
|
||||
c.MaxRounds = 5
|
||||
}
|
||||
if c.FailureRetention <= 0 {
|
||||
c.FailureRetention = 30 * 24 * time.Hour
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type CardScanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
SyncEnabled bool
|
||||
Operator int
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
Operator int `orm:"operator"`
|
||||
}
|
||||
|
||||
type DatabaseScanner struct{}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := monthlyUsageEligibleCardModel(ctx).
|
||||
Fields(cardTable + "." + columns.Id + " AS " + columns.Id).
|
||||
OrderDesc(cardTable + "." + columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read card snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func monthlyUsageEligibleCardModel(ctx context.Context) *gdb.Model {
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
deptTable := dao.SysDept.Table()
|
||||
cardColumns := dao.TSimCardInfo.Columns()
|
||||
deptColumns := dao.SysDept.Columns()
|
||||
|
||||
return dao.TSimCardInfo.Ctx(ctx).
|
||||
InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId).
|
||||
Where(cardTable+"."+cardColumns.Iccid+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.ChanKey+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", "").
|
||||
WhereNull(deptTable + "." + deptColumns.DeletedAt)
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner CardScanner
|
||||
store *zset.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
observation *supply.Observer
|
||||
}
|
||||
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
eligible int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
skippedChannels int64
|
||||
stoppedTasks int64
|
||||
channels map[string]struct{}
|
||||
scopeChannels int
|
||||
}
|
||||
|
||||
func (s *productionStats) channelCount() int {
|
||||
return len(s.channels)
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, queryMonth, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=monthly_usage_producer status=%s runId=%s queryMonth=%s scanUpperID=%d pages=%d scanned=%d eligible=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d stoppedTasks=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, queryMonth, maxID, s.pages, s.scanned, s.eligible, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.stoppedTasks, s.channelCount(), s.scopeChannels, time.Since(startedAt).Milliseconds())
|
||||
message += fmt.Sprintf(" skippedChannels=%d", s.skippedChannels)
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
if status == "failed" {
|
||||
g.Log().Error(ctx, message)
|
||||
return
|
||||
}
|
||||
if status == "skipped" {
|
||||
g.Log().Warning(ctx, message)
|
||||
return
|
||||
}
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
|
||||
func New(scanner CardScanner, store *zset.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("monthly usage producer dependencies are incomplete")
|
||||
}
|
||||
config = config.withDefaults()
|
||||
if err := ValidateScanSettings(config.ScanConcurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.RoundRetryEnabled {
|
||||
if _, ok := scanner.(channelScanner); !ok {
|
||||
return nil, fmt.Errorf("channel scan requires ListChannels and ScanChannel capabilities")
|
||||
}
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config, owner: newOwner()}, nil
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("monthly usage producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected; callers
|
||||
// should log and ack instead of retrying.
|
||||
var ErrRunFinished = fmt.Errorf("monthly usage run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
func (p *Producer) Produce(ctx context.Context, message model.TrafficUsageMonthlyStartMessage) (err error) {
|
||||
startedAt := time.Now()
|
||||
stats := productionStats{channels: make(map[string]struct{})}
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
|
||||
if message.RunID == "" {
|
||||
err = fmt.Errorf("runId is empty")
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats.scopeChannels = scope.ChannelCount()
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if p.config.RoundRetryEnabled {
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
}
|
||||
acquired, err := p.store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer releaseCancel()
|
||||
_, _ = p.store.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
pending, err := p.hasPendingTasks(ctx, message.RunID)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "pending_check", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if pending {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "pending_check", "pending_tasks", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
renewErr := make(chan error, 1)
|
||||
renewDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(renewDone)
|
||||
p.renewLock(runCtx, cancel, renewErr)
|
||||
}()
|
||||
defer func() {
|
||||
cancel()
|
||||
<-renewDone
|
||||
select {
|
||||
case lockErr := <-renewErr:
|
||||
err = lockErr
|
||||
default:
|
||||
}
|
||||
}()
|
||||
ctx = runCtx
|
||||
if message.Operation == "" {
|
||||
message.Operation = operation
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
if err = p.store.CreateRun(ctx, zset.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: message.Operation,
|
||||
QueryMonth: message.QueryMonth,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_init", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
maxID, err := p.scanner.SnapshotMaxID(ctx)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "snapshot", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_producer status=started runId=%s queryMonth=%s scanUpperID=%d scopeChannels=%d scanMode=channel scanConcurrency=%d pageSize=%d", message.RunID, message.QueryMonth, maxID, scope.ChannelCount(), p.config.ScanConcurrency, p.config.PageSize)
|
||||
stage := "scan"
|
||||
defer func() {
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, message.QueryMonth, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
if p.config.RoundRetryEnabled {
|
||||
stage = "channel_scan"
|
||||
if err = p.produceChannels(ctx, message, scope, maxID, startedAt, &stats); err != nil {
|
||||
if isProductionClosed(err) {
|
||||
// A same-Run redelivery after producerDone leaves consumption alone.
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Verify ownership immediately before closing production. The periodic
|
||||
// renewer still cancels scanners and capacity waits on lease loss.
|
||||
var owned bool
|
||||
owned, err = p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owned {
|
||||
return fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "producer_done"
|
||||
err = p.store.MarkProducerDone(ctx, message.RunID)
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("monthly usage full sync requires Run queue; legacy serial production has been removed")
|
||||
}
|
||||
|
||||
// isRunNotFound reports whether err means the run's metadata hash is absent, as
|
||||
// opposed to a genuine read failure.
|
||||
func isRunNotFound(err error) bool {
|
||||
return errors.Is(err, zset.ErrRunNotFound)
|
||||
}
|
||||
|
||||
func (p *Producer) hasPendingTasks(ctx context.Context, currentRunID ...string) (bool, error) {
|
||||
currentID := ""
|
||||
if len(currentRunID) > 0 {
|
||||
currentID = currentRunID[0]
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, runID := range runs {
|
||||
if runID == currentID {
|
||||
// A redelivered message for the same run may resume a producer
|
||||
// that stopped before producerDone. Stable task IDs make the
|
||||
// rescan idempotent; other runs still provide backpressure.
|
||||
continue
|
||||
}
|
||||
meta, metaErr := p.store.GetRunMeta(ctx, runID)
|
||||
if metaErr != nil {
|
||||
// The active index is a global key written outside the run-scoped
|
||||
// Lua transaction, so it can list a run whose hash is already gone
|
||||
// (expired, evicted, or cleaned by hand). Returning an error here
|
||||
// would surface as a non-retryable failure and make the consumer
|
||||
// redeliver this request forever. Treat it as a stale index entry
|
||||
// and drop it.
|
||||
if isRunNotFound(metaErr) {
|
||||
if _, removeErr := p.store.RedisClient().Do(ctx, "SREM", p.store.ActiveRunsKey(), runID); removeErr != nil {
|
||||
return false, removeErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=stale_index runId=%s", runID)
|
||||
continue
|
||||
}
|
||||
return false, metaErr
|
||||
}
|
||||
// A "running" run has already finished producing (producerDone=1)
|
||||
// and the consumer owns the remaining work: it is genuinely in-flight
|
||||
// and must keep providing backpressure.
|
||||
if meta.Status == "running" {
|
||||
return true, nil
|
||||
}
|
||||
// A "producing" run for another ID cannot have a live producer. This
|
||||
// function only runs after AcquireLockOrTakeover succeeded on the
|
||||
// single producer lock, so any live producer would still hold it and
|
||||
// we could not have acquired it. Such a run is a zombie left behind by
|
||||
// a producer that died before MarkProducerDone (for example the
|
||||
// process was replaced mid-run).
|
||||
//
|
||||
// Promote it so the normal lifecycle can drain and close it. Blocking
|
||||
// would reject every later request forever, and merely warning would
|
||||
// leak the run's keys and its active-index entry indefinitely.
|
||||
reclaimed, reclaimErr := p.store.MarkProducerAbandoned(ctx, runID)
|
||||
if reclaimErr != nil {
|
||||
return false, reclaimErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=zombie_run runId=%s reclaimed=%t activeCount=%d inflightCount=%d nextRoundCount=%d", runID, reclaimed, meta.ActiveCount, meta.InflightCount, meta.NextRoundCount)
|
||||
}
|
||||
}
|
||||
channels, err := p.store.ActiveChannels(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, chanKey := range channels {
|
||||
size, sizeErr := p.store.Size(ctx, chanKey)
|
||||
if sizeErr != nil {
|
||||
return false, sizeErr
|
||||
}
|
||||
if size > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelFunc, errors chan<- error) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ok, err := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil || !ok {
|
||||
// Cancellation during a renewal RPC is shutdown, not lease loss.
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
err = fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
select {
|
||||
case errors <- fmt.Errorf("renew producer lock: %w", err):
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var bytes [12]byte
|
||||
if _, err := rand.Read(bytes[:]); err == nil {
|
||||
return "producer-" + hex.EncodeToString(bytes[:])
|
||||
}
|
||||
return fmt.Sprintf("producer-%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, duration time.Duration) error {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
type pendingRedis struct {
|
||||
channels []string
|
||||
sizes map[string]int64
|
||||
failKey string
|
||||
}
|
||||
|
||||
func (r *pendingRedis) Do(_ context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
switch strings.ToUpper(command) {
|
||||
case "SMEMBERS":
|
||||
return gvar.New(r.channels), nil
|
||||
case "ZCARD":
|
||||
key := fmt.Sprint(args[0])
|
||||
if key == r.failKey {
|
||||
return nil, fmt.Errorf("redis unavailable")
|
||||
}
|
||||
return gvar.New(r.sizes[key]), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected command %s", command)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *pendingRedis) Eval(_ context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
return nil, fmt.Errorf("unexpected Eval call")
|
||||
}
|
||||
|
||||
func TestHasPendingTasks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
redis *pendingRedis
|
||||
want bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "all registered channels are empty",
|
||||
redis: &pendingRedis{
|
||||
channels: []string{"mobile-a", "telecom-a"},
|
||||
sizes: map[string]int64{
|
||||
"flow:mobile-a": 0,
|
||||
"flow:telecom-a": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "previous run still has tasks",
|
||||
redis: &pendingRedis{
|
||||
channels: []string{"mobile-a", "telecom-a"},
|
||||
sizes: map[string]int64{
|
||||
"flow:mobile-a": 0,
|
||||
"flow:telecom-a": 3,
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "redis error is not treated as empty",
|
||||
redis: &pendingRedis{
|
||||
channels: []string{"mobile-a"},
|
||||
sizes: map[string]int64{},
|
||||
failKey: "flow:mobile-a",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
store, err := zset.New(test.redis, "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(&fakeScanner{}, store, Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := func() (bool, error) {
|
||||
producer.config.RoundRetryEnabled = false
|
||||
return producer.hasPendingTasks(context.Background())
|
||||
}()
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("hasPendingTasks() error = %v, wantErr %v", err, test.wantErr)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("hasPendingTasks() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type fakeScanner struct{}
|
||||
|
||||
func (*fakeScanner) SnapshotMaxID(context.Context) (int64, error) { return 0, nil }
|
||||
func (*fakeScanner) Scan(context.Context, int64, int64, int) ([]Card, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (*fakeScanner) ListChannels(context.Context) ([]channelMetadata, error) { return nil, nil }
|
||||
func (*fakeScanner) ScanChannel(context.Context, string, int64, int) ([]Card, error) { return nil, nil }
|
||||
|
||||
type pageScanner struct {
|
||||
cards []Card
|
||||
}
|
||||
|
||||
func (s *pageScanner) SnapshotMaxID(context.Context) (int64, error) {
|
||||
if len(s.cards) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return s.cards[len(s.cards)-1].ID, nil
|
||||
}
|
||||
|
||||
func (s *pageScanner) Scan(_ context.Context, lastID, _ int64, _ int) ([]Card, error) {
|
||||
if lastID != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.cards, nil
|
||||
}
|
||||
|
||||
type producerRedis struct {
|
||||
mu sync.Mutex
|
||||
enqueueBatches map[string]int
|
||||
}
|
||||
|
||||
func (r *producerRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
|
||||
switch strings.ToUpper(command) {
|
||||
case "SET":
|
||||
return gvar.New("OK"), nil
|
||||
case "SMEMBERS":
|
||||
return gvar.New([]string{}), nil
|
||||
case "HGETALL":
|
||||
// No stored run metadata: the producer treats this as a first delivery.
|
||||
return gvar.New(map[string]string{}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected command %s", command)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *producerRedis) Eval(_ context.Context, _ string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if numKeys == 1 {
|
||||
return gvar.New(int64(1)), nil
|
||||
}
|
||||
if numKeys != 3 {
|
||||
return nil, fmt.Errorf("unexpected key count %d", numKeys)
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.enqueueBatches[keys[0]] = len(args) - 3
|
||||
r.mu.Unlock()
|
||||
return gvar.New(len(args) - 3), nil
|
||||
}
|
||||
|
||||
func (s *pageScanner) ListChannels(ctx context.Context) ([]channelMetadata, error) {
|
||||
return scannerFor(s.cards, 0).ListChannels(ctx)
|
||||
}
|
||||
func (s *pageScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
return scannerFor(s.cards, 0).ScanChannel(ctx, key, last, size)
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func newScopeProducer(t *testing.T, cards []Card) (*Producer, *producerRedis) {
|
||||
t.Helper()
|
||||
store := testStore(t)
|
||||
scanner := scannerFor(cards, cards[len(cards)-1].ID)
|
||||
for i := range scanner.channels {
|
||||
for _, c := range cards {
|
||||
if c.ChanKey == scanner.channels[i].ChanKey {
|
||||
scanner.channels[i].TempKey = c.TempKey
|
||||
scanner.channels[i].SyncEnabled = c.SyncEnabled
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
p, err := New(scanner, store, Config{PageSize: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p, &producerRedis{enqueueBatches: map[string]int{}}
|
||||
}
|
||||
func scopedMessage(runID string, channels ...string) model.TrafficUsageMonthlyStartMessage {
|
||||
return model.TrafficUsageMonthlyStartMessage{
|
||||
RunID: runID,
|
||||
Operation: "current_month_usage_full_sync",
|
||||
QueryMonth: "202608",
|
||||
Scope: &model.SyncScope{Channels: channels},
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeEnqueuesOnlySelectedChannels(t *testing.T) {
|
||||
producer, redis := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
{ID: 2, ChanKey: "173", TempKey: "sim_yd_v5", Operator: 1},
|
||||
{ID: 3, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
{ID: 4, ChanKey: "189", TempKey: "sim_yd_v5", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), scopedMessage("scoped", "217")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.enqueueBatches = map[string]int{}
|
||||
for _, ch := range []string{"217", "173", "189", "ghost-channel", "a", "b", "mobile-a", "mobile-b"} {
|
||||
key, _ := producer.store.RoundActiveKey("scoped", 1, ch)
|
||||
n, _ := producer.store.SizeKey(context.Background(), key)
|
||||
if n > 0 {
|
||||
redis.enqueueBatches["flow:"+ch] = int(n)
|
||||
}
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.enqueueBatches) != 1 {
|
||||
t.Fatalf("enqueue batches = %+v, want only the 217 channel", redis.enqueueBatches)
|
||||
}
|
||||
if redis.enqueueBatches["flow:217"] != 2 {
|
||||
t.Fatalf("enqueue batches = %+v, want 2 tasks on flow:217", redis.enqueueBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeMultipleChannels(t *testing.T) {
|
||||
producer, redis := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
{ID: 2, ChanKey: "173", TempKey: "sim_yd_v5", Operator: 1},
|
||||
{ID: 3, ChanKey: "189", TempKey: "sim_yd_v5", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), scopedMessage("scoped", "189", "217")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.enqueueBatches = map[string]int{}
|
||||
for _, ch := range []string{"217", "173", "189", "ghost-channel", "a", "b", "mobile-a", "mobile-b"} {
|
||||
key, _ := producer.store.RoundActiveKey("scoped", 1, ch)
|
||||
n, _ := producer.store.SizeKey(context.Background(), key)
|
||||
if n > 0 {
|
||||
redis.enqueueBatches["flow:"+ch] = int(n)
|
||||
}
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.enqueueBatches) != 2 {
|
||||
t.Fatalf("enqueue batches = %+v, want 217 and 189", redis.enqueueBatches)
|
||||
}
|
||||
if _, ok := redis.enqueueBatches["flow:173"]; ok {
|
||||
t.Fatalf("channel 173 was enqueued despite not being selected: %+v", redis.enqueueBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceFullScopeKeepsEveryChannel(t *testing.T) {
|
||||
producer, redis := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
{ID: 2, ChanKey: "173", TempKey: "sim_yd_v5", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), model.TrafficUsageMonthlyStartMessage{
|
||||
RunID: "full",
|
||||
Operation: "current_month_usage_full_sync",
|
||||
QueryMonth: "202608",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.enqueueBatches = map[string]int{}
|
||||
for _, ch := range []string{"217", "173", "189", "ghost-channel", "a", "b", "mobile-a", "mobile-b"} {
|
||||
key, _ := producer.store.RoundActiveKey("full", 1, ch)
|
||||
n, _ := producer.store.SizeKey(context.Background(), key)
|
||||
if n > 0 {
|
||||
redis.enqueueBatches["flow:"+ch] = int(n)
|
||||
}
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.enqueueBatches) != 2 {
|
||||
t.Fatalf("enqueue batches = %+v, want both channels for a full sync", redis.enqueueBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeCombinesWithSyncDisabledGuard(t *testing.T) {
|
||||
producer, redis := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
// Selected but excluded by sync_enabled: scope must not override it.
|
||||
{ID: 2, ChanKey: "173", TempKey: "sim_yd_v5", SyncEnabled: true, Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), scopedMessage("scoped", "173")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.enqueueBatches = map[string]int{}
|
||||
for _, ch := range []string{"217", "173", "189", "ghost-channel", "a", "b", "mobile-a", "mobile-b"} {
|
||||
key, _ := producer.store.RoundActiveKey("scoped", 1, ch)
|
||||
n, _ := producer.store.SizeKey(context.Background(), key)
|
||||
if n > 0 {
|
||||
redis.enqueueBatches["flow:"+ch] = int(n)
|
||||
}
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.enqueueBatches) != 0 {
|
||||
t.Fatalf("enqueue batches = %+v, want none (selected channel is sync-disabled)", redis.enqueueBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeSelectingUnknownChannelEnqueuesNothing(t *testing.T) {
|
||||
producer, redis := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), scopedMessage("scoped", "does-not-exist")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.enqueueBatches = map[string]int{}
|
||||
for _, ch := range []string{"217", "173", "189", "ghost-channel", "a", "b", "mobile-a", "mobile-b"} {
|
||||
key, _ := producer.store.RoundActiveKey("scoped", 1, ch)
|
||||
n, _ := producer.store.SizeKey(context.Background(), key)
|
||||
if n > 0 {
|
||||
redis.enqueueBatches["flow:"+ch] = int(n)
|
||||
}
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.enqueueBatches) != 0 {
|
||||
t.Fatalf("enqueue batches = %+v, want none", redis.enqueueBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceRejectsInvalidScope(t *testing.T) {
|
||||
producer, redis := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), scopedMessage("bad", "invalid/chan")); err == nil {
|
||||
t.Fatal("expected invalid chanKey to be rejected")
|
||||
}
|
||||
redis.enqueueBatches = map[string]int{}
|
||||
for _, ch := range []string{"217", "173", "189", "ghost-channel", "a", "b", "mobile-a", "mobile-b"} {
|
||||
key, _ := producer.store.RoundActiveKey("scoped", 1, ch)
|
||||
n, _ := producer.store.SizeKey(context.Background(), key)
|
||||
if n > 0 {
|
||||
redis.enqueueBatches["flow:"+ch] = int(n)
|
||||
}
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.enqueueBatches) != 0 {
|
||||
t.Fatalf("rejected scope must not enqueue: %+v", redis.enqueueBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeDeduplicatesChannels(t *testing.T) {
|
||||
producer, redis := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), scopedMessage("scoped", "217", "217")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.enqueueBatches = map[string]int{}
|
||||
for _, ch := range []string{"217", "173", "189", "ghost-channel", "a", "b", "mobile-a", "mobile-b"} {
|
||||
key, _ := producer.store.RoundActiveKey("scoped", 1, ch)
|
||||
n, _ := producer.store.SizeKey(context.Background(), key)
|
||||
if n > 0 {
|
||||
redis.enqueueBatches["flow:"+ch] = int(n)
|
||||
}
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if redis.enqueueBatches["flow:217"] != 1 {
|
||||
t.Fatalf("enqueue batches = %+v, want 1 task despite a duplicated chanKey", redis.enqueueBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopeDigestIsOrderIndependent(t *testing.T) {
|
||||
first := &model.SyncScope{Channels: []string{"173", "217"}}
|
||||
second := &model.SyncScope{Channels: []string{"217", "173"}}
|
||||
// Normalization sorts the channels, so both requests produce one digest.
|
||||
normalizedFirst, err := model.NormalizeScope(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalizedSecond, err := model.NormalizeScope(second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scopeDigest(normalizedFirst) != scopeDigest(normalizedSecond) {
|
||||
t.Fatalf("digest depends on channel order: %s vs %s", scopeDigest(normalizedFirst), scopeDigest(normalizedSecond))
|
||||
}
|
||||
if scopeDigest(nil) != "" {
|
||||
t.Fatalf("full sync must have an empty digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAlreadyFinishedTreatsMissingRunAsFirstDelivery(t *testing.T) {
|
||||
producer, _ := newScopeProducer(t, []Card{
|
||||
{ID: 1, ChanKey: "217", TempKey: "sim_yd_v5", Operator: 1},
|
||||
})
|
||||
// The fake Redis returns no metadata, which must not be treated as finished.
|
||||
finished, err := producer.runAlreadyFinished(context.Background(), "fresh")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if finished {
|
||||
t.Fatal("a run without stored metadata must not be reported as finished")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (p *Producer) observe(key, stage string, pending int, added, ended bool) {
|
||||
if p.observation != nil {
|
||||
p.observation.Event(key, stage, pending, added, ended, time.Now())
|
||||
}
|
||||
}
|
||||
func (p *Producer) startObservation(ctx context.Context, run string, keys []string, started time.Time) func(bool) {
|
||||
tracker := p.config.SupplyObservation
|
||||
if tracker == nil {
|
||||
return func(bool) {}
|
||||
}
|
||||
o := supply.New(newOwner(), started, keys)
|
||||
p.observation = o
|
||||
readCtx, readCancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
meta, err := p.store.GetRunMeta(readCtx, run)
|
||||
readCancel()
|
||||
if err != nil {
|
||||
o.Partial()
|
||||
} else if meta.SupplyObservation != nil {
|
||||
o.Restore(*meta.SupplyObservation)
|
||||
}
|
||||
tracker.Put(run, o)
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
flush := func() {
|
||||
writeCtx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
encoded, e := o.Encode()
|
||||
if e == nil {
|
||||
var saved bool
|
||||
saved, e = p.store.SaveSupply(writeCtx, run, p.config.RunLockKey, p.owner, encoded)
|
||||
if !saved && e == nil {
|
||||
o.Partial()
|
||||
}
|
||||
}
|
||||
if e != nil {
|
||||
o.Partial()
|
||||
g.Log().Warningf(writeCtx, "event=supply_observation status=partial runId=%s error=%q", run, e.Error())
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
defer close(done)
|
||||
sampleTicker := time.NewTicker(time.Second)
|
||||
defer sampleTicker.Stop()
|
||||
flushTicker := time.NewTicker(10 * time.Second)
|
||||
defer flushTicker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
o.Partial()
|
||||
return
|
||||
case <-flushTicker.C:
|
||||
flush()
|
||||
case <-sampleTicker.C:
|
||||
views := o.Views()
|
||||
sampleCtx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
samples, e := p.store.ReadSupply(sampleCtx, run, keys, time.Now())
|
||||
cancel()
|
||||
now := time.Now()
|
||||
for _, key := range keys {
|
||||
s := samples[key]
|
||||
if e != nil || !tracker.LocalConsumer || !tracker.ConsumerActive.Load() {
|
||||
s.Known = false
|
||||
}
|
||||
o.Apply(key, views[key], s, now)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func(success bool) {
|
||||
close(stop)
|
||||
<-done
|
||||
if !success {
|
||||
o.Partial()
|
||||
}
|
||||
flush()
|
||||
tracker.Remove(run)
|
||||
p.observation = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type telemetryFailRedis struct{ zset.RedisClient }
|
||||
|
||||
func (r telemetryFailRedis) Eval(ctx context.Context, script string, n int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if strings.Contains(script, "'supplyObservation'") {
|
||||
return nil, fmt.Errorf("injected telemetry write failure")
|
||||
}
|
||||
return r.RedisClient.Eval(ctx, script, n, keys, args)
|
||||
}
|
||||
func TestSupplyFlushFailureDoesNotBlockProducerDone(t *testing.T) {
|
||||
client := testutil.Redis(t)
|
||||
store, err := zset.New(telemetryFailRedis{client}, "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 1)
|
||||
p := testProducer(t, scanner, store, Config{SupplyObservation: supply.NewTracker(true)})
|
||||
if err = p.Produce(context.Background(), model.TrafficUsageMonthlyStartMessage{RunID: "telemetry-failure"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMeta(t, store, "telemetry-failure", true, 1)
|
||||
}
|
||||
|
||||
func TestSupplyObservesEmptyTargetAndFinalFlush(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 2)
|
||||
scanner.channels = append(scanner.channels, channelMetadata{ChanKey: "empty", TempKey: "sim_yd_v5"})
|
||||
tracker := supply.NewTracker(true)
|
||||
tracker.ConsumerActive.Store(true)
|
||||
scanner.hook = func(ctx context.Context, key string, last int64) error {
|
||||
if key == "empty" {
|
||||
return sleepContext(ctx, 2200*time.Millisecond)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{SupplyObservation: tracker})
|
||||
if err := p.Produce(context.Background(), model.TrafficUsageMonthlyStartMessage{RunID: "supply-end", QueryMonth: "202610"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := store.GetRunMeta(context.Background(), "supply-end")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.ProducerDone || m.SupplyObservation == nil {
|
||||
t.Fatalf("missing final observation %+v", m)
|
||||
}
|
||||
snapshot := m.SupplyObservation
|
||||
if snapshot.Summary.Targets != 2 || snapshot.Summary.Confirmed != 0 {
|
||||
t.Fatalf("bad summary %+v", snapshot)
|
||||
}
|
||||
c := snapshot.Channels[1]
|
||||
if c.Key != "empty" || c.FirstEnqueueMS != nil || c.Termination != "empty_result" || c.Unconfirmed.TotalMS < 900 {
|
||||
t.Fatalf("bad empty target %+v", c)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
func newZombieTestProducer(t *testing.T) (*Producer, *zset.Store) {
|
||||
t.Helper()
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(&pageScanner{cards: nil}, store, Config{PageSize: 100, RoundRetryEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return producer, store
|
||||
}
|
||||
|
||||
// TestHasPendingTasksReclaimsZombieRun guards the "zombie run" regression.
|
||||
//
|
||||
// A producer that dies before MarkProducerDone (for example the container is
|
||||
// replaced mid-run) leaves its run in status=producing with producerDone=0.
|
||||
// completeRunIfIdle and advanceRunRound both require producerDone=1, so such a
|
||||
// run can never be closed, never leaves the active index, and never gets the
|
||||
// retention TTL. Treating it as backpressure rejected every later request
|
||||
// forever; merely warning about it leaked keys and an index entry indefinitely.
|
||||
func TestHasPendingTasksReclaimsZombieRun(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// A genuinely in-flight run (producerDone set, consumer working) must keep
|
||||
// providing backpressure.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "running-run", Status: "running", ProducerDone: true, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := producer.hasPendingTasks(ctx, "new-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !pending {
|
||||
t.Fatal("a running run must still provide backpressure")
|
||||
}
|
||||
if _, err := store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), "running-run"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A zombie run must not block a new request, and must be promoted so the
|
||||
// normal lifecycle can drain and close it.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err = producer.hasPendingTasks(ctx, "new-run-2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending {
|
||||
t.Fatal("a producing run with producerDone=0 must not block new requests (zombie run)")
|
||||
}
|
||||
|
||||
meta, err := store.GetRunMeta(ctx, "zombie-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.ProducerDone {
|
||||
t.Fatal("the zombie run must be promoted (producerDone=1) so it can be closed")
|
||||
}
|
||||
if meta.Status != "running" {
|
||||
t.Fatalf("promoted run status = %q, want running", meta.Status)
|
||||
}
|
||||
// Promotion makes the run closable, which is what applies the retention TTL
|
||||
// and releases the active index entry.
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "zombie-run"); err != nil || !completed {
|
||||
t.Fatalf("promoted run must be closable: completed=%v err=%v", completed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPendingTasksNeverReclaimsReplayRun guards the boundary: a replay target
|
||||
// is also created as "producing", but it is driven by the replay engine under
|
||||
// its own lock, so its producerDone is not the producer's to set.
|
||||
func TestHasPendingTasksNeverReclaimsReplayRun(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{
|
||||
RunID: "replay-run", Status: "producing", ProducerDone: false, CurrentRound: 1,
|
||||
SyncType: "replay", SourceRunID: "source-run", ReplayCycle: 1,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := producer.hasPendingTasks(ctx, "new-run"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "replay-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ProducerDone {
|
||||
t.Fatal("a replay run must not be promoted by the full-sync producer")
|
||||
}
|
||||
if meta.Status != "producing" {
|
||||
t.Fatalf("replay run status = %q, want producing", meta.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPendingTasksDropsStaleIndexEntry covers a second unrecoverable path:
|
||||
// the active index can list a run whose hash is already gone. Returning that as
|
||||
// an error made the request fail permanently (the consumer redelivers it
|
||||
// forever), which is the same "only fixable by hand-cleaning Redis" failure.
|
||||
func TestHasPendingTasksDropsStaleIndexEntry(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Index a run that has no metadata hash.
|
||||
if _, err := store.RedisClient().Do(ctx, "SADD", store.ActiveRunsKey(), "ghost-run"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := producer.hasPendingTasks(ctx, "new-run")
|
||||
if err != nil {
|
||||
t.Fatalf("a stale index entry must not fail the request: %v", err)
|
||||
}
|
||||
if pending {
|
||||
t.Fatal("a stale index entry must not provide backpressure")
|
||||
}
|
||||
// The stale entry is removed so it does not have to be handled again.
|
||||
members, err := store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, member := range members {
|
||||
if member == "ghost-run" {
|
||||
t.Fatal("the stale index entry must be removed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceCreatesRunAfterZombie covers the end-to-end path through Produce:
|
||||
// a zombie run must not stop a new run from being created.
|
||||
func TestProduceCreatesRunAfterZombie(t *testing.T) {
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
producer, err := New(&pageScanner{cards: []Card{
|
||||
{ID: 1, ChanKey: "channel-a", TempKey: "sim_yd_v5", Operator: 1},
|
||||
}}, store, Config{PageSize: 100, RoundRetryEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A zombie run from a crashed producer.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A new request must still produce its own run.
|
||||
if err := producer.Produce(ctx, scopedMessage("fresh-run", "channel-a")); err != nil {
|
||||
t.Fatalf("a zombie run must not stop a new run: %v", err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "fresh-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The producer enqueues into the active ZSet; ActiveCount is the count it
|
||||
// maintains. EnqueuedCount is an accounting field the producer does not set.
|
||||
if !meta.ProducerDone || meta.ActiveCount != 1 {
|
||||
t.Fatalf("fresh run metadata = %+v, want producerDone with 1 active task", meta)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// Pause the real producer after its first enqueue, before MarkProducerDone.
|
||||
type interruptedScanner struct {
|
||||
*datasetScanner
|
||||
paused chan struct{}
|
||||
}
|
||||
|
||||
func TestInterruptedRunReclamationGuards(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
for _, status := range []string{"producing", "running"} {
|
||||
for _, guard := range []string{"no_lock", "foreign_lock", "no_current_run", "same_run", "producer_done", "replay"} {
|
||||
t.Run(mode+"/"+status+"/"+guard, func(t *testing.T) {
|
||||
ctx := contextForTest(t)
|
||||
store := testStore(t)
|
||||
p := testProducer(t, scannerFor(nil, 0), store, Config{ScanMode: mode})
|
||||
meta := zset.RunMeta{RunID: "guarded", Status: status, ProducerDone: guard == "producer_done"}
|
||||
if guard == "replay" {
|
||||
meta.SyncType = "replay"
|
||||
}
|
||||
if err := store.CreateRun(ctx, meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := "fresh"
|
||||
owner := p.owner
|
||||
if guard == "same_run" {
|
||||
current = meta.RunID
|
||||
}
|
||||
if guard == "no_current_run" {
|
||||
current = ""
|
||||
}
|
||||
if guard == "foreign_lock" {
|
||||
owner = "other-owner"
|
||||
}
|
||||
if guard != "no_lock" {
|
||||
if ok, err := store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, owner, current, p.config.RunLockTTL); err != nil || !ok {
|
||||
t.Fatalf("acquire=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
key, _ := store.RunMetaKey(meta.RunID)
|
||||
before, err := store.RedisClient().Do(ctx, "HGETALL", key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := p.hasPendingTasks(ctx, current)
|
||||
if err != nil || pending != (guard != "same_run") {
|
||||
t.Fatalf("guard=%s pending=%t err=%v", guard, pending, err)
|
||||
}
|
||||
after, err := store.RedisClient().Do(ctx, "HGETALL", key)
|
||||
if err != nil || !reflect.DeepEqual(before.MapStrStr(), after.MapStrStr()) {
|
||||
t.Fatalf("guarded metadata changed: before=%v after=%v err=%v", before, after, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *interruptedScanner) Scan(ctx context.Context, last, upper int64, size int) ([]Card, error) {
|
||||
if last > 0 {
|
||||
close(s.paused)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return s.datasetScanner.Scan(ctx, last, upper, size)
|
||||
}
|
||||
|
||||
func (s *interruptedScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
if last > 0 {
|
||||
close(s.paused)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return s.datasetScanner.ScanChannel(ctx, key, last, size)
|
||||
}
|
||||
|
||||
func TestBlockedInterruptedProducerRecovery(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
for _, backlog := range []string{"none", "queued", "leased", "waiting"} {
|
||||
t.Run(mode+"/"+backlog, func(t *testing.T) {
|
||||
ctx := contextForTest(t)
|
||||
store := testStore(t)
|
||||
scanner := &interruptedScanner{datasetScanner: scannerFor([]Card{{ID: 1, ChanKey: "blocked"}, {ID: 2, ChanKey: "blocked"}}, 2), paused: make(chan struct{})}
|
||||
old := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 1})
|
||||
oldCtx, interrupt := context.WithCancel(ctx)
|
||||
defer interrupt()
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- old.Produce(oldCtx, scopedMessage("interrupted")) }()
|
||||
select {
|
||||
case <-scanner.paused:
|
||||
case err := <-result:
|
||||
t.Fatalf("producer exited before pause: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
claim := func(channel string) zset.ClaimedTask {
|
||||
t.Helper()
|
||||
tasks, err := store.ClaimRun(ctx, "interrupted", channel, time.Now().Add(time.Second), time.Now().Add(time.Minute), 1)
|
||||
if err != nil || len(tasks) != 1 {
|
||||
t.Fatalf("claim %s=%v err=%v", channel, tasks, err)
|
||||
}
|
||||
return tasks[0]
|
||||
}
|
||||
if n, err := store.BlockRunScopedAsFailed(ctx, claim("blocked"), zset.TaskOutcome{Code: 797, ErrCode: "S00004"}); err != nil || n != 1 {
|
||||
t.Fatalf("blocked=%d err=%v", n, err)
|
||||
}
|
||||
var remaining zset.ClaimedTask
|
||||
if backlog != "none" {
|
||||
tasks := []zset.Task{{TaskID: "remaining", RunID: "interrupted", ChanKey: "other"}}
|
||||
if n, err := store.EnqueueRunBatch(ctx, "interrupted", "other", tasks, time.Now()); err != nil || n != 1 {
|
||||
t.Fatalf("enqueue=%d err=%v", n, err)
|
||||
}
|
||||
if backlog != "queued" {
|
||||
remaining = claim("other")
|
||||
}
|
||||
if backlog == "waiting" {
|
||||
if ok, err := store.FinalizeRunTask(ctx, remaining, zset.TaskOutcome{Retry: true}); err != nil || !ok {
|
||||
t.Fatalf("defer=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
before, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || before.Status != "running" || before.ProducerDone || before.FailedCount != 1 {
|
||||
t.Fatalf("blocked metadata=%+v err=%v", before, err)
|
||||
}
|
||||
fresh := testProducer(t, scannerFor([]Card{{ID: 3, ChanKey: "fresh"}}, 3), store, Config{ScanMode: mode})
|
||||
if err := fresh.Produce(ctx, scopedMessage("fresh")); !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("live producer lock must prevent reclamation: %v", err)
|
||||
}
|
||||
requireMeta(t, store, "interrupted", false, before.ActiveCount)
|
||||
interrupt()
|
||||
if err := <-result; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("interruption=%v", err)
|
||||
}
|
||||
lock, err := store.RedisClient().Do(ctx, "EXISTS", old.config.RunLockKey)
|
||||
if err != nil || lock.Int() != 0 {
|
||||
t.Fatalf("old lock=%v err=%v", lock, err)
|
||||
}
|
||||
err = fresh.Produce(ctx, scopedMessage("fresh"))
|
||||
if backlog == "none" && err != nil || backlog != "none" && !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("fresh start with %s backlog: %v", backlog, err)
|
||||
}
|
||||
after, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || !after.ProducerDone || after.Status != "running" || after.ActiveCount != before.ActiveCount || after.InflightCount != before.InflightCount || after.NextRoundCount != before.NextRoundCount || after.FailedCount != before.FailedCount {
|
||||
t.Fatalf("reclaimed metadata=%+v before=%+v err=%v", after, before, err)
|
||||
}
|
||||
if backlog != "none" {
|
||||
if _, err := store.GetRunMeta(ctx, "fresh"); !errors.Is(err, zset.ErrRunNotFound) {
|
||||
t.Fatalf("backlog must prevent creating fresh run: %v", err)
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "interrupted"); err != nil || complete {
|
||||
t.Fatalf("premature completion=%t err=%v", complete, err)
|
||||
}
|
||||
if backlog == "waiting" {
|
||||
if advanced, err := store.AdvanceRunRound(ctx, "interrupted"); err != nil || !advanced {
|
||||
t.Fatalf("advance=%t err=%v", advanced, err)
|
||||
}
|
||||
}
|
||||
if backlog != "leased" {
|
||||
remaining = claim("other")
|
||||
}
|
||||
if ok, err := store.FinalizeRunTask(ctx, remaining, zset.TaskOutcome{Success: true}); err != nil || !ok {
|
||||
t.Fatalf("drain=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "interrupted", time.Hour); err != nil || !complete {
|
||||
t.Fatalf("normal completion=%t err=%v", complete, err)
|
||||
}
|
||||
closed, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || closed.Status != "completed_with_failures" || closed.FailedCount != 1 || closed.ActiveCount != 0 || closed.InflightCount != 0 || closed.NextRoundCount != 0 {
|
||||
t.Fatalf("closed metadata=%+v err=%v", closed, err)
|
||||
}
|
||||
metaKey, _ := store.RunMetaKey("interrupted")
|
||||
failedKey, _ := store.FailedKey("interrupted")
|
||||
stateKey, _ := store.TaskStateKey("interrupted")
|
||||
channelsKey, _ := store.RunChannelsKey("interrupted")
|
||||
for _, key := range []string{metaKey, failedKey, stateKey, channelsKey} {
|
||||
ttl, err := store.RedisClient().Do(ctx, "PTTL", key)
|
||||
if err != nil || ttl.Int64() <= 0 || ttl.Int64() > time.Hour.Milliseconds() {
|
||||
t.Fatalf("retention %s=%v err=%v", key, ttl, err)
|
||||
}
|
||||
}
|
||||
if backlog != "none" {
|
||||
if err := fresh.Produce(ctx, scopedMessage("fresh")); err != nil {
|
||||
t.Fatalf("fresh start after draining: %v", err)
|
||||
}
|
||||
}
|
||||
requireMeta(t, store, "fresh", true, 1)
|
||||
active, err := store.ActiveRuns(ctx)
|
||||
if err != nil || len(active) != 1 || active[0] != "fresh" {
|
||||
t.Fatalf("only closed run must leave active index: %v err=%v", active, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,491 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
var errProducerDone = gerror.New("run production already closed")
|
||||
|
||||
func producerStateError(state zset.ProducerState) error {
|
||||
if state.Status == "completed" || state.Status == "completed_with_failures" {
|
||||
return ErrRunFinished
|
||||
}
|
||||
if state.ProducerDone {
|
||||
return errProducerDone
|
||||
}
|
||||
if state.Status != "producing" && state.Status != "running" {
|
||||
return gerror.Newf("unexpected producer run status %q", state.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// produceChannels owns all per-attempt coordination. There is one worker per
|
||||
// target, NOT a pool of four long-lived channel workers. SQL tokens and the
|
||||
// cancellable admission gate never span sleeps or goroutine joins.
|
||||
func (p *Producer) produceChannels(ctx context.Context, message model.TrafficUsageMonthlyHistoryStartMessage, scope *model.SyncScope, scanUpperID int64, startedAt time.Time, total *productionStats) error {
|
||||
scanner := p.scanner.(channelScanner)
|
||||
metadata, err := scanner.ListChannels(ctx)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "initialize channel scan")
|
||||
}
|
||||
state, err := p.store.ReadProducerState(ctx, message.RunID, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return err
|
||||
}
|
||||
targets := make(map[string]channelMetadata, len(metadata))
|
||||
known := make(map[string]bool, len(metadata))
|
||||
for _, channel := range metadata {
|
||||
if channel.ChanKey == "" || !scope.Allows(channel.ChanKey) {
|
||||
continue
|
||||
}
|
||||
if known[channel.ChanKey] {
|
||||
return gerror.Newf("duplicate channel metadata %q", channel.ChanKey)
|
||||
}
|
||||
known[channel.ChanKey] = true
|
||||
reason := ""
|
||||
if channel.SyncEnabled {
|
||||
reason = "sync_disabled"
|
||||
} else if channel.TempKey == "" {
|
||||
reason = "missing_channel_template"
|
||||
}
|
||||
if reason != "" {
|
||||
total.skippedChannels++
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_channel status=skipped runId=%s chanKey=%s reason=%s skipUnit=channel", message.RunID, channel.ChanKey, reason)
|
||||
continue
|
||||
}
|
||||
targets[channel.ChanKey] = channel
|
||||
}
|
||||
if scope != nil {
|
||||
for _, chanKey := range scope.Channels {
|
||||
if !known[chanKey] {
|
||||
total.skippedChannels++
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_channel status=skipped runId=%s chanKey=%s reason=unknown_channel skipUnit=channel", message.RunID, chanKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_channel_scan status=started runId=%s targetChannels=%d scanUpperID=%d scanConcurrency=%d pageSize=%d skipUnit=channel", message.RunID, len(targets), scanUpperID, p.config.ScanConcurrency, p.config.PageSize)
|
||||
keys := make([]string, 0, len(targets))
|
||||
for key := range targets {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
finishObservation := p.startObservation(ctx, message.RunID, keys, startedAt)
|
||||
defer finishObservation()
|
||||
var (
|
||||
workers sync.WaitGroup
|
||||
slots = make(chan struct{}, p.config.ScanConcurrency)
|
||||
gate = make(chan struct{}, 1)
|
||||
metrics = newScanActivity()
|
||||
results = make([]*channelScanStats, len(keys))
|
||||
)
|
||||
pool, hasPool := scanner.(interface {
|
||||
poolWait(context.Context) (int64, int64)
|
||||
})
|
||||
var poolCount, poolNanos int64
|
||||
if hasPool {
|
||||
poolCount, poolNanos = pool.poolWait(ctx)
|
||||
}
|
||||
workCtx, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
for i, key := range keys {
|
||||
stats := &channelScanStats{chanKey: key, lastSummary: time.Now()}
|
||||
results[i] = stats
|
||||
workers.Add(1)
|
||||
go func(channel channelMetadata, stats *channelScanStats) {
|
||||
defer workers.Done()
|
||||
scanErr := p.scanOneChannel(workCtx, scanner, message, channel, scanUpperID, startedAt, slots, gate, metrics, stats)
|
||||
if scanErr != nil {
|
||||
cancel(scanErr)
|
||||
}
|
||||
reason := stats.stopReason
|
||||
if reason == "" {
|
||||
reason = "scan_complete"
|
||||
}
|
||||
if scanErr != nil {
|
||||
reason = "cancelled"
|
||||
}
|
||||
p.observe(channel.ChanKey, reason, 0, false, true)
|
||||
stats.log(workCtx, message.RunID, "ended", startedAt, scanErr)
|
||||
}(targets[key], stats)
|
||||
}
|
||||
workers.Wait()
|
||||
var summary channelScanStats
|
||||
for _, stats := range results {
|
||||
total.pages += stats.scanPages
|
||||
total.scanned += stats.eligible
|
||||
total.eligible += stats.eligible
|
||||
total.enqueued += stats.added
|
||||
total.stoppedTasks += stats.rejected
|
||||
if stats.eligible > 0 {
|
||||
total.channels[stats.chanKey] = struct{}{}
|
||||
}
|
||||
summary.merge(stats)
|
||||
}
|
||||
metrics.log(ctx, message.RunID, &summary)
|
||||
if hasPool {
|
||||
count, nanos := pool.poolWait(ctx)
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_scan_pool runId=%s sharedPoolWaitCountDelta=%d sharedPoolWaitMsDelta=%d attribution=shared_pool", message.RunID, count-poolCount, (nanos-poolNanos)/int64(time.Millisecond))
|
||||
}
|
||||
return context.Cause(workCtx)
|
||||
}
|
||||
|
||||
func acquireContext(ctx context.Context, token chan struct{}) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case token <- struct{}{}:
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-token
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) scanOneChannel(ctx context.Context, scanner channelScanner, message model.TrafficUsageMonthlyHistoryStartMessage, channel channelMetadata, scanUpperID int64, startedAt time.Time, slots, gate chan struct{}, metrics *scanActivity, stats *channelScanStats) error {
|
||||
for stats.cursor < scanUpperID {
|
||||
// Detect a blocked/closed run before querying, including empty channels.
|
||||
p.observe(channel.ChanKey, "redis_read", 0, false, false)
|
||||
state, err := p.store.ReadProducerState(ctx, message.RunID, channel.ChanKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Blocked {
|
||||
// No page was read: the unknown unscanned remainder has no card
|
||||
// count. Never turn it into a fabricated success/failure total.
|
||||
stats.stopReason = "blocked_unscanned_remainder"
|
||||
return nil
|
||||
}
|
||||
p.observe(channel.ChanKey, "sql_slot", 0, false, false)
|
||||
waitStart := time.Now()
|
||||
if err = acquireContext(ctx, slots); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.slotWait += time.Since(waitStart)
|
||||
metrics.begin()
|
||||
p.observe(channel.ChanKey, "sql_query", 0, false, false)
|
||||
queryStart := time.Now()
|
||||
cards, scanErr := scanner.ScanChannel(ctx, channel.ChanKey, stats.cursor, p.config.PageSize)
|
||||
elapsed := time.Since(queryStart)
|
||||
metrics.end()
|
||||
<-slots // release immediately after results have been materialized
|
||||
stats.sqlTime += elapsed
|
||||
stats.sqlLatency.observe(elapsed)
|
||||
stats.scanPages++
|
||||
if scanErr != nil {
|
||||
return gerror.Wrapf(scanErr, "scan channel %s after %d", channel.ChanKey, stats.cursor)
|
||||
}
|
||||
if len(cards) > p.config.PageSize {
|
||||
return gerror.New("scanner exceeded page size")
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
stats.stopReason = "empty_result"
|
||||
return nil
|
||||
}
|
||||
var (
|
||||
pageCursor = stats.cursor
|
||||
overUpper bool
|
||||
tasks = make([]zset.Task, 0, len(cards))
|
||||
)
|
||||
for i, card := range cards {
|
||||
if card.ChanKey != channel.ChanKey || card.ID <= pageCursor {
|
||||
return gerror.New("channel scanner returned unordered or foreign cards")
|
||||
}
|
||||
if card.ID > scanUpperID {
|
||||
stats.overUpper += int64(len(cards) - i)
|
||||
overUpper = true
|
||||
break
|
||||
}
|
||||
pageCursor = card.ID
|
||||
tasks = append(tasks, zset.Task{TaskID: zset.StableTaskID(message.RunID, message.Operation, card.ID), RunID: message.RunID, Operation: message.Operation, QueryMonth: message.QueryMonth, ChanKey: channel.ChanKey, TempKey: channel.TempKey, Operator: channel.Operator, CardID: card.ID, ICCID: card.ICCID, MSISDN: card.MSISDN, IMSI: card.IMSI, PackageCode: card.PackageCode})
|
||||
}
|
||||
p.observe(channel.ChanKey, "page_ready", len(tasks), false, false)
|
||||
stats.eligible += int64(len(tasks))
|
||||
blocked, err := p.admitChannelPage(ctx, message.RunID, tasks, startedAt, gate, stats)
|
||||
if err != nil {
|
||||
return err // page cursor is not committed, even after a partial write
|
||||
}
|
||||
if blocked {
|
||||
stats.stopReason = "blocked"
|
||||
return nil
|
||||
}
|
||||
stats.cursor = pageCursor
|
||||
stats.progress(ctx, message.RunID, startedAt)
|
||||
if overUpper || stats.cursor >= scanUpperID {
|
||||
stats.stopReason = "upper_bound"
|
||||
return nil
|
||||
}
|
||||
// Do not use a short page as EOF: query-time eligibility may change.
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// admitChannelPage admits only new TaskIDs, in capacity-sized slices. Each
|
||||
// worker retains at most this page. No capacity reservation survives the gate.
|
||||
func (p *Producer) admitChannelPage(ctx context.Context, runID string, tasks []zset.Task, startedAt time.Time, gate chan struct{}, stats *channelScanStats) (bool, error) {
|
||||
for len(tasks) > 0 {
|
||||
p.observe(stats.chanKey, "admission_gate", len(tasks), false, false)
|
||||
gateStart := time.Now()
|
||||
if err := acquireContext(ctx, gate); err != nil {
|
||||
return false, err
|
||||
}
|
||||
stats.gateWait += time.Since(gateStart)
|
||||
blocked, waitKind, remaining, err := p.admitChannelSlice(ctx, runID, tasks, startedAt, stats)
|
||||
<-gate
|
||||
if err != nil || blocked {
|
||||
return blocked, err
|
||||
}
|
||||
tasks = remaining
|
||||
p.observe(stats.chanKey, "page_ready", len(tasks), false, false)
|
||||
if waitKind != "" {
|
||||
p.observe(stats.chanKey, waitKind+"_capacity", len(tasks), false, false)
|
||||
waitStart := time.Now()
|
||||
err = sleepContext(ctx, p.config.PollInterval)
|
||||
if waitKind == "global" {
|
||||
stats.globalWait += time.Since(waitStart)
|
||||
} else {
|
||||
stats.channelWait += time.Since(waitStart)
|
||||
}
|
||||
stats.progress(ctx, runID, startedAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, ctx.Err()
|
||||
}
|
||||
|
||||
// Caller holds the short admission gate; there is no sleep in this method.
|
||||
func (p *Producer) admitChannelSlice(ctx context.Context, runID string, tasks []zset.Task, startedAt time.Time, stats *channelScanStats) (blocked bool, waitKind string, remaining []zset.Task, err error) {
|
||||
p.observe(stats.chanKey, "redis_read", len(tasks), false, false)
|
||||
readStart := time.Now()
|
||||
state, err := p.store.ReadProducerState(ctx, runID, stats.chanKey)
|
||||
if err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
if err = producerStateError(state); err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
existing, err := p.store.ExistingRunTasks(ctx, runID, tasks)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if err != nil {
|
||||
return false, "", tasks, err
|
||||
}
|
||||
pending := tasks[:0]
|
||||
for i, exists := range existing {
|
||||
if exists {
|
||||
stats.duplicates++
|
||||
} else {
|
||||
pending = append(pending, tasks[i])
|
||||
}
|
||||
}
|
||||
tasks = pending
|
||||
p.observe(stats.chanKey, "redis_read", len(tasks), false, false)
|
||||
if state.Blocked {
|
||||
stats.rejected += int64(len(tasks))
|
||||
return true, "", nil, nil
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return false, "", nil, nil
|
||||
}
|
||||
space := int64(len(tasks))
|
||||
if limit := p.config.GlobalHighWaterMark; limit > 0 {
|
||||
space = min(space, limit-state.ActiveCount) // inflight is already included
|
||||
if space <= 0 {
|
||||
return false, "global", tasks, nil
|
||||
}
|
||||
}
|
||||
if limit := p.config.ChannelHighWaterMark; limit > 0 {
|
||||
key, keyErr := p.store.RoundActiveKey(runID, state.CurrentRound, stats.chanKey)
|
||||
if keyErr != nil {
|
||||
return false, "", tasks, keyErr
|
||||
}
|
||||
readStart = time.Now()
|
||||
size, sizeErr := p.store.SizeKey(ctx, key)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if sizeErr != nil {
|
||||
return false, "", tasks, sizeErr
|
||||
}
|
||||
space = min(space, limit-size)
|
||||
if space <= 0 {
|
||||
return false, "channel", tasks, nil
|
||||
}
|
||||
}
|
||||
batch := tasks[:int(space)]
|
||||
p.observe(stats.chanKey, "redis_write", len(tasks), false, false)
|
||||
enqueueStart := time.Now()
|
||||
added, err := p.store.EnqueueRunBatch(ctx, runID, stats.chanKey, batch, time.Now())
|
||||
stats.enqueueTime += time.Since(enqueueStart)
|
||||
stats.enqueueBatches++
|
||||
if err != nil {
|
||||
return false, "", tasks, err // uncertain responses require idempotent rescan
|
||||
}
|
||||
if added < 0 || added > len(batch) {
|
||||
return false, "", tasks, gerror.New("invalid enqueue count")
|
||||
}
|
||||
stats.added += int64(added)
|
||||
p.observe(stats.chanKey, "redis_write", len(tasks)-len(batch), added > 0, false)
|
||||
if added > 0 && !stats.firstEnqueueRecorded {
|
||||
stats.firstEnqueueRecorded = true
|
||||
stats.firstEnqueue = time.Since(startedAt)
|
||||
stats.log(ctx, runID, "first_enqueue", startedAt, nil)
|
||||
}
|
||||
if added < len(batch) {
|
||||
readStart = time.Now()
|
||||
confirmed, checkErr := p.store.ExistingRunTasks(ctx, runID, batch)
|
||||
if checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
state, checkErr = p.store.ReadProducerState(ctx, runID, stats.chanKey)
|
||||
stats.stateTime += time.Since(readStart)
|
||||
if checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
if checkErr = producerStateError(state); checkErr != nil {
|
||||
return false, "", tasks, checkErr
|
||||
}
|
||||
present := 0
|
||||
for _, exists := range confirmed {
|
||||
if exists {
|
||||
present++
|
||||
}
|
||||
}
|
||||
if present < added {
|
||||
return false, "", tasks, gerror.New("enqueue count exceeds confirmed task states")
|
||||
}
|
||||
stats.duplicates += int64(present - added)
|
||||
if state.Blocked {
|
||||
stats.rejected += int64(len(tasks) - present)
|
||||
return true, "", nil, nil
|
||||
}
|
||||
if present != len(batch) {
|
||||
return false, "", tasks, gerror.Newf("unexplained enqueue rejection: added=%d submitted=%d confirmed=%d", added, len(batch), present)
|
||||
}
|
||||
}
|
||||
return false, "", tasks[len(batch):], nil
|
||||
}
|
||||
|
||||
// Retained only to keep errors.Is semantics visible at the call site.
|
||||
func isProductionClosed(err error) bool { return errors.Is(err, errProducerDone) }
|
||||
|
||||
func durationMS(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) }
|
||||
|
||||
// latencyHistogram is fixed-size. Reported percentiles are bucket upper bounds,
|
||||
// never exact samples or a server-side SLA; the last bucket is unbounded.
|
||||
var latencyBounds = [...]time.Duration{time.Millisecond, 2 * time.Millisecond, 5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond, 50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond, 500 * time.Millisecond, time.Second, 2 * time.Second, 5 * time.Second, 10 * time.Second, 30 * time.Second, time.Minute}
|
||||
|
||||
type latencyHistogram struct{ counts [16]int64 }
|
||||
|
||||
func (h *latencyHistogram) observe(d time.Duration) {
|
||||
i := 0
|
||||
for i < len(latencyBounds) && d > latencyBounds[i] {
|
||||
i++
|
||||
}
|
||||
h.counts[i]++
|
||||
}
|
||||
func (h *latencyHistogram) percentile(percent int64) string {
|
||||
var total, cumulative int64
|
||||
for _, count := range h.counts {
|
||||
total += count
|
||||
}
|
||||
if total == 0 {
|
||||
return "n/a"
|
||||
}
|
||||
target := (total*percent + 99) / 100
|
||||
for i, count := range h.counts {
|
||||
cumulative += count
|
||||
if cumulative >= target {
|
||||
if i == len(latencyBounds) {
|
||||
return ">60000"
|
||||
}
|
||||
return fmt.Sprintf("%.0f", durationMS(latencyBounds[i]))
|
||||
}
|
||||
}
|
||||
return "n/a"
|
||||
}
|
||||
|
||||
type channelScanStats struct {
|
||||
firstEnqueueRecorded bool
|
||||
chanKey, stopReason string
|
||||
cursor, scanPages, eligible, overUpper, enqueueBatches, added, duplicates, rejected int64
|
||||
sqlTime, slotWait, gateWait, globalWait, channelWait, stateTime, enqueueTime, firstEnqueue time.Duration
|
||||
sqlLatency latencyHistogram
|
||||
lastSummary time.Time
|
||||
}
|
||||
|
||||
func (s *channelScanStats) log(ctx context.Context, runID, status string, startedAt time.Time, err error) {
|
||||
errorText := ""
|
||||
if err != nil {
|
||||
errorText = err.Error()
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_channel_scan status=%s runId=%s chanKey=%s cursor=%d scanPages=%d eligible=%d overUpper=%d enqueueBatches=%d added=%d duplicates=%d rejected=%d sqlMs=%.3f sqlP50UpperMs=%s sqlP95UpperMs=%s slotWaitMs=%.3f admissionWaitMs=%.3f globalCapacityWaitMs=%.3f channelCapacityWaitMs=%.3f stateReadMs=%.3f enqueueMs=%.3f firstEnqueueMs=%.3f wallMs=%d stopReason=%s error=%q", status, runID, s.chanKey, s.cursor, s.scanPages, s.eligible, s.overUpper, s.enqueueBatches, s.added, s.duplicates, s.rejected, durationMS(s.sqlTime), s.sqlLatency.percentile(50), s.sqlLatency.percentile(95), durationMS(s.slotWait), durationMS(s.gateWait), durationMS(s.globalWait), durationMS(s.channelWait), durationMS(s.stateTime), durationMS(s.enqueueTime), durationMS(s.firstEnqueue), time.Since(startedAt).Milliseconds(), s.stopReason, errorText)
|
||||
}
|
||||
func (s *channelScanStats) progress(ctx context.Context, runID string, startedAt time.Time) {
|
||||
if time.Since(s.lastSummary) >= time.Minute {
|
||||
s.lastSummary = time.Now()
|
||||
s.log(ctx, runID, "progress", startedAt, nil)
|
||||
}
|
||||
}
|
||||
func (s *channelScanStats) merge(other *channelScanStats) {
|
||||
s.scanPages += other.scanPages
|
||||
s.eligible += other.eligible
|
||||
s.overUpper += other.overUpper
|
||||
s.enqueueBatches += other.enqueueBatches
|
||||
s.added += other.added
|
||||
s.duplicates += other.duplicates
|
||||
s.rejected += other.rejected
|
||||
s.sqlTime += other.sqlTime
|
||||
s.slotWait += other.slotWait
|
||||
s.gateWait += other.gateWait
|
||||
s.globalWait += other.globalWait
|
||||
s.channelWait += other.channelWait
|
||||
s.stateTime += other.stateTime
|
||||
s.enqueueTime += other.enqueueTime
|
||||
for i, count := range other.sqlLatency.counts {
|
||||
s.sqlLatency.counts[i] += count
|
||||
}
|
||||
}
|
||||
|
||||
type scanActivity struct {
|
||||
mu sync.Mutex
|
||||
started, changed time.Time
|
||||
running, peak int
|
||||
area time.Duration
|
||||
}
|
||||
|
||||
func newScanActivity() *scanActivity {
|
||||
now := time.Now()
|
||||
return &scanActivity{started: now, changed: now}
|
||||
}
|
||||
func (s *scanActivity) change(delta int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := time.Now()
|
||||
s.area += time.Duration(s.running) * now.Sub(s.changed)
|
||||
s.changed = now
|
||||
s.running += delta
|
||||
s.peak = max(s.peak, s.running)
|
||||
}
|
||||
func (s *scanActivity) begin() { s.change(1) }
|
||||
func (s *scanActivity) end() { s.change(-1) }
|
||||
func (s *scanActivity) log(ctx context.Context, runID string, stats *channelScanStats) {
|
||||
wall := time.Since(s.started)
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_scan_summary runId=%s wallMs=%d sqlInFlightPeak=%d sqlInFlightAvg=%.4f scanPages=%d enqueueBatches=%d eligible=%d overUpper=%d added=%d duplicates=%d rejected=%d sqlCumulativeMs=%.3f sqlP50UpperMs=%s sqlP95UpperMs=%s slotWaitCumulativeMs=%.3f admissionWaitCumulativeMs=%.3f globalCapacityWaitCumulativeMs=%.3f channelCapacityWaitCumulativeMs=%.3f stateReadCumulativeMs=%.3f enqueueCumulativeMs=%.3f", runID, wall.Milliseconds(), s.peak, float64(s.area)/float64(wall), stats.scanPages, stats.enqueueBatches, stats.eligible, stats.overUpper, stats.added, stats.duplicates, stats.rejected, durationMS(stats.sqlTime), stats.sqlLatency.percentile(50), stats.sqlLatency.percentile(95), durationMS(stats.slotWait), durationMS(stats.gateWait), durationMS(stats.globalWait), durationMS(stats.channelWait), durationMS(stats.stateTime), durationMS(stats.enqueueTime))
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
)
|
||||
|
||||
// datasetScanner applies the SQL eligibility predicate before LIMIT; metadata
|
||||
// eligibility is applied by the producer. Optional hooks simulate SQL latency,
|
||||
// live additions and failures without accessing an application database.
|
||||
type datasetScanner struct {
|
||||
cards []Card
|
||||
channels []channelMetadata
|
||||
upper int64
|
||||
hook func(context.Context, string, int64) error
|
||||
queries atomic.Int32
|
||||
lists atomic.Int32
|
||||
inflight atomic.Int32
|
||||
peak atomic.Int32
|
||||
}
|
||||
|
||||
func (s *datasetScanner) SnapshotMaxID(context.Context) (int64, error) { return s.upper, nil }
|
||||
func (s *datasetScanner) ListChannels(context.Context) ([]channelMetadata, error) {
|
||||
s.lists.Add(1)
|
||||
return s.channels, nil
|
||||
}
|
||||
func (s *datasetScanner) Scan(ctx context.Context, last, upper int64, size int) ([]Card, error) {
|
||||
var rows []Card
|
||||
for _, card := range s.cards {
|
||||
if card.ID > last && card.ID <= upper {
|
||||
for _, channel := range s.channels {
|
||||
if channel.ChanKey == card.ChanKey {
|
||||
card.TempKey = channel.TempKey
|
||||
card.SyncEnabled = channel.SyncEnabled
|
||||
card.Operator = channel.Operator
|
||||
}
|
||||
}
|
||||
rows = append(rows, card)
|
||||
if len(rows) == size {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
func (s *datasetScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
s.queries.Add(1)
|
||||
running := s.inflight.Add(1)
|
||||
defer s.inflight.Add(-1)
|
||||
for {
|
||||
old := s.peak.Load()
|
||||
if running <= old || s.peak.CompareAndSwap(old, running) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if s.hook != nil {
|
||||
if err := s.hook(ctx, key, last); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var rows []Card
|
||||
for _, card := range s.cards {
|
||||
if card.ChanKey == key && card.ID > last {
|
||||
rows = append(rows, card)
|
||||
if len(rows) == size {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
func scannerFor(cards []Card, upper int64) *datasetScanner {
|
||||
keys := map[string]bool{}
|
||||
s := &datasetScanner{cards: cards, upper: upper}
|
||||
for _, card := range cards {
|
||||
if card.ChanKey != "" && !keys[card.ChanKey] {
|
||||
keys[card.ChanKey] = true
|
||||
s.channels = append(s.channels, channelMetadata{ChanKey: card.ChanKey, TempKey: "sim_yd_v5", Operator: 1})
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
func testStore(t *testing.T) *zset.Store {
|
||||
t.Helper()
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
func testProducer(t *testing.T, scanner CardScanner, store *zset.Store, config Config) *Producer {
|
||||
t.Helper()
|
||||
config.RoundRetryEnabled = true
|
||||
if config.ScanMode == "" {
|
||||
config.ScanMode = "channel"
|
||||
}
|
||||
p, err := New(scanner, store, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
func contextForTest(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
return ctx
|
||||
}
|
||||
func taskIDs(t *testing.T, store *zset.Store, run string) []string {
|
||||
t.Helper()
|
||||
key, _ := store.TaskStateKey(run)
|
||||
result, err := store.RedisClient().Do(context.Background(), "HKEYS", key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := result.Strings()
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
func requireMeta(t *testing.T, store *zset.Store, run string, done bool, active int64) {
|
||||
t.Helper()
|
||||
meta, err := store.GetRunMeta(context.Background(), run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ProducerDone != done || meta.ActiveCount != active {
|
||||
t.Fatalf("meta=%+v, want done=%t active=%d", meta, done, active)
|
||||
}
|
||||
}
|
||||
func waitUntil(t *testing.T, ctx context.Context, condition func() bool) {
|
||||
t.Helper()
|
||||
for !condition() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelModeSettings(t *testing.T) {
|
||||
store, _ := zset.New(&producerRedis{}, "flow:")
|
||||
p, err := New(&fakeScanner{}, store, Config{})
|
||||
if err != nil || p.config.ScanMode != "serial" || p.config.ScanConcurrency != 4 {
|
||||
t.Fatalf("defaults: p=%+v err=%v", p, err)
|
||||
}
|
||||
for _, config := range []Config{{ScanMode: "parallel"}, {ScanConcurrency: -1}, {ScanMode: "channel"}} {
|
||||
if _, err = New(&fakeScanner{}, store, config); err == nil {
|
||||
t.Fatalf("accepted invalid config/capability %+v", config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanFiniteMemoryBoundary(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
upper int64
|
||||
cards []Card
|
||||
ids []int64
|
||||
queries int32
|
||||
}{
|
||||
{"zero upper", 0, []Card{{ID: 1, ChanKey: "a"}}, nil, 0},
|
||||
{"exact page", 4, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}}, []int64{1, 2, 3, 4}, 2},
|
||||
{"tail then empty", 9, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}}, []int64{1, 2, 3}, 3},
|
||||
{"first outside", 3, []Card{{ID: 4, ChanKey: "a"}, {ID: 5, ChanKey: "a"}}, nil, 1},
|
||||
{"mixed final page", 3, []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}, {ID: 5, ChanKey: "a"}}, []int64{1, 2, 3}, 2},
|
||||
{"empty channel", 9, nil, nil, 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor(test.cards, test.upper)
|
||||
if len(scanner.channels) == 0 {
|
||||
scanner.channels = []channelMetadata{{ChanKey: "a", TempKey: "sim_yd_v5"}}
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("boundary")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []string
|
||||
for _, id := range test.ids {
|
||||
want = append(want, zset.StableTaskID("boundary", operation, id))
|
||||
}
|
||||
sort.Strings(want)
|
||||
got := taskIDs(t, store, "boundary")
|
||||
if len(got) != len(want) || (len(want) > 0 && !reflect.DeepEqual(got, want)) {
|
||||
t.Fatalf("got=%v want=%v", got, want)
|
||||
}
|
||||
if scanner.queries.Load() != test.queries {
|
||||
t.Fatalf("queries=%d want=%d", scanner.queries.Load(), test.queries)
|
||||
}
|
||||
requireMeta(t, store, "boundary", true, int64(len(want)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanLiveHigherIDsDoNotExtendRun(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}}, 3)
|
||||
scanner.hook = func(_ context.Context, _ string, last int64) error {
|
||||
if last > 0 {
|
||||
scanner.cards = append(scanner.cards, Card{ID: 100 + last, ChanKey: "a"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("live")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(taskIDs(t, store, "live")) != 3 || scanner.queries.Load() != 2 {
|
||||
t.Fatal("higher IDs extended finite range")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanSerialTaskIDSetParity(t *testing.T) {
|
||||
for _, scope := range [][]string{nil, {"a", "b"}, {"disabled", "unknown"}} {
|
||||
t.Run(fmt.Sprint(scope), func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
cards := []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "b"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "disabled"}, {ID: 5, ChanKey: "no-template"}, {ID: 6, ChanKey: "unknown"}, {ID: 7, ChanKey: "b"}, {ID: 8, ChanKey: "a"}}
|
||||
var sets [][]string
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
scanner := scannerFor(cards, 7)
|
||||
scanner.channels = []channelMetadata{{ChanKey: "a", TempKey: "sim_yd_v5", Operator: 1}, {ChanKey: "b", TempKey: "sim_dx_v1", Operator: 2}, {ChanKey: "disabled", TempKey: "sim_yd_v5", SyncEnabled: true}, {ChanKey: "no-template"}}
|
||||
p := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 2})
|
||||
run := "parity-" + mode
|
||||
if err := p.Produce(contextForTest(t), scopedMessage(run, scope...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := taskIDs(t, store, run)
|
||||
for i, id := range ids {
|
||||
ids[i] = strings.TrimPrefix(id, run+":")
|
||||
}
|
||||
sets = append(sets, ids)
|
||||
if mode == "channel" && scanner.lists.Load() != 1 {
|
||||
t.Fatal("metadata not loaded once")
|
||||
}
|
||||
_, _ = store.RedisClient().Do(context.Background(), "SREM", store.ActiveRunsKey(), run)
|
||||
}
|
||||
if !reflect.DeepEqual(sets[0], sets[1]) {
|
||||
t.Fatalf("serial=%v channel=%v", sets[0], sets[1])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelSQLSlotsReleasedWhileChannelsWait(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
var cards []Card
|
||||
for i := 0; i < 9; i++ {
|
||||
cards = append(cards, Card{ID: int64(i + 1), ChanKey: fmt.Sprint(i)}, Card{ID: int64(i + 10), ChanKey: fmt.Sprint(i)})
|
||||
}
|
||||
sort.Slice(cards, func(i, j int) bool { return cards[i].ID < cards[j].ID })
|
||||
scanner := scannerFor(cards, 100)
|
||||
scanner.hook = func(ctx context.Context, _ string, _ int64) error { return sleepContext(ctx, 5*time.Millisecond) }
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2, ScanConcurrency: 4, ChannelHighWaterMark: 1, GlobalHighWaterMark: 100, PollInterval: time.Millisecond})
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- p.Produce(ctx, scopedMessage("isolation")) }()
|
||||
waitUntil(t, ctx, func() bool { return len(taskIDs(t, store, "isolation")) == 9 })
|
||||
if peak := scanner.peak.Load(); peak > 4 || peak < 2 {
|
||||
t.Fatalf("SQL peak=%d", peak)
|
||||
}
|
||||
// All nine channels got a first task despite the first four waiting at
|
||||
// channel capacity. SQL slots are free while every channel is paused.
|
||||
if scanner.inflight.Load() != 0 {
|
||||
t.Fatal("SQL tokens held during channel backpressure")
|
||||
}
|
||||
for _, channel := range scanner.channels {
|
||||
if _, err := store.BlockRunScoped(ctx, "isolation", channel.ChanKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "isolation")
|
||||
if err != nil || !meta.ProducerDone {
|
||||
t.Fatalf("meta=%+v err=%v", meta, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelRemainingCapacityAndDuplicateAdmission(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
scanner := scannerFor(nil, 0)
|
||||
p := testProducer(t, scanner, store, Config{GlobalHighWaterMark: 3, ChannelHighWaterMark: 2, PollInterval: time.Millisecond})
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "capacity", Status: "producing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Existing success and waiting states still deduplicate and consume no new
|
||||
// capacity. Active includes leases: inflight must not be added again.
|
||||
stateKey, _ := store.TaskStateKey("capacity")
|
||||
metaKey, _ := store.RunMetaKey("capacity")
|
||||
activeKey, _ := store.RoundActiveKey("capacity", 1, "a")
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", stateKey, "one", `{"status":"success"}`, "two", `{"status":"waiting"}`)
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", metaKey, "activeCount", 2, "inflightCount", 2)
|
||||
stats := &channelScanStats{chanKey: "a", lastSummary: time.Now()}
|
||||
tasks := []zset.Task{{TaskID: "one", RunID: "capacity", ChanKey: "a"}, {TaskID: "two", RunID: "capacity", ChanKey: "a"}, {TaskID: "three", RunID: "capacity", ChanKey: "a"}, {TaskID: "four", RunID: "capacity", ChanKey: "a"}}
|
||||
gate := make(chan struct{}, 1)
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := p.admitChannelPage(ctx, "capacity", tasks, time.Now(), gate, stats); result <- err }()
|
||||
waitUntil(t, ctx, func() bool { meta, e := store.GetRunMeta(ctx, "capacity"); return e == nil && meta.ActiveCount == 3 })
|
||||
if size, err := store.SizeKey(ctx, activeKey); err != nil || size != 1 {
|
||||
t.Fatalf("initial slice size=%d err=%v", size, err)
|
||||
}
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", metaKey, "activeCount", 0, "inflightCount", 0)
|
||||
if err := <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats.duplicates != 2 || stats.added != 2 || stats.enqueueBatches != 2 || stats.globalWait == 0 {
|
||||
t.Fatalf("stats=%+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
// faultRedis forwards to a real isolated Redis, optionally returning a lost
|
||||
// response after Lua succeeded. Never mocks the Lua semantics under test.
|
||||
type faultRedis struct {
|
||||
zset.RedisClient
|
||||
failEnqueue atomic.Int32
|
||||
failRead atomic.Bool
|
||||
failRenew atomic.Bool
|
||||
lostResponse bool
|
||||
reject bool
|
||||
}
|
||||
|
||||
func (r *faultRedis) Do(ctx context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
if r.failRead.Load() && command == "HMGET" {
|
||||
return nil, errors.New("Redis read failure")
|
||||
}
|
||||
return r.RedisClient.Do(ctx, command, args...)
|
||||
}
|
||||
func (r *faultRedis) Eval(ctx context.Context, script string, n int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if r.failRenew.Load() && strings.Contains(script, "PEXPIRE") && strings.Contains(script, "owner ~=") {
|
||||
return gvar.New(0), nil
|
||||
}
|
||||
if n == 4 && strings.Contains(script, "HEXISTS") {
|
||||
if r.reject {
|
||||
return gvar.New(0), nil
|
||||
}
|
||||
if r.failEnqueue.CompareAndSwap(1, 0) {
|
||||
if r.lostResponse {
|
||||
if _, err := r.RedisClient.Eval(ctx, script, n, keys, args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, errors.New("enqueue response unavailable")
|
||||
}
|
||||
}
|
||||
return r.RedisClient.Eval(ctx, script, n, keys, args)
|
||||
}
|
||||
|
||||
func TestChannelRescanPartialAndUncertainWrites(t *testing.T) {
|
||||
for _, uncertain := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(uncertain), func(t *testing.T) {
|
||||
client := &faultRedis{RedisClient: testutil.Redis(t), lostResponse: uncertain}
|
||||
store, _ := zset.New(client, "flow:")
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "a"}, {ID: 3, ChanKey: "a"}, {ID: 4, ChanKey: "a"}}, 4)
|
||||
// Fail the second page: first page must survive; uncertain success is
|
||||
// confirmed only on a same-Run rescan, with stable IDs and no double count.
|
||||
scanner.hook = func(_ context.Context, _ string, last int64) error {
|
||||
if last == 2 {
|
||||
client.failEnqueue.Store(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err == nil {
|
||||
t.Fatal("expected enqueue failure")
|
||||
}
|
||||
active := int64(2)
|
||||
if uncertain {
|
||||
active = 4
|
||||
}
|
||||
requireMeta(t, store, "rescan", false, active)
|
||||
scanner.hook = nil
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMeta(t, store, "rescan", true, 4)
|
||||
if len(taskIDs(t, store, "rescan")) != 4 {
|
||||
t.Fatal("wrong stable task set")
|
||||
}
|
||||
before := scanner.queries.Load()
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("rescan")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if before != scanner.queries.Load() {
|
||||
t.Fatal("producerDone redelivery rescanned cards")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelFailureCancelsAllWorkersWithoutProducerDone(t *testing.T) {
|
||||
for _, kind := range []string{"sql", "redis", "unexplained_rejection", "lease"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
client := &faultRedis{RedisClient: testutil.Redis(t)}
|
||||
store, _ := zset.New(client, "flow:")
|
||||
cards := []Card{{ID: 1, ChanKey: "a"}, {ID: 2, ChanKey: "b"}, {ID: 3, ChanKey: "c"}}
|
||||
scanner := scannerFor(cards, 10)
|
||||
allStarted := make(chan struct{})
|
||||
var started atomic.Int32
|
||||
var once sync.Once
|
||||
scanner.hook = func(ctx context.Context, key string, _ int64) error {
|
||||
if started.Add(1) >= 3 {
|
||||
once.Do(func() { close(allStarted) })
|
||||
}
|
||||
select {
|
||||
case <-allStarted:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if kind == "lease" {
|
||||
client.failRenew.Store(true)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
if key == "a" {
|
||||
switch kind {
|
||||
case "sql":
|
||||
return errors.New("SQL failed")
|
||||
case "redis":
|
||||
client.failRead.Store(true)
|
||||
case "unexplained_rejection":
|
||||
client.reject = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{PageSize: 2, RunLockTTL: 90 * time.Millisecond})
|
||||
if err := p.Produce(contextForTest(t), scopedMessage("failure")); err == nil {
|
||||
t.Fatal("expected fatal failure")
|
||||
}
|
||||
if scanner.inflight.Load() != 0 {
|
||||
t.Fatal("scanner goroutines leaked")
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "failure")
|
||||
if err != nil || meta.ProducerDone {
|
||||
t.Fatalf("partial scan marked done: %+v err=%v", meta, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelScanStateStopsAndNoTarget(t *testing.T) {
|
||||
for _, kind := range []string{"terminal", "producerDone", "blocked", "missing", "no-target", "no-cards"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 2)
|
||||
if kind == "no-target" {
|
||||
scanner.channels[0].SyncEnabled = true
|
||||
}
|
||||
if kind == "no-cards" {
|
||||
scanner.cards = nil
|
||||
}
|
||||
p := testProducer(t, scanner, store, Config{})
|
||||
if kind != "no-target" && kind != "no-cards" {
|
||||
if err := store.CreateRun(context.Background(), zset.RunMeta{RunID: "stop", Status: "producing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := store.RunMetaKey("stop")
|
||||
switch kind {
|
||||
case "terminal":
|
||||
_, _ = store.RedisClient().Do(context.Background(), "HSET", key, "status", "completed")
|
||||
case "producerDone":
|
||||
if err := store.MarkProducerDone(context.Background(), "stop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "blocked":
|
||||
_, _ = store.BlockRunScoped(context.Background(), "stop", "a")
|
||||
case "missing":
|
||||
scanner.hook = func(ctx context.Context, _ string, _ int64) error {
|
||||
_, err := store.RedisClient().Do(ctx, "DEL", key)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
err := p.Produce(contextForTest(t), scopedMessage("stop"))
|
||||
if kind == "terminal" {
|
||||
if !errors.Is(err, ErrRunFinished) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
} else if kind == "missing" {
|
||||
if !errors.Is(err, zset.ErrRunNotFound) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(taskIDs(t, store, "stop")) != 0 {
|
||||
t.Fatal("stopped/empty run enqueued tasks")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelOldNonRunPathRemainsSerial(t *testing.T) {
|
||||
redis := &producerRedis{batches: make(map[string]int)}
|
||||
store, _ := zset.New(redis, "flow:")
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "a"}}, 1)
|
||||
p, err := New(scanner, store, Config{ScanMode: "channel", RoundRetryEnabled: false})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = p.Produce(context.Background(), scopedMessage("legacy")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scanner.queries.Load() != 0 || scanner.lists.Load() != 0 {
|
||||
t.Fatal("non-Run path invoked channel scanner")
|
||||
}
|
||||
}
|
||||
|
||||
const operation = FullSyncOperation
|
||||
|
||||
type fakeScanner struct{}
|
||||
|
||||
func (*fakeScanner) SnapshotMaxID(context.Context) (int64, error) { return 0, nil }
|
||||
func (*fakeScanner) Scan(context.Context, int64, int64, int) ([]Card, error) { return nil, nil }
|
||||
func scopedMessage(run string, channels ...string) model.TrafficUsageMonthlyHistoryStartMessage {
|
||||
m := model.TrafficUsageMonthlyHistoryStartMessage{RunID: run, QueryMonth: "202609", Operation: FullSyncOperation}
|
||||
if len(channels) > 0 {
|
||||
m.Scope = &model.SyncScope{Channels: channels}
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
// channelScanner augments, rather than replaces, the legacy scanner contract.
|
||||
// ListChannels is called once before any workers start. ScanChannel must not
|
||||
// mutate the legacy DatabaseScanner.channelMetadata cache.
|
||||
type channelScanner interface {
|
||||
ListChannels(context.Context) ([]channelMetadata, error)
|
||||
ScanChannel(context.Context, string, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
// This exact MySQL query shape is a release gate. Do not silently remove hints,
|
||||
// add an SQL upper bound, or move eligibility filtering after LIMIT.
|
||||
const channelPageSQL = `SELECT c.id, c.iccid, c.msisdn, c.imsi, c.chan_key, c.current_package_code
|
||||
FROM t_sim_card_info AS c FORCE INDEX (idx_card_chan_key)
|
||||
STRAIGHT_JOIN sys_dept AS d
|
||||
ON c.dept_id = d.dept_id
|
||||
AND c.tenant_id = d.tenant_id
|
||||
WHERE c.chan_key = ?
|
||||
AND c.id > ?
|
||||
AND c.iccid <> ''
|
||||
AND c.current_package_code <> ''
|
||||
AND d.deleted_at IS NULL
|
||||
ORDER BY c.id ASC
|
||||
LIMIT ?`
|
||||
|
||||
func (s *DatabaseScanner) ListChannels(ctx context.Context) ([]channelMetadata, error) {
|
||||
var metadata []channelMetadata
|
||||
if err := dao.TSimCardInfo.DB().Model("t_sim_channels").Ctx(ctx).Fields("chan_key,temp_key,sync_enabled,operator").Scan(&metadata); err != nil {
|
||||
return nil, gerror.Wrap(err, "list channel metadata")
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) ScanChannel(ctx context.Context, chanKey string, lastID int64, pageSize int) ([]Card, error) {
|
||||
if chanKey == "" || lastID < 0 || pageSize <= 0 {
|
||||
return nil, gerror.New("invalid channel pagination parameters")
|
||||
}
|
||||
var rows []entity.TSimCardInfo
|
||||
// Use the card DAO's database group, not a separately configured connection.
|
||||
if err := dao.TSimCardInfo.DB().Raw(channelPageSQL, chanKey, lastID, pageSize).Ctx(ctx).Scan(&rows); err != nil {
|
||||
return nil, gerror.Wrap(err, "scan channel cards")
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
cards = append(cards, Card{ID: row.Id, ICCID: row.Iccid, MSISDN: row.Msisdn, IMSI: row.Imsi, ChanKey: row.ChanKey, PackageCode: row.CurrentPackageCode})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
// Pool counters are shared with other worker traffic. Their deltas are direct
|
||||
// observations, not an attribution of all connection waits to this producer.
|
||||
func (*DatabaseScanner) poolWait(ctx context.Context) (int64, int64) {
|
||||
var count, nanos int64
|
||||
for _, item := range dao.TSimCardInfo.DB().Stats(ctx) {
|
||||
stats := item.Stats()
|
||||
count += stats.WaitCount
|
||||
nanos += int64(stats.WaitDuration)
|
||||
}
|
||||
return count, nanos
|
||||
}
|
||||
|
||||
// ValidateScanSettings is strict for values read from configuration. The Go
|
||||
// constructor first fills omitted zero values; explicit YAML zero is rejected.
|
||||
func ValidateScanSettings(mode string, concurrency int) error {
|
||||
if mode != "serial" && mode != "channel" {
|
||||
return gerror.Newf("invalid trafficUsageMonthlyHistorySync.producer.scanMode %q (serial or channel required)", mode)
|
||||
}
|
||||
if concurrency <= 0 {
|
||||
return gerror.New("trafficUsageMonthlyHistorySync.producer.scanConcurrency must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
)
|
||||
|
||||
// SQLite runs the SAME predicate and ordering with only the two MySQL hints
|
||||
// removed. This checks full eligibility/TaskID parity, not MySQL access plans.
|
||||
// Production EXPLAIN must still validate the unmodified channelPageSQL.
|
||||
type sqliteChannelScanner struct{ DatabaseScanner }
|
||||
|
||||
func (s *sqliteChannelScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
query := strings.ReplaceAll(channelPageSQL, "FORCE INDEX (idx_card_chan_key)", "")
|
||||
query = strings.ReplaceAll(query, "STRAIGHT_JOIN", "INNER JOIN")
|
||||
var rows []struct {
|
||||
ID int64 `orm:"id"`
|
||||
ICCID string `orm:"iccid"`
|
||||
MSISDN string `orm:"msisdn"`
|
||||
IMSI string `orm:"imsi"`
|
||||
ChanKey string `orm:"chan_key"`
|
||||
PackageCode string `orm:"current_package_code"`
|
||||
}
|
||||
if err := dao.TSimCardInfo.DB().Raw(query, key, last, size).Ctx(ctx).Scan(&rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
cards = append(cards, Card{ID: row.ID, ICCID: row.ICCID, MSISDN: row.MSISDN, IMSI: row.IMSI, ChanKey: row.ChanKey, PackageCode: row.PackageCode})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
func TestDatabaseQualificationAndCompleteTaskIDParity(t *testing.T) {
|
||||
// No application DB/config: isolated SQLite file and loopback Redis only.
|
||||
if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "sqlite", Link: "sqlite::@file(" + t.TempDir() + "/cards.db)"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := contextForTest(t)
|
||||
db := dao.TSimCardInfo.DB()
|
||||
t.Cleanup(func() { _ = db.Close(context.Background()) })
|
||||
for _, sql := range []string{
|
||||
`CREATE TABLE sys_dept (dept_id INTEGER PRIMARY KEY, tenant_id TEXT, deleted_at DATETIME)`,
|
||||
`CREATE TABLE t_sim_card_info (id INTEGER PRIMARY KEY, iccid TEXT, msisdn TEXT, imsi TEXT, chan_key TEXT, current_package_code TEXT, dept_id INTEGER, tenant_id TEXT)`,
|
||||
`CREATE INDEX idx_card_chan_key ON t_sim_card_info(chan_key)`,
|
||||
`CREATE TABLE t_sim_channels (chan_key TEXT, temp_key TEXT, sync_enabled INTEGER, operator INTEGER)`,
|
||||
`INSERT INTO sys_dept VALUES (1,'t1',NULL),(2,'t2',NULL),(3,'t1','2026-01-01')`,
|
||||
`INSERT INTO t_sim_channels VALUES ('a','sim_yd_v5',0,1),('b','sim_dx_v1',0,2),('disabled','sim_yd_v5',1,1),('no-template','',0,1)`,
|
||||
`INSERT INTO t_sim_card_info VALUES
|
||||
(1,'i1','m1','s1','a','pkg',1,'t1'),
|
||||
(2,'i2','m2','s2','b','pkg',2,'t2'),
|
||||
(3,'i3','','','a','pkg',1,'t1'),
|
||||
(4,'','','','a','pkg',1,'t1'),
|
||||
(5,'i5','','','a','',1,'t1'),
|
||||
(6,'i6','','','a','pkg',1,'wrong-tenant'),
|
||||
(7,'i7','','','a','pkg',3,'t1'),
|
||||
(8,'i8','','','b','pkg',99,'t2'),
|
||||
(9,'i9','','','disabled','pkg',1,'t1'),
|
||||
(10,'i10','','','no-template','pkg',1,'t1'),
|
||||
(11,'i11','','','unknown','pkg',1,'t1'),
|
||||
(12,'i12','','','','pkg',1,'t1'),
|
||||
(13,'i13','','','b','pkg',2,'t2')`,
|
||||
} {
|
||||
if _, err := db.Exec(ctx, sql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store := testStore(t)
|
||||
for _, scope := range [][]string{nil, {"a"}, {"b", "disabled", "no-template", "unknown"}, {"unknown"}} {
|
||||
var sets [][]string
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
scanner := &sqliteChannelScanner{}
|
||||
p := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 2})
|
||||
run := "db-" + mode
|
||||
if err := p.Produce(ctx, scopedMessage(run, scope...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := taskIDs(t, store, run)
|
||||
for i, id := range ids {
|
||||
ids[i] = strings.TrimPrefix(id, run+":")
|
||||
}
|
||||
sort.Strings(ids)
|
||||
sets = append(sets, ids)
|
||||
stateKey, _ := store.TaskStateKey(run)
|
||||
metaKey, _ := store.RunMetaKey(run)
|
||||
chanKey, _ := store.RunChannelsKey(run)
|
||||
_, _ = store.RedisClient().Do(ctx, "DEL", stateKey, metaKey, chanKey)
|
||||
for _, key := range []string{"a", "b"} {
|
||||
activeKey, _ := store.RoundActiveKey(run, 1, key)
|
||||
_, _ = store.RedisClient().Do(ctx, "DEL", activeKey)
|
||||
}
|
||||
_, _ = store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), run)
|
||||
}
|
||||
if !reflect.DeepEqual(sets[0], sets[1]) {
|
||||
t.Fatalf("scope=%v serial=%v channel=%v", scope, sets[0], sets[1])
|
||||
}
|
||||
if scope == nil {
|
||||
want := []string{operation + ":1", operation + ":13", operation + ":2", operation + ":3"}
|
||||
sort.Strings(want)
|
||||
if !reflect.DeepEqual(sets[0], want) {
|
||||
t.Fatalf("qualification set=%v want=%v", sets[0], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelSQLShapeAndParameters(t *testing.T) {
|
||||
for _, term := range []string{"FORCE INDEX (idx_card_chan_key)", "STRAIGHT_JOIN sys_dept", "c.chan_key = ?", "c.id > ?", "c.iccid <> ''", "c.current_package_code", "c.current_package_code <> ''", "d.deleted_at IS NULL", "c.tenant_id = d.tenant_id", "ORDER BY c.id ASC", "LIMIT ?"} {
|
||||
if !strings.Contains(channelPageSQL, term) {
|
||||
t.Fatalf("missing %q", term)
|
||||
}
|
||||
}
|
||||
if strings.Contains(channelPageSQL, "<=") || strings.Count(channelPageSQL, "?") != 3 {
|
||||
t.Fatal("SQL contains upper bound or changed bind parameter count")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
func TestChannelBackpressureDoesNotStarveHighIDChannel(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx, cancel := context.WithCancel(contextForTest(t))
|
||||
defer cancel()
|
||||
scanner := scannerFor([]Card{{ID: 1, ChanKey: "slow"}, {ID: 2, ChanKey: "slow"}, {ID: 3, ChanKey: "slow"}, {ID: 1000, ChanKey: "fast"}}, 1000)
|
||||
p := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 1, ScanConcurrency: 1, ChannelHighWaterMark: 1, GlobalHighWaterMark: 10, PollInterval: time.Millisecond})
|
||||
result := make(chan error, 1)
|
||||
started := time.Now()
|
||||
go func() { result <- p.Produce(ctx, scopedMessage("starvation")) }()
|
||||
slowKey, _ := store.RoundActiveKey("starvation", 1, "slow")
|
||||
fastKey, _ := store.RoundActiveKey("starvation", 1, "fast")
|
||||
waitUntil(t, ctx, func() bool { n, e := store.SizeKey(ctx, slowKey); return e == nil && n == 1 })
|
||||
if mode == "channel" {
|
||||
waitUntil(t, ctx, func() bool { n, e := store.SizeKey(ctx, fastKey); return e == nil && n == 1 })
|
||||
} else {
|
||||
// No consumer drains slow. The serial cursor cannot reach card 1000.
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
if n, err := store.SizeKey(ctx, fastKey); err != nil || n != 0 {
|
||||
t.Fatalf("serial fast size=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
slow, _ := store.SizeKey(ctx, slowKey)
|
||||
fast, _ := store.SizeKey(ctx, fastKey)
|
||||
t.Logf("mode=%s slowQueued=%d fastQueued=%d scanConcurrency=1 channelHWM=1 elapsedMs=%d", mode, slow, fast, time.Since(started).Milliseconds())
|
||||
cancel()
|
||||
if err := <-result; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation err=%v", err)
|
||||
}
|
||||
requireMeta(t, store, "starvation", false, slow+fast)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelPayloadRetainsPeriodPackageAndIdentifiers(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
card := Card{ID: 10, ICCID: "iccid", MSISDN: "msisdn", IMSI: "imsi", ChanKey: "a", PackageCode: "pkg-history"}
|
||||
p := testProducer(t, scannerFor([]Card{card}, card.ID), store, Config{PageSize: 1})
|
||||
message := scopedMessage("payload", "a")
|
||||
if err := p.Produce(ctx, message); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := store.ClaimRun(ctx, message.RunID, "a", time.Now().Add(time.Second), time.Now().Add(time.Minute), 1)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
task := claimed[0].Task
|
||||
if task.TaskID != zset.StableTaskID(message.RunID, FullSyncOperation, card.ID) || task.Operation != FullSyncOperation || task.QueryMonth != message.QueryMonth || task.PackageCode != card.PackageCode || task.ICCID != card.ICCID || task.MSISDN != card.MSISDN || task.IMSI != card.IMSI || task.TempKey != "sim_yd_v5" || task.Operator != 1 {
|
||||
t.Fatalf("task=%+v", task)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZombieWithBacklogDrainsBeforeCompletion(t *testing.T) {
|
||||
p, store := newZombieTestProducer(t)
|
||||
ctx := contextForTest(t)
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-backlog"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks := []zset.Task{{TaskID: "zombie-card", RunID: "zombie-backlog", ChanKey: "a"}}
|
||||
if _, err := store.EnqueueRunBatch(ctx, "zombie-backlog", "a", tasks, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending, err := p.hasPendingTasks(ctx, "new"); err != nil || !pending {
|
||||
t.Fatalf("backlog must retain backpressure: pending=%t err=%v", pending, err)
|
||||
}
|
||||
requireMeta(t, store, "zombie-backlog", true, 1)
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "zombie-backlog"); err != nil || complete {
|
||||
t.Fatalf("premature complete=%t err=%v", complete, err)
|
||||
}
|
||||
claimed, err := store.ClaimRun(ctx, "zombie-backlog", "a", time.Now().Add(time.Second), time.Now().Add(time.Minute), 1)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
if ok, err := store.FinalizeRunTask(ctx, claimed[0], zset.TaskOutcome{Success: true}); err != nil || !ok {
|
||||
t.Fatalf("finalize=%t err=%v", ok, err)
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "zombie-backlog", time.Hour); err != nil || !complete {
|
||||
t.Fatalf("complete=%t err=%v", complete, err)
|
||||
}
|
||||
meta, _ := store.RunMetaKey("zombie-backlog")
|
||||
ttl, err := store.RedisClient().Do(ctx, "TTL", meta)
|
||||
if err != nil || ttl.Int() <= 0 {
|
||||
t.Fatalf("retention ttl=%v err=%v", ttl, err)
|
||||
}
|
||||
active, err := store.ActiveRuns(ctx)
|
||||
if err != nil || len(active) != 0 {
|
||||
t.Fatalf("active=%v err=%v", active, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,741 @@
|
||||
// Package monthlyusagehistory produces the previous-month usage history run.
|
||||
//
|
||||
// It writes the TDengine monthly table, a different target from
|
||||
// producer/monthlyusage (which fills the MySQL current-month snapshot). Apart
|
||||
// from the period and the destination it is structurally identical: scan the
|
||||
// eligible cards once, group by channel, and enqueue one Redis ZSet task per
|
||||
// card. The zset consumer owns pacing, retries, failure pooling, archiving and
|
||||
// replay.
|
||||
//
|
||||
// The run period is a month (yyyyMM) and travels in zset.Task.QueryMonth, which
|
||||
// is a period identifier rather than a month name; see that field's
|
||||
// documentation.
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// FullSyncOperation identifies a monthly usage history run in run metadata,
|
||||
// archives and replay. contract.SyncKindForOperation maps it to
|
||||
// traffic_usage_monthly.
|
||||
const FullSyncOperation = model.OperationPreviousMonthUsageFullSync
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
// The run hash and its archive JSON must stay bounded regardless of how many
|
||||
// channels a scope selects.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels. It lets
|
||||
// operators confirm two runs covered the same scope without storing the list.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ScanMode string
|
||||
ScanConcurrency int
|
||||
PageSize int
|
||||
GlobalHighWaterMark int64
|
||||
ChannelHighWaterMark int64
|
||||
PollInterval time.Duration
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
RoundRetryEnabled bool
|
||||
MaxRounds int
|
||||
FailureRetention time.Duration
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.ScanMode == "" {
|
||||
c.ScanMode = "serial"
|
||||
}
|
||||
if c.ScanConcurrency == 0 {
|
||||
c.ScanConcurrency = 4
|
||||
}
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 100
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = 2 * time.Second
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "monthly-history:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.MaxRounds <= 0 {
|
||||
c.MaxRounds = 3
|
||||
}
|
||||
if c.FailureRetention <= 0 {
|
||||
c.FailureRetention = 30 * 24 * time.Hour
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type CardScanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
Scan(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
PackageCode string
|
||||
SyncEnabled bool
|
||||
Operator int
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
Operator int `orm:"operator"`
|
||||
}
|
||||
|
||||
type DatabaseScanner struct {
|
||||
channelMetadata map[string]channelMetadata
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := monthlyHistoryEligibleCardModel(ctx).
|
||||
Fields(cardTable + "." + columns.Id + " AS " + columns.Id).
|
||||
OrderDesc(cardTable + "." + columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read card snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) Scan(ctx context.Context, lastID, maxID int64, pageSize int) ([]Card, error) {
|
||||
if maxID <= lastID {
|
||||
return nil, nil
|
||||
}
|
||||
if lastID == 0 || s.channelMetadata == nil {
|
||||
var metadata []channelMetadata
|
||||
if err := g.DB().Model("t_sim_channels").Fields("chan_key,temp_key,sync_enabled,operator").Scan(&metadata); err != nil {
|
||||
return nil, fmt.Errorf("scan channel metadata: %w", err)
|
||||
}
|
||||
s.channelMetadata = make(map[string]channelMetadata, len(metadata))
|
||||
for _, item := range metadata {
|
||||
s.channelMetadata[item.ChanKey] = item
|
||||
}
|
||||
}
|
||||
var rows []entity.TSimCardInfo
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
query := monthlyHistoryEligibleCardModel(ctx).
|
||||
Fields(
|
||||
cardTable+"."+columns.Id+" AS "+columns.Id,
|
||||
cardTable+"."+columns.Iccid+" AS "+columns.Iccid,
|
||||
cardTable+"."+columns.Msisdn+" AS "+columns.Msisdn,
|
||||
cardTable+"."+columns.Imsi+" AS "+columns.Imsi,
|
||||
cardTable+"."+columns.ChanKey+" AS "+columns.ChanKey,
|
||||
// The monthly usage child table stores the package code and the
|
||||
// diagnosis page reads it back, so it must be carried on the task.
|
||||
cardTable+"."+columns.CurrentPackageCode+" AS "+columns.CurrentPackageCode,
|
||||
).
|
||||
WhereGT(cardTable+"."+columns.Id, lastID).
|
||||
WhereLTE(cardTable+"."+columns.Id, maxID).
|
||||
OrderAsc(cardTable + "." + columns.Id).
|
||||
Limit(pageSize)
|
||||
if err := query.Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan card information: %w", err)
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
channel := s.channelMetadata[row.ChanKey]
|
||||
cards = append(cards, Card{
|
||||
ID: row.Id,
|
||||
ICCID: row.Iccid,
|
||||
MSISDN: row.Msisdn,
|
||||
IMSI: row.Imsi,
|
||||
ChanKey: row.ChanKey,
|
||||
TempKey: channel.TempKey,
|
||||
PackageCode: row.CurrentPackageCode,
|
||||
SyncEnabled: channel.SyncEnabled,
|
||||
Operator: channel.Operator,
|
||||
})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
// monthlyHistoryEligibleCardModel mirrors the monthly usage eligibility rule and
|
||||
// the legacy admin SyncMonthly scan: the card must belong to a live department,
|
||||
// carry an ICCID and channel, and have a current package code.
|
||||
func monthlyHistoryEligibleCardModel(ctx context.Context) *gdb.Model {
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
deptTable := dao.SysDept.Table()
|
||||
cardColumns := dao.TSimCardInfo.Columns()
|
||||
deptColumns := dao.SysDept.Columns()
|
||||
|
||||
return dao.TSimCardInfo.Ctx(ctx).
|
||||
InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId).
|
||||
Where(cardTable+"."+cardColumns.Iccid+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.ChanKey+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", "").
|
||||
WhereNull(deptTable + "." + deptColumns.DeletedAt)
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner CardScanner
|
||||
store *zset.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
observation *supplyObservation
|
||||
}
|
||||
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
eligible int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
stoppedTasks int64
|
||||
channels map[string]struct{}
|
||||
scopeChannels int
|
||||
skippedChannels int
|
||||
}
|
||||
|
||||
func (s *productionStats) channelCount() int {
|
||||
return len(s.channels)
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, queryMonth, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=monthly_usage_history_producer status=%s runId=%s queryMonth=%s snapshotMaxId=%d pages=%d scanned=%d eligible=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d stoppedTasks=%d channelCount=%d scopeChannels=%d skippedChannels=%d durationMs=%d", status, runID, queryMonth, maxID, s.pages, s.scanned, s.eligible, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.stoppedTasks, s.channelCount(), s.scopeChannels, s.skippedChannels, time.Since(startedAt).Milliseconds())
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
if status == "failed" {
|
||||
g.Log().Error(ctx, message)
|
||||
return
|
||||
}
|
||||
if status == "skipped" {
|
||||
g.Log().Warning(ctx, message)
|
||||
return
|
||||
}
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
|
||||
func New(scanner CardScanner, store *zset.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("monthly usage history producer dependencies are incomplete")
|
||||
}
|
||||
config = config.withDefaults()
|
||||
if err := ValidateScanSettings(config.ScanMode, config.ScanConcurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.ScanMode == "channel" {
|
||||
if _, ok := scanner.(channelScanner); !ok {
|
||||
return nil, fmt.Errorf("channel scan requires ListChannels and ScanChannel")
|
||||
}
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config, owner: newOwner()}, nil
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("monthly usage history producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected; callers
|
||||
// should log and ack instead of retrying.
|
||||
var ErrRunFinished = fmt.Errorf("monthly usage history run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if errors.Is(err, zset.ErrRunNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
// Produce scans the eligible cards once and enqueues one task per card.
|
||||
//
|
||||
// message.QueryMonth (yyyyMM) is stored in zset.Task.QueryMonth, the queue's
|
||||
// period-identifier field, so the zset store, run metadata, archive, replay and
|
||||
// Lua scripts need no change for a monthly history run.
|
||||
func (p *Producer) Produce(ctx context.Context, message model.TrafficUsageMonthlyHistoryStartMessage) (err error) {
|
||||
startedAt := time.Now()
|
||||
stats := productionStats{channels: make(map[string]struct{})}
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
|
||||
if message.RunID == "" {
|
||||
err = fmt.Errorf("runId is empty")
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if message.QueryMonth == "" {
|
||||
err = fmt.Errorf("queryMonth is empty")
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats.scopeChannels = scope.ChannelCount()
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if p.config.RoundRetryEnabled {
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
}
|
||||
acquired, err := p.store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer releaseCancel()
|
||||
_, _ = p.store.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
pending, err := p.hasPendingTasks(ctx, message.RunID)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "pending_check", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if pending {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "pending_check", "pending_tasks", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
renewErr := make(chan error, 1)
|
||||
go p.renewLock(runCtx, cancel, renewErr)
|
||||
ctx = runCtx
|
||||
if message.Operation == "" {
|
||||
message.Operation = FullSyncOperation
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
if err = p.store.CreateRun(ctx, zset.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: message.Operation,
|
||||
QueryMonth: message.QueryMonth,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_init", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
maxID, err := p.scanner.SnapshotMaxID(ctx)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "snapshot", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_producer status=started runId=%s queryMonth=%s snapshotMaxId=%d scopeChannels=%d scanMode=%s scanConcurrency=%d pageSize=%d", message.RunID, message.QueryMonth, maxID, scope.ChannelCount(), p.config.ScanMode, p.config.ScanConcurrency, p.config.PageSize)
|
||||
stage := "scan"
|
||||
defer func() {
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, message.QueryMonth, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
if p.config.RoundRetryEnabled && p.config.ScanMode == "channel" {
|
||||
stage = "channel_scan"
|
||||
if err = p.produceChannels(ctx, message, scope, maxID, startedAt, &stats); err != nil {
|
||||
if isProductionClosed(err) {
|
||||
// A same-Run redelivery after producerDone leaves consumption alone.
|
||||
err = nil
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Verify ownership immediately before closing production. The periodic
|
||||
// renewer still cancels scanners and capacity waits on lease loss.
|
||||
var owned bool
|
||||
owned, err = p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owned {
|
||||
return fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "producer_done"
|
||||
err = p.store.MarkProducerDone(ctx, message.RunID)
|
||||
return err
|
||||
}
|
||||
lastID := int64(0)
|
||||
for lastID < maxID {
|
||||
stage = "capacity"
|
||||
if err = p.waitForCapacity(ctx, message.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "scan"
|
||||
cards, scanErr := p.scanner.Scan(ctx, lastID, maxID, p.config.PageSize)
|
||||
if scanErr != nil {
|
||||
err = scanErr
|
||||
return err
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
stats.pages++
|
||||
stats.scanned += int64(len(cards))
|
||||
groupedTasks := make(map[string][]zset.Task)
|
||||
for _, card := range cards {
|
||||
lastID = card.ID
|
||||
if card.ChanKey == "" {
|
||||
stats.skippedEmptyChan++
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_card status=skipped stage=validate cardId=%d reason=empty_chan_key", card.ID)
|
||||
continue
|
||||
}
|
||||
// Scope is applied before the sync_enabled and template checks so a
|
||||
// partial run only ever reports on the channels it was asked for.
|
||||
if !scope.Allows(card.ChanKey) {
|
||||
stats.skippedOutOfScope++
|
||||
continue
|
||||
}
|
||||
if card.SyncEnabled {
|
||||
stats.skippedSyncDisabled++
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_card status=skipped stage=validate cardId=%d chanKey=%s reason=sync_disabled", card.ID, card.ChanKey)
|
||||
continue
|
||||
}
|
||||
if card.TempKey == "" {
|
||||
stats.skippedNoTemplate++
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_card status=skipped stage=validate cardId=%d chanKey=%s reason=missing_channel_template", card.ID, card.ChanKey)
|
||||
continue
|
||||
}
|
||||
stats.eligible++
|
||||
stats.channels[card.ChanKey] = struct{}{}
|
||||
task := zset.Task{
|
||||
TaskID: zset.StableTaskID(message.RunID, message.Operation, card.ID),
|
||||
RunID: message.RunID,
|
||||
Operation: message.Operation,
|
||||
QueryMonth: message.QueryMonth,
|
||||
ChanKey: card.ChanKey,
|
||||
TempKey: card.TempKey,
|
||||
Operator: card.Operator,
|
||||
CardID: card.ID,
|
||||
ICCID: card.ICCID,
|
||||
MSISDN: card.MSISDN,
|
||||
IMSI: card.IMSI,
|
||||
PackageCode: card.PackageCode,
|
||||
}
|
||||
groupedTasks[card.ChanKey] = append(groupedTasks[card.ChanKey], task)
|
||||
}
|
||||
now := time.Now()
|
||||
for chanKey, tasks := range groupedTasks {
|
||||
stage = "enqueue"
|
||||
if err = p.waitForChannelCapacity(ctx, message.RunID, chanKey); err != nil {
|
||||
return err
|
||||
}
|
||||
var added int
|
||||
var enqueueErr error
|
||||
if p.config.RoundRetryEnabled {
|
||||
added, enqueueErr = p.store.EnqueueRunBatch(ctx, message.RunID, chanKey, tasks, now)
|
||||
} else {
|
||||
added, enqueueErr = p.store.EnqueueBatch(ctx, chanKey, tasks, now)
|
||||
}
|
||||
if enqueueErr != nil {
|
||||
err = enqueueErr
|
||||
return err
|
||||
}
|
||||
stats.enqueued += int64(added)
|
||||
if added < len(tasks) {
|
||||
stats.stoppedTasks += int64(len(tasks) - added)
|
||||
}
|
||||
if added == 0 {
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_channel status=skipped runId=%s queryMonth=%s chanKey=%s count=%d reason=stopped", message.RunID, message.QueryMonth, chanKey, len(tasks))
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
if err = p.store.MarkProducerDone(ctx, message.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Producer) hasPendingTasks(ctx context.Context, currentRunID ...string) (bool, error) {
|
||||
currentID := ""
|
||||
if len(currentRunID) > 0 {
|
||||
currentID = currentRunID[0]
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, runID := range runs {
|
||||
if runID == currentID {
|
||||
// A redelivered message for the same run may resume a producer
|
||||
// that stopped before producerDone. Stable task IDs make the
|
||||
// rescan idempotent; other runs still provide backpressure.
|
||||
continue
|
||||
}
|
||||
meta, metaErr := p.store.GetRunMeta(ctx, runID)
|
||||
if metaErr != nil {
|
||||
// The active index is a global key written outside the run-scoped
|
||||
// Lua transaction, so it can list a run whose hash is already gone
|
||||
// (expired, evicted, or cleaned by hand). Returning an error here
|
||||
// would surface as a non-retryable failure and make the consumer
|
||||
// redeliver this request forever. Treat it as a stale index entry
|
||||
// and drop it.
|
||||
if errors.Is(metaErr, zset.ErrRunNotFound) {
|
||||
if _, removeErr := p.store.RedisClient().Do(ctx, "SREM", p.store.ActiveRunsKey(), runID); removeErr != nil {
|
||||
return false, removeErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_producer status=skipped stage=pending_check reason=stale_index runId=%s", runID)
|
||||
continue
|
||||
}
|
||||
return false, metaErr
|
||||
}
|
||||
if meta.Status != "producing" && meta.Status != "running" {
|
||||
continue
|
||||
}
|
||||
// Channel blocking can set running before production ends. Only
|
||||
// producerDone identifies consumer-owned work; replay has its own lock.
|
||||
if meta.ProducerDone || meta.SyncType == "replay" {
|
||||
return true, nil
|
||||
}
|
||||
// Recheck the unique producer lease immediately before reclaiming
|
||||
// another run. A lost lease or an unspecified current run is not proof
|
||||
// that the old producer has stopped.
|
||||
if currentID == "" {
|
||||
return true, nil
|
||||
}
|
||||
owned, lockErr := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if lockErr != nil {
|
||||
return false, lockErr
|
||||
}
|
||||
if !owned {
|
||||
return true, nil
|
||||
}
|
||||
// Close production, not the run: consumers retain queued, leased and
|
||||
// next-round work and apply retention/index cleanup on normal completion.
|
||||
reclaimed, reclaimErr := p.store.MarkProducerAbandoned(ctx, runID)
|
||||
if reclaimErr != nil {
|
||||
return false, reclaimErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_history_producer status=skipped stage=pending_check reason=zombie_run runId=%s reclaimed=%t activeCount=%d inflightCount=%d nextRoundCount=%d", runID, reclaimed, meta.ActiveCount, meta.InflightCount, meta.NextRoundCount)
|
||||
if !reclaimed || meta.ActiveCount > 0 || meta.InflightCount > 0 || meta.NextRoundCount > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
channels, err := p.store.ActiveChannels(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, chanKey := range channels {
|
||||
size, sizeErr := p.store.Size(ctx, chanKey)
|
||||
if sizeErr != nil {
|
||||
return false, sizeErr
|
||||
}
|
||||
if size > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelFunc, errors chan<- error) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ok, err := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil || !ok {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
select {
|
||||
case errors <- fmt.Errorf("renew producer lock: %w", err):
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var bytes [12]byte
|
||||
if _, err := rand.Read(bytes[:]); err == nil {
|
||||
return "producer-" + hex.EncodeToString(bytes[:])
|
||||
}
|
||||
return fmt.Sprintf("producer-%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func (p *Producer) waitForCapacity(ctx context.Context, runID ...string) error {
|
||||
if p.config.GlobalHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
if p.config.RoundRetryEnabled && len(runID) > 0 && runID[0] != "" {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
channels, err := p.store.RunChannels(ctx, runID[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, chanKey := range channels {
|
||||
key, keyErr := p.store.RoundActiveKey(runID[0], meta.CurrentRound, chanKey)
|
||||
if keyErr != nil {
|
||||
return keyErr
|
||||
}
|
||||
size, sizeErr := p.store.SizeKey(ctx, key)
|
||||
if sizeErr != nil {
|
||||
return sizeErr
|
||||
}
|
||||
total += size
|
||||
}
|
||||
if total < p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
if err = sleepContext(ctx, p.config.PollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
channels, err := p.store.ActiveChannels(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, chanKey := range channels {
|
||||
size, sizeErr := p.store.Size(ctx, chanKey)
|
||||
if sizeErr != nil {
|
||||
return sizeErr
|
||||
}
|
||||
total += size
|
||||
}
|
||||
if total < p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
if err = sleepContext(ctx, p.config.PollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) waitForChannelCapacity(ctx context.Context, runIDAndChan ...string) error {
|
||||
if p.config.ChannelHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
chanKey := ""
|
||||
runID := ""
|
||||
if len(runIDAndChan) == 1 {
|
||||
chanKey = runIDAndChan[0]
|
||||
} else if len(runIDAndChan) >= 2 {
|
||||
runID, chanKey = runIDAndChan[0], runIDAndChan[1]
|
||||
}
|
||||
for {
|
||||
var size int64
|
||||
var err error
|
||||
if p.config.RoundRetryEnabled && runID != "" {
|
||||
meta, metaErr := p.store.GetRunMeta(ctx, runID)
|
||||
if metaErr != nil {
|
||||
return metaErr
|
||||
}
|
||||
key, keyErr := p.store.RoundActiveKey(runID, meta.CurrentRound, chanKey)
|
||||
if keyErr != nil {
|
||||
return keyErr
|
||||
}
|
||||
size, err = p.store.SizeKey(ctx, key)
|
||||
} else {
|
||||
size, err = p.store.Size(ctx, chanKey)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size < p.config.ChannelHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
if err = sleepContext(ctx, p.config.PollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, duration time.Duration) error {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
type pageScanner struct {
|
||||
cards []Card
|
||||
}
|
||||
|
||||
func (s *pageScanner) SnapshotMaxID(context.Context) (int64, error) {
|
||||
if len(s.cards) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return s.cards[len(s.cards)-1].ID, nil
|
||||
}
|
||||
|
||||
func (s *pageScanner) Scan(_ context.Context, lastID, _ int64, _ int) ([]Card, error) {
|
||||
if lastID != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.cards, nil
|
||||
}
|
||||
|
||||
// producerRedis records the enqueued members so a test can assert both the
|
||||
// batch shape and the exact task payload (period and package code).
|
||||
type producerRedis struct {
|
||||
mu sync.Mutex
|
||||
members []string
|
||||
batches map[string]int
|
||||
}
|
||||
|
||||
func (r *producerRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
|
||||
switch strings.ToUpper(command) {
|
||||
case "SET":
|
||||
return gvar.New("OK"), nil
|
||||
case "SMEMBERS":
|
||||
return gvar.New([]string{}), nil
|
||||
case "HGETALL":
|
||||
// No stored run metadata: the producer treats this as a first delivery.
|
||||
return gvar.New(map[string]string{}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected command %s", command)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *producerRedis) Eval(_ context.Context, _ string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if numKeys == 1 {
|
||||
return gvar.New(int64(1)), nil
|
||||
}
|
||||
if numKeys != 3 {
|
||||
return nil, fmt.Errorf("unexpected key count %d", numKeys)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.batches == nil {
|
||||
r.batches = make(map[string]int)
|
||||
}
|
||||
count := len(args) - 3
|
||||
r.batches[keys[0]] += count
|
||||
for _, arg := range args[3:] {
|
||||
if member, ok := arg.(string); ok {
|
||||
r.members = append(r.members, member)
|
||||
}
|
||||
}
|
||||
return gvar.New(count), nil
|
||||
}
|
||||
|
||||
func newProducer(t *testing.T, cards []Card) (*Producer, *producerRedis) {
|
||||
t.Helper()
|
||||
redis := &producerRedis{batches: make(map[string]int)}
|
||||
store, err := zset.New(redis, "monthly-history:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(&pageScanner{cards: cards}, store, Config{PageSize: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return producer, redis
|
||||
}
|
||||
|
||||
func monthlyMessage(runID string, channels ...string) model.TrafficUsageMonthlyHistoryStartMessage {
|
||||
message := model.TrafficUsageMonthlyHistoryStartMessage{
|
||||
RunID: runID,
|
||||
Operation: FullSyncOperation,
|
||||
QueryMonth: "202609",
|
||||
TriggeredAt: "2026-10-10T06:00:00+08:00",
|
||||
}
|
||||
if len(channels) > 0 {
|
||||
message.Scope = &model.SyncScope{Channels: channels}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func TestProduceCarriesQueryMonthAndPackageCode(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "89860001", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "PKG-A", Operator: 1},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), monthlyMessage("monthly-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.members) != 1 {
|
||||
t.Fatalf("members = %+v, want exactly one task", redis.members)
|
||||
}
|
||||
member := redis.members[0]
|
||||
// The period rides in queryMonth because the queue only copies that field.
|
||||
if !strings.Contains(member, `"queryMonth":"202609"`) {
|
||||
t.Fatalf("task member lost the monthly period: %s", member)
|
||||
}
|
||||
// The package code must reach the TDengine child table via the task.
|
||||
if !strings.Contains(member, `"packageCode":"PKG-A"`) {
|
||||
t.Fatalf("task member lost the package code: %s", member)
|
||||
}
|
||||
if !strings.Contains(member, `"operation":"`+FullSyncOperation+`"`) {
|
||||
t.Fatalf("task member lost the operation: %s", member)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceUsesStableTaskIDForIdempotency(t *testing.T) {
|
||||
producer, _ := newProducer(t, []Card{
|
||||
{ID: 42, ICCID: "89860042", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "PKG-A"},
|
||||
})
|
||||
want := zset.StableTaskID("monthly-1", FullSyncOperation, 42)
|
||||
if got := zset.StableTaskID("monthly-1", FullSyncOperation, 42); got != want {
|
||||
t.Fatalf("StableTaskID is not deterministic: %s vs %s", got, want)
|
||||
}
|
||||
if err := producer.Produce(context.Background(), monthlyMessage("monthly-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeEnqueuesOnlySelectedChannels(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 2, ICCID: "2", ChanKey: "173", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 3, ICCID: "3", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 4, ICCID: "4", ChanKey: "189", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), monthlyMessage("monthly-scoped", "217")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 1 {
|
||||
t.Fatalf("enqueue batches = %+v, want only the 217 channel", redis.batches)
|
||||
}
|
||||
if redis.batches["monthly-history:217"] != 2 {
|
||||
t.Fatalf("enqueue batches = %+v, want 2 tasks on monthly:217", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceFullScopeKeepsEveryChannel(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 2, ICCID: "2", ChanKey: "173", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), monthlyMessage("monthly-full")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 2 {
|
||||
t.Fatalf("enqueue batches = %+v, want both channels for a full sync", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeDoesNotOverrideSyncDisabled(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
{ID: 2, ICCID: "2", ChanKey: "173", TempKey: "sim_yd_v5", PackageCode: "P", SyncEnabled: true},
|
||||
})
|
||||
|
||||
if err := producer.Produce(context.Background(), monthlyMessage("monthly-scoped", "173")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 0 {
|
||||
t.Fatalf("enqueue batches = %+v, want none (selected channel is sync-disabled)", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceRejectsMissingPeriod(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
message := monthlyMessage("monthly-bad")
|
||||
message.QueryMonth = ""
|
||||
if err := producer.Produce(context.Background(), message); err == nil {
|
||||
t.Fatal("expected a missing queryDate to be rejected")
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 0 {
|
||||
t.Fatalf("rejected message must not enqueue: %+v", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceRejectsInvalidScope(t *testing.T) {
|
||||
producer, redis := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
if err := producer.Produce(context.Background(), monthlyMessage("monthly-bad", "invalid/chan")); err == nil {
|
||||
t.Fatal("expected invalid chanKey to be rejected")
|
||||
}
|
||||
redis.mu.Lock()
|
||||
defer redis.mu.Unlock()
|
||||
if len(redis.batches) != 0 {
|
||||
t.Fatalf("rejected scope must not enqueue: %+v", redis.batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeDigestIsOrderIndependent(t *testing.T) {
|
||||
first, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"173", "217"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"217", "173"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scopeDigest(first) != scopeDigest(second) {
|
||||
t.Fatalf("digest depends on channel order: %s vs %s", scopeDigest(first), scopeDigest(second))
|
||||
}
|
||||
if scopeDigest(nil) != "" {
|
||||
t.Fatal("a full sync must have an empty digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAlreadyFinishedTreatsMissingRunAsFirstDelivery(t *testing.T) {
|
||||
producer, _ := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "1", ChanKey: "217", TempKey: "sim_yd_v5", PackageCode: "P"},
|
||||
})
|
||||
finished, err := producer.runAlreadyFinished(context.Background(), "fresh")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if finished {
|
||||
t.Fatal("a run without stored metadata must not be reported as finished")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
// Producer-local telemetry: no run/archive schema changes and no assertion of
|
||||
// consumer readiness. Empty queues alone do not prove consumer starvation.
|
||||
type supplyChannel struct {
|
||||
stage string
|
||||
pending int
|
||||
version uint64
|
||||
ended bool
|
||||
firstQuery, firstEnqueue *int64
|
||||
changed time.Time
|
||||
stageDuration map[string]time.Duration
|
||||
validSamples, missingSamples int64
|
||||
}
|
||||
|
||||
type supplyObservation struct {
|
||||
mu sync.Mutex
|
||||
started time.Time
|
||||
channels map[string]*supplyChannel
|
||||
}
|
||||
|
||||
func (o *supplyObservation) event(key, stage string, pending int, added, ended bool, now time.Time) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
s := o.channels[key]
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.stageDuration[s.stage] += now.Sub(s.changed)
|
||||
s.changed, s.stage, s.pending, s.ended = now, stage, pending, ended
|
||||
s.version++
|
||||
if stage == "sql_query" && s.firstQuery == nil {
|
||||
v := now.Sub(o.started).Milliseconds()
|
||||
s.firstQuery = &v
|
||||
}
|
||||
if added && s.firstEnqueue == nil {
|
||||
v := now.Sub(o.started).Milliseconds()
|
||||
s.firstEnqueue = &v
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve fractional milliseconds across transitions; round only for logs.
|
||||
func (s *supplyChannel) stageMilliseconds(now time.Time) map[string]int64 {
|
||||
if !s.ended {
|
||||
s.stageDuration[s.stage] += now.Sub(s.changed)
|
||||
s.changed = now
|
||||
}
|
||||
ms := make(map[string]int64, len(s.stageDuration))
|
||||
for stage, duration := range s.stageDuration {
|
||||
ms[stage] = duration.Milliseconds()
|
||||
}
|
||||
return ms
|
||||
}
|
||||
|
||||
func (p *Producer) observe(key, stage string, pending int, added, ended bool) {
|
||||
if p.observation != nil {
|
||||
p.observation.event(key, stage, pending, added, ended, time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
// Queue-only samples mirror e6ec5b8: cardinalities, never task bodies. All keys
|
||||
// share the run hash tag, and round/producer state is checked in the same Lua.
|
||||
// Leases are token HASHes, not ZSets; future scores include leased members.
|
||||
const supplySampleScript = `
|
||||
local run=redis.call('HMGET',KEYS[1],'runId','status','currentRound','producerDone','syncType')
|
||||
local known=run[1] and run[2] and run[3] and run[4]
|
||||
local eligible=known and (run[2]=='producing' or run[2]=='running') and run[3]=='1' and run[4]=='0' and run[5]~='replay'
|
||||
local blocked=redis.call('HGET',KEYS[1],'blocked:'..ARGV[2])
|
||||
return {known and 1 or 0,(eligible and blocked~='1') and 1 or 0,redis.call('ZCOUNT',KEYS[2],'-inf',ARGV[1]),redis.call('HLEN',KEYS[3]),redis.call('ZCOUNT',KEYS[2],'('..ARGV[1],'+inf')}
|
||||
`
|
||||
|
||||
func (p *Producer) sampleSupply(ctx context.Context, run, key string) ([]int64, error) {
|
||||
meta, err := p.store.RunMetaKey(run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
active, err := p.store.RoundActiveKey(run, 1, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lease, err := p.store.RoundLeaseKey(run, 1, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := p.store.RedisClient().Eval(ctx, supplySampleScript, 3, []string{meta, active, lease}, []any{time.Now().UnixMilli(), key})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.Int64s(), nil
|
||||
}
|
||||
|
||||
func (p *Producer) startObservation(ctx context.Context, run string, keys []string, started time.Time) func() {
|
||||
o := &supplyObservation{started: started, channels: make(map[string]*supplyChannel, len(keys))}
|
||||
for _, key := range keys {
|
||||
o.channels[key] = &supplyChannel{stage: "initializing", changed: started, stageDuration: make(map[string]time.Duration)}
|
||||
}
|
||||
p.observation = o
|
||||
stop, done := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
for _, key := range keys {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
o.mu.Lock()
|
||||
s := o.channels[key]
|
||||
version, ended := s.version, s.ended
|
||||
o.mu.Unlock()
|
||||
if ended {
|
||||
continue
|
||||
}
|
||||
sampleCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
|
||||
values, err := p.sampleSupply(sampleCtx, run, key)
|
||||
cancel()
|
||||
o.mu.Lock()
|
||||
known := err == nil && len(values) == 5 && values[0] == 1 && s.version == version && !s.ended
|
||||
stage, pending := s.stage, s.pending
|
||||
if known {
|
||||
s.validSamples++
|
||||
} else {
|
||||
s.missingSamples++
|
||||
}
|
||||
o.mu.Unlock()
|
||||
// Logging must not hold the mutex used by producer progress.
|
||||
if known {
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_supply status=sample runId=%s chanKey=%s stage=%s pending=%d firstRoundEligible=%t ready=%d leases=%d futureIncludingLeases=%d consumerEligibility=unknown", run, key, stage, pending, values[1] == 1, values[2], values[3], values[4])
|
||||
} else {
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_supply status=unknown runId=%s chanKey=%s stage=%s pending=%d consumerEligibility=unknown", run, key, stage, pending)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
close(stop)
|
||||
<-done
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
for _, key := range keys {
|
||||
s := o.channels[key]
|
||||
stageMS := s.stageMilliseconds(time.Now())
|
||||
query, enqueue := int64(-1), int64(-1)
|
||||
if s.firstQuery != nil {
|
||||
query = *s.firstQuery
|
||||
}
|
||||
if s.firstEnqueue != nil {
|
||||
enqueue = *s.firstEnqueue
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_history_supply status=ended runId=%s chanKey=%s firstQueryMs=%d firstEnqueueMs=%d stageMs=%v validSamples=%d missingSamples=%d terminalStage=%s consumerEligibility=unknown", run, key, query, enqueue, stageMS, s.validSamples, s.missingSamples, s.stage)
|
||||
}
|
||||
p.observation = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
func TestLocalSupplyObservationFirstEventsAndStages(t *testing.T) {
|
||||
started := time.Now()
|
||||
o := &supplyObservation{started: started, channels: map[string]*supplyChannel{"a": {stage: "initializing", changed: started, stageDuration: map[string]time.Duration{}}}}
|
||||
o.event("a", "sql_query", 0, false, false, started.Add(10*time.Millisecond))
|
||||
o.event("a", "page_ready", 2, false, false, started.Add(20*time.Millisecond))
|
||||
o.event("a", "redis_write", 1, true, false, started.Add(30*time.Millisecond))
|
||||
o.event("a", "channel_capacity", 1, false, false, started.Add(40*time.Millisecond))
|
||||
o.event("a", "sql_query", 0, false, false, started.Add(50*time.Millisecond))
|
||||
o.event("a", "redis_write", 0, true, false, started.Add(60*time.Millisecond))
|
||||
o.event("a", "upper_bound", 0, false, true, started.Add(70*time.Millisecond))
|
||||
o.event("unknown", "ignored", 0, false, false, started)
|
||||
s := o.channels["a"]
|
||||
if s.firstQuery == nil || *s.firstQuery != 10 || s.firstEnqueue == nil || *s.firstEnqueue != 30 || !s.ended || s.pending != 0 || s.version != 7 || s.stageDuration["channel_capacity"] != 10*time.Millisecond || s.stageDuration["sql_query"] != 20*time.Millisecond {
|
||||
t.Fatalf("observation=%+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplyObservationAccumulatesSubMillisecondStages(t *testing.T) {
|
||||
started := time.Unix(0, 0)
|
||||
o := &supplyObservation{started: started, channels: map[string]*supplyChannel{"a": {stage: "redis_read", changed: started, stageDuration: map[string]time.Duration{}}}}
|
||||
for i := 1; i <= 100000; i++ {
|
||||
stage := "redis_read"
|
||||
if i%2 == 1 {
|
||||
stage = "redis_write"
|
||||
}
|
||||
o.event("a", stage, 0, false, i == 100000, started.Add(time.Duration(i)*500*time.Microsecond))
|
||||
}
|
||||
if got := o.channels["a"].stageMilliseconds(started.Add(time.Hour)); got["redis_read"] != 25000 || got["redis_write"] != 25000 {
|
||||
t.Fatalf("stageMs=%v, want 25000ms per stage", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplyObservationFinalFlushPreservesFraction(t *testing.T) {
|
||||
started := time.Unix(0, 0)
|
||||
o := &supplyObservation{started: started, channels: map[string]*supplyChannel{"a": {stage: "redis_write", changed: started, stageDuration: map[string]time.Duration{}}}}
|
||||
o.event("a", "redis_read", 0, false, false, started.Add(600*time.Microsecond))
|
||||
o.event("a", "redis_write", 0, false, false, started.Add(1200*time.Microsecond))
|
||||
s := o.channels["a"]
|
||||
if got := s.stageMilliseconds(started.Add(1800 * time.Microsecond)); got["redis_write"] != 1 || got["redis_read"] != 0 {
|
||||
t.Fatalf("final stageMs=%v", got)
|
||||
}
|
||||
if s.stageDuration["redis_write"] != 1200*time.Microsecond || s.stageDuration["redis_read"] != 600*time.Microsecond {
|
||||
t.Fatalf("fractional duration lost: %v", s.stageDuration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplySamplesDistinguishReadyLeaseFutureAndClosedRound(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
p := testProducer(t, scannerFor(nil, 0), store, Config{})
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "supply"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
active, _ := store.RoundActiveKey("supply", 1, "a")
|
||||
lease, _ := store.RoundLeaseKey("supply", 1, "a")
|
||||
_, _ = store.RedisClient().Do(ctx, "ZADD", active, time.Now().Add(-time.Minute).UnixMilli(), "ready", time.Now().Add(time.Hour).UnixMilli(), "future")
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", lease, "leased", "token")
|
||||
values, err := p.sampleSupply(ctx, "supply", "a")
|
||||
if err != nil || !reflect.DeepEqual(values, []int64{1, 1, 1, 1, 1}) {
|
||||
t.Fatalf("sample=%v err=%v", values, err)
|
||||
}
|
||||
meta, _ := store.RunMetaKey("supply")
|
||||
for _, args := range [][]any{{"blocked:a", 1}, {"blocked:a", 0, "producerDone", 1}, {"producerDone", 0, "currentRound", 2}, {"currentRound", 1, "syncType", "replay"}} {
|
||||
_, _ = store.RedisClient().Do(ctx, "HSET", append([]any{meta}, args...)...)
|
||||
values, err = p.sampleSupply(ctx, "supply", "a")
|
||||
if err != nil || values[0] != 1 || values[1] != 0 {
|
||||
t.Fatalf("ineligible sample=%v err=%v", values, err)
|
||||
}
|
||||
}
|
||||
values, err = p.sampleSupply(ctx, "missing", "a")
|
||||
if err != nil || values[0] != 0 || values[1] != 0 {
|
||||
t.Fatalf("missing sample=%v err=%v", values, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplySamplerTracksWaitingChannelsAndStops(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := contextForTest(t)
|
||||
p := testProducer(t, scannerFor(nil, 0), store, Config{})
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "sampling"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finish := p.startObservation(ctx, "sampling", []string{"a", "b"}, time.Now())
|
||||
defer finish()
|
||||
o := p.observation
|
||||
p.observe("a", "channel_capacity", 1, false, false)
|
||||
p.observe("b", "sql_slot", 0, false, false)
|
||||
waitUntil(t, ctx, func() bool {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.channels["a"].validSamples > 0 && o.channels["b"].validSamples > 0
|
||||
})
|
||||
p.observe("a", "cancelled", 0, false, true)
|
||||
p.observe("b", "cancelled", 0, false, true)
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.channels["a"].stageDuration["channel_capacity"] < 900*time.Millisecond || o.channels["b"].stageDuration["sql_slot"] < 900*time.Millisecond {
|
||||
t.Fatalf("missing wait attribution: a=%+v b=%+v", o.channels["a"], o.channels["b"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package monthlyusagehistory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
func newZombieTestProducer(t *testing.T) (*Producer, *zset.Store) {
|
||||
t.Helper()
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(&pageScanner{cards: nil}, store, Config{PageSize: 100, RoundRetryEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if acquired, err := store.AcquireLockOrTakeover(ctx, producer.config.RunLockKey, producer.owner, "pending-check", producer.config.RunLockTTL); err != nil || !acquired {
|
||||
t.Fatalf("acquire producer lock=%t err=%v", acquired, err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = store.ReleaseLock(ctx, producer.config.RunLockKey, producer.owner) })
|
||||
return producer, store
|
||||
}
|
||||
|
||||
// TestHasPendingTasksReclaimsZombieRun guards the "zombie run" regression.
|
||||
//
|
||||
// A producer that dies before MarkProducerDone (for example the container is
|
||||
// replaced mid-run) leaves its run in status=producing with producerDone=0.
|
||||
// completeRunIfIdle and advanceRunRound both require producerDone=1, so such a
|
||||
// run can never be closed, never leaves the active index, and never gets the
|
||||
// retention TTL. Treating it as backpressure rejected every later request
|
||||
// forever; merely warning about it leaked keys and an index entry indefinitely.
|
||||
func TestHasPendingTasksReclaimsZombieRun(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// A genuinely in-flight run (producerDone set, consumer working) must keep
|
||||
// providing backpressure.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "running-run", Status: "running", ProducerDone: true, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := producer.hasPendingTasks(ctx, "new-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !pending {
|
||||
t.Fatal("a running run must still provide backpressure")
|
||||
}
|
||||
if _, err := store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), "running-run"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A zombie run must not block a new request, and must be promoted so the
|
||||
// normal lifecycle can drain and close it.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err = producer.hasPendingTasks(ctx, "new-run-2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending {
|
||||
t.Fatal("a producing run with producerDone=0 must not block new requests (zombie run)")
|
||||
}
|
||||
|
||||
meta, err := store.GetRunMeta(ctx, "zombie-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.ProducerDone {
|
||||
t.Fatal("the zombie run must be promoted (producerDone=1) so it can be closed")
|
||||
}
|
||||
if meta.Status != "running" {
|
||||
t.Fatalf("promoted run status = %q, want running", meta.Status)
|
||||
}
|
||||
// Promotion makes the run closable, which is what applies the retention TTL
|
||||
// and releases the active index entry.
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "zombie-run"); err != nil || !completed {
|
||||
t.Fatalf("promoted run must be closable: completed=%v err=%v", completed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPendingTasksNeverReclaimsReplayRun guards the boundary: a replay target
|
||||
// is also created as "producing", but it is driven by the replay engine under
|
||||
// its own lock, so its producerDone is not the producer's to set.
|
||||
func TestHasPendingTasksNeverReclaimsReplayRun(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{
|
||||
RunID: "replay-run", Status: "producing", ProducerDone: false, CurrentRound: 1,
|
||||
SyncType: "replay", SourceRunID: "source-run", ReplayCycle: 1,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending, err := producer.hasPendingTasks(ctx, "new-run"); err != nil || !pending {
|
||||
t.Fatalf("replay must retain backpressure: pending=%t err=%v", pending, err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "replay-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ProducerDone {
|
||||
t.Fatal("a replay run must not be promoted by the full-sync producer")
|
||||
}
|
||||
if meta.Status != "producing" {
|
||||
t.Fatalf("replay run status = %q, want producing", meta.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPendingTasksDropsStaleIndexEntry covers a second unrecoverable path:
|
||||
// the active index can list a run whose hash is already gone. Returning that as
|
||||
// an error made the request fail permanently (the consumer redelivers it
|
||||
// forever), which is the same "only fixable by hand-cleaning Redis" failure.
|
||||
func TestHasPendingTasksDropsStaleIndexEntry(t *testing.T) {
|
||||
producer, store := newZombieTestProducer(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Index a run that has no metadata hash.
|
||||
if _, err := store.RedisClient().Do(ctx, "SADD", store.ActiveRunsKey(), "ghost-run"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := producer.hasPendingTasks(ctx, "new-run")
|
||||
if err != nil {
|
||||
t.Fatalf("a stale index entry must not fail the request: %v", err)
|
||||
}
|
||||
if pending {
|
||||
t.Fatal("a stale index entry must not provide backpressure")
|
||||
}
|
||||
// The stale entry is removed so it does not have to be handled again.
|
||||
members, err := store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, member := range members {
|
||||
if member == "ghost-run" {
|
||||
t.Fatal("the stale index entry must be removed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceCreatesRunAfterZombie covers the end-to-end path through Produce:
|
||||
// a zombie run must not stop a new run from being created.
|
||||
func TestProduceCreatesRunAfterZombie(t *testing.T) {
|
||||
store, err := zset.New(testutil.Redis(t), "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
producer, err := New(&pageScanner{cards: []Card{
|
||||
{ID: 1, ChanKey: "channel-a", TempKey: "sim_yd_v5", Operator: 1},
|
||||
}}, store, Config{PageSize: 100, RoundRetryEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A zombie run from a crashed producer.
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A new request must still produce its own run.
|
||||
if err := producer.Produce(ctx, scopedMessage("fresh-run", "channel-a")); err != nil {
|
||||
t.Fatalf("a zombie run must not stop a new run: %v", err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "fresh-run")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The producer enqueues into the active ZSet; ActiveCount is the count it
|
||||
// maintains. EnqueuedCount is an accounting field the producer does not set.
|
||||
if !meta.ProducerDone || meta.ActiveCount != 1 {
|
||||
t.Fatalf("fresh run metadata = %+v, want producerDone with 1 active task", meta)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
package simstatus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
const FullSyncOperation = "sim_status_queue_full_sync"
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// productionStats mirrors the monthly usage producer so both full-sync paths
|
||||
// report a single comparable summary line. The status path previously had no
|
||||
// producer summary at all, which made a scope change impossible to verify.
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
scopeChannels int
|
||||
channels map[string]struct{}
|
||||
}
|
||||
|
||||
func newProductionStats(scopeChannels int) productionStats {
|
||||
return productionStats{channels: make(map[string]struct{}), scopeChannels: scopeChannels}
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=sim_status_queue_producer status=%s runId=%s snapshotMaxId=%d pages=%d scanned=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, maxID, s.pages, s.scanned, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, len(s.channels), s.scopeChannels, time.Since(startedAt).Milliseconds())
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
switch status {
|
||||
case "failed":
|
||||
g.Log().Error(ctx, message)
|
||||
case "skipped":
|
||||
g.Log().Warning(ctx, message)
|
||||
default:
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
PageSize int
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
PollInterval time.Duration
|
||||
GlobalHighWaterMark int64
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 1000
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "sim-status:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = time.Second
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
Operator int
|
||||
SyncEnabled bool
|
||||
}
|
||||
|
||||
type Scanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
Scan(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
Operator int `orm:"operator"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
}
|
||||
|
||||
// DatabaseScanner uses keyset pagination so a large status run does not hold
|
||||
// the entire card table in memory and does not degrade through OFFSET scans.
|
||||
type DatabaseScanner struct {
|
||||
metadata map[string]channelMetadata
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id).
|
||||
OrderDesc(columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read status snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) Scan(ctx context.Context, lastID, maxID int64, pageSize int) ([]Card, error) {
|
||||
if maxID <= lastID {
|
||||
return nil, nil
|
||||
}
|
||||
if s.metadata == nil {
|
||||
var rows []channelMetadata
|
||||
if err := g.DB().Model("t_sim_channels").Fields("chan_key,temp_key,sync_enabled,operator").Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan status channel metadata: %w", err)
|
||||
}
|
||||
s.metadata = make(map[string]channelMetadata, len(rows))
|
||||
for _, row := range rows {
|
||||
s.metadata[row.ChanKey] = row
|
||||
}
|
||||
}
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
var rows []entity.TSimCardInfo
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id, columns.Iccid, columns.Msisdn, columns.Imsi, columns.ChanKey).
|
||||
WhereGT(columns.Id, lastID).
|
||||
WhereLTE(columns.Id, maxID).
|
||||
OrderAsc(columns.Id).
|
||||
Limit(pageSize).
|
||||
Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan status cards: %w", err)
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.Id <= 0 || row.Iccid == "" || row.ChanKey == "" {
|
||||
// Keep the page cursor moving even when a row cannot become a
|
||||
// runnable task. The producer advances by the highest returned ID.
|
||||
if row.Id > 0 {
|
||||
cards = append(cards, Card{ID: row.Id})
|
||||
}
|
||||
continue
|
||||
}
|
||||
metadata := s.metadata[row.ChanKey]
|
||||
cards = append(cards, Card{ID: row.Id, ICCID: row.Iccid, MSISDN: row.Msisdn, IMSI: row.Imsi, ChanKey: row.ChanKey, TempKey: metadata.TempKey, Operator: metadata.Operator, SyncEnabled: metadata.SyncEnabled})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner Scanner
|
||||
store *queue.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("sim status producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected.
|
||||
var ErrRunFinished = fmt.Errorf("sim status run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
func New(scanner Scanner, store *queue.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("sim status producer dependencies are incomplete")
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config.withDefaults(), owner: newOwner()}, nil
|
||||
}
|
||||
|
||||
func (p *Producer) Produce(ctx context.Context, message model.SimStatusQueueStartMessage) error {
|
||||
startedAt := time.Now()
|
||||
if message.RunID == "" {
|
||||
return fmt.Errorf("status sync runId is empty")
|
||||
}
|
||||
operation := message.Operation
|
||||
if operation == "" {
|
||||
operation = FullSyncOperation
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
failedStats := newProductionStats(0)
|
||||
failedStats.log(ctx, "failed", message.RunID, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats := newProductionStats(scope.ChannelCount())
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
acquired, err := p.store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_, _ = p.store.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
if err := p.store.CreateRun(ctx, queue.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: operation,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "run_init", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
lockCtx, stopRenew := context.WithCancel(ctx)
|
||||
renewErr := make(chan error, 1)
|
||||
go p.renewLock(lockCtx, stopRenew, renewErr)
|
||||
defer func() {
|
||||
stopRenew()
|
||||
select {
|
||||
case <-renewErr:
|
||||
default:
|
||||
}
|
||||
}()
|
||||
maxID, err := p.scanner.SnapshotMaxID(ctx)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "snapshot", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=sim_status_queue_producer status=started runId=%s snapshotMaxId=%d scopeChannels=%d", message.RunID, maxID, scope.ChannelCount())
|
||||
stage := "scan"
|
||||
defer func() {
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
lastID := int64(0)
|
||||
for lastID < maxID {
|
||||
select {
|
||||
case err := <-renewErr:
|
||||
return err
|
||||
default:
|
||||
}
|
||||
stage = "scan"
|
||||
cards, scanErr := p.scanner.Scan(ctx, lastID, maxID, p.config.PageSize)
|
||||
if scanErr != nil {
|
||||
return scanErr
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
stats.pages++
|
||||
stats.scanned += int64(len(cards))
|
||||
groups := make(map[string][]queue.Task)
|
||||
for _, card := range cards {
|
||||
if card.ID > lastID {
|
||||
lastID = card.ID
|
||||
}
|
||||
if card.ChanKey == "" {
|
||||
stats.skippedEmptyChan++
|
||||
continue
|
||||
}
|
||||
// Scope is applied before the template and sync_enabled checks so a
|
||||
// partial run only reports on the channels it was asked for.
|
||||
if !scope.Allows(card.ChanKey) {
|
||||
stats.skippedOutOfScope++
|
||||
continue
|
||||
}
|
||||
if card.TempKey == "" {
|
||||
stats.skippedNoTemplate++
|
||||
continue
|
||||
}
|
||||
if card.SyncEnabled {
|
||||
// sync_enabled=1(否/不参与) → skip
|
||||
stats.skippedSyncDisabled++
|
||||
continue
|
||||
}
|
||||
stats.channels[card.ChanKey] = struct{}{}
|
||||
task := queue.Task{TaskID: queue.StableTaskID(message.RunID, operation, card.ID), RunID: message.RunID, Operation: operation, ChanKey: card.ChanKey, TempKey: card.TempKey, Operator: card.Operator, CardID: card.ID, ICCID: card.ICCID, MSISDN: card.MSISDN, IMSI: card.IMSI}
|
||||
groups[card.ChanKey] = append(groups[card.ChanKey], task)
|
||||
}
|
||||
for chanKey, tasks := range groups {
|
||||
stage = "enqueue"
|
||||
added, enqueueErr := p.store.EnqueueRunBatch(ctx, message.RunID, chanKey, tasks, time.Now())
|
||||
if enqueueErr != nil {
|
||||
return fmt.Errorf("enqueue status channel %s: %w", chanKey, enqueueErr)
|
||||
}
|
||||
stats.enqueued += int64(added)
|
||||
}
|
||||
stage = "capacity"
|
||||
if err = p.waitForCapacity(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return p.store.MarkProducerDone(ctx, message.RunID)
|
||||
}
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelFunc, errors chan<- error) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ok, err := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err == nil && !ok {
|
||||
err = ErrAlreadyRunning
|
||||
}
|
||||
if err != nil {
|
||||
select {
|
||||
case errors <- err:
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) waitForCapacity(ctx context.Context) error {
|
||||
if p.config.GlobalHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, runID := range runs {
|
||||
meta, sizeErr := p.store.GetRunMeta(ctx, runID)
|
||||
if sizeErr != nil {
|
||||
// The active index can list a run whose hash is already gone
|
||||
// (expired, evicted, or cleaned by hand). Treat it as a stale
|
||||
// index entry and drop it instead of failing the scan.
|
||||
if errors.Is(sizeErr, queue.ErrRunNotFound) {
|
||||
if _, removeErr := p.store.RedisClient().Do(ctx, "SREM", p.store.ActiveRunsKey(), runID); removeErr != nil {
|
||||
return removeErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=sim_status_queue_producer status=skipped stage=capacity reason=stale_index runId=%s", runID)
|
||||
continue
|
||||
}
|
||||
return sizeErr
|
||||
}
|
||||
// ActiveCount includes leased tasks; do not count InflightCount twice.
|
||||
total += meta.ActiveCount + meta.NextRoundCount
|
||||
}
|
||||
if total <= p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(p.config.PollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) EnqueueCard(ctx context.Context, message model.SimStatusQueueTaskMessage) error {
|
||||
if message.CardID <= 0 || message.ICCID == "" || message.ChanKey == "" || message.TempKey == "" {
|
||||
return fmt.Errorf("invalid status sync task")
|
||||
}
|
||||
if message.RunID == "" {
|
||||
message.RunID = "status-card-" + newOwner()
|
||||
}
|
||||
if message.Operation == "" {
|
||||
message.Operation = "sim_status_queue_task"
|
||||
}
|
||||
if err := p.store.CreateRun(ctx, queue.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: message.Operation,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
task := queue.Task{TaskID: queue.StableTaskID(message.RunID, message.Operation, message.CardID), RunID: message.RunID, Operation: message.Operation, ChanKey: message.ChanKey, TempKey: message.TempKey, Operator: message.Operator, CardID: message.CardID, ICCID: message.ICCID, MSISDN: message.MSISDN, IMSI: message.IMSI}
|
||||
if _, err := p.store.EnqueueRunBatch(ctx, message.RunID, message.ChanKey, []queue.Task{task}, time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.store.MarkProducerDone(ctx, message.RunID)
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var b [12]byte
|
||||
if _, err := rand.Read(b[:]); err == nil {
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
return fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package simstatus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type cardScanner struct{ cards []Card }
|
||||
|
||||
func (s cardScanner) SnapshotMaxID(context.Context) (int64, error) { return int64(len(s.cards)), nil }
|
||||
func (s cardScanner) Scan(_ context.Context, last, max int64, limit int) ([]Card, error) {
|
||||
end := last + int64(limit)
|
||||
if end > max {
|
||||
end = max
|
||||
}
|
||||
return s.cards[last:end], nil
|
||||
}
|
||||
|
||||
func TestProducerUsesRunQueueForFullAndSingleCard(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
ctx := context.Background()
|
||||
store, err := queue.New(redis, "sim-status:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanner := cardScanner{cards: []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5", SyncEnabled: true},
|
||||
{ID: 3, ICCID: "c", ChanKey: "ch"},
|
||||
{ID: 4, ICCID: "d", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
}}
|
||||
producer, err := New(scanner, store, Config{PageSize: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err = producer.Produce(ctx, model.SimStatusQueueStartMessage{RunID: "full"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "full")
|
||||
if err != nil || !meta.ProducerDone || meta.ActiveCount != 2 || meta.Status != "running" {
|
||||
t.Fatalf("full metadata=%+v err=%v", meta, err)
|
||||
}
|
||||
claims, err := store.ClaimRun(ctx, "full", "ch", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil || len(claims) != 2 {
|
||||
t.Fatalf("full claims=%+v err=%v", claims, err)
|
||||
}
|
||||
if n, err := store.Size(ctx, "ch"); err != nil || n != 0 {
|
||||
t.Fatalf("legacy queue populated: %d %v", n, err)
|
||||
}
|
||||
message := model.SimStatusQueueTaskMessage{RunID: "single", CardID: 42, ICCID: "single", ChanKey: "ch", TempKey: "sim_yd_v5"}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err = producer.EnqueueCard(ctx, message); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
meta, err = store.GetRunMeta(ctx, "single")
|
||||
if err != nil || !meta.ProducerDone || meta.ActiveCount != 1 {
|
||||
t.Fatalf("single metadata=%+v err=%v", meta, err)
|
||||
}
|
||||
claims, err = store.ClaimRun(ctx, "single", "ch", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil || len(claims) != 1 {
|
||||
t.Fatalf("single claims=%+v err=%v", claims, err)
|
||||
}
|
||||
if n, err := store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}, 1); err != nil || n != 1 {
|
||||
t.Fatalf("single finalize: %d %v", n, err)
|
||||
}
|
||||
if ok, err := store.CompleteRunIfIdle(ctx, "single"); err != nil || !ok {
|
||||
t.Fatalf("single complete: %v %v", ok, err)
|
||||
}
|
||||
if err = producer.EnqueueCard(ctx, message); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err = store.GetRunMeta(ctx, "single")
|
||||
if err != nil || meta.Status != "completed" || meta.SuccessCount != 1 || meta.ActiveCount != 0 {
|
||||
t.Fatalf("replayed single=%+v err=%v", meta, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducerBackpressureIncludesLeasedRunTasks(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
producer, err := New(cardScanner{}, store, Config{GlobalHighWaterMark: 1, PollInterval: time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.CreateRun(ctx, queue.RunMeta{RunID: "busy"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks := []queue.Task{{RunID: "busy", TaskID: "1", ChanKey: "ch"}, {RunID: "busy", TaskID: "2", ChanKey: "ch"}}
|
||||
if _, err = store.EnqueueRunBatch(ctx, "busy", "ch", tasks, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claims, err := store.ClaimRun(ctx, "busy", "ch", time.Now(), time.Now().Add(time.Minute), 2)
|
||||
if err != nil || len(claims) != 2 {
|
||||
t.Fatalf("claim=%v err=%v", claims, err)
|
||||
}
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 30*time.Millisecond)
|
||||
defer cancel()
|
||||
if err = producer.waitForCapacity(waitCtx); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("did not wait on run backlog: %v", err)
|
||||
}
|
||||
if _, err = store.FinalizeRunTasks(ctx, claims[:1], []queue.TaskOutcome{{Success: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// One remaining lease meets the watermark: InflightCount must not be added again.
|
||||
waitCtx2, cancel2 := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel2()
|
||||
if err = producer.waitForCapacity(waitCtx2); err != nil {
|
||||
t.Fatalf("double-counted lease: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package simstatus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
func newScopeTestProducer(t *testing.T, cards []Card) (*Producer, *queue.Store) {
|
||||
t.Helper()
|
||||
store, err := queue.New(testutil.Redis(t), "sim-status:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(cardScanner{cards: cards}, store, Config{PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return producer, store
|
||||
}
|
||||
|
||||
func scopeMessage(runID string, channels ...string) model.SimStatusQueueStartMessage {
|
||||
return model.SimStatusQueueStartMessage{
|
||||
RunID: runID,
|
||||
Scope: &model.SyncScope{Channels: channels},
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeEnqueuesOnlySelectedChannels(t *testing.T) {
|
||||
cards := []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "217", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "173", TempKey: "sim_yd_v5"},
|
||||
{ID: 3, ICCID: "c", ChanKey: "217", TempKey: "sim_yd_v5"},
|
||||
{ID: 4, ICCID: "d", ChanKey: "189", TempKey: "sim_yd_v5"},
|
||||
}
|
||||
producer, store := newScopeTestProducer(t, cards)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := producer.Produce(ctx, scopeMessage("scoped", "217")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
channels, err := store.RunChannels(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(channels) != 1 || channels[0] != "217" {
|
||||
t.Fatalf("run channels = %v, want only 217", channels)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ScopeChannels != 1 {
|
||||
t.Fatalf("scopeChannels = %d, want 1", meta.ScopeChannels)
|
||||
}
|
||||
if meta.ScopeDigest == "" {
|
||||
t.Fatalf("scopeDigest must be recorded for a partial run")
|
||||
}
|
||||
if meta.ScopeChannelsSample != "217" {
|
||||
t.Fatalf("scopeChannelsSample = %q, want 217", meta.ScopeChannelsSample)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceFullScopeRecordsNoScopeMetadata(t *testing.T) {
|
||||
cards := []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "217", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "173", TempKey: "sim_yd_v5"},
|
||||
}
|
||||
producer, store := newScopeTestProducer(t, cards)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := producer.Produce(ctx, model.SimStatusQueueStartMessage{RunID: "full"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "full")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ScopeChannels != 0 || meta.ScopeDigest != "" || meta.ScopeChannelsSample != "" {
|
||||
t.Fatalf("full run must not record scope metadata: %+v", meta)
|
||||
}
|
||||
channels, err := store.RunChannels(ctx, "full")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(channels) != 2 {
|
||||
t.Fatalf("run channels = %v, want both channels", channels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeCombinesWithSyncDisabledGuard(t *testing.T) {
|
||||
cards := []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "217", TempKey: "sim_yd_v5"},
|
||||
// Selected but excluded by the sync_enabled guard: scope must not
|
||||
// override it, and the run must still complete cleanly.
|
||||
{ID: 2, ICCID: "b", ChanKey: "173", TempKey: "sim_yd_v5", SyncEnabled: true},
|
||||
{ID: 3, ICCID: "c", ChanKey: "189", TempKey: "sim_yd_v5"},
|
||||
}
|
||||
producer, store := newScopeTestProducer(t, cards)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := producer.Produce(ctx, scopeMessage("scoped", "173")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
channels, err := store.RunChannels(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(channels) != 0 {
|
||||
t.Fatalf("run channels = %v, want none (only selected channel is sync-disabled)", channels)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The producer never enqueues work for a sync-disabled channel. The run is
|
||||
// closed by the consumer's idle check, which is simulated here.
|
||||
if !meta.ProducerDone || meta.ActiveCount != 0 || meta.EnqueuedCount != 0 {
|
||||
t.Fatalf("empty scoped run metadata = %+v", meta)
|
||||
}
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "scoped"); err != nil || !completed {
|
||||
t.Fatalf("empty scoped run must be closable: completed=%v err=%v", completed, err)
|
||||
}
|
||||
meta, err = store.GetRunMeta(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.Status != "completed" {
|
||||
t.Fatalf("status = %q, want completed", meta.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeSelectingUnknownChannelProducesEmptyRun(t *testing.T) {
|
||||
cards := []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "217", TempKey: "sim_yd_v5"},
|
||||
}
|
||||
producer, store := newScopeTestProducer(t, cards)
|
||||
ctx := context.Background()
|
||||
|
||||
// An unknown channel is not an error; it simply matches no card.
|
||||
if err := producer.Produce(ctx, scopeMessage("scoped", "does-not-exist")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.ProducerDone || meta.EnqueuedCount != 0 || meta.ActiveCount != 0 {
|
||||
t.Fatalf("unknown-channel run metadata = %+v", meta)
|
||||
}
|
||||
if meta.ScopeChannels != 1 {
|
||||
t.Fatalf("scopeChannels = %d, want the requested 1", meta.ScopeChannels)
|
||||
}
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "scoped"); err != nil || !completed {
|
||||
t.Fatalf("unknown-channel run must be closable: completed=%v err=%v", completed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceRejectsInvalidScope(t *testing.T) {
|
||||
producer, store := newScopeTestProducer(t, []Card{{ID: 1, ICCID: "a", ChanKey: "217", TempKey: "sim_yd_v5"}})
|
||||
ctx := context.Background()
|
||||
|
||||
err := producer.Produce(ctx, scopeMessage("bad", "invalid/chan"))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid chanKey to be rejected")
|
||||
}
|
||||
// A rejected request must not leave a run behind.
|
||||
if _, metaErr := store.GetRunMeta(ctx, "bad"); metaErr == nil {
|
||||
t.Fatal("rejected scope created a run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceScopeDeduplicatesChannels(t *testing.T) {
|
||||
producer, store := newScopeTestProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "217", TempKey: "sim_yd_v5"},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
if err := producer.Produce(ctx, scopeMessage("scoped", "217", "217", "217")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ScopeChannels != 1 {
|
||||
t.Fatalf("scopeChannels = %d, want 1 after deduplication", meta.ScopeChannels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceRejectsRedeliveryOfFinishedRun(t *testing.T) {
|
||||
producer, store := newScopeTestProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "217", TempKey: "sim_yd_v5"},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
if err := producer.Produce(ctx, model.SimStatusQueueStartMessage{RunID: "done"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Drain the queued work so the run reaches a terminal state.
|
||||
claims, err := store.ClaimRun(ctx, "done", "217", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil || len(claims) != 1 {
|
||||
t.Fatalf("claims=%v err=%v", claims, err)
|
||||
}
|
||||
if _, err = store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "done"); err != nil || !completed {
|
||||
t.Fatalf("complete run: completed=%v err=%v", completed, err)
|
||||
}
|
||||
|
||||
// A redelivered start message must not resurrect the finished run.
|
||||
err = producer.Produce(ctx, model.SimStatusQueueStartMessage{RunID: "done"})
|
||||
if !errors.Is(err, ErrRunFinished) {
|
||||
t.Fatalf("redelivery error = %v, want ErrRunFinished", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user