package monthlyusage import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "errors" "fmt" "strings" "sync/atomic" "time" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset" ) const operation = "current_month_usage_full_sync" // scopeSampleLimit bounds how many channel names are copied into run metadata. // The run hash and its archive JSON must stay bounded regardless of how many // channels a scope selects. const scopeSampleLimit = 10 // scopeDigest is a stable short fingerprint of the selected channels. It lets // operators confirm two runs covered the same scope without storing the list. func scopeDigest(scope *model.SyncScope) string { if scope == nil || len(scope.Channels) == 0 { return "" } digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n"))) return hex.EncodeToString(digest[:8]) } type Config struct { SupplyObservation *supply.Tracker // Empty/zero mean omitted for Go callers; startup config validates explicit values. ScanConcurrency int PageSize int GlobalHighWaterMark int64 ChannelHighWaterMark int64 PollInterval time.Duration RunLockKey string RunLockTTL time.Duration RoundRetryEnabled bool MaxRounds int FailureRetention time.Duration } func (c Config) withDefaults() Config { c.RoundRetryEnabled = true if c.ScanConcurrency == 0 { c.ScanConcurrency = 4 } if c.PageSize <= 0 { c.PageSize = 100 } if c.PollInterval <= 0 { c.PollInterval = 2 * time.Second } if c.RunLockKey == "" { c.RunLockKey = "flow:producer:active" } if c.RunLockTTL <= 0 { c.RunLockTTL = 5 * time.Minute } if c.MaxRounds <= 0 { c.MaxRounds = 5 } if c.FailureRetention <= 0 { c.FailureRetention = 30 * 24 * time.Hour } return c } type CardScanner interface { SnapshotMaxID(context.Context) (int64, error) } type Card struct { ID int64 ICCID string MSISDN string IMSI string ChanKey string TempKey string SyncEnabled bool Operator int } type channelMetadata struct { ChanKey string `orm:"chan_key"` TempKey string `orm:"temp_key"` SyncEnabled bool `orm:"sync_enabled"` Operator int `orm:"operator"` } type DatabaseScanner struct{} func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) { var card entity.TSimCardInfo cardTable := dao.TSimCardInfo.Table() columns := dao.TSimCardInfo.Columns() if err := monthlyUsageEligibleCardModel(ctx). Fields(cardTable + "." + columns.Id + " AS " + columns.Id). OrderDesc(cardTable + "." + columns.Id). Limit(1). Scan(&card); err != nil { return 0, fmt.Errorf("read card snapshot max id: %w", err) } return card.Id, nil } func monthlyUsageEligibleCardModel(ctx context.Context) *gdb.Model { cardTable := dao.TSimCardInfo.Table() deptTable := dao.SysDept.Table() cardColumns := dao.TSimCardInfo.Columns() deptColumns := dao.SysDept.Columns() return dao.TSimCardInfo.Ctx(ctx). InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId). Where(cardTable+"."+cardColumns.Iccid+" <> ?", ""). Where(cardTable+"."+cardColumns.ChanKey+" <> ?", ""). Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", ""). WhereNull(deptTable + "." + deptColumns.DeletedAt) } type Producer struct { scanner CardScanner store *zset.Store config Config running atomic.Int32 owner string observation *supply.Observer } type productionStats struct { pages int64 scanned int64 eligible int64 enqueued int64 skippedEmptyChan int64 skippedNoTemplate int64 skippedSyncDisabled int64 skippedOutOfScope int64 skippedChannels int64 stoppedTasks int64 channels map[string]struct{} scopeChannels int } func (s *productionStats) channelCount() int { return len(s.channels) } func (s *productionStats) log(ctx context.Context, status, runID, queryMonth, stage, reason string, startedAt time.Time, maxID int64, err error) { message := fmt.Sprintf("event=monthly_usage_producer status=%s runId=%s queryMonth=%s scanUpperID=%d pages=%d scanned=%d eligible=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d stoppedTasks=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, queryMonth, maxID, s.pages, s.scanned, s.eligible, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.stoppedTasks, s.channelCount(), s.scopeChannels, time.Since(startedAt).Milliseconds()) message += fmt.Sprintf(" skippedChannels=%d", s.skippedChannels) if s.scanned > 0 { message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds()) } if stage != "" { message += " stage=" + stage } if reason != "" { message += " reason=" + reason } if err != nil { message += fmt.Sprintf(" error=%q", err.Error()) } if status == "failed" { g.Log().Error(ctx, message) return } if status == "skipped" { g.Log().Warning(ctx, message) return } g.Log().Info(ctx, message) } func New(scanner CardScanner, store *zset.Store, config Config) (*Producer, error) { if scanner == nil || store == nil { return nil, fmt.Errorf("monthly usage producer dependencies are incomplete") } config = config.withDefaults() if err := ValidateScanSettings(config.ScanConcurrency); err != nil { return nil, err } if config.RoundRetryEnabled { if _, ok := scanner.(channelScanner); !ok { return nil, fmt.Errorf("channel scan requires ListChannels and ScanChannel capabilities") } } return &Producer{scanner: scanner, store: store, config: config, owner: newOwner()}, nil } var ErrAlreadyRunning = fmt.Errorf("monthly usage producer already running") // ErrRunFinished is returned when a redelivered start message names a run that // already reached a terminal state. The run must not be resurrected; callers // should log and ack instead of retrying. var ErrRunFinished = fmt.Errorf("monthly usage run already finished") // runAlreadyFinished reports whether runID exists and has already completed. // A missing run is not an error: the message may be the first delivery. func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) { meta, err := p.store.GetRunMeta(ctx, runID) if err != nil { if strings.Contains(err.Error(), "not found") { return false, nil } return false, err } return meta.Status == "completed" || meta.Status == "completed_with_failures", nil } func (p *Producer) Produce(ctx context.Context, message model.TrafficUsageMonthlyStartMessage) (err error) { startedAt := time.Now() stats := productionStats{channels: make(map[string]struct{})} if !p.running.CompareAndSwap(0, 1) { stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "precheck", "already_running", startedAt, 0, nil) return ErrAlreadyRunning } defer p.running.Store(0) if message.RunID == "" { err = fmt.Errorf("runId is empty") stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "", startedAt, 0, err) return err } scope, err := model.NormalizeScope(message.Scope) if err != nil { stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "invalid_scope", startedAt, 0, err) return err } stats.scopeChannels = scope.ChannelCount() // A redelivered start message must not resurrect a run that already // finished; that would create a second producer pass over the same RunID. if p.config.RoundRetryEnabled { if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil { stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_state", "", startedAt, 0, checkErr) return checkErr } else if finished { stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "run_state", "run_finished", startedAt, 0, nil) return ErrRunFinished } } acquired, err := p.store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL) if err != nil { stats.log(ctx, "failed", message.RunID, message.QueryMonth, "lock", "", startedAt, 0, err) return err } if !acquired { stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "lock", "already_running", startedAt, 0, nil) return ErrAlreadyRunning } runCtx, cancel := context.WithCancel(ctx) defer cancel() defer func() { releaseCtx, releaseCancel := context.WithTimeout(context.Background(), time.Second) defer releaseCancel() _, _ = p.store.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner) }() pending, err := p.hasPendingTasks(ctx, message.RunID) if err != nil { stats.log(ctx, "failed", message.RunID, message.QueryMonth, "pending_check", "", startedAt, 0, err) return err } if pending { stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "pending_check", "pending_tasks", startedAt, 0, nil) return ErrAlreadyRunning } renewErr := make(chan error, 1) renewDone := make(chan struct{}) go func() { defer close(renewDone) p.renewLock(runCtx, cancel, renewErr) }() defer func() { cancel() <-renewDone select { case lockErr := <-renewErr: err = lockErr default: } }() ctx = runCtx if message.Operation == "" { message.Operation = operation } if p.config.RoundRetryEnabled { if err = p.store.CreateRun(ctx, zset.RunMeta{ RunID: message.RunID, Operation: message.Operation, QueryMonth: message.QueryMonth, Status: "producing", CurrentRound: 1, ScopeChannels: scope.ChannelCount(), ScopeDigest: scopeDigest(scope), ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","), }); err != nil { stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_init", "", startedAt, 0, err) return err } } maxID, err := p.scanner.SnapshotMaxID(ctx) if err != nil { stats.log(ctx, "failed", message.RunID, message.QueryMonth, "snapshot", "", startedAt, 0, err) return err } g.Log().Infof(ctx, "event=monthly_usage_producer status=started runId=%s queryMonth=%s scanUpperID=%d scopeChannels=%d scanMode=channel scanConcurrency=%d pageSize=%d", message.RunID, message.QueryMonth, maxID, scope.ChannelCount(), p.config.ScanConcurrency, p.config.PageSize) stage := "scan" defer func() { if err != nil { stats.log(ctx, "failed", message.RunID, message.QueryMonth, stage, "", startedAt, maxID, err) return } stats.log(ctx, "completed", message.RunID, message.QueryMonth, "", "", startedAt, maxID, nil) }() if p.config.RoundRetryEnabled { stage = "channel_scan" if err = p.produceChannels(ctx, message, scope, maxID, startedAt, &stats); err != nil { if isProductionClosed(err) { // A same-Run redelivery after producerDone leaves consumption alone. return nil } return err } // Verify ownership immediately before closing production. The periodic // renewer still cancels scanners and capacity waits on lease loss. var owned bool owned, err = p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL) if err != nil { return err } if !owned { return fmt.Errorf("producer lock token is no longer owned") } if err = ctx.Err(); err != nil { return err } stage = "producer_done" err = p.store.MarkProducerDone(ctx, message.RunID) return err } return fmt.Errorf("monthly usage full sync requires Run queue; legacy serial production has been removed") } // isRunNotFound reports whether err means the run's metadata hash is absent, as // opposed to a genuine read failure. func isRunNotFound(err error) bool { return errors.Is(err, zset.ErrRunNotFound) } func (p *Producer) hasPendingTasks(ctx context.Context, currentRunID ...string) (bool, error) { currentID := "" if len(currentRunID) > 0 { currentID = currentRunID[0] } if p.config.RoundRetryEnabled { runs, err := p.store.ActiveRuns(ctx) if err != nil { return false, err } for _, runID := range runs { if runID == currentID { // A redelivered message for the same run may resume a producer // that stopped before producerDone. Stable task IDs make the // rescan idempotent; other runs still provide backpressure. continue } meta, metaErr := p.store.GetRunMeta(ctx, runID) if metaErr != nil { // The active index is a global key written outside the run-scoped // Lua transaction, so it can list a run whose hash is already gone // (expired, evicted, or cleaned by hand). Returning an error here // would surface as a non-retryable failure and make the consumer // redeliver this request forever. Treat it as a stale index entry // and drop it. if isRunNotFound(metaErr) { if _, removeErr := p.store.RedisClient().Do(ctx, "SREM", p.store.ActiveRunsKey(), runID); removeErr != nil { return false, removeErr } g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=stale_index runId=%s", runID) continue } return false, metaErr } // A "running" run has already finished producing (producerDone=1) // and the consumer owns the remaining work: it is genuinely in-flight // and must keep providing backpressure. if meta.Status == "running" { return true, nil } // A "producing" run for another ID cannot have a live producer. This // function only runs after AcquireLockOrTakeover succeeded on the // single producer lock, so any live producer would still hold it and // we could not have acquired it. Such a run is a zombie left behind by // a producer that died before MarkProducerDone (for example the // process was replaced mid-run). // // Promote it so the normal lifecycle can drain and close it. Blocking // would reject every later request forever, and merely warning would // leak the run's keys and its active-index entry indefinitely. reclaimed, reclaimErr := p.store.MarkProducerAbandoned(ctx, runID) if reclaimErr != nil { return false, reclaimErr } g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=zombie_run runId=%s reclaimed=%t activeCount=%d inflightCount=%d nextRoundCount=%d", runID, reclaimed, meta.ActiveCount, meta.InflightCount, meta.NextRoundCount) } } channels, err := p.store.ActiveChannels(ctx) if err != nil { return false, err } for _, chanKey := range channels { size, sizeErr := p.store.Size(ctx, chanKey) if sizeErr != nil { return false, sizeErr } if size > 0 { return true, nil } } return false, nil } func (p *Producer) renewLock(ctx context.Context, cancel context.CancelFunc, errors chan<- error) { interval := p.config.RunLockTTL / 3 if interval <= 0 { interval = time.Second } ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: ok, err := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL) if err != nil || !ok { // Cancellation during a renewal RPC is shutdown, not lease loss. if ctx.Err() != nil { return } if err == nil { err = fmt.Errorf("producer lock token is no longer owned") } select { case errors <- fmt.Errorf("renew producer lock: %w", err): default: } cancel() return } } } } func newOwner() string { var bytes [12]byte if _, err := rand.Read(bytes[:]); err == nil { return "producer-" + hex.EncodeToString(bytes[:]) } return fmt.Sprintf("producer-%d", time.Now().UnixNano()) } func sleepContext(ctx context.Context, duration time.Duration) error { timer := time.NewTimer(duration) defer timer.Stop() select { case <-timer.C: return nil case <-ctx.Done(): return ctx.Err() } }