chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,485 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
const operation = "current_month_usage_full_sync"
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
// The run hash and its archive JSON must stay bounded regardless of how many
|
||||
// channels a scope selects.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels. It lets
|
||||
// operators confirm two runs covered the same scope without storing the list.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
SupplyObservation *supply.Tracker
|
||||
// Empty/zero mean omitted for Go callers; startup config validates explicit values.
|
||||
ScanConcurrency int
|
||||
PageSize int
|
||||
GlobalHighWaterMark int64
|
||||
ChannelHighWaterMark int64
|
||||
PollInterval time.Duration
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
RoundRetryEnabled bool
|
||||
MaxRounds int
|
||||
FailureRetention time.Duration
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
c.RoundRetryEnabled = true
|
||||
if c.ScanConcurrency == 0 {
|
||||
c.ScanConcurrency = 4
|
||||
}
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 100
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = 2 * time.Second
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "flow:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.MaxRounds <= 0 {
|
||||
c.MaxRounds = 5
|
||||
}
|
||||
if c.FailureRetention <= 0 {
|
||||
c.FailureRetention = 30 * 24 * time.Hour
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type CardScanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
SyncEnabled bool
|
||||
Operator int
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
Operator int `orm:"operator"`
|
||||
}
|
||||
|
||||
type DatabaseScanner struct{}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := monthlyUsageEligibleCardModel(ctx).
|
||||
Fields(cardTable + "." + columns.Id + " AS " + columns.Id).
|
||||
OrderDesc(cardTable + "." + columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read card snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func monthlyUsageEligibleCardModel(ctx context.Context) *gdb.Model {
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
deptTable := dao.SysDept.Table()
|
||||
cardColumns := dao.TSimCardInfo.Columns()
|
||||
deptColumns := dao.SysDept.Columns()
|
||||
|
||||
return dao.TSimCardInfo.Ctx(ctx).
|
||||
InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId).
|
||||
Where(cardTable+"."+cardColumns.Iccid+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.ChanKey+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", "").
|
||||
WhereNull(deptTable + "." + deptColumns.DeletedAt)
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner CardScanner
|
||||
store *zset.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
observation *supply.Observer
|
||||
}
|
||||
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
eligible int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
skippedChannels int64
|
||||
stoppedTasks int64
|
||||
channels map[string]struct{}
|
||||
scopeChannels int
|
||||
}
|
||||
|
||||
func (s *productionStats) channelCount() int {
|
||||
return len(s.channels)
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, queryMonth, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=monthly_usage_producer status=%s runId=%s queryMonth=%s scanUpperID=%d pages=%d scanned=%d eligible=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d stoppedTasks=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, queryMonth, maxID, s.pages, s.scanned, s.eligible, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.stoppedTasks, s.channelCount(), s.scopeChannels, time.Since(startedAt).Milliseconds())
|
||||
message += fmt.Sprintf(" skippedChannels=%d", s.skippedChannels)
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
if status == "failed" {
|
||||
g.Log().Error(ctx, message)
|
||||
return
|
||||
}
|
||||
if status == "skipped" {
|
||||
g.Log().Warning(ctx, message)
|
||||
return
|
||||
}
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
|
||||
func New(scanner CardScanner, store *zset.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("monthly usage producer dependencies are incomplete")
|
||||
}
|
||||
config = config.withDefaults()
|
||||
if err := ValidateScanSettings(config.ScanConcurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.RoundRetryEnabled {
|
||||
if _, ok := scanner.(channelScanner); !ok {
|
||||
return nil, fmt.Errorf("channel scan requires ListChannels and ScanChannel capabilities")
|
||||
}
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config, owner: newOwner()}, nil
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("monthly usage producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected; callers
|
||||
// should log and ack instead of retrying.
|
||||
var ErrRunFinished = fmt.Errorf("monthly usage run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
func (p *Producer) Produce(ctx context.Context, message model.TrafficUsageMonthlyStartMessage) (err error) {
|
||||
startedAt := time.Now()
|
||||
stats := productionStats{channels: make(map[string]struct{})}
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
|
||||
if message.RunID == "" {
|
||||
err = fmt.Errorf("runId is empty")
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats.scopeChannels = scope.ChannelCount()
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if p.config.RoundRetryEnabled {
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
}
|
||||
acquired, err := p.store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer releaseCancel()
|
||||
_, _ = p.store.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
pending, err := p.hasPendingTasks(ctx, message.RunID)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "pending_check", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if pending {
|
||||
stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "pending_check", "pending_tasks", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
renewErr := make(chan error, 1)
|
||||
renewDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(renewDone)
|
||||
p.renewLock(runCtx, cancel, renewErr)
|
||||
}()
|
||||
defer func() {
|
||||
cancel()
|
||||
<-renewDone
|
||||
select {
|
||||
case lockErr := <-renewErr:
|
||||
err = lockErr
|
||||
default:
|
||||
}
|
||||
}()
|
||||
ctx = runCtx
|
||||
if message.Operation == "" {
|
||||
message.Operation = operation
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
if err = p.store.CreateRun(ctx, zset.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: message.Operation,
|
||||
QueryMonth: message.QueryMonth,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_init", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
maxID, err := p.scanner.SnapshotMaxID(ctx)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, "snapshot", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_producer status=started runId=%s queryMonth=%s scanUpperID=%d scopeChannels=%d scanMode=channel scanConcurrency=%d pageSize=%d", message.RunID, message.QueryMonth, maxID, scope.ChannelCount(), p.config.ScanConcurrency, p.config.PageSize)
|
||||
stage := "scan"
|
||||
defer func() {
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, message.QueryMonth, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, message.QueryMonth, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
if p.config.RoundRetryEnabled {
|
||||
stage = "channel_scan"
|
||||
if err = p.produceChannels(ctx, message, scope, maxID, startedAt, &stats); err != nil {
|
||||
if isProductionClosed(err) {
|
||||
// A same-Run redelivery after producerDone leaves consumption alone.
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Verify ownership immediately before closing production. The periodic
|
||||
// renewer still cancels scanners and capacity waits on lease loss.
|
||||
var owned bool
|
||||
owned, err = p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owned {
|
||||
return fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "producer_done"
|
||||
err = p.store.MarkProducerDone(ctx, message.RunID)
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("monthly usage full sync requires Run queue; legacy serial production has been removed")
|
||||
}
|
||||
|
||||
// isRunNotFound reports whether err means the run's metadata hash is absent, as
|
||||
// opposed to a genuine read failure.
|
||||
func isRunNotFound(err error) bool {
|
||||
return errors.Is(err, zset.ErrRunNotFound)
|
||||
}
|
||||
|
||||
func (p *Producer) hasPendingTasks(ctx context.Context, currentRunID ...string) (bool, error) {
|
||||
currentID := ""
|
||||
if len(currentRunID) > 0 {
|
||||
currentID = currentRunID[0]
|
||||
}
|
||||
if p.config.RoundRetryEnabled {
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, runID := range runs {
|
||||
if runID == currentID {
|
||||
// A redelivered message for the same run may resume a producer
|
||||
// that stopped before producerDone. Stable task IDs make the
|
||||
// rescan idempotent; other runs still provide backpressure.
|
||||
continue
|
||||
}
|
||||
meta, metaErr := p.store.GetRunMeta(ctx, runID)
|
||||
if metaErr != nil {
|
||||
// The active index is a global key written outside the run-scoped
|
||||
// Lua transaction, so it can list a run whose hash is already gone
|
||||
// (expired, evicted, or cleaned by hand). Returning an error here
|
||||
// would surface as a non-retryable failure and make the consumer
|
||||
// redeliver this request forever. Treat it as a stale index entry
|
||||
// and drop it.
|
||||
if isRunNotFound(metaErr) {
|
||||
if _, removeErr := p.store.RedisClient().Do(ctx, "SREM", p.store.ActiveRunsKey(), runID); removeErr != nil {
|
||||
return false, removeErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=stale_index runId=%s", runID)
|
||||
continue
|
||||
}
|
||||
return false, metaErr
|
||||
}
|
||||
// A "running" run has already finished producing (producerDone=1)
|
||||
// and the consumer owns the remaining work: it is genuinely in-flight
|
||||
// and must keep providing backpressure.
|
||||
if meta.Status == "running" {
|
||||
return true, nil
|
||||
}
|
||||
// A "producing" run for another ID cannot have a live producer. This
|
||||
// function only runs after AcquireLockOrTakeover succeeded on the
|
||||
// single producer lock, so any live producer would still hold it and
|
||||
// we could not have acquired it. Such a run is a zombie left behind by
|
||||
// a producer that died before MarkProducerDone (for example the
|
||||
// process was replaced mid-run).
|
||||
//
|
||||
// Promote it so the normal lifecycle can drain and close it. Blocking
|
||||
// would reject every later request forever, and merely warning would
|
||||
// leak the run's keys and its active-index entry indefinitely.
|
||||
reclaimed, reclaimErr := p.store.MarkProducerAbandoned(ctx, runID)
|
||||
if reclaimErr != nil {
|
||||
return false, reclaimErr
|
||||
}
|
||||
g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=zombie_run runId=%s reclaimed=%t activeCount=%d inflightCount=%d nextRoundCount=%d", runID, reclaimed, meta.ActiveCount, meta.InflightCount, meta.NextRoundCount)
|
||||
}
|
||||
}
|
||||
channels, err := p.store.ActiveChannels(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, chanKey := range channels {
|
||||
size, sizeErr := p.store.Size(ctx, chanKey)
|
||||
if sizeErr != nil {
|
||||
return false, sizeErr
|
||||
}
|
||||
if size > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelFunc, errors chan<- error) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ok, err := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
if err != nil || !ok {
|
||||
// Cancellation during a renewal RPC is shutdown, not lease loss.
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
err = fmt.Errorf("producer lock token is no longer owned")
|
||||
}
|
||||
select {
|
||||
case errors <- fmt.Errorf("renew producer lock: %w", err):
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var bytes [12]byte
|
||||
if _, err := rand.Read(bytes[:]); err == nil {
|
||||
return "producer-" + hex.EncodeToString(bytes[:])
|
||||
}
|
||||
return fmt.Sprintf("producer-%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, duration time.Duration) error {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user