iotlcm/worker-svr/internal/producer/monthlyusage/producer.go

package monthlyusage

import (
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"strings"
	"sync/atomic"
	"time"

	"github.com/gogf/gf/v2/database/gdb"
	"github.com/gogf/gf/v2/frame/g"

	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
)

const operation = "current_month_usage_full_sync"

// scopeSampleLimit bounds how many channel names are copied into run metadata.
// The run hash and its archive JSON must stay bounded regardless of how many
// channels a scope selects.
const scopeSampleLimit = 10

// scopeDigest is a stable short fingerprint of the selected channels. It lets
// operators confirm two runs covered the same scope without storing the list.
func scopeDigest(scope *model.SyncScope) string {
	if scope == nil || len(scope.Channels) == 0 {
		return ""
	}
	digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
	return hex.EncodeToString(digest[:8])
}

type Config struct {
	SupplyObservation *supply.Tracker
	// Empty/zero mean omitted for Go callers; startup config validates explicit values.
	ScanConcurrency      int
	PageSize             int
	GlobalHighWaterMark  int64
	ChannelHighWaterMark int64
	PollInterval         time.Duration
	RunLockKey           string
	RunLockTTL           time.Duration
	RoundRetryEnabled    bool
	MaxRounds            int
	FailureRetention     time.Duration
}

func (c Config) withDefaults() Config {
	c.RoundRetryEnabled = true
	if c.ScanConcurrency == 0 {
		c.ScanConcurrency = 4
	}
	if c.PageSize <= 0 {
		c.PageSize = 100
	}
	if c.PollInterval <= 0 {
		c.PollInterval = 2 * time.Second
	}
	if c.RunLockKey == "" {
		c.RunLockKey = "flow:producer:active"
	}
	if c.RunLockTTL <= 0 {
		c.RunLockTTL = 5 * time.Minute
	}
	if c.MaxRounds <= 0 {
		c.MaxRounds = 5
	}
	if c.FailureRetention <= 0 {
		c.FailureRetention = 30 * 24 * time.Hour
	}
	return c
}

type CardScanner interface {
	SnapshotMaxID(context.Context) (int64, error)
}

type Card struct {
	ID          int64
	ICCID       string
	MSISDN      string
	IMSI        string
	ChanKey     string
	TempKey     string
	SyncEnabled bool
	Operator    int
}

type channelMetadata struct {
	ChanKey     string `orm:"chan_key"`
	TempKey     string `orm:"temp_key"`
	SyncEnabled bool   `orm:"sync_enabled"`
	Operator    int    `orm:"operator"`
}

type DatabaseScanner struct{}

func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
	var card entity.TSimCardInfo
	cardTable := dao.TSimCardInfo.Table()
	columns := dao.TSimCardInfo.Columns()
	if err := monthlyUsageEligibleCardModel(ctx).
		Fields(cardTable + "." + columns.Id + " AS " + columns.Id).
		OrderDesc(cardTable + "." + columns.Id).
		Limit(1).
		Scan(&card); err != nil {
		return 0, fmt.Errorf("read card snapshot max id: %w", err)
	}
	return card.Id, nil
}

func monthlyUsageEligibleCardModel(ctx context.Context) *gdb.Model {
	cardTable := dao.TSimCardInfo.Table()
	deptTable := dao.SysDept.Table()
	cardColumns := dao.TSimCardInfo.Columns()
	deptColumns := dao.SysDept.Columns()

	return dao.TSimCardInfo.Ctx(ctx).
		InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId).
		Where(cardTable+"."+cardColumns.Iccid+" <> ?", "").
		Where(cardTable+"."+cardColumns.ChanKey+" <> ?", "").
		Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", "").
		WhereNull(deptTable + "." + deptColumns.DeletedAt)
}

type Producer struct {
	scanner     CardScanner
	store       *zset.Store
	config      Config
	running     atomic.Int32
	owner       string
	observation *supply.Observer
}

type productionStats struct {
	pages               int64
	scanned             int64
	eligible            int64
	enqueued            int64
	skippedEmptyChan    int64
	skippedNoTemplate   int64
	skippedSyncDisabled int64
	skippedOutOfScope   int64
	skippedChannels     int64
	stoppedTasks        int64
	channels            map[string]struct{}
	scopeChannels       int
}

func (s *productionStats) channelCount() int {
	return len(s.channels)
}

func (s *productionStats) log(ctx context.Context, status, runID, queryMonth, stage, reason string, startedAt time.Time, maxID int64, err error) {
	message := fmt.Sprintf("event=monthly_usage_producer status=%s runId=%s queryMonth=%s scanUpperID=%d pages=%d scanned=%d eligible=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d stoppedTasks=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, queryMonth, maxID, s.pages, s.scanned, s.eligible, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.stoppedTasks, s.channelCount(), s.scopeChannels, time.Since(startedAt).Milliseconds())
	message += fmt.Sprintf(" skippedChannels=%d", s.skippedChannels)
	if s.scanned > 0 {
		message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
	}
	if stage != "" {
		message += " stage=" + stage
	}
	if reason != "" {
		message += " reason=" + reason
	}
	if err != nil {
		message += fmt.Sprintf(" error=%q", err.Error())
	}
	if status == "failed" {
		g.Log().Error(ctx, message)
		return
	}
	if status == "skipped" {
		g.Log().Warning(ctx, message)
		return
	}
	g.Log().Info(ctx, message)
}

func New(scanner CardScanner, store *zset.Store, config Config) (*Producer, error) {
	if scanner == nil || store == nil {
		return nil, fmt.Errorf("monthly usage producer dependencies are incomplete")
	}
	config = config.withDefaults()
	if err := ValidateScanSettings(config.ScanConcurrency); err != nil {
		return nil, err
	}
	if config.RoundRetryEnabled {
		if _, ok := scanner.(channelScanner); !ok {
			return nil, fmt.Errorf("channel scan requires ListChannels and ScanChannel capabilities")
		}
	}
	return &Producer{scanner: scanner, store: store, config: config, owner: newOwner()}, nil
}

var ErrAlreadyRunning = fmt.Errorf("monthly usage producer already running")

// ErrRunFinished is returned when a redelivered start message names a run that
// already reached a terminal state. The run must not be resurrected; callers
// should log and ack instead of retrying.
var ErrRunFinished = fmt.Errorf("monthly usage run already finished")

// runAlreadyFinished reports whether runID exists and has already completed.
// A missing run is not an error: the message may be the first delivery.
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
	meta, err := p.store.GetRunMeta(ctx, runID)
	if err != nil {
		if strings.Contains(err.Error(), "not found") {
			return false, nil
		}
		return false, err
	}
	return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
}

func (p *Producer) Produce(ctx context.Context, message model.TrafficUsageMonthlyStartMessage) (err error) {
	startedAt := time.Now()
	stats := productionStats{channels: make(map[string]struct{})}
	if !p.running.CompareAndSwap(0, 1) {
		stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "precheck", "already_running", startedAt, 0, nil)
		return ErrAlreadyRunning
	}
	defer p.running.Store(0)

	if message.RunID == "" {
		err = fmt.Errorf("runId is empty")
		stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "", startedAt, 0, err)
		return err
	}
	scope, err := model.NormalizeScope(message.Scope)
	if err != nil {
		stats.log(ctx, "failed", message.RunID, message.QueryMonth, "validate", "invalid_scope", startedAt, 0, err)
		return err
	}
	stats.scopeChannels = scope.ChannelCount()
	// A redelivered start message must not resurrect a run that already
	// finished; that would create a second producer pass over the same RunID.
	if p.config.RoundRetryEnabled {
		if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
			stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_state", "", startedAt, 0, checkErr)
			return checkErr
		} else if finished {
			stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "run_state", "run_finished", startedAt, 0, nil)
			return ErrRunFinished
		}
	}
	acquired, err := p.store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
	if err != nil {
		stats.log(ctx, "failed", message.RunID, message.QueryMonth, "lock", "", startedAt, 0, err)
		return err
	}
	if !acquired {
		stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "lock", "already_running", startedAt, 0, nil)
		return ErrAlreadyRunning
	}
	runCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	defer func() {
		releaseCtx, releaseCancel := context.WithTimeout(context.Background(), time.Second)
		defer releaseCancel()
		_, _ = p.store.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
	}()
	pending, err := p.hasPendingTasks(ctx, message.RunID)
	if err != nil {
		stats.log(ctx, "failed", message.RunID, message.QueryMonth, "pending_check", "", startedAt, 0, err)
		return err
	}
	if pending {
		stats.log(ctx, "skipped", message.RunID, message.QueryMonth, "pending_check", "pending_tasks", startedAt, 0, nil)
		return ErrAlreadyRunning
	}
	renewErr := make(chan error, 1)
	renewDone := make(chan struct{})
	go func() {
		defer close(renewDone)
		p.renewLock(runCtx, cancel, renewErr)
	}()
	defer func() {
		cancel()
		<-renewDone
		select {
		case lockErr := <-renewErr:
			err = lockErr
		default:
		}
	}()
	ctx = runCtx
	if message.Operation == "" {
		message.Operation = operation
	}
	if p.config.RoundRetryEnabled {
		if err = p.store.CreateRun(ctx, zset.RunMeta{
			RunID:               message.RunID,
			Operation:           message.Operation,
			QueryMonth:          message.QueryMonth,
			Status:              "producing",
			CurrentRound:        1,
			ScopeChannels:       scope.ChannelCount(),
			ScopeDigest:         scopeDigest(scope),
			ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
		}); err != nil {
			stats.log(ctx, "failed", message.RunID, message.QueryMonth, "run_init", "", startedAt, 0, err)
			return err
		}
	}
	maxID, err := p.scanner.SnapshotMaxID(ctx)
	if err != nil {
		stats.log(ctx, "failed", message.RunID, message.QueryMonth, "snapshot", "", startedAt, 0, err)
		return err
	}
	g.Log().Infof(ctx, "event=monthly_usage_producer status=started runId=%s queryMonth=%s scanUpperID=%d scopeChannels=%d scanMode=channel scanConcurrency=%d pageSize=%d", message.RunID, message.QueryMonth, maxID, scope.ChannelCount(), p.config.ScanConcurrency, p.config.PageSize)
	stage := "scan"
	defer func() {
		if err != nil {
			stats.log(ctx, "failed", message.RunID, message.QueryMonth, stage, "", startedAt, maxID, err)
			return
		}
		stats.log(ctx, "completed", message.RunID, message.QueryMonth, "", "", startedAt, maxID, nil)
	}()
	if p.config.RoundRetryEnabled {
		stage = "channel_scan"
		if err = p.produceChannels(ctx, message, scope, maxID, startedAt, &stats); err != nil {
			if isProductionClosed(err) {
				// A same-Run redelivery after producerDone leaves consumption alone.
				return nil
			}
			return err
		}
		// Verify ownership immediately before closing production. The periodic
		// renewer still cancels scanners and capacity waits on lease loss.
		var owned bool
		owned, err = p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
		if err != nil {
			return err
		}
		if !owned {
			return fmt.Errorf("producer lock token is no longer owned")
		}
		if err = ctx.Err(); err != nil {
			return err
		}
		stage = "producer_done"
		err = p.store.MarkProducerDone(ctx, message.RunID)
		return err
	}
	return fmt.Errorf("monthly usage full sync requires Run queue; legacy serial production has been removed")
}

// isRunNotFound reports whether err means the run's metadata hash is absent, as
// opposed to a genuine read failure.
func isRunNotFound(err error) bool {
	return errors.Is(err, zset.ErrRunNotFound)
}

func (p *Producer) hasPendingTasks(ctx context.Context, currentRunID ...string) (bool, error) {
	currentID := ""
	if len(currentRunID) > 0 {
		currentID = currentRunID[0]
	}
	if p.config.RoundRetryEnabled {
		runs, err := p.store.ActiveRuns(ctx)
		if err != nil {
			return false, err
		}
		for _, runID := range runs {
			if runID == currentID {
				// A redelivered message for the same run may resume a producer
				// that stopped before producerDone.  Stable task IDs make the
				// rescan idempotent; other runs still provide backpressure.
				continue
			}
			meta, metaErr := p.store.GetRunMeta(ctx, runID)
			if metaErr != nil {
				// The active index is a global key written outside the run-scoped
				// Lua transaction, so it can list a run whose hash is already gone
				// (expired, evicted, or cleaned by hand). Returning an error here
				// would surface as a non-retryable failure and make the consumer
				// redeliver this request forever. Treat it as a stale index entry
				// and drop it.
				if isRunNotFound(metaErr) {
					if _, removeErr := p.store.RedisClient().Do(ctx, "SREM", p.store.ActiveRunsKey(), runID); removeErr != nil {
						return false, removeErr
					}
					g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=stale_index runId=%s", runID)
					continue
				}
				return false, metaErr
			}
			// A "running" run has already finished producing (producerDone=1)
			// and the consumer owns the remaining work: it is genuinely in-flight
			// and must keep providing backpressure.
			if meta.Status == "running" {
				return true, nil
			}
			// A "producing" run for another ID cannot have a live producer. This
			// function only runs after AcquireLockOrTakeover succeeded on the
			// single producer lock, so any live producer would still hold it and
			// we could not have acquired it. Such a run is a zombie left behind by
			// a producer that died before MarkProducerDone (for example the
			// process was replaced mid-run).
			//
			// Promote it so the normal lifecycle can drain and close it. Blocking
			// would reject every later request forever, and merely warning would
			// leak the run's keys and its active-index entry indefinitely.
			reclaimed, reclaimErr := p.store.MarkProducerAbandoned(ctx, runID)
			if reclaimErr != nil {
				return false, reclaimErr
			}
			g.Log().Warningf(ctx, "event=monthly_usage_producer status=skipped stage=pending_check reason=zombie_run runId=%s reclaimed=%t activeCount=%d inflightCount=%d nextRoundCount=%d", runID, reclaimed, meta.ActiveCount, meta.InflightCount, meta.NextRoundCount)
		}
	}
	channels, err := p.store.ActiveChannels(ctx)
	if err != nil {
		return false, err
	}
	for _, chanKey := range channels {
		size, sizeErr := p.store.Size(ctx, chanKey)
		if sizeErr != nil {
			return false, sizeErr
		}
		if size > 0 {
			return true, nil
		}
	}
	return false, nil
}

func (p *Producer) renewLock(ctx context.Context, cancel context.CancelFunc, errors chan<- error) {
	interval := p.config.RunLockTTL / 3
	if interval <= 0 {
		interval = time.Second
	}
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			ok, err := p.store.RenewLock(ctx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
			if err != nil || !ok {
				// Cancellation during a renewal RPC is shutdown, not lease loss.
				if ctx.Err() != nil {
					return
				}
				if err == nil {
					err = fmt.Errorf("producer lock token is no longer owned")
				}
				select {
				case errors <- fmt.Errorf("renew producer lock: %w", err):
				default:
				}
				cancel()
				return
			}
		}
	}
}

func newOwner() string {
	var bytes [12]byte
	if _, err := rand.Read(bytes[:]); err == nil {
		return "producer-" + hex.EncodeToString(bytes[:])
	}
	return fmt.Sprintf("producer-%d", time.Now().UnixNano())
}

func sleepContext(ctx context.Context, duration time.Duration) error {
	timer := time.NewTimer(duration)
	defer timer.Stop()
	select {
	case <-timer.C:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}