chore: 初始开源发布
iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。 - Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件） - 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、 库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异， 不破坏「校验 MSISDN 格式」等测试断言） - 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23）， 移除写死的 git 凭据 - README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 已知：3 个测试包在源仓同一提交上同样失败（client/upstream、 service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。
This commit is contained in:
MiFaZhan
@@ -0,0 +1,498 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// FullSyncOperation is recorded on the run and used by SyncKindForOperation to
|
||||
// classify the run for the admin monitor.
|
||||
const FullSyncOperation = "imei_full_sync"
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// productionStats mirrors the status producer so every full-sync path reports a
|
||||
// single comparable summary line.
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
skippedNoImei int64
|
||||
scopeChannels int
|
||||
channels map[string]struct{}
|
||||
}
|
||||
|
||||
func newProductionStats(scopeChannels int) productionStats {
|
||||
return productionStats{channels: make(map[string]struct{}), scopeChannels: scopeChannels}
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=imei_queue_producer status=%s runId=%s snapshotMaxId=%d pages=%d scanned=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d skippedNoImei=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, maxID, s.pages, s.scanned, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.skippedNoImei, len(s.channels), s.scopeChannels, time.Since(startedAt).Milliseconds())
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
switch status {
|
||||
case "failed":
|
||||
g.Log().Error(ctx, message)
|
||||
case "skipped":
|
||||
g.Log().Warning(ctx, message)
|
||||
default:
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
PageSize int
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
PollInterval time.Duration
|
||||
GlobalHighWaterMark int64
|
||||
// LeaseRedisConfig is required only for Sentinel discovery. Standalone
|
||||
// settings come from the actual adapter; Cluster is rejected in New.
|
||||
LeaseRedisConfig *gredis.Config
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 1000
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "imei:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = time.Second
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
Operator int
|
||||
SyncEnabled bool
|
||||
Imei string
|
||||
}
|
||||
|
||||
type Scanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
Scan(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
Operator int `orm:"operator"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
}
|
||||
|
||||
// DatabaseScanner uses keyset pagination so a large IMEI run does not hold the
|
||||
// entire card table in memory and does not degrade through OFFSET scans.
|
||||
type DatabaseScanner struct {
|
||||
metadata map[string]channelMetadata
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id).
|
||||
OrderDesc(columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read imei snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) Scan(ctx context.Context, lastID, maxID int64, pageSize int) ([]Card, error) {
|
||||
if maxID <= lastID {
|
||||
return nil, nil
|
||||
}
|
||||
if s.metadata == nil {
|
||||
var rows []channelMetadata
|
||||
if err := g.DB().Model("t_sim_channels").Ctx(ctx).Fields("chan_key,temp_key,sync_enabled,operator").Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan imei channel metadata: %w", err)
|
||||
}
|
||||
s.metadata = make(map[string]channelMetadata, len(rows))
|
||||
for _, row := range rows {
|
||||
s.metadata[row.ChanKey] = row
|
||||
}
|
||||
}
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
var rows []entity.TSimCardInfo
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id, columns.Iccid, columns.Msisdn, columns.Imsi, columns.ChanKey, columns.Imei).
|
||||
WhereGT(columns.Id, lastID).
|
||||
WhereLTE(columns.Id, maxID).
|
||||
OrderAsc(columns.Id).
|
||||
Limit(pageSize).
|
||||
Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan imei cards: %w", err)
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.Id <= 0 || row.Iccid == "" || row.ChanKey == "" {
|
||||
// Keep the page cursor moving even when a row cannot become a
|
||||
// runnable task. The producer advances by the highest returned ID.
|
||||
if row.Id > 0 {
|
||||
cards = append(cards, Card{ID: row.Id})
|
||||
}
|
||||
continue
|
||||
}
|
||||
metadata := s.metadata[row.ChanKey]
|
||||
cards = append(cards, Card{ID: row.Id, ICCID: row.Iccid, MSISDN: row.Msisdn, IMSI: row.Imsi, ChanKey: row.ChanKey, TempKey: metadata.TempKey, Operator: metadata.Operator, SyncEnabled: metadata.SyncEnabled, Imei: row.Imei})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner Scanner
|
||||
store *queue.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
leaseFactory leaseClientFactory
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("imei producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected.
|
||||
var ErrRunFinished = fmt.Errorf("imei run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
func New(scanner Scanner, store *queue.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("imei producer dependencies are incomplete")
|
||||
}
|
||||
config = config.withDefaults()
|
||||
factory, err := newLeaseClientFactory(store.RedisClient(), config.LeaseRedisConfig, leaseIOBudget(config.RunLockTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config, owner: newOwner(), leaseFactory: factory}, nil
|
||||
}
|
||||
|
||||
func leaseIOBudget(ttl time.Duration) time.Duration {
|
||||
return min(time.Second, ttl/6)
|
||||
}
|
||||
|
||||
func (p *Producer) Produce(ctx context.Context, message model.ImeiQueueStartMessage) (err error) {
|
||||
startedAt := time.Now()
|
||||
if message.RunID == "" {
|
||||
return fmt.Errorf("imei sync runId is empty")
|
||||
}
|
||||
operation := message.Operation
|
||||
if operation == "" {
|
||||
operation = FullSyncOperation
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
failedStats := newProductionStats(0)
|
||||
failedStats.log(ctx, "failed", message.RunID, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats := newProductionStats(scope.ChannelCount())
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
leaseClient, closeLeaseClient, err := p.leaseFactory()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeLeaseClient()
|
||||
leaseStore, err := queue.New(leaseClient, p.store.Prefix())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
// Tokens are per lease, not per Producer: a delayed command from an earlier
|
||||
// attempt must never match a later lease on this same Producer instance.
|
||||
p.owner = newOwner()
|
||||
acquireCtx, cancelAcquire := context.WithTimeout(ctx, leaseIOBudget(p.config.RunLockTTL))
|
||||
acquired, err := leaseStore.AcquireLockOrTakeover(acquireCtx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
cancelAcquire()
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), leaseIOBudget(p.config.RunLockTTL))
|
||||
defer cancel()
|
||||
_, _ = leaseStore.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
workStore, err := p.store.WithIMEIProducerLease(p.config.RunLockKey, p.owner, message.RunID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockCtx, cancelLock := context.WithCancelCause(ctx)
|
||||
// Stop renewal separately so normal cleanup does not become a lock-loss cause.
|
||||
renewCtx, stopRenew := context.WithCancel(lockCtx)
|
||||
renewDone := make(chan struct{})
|
||||
var (
|
||||
maxID int64
|
||||
stage = "run_init"
|
||||
)
|
||||
go func() {
|
||||
defer close(renewDone)
|
||||
p.renewLock(renewCtx, cancelLock, leaseStore)
|
||||
}()
|
||||
defer func() {
|
||||
stopRenew()
|
||||
<-renewDone
|
||||
// Join before reading the cause and releasing the lease: a failed renewal
|
||||
// must not be masked by a scanner/Redis context-cancellation error.
|
||||
if cause := context.Cause(lockCtx); cause != nil {
|
||||
err = cause
|
||||
}
|
||||
if errors.Is(err, queue.ErrProducerLeaseLost) {
|
||||
err = ErrAlreadyRunning
|
||||
}
|
||||
cancelLock(nil)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
if err = workStore.CreateRun(lockCtx, queue.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: operation,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "snapshot"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
maxID, err = p.scanner.SnapshotMaxID(lockCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=imei_queue_producer status=started runId=%s snapshotMaxId=%d scopeChannels=%d", message.RunID, maxID, scope.ChannelCount())
|
||||
lastID := int64(0)
|
||||
for lastID < maxID {
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "scan"
|
||||
cards, scanErr := p.scanner.Scan(lockCtx, lastID, maxID, p.config.PageSize)
|
||||
if scanErr != nil {
|
||||
return scanErr
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
stats.pages++
|
||||
stats.scanned += int64(len(cards))
|
||||
groups := make(map[string][]queue.Task)
|
||||
for _, card := range cards {
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if card.ID > lastID {
|
||||
lastID = card.ID
|
||||
}
|
||||
if card.ChanKey == "" {
|
||||
stats.skippedEmptyChan++
|
||||
continue
|
||||
}
|
||||
// Scope is applied before the template and sync_enabled checks so a
|
||||
// partial run only reports on the channels it was asked for.
|
||||
if !scope.Allows(card.ChanKey) {
|
||||
stats.skippedOutOfScope++
|
||||
continue
|
||||
}
|
||||
if card.TempKey == "" {
|
||||
stats.skippedNoTemplate++
|
||||
continue
|
||||
}
|
||||
if card.SyncEnabled {
|
||||
// sync_enabled=1(否/不参与) → skip
|
||||
stats.skippedSyncDisabled++
|
||||
continue
|
||||
}
|
||||
// skippedNoImei counts cards that already have a value. They are
|
||||
// still enqueued (this is a full re-poll, so a device swap is
|
||||
// detected), but the count makes the "how much of this pass is
|
||||
// actually filling gaps" question answerable from one log line.
|
||||
if strings.TrimSpace(card.Imei) != "" {
|
||||
stats.skippedNoImei++
|
||||
}
|
||||
stats.channels[card.ChanKey] = struct{}{}
|
||||
task := queue.Task{TaskID: queue.StableTaskID(message.RunID, operation, card.ID), RunID: message.RunID, Operation: operation, ChanKey: card.ChanKey, TempKey: card.TempKey, Operator: card.Operator, CardID: card.ID, ICCID: card.ICCID, MSISDN: card.MSISDN, IMSI: card.IMSI}
|
||||
groups[card.ChanKey] = append(groups[card.ChanKey], task)
|
||||
}
|
||||
for chanKey, tasks := range groups {
|
||||
stage = "enqueue"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
added, enqueueErr := workStore.EnqueueRunBatch(lockCtx, message.RunID, chanKey, tasks, time.Now())
|
||||
if enqueueErr != nil {
|
||||
return fmt.Errorf("enqueue imei channel %s: %w", chanKey, enqueueErr)
|
||||
}
|
||||
stats.enqueued += int64(added)
|
||||
}
|
||||
stage = "capacity"
|
||||
if err = p.waitForCapacity(lockCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
stage = "producer_done"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return workStore.MarkProducerDone(lockCtx, message.RunID)
|
||||
}
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelCauseFunc, leaseStore *queue.Store) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
renewCtx, stop := context.WithTimeout(ctx, leaseIOBudget(p.config.RunLockTTL))
|
||||
ok, err := leaseStore.RenewLock(renewCtx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err == nil && !ok {
|
||||
err = ErrAlreadyRunning
|
||||
}
|
||||
if err != nil {
|
||||
cancel(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) waitForCapacity(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
if p.config.GlobalHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, runID := range runs {
|
||||
meta, sizeErr := p.store.GetRunMeta(ctx, runID)
|
||||
if sizeErr != nil {
|
||||
return sizeErr
|
||||
}
|
||||
// ActiveCount includes leased tasks; do not count InflightCount twice.
|
||||
total += meta.ActiveCount + meta.NextRoundCount
|
||||
}
|
||||
if total <= p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return context.Cause(ctx)
|
||||
case <-time.After(p.config.PollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var b [12]byte
|
||||
if _, err := rand.Read(b[:]); err == nil {
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
return fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
}
|
||||
Reference in New Issue
Block a user