chore: 并入 iotlcm-worker-svr 历史到 worker-svr/(原提交 96f0f4e)

This commit is contained in:
MiFaZhan
2026-11-02 13:28:03 +08:00
422 changed files with 80993 additions and 0 deletions
@@ -0,0 +1,31 @@
package worker
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
)
// Missing settings retain the serial path; explicit invalid values fail closed.
// Enabling channel mode requires validating idx_card_chan_key on the target DB.
func loadHistoryUsageScanSettings(ctx context.Context, section string) (string, int, error) {
mode := g.Cfg().MustGet(ctx, section+".producer.scanMode", "serial").String()
concurrency := g.Cfg().MustGet(ctx, section+".producer.scanConcurrency", 4).Int()
var err error
switch section {
case "trafficUsageDailySync":
err = dailyusage.ValidateScanSettings(mode, concurrency)
case "trafficUsageMonthlyHistorySync":
err = monthlyusagehistory.ValidateScanSettings(mode, concurrency)
default:
return "", 0, gerror.Newf("unknown history usage config section %q", section)
}
if err != nil {
return "", 0, gerror.Wrap(err, "load history usage scan settings")
}
return mode, concurrency, nil
}
@@ -0,0 +1,42 @@
package worker
import (
"context"
"fmt"
"testing"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gcfg"
)
func TestHistoryUsageScanSettings(t *testing.T) {
old := g.Cfg().GetAdapter()
t.Cleanup(func() { g.Cfg().SetAdapter(old) })
for _, section := range []string{"trafficUsageDailySync", "trafficUsageMonthlyHistorySync"} {
for _, test := range []struct {
name, fields, mode string
concurrency int
invalid bool
}{
{"omitted", "pageSize: 100", "serial", 4, false},
{"channel", "scanMode: channel\n scanConcurrency: 2", "channel", 2, false},
{"rollback", "scanMode: serial\n scanConcurrency: 1", "serial", 1, false},
{"zero", "scanMode: channel\n scanConcurrency: 0", "", 0, true},
{"negative", "scanConcurrency: -1", "", 0, true},
{"empty_mode", "scanMode: ''", "", 0, true},
{"invalid_mode", "scanMode: pool", "", 0, true},
} {
t.Run(section+"/"+test.name, func(t *testing.T) {
adapter, err := gcfg.NewAdapterContent(fmt.Sprintf("%s:\n producer:\n %s\n", section, test.fields))
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
mode, concurrency, err := loadHistoryUsageScanSettings(context.Background(), section)
if (err != nil) != test.invalid || mode != test.mode || concurrency != test.concurrency {
t.Fatalf("mode=%q concurrency=%d err=%v", mode, concurrency, err)
}
})
}
}
}
@@ -0,0 +1,157 @@
package worker
import (
"testing"
"github.com/gogf/gf/v2/os/gcfg"
)
// loadImeiTestConfig loads the real shipped config file.
//
// A plain g.Cfg() lookup is not enough here: the test binary does not resolve
// `config_default.yaml` (GoFrame searches for a file literally named
// `config`), so every MustGet would silently fall back to the default argument
// and a typo in the YAML key would go undetected. Loading the adapter
// explicitly makes the assertions below test the file, not the defaults.
func loadImeiTestConfig(t *testing.T) *gcfg.Config {
t.Helper()
adapter, err := gcfg.NewAdapterFile("config_default.yaml")
if err != nil {
t.Fatalf("load config_default.yaml: %v", err)
}
// Resolve relative to the package directory (internal/worker) up to the
// repository root, where config/ lives.
adapter.AddPath("../../config")
return gcfg.NewWithAdapter(adapter)
}
// TestImeiConfigFileResolves 验证 imeiFullSync 配置段真实存在于发布配置中,
// 且与状态链路结构对齐。键名拼错或段落缺失会让链路静默使用零值。
func TestImeiConfigFileResolves(t *testing.T) {
config := loadImeiTestConfig(t)
// enabled 必须存在且为 false:未完成预演验证前不得默认开启。
enabled, err := config.Get(nil, "imeiFullSync.enabled")
if err != nil {
t.Fatalf("读取 imeiFullSync.enabled 失败(配置段可能缺失): %v", err)
}
if enabled.Bool() {
t.Fatal("imeiFullSync.enabled 必须为 false(预演验证后再开)")
}
prefix, err := config.Get(nil, "imeiFullSync.redisKeyPrefix")
if err != nil || prefix.String() != "imei:" {
t.Fatalf("redisKeyPrefix = %q (err=%v), want imei:", prefix.String(), err)
}
// 三条链路的前缀必须互不相同,否则 Run 命名空间会串场。
for _, other := range []struct {
key string
want string
}{
{"trafficUsageFullSync.redisKeyPrefix", "flow:"},
{"simStatusQueueSync.redisKeyPrefix", "sim-status:"},
} {
value, err := config.Get(nil, other.key)
if err != nil {
t.Fatalf("读取 %s 失败: %v", other.key, err)
}
if value.String() == prefix.String() {
t.Fatalf("imei 前缀 %q 与 %s (%q) 重复", prefix.String(), other.key, value.String())
}
}
// 关键数值项逐条断言(对齐状态链路的键名)。
checks := []struct {
key string
want string
}{
{"imeiFullSync.producer.pageSize", "1000"},
{"imeiFullSync.producer.runLockKey", "imei:producer:active"},
{"imeiFullSync.producer.runLockTTL", "5m"},
{"imeiFullSync.consumer.lockKey", "imei:consumer:active"},
{"imeiFullSync.consumer.claimBatch", "100"},
{"imeiFullSync.consumer.mobileConcurrency", "100"},
{"imeiFullSync.consumer.telecomConcurrency", "8"},
{"imeiFullSync.consumer.networkRetries", "2"},
{"imeiFullSync.consumer.chanClientTimeout", "30s"},
{"imeiFullSync.retry.maxRounds", "5"},
{"imeiFullSync.retry.failureRetention", "720h"},
// 失败池告警阀值:漏配会静默退化为默认 50000,但键名拼错时
// 我们希望测试直接报错,而不是让链路跑在一个意外阀值上。
{"imeiFullSync.consumer.failurePoolAlertThreshold", "50000"},
}
for _, check := range checks {
value, err := config.Get(nil, check.key)
if err != nil {
t.Fatalf("读取 %s 失败: %v", check.key, err)
}
if value.String() != check.want {
t.Fatalf("%s = %q, want %q", check.key, value.String(), check.want)
}
}
// consumer.enabled 与 producer 同为 false:链路默认不跑。
consumerEnabled, err := config.Get(nil, "imeiFullSync.consumer.enabled")
if err != nil {
t.Fatalf("读取 consumer.enabled 失败: %v", err)
}
if consumerEnabled.Bool() {
t.Fatal("imeiFullSync.consumer.enabled 必须为 false")
}
}
// TestImeiTemplateQPSInConfigFile 模板限速必须覆盖生产实际存在的移动族模板,
// 否则会退回通用 1.5 QPS(255,976 张卡将耗时数天)。
func TestImeiTemplateQPSInConfigFile(t *testing.T) {
config := loadImeiTestConfig(t)
for _, tempKey := range []string{
"sim_yd_v5",
"sim_yd_mock_v5",
"sim_onelink_tengyu_v1",
"sim_onelink_xuzhou_mobile_v1",
"sim_onelink_lelian_v1",
"sim_onelink_beixin_v1",
} {
value, err := config.Get(nil, "imeiFullSync.consumer.templateQPS."+tempKey)
if err != nil {
t.Fatalf("templateQPS 缺少 %s(会退回通用 1.5 QPS): %v", tempKey, err)
}
if value.Float64() != 100 {
t.Fatalf("templateQPS.%s = %v, want 100", tempKey, value.Float64())
}
}
// 电信额度与状态/流量链路共享,必须压低。
telecom, err := config.Get(nil, "imeiFullSync.consumer.templateQPS.sim_dx_v1")
if err != nil {
t.Fatalf("templateQPS 缺少 sim_dx_v1: %v", err)
}
if telecom.Float64() != 5 {
t.Fatalf("templateQPS.sim_dx_v1 = %v, want 5(电信额度共享,必须压低)", telecom.Float64())
}
}
// TestFailurePoolAlertConfiguredForEveryChain 三条全量同步链路都要有失败池
// 告警阀值。漏配某一条时该链路会退化为「无告警」,而这正是内存问题最需要
// 被看见的地方,所以这里显式要求三条都在配置文件里出现。
func TestFailurePoolAlertConfiguredForEveryChain(t *testing.T) {
config := loadImeiTestConfig(t)
chains := []string{
"trafficUsageFullSync",
"simStatusQueueSync",
"imeiFullSync",
}
for _, chain := range chains {
key := chain + ".consumer.failurePoolAlertThreshold"
value, err := config.Get(nil, key)
if err != nil {
t.Fatalf("%s 未配置(该链路将无失败池告警): %v", key, err)
}
if value.Int64() <= 0 {
t.Fatalf("%s = %d,应为正数(0 表示关闭告警)", key, value.Int64())
}
}
}
@@ -0,0 +1,31 @@
package worker
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusage"
)
// Startup-only. Explicit zero/negative concurrency and invalid modes fail
// initialization; omitted concurrency defaults to four channel queries.
func loadMonthlyUsageProducerConfig(ctx context.Context) (monthlyusage.Config, error) {
config := monthlyusage.Config{
ScanConcurrency: g.Cfg().MustGet(ctx, "trafficUsageFullSync.producer.scanConcurrency", 4).Int(),
PageSize: g.Cfg().MustGet(ctx, "trafficUsageFullSync.producer.pageSize", 100).Int(),
GlobalHighWaterMark: g.Cfg().MustGet(ctx, "trafficUsageFullSync.producer.globalHighWaterMark", 0).Int64(),
ChannelHighWaterMark: g.Cfg().MustGet(ctx, "trafficUsageFullSync.producer.channelHighWaterMark", 0).Int64(),
PollInterval: g.Cfg().MustGet(ctx, "trafficUsageFullSync.producer.pollInterval", "2s").Duration(),
RunLockKey: g.Cfg().MustGet(ctx, "trafficUsageFullSync.producer.runLockKey", "flow:producer:active").String(),
RunLockTTL: g.Cfg().MustGet(ctx, "trafficUsageFullSync.producer.runLockTTL", "5m").Duration(),
RoundRetryEnabled: true,
MaxRounds: g.Cfg().MustGet(ctx, "trafficUsageFullSync.retry.maxRounds", 5).Int(),
FailureRetention: g.Cfg().MustGet(ctx, "trafficUsageFullSync.retry.failureRetention", "720h").Duration(),
}
if err := monthlyusage.ValidateScanSettings(config.ScanConcurrency); err != nil {
return monthlyusage.Config{}, gerror.Wrap(err, "load monthly usage producer config")
}
return config, nil
}
@@ -0,0 +1,39 @@
package worker
import (
"context"
"testing"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gcfg"
)
func TestMonthlyUsageScanStartupConfig(t *testing.T) {
old := g.Cfg().GetAdapter()
defer g.Cfg().SetAdapter(old)
for _, test := range []struct {
name, yaml string
concurrency int
invalid bool
}{
{"omitted", "{}", 4, false},
{"channel", "trafficUsageFullSync:\n producer:\n scanMode: channel\n scanConcurrency: 4", 4, false},
{"zero", "trafficUsageFullSync:\n producer:\n scanConcurrency: 0", 0, true},
{"negative", "trafficUsageFullSync:\n producer:\n scanConcurrency: -1", 0, true},
} {
t.Run(test.name, func(t *testing.T) {
adapter, err := gcfg.NewAdapterContent(test.yaml)
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
config, err := loadMonthlyUsageProducerConfig(context.Background())
if (err != nil) != test.invalid {
t.Fatalf("config=%+v err=%v", config, err)
}
if !test.invalid && config.ScanConcurrency != test.concurrency {
t.Fatalf("config=%+v", config)
}
})
}
}
+770
View File
@@ -0,0 +1,770 @@
package worker
import (
"context"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/gogf/gf/v2/database/gredis"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gmode"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/client/upstream"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
rabbitmqconsumer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consumer/rabbitmq"
zsetconsumer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consumer/zset"
fullsyncarchive "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/archive"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/httpapi"
fullsyncruntime "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/runtime"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/supply"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/tdengine"
dailyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/imei"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusage"
monthlyhistoryproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
simstatusproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/simstatus"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
batchrepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/batch"
cancelrepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/cancel"
datarepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/data"
imeirepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/imei"
inboundrepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/inbound"
lifecyclerepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/lifecycle"
monthlyusagerepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/monthlyusage"
simstatusrepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/simstatus"
trafficusagerepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/trafficusage"
unsubrepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/unsub"
addpackageservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/addpackage"
allocateservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/allocate"
balanceservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/balance"
cancelservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/cancel"
dailyusageservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/dailyusage"
dataservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data"
imeiservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/imei"
inboundservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/inbound"
lifecycleservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/lifecycle"
monthlyusageservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/monthlyusage"
monthlyhistoryservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/monthlyusagehistory"
packagecustomerservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/packagecustomer"
renewchangeservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/renewchange"
rolesyncservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/rolesync"
simstatusservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/simstatus"
trafficcutoffconfigservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficcutoffconfig"
trafficusageservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficusage"
unsubservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/unsub"
warningconfigservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/warningconfig"
workorderservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/workorder"
)
const (
shutdownTimeout = 10 * time.Second
consumerRestartInitialDelay = 2 * time.Second
consumerRestartMaxDelay = 30 * time.Second
consumerRestartBackoffResetAfter = time.Minute
)
const monthlyUsageTemplateQPSConfigKey = "trafficUsageFullSync.consumer.templateQPS"
func loadMonthlyUsageTemplateQPS(ctx context.Context) map[string]float64 {
value, err := g.Cfg().Get(ctx, monthlyUsageTemplateQPSConfigKey)
if err != nil {
g.Log().Warningf(ctx, "load monthly usage template QPS config failed: %+v", err)
return nil
}
if value == nil || value.IsNil() {
return nil
}
var configured map[string]float64
if err = value.Scan(&configured); err != nil {
g.Log().Warningf(ctx, "parse monthly usage template QPS config failed: %+v", err)
return nil
}
return configured
}
func loadSimStatusTemplateQPS(ctx context.Context) map[string]float64 {
value, err := g.Cfg().Get(ctx, "simStatusQueueSync.consumer.templateQPS")
if err != nil || value == nil || value.IsNil() {
return nil
}
var configured map[string]float64
if err = value.Scan(&configured); err != nil {
g.Log().Warningf(ctx, "parse sim status template QPS config failed: %+v", err)
return nil
}
return configured
}
func loadImeiTemplateQPS(ctx context.Context) map[string]float64 {
value, err := g.Cfg().Get(ctx, "imeiFullSync.consumer.templateQPS")
if err != nil || value == nil || value.IsNil() {
return nil
}
var configured map[string]float64
if err = value.Scan(&configured); err != nil {
g.Log().Warningf(ctx, "parse imei template QPS config failed: %+v", err)
return nil
}
return configured
}
func loadSimStatusRunPrefix(ctx context.Context) (string, error) {
prefix := strings.TrimSpace(g.Cfg().MustGet(ctx, "simStatusQueueSync.redisKeyPrefix", "sim-status:").String())
if prefix == "" {
return "", gerror.New("sim status redisKeyPrefix must be nonempty")
}
return prefix, nil
}
func loadDailyUsageTemplateQPS(ctx context.Context) map[string]float64 {
value, err := g.Cfg().Get(ctx, "trafficUsageDailySync.consumer.templateQPS")
if err != nil || value == nil || value.IsNil() {
return nil
}
var configured map[string]float64
if err = value.Scan(&configured); err != nil {
g.Log().Warningf(ctx, "parse daily usage template QPS config failed: %+v", err)
return nil
}
return configured
}
func loadDailyUsageRunPrefix(ctx context.Context) (string, error) {
prefix := strings.TrimSpace(g.Cfg().MustGet(ctx, "trafficUsageDailySync.redisKeyPrefix", "daily:").String())
if prefix == "" {
return "", gerror.New("daily usage redisKeyPrefix must be nonempty")
}
return prefix, nil
}
func loadImeiRunPrefix(ctx context.Context) (string, error) {
prefix := strings.TrimSpace(g.Cfg().MustGet(ctx, "imeiFullSync.redisKeyPrefix", "imei:").String())
if prefix == "" {
return "", gerror.New("imei redisKeyPrefix must be nonempty")
}
return prefix, nil
}
func loadMonthlyHistoryRunPrefix(ctx context.Context) (string, error) {
prefix := strings.TrimSpace(g.Cfg().MustGet(ctx, "trafficUsageMonthlyHistorySync.redisKeyPrefix", "monthly-history:").String())
if prefix == "" {
return "", gerror.New("monthly usage history redisKeyPrefix must be nonempty")
}
return prefix, nil
}
func loadMonthlyHistoryTemplateQPS(ctx context.Context) map[string]float64 {
value, err := g.Cfg().Get(ctx, "trafficUsageMonthlyHistorySync.consumer.templateQPS")
if err != nil || value == nil || value.IsNil() {
return nil
}
var configured map[string]float64
if err = value.Scan(&configured); err != nil {
g.Log().Warningf(ctx, "parse monthly usage history template QPS config failed: %+v", err)
return nil
}
return configured
}
func superviseZSetConsumer(ctx context.Context, eventPrefix string, run func(context.Context) error) {
retryDelay := consumerRestartInitialDelay
for {
startedAt := time.Now()
runErr := run(ctx)
if ctx.Err() != nil {
return
}
if time.Since(startedAt) >= consumerRestartBackoffResetAfter {
retryDelay = consumerRestartInitialDelay
}
errorText := ""
if runErr != nil {
errorText = runErr.Error()
}
g.Log().Warningf(ctx, "event=%s_consumer status=restart_scheduled retryDelayMs=%d error=%q", eventPrefix, retryDelay.Milliseconds(), errorText)
timer := time.NewTimer(retryDelay)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
retryDelay *= 2
if retryDelay > consumerRestartMaxDelay {
retryDelay = consumerRestartMaxDelay
}
}
}
func loadFullSyncArchivePrefixes(ctx context.Context) ([]string, error) {
trafficPrefix := strings.TrimSpace(g.Cfg().MustGet(ctx, "trafficUsageFullSync.redisKeyPrefix", "flow:").String())
if trafficPrefix == "" {
trafficPrefix = "flow:"
}
statusPrefix, err := loadSimStatusRunPrefix(ctx)
if err != nil {
return nil, err
}
dailyPrefix, err := loadDailyUsageRunPrefix(ctx)
if err != nil {
return nil, err
}
monthlyHistoryPrefix, err := loadMonthlyHistoryRunPrefix(ctx)
if err != nil {
return nil, err
}
imeiPrefix, err := loadImeiRunPrefix(ctx)
if err != nil {
return nil, err
}
// Duplicate prefixes would make Catalog and SimpleReplay read the same run
// hash twice. Deduplicate while preserving the traffic-first order.
prefixes := make([]string, 0, 5)
seen := make(map[string]struct{}, 5)
for _, prefix := range []string{trafficPrefix, statusPrefix, dailyPrefix, monthlyHistoryPrefix, imeiPrefix} {
if _, exists := seen[prefix]; exists {
continue
}
seen[prefix] = struct{}{}
prefixes = append(prefixes, prefix)
}
return prefixes, nil
}
func Run(ctx context.Context) error {
workerCtx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
defer stop()
if err := initialize(workerCtx); err != nil {
return err
}
var (
upstreamClient = upstream.New()
usageRepository = trafficusagerepository.New()
trafficUsage = trafficusageservice.New(upstreamClient, usageRepository)
workOrder = workorderservice.New()
balance = balanceservice.New()
inboundRepo = inboundrepository.New()
inboundExecutor = inboundservice.NewExecutorWithUpstream(inboundRepo, upstreamClient)
lifecycleRepo = lifecyclerepository.New()
lifecycleExecutor = lifecycleservice.NewExecutor(lifecycleRepo, upstreamClient)
cancelRepo = cancelrepository.New()
cancelExecutor = cancelservice.NewExecutor(cancelRepo)
renewChangeRepo = renewchangeservice.NewRepository()
addPackageCardExecutor = addpackageservice.NewCardExecutor(renewChangeRepo)
addPackagePoolExecutor = addpackageservice.NewPoolExecutor(renewChangeRepo)
renewPrepareExecutor = renewchangeservice.NewPrepareExecutor(renewChangeRepo)
prepareExecutor = addpackageservice.NewPrepareMux(renewChangeRepo, renewPrepareExecutor)
renewExecutor = renewchangeservice.NewRenewExecutor(renewChangeRepo)
changeExecutor = renewchangeservice.NewChangeExecutor(renewChangeRepo)
dataRepo = datarepository.New()
dataExecutor = dataservice.NewExecutor(dataRepo, upstreamClient)
unsubRepo = unsubrepository.New()
unsubExecutor = unsubservice.NewExecutor(unsubRepo)
restartExecutor = lifecycleservice.NewRestartExecutor(lifecycleRepo, upstreamClient)
batchRepository = batchrepository.New()
batchDispatcher = batch.NewDispatcher(lifecycleExecutor)
batchRuntime = batch.NewRuntime(
batchRepository,
batchDispatcher,
batch.WithExecutionTimeout(g.Cfg().MustGet(workerCtx, "rabbitmq.batchExecute.executionTimeout", "0s").Duration()),
batch.WithFinalizationTimeout(g.Cfg().MustGet(workerCtx, "rabbitmq.batchExecute.finalizationTimeout", "10s").Duration()),
)
allocateExecutor = allocateservice.NewExecutor()
warningConfigExecutor = warningconfigservice.NewExecutor()
trafficCutoffConfigExecutor = trafficcutoffconfigservice.NewExecutor()
packageCustomerSync = packagecustomerservice.New()
)
var (
monthlyProducer *monthlyusage.Producer
monthlyConsumer *zsetconsumer.Consumer
statusProducer *simstatusproducer.Producer
statusConsumer *zsetconsumer.Consumer
dailyProducer *dailyusageproducer.Producer
dailyConsumer *zsetconsumer.Consumer
monthlyHistoryProducer *monthlyhistoryproducer.Producer
monthlyHistoryConsumer *zsetconsumer.Consumer
imeiProducer *imei.Producer
imeiConsumer *zsetconsumer.Consumer
)
if g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.enabled", false).Bool() {
prefix := g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.redisKeyPrefix", "flow:").String()
store, err := zset.New(g.Redis(), prefix)
if err != nil {
return gerror.Wrap(err, "initialize monthly usage Redis store failed")
}
store.EnableArchiveIndex(g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.enabled", false).Bool())
producerConfig, configErr := loadMonthlyUsageProducerConfig(workerCtx)
if configErr != nil {
return configErr
}
var supplyTracker *supply.Tracker
if g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.supplyObservation.enabled", false).Bool() {
supplyTracker = supply.NewTracker(g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.enabled", false).Bool())
producerConfig.SupplyObservation = supplyTracker
}
monthlyProducer, err = monthlyusage.New(&monthlyusage.DatabaseScanner{}, store, producerConfig)
if err != nil {
return gerror.Wrap(err, "initialize monthly usage producer failed")
}
monthlyUsageClient := upstream.NewWithCurrentUsageClient(
g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.chanClientName", "").String(),
g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.chanClientTimeout", "90s").Duration(),
)
usageHandler, handlerErr := monthlyusageservice.NewHandler(monthlyUsageClient, monthlyusagerepository.NewRepository())
if handlerErr != nil {
return gerror.Wrap(handlerErr, "initialize monthly usage handler failed")
}
monthlyConsumer, err = zsetconsumer.New(store, usageHandler, zsetconsumer.Config{
SupplyObservation: supplyTracker,
LockKey: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.lockKey", "flow:consumer:active").String(),
LockTTL: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.lockTTL", "1m").Duration(),
PollInterval: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.pollInterval", "1s").Duration(),
LeaseTTL: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.leaseTTL", "3m").Duration(),
TaskTimeout: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.taskTimeout", "2m").Duration(),
ClaimBatch: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.claimBatch", 100).Int(),
MobileConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.mobileConcurrency", 30).Int(),
TelecomConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.telecomConcurrency", 20).Int(),
GenericConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.genericConcurrency", 1).Int(),
GenericQPS: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.genericQPS", 1.5).Float64(),
TemplateQPS: loadMonthlyUsageTemplateQPS(workerCtx),
RoundRetryEnabled: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.retry.enabled", true).Bool(),
MaxRounds: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.retry.maxRounds", 5).Int(),
FailureRetention: g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.retry.failureRetention", "720h").Duration(),
ShouldRetryNextRound: func(result zsetconsumer.Result) bool {
return !result.Success && result.Retryable
},
})
if err != nil {
return gerror.Wrap(err, "initialize monthly usage consumer failed")
}
if g.Cfg().MustGet(workerCtx, "trafficUsageFullSync.consumer.enabled", false).Bool() {
go superviseZSetConsumer(workerCtx, "monthly_usage", monthlyConsumer.Run)
}
}
if g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.enabled", false).Bool() {
prefix, prefixErr := loadDailyUsageRunPrefix(workerCtx)
if prefixErr != nil {
return prefixErr
}
store, err := zset.New(g.Redis(), prefix)
if err != nil {
return gerror.Wrap(err, "initialize daily usage Redis store failed")
}
store.EnableArchiveIndex(g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.enabled", false).Bool())
dailyScanMode, dailyScanConcurrency, scanErr := loadHistoryUsageScanSettings(workerCtx, "trafficUsageDailySync")
if scanErr != nil {
return scanErr
}
dailyProducer, err = dailyusageproducer.New(&dailyusageproducer.DatabaseScanner{}, store, dailyusageproducer.Config{
ScanMode: dailyScanMode,
ScanConcurrency: dailyScanConcurrency,
PageSize: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.producer.pageSize", 100).Int(),
GlobalHighWaterMark: int64(g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.producer.globalHighWaterMark", 0).Int64()),
ChannelHighWaterMark: int64(g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.producer.channelHighWaterMark", 0).Int64()),
PollInterval: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.producer.pollInterval", "2s").Duration(),
RunLockKey: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.producer.runLockKey", "daily:producer:active").String(),
RunLockTTL: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.producer.runLockTTL", "5m").Duration(),
RoundRetryEnabled: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.retry.enabled", true).Bool(),
MaxRounds: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.retry.maxRounds", 3).Int(),
FailureRetention: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.retry.failureRetention", "720h").Duration(),
})
if err != nil {
return gerror.Wrap(err, "initialize daily usage producer failed")
}
dailyHandler, handlerErr := dailyusageservice.NewHandler(upstream.New(), trafficusagerepository.New())
if handlerErr != nil {
return gerror.Wrap(handlerErr, "initialize daily usage handler failed")
}
dailyConsumer, err = zsetconsumer.New(store, dailyHandler, zsetconsumer.Config{
RunQueueEnabled: true,
RoundRetryEnabled: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.retry.enabled", true).Bool(),
MaxRounds: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.retry.maxRounds", 3).Int(),
FailureRetention: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.retry.failureRetention", "720h").Duration(),
ShouldRetryNextRound: dailyusageservice.ShouldRetryNextRound,
BlockedTasksAsFailures: true,
EventPrefix: "daily_usage",
LockKey: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.lockKey", "daily:consumer:active").String(),
LockTTL: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.lockTTL", "1m").Duration(),
PollInterval: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.pollInterval", "1s").Duration(),
LeaseTTL: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.leaseTTL", "3m").Duration(),
TaskTimeout: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.taskTimeout", "2m").Duration(),
ClaimBatch: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.claimBatch", 100).Int(),
MobileConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.mobileConcurrency", 30).Int(),
TelecomConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.telecomConcurrency", 8).Int(),
GenericConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.genericConcurrency", 1).Int(),
GenericQPS: g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.genericQPS", 1.5).Float64(),
TemplateQPS: loadDailyUsageTemplateQPS(workerCtx),
})
if err != nil {
return gerror.Wrap(err, "initialize daily usage consumer failed")
}
if g.Cfg().MustGet(workerCtx, "trafficUsageDailySync.consumer.enabled", false).Bool() {
go superviseZSetConsumer(workerCtx, "daily_usage", dailyConsumer.Run)
}
}
if g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.enabled", false).Bool() {
prefix, prefixErr := loadMonthlyHistoryRunPrefix(workerCtx)
if prefixErr != nil {
return prefixErr
}
store, err := zset.New(g.Redis(), prefix)
if err != nil {
return gerror.Wrap(err, "initialize monthly usage history Redis store failed")
}
store.EnableArchiveIndex(g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.enabled", false).Bool())
monthlyHistoryScanMode, monthlyHistoryScanConcurrency, scanErr := loadHistoryUsageScanSettings(workerCtx, "trafficUsageMonthlyHistorySync")
if scanErr != nil {
return scanErr
}
monthlyHistoryProducer, err = monthlyhistoryproducer.New(&monthlyhistoryproducer.DatabaseScanner{}, store, monthlyhistoryproducer.Config{
ScanMode: monthlyHistoryScanMode,
ScanConcurrency: monthlyHistoryScanConcurrency,
PageSize: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.producer.pageSize", 100).Int(),
GlobalHighWaterMark: int64(g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.producer.globalHighWaterMark", 0).Int64()),
ChannelHighWaterMark: int64(g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.producer.channelHighWaterMark", 0).Int64()),
PollInterval: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.producer.pollInterval", "2s").Duration(),
RunLockKey: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.producer.runLockKey", "monthly-history:producer:active").String(),
RunLockTTL: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.producer.runLockTTL", "5m").Duration(),
RoundRetryEnabled: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.retry.enabled", true).Bool(),
MaxRounds: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.retry.maxRounds", 3).Int(),
FailureRetention: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.retry.failureRetention", "720h").Duration(),
})
if err != nil {
return gerror.Wrap(err, "initialize monthly usage history producer failed")
}
monthlyHistoryHandler, handlerErr := monthlyhistoryservice.NewHandler(upstream.New(), trafficusagerepository.New())
if handlerErr != nil {
return gerror.Wrap(handlerErr, "initialize monthly usage history handler failed")
}
monthlyHistoryConsumer, err = zsetconsumer.New(store, monthlyHistoryHandler, zsetconsumer.Config{
RunQueueEnabled: true,
RoundRetryEnabled: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.retry.enabled", true).Bool(),
MaxRounds: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.retry.maxRounds", 3).Int(),
FailureRetention: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.retry.failureRetention", "720h").Duration(),
ShouldRetryNextRound: monthlyhistoryservice.ShouldRetryNextRound,
BlockedTasksAsFailures: true,
EventPrefix: "monthly_usage_history",
LockKey: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.lockKey", "monthly-history:consumer:active").String(),
LockTTL: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.lockTTL", "1m").Duration(),
PollInterval: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.pollInterval", "1s").Duration(),
LeaseTTL: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.leaseTTL", "3m").Duration(),
TaskTimeout: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.taskTimeout", "2m").Duration(),
ClaimBatch: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.claimBatch", 100).Int(),
MobileConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.mobileConcurrency", 30).Int(),
TelecomConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.telecomConcurrency", 8).Int(),
GenericConcurrency: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.genericConcurrency", 1).Int(),
GenericQPS: g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.genericQPS", 1.5).Float64(),
TemplateQPS: loadMonthlyHistoryTemplateQPS(workerCtx),
})
if err != nil {
return gerror.Wrap(err, "initialize monthly usage history consumer failed")
}
if g.Cfg().MustGet(workerCtx, "trafficUsageMonthlyHistorySync.consumer.enabled", false).Bool() {
go superviseZSetConsumer(workerCtx, "monthly_usage_history", monthlyHistoryConsumer.Run)
}
}
if g.Cfg().MustGet(workerCtx, "simStatusQueueSync.enabled", false).Bool() {
prefix, prefixErr := loadSimStatusRunPrefix(workerCtx)
if prefixErr != nil {
return prefixErr
}
store, err := zset.New(g.Redis(), prefix)
if err != nil {
return gerror.Wrap(err, "initialize sim status Redis store failed")
}
store.EnableArchiveIndex(g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.enabled", false).Bool())
statusProducer, err = simstatusproducer.New(&simstatusproducer.DatabaseScanner{}, store, simstatusproducer.Config{
PageSize: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.producer.pageSize", 1000).Int(),
GlobalHighWaterMark: int64(g.Cfg().MustGet(workerCtx, "simStatusQueueSync.producer.globalHighWaterMark", 0).Int64()),
PollInterval: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.producer.pollInterval", "1s").Duration(),
RunLockKey: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.producer.runLockKey", "sim-status:producer:active").String(),
RunLockTTL: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.producer.runLockTTL", "5m").Duration(),
})
if err != nil {
return gerror.Wrap(err, "initialize sim status producer failed")
}
statusHandler, handlerErr := simstatusservice.NewHandler(
upstream.New(),
simstatusrepository.NewRepository(),
g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.networkRetries", 2).Int(),
)
if handlerErr != nil {
return gerror.Wrap(handlerErr, "initialize sim status handler failed")
}
statusConsumer, err = zsetconsumer.New(store, statusHandler, zsetconsumer.Config{
RunQueueEnabled: true,
RoundRetryEnabled: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.retry.enabled", true).Bool(),
MaxRounds: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.retry.maxRounds", 3).Int(),
FailureRetention: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.retry.failureRetention", "720h").Duration(),
ShouldRetryNextRound: simstatusservice.ShouldRetryNextRound,
BlockedTasksAsFailures: true,
EventPrefix: "sim_status_queue",
LockKey: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.lockKey", "sim-status:consumer:active").String(),
LockTTL: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.lockTTL", "1m").Duration(),
PollInterval: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.pollInterval", "1s").Duration(),
LeaseTTL: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.leaseTTL", "3m").Duration(),
TaskTimeout: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.taskTimeout", "2m").Duration(),
ClaimBatch: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.claimBatch", 100).Int(),
MobileConcurrency: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.mobileConcurrency", 100).Int(),
TelecomConcurrency: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.telecomConcurrency", 8).Int(),
GenericConcurrency: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.genericConcurrency", 1).Int(),
GenericQPS: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.genericQPS", 1.5).Float64(),
TemplateQPS: loadSimStatusTemplateQPS(workerCtx),
FailurePoolAlertThreshold: g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.failurePoolAlertThreshold", 50000).Int64(),
})
if err != nil {
return gerror.Wrap(err, "initialize sim status consumer failed")
}
if g.Cfg().MustGet(workerCtx, "simStatusQueueSync.consumer.enabled", false).Bool() {
go superviseZSetConsumer(workerCtx, "sim_status_queue", statusConsumer.Run)
}
}
if g.Cfg().MustGet(workerCtx, "imeiFullSync.enabled", false).Bool() {
prefix, prefixErr := loadImeiRunPrefix(workerCtx)
if prefixErr != nil {
return prefixErr
}
store, err := zset.New(g.Redis(), prefix)
if err != nil {
return gerror.Wrap(err, "initialize imei Redis store failed")
}
store.EnableArchiveIndex(g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.enabled", false).Bool())
// g.Redis may be initialized from g.Cfg without registering GetConfig.
leaseConfig, configured := gredis.GetConfig()
if !configured {
value, configErr := g.Cfg().Get(workerCtx, "redis.default")
if configErr != nil {
return gerror.Wrap(configErr, "load IMEI lease Redis configuration failed")
}
leaseConfig, err = gredis.ConfigFromMap(value.Map())
if err != nil {
return gerror.Wrap(err, "parse IMEI lease Redis configuration failed")
}
}
imeiProducer, err = imei.New(&imei.DatabaseScanner{}, store, imei.Config{
PageSize: g.Cfg().MustGet(workerCtx, "imeiFullSync.producer.pageSize", 1000).Int(),
GlobalHighWaterMark: int64(g.Cfg().MustGet(workerCtx, "imeiFullSync.producer.globalHighWaterMark", 0).Int64()),
PollInterval: g.Cfg().MustGet(workerCtx, "imeiFullSync.producer.pollInterval", "1s").Duration(),
RunLockKey: g.Cfg().MustGet(workerCtx, "imeiFullSync.producer.runLockKey", "imei:producer:active").String(),
RunLockTTL: g.Cfg().MustGet(workerCtx, "imeiFullSync.producer.runLockTTL", "5m").Duration(),
LeaseRedisConfig: leaseConfig,
})
if err != nil {
return gerror.Wrap(err, "initialize imei producer failed")
}
imeiHandler, handlerErr := imeiservice.NewHandler(
upstream.NewWithImeiClient(
g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.chanClientName", "").String(),
g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.chanClientTimeout", "30s").Duration(),
),
imeirepository.NewRepository(),
g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.networkRetries", 2).Int(),
)
if handlerErr != nil {
return gerror.Wrap(handlerErr, "initialize imei handler failed")
}
imeiConsumer, err = zsetconsumer.New(store, imeiHandler, zsetconsumer.Config{
RunQueueEnabled: true,
RoundRetryEnabled: g.Cfg().MustGet(workerCtx, "imeiFullSync.retry.enabled", true).Bool(),
MaxRounds: g.Cfg().MustGet(workerCtx, "imeiFullSync.retry.maxRounds", 5).Int(),
FailureRetention: g.Cfg().MustGet(workerCtx, "imeiFullSync.retry.failureRetention", "720h").Duration(),
ShouldRetryNextRound: imeiservice.ShouldRetryNextRound,
BlockedTasksAsFailures: true,
EventPrefix: "imei_queue",
LockKey: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.lockKey", "imei:consumer:active").String(),
LockTTL: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.lockTTL", "1m").Duration(),
PollInterval: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.pollInterval", "1s").Duration(),
LeaseTTL: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.leaseTTL", "3m").Duration(),
TaskTimeout: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.taskTimeout", "2m").Duration(),
ClaimBatch: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.claimBatch", 100).Int(),
MobileConcurrency: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.mobileConcurrency", 100).Int(),
TelecomConcurrency: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.telecomConcurrency", 8).Int(),
GenericConcurrency: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.genericConcurrency", 1).Int(),
GenericQPS: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.genericQPS", 1.5).Float64(),
TemplateQPS: loadImeiTemplateQPS(workerCtx),
// 失败池是唯一随卡数线性增长的每卡结构(单条约 1KB),
// 且会保留整个 failureRetention。超阀值只告警不截断。
FailurePoolAlertThreshold: g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.failurePoolAlertThreshold", 50000).Int64(),
})
if err != nil {
return gerror.Wrap(err, "initialize imei consumer failed")
}
if g.Cfg().MustGet(workerCtx, "imeiFullSync.consumer.enabled", false).Bool() {
go superviseZSetConsumer(workerCtx, "imei_queue", imeiConsumer.Run)
}
}
var rabbitMQConsumers *rabbitmqconsumer.Consumers
if monthlyProducer != nil {
rabbitMQConsumers = rabbitmqconsumer.NewWithPackageCustomerSyncAndMonthlyUsage(trafficUsage, workOrder, balance, packageCustomerSync, monthlyProducer, batchRuntime)
} else {
rabbitMQConsumers = rabbitmqconsumer.NewWithPackageCustomerSync(trafficUsage, workOrder, balance, packageCustomerSync, batchRuntime)
}
if statusProducer != nil {
rabbitMQConsumers.WithSimStatusQueue(statusProducer, statusProducer)
}
if dailyProducer != nil {
rabbitMQConsumers.WithDailyUsageStart(dailyProducer)
}
if monthlyHistoryProducer != nil {
rabbitMQConsumers.WithMonthlyHistoryStart(monthlyHistoryProducer)
}
if imeiProducer != nil {
rabbitMQConsumers.WithImeiQueue(imeiProducer)
}
rabbitMQConsumers.WithDataReviewQuery(dataExecutor)
var (
fullSyncHTTP *httpapi.Server
fullSyncArchives []*fullsyncarchive.Service
)
{ // The authenticated control plane is always available.
prefixes, err := loadFullSyncArchivePrefixes(workerCtx)
if err != nil {
return err
}
stores := make([]*zset.Store, 0, len(prefixes))
for _, prefix := range prefixes {
store, e := zset.New(g.Redis(), prefix)
if e != nil {
return e
}
stores = append(stores, store)
}
archiveRoot := g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.rootDir", "/app/logs/full-sync").String()
repositories := make([]*fullsyncruntime.Repository, 0, len(stores))
for _, store := range stores {
repo, e := fullsyncruntime.New(store, archiveRoot)
if e != nil {
return gerror.Wrap(e, "initialize full-sync runtime repository failed")
}
repositories = append(repositories, repo)
}
catalog, err := fullsyncruntime.NewCatalog(repositories...)
if err != nil {
return gerror.Wrap(err, "initialize full-sync run catalog failed")
}
simple, err := fullsyncruntime.NewSimpleReplayWithDefault(g.Cfg().MustGet(workerCtx, "fullSyncOps.replay.maxRounds", fullsyncruntime.DefaultReplayMaxRounds).Int(), stores...)
if err != nil {
return err
}
// Only Admin exposes replay to users. This listener stays inside the
// Docker network; the Worker service must not publish port 18081.
fullSyncHTTP, err = httpapi.NewServer(httpapi.Config{Addr: ":18081", ShutdownTimeout: g.Cfg().MustGet(workerCtx, "fullSyncOps.control.shutdownTimeout", "10s").Duration()}, httpapi.Dependencies{Runs: catalog, List: catalog, Failures: catalog, Replays: repositories[0], Actions: repositories[0], Simple: simple})
if err != nil {
return gerror.Wrap(err, "initialize full-sync control server failed")
}
if err := fullSyncHTTP.Start(workerCtx); err != nil {
return gerror.Wrap(err, "start full-sync control server failed")
}
simple.Start(workerCtx)
}
if g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.enabled", false).Bool() {
prefixes, err := loadFullSyncArchivePrefixes(workerCtx)
if err != nil {
return gerror.Wrap(err, "load full-sync archive Redis prefixes failed")
}
for _, prefix := range prefixes {
store, err := zset.New(g.Redis(), prefix)
if err != nil {
return gerror.Wrapf(err, "initialize full-sync archive Redis store failed for prefix %q", prefix)
}
store.EnableArchiveIndex(true)
repo, err := fullsyncruntime.New(store, g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.rootDir", "/app/logs/full-sync").String())
if err != nil {
return gerror.Wrapf(err, "initialize full-sync archive repository failed for prefix %q", prefix)
}
service := fullsyncarchive.NewService(fullsyncarchive.Config{Root: g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.rootDir", "/app/logs/full-sync").String(), Interval: g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.pollInterval", "10s").Duration(), InitialBackoff: g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.retryMin", "5s").Duration(), MaxBackoff: g.Cfg().MustGet(workerCtx, "fullSyncOps.archive.retryMax", "5m").Duration()}, repo, repo, repo)
if err := service.Start(workerCtx); err != nil {
return gerror.Wrapf(err, "start full-sync archive service failed for prefix %q", prefix)
}
fullSyncArchives = append(fullSyncArchives, service)
}
}
batchDispatcher.Register(consts.BatchTypeInbound, inboundExecutor)
batchDispatcher.Register(consts.BatchTypeAllocate, allocateExecutor)
batchDispatcher.Register(consts.BatchTypeCardAddPackage, addPackageCardExecutor)
batchDispatcher.Register(consts.BatchTypePoolAddPackage, addPackagePoolExecutor)
batchDispatcher.Register(consts.BatchTypePrepare, prepareExecutor)
batchDispatcher.Register(consts.BatchTypeRenew, renewExecutor)
batchDispatcher.Register(consts.BatchTypeChange, changeExecutor)
batchDispatcher.Register(consts.BatchTypeDataCut, dataExecutor)
batchDispatcher.Register(consts.BatchTypeDataStart, dataExecutor)
batchDispatcher.Register(consts.BatchTypeDataStop, dataExecutor)
// Resume durable APN sub-batches independently of RabbitMQ delivery and process restarts.
if reconciler, ok := dataExecutor.(interface{ RunReconciler(context.Context) error }); ok {
go func() {
if err := reconciler.RunReconciler(workerCtx); err != nil && workerCtx.Err() == nil {
g.Log().Errorf(workerCtx, "APN operation reconciler stopped: %v", err)
}
}()
}
batchDispatcher.Register(consts.BatchTypeSimCardUnsub, unsubExecutor)
batchDispatcher.Register(consts.BatchTypeSimCardRestart, restartExecutor)
batchDispatcher.Register(consts.BatchTypeSimCardCancel, cancelExecutor)
batchDispatcher.Register(consts.BatchTypeSimWarningConfig, warningConfigExecutor)
batchDispatcher.Register(consts.BatchTypeTrafficCutoffConfig, trafficCutoffConfigExecutor)
if err := rabbitMQConsumers.Start(workerCtx); err != nil {
_ = rabbitMQConsumers.Stop(context.Background())
return gerror.Wrap(err, "start RabbitMQ consumers failed")
}
stopRoleSync, err := rolesyncservice.Start(workerCtx)
if err != nil {
_ = rabbitMQConsumers.Stop(context.Background())
return gerror.Wrap(err, "start role sync Worker failed")
}
defer stopRoleSync()
g.Log().Info(workerCtx, "iotlcm worker started")
<-workerCtx.Done()
g.Log().Info(ctx, "iotlcm worker stopping")
stopRoleSync()
shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
if err := rabbitMQConsumers.Stop(shutdownCtx); err != nil {
return gerror.Wrap(err, "stop RabbitMQ consumers failed")
}
for _, service := range fullSyncArchives {
if err := service.Stop(); err != nil {
return gerror.Wrap(err, "stop full-sync archive service failed")
}
}
if fullSyncHTTP != nil {
if err := fullSyncHTTP.Wait(); err != nil {
return gerror.Wrap(err, "stop full-sync control server failed")
}
}
if err := tdengine.Close(); err != nil {
return gerror.Wrap(err, "close TDengine failed")
}
return nil
}
func initialize(ctx context.Context) error {
if err := g.DB().PingMaster(); err != nil {
return gerror.Wrap(err, "connect database failed")
}
if err := gtime.SetTimeZone("Asia/Shanghai"); err != nil {
return gerror.Wrap(err, "set timezone failed")
}
mode := g.Cfg().MustGet(ctx, "system.gfmode", gmode.NOT_SET).String()
for _, supported := range []string{gmode.DEVELOP, gmode.TESTING, gmode.STAGING, gmode.PRODUCT} {
if mode == supported {
gmode.Set(mode)
break
}
}
return nil
}
+198
View File
@@ -0,0 +1,198 @@
package worker
import (
"context"
"reflect"
"testing"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gcfg"
)
func TestLoadMonthlyUsageTemplateQPS(t *testing.T) {
oldAdapter := g.Cfg().GetAdapter()
adapter, err := gcfg.NewAdapterContent(`
trafficUsageFullSync:
consumer:
templateQPS:
sim_yd_v5: 30
sim_dx_v1: 25
qibin_direct: 1.5
`)
if err != nil {
t.Fatalf("create config adapter: %v", err)
}
g.Cfg().SetAdapter(adapter)
defer g.Cfg().SetAdapter(oldAdapter)
got := loadMonthlyUsageTemplateQPS(context.Background())
if got["sim_yd_v5"] != 30 || got["sim_dx_v1"] != 25 || got["qibin_direct"] != 1.5 {
t.Fatalf("template QPS = %#v, want configured values", got)
}
}
func TestLoadSimStatusRunPrefix(t *testing.T) {
oldAdapter := g.Cfg().GetAdapter()
defer g.Cfg().SetAdapter(oldAdapter)
for _, test := range []struct {
name, config, want string
invalid bool
}{
{"default", "simStatusQueueSync:\n enabled: true\n", "sim-status:", false},
{"custom", "simStatusQueueSync:\n redisKeyPrefix: 'status:'", "status:", false},
{"empty", "simStatusQueueSync:\n redisKeyPrefix: ''", "", true},
} {
t.Run(test.name, func(t *testing.T) {
adapter, err := gcfg.NewAdapterContent(test.config)
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
got, err := loadSimStatusRunPrefix(context.Background())
if got != test.want || (err != nil) != test.invalid {
t.Fatalf("prefix=%q err=%v", got, err)
}
})
}
}
func TestLoadDailyUsageTemplateQPS(t *testing.T) {
oldAdapter := g.Cfg().GetAdapter()
adapter, err := gcfg.NewAdapterContent(`
trafficUsageDailySync:
consumer:
templateQPS:
sim_yd_v5: 30
sim_dx_v1: 25
`)
if err != nil {
t.Fatalf("create config adapter: %v", err)
}
g.Cfg().SetAdapter(adapter)
defer g.Cfg().SetAdapter(oldAdapter)
got := loadDailyUsageTemplateQPS(context.Background())
if got["sim_yd_v5"] != 30 || got["sim_dx_v1"] != 25 {
t.Fatalf("daily usage template QPS = %#v, want the daily-configured values", got)
}
// The daily chain must not silently inherit the monthly chain's rates; the
// two run against the same carrier quota and must be tuned independently.
monthly := loadMonthlyUsageTemplateQPS(context.Background())
if len(monthly) != 0 {
t.Fatalf("monthly template QPS = %#v, want it unaffected by the daily block", monthly)
}
}
func TestLoadDailyUsageRunPrefix(t *testing.T) {
oldAdapter := g.Cfg().GetAdapter()
defer g.Cfg().SetAdapter(oldAdapter)
for _, test := range []struct {
name, config, want string
invalid bool
}{
{"default", "trafficUsageDailySync:\n enabled: true\n", "daily:", false},
{"custom", "trafficUsageDailySync:\n redisKeyPrefix: 'daily-usage:'", "daily-usage:", false},
{"empty", "trafficUsageDailySync:\n redisKeyPrefix: ''", "", true},
} {
t.Run(test.name, func(t *testing.T) {
adapter, err := gcfg.NewAdapterContent(test.config)
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
got, err := loadDailyUsageRunPrefix(context.Background())
if got != test.want || (err != nil) != test.invalid {
t.Fatalf("prefix=%q err=%v", got, err)
}
})
}
}
func TestLoadMonthlyHistoryRunPrefix(t *testing.T) {
oldAdapter := g.Cfg().GetAdapter()
defer g.Cfg().SetAdapter(oldAdapter)
for _, test := range []struct {
name, config, want string
invalid bool
}{
{"default", "trafficUsageMonthlyHistorySync:\n enabled: true\n", "monthly-history:", false},
{"custom", "trafficUsageMonthlyHistorySync:\n redisKeyPrefix: 'mh:'", "mh:", false},
{"empty", "trafficUsageMonthlyHistorySync:\n redisKeyPrefix: ''", "", true},
} {
t.Run(test.name, func(t *testing.T) {
adapter, err := gcfg.NewAdapterContent(test.config)
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
got, err := loadMonthlyHistoryRunPrefix(context.Background())
if got != test.want || (err != nil) != test.invalid {
t.Fatalf("prefix=%q err=%v", got, err)
}
})
}
}
// TestMonthlyHistoryPrefixIsIsolatedFromCurrentMonthSync pins the separation of
// the two monthly chains: trafficUsageFullSync fills the MySQL current-month
// snapshot, while trafficUsageMonthlyHistorySync fills the TDengine monthly
// table. Sharing a prefix would let one chain's producer lock block the other.
func TestMonthlyHistoryPrefixIsIsolatedFromCurrentMonthSync(t *testing.T) {
oldAdapter := g.Cfg().GetAdapter()
defer g.Cfg().SetAdapter(oldAdapter)
adapter, err := gcfg.NewAdapterContent(`
trafficUsageFullSync:
redisKeyPrefix: 'flow:'
trafficUsageMonthlyHistorySync:
redisKeyPrefix: 'monthly-history:'
`)
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
prefixes, err := loadFullSyncArchivePrefixes(context.Background())
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, p := range prefixes {
seen[p] = true
}
if !seen["flow:"] || !seen["monthly-history:"] {
t.Fatalf("prefixes=%q, want both the current-month and monthly-history chains present", prefixes)
}
}
func TestLoadFullSyncArchivePrefixes(t *testing.T) {
oldAdapter := g.Cfg().GetAdapter()
defer g.Cfg().SetAdapter(oldAdapter)
for _, test := range []struct {
name, config string
want []string
}{
{"defaults", "{}", []string{"flow:", "sim-status:", "daily:", "monthly-history:", "imei:"}},
{"disabled IMEI remains visible", "imeiFullSync:\n enabled: false", []string{"flow:", "sim-status:", "daily:", "monthly-history:", "imei:"}},
{"custom", "trafficUsageFullSync:\n redisKeyPrefix: 'usage:'\nsimStatusQueueSync:\n redisKeyPrefix: 'status:'\ntrafficUsageDailySync:\n redisKeyPrefix: 'daily-usage:'\ntrafficUsageMonthlyHistorySync:\n redisKeyPrefix: 'monthly-history:'\nimeiFullSync:\n redisKeyPrefix: 'imei-usage:'", []string{"usage:", "status:", "daily-usage:", "monthly-history:", "imei-usage:"}},
// Duplicate prefixes would make Catalog and SimpleReplay read the same
// run hash twice, so the list is deduplicated with the first occurrence
// keeping its position.
{"shared prefix", "trafficUsageFullSync:\n redisKeyPrefix: 'sync:'\nsimStatusQueueSync:\n redisKeyPrefix: 'sync:'\ntrafficUsageDailySync:\n redisKeyPrefix: 'sync:'\ntrafficUsageMonthlyHistorySync:\n redisKeyPrefix: 'sync:'\nimeiFullSync:\n redisKeyPrefix: 'sync:'", []string{"sync:"}},
{"daily shares the status prefix", "simStatusQueueSync:\n redisKeyPrefix: 'daily:'\ntrafficUsageDailySync:\n redisKeyPrefix: 'daily:'", []string{"flow:", "daily:", "monthly-history:", "imei:"}},
{"monthly history shares the daily prefix", "trafficUsageDailySync:\n redisKeyPrefix: 'usage:'\ntrafficUsageMonthlyHistorySync:\n redisKeyPrefix: 'usage:'", []string{"flow:", "sim-status:", "usage:", "imei:"}},
} {
t.Run(test.name, func(t *testing.T) {
adapter, err := gcfg.NewAdapterContent(test.config)
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
got, err := loadFullSyncArchivePrefixes(context.Background())
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, test.want) {
t.Fatalf("prefixes=%q, want %q", got, test.want)
}
})
}
}