MiFaZhan
2bec257289
iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。 - Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件） - 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、 库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异， 不破坏「校验 MSISDN 格式」等测试断言） - 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23）， 移除写死的 git 凭据 - README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 已知：3 个测试包在源仓同一提交上同样失败（client/upstream、 service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。
377 lines
13 KiB
YAML
377 lines
13 KiB
YAML
# config_default.yaml 配置文件
|
||
# @Link https://github.com/xiujiecn/xiu-admin
|
||
# @Copyright Copyright (c) 2025 LiXiujie
|
||
# @Author Lxj <li@xiujie.cn>
|
||
# @License https://github.com/xiujiecn/xiu-admin/blob/master/LICENSE
|
||
|
||
# 系统配置
|
||
workerDiagnostic:
|
||
enabled: false
|
||
|
||
fullSyncOps:
|
||
control:
|
||
# Worker control is reachable only on the private Docker network.
|
||
listenAddress: ":18081"
|
||
readTimeout: "5s"
|
||
writeTimeout: "15s"
|
||
shutdownTimeout: "10s"
|
||
archive:
|
||
enabled: false
|
||
rootDir: "/app/logs/full-sync"
|
||
pollInterval: "10s"
|
||
retryMin: "5s"
|
||
retryMax: "5m"
|
||
replay:
|
||
# Retry-round budget for a replayed Run when the request omits maxRounds.
|
||
# Requests may override it up to 100; larger requests are rejected.
|
||
maxRounds: 3
|
||
|
||
# Alipay credentials are injected by the deployment configuration.
|
||
alipay:
|
||
appId: ""
|
||
privateKey: ""
|
||
alipayPublicKey: ""
|
||
isProduction: false
|
||
|
||
rabbitmq:
|
||
balanceRecharge:
|
||
retryDelay: "1m"
|
||
# 充值超时消息最大重试次数，超过后投递到 iotlcm.balance.recharge.dlq
|
||
maxRetry: 100
|
||
batchExecute:
|
||
executionTimeout: "0s"
|
||
finalizationTimeout: "60s"
|
||
retryDelay: "30s"
|
||
maxRetries: 3
|
||
|
||
batch:
|
||
allocate:
|
||
processingRetryAfter: "30m"
|
||
deadlockMaxRetries: 3
|
||
deadlockRetryDelay: "200ms"
|
||
inbound:
|
||
upstreamBatchSize: 1
|
||
upstreamConcurrency: 8
|
||
upstreamRateLimit:
|
||
enabled: true
|
||
mode: "redis"
|
||
redisGroup: "default"
|
||
keyPrefix: "iotlcm:worker:inbound:upstream_basic_info"
|
||
redisFailureFallback: "error"
|
||
default:
|
||
qps: 20
|
||
burst: 1
|
||
requestCost: 1
|
||
channels: {}
|
||
|
||
# 本月流量全量同步（首版默认关闭，完成 PoC 后按环境开启）
|
||
trafficUsageFullSync:
|
||
enabled: false
|
||
supplyObservation:
|
||
enabled: false # 1s sampling, 10s snapshots; Run first-round channel scanning only
|
||
redisKeyPrefix: "flow:"
|
||
retry:
|
||
enabled: true
|
||
maxRounds: 5
|
||
failureRetention: "720h"
|
||
producer:
|
||
# channel requires idx_card_chan_key; never creates or rebuilds an index.
|
||
scanConcurrency: 4 # SQL burst protection, not a measured optimum
|
||
pageSize: 100
|
||
globalHighWaterMark: 10000
|
||
channelHighWaterMark: 2000
|
||
pollInterval: "2s"
|
||
runLockKey: "flow:producer:active"
|
||
runLockTTL: "5m"
|
||
consumer:
|
||
enabled: false
|
||
# Full-sync consumer calls Chan directly; leave empty only for local tests.
|
||
chanClientName: "localhost:8110"
|
||
# A batch of 100 cards may queue behind the gateway's per-IP Tibiot
|
||
# limiter (about 650ms per request) before its request is served.
|
||
chanClientTimeout: "90s"
|
||
lockKey: "flow:consumer:active"
|
||
lockTTL: "1m"
|
||
pollInterval: "1s"
|
||
leaseTTL: "3m"
|
||
taskTimeout: "2m"
|
||
claimBatch: 100
|
||
mobileConcurrency: 30
|
||
telecomConcurrency: 20
|
||
genericConcurrency: 1
|
||
genericQPS: 1.5
|
||
# 失败池告警阀值（条目数）：单条 FailureRecord 约 1KB 且保留整个
|
||
# failureRetention，超阀值只打 WARN 日志，绝不截断 Run。
|
||
failurePoolAlertThreshold: 50000
|
||
templateQPS:
|
||
sim_yd_v5: 30
|
||
sim_yd_mock_v5: 30
|
||
sim_onelink_tengyu_v1: 30
|
||
sim_onelink_beixin_v1: 30
|
||
lanshang_8000: 30
|
||
lanshang_8003: 30
|
||
sim_onelink_xuzhou_mobile_v1: 30
|
||
sim_onelink_lelian_v1: 30
|
||
sim_onelink_qianxun_v1: 30
|
||
sim_onelink_henan_xinjing_v1: 30
|
||
sim_onelink_daihe_v1: 30
|
||
sim_dx_v1: 25
|
||
|
||
# 独立的日流量同步链路，与月流量/状态同步同构。
|
||
# 默认关闭：打开后 worker 才装配日流量 producer 与 consumer。
|
||
# 与 trafficUsageFullSync 使用独立前缀与 templateQPS，避免三条链路互相抢运营商配额；
|
||
# 定时窗口建议与全量流量同步错峰 2-3 小时（避开 0-4 点出账窗口）。
|
||
trafficUsageDailySync:
|
||
enabled: false
|
||
redisKeyPrefix: "daily:"
|
||
retry:
|
||
enabled: true
|
||
# 默认 3 轮，对齐状态同步；失败卡最终进失败池，可 replay。
|
||
maxRounds: 3
|
||
failureRetention: "720h"
|
||
producer:
|
||
pageSize: 100
|
||
globalHighWaterMark: 0
|
||
channelHighWaterMark: 0
|
||
pollInterval: "2s"
|
||
runLockKey: "daily:producer:active"
|
||
runLockTTL: "5m"
|
||
consumer:
|
||
enabled: false
|
||
lockKey: "daily:consumer:active"
|
||
lockTTL: "1m"
|
||
pollInterval: "1s"
|
||
leaseTTL: "3m"
|
||
taskTimeout: "2m"
|
||
claimBatch: 100
|
||
mobileConcurrency: 30
|
||
telecomConcurrency: 8
|
||
genericConcurrency: 1
|
||
genericQPS: 1.5
|
||
# 与 trafficUsageFullSync 分开配置：日流量与月流量走同一出口 IP，
|
||
# 共用一份 QPS 会让两轮同步互相挤占运营商配额。
|
||
templateQPS:
|
||
sim_yd_v5: 30
|
||
sim_yd_mock_v5: 30
|
||
sim_onelink_tengyu_v1: 30
|
||
sim_onelink_beixin_v1: 30
|
||
lanshang_8000: 30
|
||
lanshang_8003: 30
|
||
sim_onelink_xuzhou_mobile_v1: 30
|
||
sim_onelink_lelian_v1: 30
|
||
sim_onelink_qianxun_v1: 30
|
||
sim_onelink_henan_xinjing_v1: 30
|
||
sim_onelink_daihe_v1: 30
|
||
sim_dx_v1: 25
|
||
|
||
# 独立的月流量历史同步链路（写 TDengine 月表），与日流量/状态同步同构。
|
||
# 注意：这与 trafficUsageFullSync 不是一回事——后者填的是 MySQL 当月快照。
|
||
# 默认关闭：日流量验证通过后再单独打开。
|
||
trafficUsageMonthlyHistorySync:
|
||
enabled: false
|
||
redisKeyPrefix: "monthly-history:"
|
||
retry:
|
||
enabled: true
|
||
maxRounds: 3
|
||
failureRetention: "720h"
|
||
producer:
|
||
pageSize: 100
|
||
globalHighWaterMark: 0
|
||
channelHighWaterMark: 0
|
||
pollInterval: "2s"
|
||
runLockKey: "monthly-history:producer:active"
|
||
runLockTTL: "5m"
|
||
consumer:
|
||
enabled: false
|
||
lockKey: "monthly-history:consumer:active"
|
||
lockTTL: "1m"
|
||
pollInterval: "1s"
|
||
leaseTTL: "3m"
|
||
taskTimeout: "2m"
|
||
claimBatch: 100
|
||
mobileConcurrency: 30
|
||
telecomConcurrency: 8
|
||
genericConcurrency: 1
|
||
genericQPS: 1.5
|
||
# 三条链路共用出口 IP，各自独立限速，避免互相挤占运营商配额。
|
||
templateQPS:
|
||
sim_yd_v5: 30
|
||
sim_yd_mock_v5: 30
|
||
sim_onelink_tengyu_v1: 30
|
||
sim_onelink_beixin_v1: 30
|
||
lanshang_8000: 30
|
||
lanshang_8003: 30
|
||
sim_onelink_xuzhou_mobile_v1: 30
|
||
sim_onelink_lelian_v1: 30
|
||
sim_onelink_qianxun_v1: 30
|
||
sim_onelink_henan_xinjing_v1: 30
|
||
sim_onelink_daihe_v1: 30
|
||
sim_dx_v1: 25
|
||
|
||
# 独立的状态同步队列链路，作为消费 worker 常驻启用。
|
||
simStatusQueueSync:
|
||
enabled: true
|
||
redisKeyPrefix: "sim-status:"
|
||
retry:
|
||
enabled: true
|
||
maxRounds: 3
|
||
failureRetention: "720h"
|
||
producer:
|
||
pageSize: 1000
|
||
globalHighWaterMark: 0
|
||
pollInterval: "1s"
|
||
runLockKey: "sim-status:producer:active"
|
||
runLockTTL: "5m"
|
||
consumer:
|
||
enabled: true
|
||
lockKey: "sim-status:consumer:active"
|
||
lockTTL: "1m"
|
||
pollInterval: "1s"
|
||
leaseTTL: "3m"
|
||
taskTimeout: "2m"
|
||
claimBatch: 100
|
||
mobileConcurrency: 100
|
||
telecomConcurrency: 8
|
||
genericConcurrency: 1
|
||
genericQPS: 1.5
|
||
networkRetries: 2
|
||
# 失败池告警阀值（条目数）：单条 FailureRecord 约 1KB 且保留整个
|
||
# failureRetention，超阀值只打 WARN 日志，绝不截断 Run。
|
||
# 50000 条 ≈ 50MB，约当 255,976 张卡的 20% 失败率。
|
||
failurePoolAlertThreshold: 50000
|
||
templateQPS:
|
||
sim_yd_v5: 100
|
||
sim_yd_mock_v5: 100
|
||
sim_onelink_tengyu_v1: 100
|
||
sim_onelink_beixin_v1: 100
|
||
lanshang_8000: 100
|
||
lanshang_8003: 100
|
||
sim_onelink_xuzhou_mobile_v1: 100
|
||
sim_onelink_lelian_v1: 100
|
||
sim_dx_v1: 5
|
||
|
||
# 独立的 IMEI 全量同步链路（首版默认关闭，PoC 后按环境开启）。
|
||
#
|
||
# 该链路是"只增不减"的：只写非空 IMEI，上游返回空值只记心跳不写值，
|
||
# 所以一次上游抖动不会清空全库 IMEI。
|
||
#
|
||
# 移动 IMEI 没有批量接口，chan 内部逐卡请求，吞吐 = 卡数 × 单请求耗时。
|
||
# 电信 18 QPS 额度与状态/流量链路共享，故 templateQPS 压到 5。
|
||
imeiFullSync:
|
||
enabled: false
|
||
redisKeyPrefix: "imei:"
|
||
retry:
|
||
enabled: true
|
||
# 移动 13000 系统忙是常态（1 QPS 下仍有约 70%），轮数给足
|
||
maxRounds: 5
|
||
failureRetention: "720h"
|
||
producer:
|
||
pageSize: 1000
|
||
globalHighWaterMark: 0
|
||
pollInterval: "1s"
|
||
runLockKey: "imei:producer:active"
|
||
runLockTTL: "5m"
|
||
consumer:
|
||
enabled: false
|
||
# 全量同步直连 chan；留空仅供本地测试
|
||
chanClientName: ""
|
||
chanClientTimeout: "30s"
|
||
lockKey: "imei:consumer:active"
|
||
lockTTL: "1m"
|
||
pollInterval: "1s"
|
||
leaseTTL: "3m"
|
||
taskTimeout: "2m"
|
||
claimBatch: 100
|
||
mobileConcurrency: 100
|
||
telecomConcurrency: 8
|
||
genericConcurrency: 1
|
||
genericQPS: 1.5
|
||
networkRetries: 2
|
||
# 失败池告警阀值（条目数）：单条 FailureRecord 约 1KB 且保留整个
|
||
# failureRetention（30 天），超阀值只打 WARN 日志，绝不截断 Run。
|
||
# 50000 条 ≈ 50MB，约当 255,976 张卡的 20% 失败率。
|
||
failurePoolAlertThreshold: 50000
|
||
templateQPS:
|
||
sim_yd_v5: 100
|
||
sim_yd_mock_v5: 100
|
||
sim_onelink_tengyu_v1: 100
|
||
sim_onelink_beixin_v1: 100
|
||
lanshang_8000: 100
|
||
lanshang_8003: 100
|
||
sim_onelink_xuzhou_mobile_v1: 100
|
||
sim_onelink_lelian_v1: 100
|
||
# 电信额度与状态/流量链路共享，压到最低
|
||
sim_dx_v1: 5
|
||
|
||
upstream:
|
||
proxy:
|
||
address: "localhost:8510"
|
||
timeout: 10s
|
||
gateway:
|
||
address: "localhost:8110"
|
||
timeout: 10s
|
||
# 入库前卡还未进入本地卡表，三码校验直连 upstream-chan，避免走 8510 proxy 的本地卡拦截。
|
||
|
||
system:
|
||
name: "IotLCM"
|
||
version: "1.0"
|
||
description: "IotLCM Server"
|
||
gfmode: "dev"
|
||
upload:
|
||
local:
|
||
path: "resource/upload"
|
||
|
||
# 日志配置
|
||
logger:
|
||
path: "/app/logs" # 日志文件路径。默认为空，表示关闭，仅输出到终端
|
||
file: "{Y-m-d-H}.log" # 日志文件格式。默认为"{Y-m-d}.log"
|
||
prefix: "" # 日志内容输出前缀。默认为空
|
||
level: "dev" # 日志输出级别
|
||
timeFormat: "2006-01-02T15:04:05" # 自定义日志输出的时间格式，使用Golang标准的时间格式配置
|
||
ctxKeys: [] # 自定义Context上下文变量名称，自动打印Context的变量到日志中。默认为空
|
||
header: true # 是否打印日志的头信息。默认true
|
||
stdout: true # 日志是否同时输出到终端。默认true
|
||
rotateSize: 0 # 按照日志文件大小对文件进行滚动切分。默认为0，表示关闭滚动切分特性
|
||
rotateExpire: 0 # 按照日志文件时间间隔对文件滚动切分。默认为0，表示关闭滚动切分特性
|
||
rotateBackupLimit: 0 # 按照切分的文件数量清理切分文件，当滚动切分特性开启时有效。默认为0，表示不备份，切分则删除
|
||
rotateBackupExpire: 0 # 按照切分的文件有效期清理切分文件，当滚动切分特性开启时有效。默认为0，表示不备份，切分则删除
|
||
rotateBackupCompress: 0 # 滚动切分文件的压缩比（0-9）。默认为0，表示不压缩
|
||
rotateCheckInterval: "1h" # 滚动切分的时间检测间隔，一般不需要设置。默认为1小时
|
||
stdoutColorDisabled: false # 关闭终端的颜色打印。默认开启
|
||
writerColorEnable: false # 日志文件是否带上颜色。默认false，表示不带颜色
|
||
|
||
# 数据库配置
|
||
database:
|
||
logger:
|
||
path: "/app/logs/sql"
|
||
level: "all"
|
||
stdout: true
|
||
default: # 分组名称，可自定义，默认为default
|
||
link: "mysql:${DB_USER}:${DB_PASSWORD}@tcp(${DB_HOST}:3306)/iotlcm?charset=utf8mb4&parseTime=True&loc=Asia%2FShanghai"
|
||
debug: true #开启调试模式
|
||
charset: "utf8mb4" # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。
|
||
dryRun: false #ORM空跑(只读不写)
|
||
maxIdle: 30 #连接池最大闲置的连接数
|
||
maxOpen: 100 #连接池最大打开的连接数
|
||
maxLifetime: 120 #(单位秒)连接对象可重复使用的时间长度
|
||
|
||
# 缓存配置
|
||
cache:
|
||
adapter: "redis" # 缓存适配器，支持：file|redis
|
||
fileDir: "resource/cache" # 缓存文件目录
|
||
|
||
# Redis 配置
|
||
redis:
|
||
# 单实例配置
|
||
default:
|
||
mode: "single" #redis模式，支持：single|cluster|sentinel
|
||
sentinelMasterName: "${REDIS_SENTINEL_MASTER}" #哨兵模式主节点名称
|
||
address: 192.0.2.200:6379
|
||
db: 0
|
||
pass: "${REDIS_PASSWORD}"
|
||
minIdle: 5 #最小空闲连接数
|
||
maxIdle: 30 #允许闲置的最大连接数
|
||
maxActive: 5000 #最大活跃连接数
|
||
idleTimeout: "60s" #连接最大空闲时间，使用时间字符串例如30s/2m/2d
|
||
ClientName: "SystemCache"
|