chore: 初始开源发布

iotlcm-admin-svr:管理后台服务。

- Go module:iotlcm → git.5iot.com/iotlcm-public/iotlcm-admin-svr(729 文件)
- 凭据与地址脱敏:JWT signingKey(含代码里的 MustGet 默认值)、
  MySQL DSN、Redis 口令、告警邮件发件人邮箱全部改为占位;
  私网 IP → 文档地址段;库名 xiujie_admin → iotlcm
- 真实手机号/邮箱 → 合法 11 位编造号 / example.com
- 移除内部资产:CI 触发器、过时 DDL(sql/db_iotlcm.sql)、
  私有依赖 go get 行、明文 registry 登录口令
- 启动 banner 统一 IOTLCM 方块字 + 社区惯例署名
- Dockerfile 改用公共基础镜像
- 拼错的 RADEME.md → README.md,README 重写
- 需要真实数据库的用例改为可用 IOTLCM_TEST_WITH_DB=1 显式启用
- 补 Apache-2.0 LICENSE,LF 行尾 + .gitattributes

数据库 schema 不随本仓发布(生产 DDL 已过时),README 已说明。
This commit is contained in:
MiFaZhan
2026-10-23 14:46:05 +08:00
commit 59eb34976a
1132 changed files with 138197 additions and 0 deletions
@@ -0,0 +1,72 @@
package trafficusage
import (
"math"
"time"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
)
var historyUsageLocation = time.FixedZone("UTC+8", 8*60*60)
func previousUsageMonth(now time.Time) time.Time {
now = now.In(historyUsageLocation)
return time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, historyUsageLocation).AddDate(0, -1, 0)
}
func reconcileUsage[T any](keys []*model.SimKey, output []model.SimBaseRes[T], amount func(T) float64) ([]model.SimBaseRes[T], []*model.SimKey) {
return reconcileUsageWithPeriod(keys, output, amount, func(T) string { return "" }, func(string) (string, bool) { return "", true }, "")
}
func reconcileUsageWithPeriod[T any](keys []*model.SimKey, output []model.SimBaseRes[T], amount func(T) float64, period func(T) string, normalize func(string) (string, bool), expected string) ([]model.SimBaseRes[T], []*model.SimKey) {
results := make(map[string][]model.SimBaseRes[T])
for _, item := range output {
if item.SimKey != nil {
results[item.SimKey.ICCID] = append(results[item.SimKey.ICCID], item)
}
}
success := make([]model.SimBaseRes[T], 0, len(keys))
failed := make([]*model.SimKey, 0)
for _, key := range keys {
items := results[key.ICCID]
if len(items) != 1 || items[0].Code != 0 {
failed = append(failed, key)
continue
}
usage := amount(items[0].ResData)
if math.IsNaN(usage) || math.IsInf(usage, 0) || usage < 0 {
failed = append(failed, key)
continue
}
if expected != "" {
periodValue, ok := normalize(period(items[0].ResData))
if !ok || periodValue != expected {
failed = append(failed, key)
continue
}
}
items[0].SimKey = key
success = append(success, items[0])
}
return success, failed
}
func normalizeUsageDay(value string) (string, bool) {
for _, layout := range []string{"20060102", "2006-01-02"} {
parsed, err := time.ParseInLocation(layout, value, historyUsageLocation)
if err == nil {
return parsed.Format("20060102"), true
}
}
return "", false
}
func normalizeUsageMonth(value string) (string, bool) {
for _, layout := range []string{"200601", "2006-01", "20060102", "2006-01-02"} {
parsed, err := time.ParseInLocation(layout, value, historyUsageLocation)
if err == nil {
return parsed.Format("200601"), true
}
}
return "", false
}
@@ -0,0 +1,41 @@
package trafficusage
import (
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"testing"
"time"
)
func TestPreviousUsageMonth(t *testing.T) {
for input, want := range map[string]string{"2026-03-31T10:00:00+08:00": "202602", "2024-03-31T10:00:00+08:00": "202402", "2026-01-31T00:00:00+08:00": "202512", "2026-08-31T16:00:00Z": "202608"} {
date, err := time.Parse(time.RFC3339, input)
if err != nil {
t.Fatal(err)
}
if got := previousUsageMonth(date).Format("200601"); got != want {
t.Fatalf("%s: got %s want %s", input, got, want)
}
}
}
func TestReconcileUsage(t *testing.T) {
keys := []*model.SimKey{{ICCID: "ok"}, {ICCID: "missing"}, {ICCID: "duplicate"}, {ICCID: "failed"}}
output := []model.SimBaseRes[model.SimDataUsageDailyOutputItem]{
{SimKey: keys[0]}, {SimKey: keys[2]}, {SimKey: keys[2]}, {SimKey: keys[3], Code: 1}, {}, {SimKey: &model.SimKey{ICCID: "unknown"}},
}
success, failed := reconcileUsage(keys, output, func(item model.SimDataUsageDailyOutputItem) float64 { return item.DataAmount })
if len(success) != 1 || success[0].SimKey != keys[0] || len(failed) != 3 {
t.Fatalf("success=%+v failed=%+v", success, failed)
}
}
func TestReconcileUsageWithPeriod(t *testing.T) {
key := &model.SimKey{ICCID: "period"}
output := []model.SimBaseRes[model.SimDataUsageDailyOutputItem]{
{SimKey: key, ResData: model.SimDataUsageDailyOutputItem{QueryDate: "2026-07-14", DataAmount: 1}},
}
success, failed := reconcileUsageWithPeriod([]*model.SimKey{key}, output, func(item model.SimDataUsageDailyOutputItem) float64 { return item.DataAmount }, func(item model.SimDataUsageDailyOutputItem) string { return item.QueryDate }, normalizeUsageDay, "20260713")
if len(success) != 0 || len(failed) != 1 {
t.Fatalf("expected period mismatch to fail, success=%+v failed=%+v", success, failed)
}
}
@@ -0,0 +1,748 @@
package trafficusage
import (
"context"
"database/sql"
"fmt"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/cmd/inithttp"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/tdengine"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
"hash/fnv"
"math"
"regexp"
"strconv"
"strings"
"time"
"unicode"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
)
// identifierPattern 限制 database/stable 只能是普通标识符,防止拼 SQL 时传入非法名称。
var identifierPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
type sTrafficUsage struct {
}
func NewTrafficUsage() *sTrafficUsage {
return &sTrafficUsage{}
}
func init() {
service.RegisterTrafficUsage(NewTrafficUsage())
// 只注册连接释放逻辑,不在应用启动时自动创建 TDengine 库表。
inithttp.RegisterHttpCleanupFunc("trafficUsage", func(ctx context.Context) error {
return tdengine.Close()
})
}
// WriteDaily 批量写入日流量数据。
// 每个 ICCID 会落到独立子表,子表通过 t_sim_card_usage_daily 超级表自动创建。
func (s *sTrafficUsage) WriteDaily(ctx context.Context, items []model.TrafficUsageDailyItem) error {
if len(items) == 0 {
return nil
}
if !tdengine.Enabled(ctx) {
g.Log().Debug(ctx, "sTrafficUsage.WriteDaily skipped, tdengine disabled")
return gerror.New("TDengine disabled; daily usage was not persisted")
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(cfg.DailyStable)
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
return err
}
batchSize := normalizeBatchSize(cfg.BatchSize)
for start := 0; start < len(items); start += batchSize {
end := start + batchSize
if end > len(items) {
end = len(items)
}
batch := items[start:end]
var builder strings.Builder
builder.WriteString("INSERT INTO ")
for i, item := range batch {
if item.Iccid == "" {
return gerror.New("traffic usage daily item iccid is empty")
}
if err := validateUsageMb("traffic usage daily item usage mb", item.UsageMb); err != nil {
return err
}
if i > 0 {
builder.WriteByte(' ')
}
builder.WriteString(qualify(dbName, childTrafficTableName(cfg.DailyStable, item.Iccid)))
builder.WriteString(" USING ")
builder.WriteString(qualify(dbName, cfg.DailyStable))
builder.WriteString(" TAGS (")
builder.WriteString(sqlString(item.Iccid))
builder.WriteString(") VALUES (")
builder.WriteString(strings.Join([]string{
sqlTime(dayStart(item.Date)),
sqlFloat(item.UsageMb),
sqlString(item.PackageCode),
}, ", "))
builder.WriteByte(')')
}
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
return gerror.Wrap(err, "tdengine write daily traffic usage failed")
}
}
return nil
}
// WriteMonthly 批量写入月流量数据。
// 月流量和日流量共用同一个 TDengine database,但使用不同超级表。
func (s *sTrafficUsage) WriteMonthly(ctx context.Context, items []model.TrafficUsageMonthlyItem) error {
if len(items) == 0 {
return nil
}
if !tdengine.Enabled(ctx) {
g.Log().Debug(ctx, "sTrafficUsage.WriteMonthly skipped, tdengine disabled")
return gerror.New("TDengine disabled; monthly usage was not persisted")
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(cfg.MonthlyStable)
if err := validateTableConfig(dbName, cfg.MonthlyStable); err != nil {
return err
}
batchSize := normalizeBatchSize(cfg.BatchSize)
for start := 0; start < len(items); start += batchSize {
end := start + batchSize
if end > len(items) {
end = len(items)
}
batch := items[start:end]
var builder strings.Builder
builder.WriteString("INSERT INTO ")
for i, item := range batch {
if item.Iccid == "" {
return gerror.New("traffic usage monthly item iccid is empty")
}
if err := validateUsageMb("traffic usage monthly item usage mb", item.UsageMb); err != nil {
return err
}
if i > 0 {
builder.WriteByte(' ')
}
builder.WriteString(qualify(dbName, childTrafficTableName(cfg.MonthlyStable, item.Iccid)))
builder.WriteString(" USING ")
builder.WriteString(qualify(dbName, cfg.MonthlyStable))
builder.WriteString(" TAGS (")
builder.WriteString(sqlString(item.Iccid))
builder.WriteString(") VALUES (")
builder.WriteString(strings.Join([]string{
sqlTime(monthStart(item.Month)),
sqlFloat(item.UsageMb),
sqlString(item.PackageCode),
}, ", "))
builder.WriteByte(')')
}
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
return gerror.Wrap(err, "tdengine write monthly traffic usage failed")
}
}
return nil
}
// WritePoolUsageDaily 批量写入流量池的流量使用记录。
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_usage_daily 超级表自动创建。
func (s *sTrafficUsage) WritePoolUsageDaily(ctx context.Context, items []model.TrafficPoolUsageItem) error {
return s.writePoolUsage(ctx, items, tdengine.GetConfig(ctx).PoolDailyStable, "flow_pool_usage_d_", dayStart, "daily")
}
// WritePoolUsageMonthly 批量写入流量池的流量使用记录。
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_usage_monthly 超级表自动创建。
func (s *sTrafficUsage) WritePoolUsageMonthly(ctx context.Context, items []model.TrafficPoolUsageItem) error {
return s.writePoolUsage(ctx, items, tdengine.GetConfig(ctx).PoolMonthlyStable, "flow_pool_usage_m_", monthStart, "monthly")
}
// WritePoolCardCountDaily 批量写入流量池的卡数量记录。
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_card_count_daily 超级表自动创建。
func (s *sTrafficUsage) WritePoolCardCountDaily(ctx context.Context, items []model.TrafficPoolCardCountItem) error {
return s.writePoolCardCount(ctx, items, tdengine.GetConfig(ctx).PoolCardCountDailyStable, "flow_pool_card_count_d_", dayStart, "daily")
}
// WritePoolCardCountMonthly 批量写入流量池的卡数量记录。
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_card_count_monthly 超级表自动创建。
func (s *sTrafficUsage) WritePoolCardCountMonthly(ctx context.Context, items []model.TrafficPoolCardCountItem) error {
return s.writePoolCardCount(ctx, items, tdengine.GetConfig(ctx).PoolCardCountMonthlyStable, "flow_pool_card_count_m_", monthStart, "monthly")
}
// UpdateDaily 更新日流量数据。
func (s *sTrafficUsage) UpdateDaily(ctx context.Context, items []model.TrafficUsageDailyItem) error {
return s.WriteDaily(ctx, items)
}
// UpdateMonthly 更新月流量数据。
func (s *sTrafficUsage) UpdateMonthly(ctx context.Context, items []model.TrafficUsageMonthlyItem) error {
return s.WriteMonthly(ctx, items)
}
// DeleteDaily 删除日流量数据。
func (s *sTrafficUsage) DeleteDaily(ctx context.Context, input model.TrafficUsageDeleteInput) error {
if !tdengine.Enabled(ctx) {
g.Log().Debug(ctx, "sTrafficUsage.DeleteDaily skipped, tdengine disabled")
return nil
}
if input.Iccid == "" {
return gerror.New("traffic usage daily delete iccid is empty")
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(cfg.DailyStable)
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
return err
}
return deleteRange(ctx, qualify(dbName, childTrafficTableName(cfg.DailyStable, input.Iccid)), input.StartTime, input.EndTime)
}
// DeleteMonthly 删除月流量数据。
func (s *sTrafficUsage) DeleteMonthly(ctx context.Context, input model.TrafficUsageDeleteInput) error {
if !tdengine.Enabled(ctx) {
g.Log().Debug(ctx, "sTrafficUsage.DeleteMonthly skipped, tdengine disabled")
return nil
}
if input.Iccid == "" {
return gerror.New("traffic usage monthly delete iccid is empty")
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(cfg.MonthlyStable)
if err := validateTableConfig(dbName, cfg.MonthlyStable); err != nil {
return err
}
return deleteRange(ctx, qualify(dbName, childTrafficTableName(cfg.MonthlyStable, input.Iccid)), input.StartTime, input.EndTime)
}
// QueryDaily 查询日流量数据。
func (s *sTrafficUsage) QueryDaily(ctx context.Context, input model.TrafficUsageQueryInput) ([]model.TrafficUsageDailyRecord, int, error) {
if !tdengine.Enabled(ctx) {
return nil, 0, nil
}
if input.Iccid == "" {
return nil, 0, gerror.New("traffic usage daily query iccid is empty")
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(cfg.DailyStable)
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
return nil, 0, err
}
table := qualify(dbName, childTrafficTableName(cfg.DailyStable, input.Iccid))
total, err := countRows(ctx, table, input.StartTime, input.EndTime)
if err != nil {
if isTableNotExist(err) {
return []model.TrafficUsageDailyRecord{}, 0, nil
}
return nil, 0, err
}
rows, err := queryRows(ctx, table, input)
if err != nil {
if isTableNotExist(err) {
return []model.TrafficUsageDailyRecord{}, 0, nil
}
return nil, 0, err
}
defer rows.Close()
list := make([]model.TrafficUsageDailyRecord, 0)
for rows.Next() {
var item model.TrafficUsageDailyRecord
if err = rows.Scan(&item.Ts, &item.UsageMb, &item.PackageCode, &item.Iccid); err != nil {
return nil, 0, err
}
list = append(list, item)
}
if err = rows.Err(); err != nil {
return nil, 0, err
}
return list, total, nil
}
// QueryMonthly 查询月流量数据。
func (s *sTrafficUsage) QueryMonthly(ctx context.Context, input model.TrafficUsageQueryInput) ([]model.TrafficUsageMonthlyRecord, int, error) {
if !tdengine.Enabled(ctx) {
return nil, 0, nil
}
if input.Iccid == "" {
return nil, 0, gerror.New("traffic usage monthly query iccid is empty")
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(cfg.MonthlyStable)
if err := validateTableConfig(dbName, cfg.MonthlyStable); err != nil {
return nil, 0, err
}
table := qualify(dbName, childTrafficTableName(cfg.MonthlyStable, input.Iccid))
total, err := countRows(ctx, table, input.StartTime, input.EndTime)
if err != nil {
if isTableNotExist(err) {
return []model.TrafficUsageMonthlyRecord{}, 0, nil
}
return nil, 0, err
}
rows, err := queryRows(ctx, table, input)
if err != nil {
if isTableNotExist(err) {
return []model.TrafficUsageMonthlyRecord{}, 0, nil
}
return nil, 0, err
}
defer rows.Close()
list := make([]model.TrafficUsageMonthlyRecord, 0)
for rows.Next() {
var item model.TrafficUsageMonthlyRecord
if err = rows.Scan(&item.Ts, &item.UsageMb, &item.PackageCode, &item.Iccid); err != nil {
return nil, 0, err
}
list = append(list, item)
}
if err = rows.Err(); err != nil {
return nil, 0, err
}
return list, total, nil
}
// AggregatePoolUsageDaily 汇总日流量数据。
func (s *sTrafficUsage) AggregatePoolUsageDaily(ctx context.Context, date time.Time, cards []model.TrafficPoolCard) ([]model.TrafficPoolUsageAggregate, error) {
return s.aggregatePoolUsage(ctx, dayStart(date), cards, tdengine.GetConfig(ctx).DailyStable, "daily")
}
// AggregatePoolUsageMonthly 汇总月流量数据。
func (s *sTrafficUsage) AggregatePoolUsageMonthly(ctx context.Context, month time.Time, cards []model.TrafficPoolCard) ([]model.TrafficPoolUsageAggregate, error) {
return s.aggregatePoolUsage(ctx, monthStart(month), cards, tdengine.GetConfig(ctx).MonthlyStable, "monthly")
}
// aggregatePoolUsage 汇总流量数据。
func (s *sTrafficUsage) aggregatePoolUsage(ctx context.Context, ts time.Time, cards []model.TrafficPoolCard, stable string, name string) ([]model.TrafficPoolUsageAggregate, error) {
if len(cards) == 0 {
return []model.TrafficPoolUsageAggregate{}, nil
}
if !tdengine.Enabled(ctx) {
g.Log().Debugf(ctx, "sTrafficUsage.aggregatePoolUsage %s skipped, tdengine disabled", name)
return []model.TrafficPoolUsageAggregate{}, nil
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(stable)
if err := validateTableConfig(dbName, stable); err != nil {
return nil, err
}
grouped := make(map[string][]string)
for _, card := range cards {
if card.FlowPoolCode == "" || card.Iccid == "" {
continue
}
grouped[card.FlowPoolCode] = append(grouped[card.FlowPoolCode], card.Iccid)
}
result := make([]model.TrafficPoolUsageAggregate, 0, len(grouped))
for flowPoolCode, iccids := range grouped {
usageMb, recordCount, err := aggregatePoolUsageByIccids(ctx, dbName, stable, ts, iccids)
if err != nil {
return nil, gerror.Wrapf(err, "tdengine aggregate %s traffic pool usage failed, flowPoolCode:%s", name, flowPoolCode)
}
if recordCount != int64(len(iccids)) {
return nil, gerror.Newf(
"tdengine aggregate %s traffic pool usage incomplete, flowPoolCode:%s, expected:%d, actual:%d",
name,
flowPoolCode,
len(iccids),
recordCount,
)
}
result = append(result, model.TrafficPoolUsageAggregate{
FlowPoolCode: flowPoolCode,
UsageMb: usageMb,
RecordCount: recordCount,
})
}
return result, nil
}
// writePoolUsage 批量写入流量池使用量数据。
// 每个 FlowPoolCode 会落到独立子表,子表通过对应的超级表自动创建。
func (s *sTrafficUsage) writePoolUsage(ctx context.Context, items []model.TrafficPoolUsageItem, stable string, prefix string, normalizeTime func(time.Time) time.Time, name string) error {
if len(items) == 0 {
return nil
}
if !tdengine.Enabled(ctx) {
g.Log().Debugf(ctx, "sTrafficUsage.writePoolUsage %s skipped, tdengine disabled", name)
return nil
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(stable)
if err := validateTableConfig(dbName, stable); err != nil {
return err
}
batchSize := normalizeBatchSize(cfg.BatchSize)
for start := 0; start < len(items); start += batchSize {
end := start + batchSize
if end > len(items) {
end = len(items)
}
batch := items[start:end]
var builder strings.Builder
builder.WriteString("INSERT INTO ")
for i, item := range batch {
if item.FlowPoolCode == "" {
return gerror.New("traffic pool usage item flow pool code is empty")
}
if err := validateUsageMb("traffic pool usage item usage mb", item.UsageMb); err != nil {
return err
}
if i > 0 {
builder.WriteByte(' ')
}
builder.WriteString(qualify(dbName, childTableNameWithPrefix(prefix, item.FlowPoolCode)))
builder.WriteString(" USING ")
builder.WriteString(qualify(dbName, stable))
builder.WriteString(" TAGS (")
builder.WriteString(sqlString(item.FlowPoolCode))
builder.WriteString(") VALUES (")
builder.WriteString(strings.Join([]string{
sqlTime(normalizeTime(item.Date)),
sqlFloat(item.UsageMb),
}, ", "))
builder.WriteByte(')')
}
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
return gerror.Wrapf(err, "tdengine write %s traffic pool usage failed", name)
}
}
return nil
}
// writePoolCardCount 批量写入流量池卡数量数据。
// 每个 FlowPoolCode 会落到独立子表,子表通过对应的超级表自动创建。
func (s *sTrafficUsage) writePoolCardCount(ctx context.Context, items []model.TrafficPoolCardCountItem, stable string, prefix string, normalizeTime func(time.Time) time.Time, name string) error {
if len(items) == 0 {
return nil
}
if !tdengine.Enabled(ctx) {
g.Log().Debugf(ctx, "sTrafficUsage.writePoolCardCount %s skipped, tdengine disabled", name)
return nil
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(stable)
if err := validateTableConfig(dbName, stable); err != nil {
return err
}
batchSize := normalizeBatchSize(cfg.BatchSize)
for start := 0; start < len(items); start += batchSize {
end := start + batchSize
if end > len(items) {
end = len(items)
}
batch := items[start:end]
var builder strings.Builder
builder.WriteString("INSERT INTO ")
for i, item := range batch {
if item.FlowPoolCode == "" {
return gerror.New("traffic pool card count item flow pool code is empty")
}
if item.CardCount < 0 {
return gerror.Newf("traffic pool card count item card count cannot be negative: %d", item.CardCount)
}
if i > 0 {
builder.WriteByte(' ')
}
builder.WriteString(qualify(dbName, childTableNameWithPrefix(prefix, item.FlowPoolCode)))
builder.WriteString(" USING ")
builder.WriteString(qualify(dbName, stable))
builder.WriteString(" TAGS (")
builder.WriteString(sqlString(item.FlowPoolCode))
builder.WriteString(") VALUES (")
builder.WriteString(strings.Join([]string{
sqlTime(normalizeTime(item.Date)),
strconv.FormatInt(item.CardCount, 10),
}, ", "))
builder.WriteByte(')')
}
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
return gerror.Wrapf(err, "tdengine write %s traffic pool card count failed", name)
}
}
return nil
}
// aggregatePoolUsageByIccids 汇总流量数据。
// 每个 ICCID 会落到独立子表,子表通过 t_sim_card_usage_daily 或 t_sim_card_usage_monthly 超级表自动创建。
func aggregatePoolUsageByIccids(ctx context.Context, database string, stable string, ts time.Time, iccids []string) (float64, int64, error) {
var usageMb float64
var recordCount int64
for _, iccid := range iccids {
table := qualify(database, childTrafficTableName(stable, iccid))
row, err := tdengine.QueryRow(ctx, fmt.Sprintf("SELECT usage_mb FROM %s WHERE ts = %s", table, sqlTime(ts)))
if err != nil {
return 0, 0, gerror.Wrap(err, "tdengine query traffic usage row failed")
}
var value float64
if err = row.Scan(&value); err != nil {
if err == sql.ErrNoRows || isTableNotExist(err) {
continue
}
return 0, 0, gerror.Wrap(err, "tdengine scan traffic usage row failed")
}
usageMb += value
recordCount++
}
return usageMb, recordCount, nil
}
// countRows 统计流量数据记录数。
func countRows(ctx context.Context, table string, startTime, endTime time.Time) (int, error) {
where := whereRangeSQL(startTime, endTime)
row, err := tdengine.QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s%s", table, where))
if err != nil {
return 0, gerror.Wrap(err, "tdengine count traffic usage failed")
}
total := 0
if err = row.Scan(&total); err != nil {
return 0, gerror.Wrap(err, "tdengine count traffic usage failed")
}
return total, nil
}
// queryRows 查询流量数据记录。
func queryRows(ctx context.Context, table string, input model.TrafficUsageQueryInput) (*sql.Rows, error) {
fields := "ts, usage_mb, package_code, iccid"
where := whereRangeSQL(input.StartTime, input.EndTime)
order := "DESC"
if input.Asc {
order = "ASC"
}
limit := ""
page, pageSize := normalizePage(input.Page, input.PageSize)
if pageSize > 0 {
limit = fmt.Sprintf(" LIMIT %d OFFSET %d", pageSize, (page-1)*pageSize)
}
rows, err := tdengine.Query(ctx, fmt.Sprintf("SELECT %s FROM %s%s ORDER BY ts %s%s", fields, table, where, order, limit))
if err != nil {
return nil, gerror.Wrap(err, "tdengine query traffic usage failed")
}
return rows, nil
}
// deleteRange 删除流量数据记录。
func deleteRange(ctx context.Context, table string, startTime, endTime time.Time) error {
where := whereRangeSQL(startTime, endTime)
if where == "" {
return gerror.New("traffic usage delete time range is empty")
}
if _, err := tdengine.Exec(ctx, fmt.Sprintf("DELETE FROM %s%s", table, where)); err != nil {
if isTableNotExist(err) {
return nil
}
return gerror.Wrap(err, "tdengine delete traffic usage failed")
}
return nil
}
// whereRangeSQL 构建时间范围查询条件。
func whereRangeSQL(startTime, endTime time.Time) string {
clauses := make([]string, 0, 2)
if !startTime.IsZero() {
clauses = append(clauses, "ts >= "+sqlTime(startTime))
}
if !endTime.IsZero() {
clauses = append(clauses, "ts <= "+sqlTime(endTime))
}
if len(clauses) == 0 {
return ""
}
return " WHERE " + strings.Join(clauses, " AND ")
}
// validateTableConfig 验证表配置是否有效。
func validateTableConfig(database, stable string) error {
if !identifierPattern.MatchString(database) {
return gerror.Newf("invalid tdengine database name: %s", database)
}
if !identifierPattern.MatchString(stable) {
return gerror.Newf("invalid tdengine stable name: %s", stable)
}
return nil
}
// qualify 格式化表名。
func qualify(database, table string) string {
return fmt.Sprintf("%s.%s", database, table)
}
// childTrafficTableName 构建子表名。
// 每个 ICCID 会落到独立子表,子表通过 t_sim_card_usage_daily 或 t_sim_card_usage_monthly 超级表自动创建。
func childTrafficTableName(stable string, iccid string) string {
prefix := "sim_card_usage_m_"
if stable == "t_sim_card_usage_daily" {
prefix = "sim_card_usage_d_"
}
return childTableNameWithPrefix(prefix, iccid)
}
// childTableNameWithPrefix 构建子表名。
func childTableNameWithPrefix(prefix string, value string) string {
var builder strings.Builder
builder.WriteString(prefix)
for _, r := range strings.ToLower(value) {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
builder.WriteRune(r)
}
}
name := builder.String()
if name == prefix || len(name) > 190 {
return fmt.Sprintf("%s%x", prefix, hashString(value))
}
return name
}
// hashString 对字符串进行哈希处理。
// 用于生成子表名。
func hashString(value string) uint32 {
hash := fnv.New32a()
_, _ = hash.Write([]byte(value))
return hash.Sum32()
}
// normalizeBatchSize 归一化批量大小。
func normalizeBatchSize(size int) int {
if size <= 0 {
return 200
}
return size
}
// normalizePage 归一化分页参数。
func normalizePage(page, pageSize int) (int, int) {
if page <= 0 {
page = 1
}
if pageSize < 0 {
pageSize = 0
}
return page, pageSize
}
// dayStart 获取指定日期的快照时间。
// 用于查询指定日期的流量数据记录。
func dayStart(t time.Time) time.Time {
if t.IsZero() {
t = time.Now()
}
return time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location())
}
// monthStart 获取指定月份的快照时间。
// 用于查询指定月份的流量数据记录。
func monthStart(t time.Time) time.Time {
if t.IsZero() {
t = time.Now()
}
lastDay := time.Date(t.Year(), t.Month()+1, 0, 23, 59, 59, 0, t.Location())
return lastDay
}
// sqlString 格式化字符串为SQL字符串。
func sqlString(value string) string {
return "'" + strings.ReplaceAll(value, "'", "''") + "'"
}
// sqlTime 格式化时间时间为SQL时间。
func sqlTime(value time.Time) string {
return sqlString(value.Format("2006-01-02 15:04:05.000"))
}
// sqlFloat 格式化浮点数为SQL浮点数。
func sqlFloat(value float64) string {
return strconv.FormatFloat(value, 'f', -1, 64)
}
// validateUsageMb 验证流量使用记录是否有效。
// 用于检查流量使用记录是否为有限的浮点数,且是否为负数。
func validateUsageMb(name string, value float64) error {
if math.IsNaN(value) || math.IsInf(value, 0) {
return gerror.Newf("%s must be finite: %v", name, value)
}
if value < 0 {
return gerror.Newf("%s cannot be negative: %v", name, value)
}
return nil
}
// isTableNotExist 检查表是否存在。
// 用于在删除流量数据记录时,检查表是否存在。
func isTableNotExist(err error) bool {
if err == nil {
return false
}
text := strings.ToLower(err.Error())
return strings.Contains(text, "table does not exist") ||
strings.Contains(text, "table not exist") ||
strings.Contains(text, "does not exist")
}
// GetTopUsageCardsYesterday 查询前一天流量使用前五名的卡片
func (s *sTrafficUsage) GetTopUsageCardsYesterday(ctx context.Context) ([]model.TopUsageCard, error) {
if !tdengine.Enabled(ctx) {
return nil, nil
}
cfg := tdengine.GetConfig(ctx)
dbName := cfg.DbNameFor(cfg.DailyStable)
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
return nil, err
}
// 获取前一天日期
yesterday := time.Now().AddDate(0, 0, -1)
startStr := yesterday.Format("2006-01-02")
endStr := dayStart(yesterday).Format("2006-01-02 15:04:05")
table := qualify(dbName, cfg.DailyStable)
sql := fmt.Sprintf(
"SELECT iccid, FIRST(usage_mb), LAST(usage_mb) FROM %s WHERE ts >= '%s' AND ts <= '%s' GROUP BY iccid ORDER BY LAST(usage_mb) DESC LIMIT 5",
table, startStr, endStr)
rows, err := tdengine.Query(ctx, sql)
if err != nil {
if isTableNotExist(err) {
return []model.TopUsageCard{}, nil
}
return nil, err
}
defer rows.Close()
result := make([]model.TopUsageCard, 0)
for rows.Next() {
var iccid string
var firstUsage, lastUsage float64
if err = rows.Scan(&iccid, &firstUsage, &lastUsage); err != nil {
return nil, err
}
result = append(result, model.TopUsageCard{
Iccid: iccid,
UsageMb: lastUsage,
StatDate: yesterday.Format("2006-01-02"),
})
}
if err = rows.Err(); err != nil {
return nil, err
}
return result, nil
}
@@ -0,0 +1,37 @@
package trafficusage
import (
"context"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
"github.com/gogf/gf/v2/database/gdb"
)
func trafficUsageEligibleCardModel(ctx context.Context) *gdb.Model {
cardTable := dao.TSimCardInfo.Table()
deptTable := dao.SysDept.Table()
cardColumns := dao.TSimCardInfo.Columns()
deptColumns := dao.SysDept.Columns()
return dao.TSimCardInfo.Ctx(ctx).
InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId).
Where(cardTable+"."+cardColumns.Iccid+" <> ?", "").
Where(cardTable+"."+cardColumns.ChanKey+" <> ?", "").
Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", "").
WhereNull(deptTable + "." + deptColumns.DeletedAt)
}
func trafficUsageCardFields() []string {
cardTable := dao.TSimCardInfo.Table()
columns := dao.TSimCardInfo.Columns()
return []string{
cardTable + "." + columns.Id + " AS " + columns.Id,
cardTable + "." + columns.Iccid + " AS " + columns.Iccid,
cardTable + "." + columns.Imsi + " AS " + columns.Imsi,
cardTable + "." + columns.Msisdn + " AS " + columns.Msisdn,
cardTable + "." + columns.ChanKey + " AS " + columns.ChanKey,
cardTable + "." + columns.CurrentPackageCode + " AS " + columns.CurrentPackageCode,
cardTable + "." + columns.FlowPoolCode + " AS " + columns.FlowPoolCode,
cardTable + "." + columns.IsFlowPool + " AS " + columns.IsFlowPool,
}
}
@@ -0,0 +1,268 @@
package trafficusage
import (
"context"
"fmt"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/rabbitmq"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
"time"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/utility/uuid32"
"github.com/gogf/gf/v2/frame/g"
)
const (
trafficUsageDailyPageSize = 100
)
type sTrafficUsageDailySync struct {
}
// These indirections keep the trigger deterministic in unit tests without changing
// the production publisher, clock, or run-id generator.
var (
dailyUsageNow = time.Now
dailyUsageRunID = uuid32.S
dailyUsageRabbitEnabled = rabbitmq.Enabled
dailyUsagePublish = rabbitmq.Publish
)
func NewTrafficUsageDailySync() *sTrafficUsageDailySync {
return &sTrafficUsageDailySync{}
}
func init() {
service.RegisterTrafficUsageDailySync(NewTrafficUsageDailySync())
}
// StartDaily publishes one command for Worker to execute the previous-day usage
// full sync. Admin deliberately does not scan cards or call an upstream service;
// Worker owns pacing, retries, the failure pool and replay.
func (s *sTrafficUsageDailySync) StartDaily(ctx context.Context) error {
_, err := s.StartDailyRun(ctx)
return err
}
func (s *sTrafficUsageDailySync) StartDailyRun(ctx context.Context) (string, error) {
return s.StartDailyRunScoped(ctx, nil)
}
// StartDailyRunScoped starts a run restricted to scope.Channels. A nil scope, or
// one whose channels normalize to empty, starts a full sync.
func (s *sTrafficUsageDailySync) StartDailyRunScoped(ctx context.Context, scope *model.SyncScope) (string, error) {
return s.StartDailyRunScopedAt(ctx, scope, "")
}
// StartDailyRunScopedAt starts a run for the given day (yyyyMMdd). An empty
// queryDate keeps the historical behaviour and covers the previous calendar day
// in the business timezone; a non-empty value lets an operator backfill one
// specific past day.
func (s *sTrafficUsageDailySync) StartDailyRunScopedAt(ctx context.Context, scope *model.SyncScope, queryDate string) (string, error) {
if ctx == nil {
ctx = context.Background()
}
startedAt := time.Now()
normalized, err := model.NormalizeScope(scope)
if err != nil {
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=validate durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
return "", err
}
if !dailyUsageRabbitEnabled(ctx) {
err := fmt.Errorf("daily usage start skipped: rabbitmq is disabled")
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=precheck durationMs=%d reason=%q", elapsedMilliseconds(startedAt), err.Error())
return "", err
}
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=timezone durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
return "", fmt.Errorf("load daily usage timezone: %w", err)
}
resolvedDate, err := resolveDailyQueryDate(dailyUsageNow().In(location), location, queryDate)
if err != nil {
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=validate reason=invalid_query_date queryDate=%q durationMs=%d error=%q", queryDate, elapsedMilliseconds(startedAt), err.Error())
return "", err
}
message := model.TrafficUsageDailyStartMessage{
RunID: fmt.Sprintf("daily-usage-%s-%s", resolvedDate, dailyUsageRunID()),
Operation: model.OperationPreviousDayUsageFullSync,
QueryDate: resolvedDate,
TriggeredAt: dailyUsageNow().In(location).Format(time.RFC3339),
Scope: normalized,
}
if err = dailyUsagePublish(ctx, consts.QueueTrafficUsageDailyStart, message); err != nil {
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=publish runId=%s queryDate=%s durationMs=%d error=%q", message.RunID, message.QueryDate, elapsedMilliseconds(startedAt), err.Error())
return "", fmt.Errorf("publish daily usage start: %w", err)
}
g.Log().Infof(ctx, "event=daily_usage_trigger status=published runId=%s queryDate=%s triggeredAt=%s durationMs=%d", message.RunID, message.QueryDate, message.TriggeredAt, elapsedMilliseconds(startedAt))
return message.RunID, nil
}
// minBackfillDay is the earliest day a manual backfill may target. It only
// catches obvious typos; it is not a business boundary. Comparing the
// zero-padded yyyyMMdd strings sorts chronologically.
const minBackfillDay = "20200101"
// resolveDailyQueryDate returns the day a run must cover. An empty requested
// value keeps the legacy default (the previous day in the business timezone);
// otherwise the value is validated and returned in canonical yyyyMMdd form.
func resolveDailyQueryDate(now time.Time, location *time.Location, requested string) (string, error) {
// The default covers the previous calendar day, which is the same period the
// legacy admin scan used.
yesterday := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location).AddDate(0, 0, -1)
if requested == "" {
return yesterday.Format("20060102"), nil
}
parsed, err := time.ParseInLocation("20060102", requested, location)
if err != nil {
return "", fmt.Errorf("invalid queryDate %q: want yyyyMMdd", requested)
}
if parsed.After(yesterday) {
return "", fmt.Errorf("invalid queryDate %q: must not be later than %s", requested, yesterday.Format("20060102"))
}
if parsed.Format("20060102") < minBackfillDay {
return "", fmt.Errorf("invalid queryDate %q: must not be earlier than %s", requested, minBackfillDay)
}
return parsed.Format("20060102"), nil
}
// SyncDaily is the legacy in-process scan. It is retained as the rollback path
// until the worker chain is verified in production, and it is no longer
// registered on any timer.
func (s *sTrafficUsageDailySync) SyncDaily(ctx context.Context) {
now := time.Now().In(historyUsageLocation)
yesterday := now.AddDate(0, 0, -1)
queryDate := yesterday.Format("20060102")
date := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, historyUsageLocation)
cardColumns := dao.TSimCardInfo.Columns()
cardTable := dao.TSimCardInfo.Table()
snapshotMaxID, err := trafficUsageEligibleCardModel(ctx).Max(cardTable + "." + cardColumns.Id)
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageDaily snapshot sim cards failed: %+v", err)
return
}
maxID := int64(snapshotMaxID)
if maxID <= 0 {
return
}
var lastID int64
for lastID < maxID {
list := []entity.TSimCardInfo{}
if err = trafficUsageEligibleCardModel(ctx).
Fields(trafficUsageCardFields()).
WhereGT(cardTable+"."+cardColumns.Id, lastID).
WhereLTE(cardTable+"."+cardColumns.Id, maxID).
OrderAsc(cardTable + "." + cardColumns.Id).
Limit(trafficUsageDailyPageSize).
Scan(&list); err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageDaily list sim cards failed, lastId:%d, err:%+v", lastID, err)
return
}
if len(list) == 0 {
return
}
lastID = list[len(list)-1].Id
cardMap := trafficUsageDailyCardMap(list)
simKeys := trafficUsageDailySimKeys(list)
output, err := service.UpstreamChan().QuerySimDatausageDaily(ctx, &model.SimDataUsageDailyInput{
SimKeys: simKeys,
ReqData: model.SimDataUsageDailyInputItem{
QueryDate: queryDate,
},
})
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageDaily query daily failed, lastId:%d, count:%d, err:%+v", lastID, len(simKeys), err)
for _, simKey := range simKeys {
s.publishRetry(ctx, simKey, cardMap[simKey.ICCID], queryDate)
}
continue
}
success, failed := reconcileUsageWithPeriod(simKeys, output, func(item model.SimDataUsageDailyOutputItem) float64 { return item.DataAmount }, func(item model.SimDataUsageDailyOutputItem) string { return item.QueryDate }, normalizeUsageDay, queryDate)
for _, key := range failed {
s.publishRetry(ctx, key, cardMap[key.ICCID], queryDate)
}
if items := s.buildDailyItems(success, cardMap, date); len(items) > 0 {
if err = service.TrafficUsage().UpdateDaily(ctx, items); err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageDaily write tdengine failed, lastId:%d, err:%+v", lastID, err)
for _, item := range success {
s.publishRetry(ctx, item.SimKey, cardMap[item.SimKey.ICCID], queryDate, item.ResData.DataAmount)
}
}
}
}
}
// buildDailyItems 构建SIM卡的流量使用记录
func (s *sTrafficUsageDailySync) buildDailyItems(output []model.SimBaseRes[model.SimDataUsageDailyOutputItem], cardMap map[string]model.TrafficUsageSimCard, date time.Time) []model.TrafficUsageDailyItem {
items := make([]model.TrafficUsageDailyItem, 0, len(output))
for _, item := range output {
if item.SimKey == nil {
continue
}
card := cardMap[item.SimKey.ICCID]
items = append(items, model.TrafficUsageDailyItem{
TrafficUsageBase: model.TrafficUsageBase{
Iccid: item.SimKey.ICCID,
UsageMb: item.ResData.DataAmount,
PackageCode: card.PackageCode,
},
Date: date,
})
}
return items
}
// publishRetry 发布重试消息
func (s *sTrafficUsageDailySync) publishRetry(ctx context.Context, simKey *model.SimKey, card model.TrafficUsageSimCard, queryDate string, usage ...float64) {
var snapshot *float64
if len(usage) > 0 {
snapshot = &usage[0]
}
if !rabbitmq.Enabled(ctx) {
g.Log().Errorf(ctx, "traffic usage daily retry skipped, rabbitmq disabled, simKey:%+v, queryDate:%s", simKey, queryDate)
return
}
if err := rabbitmq.PublishConfirmed(ctx, consts.QueueTrafficUsageDailyRetry, model.TrafficUsageDailyRetryMessage{
UsageMb: snapshot,
SimKey: simKey,
Card: card,
QueryDate: queryDate,
}); err != nil {
g.Log().Errorf(ctx, "publish traffic usage daily retry failed, simKey:%+v, queryDate:%s, err:%+v", simKey, queryDate, err)
}
}
// trafficUsageDailySimKeys 提取SIM卡的ChanKey、ICCID、IMSI、MSISDN
func trafficUsageDailySimKeys(list []entity.TSimCardInfo) []*model.SimKey {
simKeys := make([]*model.SimKey, 0, len(list))
for _, item := range list {
simKeys = append(simKeys, &model.SimKey{
ChanKey: item.ChanKey,
ICCID: item.Iccid,
IMSI: item.Imsi,
MSISDN: item.Msisdn,
})
}
return simKeys
}
// trafficUsageDailyCardMap 提取SIM卡的ChanKey、ICCID、IMSI、MSISDN、PackageCode、FlowPoolCode、IsFlowPool
func trafficUsageDailyCardMap(list []entity.TSimCardInfo) map[string]model.TrafficUsageSimCard {
cardMap := make(map[string]model.TrafficUsageSimCard, len(list))
for _, item := range list {
cardMap[item.Iccid] = model.TrafficUsageSimCard{
Iccid: item.Iccid,
PackageCode: item.CurrentPackageCode,
}
}
return cardMap
}
@@ -0,0 +1,329 @@
package trafficusage
import (
"context"
"fmt"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/rabbitmq"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
"time"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/utility/uuid32"
"github.com/gogf/gf/v2/frame/g"
)
const (
trafficUsageMonthlyPageSize = 100
)
type sTrafficUsageMonthlySync struct {
}
const monthlyUsageFullSyncOperation = "current_month_usage_full_sync"
// These indirections keep the trigger deterministic in unit tests without changing
// the production publisher, clock, or run-id generator.
var (
monthlyUsageNow = time.Now
monthlyUsageRunID = uuid32.S
monthlyUsageRabbitEnabled = rabbitmq.Enabled
monthlyUsagePublish = rabbitmq.Publish
monthlyHistoryNow = time.Now
monthlyHistoryRunID = uuid32.S
monthlyHistoryRabbitEnabled = rabbitmq.Enabled
monthlyHistoryPublish = rabbitmq.Publish
)
func NewTrafficUsageMonthlySync() *sTrafficUsageMonthlySync {
return &sTrafficUsageMonthlySync{}
}
func init() {
service.RegisterTrafficUsageMonthlySync(NewTrafficUsageMonthlySync())
}
// StartMonthly publishes one command for Worker to execute the full monthly
// usage sync. Admin deliberately does not scan cards or call an upstream service.
func (s *sTrafficUsageMonthlySync) StartMonthly(ctx context.Context) error {
_, err := s.StartMonthlyRun(ctx)
return err
}
func (s *sTrafficUsageMonthlySync) StartMonthlyRun(ctx context.Context) (string, error) {
return s.StartMonthlyRunScoped(ctx, nil)
}
// StartMonthlyRunScoped starts a run restricted to scope.Channels. A nil scope,
// or one whose channels normalize to empty, starts a full sync.
func (s *sTrafficUsageMonthlySync) StartMonthlyRunScoped(ctx context.Context, scope *model.SyncScope) (string, error) {
if ctx == nil {
ctx = context.Background()
}
startedAt := time.Now()
normalized, err := model.NormalizeScope(scope)
if err != nil {
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=validate durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
return "", err
}
if !monthlyUsageRabbitEnabled(ctx) {
err := fmt.Errorf("monthly usage start skipped: rabbitmq is disabled")
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=precheck durationMs=%d reason=%q", elapsedMilliseconds(startedAt), err.Error())
return "", err
}
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=timezone durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
return "", fmt.Errorf("load monthly usage timezone: %w", err)
}
now := monthlyUsageNow().In(location)
message := model.TrafficUsageMonthlyStartMessage{
RunID: fmt.Sprintf("current-month-usage-%s-%s", now.Format("20060102"), monthlyUsageRunID()),
Operation: monthlyUsageFullSyncOperation,
QueryMonth: now.Format("200601"),
TriggeredAt: now.Format(time.RFC3339),
Scope: normalized,
}
if err = monthlyUsagePublish(ctx, consts.QueueTrafficUsageMonthlyStart, message); err != nil {
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=publish runId=%s queryMonth=%s durationMs=%d error=%q", message.RunID, message.QueryMonth, elapsedMilliseconds(startedAt), err.Error())
return "", fmt.Errorf("publish monthly usage start: %w", err)
}
g.Log().Infof(ctx, "event=monthly_usage_trigger status=published runId=%s queryMonth=%s triggeredAt=%s durationMs=%d", message.RunID, message.QueryMonth, message.TriggeredAt, elapsedMilliseconds(startedAt))
return message.RunID, nil
}
func elapsedMilliseconds(startedAt time.Time) int64 {
return time.Since(startedAt).Milliseconds()
}
// StartMonthlyHistory publishes one command for Worker to execute the
// previous-month usage history sync (the TDengine monthly table).
func (s *sTrafficUsageMonthlySync) StartMonthlyHistory(ctx context.Context) error {
_, err := s.StartMonthlyHistoryRun(ctx)
return err
}
func (s *sTrafficUsageMonthlySync) StartMonthlyHistoryRun(ctx context.Context) (string, error) {
return s.StartMonthlyHistoryRunScoped(ctx, nil)
}
// StartMonthlyHistoryRunScoped starts a monthly history run restricted to
// scope.Channels. A nil scope, or one whose channels normalize to empty, starts
// a full sync.
//
// The period is the previous month, matching what the legacy SyncMonthly scan
// wrote. Do not confuse this with StartMonthlyRunScoped, which fills the
// current-month snapshot in MySQL.
func (s *sTrafficUsageMonthlySync) StartMonthlyHistoryRunScoped(ctx context.Context, scope *model.SyncScope) (string, error) {
return s.StartMonthlyHistoryRunScopedAt(ctx, scope, "")
}
// StartMonthlyHistoryRunScopedAt starts a monthly history run for the given
// month (yyyyMM). An empty queryMonth keeps the historical behaviour and covers
// the previous calendar month; a non-empty value lets an operator backfill one
// specific past month.
func (s *sTrafficUsageMonthlySync) StartMonthlyHistoryRunScopedAt(ctx context.Context, scope *model.SyncScope, queryMonth string) (string, error) {
if ctx == nil {
ctx = context.Background()
}
startedAt := time.Now()
normalized, err := model.NormalizeScope(scope)
if err != nil {
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=validate durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
return "", err
}
if !monthlyHistoryRabbitEnabled(ctx) {
err := fmt.Errorf("monthly history start skipped: rabbitmq is disabled")
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=precheck durationMs=%d reason=%q", elapsedMilliseconds(startedAt), err.Error())
return "", err
}
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=timezone durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
return "", fmt.Errorf("load monthly history timezone: %w", err)
}
now := monthlyHistoryNow().In(location)
resolvedMonth, err := resolveMonthlyHistoryQueryMonth(now, location, queryMonth)
if err != nil {
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=validate reason=invalid_query_month queryMonth=%q durationMs=%d error=%q", queryMonth, elapsedMilliseconds(startedAt), err.Error())
return "", err
}
message := model.TrafficUsageMonthlyHistoryStartMessage{
RunID: fmt.Sprintf("monthly-history-%s-%s", resolvedMonth, monthlyHistoryRunID()),
Operation: model.OperationPreviousMonthUsageFullSync,
QueryMonth: resolvedMonth,
TriggeredAt: now.Format(time.RFC3339),
Scope: normalized,
}
if err = monthlyHistoryPublish(ctx, consts.QueueTrafficUsageMonthlyHistoryStart, message); err != nil {
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=publish runId=%s queryMonth=%s durationMs=%d error=%q", message.RunID, message.QueryMonth, elapsedMilliseconds(startedAt), err.Error())
return "", fmt.Errorf("publish monthly history start: %w", err)
}
g.Log().Infof(ctx, "event=monthly_history_trigger status=published runId=%s queryMonth=%s triggeredAt=%s durationMs=%d", message.RunID, message.QueryMonth, message.TriggeredAt, elapsedMilliseconds(startedAt))
return message.RunID, nil
}
// minBackfillMonth is the earliest month a manual backfill may target. It only
// catches obvious typos; it is not a business boundary.
const minBackfillMonth = "202001"
// resolveMonthlyHistoryQueryMonth returns the month a history run must cover. An
// empty requested value keeps the legacy default (the previous calendar month);
// otherwise the value is validated and returned in canonical yyyyMM form.
func resolveMonthlyHistoryQueryMonth(now time.Time, location *time.Location, requested string) (string, error) {
currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location)
if requested == "" {
// The previous month is what the legacy scan backfilled, and the
// month-first day is what the worker's snapshotMonth derives the
// end-of-month ts from.
return currentMonth.AddDate(0, -1, 0).Format("200601"), nil
}
parsed, err := time.ParseInLocation("200601", requested, location)
if err != nil {
return "", fmt.Errorf("invalid queryMonth %q: want yyyyMM", requested)
}
// The current month is served by the MySQL snapshot chain
// (StartMonthlyRunScoped), not by the TDengine history chain.
if !parsed.Before(currentMonth) {
return "", fmt.Errorf("invalid queryMonth %q: must be earlier than the current month %s", requested, currentMonth.Format("200601"))
}
if parsed.Format("200601") < minBackfillMonth {
return "", fmt.Errorf("invalid queryMonth %q: must not be earlier than %s", requested, minBackfillMonth)
}
return parsed.Format("200601"), nil
}
// SyncMonthly is the legacy in-process scan. It is retained as the rollback path
// until the worker monthly history chain is verified in production.
func (s *sTrafficUsageMonthlySync) SyncMonthly(ctx context.Context) {
month := previousUsageMonth(time.Now())
queryDate := month.Format("200601")
cardColumns := dao.TSimCardInfo.Columns()
cardTable := dao.TSimCardInfo.Table()
snapshotMaxID, err := trafficUsageEligibleCardModel(ctx).Max(cardTable + "." + cardColumns.Id)
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly snapshot sim cards failed: %+v", err)
return
}
maxID := int64(snapshotMaxID)
if maxID <= 0 {
return
}
var lastID int64
for lastID < maxID {
list := []entity.TSimCardInfo{}
if err = trafficUsageEligibleCardModel(ctx).
Fields(trafficUsageCardFields()).
WhereGT(cardTable+"."+cardColumns.Id, lastID).
WhereLTE(cardTable+"."+cardColumns.Id, maxID).
OrderAsc(cardTable + "." + cardColumns.Id).
Limit(trafficUsageMonthlyPageSize).
Scan(&list); err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly list sim cards failed, lastId:%d, err:%+v", lastID, err)
return
}
if len(list) == 0 {
return
}
lastID = list[len(list)-1].Id
cardMap := trafficUsageMonthlyCardMap(list)
simKeys := trafficUsageMonthlySimKeys(list)
output, err := service.UpstreamChan().QuerySimDatausageMonthly(ctx, &model.SimDataUsageMonthlyInput{
SimKeys: simKeys,
ReqData: model.SimDataUsageMonthlyInputItem{
QueryDate: queryDate,
},
})
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly query monthly failed, lastId:%d, count:%d, err:%+v", lastID, len(simKeys), err)
for _, simKey := range simKeys {
s.publishRetry(ctx, simKey, cardMap[simKey.ICCID], queryDate)
}
continue
}
success, failed := reconcileUsageWithPeriod(simKeys, output, func(item model.SimDataUsageMonthlyOutputItem) float64 { return item.DataAmount }, func(item model.SimDataUsageMonthlyOutputItem) string { return item.DeadLine }, normalizeUsageMonth, queryDate)
for _, key := range failed {
s.publishRetry(ctx, key, cardMap[key.ICCID], queryDate)
}
if items := s.buildMonthlyItems(success, cardMap, month); len(items) > 0 {
if err = service.TrafficUsage().UpdateMonthly(ctx, items); err != nil {
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly write tdengine failed, lastId:%d, err:%+v", lastID, err)
for _, item := range success {
s.publishRetry(ctx, item.SimKey, cardMap[item.SimKey.ICCID], queryDate, item.ResData.DataAmount)
}
}
}
}
}
func (s *sTrafficUsageMonthlySync) buildMonthlyItems(output []model.SimBaseRes[model.SimDataUsageMonthlyOutputItem], cardMap map[string]model.TrafficUsageSimCard, month time.Time) []model.TrafficUsageMonthlyItem {
items := make([]model.TrafficUsageMonthlyItem, 0, len(output))
for _, item := range output {
if item.SimKey == nil {
continue
}
card := cardMap[item.SimKey.ICCID]
items = append(items, model.TrafficUsageMonthlyItem{
TrafficUsageBase: model.TrafficUsageBase{
Iccid: item.SimKey.ICCID,
UsageMb: item.ResData.DataAmount,
PackageCode: card.PackageCode,
},
Month: month,
})
}
return items
}
func (s *sTrafficUsageMonthlySync) publishRetry(ctx context.Context, simKey *model.SimKey, card model.TrafficUsageSimCard, queryDate string, usage ...float64) {
var snapshot *float64
if len(usage) > 0 {
snapshot = &usage[0]
}
if !rabbitmq.Enabled(ctx) {
g.Log().Errorf(ctx, "traffic usage monthly retry skipped, rabbitmq disabled, simKey:%+v, queryDate:%s", simKey, queryDate)
return
}
if err := rabbitmq.PublishConfirmed(ctx, consts.QueueTrafficUsageMonthlyRetry, model.TrafficUsageMonthlyRetryMessage{
UsageMb: snapshot,
SimKey: simKey,
Card: card,
QueryDate: queryDate,
}); err != nil {
g.Log().Errorf(ctx, "publish traffic usage monthly retry failed, simKey:%+v, queryDate:%s, err:%+v", simKey, queryDate, err)
}
}
func trafficUsageMonthlySimKeys(list []entity.TSimCardInfo) []*model.SimKey {
simKeys := make([]*model.SimKey, 0, len(list))
for _, item := range list {
simKeys = append(simKeys, &model.SimKey{
ChanKey: item.ChanKey,
ICCID: item.Iccid,
IMSI: item.Imsi,
MSISDN: item.Msisdn,
})
}
return simKeys
}
func trafficUsageMonthlyCardMap(list []entity.TSimCardInfo) map[string]model.TrafficUsageSimCard {
cardMap := make(map[string]model.TrafficUsageSimCard, len(list))
for _, item := range list {
cardMap[item.Iccid] = model.TrafficUsageSimCard{
Iccid: item.Iccid,
PackageCode: item.CurrentPackageCode,
}
}
return cardMap
}
@@ -0,0 +1,156 @@
package trafficusage
import (
"context"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
"time"
"github.com/gogf/gf/v2/frame/g"
)
type sTrafficPoolUsageSync struct {
}
type trafficPoolCardRow struct {
Iccid string `gconv:"iccid"`
FlowPoolCode string `gconv:"flow_pool_code"`
}
func NewTrafficPoolUsageSync() *sTrafficPoolUsageSync {
return &sTrafficPoolUsageSync{}
}
func init() {
service.RegisterTrafficPoolUsageSync(NewTrafficPoolUsageSync())
}
// TODO: 流量池查询逻辑还未确定
// SyncDaily 同步流量池的流量使用记录
func (s *sTrafficPoolUsageSync) SyncDaily(ctx context.Context) {
yesterday := time.Now().AddDate(0, 0, -1)
date := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, time.Local)
cards, err := queryTrafficPoolCards(ctx)
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
return
}
if len(cards) == 0 {
g.Log().Debug(ctx, "SyncTrafficPoolUsageDaily skipped, no flow pool cards")
return
}
aggregates, err := service.TrafficUsage().AggregatePoolUsageDaily(ctx, date, cards)
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
return
}
if items := buildTrafficPoolUsageItems(aggregates, date); len(items) > 0 {
if err = service.TrafficUsage().WritePoolUsageDaily(ctx, items); err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
return
}
}
if countItems := buildTrafficPoolCardCountItems(cards, date); len(countItems) > 0 {
if err = service.TrafficUsage().WritePoolCardCountDaily(ctx, countItems); err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
return
}
}
}
// SyncMonthly 同步流量池的流量使用记录
func (s *sTrafficPoolUsageSync) SyncMonthly(ctx context.Context) {
lastMonth := time.Now().AddDate(0, -1, 0)
month := time.Date(lastMonth.Year(), lastMonth.Month(), 1, 0, 0, 0, 0, time.Local)
cards, err := queryTrafficPoolCards(ctx)
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
return
}
if len(cards) == 0 {
g.Log().Debug(ctx, "SyncTrafficPoolUsageMonthly skipped, no flow pool cards")
return
}
aggregates, err := service.TrafficUsage().AggregatePoolUsageMonthly(ctx, month, cards)
if err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
return
}
if items := buildTrafficPoolUsageItems(aggregates, month); len(items) > 0 {
if err = service.TrafficUsage().WritePoolUsageMonthly(ctx, items); err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
return
}
}
if countItems := buildTrafficPoolCardCountItems(cards, month); len(countItems) > 0 {
if err = service.TrafficUsage().WritePoolCardCountMonthly(ctx, countItems); err != nil {
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
return
}
}
}
func queryTrafficPoolCards(ctx context.Context) ([]model.TrafficPoolCard, error) {
rows := make([]trafficPoolCardRow, 0)
cardTable := dao.TSimCardInfo.Table()
columns := dao.TSimCardInfo.Columns()
if err := trafficUsageEligibleCardModel(ctx).
Fields(cardTable+"."+columns.Iccid+" AS iccid", cardTable+"."+columns.FlowPoolCode+" AS flow_pool_code").
Where(cardTable+"."+columns.IsFlowPool, 1).
Where(cardTable+"."+columns.FlowPoolCode+" <> ?", "").
Scan(&rows); err != nil {
return nil, err
}
cards := make([]model.TrafficPoolCard, 0, len(rows))
for _, row := range rows {
if row.Iccid == "" || row.FlowPoolCode == "" {
continue
}
cards = append(cards, model.TrafficPoolCard{
Iccid: row.Iccid,
FlowPoolCode: row.FlowPoolCode,
})
}
return cards, nil
}
func buildTrafficPoolUsageItems(aggregates []model.TrafficPoolUsageAggregate, date time.Time) []model.TrafficPoolUsageItem {
items := make([]model.TrafficPoolUsageItem, 0, len(aggregates))
for _, aggregate := range aggregates {
if aggregate.FlowPoolCode == "" {
continue
}
items = append(items, model.TrafficPoolUsageItem{
FlowPoolCode: aggregate.FlowPoolCode,
UsageMb: aggregate.UsageMb,
Date: date,
})
}
return items
}
func buildTrafficPoolCardCountItems(cards []model.TrafficPoolCard, date time.Time) []model.TrafficPoolCardCountItem {
countMap := make(map[string]int64)
for _, card := range cards {
if card.FlowPoolCode == "" {
continue
}
countMap[card.FlowPoolCode]++
}
items := make([]model.TrafficPoolCardCountItem, 0, len(countMap))
for flowPoolCode, cardCount := range countMap {
items = append(items, model.TrafficPoolCardCountItem{
FlowPoolCode: flowPoolCode,
CardCount: cardCount,
Date: date,
})
}
return items
}