chore: 初始开源发布
iotlcm-admin-svr:管理后台服务。 - Go module:iotlcm → git.5iot.com/iotlcm-public/iotlcm-admin-svr(729 文件) - 凭据与地址脱敏:JWT signingKey(含代码里的 MustGet 默认值)、 MySQL DSN、Redis 口令、告警邮件发件人邮箱全部改为占位; 私网 IP → 文档地址段;库名 xiujie_admin → iotlcm - 真实手机号/邮箱 → 合法 11 位编造号 / example.com - 移除内部资产:CI 触发器、过时 DDL(sql/db_iotlcm.sql)、 私有依赖 go get 行、明文 registry 登录口令 - 启动 banner 统一 IOTLCM 方块字 + 社区惯例署名 - Dockerfile 改用公共基础镜像 - 拼错的 RADEME.md → README.md,README 重写 - 需要真实数据库的用例改为可用 IOTLCM_TEST_WITH_DB=1 显式启用 - 补 Apache-2.0 LICENSE,LF 行尾 + .gitattributes 数据库 schema 不随本仓发布(生产 DDL 已过时),README 已说明。
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
)
|
||||
|
||||
var historyUsageLocation = time.FixedZone("UTC+8", 8*60*60)
|
||||
|
||||
func previousUsageMonth(now time.Time) time.Time {
|
||||
now = now.In(historyUsageLocation)
|
||||
return time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, historyUsageLocation).AddDate(0, -1, 0)
|
||||
}
|
||||
|
||||
func reconcileUsage[T any](keys []*model.SimKey, output []model.SimBaseRes[T], amount func(T) float64) ([]model.SimBaseRes[T], []*model.SimKey) {
|
||||
return reconcileUsageWithPeriod(keys, output, amount, func(T) string { return "" }, func(string) (string, bool) { return "", true }, "")
|
||||
}
|
||||
|
||||
func reconcileUsageWithPeriod[T any](keys []*model.SimKey, output []model.SimBaseRes[T], amount func(T) float64, period func(T) string, normalize func(string) (string, bool), expected string) ([]model.SimBaseRes[T], []*model.SimKey) {
|
||||
results := make(map[string][]model.SimBaseRes[T])
|
||||
for _, item := range output {
|
||||
if item.SimKey != nil {
|
||||
results[item.SimKey.ICCID] = append(results[item.SimKey.ICCID], item)
|
||||
}
|
||||
}
|
||||
success := make([]model.SimBaseRes[T], 0, len(keys))
|
||||
failed := make([]*model.SimKey, 0)
|
||||
for _, key := range keys {
|
||||
items := results[key.ICCID]
|
||||
if len(items) != 1 || items[0].Code != 0 {
|
||||
failed = append(failed, key)
|
||||
continue
|
||||
}
|
||||
usage := amount(items[0].ResData)
|
||||
if math.IsNaN(usage) || math.IsInf(usage, 0) || usage < 0 {
|
||||
failed = append(failed, key)
|
||||
continue
|
||||
}
|
||||
if expected != "" {
|
||||
periodValue, ok := normalize(period(items[0].ResData))
|
||||
if !ok || periodValue != expected {
|
||||
failed = append(failed, key)
|
||||
continue
|
||||
}
|
||||
}
|
||||
items[0].SimKey = key
|
||||
success = append(success, items[0])
|
||||
}
|
||||
return success, failed
|
||||
}
|
||||
|
||||
func normalizeUsageDay(value string) (string, bool) {
|
||||
for _, layout := range []string{"20060102", "2006-01-02"} {
|
||||
parsed, err := time.ParseInLocation(layout, value, historyUsageLocation)
|
||||
if err == nil {
|
||||
return parsed.Format("20060102"), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func normalizeUsageMonth(value string) (string, bool) {
|
||||
for _, layout := range []string{"200601", "2006-01", "20060102", "2006-01-02"} {
|
||||
parsed, err := time.ParseInLocation(layout, value, historyUsageLocation)
|
||||
if err == nil {
|
||||
return parsed.Format("200601"), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPreviousUsageMonth(t *testing.T) {
|
||||
for input, want := range map[string]string{"2026-03-31T10:00:00+08:00": "202602", "2024-03-31T10:00:00+08:00": "202402", "2026-01-31T00:00:00+08:00": "202512", "2026-08-31T16:00:00Z": "202608"} {
|
||||
date, err := time.Parse(time.RFC3339, input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := previousUsageMonth(date).Format("200601"); got != want {
|
||||
t.Fatalf("%s: got %s want %s", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileUsage(t *testing.T) {
|
||||
keys := []*model.SimKey{{ICCID: "ok"}, {ICCID: "missing"}, {ICCID: "duplicate"}, {ICCID: "failed"}}
|
||||
output := []model.SimBaseRes[model.SimDataUsageDailyOutputItem]{
|
||||
{SimKey: keys[0]}, {SimKey: keys[2]}, {SimKey: keys[2]}, {SimKey: keys[3], Code: 1}, {}, {SimKey: &model.SimKey{ICCID: "unknown"}},
|
||||
}
|
||||
success, failed := reconcileUsage(keys, output, func(item model.SimDataUsageDailyOutputItem) float64 { return item.DataAmount })
|
||||
if len(success) != 1 || success[0].SimKey != keys[0] || len(failed) != 3 {
|
||||
t.Fatalf("success=%+v failed=%+v", success, failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileUsageWithPeriod(t *testing.T) {
|
||||
key := &model.SimKey{ICCID: "period"}
|
||||
output := []model.SimBaseRes[model.SimDataUsageDailyOutputItem]{
|
||||
{SimKey: key, ResData: model.SimDataUsageDailyOutputItem{QueryDate: "2026-07-14", DataAmount: 1}},
|
||||
}
|
||||
success, failed := reconcileUsageWithPeriod([]*model.SimKey{key}, output, func(item model.SimDataUsageDailyOutputItem) float64 { return item.DataAmount }, func(item model.SimDataUsageDailyOutputItem) string { return item.QueryDate }, normalizeUsageDay, "20260713")
|
||||
if len(success) != 0 || len(failed) != 1 {
|
||||
t.Fatalf("expected period mismatch to fail, success=%+v failed=%+v", success, failed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,748 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/cmd/inithttp"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/tdengine"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
|
||||
"hash/fnv"
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
// identifierPattern 限制 database/stable 只能是普通标识符,防止拼 SQL 时传入非法名称。
|
||||
var identifierPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type sTrafficUsage struct {
|
||||
}
|
||||
|
||||
func NewTrafficUsage() *sTrafficUsage {
|
||||
return &sTrafficUsage{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
service.RegisterTrafficUsage(NewTrafficUsage())
|
||||
// 只注册连接释放逻辑,不在应用启动时自动创建 TDengine 库表。
|
||||
inithttp.RegisterHttpCleanupFunc("trafficUsage", func(ctx context.Context) error {
|
||||
return tdengine.Close()
|
||||
})
|
||||
}
|
||||
|
||||
// WriteDaily 批量写入日流量数据。
|
||||
// 每个 ICCID 会落到独立子表,子表通过 t_sim_card_usage_daily 超级表自动创建。
|
||||
func (s *sTrafficUsage) WriteDaily(ctx context.Context, items []model.TrafficUsageDailyItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !tdengine.Enabled(ctx) {
|
||||
g.Log().Debug(ctx, "sTrafficUsage.WriteDaily skipped, tdengine disabled")
|
||||
return gerror.New("TDengine disabled; daily usage was not persisted")
|
||||
}
|
||||
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(cfg.DailyStable)
|
||||
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
batchSize := normalizeBatchSize(cfg.BatchSize)
|
||||
for start := 0; start < len(items); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
|
||||
var builder strings.Builder
|
||||
builder.WriteString("INSERT INTO ")
|
||||
for i, item := range batch {
|
||||
if item.Iccid == "" {
|
||||
return gerror.New("traffic usage daily item iccid is empty")
|
||||
}
|
||||
if err := validateUsageMb("traffic usage daily item usage mb", item.UsageMb); err != nil {
|
||||
return err
|
||||
}
|
||||
if i > 0 {
|
||||
builder.WriteByte(' ')
|
||||
}
|
||||
builder.WriteString(qualify(dbName, childTrafficTableName(cfg.DailyStable, item.Iccid)))
|
||||
builder.WriteString(" USING ")
|
||||
builder.WriteString(qualify(dbName, cfg.DailyStable))
|
||||
builder.WriteString(" TAGS (")
|
||||
builder.WriteString(sqlString(item.Iccid))
|
||||
builder.WriteString(") VALUES (")
|
||||
builder.WriteString(strings.Join([]string{
|
||||
sqlTime(dayStart(item.Date)),
|
||||
sqlFloat(item.UsageMb),
|
||||
sqlString(item.PackageCode),
|
||||
}, ", "))
|
||||
builder.WriteByte(')')
|
||||
}
|
||||
|
||||
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
|
||||
return gerror.Wrap(err, "tdengine write daily traffic usage failed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteMonthly 批量写入月流量数据。
|
||||
// 月流量和日流量共用同一个 TDengine database,但使用不同超级表。
|
||||
func (s *sTrafficUsage) WriteMonthly(ctx context.Context, items []model.TrafficUsageMonthlyItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !tdengine.Enabled(ctx) {
|
||||
g.Log().Debug(ctx, "sTrafficUsage.WriteMonthly skipped, tdengine disabled")
|
||||
return gerror.New("TDengine disabled; monthly usage was not persisted")
|
||||
}
|
||||
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(cfg.MonthlyStable)
|
||||
if err := validateTableConfig(dbName, cfg.MonthlyStable); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
batchSize := normalizeBatchSize(cfg.BatchSize)
|
||||
for start := 0; start < len(items); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
|
||||
var builder strings.Builder
|
||||
builder.WriteString("INSERT INTO ")
|
||||
for i, item := range batch {
|
||||
if item.Iccid == "" {
|
||||
return gerror.New("traffic usage monthly item iccid is empty")
|
||||
}
|
||||
if err := validateUsageMb("traffic usage monthly item usage mb", item.UsageMb); err != nil {
|
||||
return err
|
||||
}
|
||||
if i > 0 {
|
||||
builder.WriteByte(' ')
|
||||
}
|
||||
builder.WriteString(qualify(dbName, childTrafficTableName(cfg.MonthlyStable, item.Iccid)))
|
||||
builder.WriteString(" USING ")
|
||||
builder.WriteString(qualify(dbName, cfg.MonthlyStable))
|
||||
builder.WriteString(" TAGS (")
|
||||
builder.WriteString(sqlString(item.Iccid))
|
||||
builder.WriteString(") VALUES (")
|
||||
builder.WriteString(strings.Join([]string{
|
||||
sqlTime(monthStart(item.Month)),
|
||||
sqlFloat(item.UsageMb),
|
||||
sqlString(item.PackageCode),
|
||||
}, ", "))
|
||||
builder.WriteByte(')')
|
||||
}
|
||||
|
||||
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
|
||||
return gerror.Wrap(err, "tdengine write monthly traffic usage failed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WritePoolUsageDaily 批量写入流量池的流量使用记录。
|
||||
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_usage_daily 超级表自动创建。
|
||||
func (s *sTrafficUsage) WritePoolUsageDaily(ctx context.Context, items []model.TrafficPoolUsageItem) error {
|
||||
return s.writePoolUsage(ctx, items, tdengine.GetConfig(ctx).PoolDailyStable, "flow_pool_usage_d_", dayStart, "daily")
|
||||
}
|
||||
|
||||
// WritePoolUsageMonthly 批量写入流量池的流量使用记录。
|
||||
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_usage_monthly 超级表自动创建。
|
||||
func (s *sTrafficUsage) WritePoolUsageMonthly(ctx context.Context, items []model.TrafficPoolUsageItem) error {
|
||||
return s.writePoolUsage(ctx, items, tdengine.GetConfig(ctx).PoolMonthlyStable, "flow_pool_usage_m_", monthStart, "monthly")
|
||||
}
|
||||
|
||||
// WritePoolCardCountDaily 批量写入流量池的卡数量记录。
|
||||
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_card_count_daily 超级表自动创建。
|
||||
func (s *sTrafficUsage) WritePoolCardCountDaily(ctx context.Context, items []model.TrafficPoolCardCountItem) error {
|
||||
return s.writePoolCardCount(ctx, items, tdengine.GetConfig(ctx).PoolCardCountDailyStable, "flow_pool_card_count_d_", dayStart, "daily")
|
||||
}
|
||||
|
||||
// WritePoolCardCountMonthly 批量写入流量池的卡数量记录。
|
||||
// 每个 FlowPoolCode 会落到独立子表,子表通过 t_flow_pool_card_count_monthly 超级表自动创建。
|
||||
func (s *sTrafficUsage) WritePoolCardCountMonthly(ctx context.Context, items []model.TrafficPoolCardCountItem) error {
|
||||
return s.writePoolCardCount(ctx, items, tdengine.GetConfig(ctx).PoolCardCountMonthlyStable, "flow_pool_card_count_m_", monthStart, "monthly")
|
||||
}
|
||||
|
||||
// UpdateDaily 更新日流量数据。
|
||||
func (s *sTrafficUsage) UpdateDaily(ctx context.Context, items []model.TrafficUsageDailyItem) error {
|
||||
return s.WriteDaily(ctx, items)
|
||||
}
|
||||
|
||||
// UpdateMonthly 更新月流量数据。
|
||||
func (s *sTrafficUsage) UpdateMonthly(ctx context.Context, items []model.TrafficUsageMonthlyItem) error {
|
||||
return s.WriteMonthly(ctx, items)
|
||||
}
|
||||
|
||||
// DeleteDaily 删除日流量数据。
|
||||
func (s *sTrafficUsage) DeleteDaily(ctx context.Context, input model.TrafficUsageDeleteInput) error {
|
||||
if !tdengine.Enabled(ctx) {
|
||||
g.Log().Debug(ctx, "sTrafficUsage.DeleteDaily skipped, tdengine disabled")
|
||||
return nil
|
||||
}
|
||||
if input.Iccid == "" {
|
||||
return gerror.New("traffic usage daily delete iccid is empty")
|
||||
}
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(cfg.DailyStable)
|
||||
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteRange(ctx, qualify(dbName, childTrafficTableName(cfg.DailyStable, input.Iccid)), input.StartTime, input.EndTime)
|
||||
}
|
||||
|
||||
// DeleteMonthly 删除月流量数据。
|
||||
func (s *sTrafficUsage) DeleteMonthly(ctx context.Context, input model.TrafficUsageDeleteInput) error {
|
||||
if !tdengine.Enabled(ctx) {
|
||||
g.Log().Debug(ctx, "sTrafficUsage.DeleteMonthly skipped, tdengine disabled")
|
||||
return nil
|
||||
}
|
||||
if input.Iccid == "" {
|
||||
return gerror.New("traffic usage monthly delete iccid is empty")
|
||||
}
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(cfg.MonthlyStable)
|
||||
if err := validateTableConfig(dbName, cfg.MonthlyStable); err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteRange(ctx, qualify(dbName, childTrafficTableName(cfg.MonthlyStable, input.Iccid)), input.StartTime, input.EndTime)
|
||||
}
|
||||
|
||||
// QueryDaily 查询日流量数据。
|
||||
func (s *sTrafficUsage) QueryDaily(ctx context.Context, input model.TrafficUsageQueryInput) ([]model.TrafficUsageDailyRecord, int, error) {
|
||||
if !tdengine.Enabled(ctx) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if input.Iccid == "" {
|
||||
return nil, 0, gerror.New("traffic usage daily query iccid is empty")
|
||||
}
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(cfg.DailyStable)
|
||||
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
table := qualify(dbName, childTrafficTableName(cfg.DailyStable, input.Iccid))
|
||||
total, err := countRows(ctx, table, input.StartTime, input.EndTime)
|
||||
if err != nil {
|
||||
if isTableNotExist(err) {
|
||||
return []model.TrafficUsageDailyRecord{}, 0, nil
|
||||
}
|
||||
return nil, 0, err
|
||||
}
|
||||
rows, err := queryRows(ctx, table, input)
|
||||
if err != nil {
|
||||
if isTableNotExist(err) {
|
||||
return []model.TrafficUsageDailyRecord{}, 0, nil
|
||||
}
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
list := make([]model.TrafficUsageDailyRecord, 0)
|
||||
for rows.Next() {
|
||||
var item model.TrafficUsageDailyRecord
|
||||
if err = rows.Scan(&item.Ts, &item.UsageMb, &item.PackageCode, &item.Iccid); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
list = append(list, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return list, total, nil
|
||||
}
|
||||
|
||||
// QueryMonthly 查询月流量数据。
|
||||
func (s *sTrafficUsage) QueryMonthly(ctx context.Context, input model.TrafficUsageQueryInput) ([]model.TrafficUsageMonthlyRecord, int, error) {
|
||||
if !tdengine.Enabled(ctx) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if input.Iccid == "" {
|
||||
return nil, 0, gerror.New("traffic usage monthly query iccid is empty")
|
||||
}
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(cfg.MonthlyStable)
|
||||
if err := validateTableConfig(dbName, cfg.MonthlyStable); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
table := qualify(dbName, childTrafficTableName(cfg.MonthlyStable, input.Iccid))
|
||||
total, err := countRows(ctx, table, input.StartTime, input.EndTime)
|
||||
if err != nil {
|
||||
if isTableNotExist(err) {
|
||||
return []model.TrafficUsageMonthlyRecord{}, 0, nil
|
||||
}
|
||||
return nil, 0, err
|
||||
}
|
||||
rows, err := queryRows(ctx, table, input)
|
||||
if err != nil {
|
||||
if isTableNotExist(err) {
|
||||
return []model.TrafficUsageMonthlyRecord{}, 0, nil
|
||||
}
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
list := make([]model.TrafficUsageMonthlyRecord, 0)
|
||||
for rows.Next() {
|
||||
var item model.TrafficUsageMonthlyRecord
|
||||
if err = rows.Scan(&item.Ts, &item.UsageMb, &item.PackageCode, &item.Iccid); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
list = append(list, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return list, total, nil
|
||||
}
|
||||
|
||||
// AggregatePoolUsageDaily 汇总日流量数据。
|
||||
func (s *sTrafficUsage) AggregatePoolUsageDaily(ctx context.Context, date time.Time, cards []model.TrafficPoolCard) ([]model.TrafficPoolUsageAggregate, error) {
|
||||
return s.aggregatePoolUsage(ctx, dayStart(date), cards, tdengine.GetConfig(ctx).DailyStable, "daily")
|
||||
}
|
||||
|
||||
// AggregatePoolUsageMonthly 汇总月流量数据。
|
||||
func (s *sTrafficUsage) AggregatePoolUsageMonthly(ctx context.Context, month time.Time, cards []model.TrafficPoolCard) ([]model.TrafficPoolUsageAggregate, error) {
|
||||
return s.aggregatePoolUsage(ctx, monthStart(month), cards, tdengine.GetConfig(ctx).MonthlyStable, "monthly")
|
||||
}
|
||||
|
||||
// aggregatePoolUsage 汇总流量数据。
|
||||
func (s *sTrafficUsage) aggregatePoolUsage(ctx context.Context, ts time.Time, cards []model.TrafficPoolCard, stable string, name string) ([]model.TrafficPoolUsageAggregate, error) {
|
||||
if len(cards) == 0 {
|
||||
return []model.TrafficPoolUsageAggregate{}, nil
|
||||
}
|
||||
if !tdengine.Enabled(ctx) {
|
||||
g.Log().Debugf(ctx, "sTrafficUsage.aggregatePoolUsage %s skipped, tdengine disabled", name)
|
||||
return []model.TrafficPoolUsageAggregate{}, nil
|
||||
}
|
||||
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(stable)
|
||||
if err := validateTableConfig(dbName, stable); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
grouped := make(map[string][]string)
|
||||
for _, card := range cards {
|
||||
if card.FlowPoolCode == "" || card.Iccid == "" {
|
||||
continue
|
||||
}
|
||||
grouped[card.FlowPoolCode] = append(grouped[card.FlowPoolCode], card.Iccid)
|
||||
}
|
||||
|
||||
result := make([]model.TrafficPoolUsageAggregate, 0, len(grouped))
|
||||
for flowPoolCode, iccids := range grouped {
|
||||
usageMb, recordCount, err := aggregatePoolUsageByIccids(ctx, dbName, stable, ts, iccids)
|
||||
if err != nil {
|
||||
return nil, gerror.Wrapf(err, "tdengine aggregate %s traffic pool usage failed, flowPoolCode:%s", name, flowPoolCode)
|
||||
}
|
||||
if recordCount != int64(len(iccids)) {
|
||||
return nil, gerror.Newf(
|
||||
"tdengine aggregate %s traffic pool usage incomplete, flowPoolCode:%s, expected:%d, actual:%d",
|
||||
name,
|
||||
flowPoolCode,
|
||||
len(iccids),
|
||||
recordCount,
|
||||
)
|
||||
}
|
||||
result = append(result, model.TrafficPoolUsageAggregate{
|
||||
FlowPoolCode: flowPoolCode,
|
||||
UsageMb: usageMb,
|
||||
RecordCount: recordCount,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// writePoolUsage 批量写入流量池使用量数据。
|
||||
// 每个 FlowPoolCode 会落到独立子表,子表通过对应的超级表自动创建。
|
||||
func (s *sTrafficUsage) writePoolUsage(ctx context.Context, items []model.TrafficPoolUsageItem, stable string, prefix string, normalizeTime func(time.Time) time.Time, name string) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !tdengine.Enabled(ctx) {
|
||||
g.Log().Debugf(ctx, "sTrafficUsage.writePoolUsage %s skipped, tdengine disabled", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(stable)
|
||||
if err := validateTableConfig(dbName, stable); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
batchSize := normalizeBatchSize(cfg.BatchSize)
|
||||
for start := 0; start < len(items); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
|
||||
var builder strings.Builder
|
||||
builder.WriteString("INSERT INTO ")
|
||||
for i, item := range batch {
|
||||
if item.FlowPoolCode == "" {
|
||||
return gerror.New("traffic pool usage item flow pool code is empty")
|
||||
}
|
||||
if err := validateUsageMb("traffic pool usage item usage mb", item.UsageMb); err != nil {
|
||||
return err
|
||||
}
|
||||
if i > 0 {
|
||||
builder.WriteByte(' ')
|
||||
}
|
||||
builder.WriteString(qualify(dbName, childTableNameWithPrefix(prefix, item.FlowPoolCode)))
|
||||
builder.WriteString(" USING ")
|
||||
builder.WriteString(qualify(dbName, stable))
|
||||
builder.WriteString(" TAGS (")
|
||||
builder.WriteString(sqlString(item.FlowPoolCode))
|
||||
builder.WriteString(") VALUES (")
|
||||
builder.WriteString(strings.Join([]string{
|
||||
sqlTime(normalizeTime(item.Date)),
|
||||
sqlFloat(item.UsageMb),
|
||||
}, ", "))
|
||||
builder.WriteByte(')')
|
||||
}
|
||||
|
||||
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
|
||||
return gerror.Wrapf(err, "tdengine write %s traffic pool usage failed", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writePoolCardCount 批量写入流量池卡数量数据。
|
||||
// 每个 FlowPoolCode 会落到独立子表,子表通过对应的超级表自动创建。
|
||||
func (s *sTrafficUsage) writePoolCardCount(ctx context.Context, items []model.TrafficPoolCardCountItem, stable string, prefix string, normalizeTime func(time.Time) time.Time, name string) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !tdengine.Enabled(ctx) {
|
||||
g.Log().Debugf(ctx, "sTrafficUsage.writePoolCardCount %s skipped, tdengine disabled", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(stable)
|
||||
if err := validateTableConfig(dbName, stable); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
batchSize := normalizeBatchSize(cfg.BatchSize)
|
||||
for start := 0; start < len(items); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
|
||||
var builder strings.Builder
|
||||
builder.WriteString("INSERT INTO ")
|
||||
for i, item := range batch {
|
||||
if item.FlowPoolCode == "" {
|
||||
return gerror.New("traffic pool card count item flow pool code is empty")
|
||||
}
|
||||
if item.CardCount < 0 {
|
||||
return gerror.Newf("traffic pool card count item card count cannot be negative: %d", item.CardCount)
|
||||
}
|
||||
if i > 0 {
|
||||
builder.WriteByte(' ')
|
||||
}
|
||||
builder.WriteString(qualify(dbName, childTableNameWithPrefix(prefix, item.FlowPoolCode)))
|
||||
builder.WriteString(" USING ")
|
||||
builder.WriteString(qualify(dbName, stable))
|
||||
builder.WriteString(" TAGS (")
|
||||
builder.WriteString(sqlString(item.FlowPoolCode))
|
||||
builder.WriteString(") VALUES (")
|
||||
builder.WriteString(strings.Join([]string{
|
||||
sqlTime(normalizeTime(item.Date)),
|
||||
strconv.FormatInt(item.CardCount, 10),
|
||||
}, ", "))
|
||||
builder.WriteByte(')')
|
||||
}
|
||||
|
||||
if _, err := tdengine.Exec(ctx, builder.String()); err != nil {
|
||||
return gerror.Wrapf(err, "tdengine write %s traffic pool card count failed", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// aggregatePoolUsageByIccids 汇总流量数据。
|
||||
// 每个 ICCID 会落到独立子表,子表通过 t_sim_card_usage_daily 或 t_sim_card_usage_monthly 超级表自动创建。
|
||||
func aggregatePoolUsageByIccids(ctx context.Context, database string, stable string, ts time.Time, iccids []string) (float64, int64, error) {
|
||||
var usageMb float64
|
||||
var recordCount int64
|
||||
for _, iccid := range iccids {
|
||||
table := qualify(database, childTrafficTableName(stable, iccid))
|
||||
row, err := tdengine.QueryRow(ctx, fmt.Sprintf("SELECT usage_mb FROM %s WHERE ts = %s", table, sqlTime(ts)))
|
||||
if err != nil {
|
||||
return 0, 0, gerror.Wrap(err, "tdengine query traffic usage row failed")
|
||||
}
|
||||
var value float64
|
||||
if err = row.Scan(&value); err != nil {
|
||||
if err == sql.ErrNoRows || isTableNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return 0, 0, gerror.Wrap(err, "tdengine scan traffic usage row failed")
|
||||
}
|
||||
usageMb += value
|
||||
recordCount++
|
||||
}
|
||||
return usageMb, recordCount, nil
|
||||
}
|
||||
|
||||
// countRows 统计流量数据记录数。
|
||||
func countRows(ctx context.Context, table string, startTime, endTime time.Time) (int, error) {
|
||||
where := whereRangeSQL(startTime, endTime)
|
||||
row, err := tdengine.QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s%s", table, where))
|
||||
if err != nil {
|
||||
return 0, gerror.Wrap(err, "tdengine count traffic usage failed")
|
||||
}
|
||||
total := 0
|
||||
if err = row.Scan(&total); err != nil {
|
||||
return 0, gerror.Wrap(err, "tdengine count traffic usage failed")
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// queryRows 查询流量数据记录。
|
||||
func queryRows(ctx context.Context, table string, input model.TrafficUsageQueryInput) (*sql.Rows, error) {
|
||||
fields := "ts, usage_mb, package_code, iccid"
|
||||
where := whereRangeSQL(input.StartTime, input.EndTime)
|
||||
order := "DESC"
|
||||
if input.Asc {
|
||||
order = "ASC"
|
||||
}
|
||||
limit := ""
|
||||
page, pageSize := normalizePage(input.Page, input.PageSize)
|
||||
if pageSize > 0 {
|
||||
limit = fmt.Sprintf(" LIMIT %d OFFSET %d", pageSize, (page-1)*pageSize)
|
||||
}
|
||||
rows, err := tdengine.Query(ctx, fmt.Sprintf("SELECT %s FROM %s%s ORDER BY ts %s%s", fields, table, where, order, limit))
|
||||
if err != nil {
|
||||
return nil, gerror.Wrap(err, "tdengine query traffic usage failed")
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// deleteRange 删除流量数据记录。
|
||||
func deleteRange(ctx context.Context, table string, startTime, endTime time.Time) error {
|
||||
where := whereRangeSQL(startTime, endTime)
|
||||
if where == "" {
|
||||
return gerror.New("traffic usage delete time range is empty")
|
||||
}
|
||||
if _, err := tdengine.Exec(ctx, fmt.Sprintf("DELETE FROM %s%s", table, where)); err != nil {
|
||||
if isTableNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return gerror.Wrap(err, "tdengine delete traffic usage failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// whereRangeSQL 构建时间范围查询条件。
|
||||
func whereRangeSQL(startTime, endTime time.Time) string {
|
||||
clauses := make([]string, 0, 2)
|
||||
if !startTime.IsZero() {
|
||||
clauses = append(clauses, "ts >= "+sqlTime(startTime))
|
||||
}
|
||||
if !endTime.IsZero() {
|
||||
clauses = append(clauses, "ts <= "+sqlTime(endTime))
|
||||
}
|
||||
if len(clauses) == 0 {
|
||||
return ""
|
||||
}
|
||||
return " WHERE " + strings.Join(clauses, " AND ")
|
||||
}
|
||||
|
||||
// validateTableConfig 验证表配置是否有效。
|
||||
func validateTableConfig(database, stable string) error {
|
||||
if !identifierPattern.MatchString(database) {
|
||||
return gerror.Newf("invalid tdengine database name: %s", database)
|
||||
}
|
||||
if !identifierPattern.MatchString(stable) {
|
||||
return gerror.Newf("invalid tdengine stable name: %s", stable)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// qualify 格式化表名。
|
||||
func qualify(database, table string) string {
|
||||
return fmt.Sprintf("%s.%s", database, table)
|
||||
}
|
||||
|
||||
// childTrafficTableName 构建子表名。
|
||||
// 每个 ICCID 会落到独立子表,子表通过 t_sim_card_usage_daily 或 t_sim_card_usage_monthly 超级表自动创建。
|
||||
func childTrafficTableName(stable string, iccid string) string {
|
||||
prefix := "sim_card_usage_m_"
|
||||
if stable == "t_sim_card_usage_daily" {
|
||||
prefix = "sim_card_usage_d_"
|
||||
}
|
||||
return childTableNameWithPrefix(prefix, iccid)
|
||||
}
|
||||
|
||||
// childTableNameWithPrefix 构建子表名。
|
||||
func childTableNameWithPrefix(prefix string, value string) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(prefix)
|
||||
for _, r := range strings.ToLower(value) {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
|
||||
builder.WriteRune(r)
|
||||
}
|
||||
}
|
||||
name := builder.String()
|
||||
if name == prefix || len(name) > 190 {
|
||||
return fmt.Sprintf("%s%x", prefix, hashString(value))
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// hashString 对字符串进行哈希处理。
|
||||
// 用于生成子表名。
|
||||
func hashString(value string) uint32 {
|
||||
hash := fnv.New32a()
|
||||
_, _ = hash.Write([]byte(value))
|
||||
return hash.Sum32()
|
||||
}
|
||||
|
||||
// normalizeBatchSize 归一化批量大小。
|
||||
func normalizeBatchSize(size int) int {
|
||||
if size <= 0 {
|
||||
return 200
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// normalizePage 归一化分页参数。
|
||||
func normalizePage(page, pageSize int) (int, int) {
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 0 {
|
||||
pageSize = 0
|
||||
}
|
||||
return page, pageSize
|
||||
}
|
||||
|
||||
// dayStart 获取指定日期的快照时间。
|
||||
// 用于查询指定日期的流量数据记录。
|
||||
func dayStart(t time.Time) time.Time {
|
||||
if t.IsZero() {
|
||||
t = time.Now()
|
||||
}
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location())
|
||||
}
|
||||
|
||||
// monthStart 获取指定月份的快照时间。
|
||||
// 用于查询指定月份的流量数据记录。
|
||||
func monthStart(t time.Time) time.Time {
|
||||
if t.IsZero() {
|
||||
t = time.Now()
|
||||
}
|
||||
lastDay := time.Date(t.Year(), t.Month()+1, 0, 23, 59, 59, 0, t.Location())
|
||||
return lastDay
|
||||
}
|
||||
|
||||
// sqlString 格式化字符串为SQL字符串。
|
||||
func sqlString(value string) string {
|
||||
return "'" + strings.ReplaceAll(value, "'", "''") + "'"
|
||||
}
|
||||
|
||||
// sqlTime 格式化时间时间为SQL时间。
|
||||
func sqlTime(value time.Time) string {
|
||||
return sqlString(value.Format("2006-01-02 15:04:05.000"))
|
||||
}
|
||||
|
||||
// sqlFloat 格式化浮点数为SQL浮点数。
|
||||
func sqlFloat(value float64) string {
|
||||
return strconv.FormatFloat(value, 'f', -1, 64)
|
||||
}
|
||||
|
||||
// validateUsageMb 验证流量使用记录是否有效。
|
||||
// 用于检查流量使用记录是否为有限的浮点数,且是否为负数。
|
||||
func validateUsageMb(name string, value float64) error {
|
||||
if math.IsNaN(value) || math.IsInf(value, 0) {
|
||||
return gerror.Newf("%s must be finite: %v", name, value)
|
||||
}
|
||||
if value < 0 {
|
||||
return gerror.Newf("%s cannot be negative: %v", name, value)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isTableNotExist 检查表是否存在。
|
||||
// 用于在删除流量数据记录时,检查表是否存在。
|
||||
func isTableNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "table does not exist") ||
|
||||
strings.Contains(text, "table not exist") ||
|
||||
strings.Contains(text, "does not exist")
|
||||
}
|
||||
|
||||
// GetTopUsageCardsYesterday 查询前一天流量使用前五名的卡片
|
||||
func (s *sTrafficUsage) GetTopUsageCardsYesterday(ctx context.Context) ([]model.TopUsageCard, error) {
|
||||
if !tdengine.Enabled(ctx) {
|
||||
return nil, nil
|
||||
}
|
||||
cfg := tdengine.GetConfig(ctx)
|
||||
dbName := cfg.DbNameFor(cfg.DailyStable)
|
||||
if err := validateTableConfig(dbName, cfg.DailyStable); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 获取前一天日期
|
||||
yesterday := time.Now().AddDate(0, 0, -1)
|
||||
startStr := yesterday.Format("2006-01-02")
|
||||
endStr := dayStart(yesterday).Format("2006-01-02 15:04:05")
|
||||
|
||||
table := qualify(dbName, cfg.DailyStable)
|
||||
sql := fmt.Sprintf(
|
||||
"SELECT iccid, FIRST(usage_mb), LAST(usage_mb) FROM %s WHERE ts >= '%s' AND ts <= '%s' GROUP BY iccid ORDER BY LAST(usage_mb) DESC LIMIT 5",
|
||||
table, startStr, endStr)
|
||||
rows, err := tdengine.Query(ctx, sql)
|
||||
if err != nil {
|
||||
if isTableNotExist(err) {
|
||||
return []model.TopUsageCard{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := make([]model.TopUsageCard, 0)
|
||||
for rows.Next() {
|
||||
var iccid string
|
||||
var firstUsage, lastUsage float64
|
||||
if err = rows.Scan(&iccid, &firstUsage, &lastUsage); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, model.TopUsageCard{
|
||||
Iccid: iccid,
|
||||
UsageMb: lastUsage,
|
||||
StatDate: yesterday.Format("2006-01-02"),
|
||||
})
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
)
|
||||
|
||||
func trafficUsageEligibleCardModel(ctx context.Context) *gdb.Model {
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
deptTable := dao.SysDept.Table()
|
||||
cardColumns := dao.TSimCardInfo.Columns()
|
||||
deptColumns := dao.SysDept.Columns()
|
||||
|
||||
return dao.TSimCardInfo.Ctx(ctx).
|
||||
InnerJoin(deptTable, cardTable+"."+cardColumns.DeptId+" = "+deptTable+"."+deptColumns.DeptId+" AND "+cardTable+"."+cardColumns.TenantId+" = "+deptTable+"."+deptColumns.TenantId).
|
||||
Where(cardTable+"."+cardColumns.Iccid+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.ChanKey+" <> ?", "").
|
||||
Where(cardTable+"."+cardColumns.CurrentPackageCode+" <> ?", "").
|
||||
WhereNull(deptTable + "." + deptColumns.DeletedAt)
|
||||
}
|
||||
|
||||
func trafficUsageCardFields() []string {
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
return []string{
|
||||
cardTable + "." + columns.Id + " AS " + columns.Id,
|
||||
cardTable + "." + columns.Iccid + " AS " + columns.Iccid,
|
||||
cardTable + "." + columns.Imsi + " AS " + columns.Imsi,
|
||||
cardTable + "." + columns.Msisdn + " AS " + columns.Msisdn,
|
||||
cardTable + "." + columns.ChanKey + " AS " + columns.ChanKey,
|
||||
cardTable + "." + columns.CurrentPackageCode + " AS " + columns.CurrentPackageCode,
|
||||
cardTable + "." + columns.FlowPoolCode + " AS " + columns.FlowPoolCode,
|
||||
cardTable + "." + columns.IsFlowPool + " AS " + columns.IsFlowPool,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/utility/uuid32"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
const (
|
||||
trafficUsageDailyPageSize = 100
|
||||
)
|
||||
|
||||
type sTrafficUsageDailySync struct {
|
||||
}
|
||||
|
||||
// These indirections keep the trigger deterministic in unit tests without changing
|
||||
// the production publisher, clock, or run-id generator.
|
||||
var (
|
||||
dailyUsageNow = time.Now
|
||||
dailyUsageRunID = uuid32.S
|
||||
dailyUsageRabbitEnabled = rabbitmq.Enabled
|
||||
dailyUsagePublish = rabbitmq.Publish
|
||||
)
|
||||
|
||||
func NewTrafficUsageDailySync() *sTrafficUsageDailySync {
|
||||
return &sTrafficUsageDailySync{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
service.RegisterTrafficUsageDailySync(NewTrafficUsageDailySync())
|
||||
}
|
||||
|
||||
// StartDaily publishes one command for Worker to execute the previous-day usage
|
||||
// full sync. Admin deliberately does not scan cards or call an upstream service;
|
||||
// Worker owns pacing, retries, the failure pool and replay.
|
||||
func (s *sTrafficUsageDailySync) StartDaily(ctx context.Context) error {
|
||||
_, err := s.StartDailyRun(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *sTrafficUsageDailySync) StartDailyRun(ctx context.Context) (string, error) {
|
||||
return s.StartDailyRunScoped(ctx, nil)
|
||||
}
|
||||
|
||||
// StartDailyRunScoped starts a run restricted to scope.Channels. A nil scope, or
|
||||
// one whose channels normalize to empty, starts a full sync.
|
||||
func (s *sTrafficUsageDailySync) StartDailyRunScoped(ctx context.Context, scope *model.SyncScope) (string, error) {
|
||||
return s.StartDailyRunScopedAt(ctx, scope, "")
|
||||
}
|
||||
|
||||
// StartDailyRunScopedAt starts a run for the given day (yyyyMMdd). An empty
|
||||
// queryDate keeps the historical behaviour and covers the previous calendar day
|
||||
// in the business timezone; a non-empty value lets an operator backfill one
|
||||
// specific past day.
|
||||
func (s *sTrafficUsageDailySync) StartDailyRunScopedAt(ctx context.Context, scope *model.SyncScope, queryDate string) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
startedAt := time.Now()
|
||||
normalized, err := model.NormalizeScope(scope)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=validate durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
if !dailyUsageRabbitEnabled(ctx) {
|
||||
err := fmt.Errorf("daily usage start skipped: rabbitmq is disabled")
|
||||
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=precheck durationMs=%d reason=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
|
||||
location, err := time.LoadLocation("Asia/Shanghai")
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=timezone durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", fmt.Errorf("load daily usage timezone: %w", err)
|
||||
}
|
||||
resolvedDate, err := resolveDailyQueryDate(dailyUsageNow().In(location), location, queryDate)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=validate reason=invalid_query_date queryDate=%q durationMs=%d error=%q", queryDate, elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
message := model.TrafficUsageDailyStartMessage{
|
||||
RunID: fmt.Sprintf("daily-usage-%s-%s", resolvedDate, dailyUsageRunID()),
|
||||
Operation: model.OperationPreviousDayUsageFullSync,
|
||||
QueryDate: resolvedDate,
|
||||
TriggeredAt: dailyUsageNow().In(location).Format(time.RFC3339),
|
||||
Scope: normalized,
|
||||
}
|
||||
if err = dailyUsagePublish(ctx, consts.QueueTrafficUsageDailyStart, message); err != nil {
|
||||
g.Log().Errorf(ctx, "event=daily_usage_trigger status=failed stage=publish runId=%s queryDate=%s durationMs=%d error=%q", message.RunID, message.QueryDate, elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", fmt.Errorf("publish daily usage start: %w", err)
|
||||
}
|
||||
g.Log().Infof(ctx, "event=daily_usage_trigger status=published runId=%s queryDate=%s triggeredAt=%s durationMs=%d", message.RunID, message.QueryDate, message.TriggeredAt, elapsedMilliseconds(startedAt))
|
||||
return message.RunID, nil
|
||||
}
|
||||
|
||||
// minBackfillDay is the earliest day a manual backfill may target. It only
|
||||
// catches obvious typos; it is not a business boundary. Comparing the
|
||||
// zero-padded yyyyMMdd strings sorts chronologically.
|
||||
const minBackfillDay = "20200101"
|
||||
|
||||
// resolveDailyQueryDate returns the day a run must cover. An empty requested
|
||||
// value keeps the legacy default (the previous day in the business timezone);
|
||||
// otherwise the value is validated and returned in canonical yyyyMMdd form.
|
||||
func resolveDailyQueryDate(now time.Time, location *time.Location, requested string) (string, error) {
|
||||
// The default covers the previous calendar day, which is the same period the
|
||||
// legacy admin scan used.
|
||||
yesterday := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location).AddDate(0, 0, -1)
|
||||
if requested == "" {
|
||||
return yesterday.Format("20060102"), nil
|
||||
}
|
||||
parsed, err := time.ParseInLocation("20060102", requested, location)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid queryDate %q: want yyyyMMdd", requested)
|
||||
}
|
||||
if parsed.After(yesterday) {
|
||||
return "", fmt.Errorf("invalid queryDate %q: must not be later than %s", requested, yesterday.Format("20060102"))
|
||||
}
|
||||
if parsed.Format("20060102") < minBackfillDay {
|
||||
return "", fmt.Errorf("invalid queryDate %q: must not be earlier than %s", requested, minBackfillDay)
|
||||
}
|
||||
return parsed.Format("20060102"), nil
|
||||
}
|
||||
|
||||
// SyncDaily is the legacy in-process scan. It is retained as the rollback path
|
||||
// until the worker chain is verified in production, and it is no longer
|
||||
// registered on any timer.
|
||||
func (s *sTrafficUsageDailySync) SyncDaily(ctx context.Context) {
|
||||
now := time.Now().In(historyUsageLocation)
|
||||
yesterday := now.AddDate(0, 0, -1)
|
||||
queryDate := yesterday.Format("20060102")
|
||||
date := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, historyUsageLocation)
|
||||
|
||||
cardColumns := dao.TSimCardInfo.Columns()
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
snapshotMaxID, err := trafficUsageEligibleCardModel(ctx).Max(cardTable + "." + cardColumns.Id)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageDaily snapshot sim cards failed: %+v", err)
|
||||
return
|
||||
}
|
||||
maxID := int64(snapshotMaxID)
|
||||
if maxID <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var lastID int64
|
||||
for lastID < maxID {
|
||||
list := []entity.TSimCardInfo{}
|
||||
if err = trafficUsageEligibleCardModel(ctx).
|
||||
Fields(trafficUsageCardFields()).
|
||||
WhereGT(cardTable+"."+cardColumns.Id, lastID).
|
||||
WhereLTE(cardTable+"."+cardColumns.Id, maxID).
|
||||
OrderAsc(cardTable + "." + cardColumns.Id).
|
||||
Limit(trafficUsageDailyPageSize).
|
||||
Scan(&list); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageDaily list sim cards failed, lastId:%d, err:%+v", lastID, err)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
lastID = list[len(list)-1].Id
|
||||
|
||||
cardMap := trafficUsageDailyCardMap(list)
|
||||
simKeys := trafficUsageDailySimKeys(list)
|
||||
output, err := service.UpstreamChan().QuerySimDatausageDaily(ctx, &model.SimDataUsageDailyInput{
|
||||
SimKeys: simKeys,
|
||||
ReqData: model.SimDataUsageDailyInputItem{
|
||||
QueryDate: queryDate,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageDaily query daily failed, lastId:%d, count:%d, err:%+v", lastID, len(simKeys), err)
|
||||
for _, simKey := range simKeys {
|
||||
s.publishRetry(ctx, simKey, cardMap[simKey.ICCID], queryDate)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
success, failed := reconcileUsageWithPeriod(simKeys, output, func(item model.SimDataUsageDailyOutputItem) float64 { return item.DataAmount }, func(item model.SimDataUsageDailyOutputItem) string { return item.QueryDate }, normalizeUsageDay, queryDate)
|
||||
for _, key := range failed {
|
||||
s.publishRetry(ctx, key, cardMap[key.ICCID], queryDate)
|
||||
}
|
||||
|
||||
if items := s.buildDailyItems(success, cardMap, date); len(items) > 0 {
|
||||
if err = service.TrafficUsage().UpdateDaily(ctx, items); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageDaily write tdengine failed, lastId:%d, err:%+v", lastID, err)
|
||||
for _, item := range success {
|
||||
s.publishRetry(ctx, item.SimKey, cardMap[item.SimKey.ICCID], queryDate, item.ResData.DataAmount)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// buildDailyItems 构建SIM卡的流量使用记录
|
||||
func (s *sTrafficUsageDailySync) buildDailyItems(output []model.SimBaseRes[model.SimDataUsageDailyOutputItem], cardMap map[string]model.TrafficUsageSimCard, date time.Time) []model.TrafficUsageDailyItem {
|
||||
items := make([]model.TrafficUsageDailyItem, 0, len(output))
|
||||
for _, item := range output {
|
||||
if item.SimKey == nil {
|
||||
continue
|
||||
}
|
||||
card := cardMap[item.SimKey.ICCID]
|
||||
items = append(items, model.TrafficUsageDailyItem{
|
||||
TrafficUsageBase: model.TrafficUsageBase{
|
||||
Iccid: item.SimKey.ICCID,
|
||||
UsageMb: item.ResData.DataAmount,
|
||||
PackageCode: card.PackageCode,
|
||||
},
|
||||
Date: date,
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// publishRetry 发布重试消息
|
||||
func (s *sTrafficUsageDailySync) publishRetry(ctx context.Context, simKey *model.SimKey, card model.TrafficUsageSimCard, queryDate string, usage ...float64) {
|
||||
var snapshot *float64
|
||||
if len(usage) > 0 {
|
||||
snapshot = &usage[0]
|
||||
}
|
||||
if !rabbitmq.Enabled(ctx) {
|
||||
g.Log().Errorf(ctx, "traffic usage daily retry skipped, rabbitmq disabled, simKey:%+v, queryDate:%s", simKey, queryDate)
|
||||
return
|
||||
}
|
||||
if err := rabbitmq.PublishConfirmed(ctx, consts.QueueTrafficUsageDailyRetry, model.TrafficUsageDailyRetryMessage{
|
||||
UsageMb: snapshot,
|
||||
SimKey: simKey,
|
||||
Card: card,
|
||||
QueryDate: queryDate,
|
||||
}); err != nil {
|
||||
g.Log().Errorf(ctx, "publish traffic usage daily retry failed, simKey:%+v, queryDate:%s, err:%+v", simKey, queryDate, err)
|
||||
}
|
||||
}
|
||||
|
||||
// trafficUsageDailySimKeys 提取SIM卡的ChanKey、ICCID、IMSI、MSISDN
|
||||
func trafficUsageDailySimKeys(list []entity.TSimCardInfo) []*model.SimKey {
|
||||
simKeys := make([]*model.SimKey, 0, len(list))
|
||||
for _, item := range list {
|
||||
simKeys = append(simKeys, &model.SimKey{
|
||||
ChanKey: item.ChanKey,
|
||||
ICCID: item.Iccid,
|
||||
IMSI: item.Imsi,
|
||||
MSISDN: item.Msisdn,
|
||||
})
|
||||
}
|
||||
return simKeys
|
||||
}
|
||||
|
||||
// trafficUsageDailyCardMap 提取SIM卡的ChanKey、ICCID、IMSI、MSISDN、PackageCode、FlowPoolCode、IsFlowPool
|
||||
func trafficUsageDailyCardMap(list []entity.TSimCardInfo) map[string]model.TrafficUsageSimCard {
|
||||
cardMap := make(map[string]model.TrafficUsageSimCard, len(list))
|
||||
for _, item := range list {
|
||||
cardMap[item.Iccid] = model.TrafficUsageSimCard{
|
||||
Iccid: item.Iccid,
|
||||
PackageCode: item.CurrentPackageCode,
|
||||
}
|
||||
}
|
||||
return cardMap
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/utility/uuid32"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
const (
|
||||
trafficUsageMonthlyPageSize = 100
|
||||
)
|
||||
|
||||
type sTrafficUsageMonthlySync struct {
|
||||
}
|
||||
|
||||
const monthlyUsageFullSyncOperation = "current_month_usage_full_sync"
|
||||
|
||||
// These indirections keep the trigger deterministic in unit tests without changing
|
||||
// the production publisher, clock, or run-id generator.
|
||||
var (
|
||||
monthlyUsageNow = time.Now
|
||||
monthlyUsageRunID = uuid32.S
|
||||
monthlyUsageRabbitEnabled = rabbitmq.Enabled
|
||||
monthlyUsagePublish = rabbitmq.Publish
|
||||
|
||||
monthlyHistoryNow = time.Now
|
||||
monthlyHistoryRunID = uuid32.S
|
||||
monthlyHistoryRabbitEnabled = rabbitmq.Enabled
|
||||
monthlyHistoryPublish = rabbitmq.Publish
|
||||
)
|
||||
|
||||
func NewTrafficUsageMonthlySync() *sTrafficUsageMonthlySync {
|
||||
return &sTrafficUsageMonthlySync{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
service.RegisterTrafficUsageMonthlySync(NewTrafficUsageMonthlySync())
|
||||
}
|
||||
|
||||
// StartMonthly publishes one command for Worker to execute the full monthly
|
||||
// usage sync. Admin deliberately does not scan cards or call an upstream service.
|
||||
func (s *sTrafficUsageMonthlySync) StartMonthly(ctx context.Context) error {
|
||||
_, err := s.StartMonthlyRun(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *sTrafficUsageMonthlySync) StartMonthlyRun(ctx context.Context) (string, error) {
|
||||
return s.StartMonthlyRunScoped(ctx, nil)
|
||||
}
|
||||
|
||||
// StartMonthlyRunScoped starts a run restricted to scope.Channels. A nil scope,
|
||||
// or one whose channels normalize to empty, starts a full sync.
|
||||
func (s *sTrafficUsageMonthlySync) StartMonthlyRunScoped(ctx context.Context, scope *model.SyncScope) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
startedAt := time.Now()
|
||||
normalized, err := model.NormalizeScope(scope)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=validate durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
if !monthlyUsageRabbitEnabled(ctx) {
|
||||
err := fmt.Errorf("monthly usage start skipped: rabbitmq is disabled")
|
||||
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=precheck durationMs=%d reason=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
|
||||
location, err := time.LoadLocation("Asia/Shanghai")
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=timezone durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", fmt.Errorf("load monthly usage timezone: %w", err)
|
||||
}
|
||||
now := monthlyUsageNow().In(location)
|
||||
message := model.TrafficUsageMonthlyStartMessage{
|
||||
RunID: fmt.Sprintf("current-month-usage-%s-%s", now.Format("20060102"), monthlyUsageRunID()),
|
||||
Operation: monthlyUsageFullSyncOperation,
|
||||
QueryMonth: now.Format("200601"),
|
||||
TriggeredAt: now.Format(time.RFC3339),
|
||||
Scope: normalized,
|
||||
}
|
||||
if err = monthlyUsagePublish(ctx, consts.QueueTrafficUsageMonthlyStart, message); err != nil {
|
||||
g.Log().Errorf(ctx, "event=monthly_usage_trigger status=failed stage=publish runId=%s queryMonth=%s durationMs=%d error=%q", message.RunID, message.QueryMonth, elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", fmt.Errorf("publish monthly usage start: %w", err)
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_usage_trigger status=published runId=%s queryMonth=%s triggeredAt=%s durationMs=%d", message.RunID, message.QueryMonth, message.TriggeredAt, elapsedMilliseconds(startedAt))
|
||||
return message.RunID, nil
|
||||
}
|
||||
|
||||
func elapsedMilliseconds(startedAt time.Time) int64 {
|
||||
return time.Since(startedAt).Milliseconds()
|
||||
}
|
||||
|
||||
// StartMonthlyHistory publishes one command for Worker to execute the
|
||||
// previous-month usage history sync (the TDengine monthly table).
|
||||
func (s *sTrafficUsageMonthlySync) StartMonthlyHistory(ctx context.Context) error {
|
||||
_, err := s.StartMonthlyHistoryRun(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *sTrafficUsageMonthlySync) StartMonthlyHistoryRun(ctx context.Context) (string, error) {
|
||||
return s.StartMonthlyHistoryRunScoped(ctx, nil)
|
||||
}
|
||||
|
||||
// StartMonthlyHistoryRunScoped starts a monthly history run restricted to
|
||||
// scope.Channels. A nil scope, or one whose channels normalize to empty, starts
|
||||
// a full sync.
|
||||
//
|
||||
// The period is the previous month, matching what the legacy SyncMonthly scan
|
||||
// wrote. Do not confuse this with StartMonthlyRunScoped, which fills the
|
||||
// current-month snapshot in MySQL.
|
||||
func (s *sTrafficUsageMonthlySync) StartMonthlyHistoryRunScoped(ctx context.Context, scope *model.SyncScope) (string, error) {
|
||||
return s.StartMonthlyHistoryRunScopedAt(ctx, scope, "")
|
||||
}
|
||||
|
||||
// StartMonthlyHistoryRunScopedAt starts a monthly history run for the given
|
||||
// month (yyyyMM). An empty queryMonth keeps the historical behaviour and covers
|
||||
// the previous calendar month; a non-empty value lets an operator backfill one
|
||||
// specific past month.
|
||||
func (s *sTrafficUsageMonthlySync) StartMonthlyHistoryRunScopedAt(ctx context.Context, scope *model.SyncScope, queryMonth string) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
startedAt := time.Now()
|
||||
normalized, err := model.NormalizeScope(scope)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=validate durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
if !monthlyHistoryRabbitEnabled(ctx) {
|
||||
err := fmt.Errorf("monthly history start skipped: rabbitmq is disabled")
|
||||
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=precheck durationMs=%d reason=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
|
||||
location, err := time.LoadLocation("Asia/Shanghai")
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=timezone durationMs=%d error=%q", elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", fmt.Errorf("load monthly history timezone: %w", err)
|
||||
}
|
||||
now := monthlyHistoryNow().In(location)
|
||||
resolvedMonth, err := resolveMonthlyHistoryQueryMonth(now, location, queryMonth)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=validate reason=invalid_query_month queryMonth=%q durationMs=%d error=%q", queryMonth, elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", err
|
||||
}
|
||||
message := model.TrafficUsageMonthlyHistoryStartMessage{
|
||||
RunID: fmt.Sprintf("monthly-history-%s-%s", resolvedMonth, monthlyHistoryRunID()),
|
||||
Operation: model.OperationPreviousMonthUsageFullSync,
|
||||
QueryMonth: resolvedMonth,
|
||||
TriggeredAt: now.Format(time.RFC3339),
|
||||
Scope: normalized,
|
||||
}
|
||||
if err = monthlyHistoryPublish(ctx, consts.QueueTrafficUsageMonthlyHistoryStart, message); err != nil {
|
||||
g.Log().Errorf(ctx, "event=monthly_history_trigger status=failed stage=publish runId=%s queryMonth=%s durationMs=%d error=%q", message.RunID, message.QueryMonth, elapsedMilliseconds(startedAt), err.Error())
|
||||
return "", fmt.Errorf("publish monthly history start: %w", err)
|
||||
}
|
||||
g.Log().Infof(ctx, "event=monthly_history_trigger status=published runId=%s queryMonth=%s triggeredAt=%s durationMs=%d", message.RunID, message.QueryMonth, message.TriggeredAt, elapsedMilliseconds(startedAt))
|
||||
return message.RunID, nil
|
||||
}
|
||||
|
||||
// minBackfillMonth is the earliest month a manual backfill may target. It only
|
||||
// catches obvious typos; it is not a business boundary.
|
||||
const minBackfillMonth = "202001"
|
||||
|
||||
// resolveMonthlyHistoryQueryMonth returns the month a history run must cover. An
|
||||
// empty requested value keeps the legacy default (the previous calendar month);
|
||||
// otherwise the value is validated and returned in canonical yyyyMM form.
|
||||
func resolveMonthlyHistoryQueryMonth(now time.Time, location *time.Location, requested string) (string, error) {
|
||||
currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location)
|
||||
if requested == "" {
|
||||
// The previous month is what the legacy scan backfilled, and the
|
||||
// month-first day is what the worker's snapshotMonth derives the
|
||||
// end-of-month ts from.
|
||||
return currentMonth.AddDate(0, -1, 0).Format("200601"), nil
|
||||
}
|
||||
parsed, err := time.ParseInLocation("200601", requested, location)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid queryMonth %q: want yyyyMM", requested)
|
||||
}
|
||||
// The current month is served by the MySQL snapshot chain
|
||||
// (StartMonthlyRunScoped), not by the TDengine history chain.
|
||||
if !parsed.Before(currentMonth) {
|
||||
return "", fmt.Errorf("invalid queryMonth %q: must be earlier than the current month %s", requested, currentMonth.Format("200601"))
|
||||
}
|
||||
if parsed.Format("200601") < minBackfillMonth {
|
||||
return "", fmt.Errorf("invalid queryMonth %q: must not be earlier than %s", requested, minBackfillMonth)
|
||||
}
|
||||
return parsed.Format("200601"), nil
|
||||
}
|
||||
|
||||
// SyncMonthly is the legacy in-process scan. It is retained as the rollback path
|
||||
// until the worker monthly history chain is verified in production.
|
||||
func (s *sTrafficUsageMonthlySync) SyncMonthly(ctx context.Context) {
|
||||
month := previousUsageMonth(time.Now())
|
||||
queryDate := month.Format("200601")
|
||||
|
||||
cardColumns := dao.TSimCardInfo.Columns()
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
snapshotMaxID, err := trafficUsageEligibleCardModel(ctx).Max(cardTable + "." + cardColumns.Id)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly snapshot sim cards failed: %+v", err)
|
||||
return
|
||||
}
|
||||
maxID := int64(snapshotMaxID)
|
||||
if maxID <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var lastID int64
|
||||
for lastID < maxID {
|
||||
list := []entity.TSimCardInfo{}
|
||||
if err = trafficUsageEligibleCardModel(ctx).
|
||||
Fields(trafficUsageCardFields()).
|
||||
WhereGT(cardTable+"."+cardColumns.Id, lastID).
|
||||
WhereLTE(cardTable+"."+cardColumns.Id, maxID).
|
||||
OrderAsc(cardTable + "." + cardColumns.Id).
|
||||
Limit(trafficUsageMonthlyPageSize).
|
||||
Scan(&list); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly list sim cards failed, lastId:%d, err:%+v", lastID, err)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
lastID = list[len(list)-1].Id
|
||||
|
||||
cardMap := trafficUsageMonthlyCardMap(list)
|
||||
simKeys := trafficUsageMonthlySimKeys(list)
|
||||
output, err := service.UpstreamChan().QuerySimDatausageMonthly(ctx, &model.SimDataUsageMonthlyInput{
|
||||
SimKeys: simKeys,
|
||||
ReqData: model.SimDataUsageMonthlyInputItem{
|
||||
QueryDate: queryDate,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly query monthly failed, lastId:%d, count:%d, err:%+v", lastID, len(simKeys), err)
|
||||
for _, simKey := range simKeys {
|
||||
s.publishRetry(ctx, simKey, cardMap[simKey.ICCID], queryDate)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
success, failed := reconcileUsageWithPeriod(simKeys, output, func(item model.SimDataUsageMonthlyOutputItem) float64 { return item.DataAmount }, func(item model.SimDataUsageMonthlyOutputItem) string { return item.DeadLine }, normalizeUsageMonth, queryDate)
|
||||
for _, key := range failed {
|
||||
s.publishRetry(ctx, key, cardMap[key.ICCID], queryDate)
|
||||
}
|
||||
|
||||
if items := s.buildMonthlyItems(success, cardMap, month); len(items) > 0 {
|
||||
if err = service.TrafficUsage().UpdateMonthly(ctx, items); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficUsageMonthly write tdengine failed, lastId:%d, err:%+v", lastID, err)
|
||||
for _, item := range success {
|
||||
s.publishRetry(ctx, item.SimKey, cardMap[item.SimKey.ICCID], queryDate, item.ResData.DataAmount)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sTrafficUsageMonthlySync) buildMonthlyItems(output []model.SimBaseRes[model.SimDataUsageMonthlyOutputItem], cardMap map[string]model.TrafficUsageSimCard, month time.Time) []model.TrafficUsageMonthlyItem {
|
||||
items := make([]model.TrafficUsageMonthlyItem, 0, len(output))
|
||||
for _, item := range output {
|
||||
if item.SimKey == nil {
|
||||
continue
|
||||
}
|
||||
card := cardMap[item.SimKey.ICCID]
|
||||
items = append(items, model.TrafficUsageMonthlyItem{
|
||||
TrafficUsageBase: model.TrafficUsageBase{
|
||||
Iccid: item.SimKey.ICCID,
|
||||
UsageMb: item.ResData.DataAmount,
|
||||
PackageCode: card.PackageCode,
|
||||
},
|
||||
Month: month,
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func (s *sTrafficUsageMonthlySync) publishRetry(ctx context.Context, simKey *model.SimKey, card model.TrafficUsageSimCard, queryDate string, usage ...float64) {
|
||||
var snapshot *float64
|
||||
if len(usage) > 0 {
|
||||
snapshot = &usage[0]
|
||||
}
|
||||
if !rabbitmq.Enabled(ctx) {
|
||||
g.Log().Errorf(ctx, "traffic usage monthly retry skipped, rabbitmq disabled, simKey:%+v, queryDate:%s", simKey, queryDate)
|
||||
return
|
||||
}
|
||||
if err := rabbitmq.PublishConfirmed(ctx, consts.QueueTrafficUsageMonthlyRetry, model.TrafficUsageMonthlyRetryMessage{
|
||||
UsageMb: snapshot,
|
||||
SimKey: simKey,
|
||||
Card: card,
|
||||
QueryDate: queryDate,
|
||||
}); err != nil {
|
||||
g.Log().Errorf(ctx, "publish traffic usage monthly retry failed, simKey:%+v, queryDate:%s, err:%+v", simKey, queryDate, err)
|
||||
}
|
||||
}
|
||||
|
||||
func trafficUsageMonthlySimKeys(list []entity.TSimCardInfo) []*model.SimKey {
|
||||
simKeys := make([]*model.SimKey, 0, len(list))
|
||||
for _, item := range list {
|
||||
simKeys = append(simKeys, &model.SimKey{
|
||||
ChanKey: item.ChanKey,
|
||||
ICCID: item.Iccid,
|
||||
IMSI: item.Imsi,
|
||||
MSISDN: item.Msisdn,
|
||||
})
|
||||
}
|
||||
return simKeys
|
||||
}
|
||||
|
||||
func trafficUsageMonthlyCardMap(list []entity.TSimCardInfo) map[string]model.TrafficUsageSimCard {
|
||||
cardMap := make(map[string]model.TrafficUsageSimCard, len(list))
|
||||
for _, item := range list {
|
||||
cardMap[item.Iccid] = model.TrafficUsageSimCard{
|
||||
Iccid: item.Iccid,
|
||||
PackageCode: item.CurrentPackageCode,
|
||||
}
|
||||
}
|
||||
return cardMap
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type sTrafficPoolUsageSync struct {
|
||||
}
|
||||
|
||||
type trafficPoolCardRow struct {
|
||||
Iccid string `gconv:"iccid"`
|
||||
FlowPoolCode string `gconv:"flow_pool_code"`
|
||||
}
|
||||
|
||||
func NewTrafficPoolUsageSync() *sTrafficPoolUsageSync {
|
||||
return &sTrafficPoolUsageSync{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
service.RegisterTrafficPoolUsageSync(NewTrafficPoolUsageSync())
|
||||
}
|
||||
|
||||
// TODO: 流量池查询逻辑还未确定
|
||||
// SyncDaily 同步流量池的流量使用记录
|
||||
func (s *sTrafficPoolUsageSync) SyncDaily(ctx context.Context) {
|
||||
yesterday := time.Now().AddDate(0, 0, -1)
|
||||
date := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, time.Local)
|
||||
|
||||
cards, err := queryTrafficPoolCards(ctx)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
|
||||
return
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
g.Log().Debug(ctx, "SyncTrafficPoolUsageDaily skipped, no flow pool cards")
|
||||
return
|
||||
}
|
||||
|
||||
aggregates, err := service.TrafficUsage().AggregatePoolUsageDaily(ctx, date, cards)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
|
||||
return
|
||||
}
|
||||
if items := buildTrafficPoolUsageItems(aggregates, date); len(items) > 0 {
|
||||
if err = service.TrafficUsage().WritePoolUsageDaily(ctx, items); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if countItems := buildTrafficPoolCardCountItems(cards, date); len(countItems) > 0 {
|
||||
if err = service.TrafficUsage().WritePoolCardCountDaily(ctx, countItems); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageDaily failed, date:%s, err:%+v", date.Format("2006-01-02"), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SyncMonthly 同步流量池的流量使用记录
|
||||
func (s *sTrafficPoolUsageSync) SyncMonthly(ctx context.Context) {
|
||||
lastMonth := time.Now().AddDate(0, -1, 0)
|
||||
month := time.Date(lastMonth.Year(), lastMonth.Month(), 1, 0, 0, 0, 0, time.Local)
|
||||
|
||||
cards, err := queryTrafficPoolCards(ctx)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
|
||||
return
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
g.Log().Debug(ctx, "SyncTrafficPoolUsageMonthly skipped, no flow pool cards")
|
||||
return
|
||||
}
|
||||
|
||||
aggregates, err := service.TrafficUsage().AggregatePoolUsageMonthly(ctx, month, cards)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
|
||||
return
|
||||
}
|
||||
if items := buildTrafficPoolUsageItems(aggregates, month); len(items) > 0 {
|
||||
if err = service.TrafficUsage().WritePoolUsageMonthly(ctx, items); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if countItems := buildTrafficPoolCardCountItems(cards, month); len(countItems) > 0 {
|
||||
if err = service.TrafficUsage().WritePoolCardCountMonthly(ctx, countItems); err != nil {
|
||||
g.Log().Errorf(ctx, "SyncTrafficPoolUsageMonthly failed, month:%s, err:%+v", month.Format("2006-01"), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func queryTrafficPoolCards(ctx context.Context) ([]model.TrafficPoolCard, error) {
|
||||
rows := make([]trafficPoolCardRow, 0)
|
||||
cardTable := dao.TSimCardInfo.Table()
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := trafficUsageEligibleCardModel(ctx).
|
||||
Fields(cardTable+"."+columns.Iccid+" AS iccid", cardTable+"."+columns.FlowPoolCode+" AS flow_pool_code").
|
||||
Where(cardTable+"."+columns.IsFlowPool, 1).
|
||||
Where(cardTable+"."+columns.FlowPoolCode+" <> ?", "").
|
||||
Scan(&rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cards := make([]model.TrafficPoolCard, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.Iccid == "" || row.FlowPoolCode == "" {
|
||||
continue
|
||||
}
|
||||
cards = append(cards, model.TrafficPoolCard{
|
||||
Iccid: row.Iccid,
|
||||
FlowPoolCode: row.FlowPoolCode,
|
||||
})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
func buildTrafficPoolUsageItems(aggregates []model.TrafficPoolUsageAggregate, date time.Time) []model.TrafficPoolUsageItem {
|
||||
items := make([]model.TrafficPoolUsageItem, 0, len(aggregates))
|
||||
for _, aggregate := range aggregates {
|
||||
if aggregate.FlowPoolCode == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, model.TrafficPoolUsageItem{
|
||||
FlowPoolCode: aggregate.FlowPoolCode,
|
||||
UsageMb: aggregate.UsageMb,
|
||||
Date: date,
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func buildTrafficPoolCardCountItems(cards []model.TrafficPoolCard, date time.Time) []model.TrafficPoolCardCountItem {
|
||||
countMap := make(map[string]int64)
|
||||
for _, card := range cards {
|
||||
if card.FlowPoolCode == "" {
|
||||
continue
|
||||
}
|
||||
countMap[card.FlowPoolCode]++
|
||||
}
|
||||
|
||||
items := make([]model.TrafficPoolCardCountItem, 0, len(countMap))
|
||||
for flowPoolCode, cardCount := range countMap {
|
||||
items = append(items, model.TrafficPoolCardCountItem{
|
||||
FlowPoolCode: flowPoolCode,
|
||||
CardCount: cardCount,
|
||||
Date: date,
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
Reference in New Issue
Block a user