chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,159 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/tdengine"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
var identifierPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type Repository struct{}
|
||||
|
||||
func New() *Repository {
|
||||
return &Repository{}
|
||||
}
|
||||
|
||||
func (r *Repository) UpdateDaily(ctx context.Context, items []model.TrafficUsageDailyItem) error {
|
||||
config := tdengine.GetConfig(ctx)
|
||||
return writeUsage(ctx, config, config.DbNameFor(config.DailyStable), config.DailyStable, items, func(item model.TrafficUsageDailyItem) time.Time {
|
||||
return snapshotDay(item.Date)
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Repository) UpdateMonthly(ctx context.Context, items []model.TrafficUsageMonthlyItem) error {
|
||||
config := tdengine.GetConfig(ctx)
|
||||
return writeUsage(ctx, config, config.DbNameFor(config.MonthlyStable), config.MonthlyStable, items, func(item model.TrafficUsageMonthlyItem) time.Time {
|
||||
return snapshotMonth(item.Month)
|
||||
})
|
||||
}
|
||||
|
||||
type usageItem interface {
|
||||
model.TrafficUsageDailyItem | model.TrafficUsageMonthlyItem
|
||||
}
|
||||
|
||||
func writeUsage[T usageItem](ctx context.Context, config tdengine.Config, dbName, stable string, items []T, timestamp func(T) time.Time) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !tdengine.Enabled(ctx) {
|
||||
return gerror.New("TDengine disabled; usage was not persisted")
|
||||
}
|
||||
if !identifierPattern.MatchString(dbName) || !identifierPattern.MatchString(stable) {
|
||||
return gerror.New("invalid tdengine database or stable name")
|
||||
}
|
||||
batchSize := config.BatchSize
|
||||
if batchSize <= 0 {
|
||||
batchSize = 200
|
||||
}
|
||||
for start := 0; start < len(items); start += batchSize {
|
||||
end := min(start+batchSize, len(items))
|
||||
var builder strings.Builder
|
||||
builder.WriteString("INSERT INTO ")
|
||||
for index, item := range items[start:end] {
|
||||
base := usageBase(item)
|
||||
if base.Iccid == "" {
|
||||
return gerror.New("traffic usage item ICCID is empty")
|
||||
}
|
||||
if math.IsNaN(base.UsageMb) || math.IsInf(base.UsageMb, 0) || base.UsageMb < 0 {
|
||||
return gerror.Newf("invalid traffic usage value: %v", base.UsageMb)
|
||||
}
|
||||
if index > 0 {
|
||||
builder.WriteByte(' ')
|
||||
}
|
||||
builder.WriteString(fmt.Sprintf("%s.%s USING %s.%s TAGS (%s) VALUES (%s, %s, %s)",
|
||||
dbName,
|
||||
childTableName(stable, base.Iccid),
|
||||
dbName,
|
||||
stable,
|
||||
sqlString(base.Iccid),
|
||||
sqlTime(timestamp(item)),
|
||||
strconv.FormatFloat(base.UsageMb, 'f', -1, 64),
|
||||
sqlString(base.PackageCode),
|
||||
))
|
||||
}
|
||||
if _, err := tdengine.ExecForDb(ctx, dbName, builder.String()); err != nil {
|
||||
return gerror.Wrap(err, "write traffic usage failed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func usageBase[T usageItem](item T) model.TrafficUsageBase {
|
||||
switch value := any(item).(type) {
|
||||
case model.TrafficUsageDailyItem:
|
||||
return value.TrafficUsageBase
|
||||
case model.TrafficUsageMonthlyItem:
|
||||
return value.TrafficUsageBase
|
||||
default:
|
||||
return model.TrafficUsageBase{}
|
||||
}
|
||||
}
|
||||
|
||||
// childTableName builds the TDengine child-table name for one ICCID.
|
||||
//
|
||||
// The rule must stay byte-identical to Admin's childTableNameWithPrefix
|
||||
// (iotlcm-admin-svr internal/logic/trafficusage/traffic_usage.go): drop every
|
||||
// character that is not a letter, digit or underscore, and fall back to an FNV
|
||||
// hash when the result is empty or longer than 190 characters. Admin's QueryDaily
|
||||
// reads child tables by that same rule, so diverging here would silently split
|
||||
// one card across two child tables and hide its data from the diagnosis page.
|
||||
func childTableName(stable, iccid string) string {
|
||||
prefix := "sim_card_usage_m_"
|
||||
if stable == "t_sim_card_usage_daily" {
|
||||
prefix = "sim_card_usage_d_"
|
||||
}
|
||||
var builder strings.Builder
|
||||
builder.WriteString(prefix)
|
||||
for _, char := range strings.ToLower(iccid) {
|
||||
if unicode.IsLetter(char) || unicode.IsDigit(char) || char == '_' {
|
||||
builder.WriteRune(char)
|
||||
}
|
||||
}
|
||||
name := builder.String()
|
||||
if name == prefix || len(name) > 190 {
|
||||
return fmt.Sprintf("%s%x", prefix, hashString(iccid))
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// hashString mirrors Admin's FNV-1a fallback so both sides derive the same
|
||||
// child-table name for identifiers that cannot be embedded verbatim.
|
||||
func hashString(value string) uint32 {
|
||||
hash := fnv.New32a()
|
||||
_, _ = hash.Write([]byte(value))
|
||||
return hash.Sum32()
|
||||
}
|
||||
|
||||
func snapshotDay(value time.Time) time.Time {
|
||||
if value.IsZero() {
|
||||
value = time.Now()
|
||||
}
|
||||
return time.Date(value.Year(), value.Month(), value.Day(), 23, 59, 59, 0, value.Location())
|
||||
}
|
||||
|
||||
func snapshotMonth(value time.Time) time.Time {
|
||||
if value.IsZero() {
|
||||
value = time.Now()
|
||||
}
|
||||
return time.Date(value.Year(), value.Month()+1, 0, 23, 59, 59, 0, value.Location())
|
||||
}
|
||||
|
||||
func sqlString(value string) string {
|
||||
return "'" + strings.ReplaceAll(value, "'", "''") + "'"
|
||||
}
|
||||
|
||||
func sqlTime(value time.Time) string {
|
||||
return sqlString(value.Format("2006-01-02 15:04:05.000"))
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// adminChildTableName is a verbatim copy of Admin's childTableNameWithPrefix
|
||||
// (iotlcm-admin-svr internal/logic/trafficusage/traffic_usage.go). It exists so
|
||||
// the worker rule is asserted against the production reader rule rather than
|
||||
// against itself: Admin's QueryDaily resolves child tables with this rule, so
|
||||
// any divergence here would silently hide a card's daily usage.
|
||||
func adminChildTableName(prefix, value string) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(prefix)
|
||||
for _, r := range strings.ToLower(value) {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
|
||||
builder.WriteRune(r)
|
||||
}
|
||||
}
|
||||
name := builder.String()
|
||||
if name == prefix || len(name) > 190 {
|
||||
hash := fnv.New32a()
|
||||
_, _ = hash.Write([]byte(value))
|
||||
return fmt.Sprintf("%s%x", prefix, hash.Sum32())
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func adminChildTrafficTableName(stable, iccid string) string {
|
||||
prefix := "sim_card_usage_m_"
|
||||
if stable == "t_sim_card_usage_daily" {
|
||||
prefix = "sim_card_usage_d_"
|
||||
}
|
||||
return adminChildTableName(prefix, iccid)
|
||||
}
|
||||
|
||||
func TestChildTableNameMatchesAdminRule(t *testing.T) {
|
||||
longICCID := strings.Repeat("8", 191)
|
||||
veryLongICCID := strings.Repeat("89", 120)
|
||||
cases := []string{
|
||||
"",
|
||||
"89860001123456789012",
|
||||
"8986-0001",
|
||||
"8986 0001",
|
||||
"8986.0001",
|
||||
"8986/0001",
|
||||
"ICCID_8986",
|
||||
"8986-0001-中文",
|
||||
"中文卡号",
|
||||
"____",
|
||||
"8986+0001",
|
||||
"8986'0001",
|
||||
longICCID,
|
||||
veryLongICCID,
|
||||
}
|
||||
for _, stable := range []string{"t_sim_card_usage_daily", "t_sim_card_usage_monthly"} {
|
||||
for _, iccid := range cases {
|
||||
want := adminChildTrafficTableName(stable, iccid)
|
||||
got := childTableName(stable, iccid)
|
||||
if got != want {
|
||||
t.Errorf("childTableName(%q, %q) = %q, admin rule yields %q", stable, iccid, got, want)
|
||||
}
|
||||
if !strings.HasPrefix(got, "sim_card_usage_") {
|
||||
t.Errorf("childTableName(%q, %q) = %q loses the table prefix", stable, iccid, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestChildTableNameDirtyICCIDSplitsOnce guards the original defect: the old
|
||||
// worker rule replaced illegal characters with "_" while Admin dropped them, so
|
||||
// a dirty ICCID could land in a different child table than the reader queried.
|
||||
func TestChildTableNameDirtyICCIDSplitsOnce(t *testing.T) {
|
||||
for _, iccid := range []string{"8986-0001", "8986 0001", "8986.0001"} {
|
||||
got := childTableName("t_sim_card_usage_daily", iccid)
|
||||
if strings.Contains(got, "__") {
|
||||
t.Errorf("childTableName(%q) = %q still substitutes instead of dropping", iccid, got)
|
||||
}
|
||||
if got != "sim_card_usage_d_89860001" {
|
||||
t.Errorf("childTableName(%q) = %q, want the illegal characters dropped", iccid, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildTableNameLongIdentifierUsesHashFallback(t *testing.T) {
|
||||
iccid := strings.Repeat("8", 191)
|
||||
got := childTableName("t_sim_card_usage_daily", iccid)
|
||||
if len(got) > 190 {
|
||||
t.Fatalf("childTableName kept an over-long name: len=%d", len(got))
|
||||
}
|
||||
if got != adminChildTrafficTableName("t_sim_card_usage_daily", iccid) {
|
||||
t.Fatalf("hash fallback diverged from the admin rule: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotDayUsesEndOfDay(t *testing.T) {
|
||||
// The dashboard "yesterday top 5" and the diagnosis page both select the
|
||||
// 23:59:59.000 row, so the timestamp contract must not drift.
|
||||
day := snapshotDay(mustParse(t, "2026-10-09T13:45:00+08:00"))
|
||||
if got := day.Format("2006-01-02 15:04:05.000"); got != "2026-10-09 23:59:59.000" {
|
||||
t.Fatalf("snapshotDay = %s, want 2026-10-09 23:59:59.000", got)
|
||||
}
|
||||
}
|
||||
|
||||
func mustParse(t *testing.T, value string) time.Time {
|
||||
t.Helper()
|
||||
parsed, err := time.Parse(time.RFC3339, value)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", value, err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
// TestSnapshotMonthMatchesAdminRule pins the monthly timestamp contract.
|
||||
//
|
||||
// The diagnosis page reads monthly history by ts and groups it with
|
||||
// ts.Format("2006-01"), while admin's monthStart wrote the end-of-month
|
||||
// 23:59:59.000 row. A migration that changed this would leave the page reading
|
||||
// a month that never gets written.
|
||||
func TestSnapshotMonthMatchesAdminRule(t *testing.T) {
|
||||
// adminMonthStart is a verbatim copy of Admin's monthStart
|
||||
// (iotlcm-admin-svr internal/logic/trafficusage/traffic_usage.go), despite
|
||||
// its name returning the end of the month.
|
||||
adminMonthStart := func(tm time.Time) time.Time {
|
||||
if tm.IsZero() {
|
||||
tm = time.Now()
|
||||
}
|
||||
return time.Date(tm.Year(), tm.Month()+1, 0, 23, 59, 59, 0, tm.Location())
|
||||
}
|
||||
location := time.FixedZone("UTC+8", 8*60*60)
|
||||
for _, month := range []time.Time{
|
||||
time.Date(2026, 9, 1, 0, 0, 0, 0, location),
|
||||
time.Date(2026, 2, 1, 0, 0, 0, 0, location),
|
||||
time.Date(2026, 12, 1, 0, 0, 0, 0, location),
|
||||
// Leap February must land on the 29th.
|
||||
time.Date(2024, 2, 1, 0, 0, 0, 0, location),
|
||||
} {
|
||||
want := adminMonthStart(month).Format("2006-01-02 15:04:05.000")
|
||||
got := snapshotMonth(month).Format("2006-01-02 15:04:05.000")
|
||||
if got != want {
|
||||
t.Errorf("snapshotMonth(%s) = %s, admin rule yields %s", month.Format("2006-01"), got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package trafficusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/tdengine"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
type routingDriver struct{ statements []string }
|
||||
type routingConnection struct{ owner *routingDriver }
|
||||
|
||||
func (recorder *routingDriver) Open(string) (driver.Conn, error) {
|
||||
return &routingConnection{owner: recorder}, nil
|
||||
}
|
||||
func (connection *routingConnection) Prepare(string) (driver.Stmt, error) {
|
||||
return nil, errors.New("unexpected prepare")
|
||||
}
|
||||
func (connection *routingConnection) Close() error { return nil }
|
||||
func (connection *routingConnection) Begin() (driver.Tx, error) {
|
||||
return nil, errors.New("unexpected transaction")
|
||||
}
|
||||
func (connection *routingConnection) Ping(context.Context) error { return nil }
|
||||
func (connection *routingConnection) ExecContext(_ context.Context, statement string, _ []driver.NamedValue) (driver.Result, error) {
|
||||
connection.owner.statements = append(connection.owner.statements, statement)
|
||||
return driver.RowsAffected(1), nil
|
||||
}
|
||||
|
||||
var routingRecorder = &routingDriver{}
|
||||
|
||||
func init() {
|
||||
sql.Register("traffic_usage_split_routing", routingRecorder)
|
||||
}
|
||||
|
||||
func TestRepositoryWritesToConfiguredSplitDatabase(t *testing.T) {
|
||||
recorder := routingRecorder
|
||||
recorder.statements = nil
|
||||
previous := g.Cfg().GetAdapter()
|
||||
t.Cleanup(func() { _ = tdengine.Close(); g.Cfg().SetAdapter(previous) })
|
||||
adapter, err := gcfg.NewAdapterContent("tsdb:\n database: TdEngine\n tdengine:\n type: traffic_usage_split_routing\n dsn: test\n dbName: legacy\n dailyDbName: iotlcm_daily\n monthlyDbName: iotlcm_monthly\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
_ = tdengine.Close()
|
||||
repository := New()
|
||||
period := time.Date(2026, 10, 21, 0, 0, 0, 0, time.FixedZone("CST", 8*60*60))
|
||||
base := model.TrafficUsageBase{Iccid: "8986112520609988658", UsageMb: 12.5, PackageCode: "test"}
|
||||
if err := repository.UpdateDaily(context.Background(), []model.TrafficUsageDailyItem{{TrafficUsageBase: base, Date: period}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repository.UpdateMonthly(context.Background(), []model.TrafficUsageMonthlyItem{{TrafficUsageBase: base, Month: period}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recorder.statements) != 2 {
|
||||
t.Fatalf("statements=%v", recorder.statements)
|
||||
}
|
||||
for index, expected := range []string{
|
||||
"INSERT INTO iotlcm_daily.sim_card_usage_d_8986112520609988658 USING iotlcm_daily.t_sim_card_usage_daily",
|
||||
"INSERT INTO iotlcm_monthly.sim_card_usage_m_8986112520609988658 USING iotlcm_monthly.t_sim_card_usage_monthly",
|
||||
} {
|
||||
if !strings.HasPrefix(recorder.statements[index], expected) {
|
||||
t.Errorf("statement=%s want prefix=%s", recorder.statements[index], expected)
|
||||
}
|
||||
}
|
||||
if err := repository.UpdateDaily(context.Background(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repository.UpdateMonthly(context.Background(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recorder.statements) != 2 {
|
||||
t.Fatal("empty updates issued SQL")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user