chore: 初始开源发布

iotlcm-worker-svr:异步执行中枢(RabbitMQ 消费者 + 全量同步编排)。

- Go module:iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr(214 文件)
- 凭据与内部标识脱敏:MySQL/Redis 凭据、私网与生产 IP、内网域名、
  库名 xiujie_admin → iotlcm、哨兵名占位
- 159 处真实手机号 → 合法 11 位编造号(保留号段与后 4 位,维持互异,
  不破坏「校验 MSISDN 格式」等测试断言)
- 移除内部资产:CI 触发器、Jenkins 触发测试文档、预演门禁与镜像
  digest 记录
- Dockerfile 改用公共基础镜像(golang:1.25-alpine → alpine:3.23),
  移除写死的 git 凭据
- README 重写(架构/全量同步模型/配置/已知失败),补 Apache-2.0
  LICENSE,LF 行尾 + .gitattributes

已知:3 个测试包在源仓同一提交上同样失败(client/upstream、
service/allocate、service/inbound),属既有缺陷,本仓未引入亦未修复。
This commit is contained in:
MiFaZhan
2026-10-22 16:37:56 +08:00
commit 2bec257289
361 changed files with 71524 additions and 0 deletions
@@ -0,0 +1,35 @@
package batch
import (
"context"
batchdomain "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
"testing"
)
func TestExecutionPreservesOriginMetadata(t *testing.T) {
ctx := context.Background()
r := newTestRepository(t, ctx)
insertBatch(t, ctx, r.db, "META", batchdomain.Type103)
if _, err := r.db.Model(dao.BatchMain.Table()).Ctx(ctx).Where("batch_no", "META").Data(do.BatchMain{SourceChannel: "mini_program", ExecutionMode: 1}).Update(); err != nil {
t.Fatal(err)
}
if claimed, err := r.MarkProcessing(ctx, "META", batchdomain.Type103); err != nil || !claimed {
t.Fatal(claimed, err)
}
if marked, err := r.MarkFailed(ctx, "META", batchdomain.Type103, "test failure"); err != nil || !marked {
t.Fatal(marked, err)
}
main, err := r.FindMain(ctx, "META")
if err != nil || main.SourceChannel != "mini_program" || main.ExecutionMode != 1 {
t.Fatalf("metadata changed: %+v err=%v", main, err)
}
if marked, err := r.MarkFailed(ctx, "META", batchdomain.Type103, "duplicate"); err != nil || marked {
t.Fatal(marked, err)
}
main, err = r.FindMain(ctx, "META")
if err != nil || main.SourceChannel != "mini_program" || main.ExecutionMode != 1 {
t.Fatal("duplicate mutated origin", err)
}
}
+200
View File
@@ -0,0 +1,200 @@
package batch
import (
"context"
"strings"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/os/gtime"
batchdomain "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
)
const (
AsyncStatusSuccess = 3
RunResultSuccess = 2
batchDetailErrorReasonLimit = 512
)
type Repository struct {
db gdb.DB
}
func New() *Repository {
return &Repository{db: dao.BatchMain.DB()}
}
func (r *Repository) FindMain(ctx context.Context, batchNo string) (*entity.BatchMain, error) {
var main *entity.BatchMain
if err := r.db.Model(dao.BatchMain.Table()).Ctx(ctx).
Where(dao.BatchMain.Columns().BatchNo, batchNo).
Scan(&main); err != nil {
return nil, gerror.Wrap(err, "query batch main failed")
}
return main, nil
}
func (r *Repository) MarkProcessing(ctx context.Context, batchNo string, batchType int) (claimed bool, err error) {
err = r.db.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
mainColumns := dao.BatchMain.Columns()
result, updateErr := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
Where(mainColumns.BatchNo, batchNo).
Where(mainColumns.BatchType, batchType).
Where(mainColumns.BatchAsyncStatus, batchdomain.AsyncStatusPending).
Data(do.BatchMain{
BatchAsyncStatus: batchdomain.AsyncStatusProcessing,
RunResult: batchdomain.RunResultRunning,
StartTime: gtime.Now(),
}).
Update()
if updateErr != nil {
return gerror.Wrap(updateErr, "CAS batch main failed")
}
affected, affectedErr := result.RowsAffected()
if affectedErr != nil {
return gerror.Wrap(affectedErr, "read batch main CAS result failed")
}
if affected == 0 {
return nil
}
detailColumns := dao.BatchDetail.Columns()
if _, updateErr = tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
Where(detailColumns.BatchNo, batchNo).
Where(detailColumns.BatchType, batchType).
Where(detailColumns.DetailStatus, batchdomain.DetailStatusPending).
Data(do.BatchDetail{DetailStatus: batchdomain.DetailStatusRunning}).
Update(); updateErr != nil {
return gerror.Wrap(updateErr, "mark pending batch details running failed")
}
claimed = true
return nil
})
return claimed, gerror.Wrap(err, "claim batch transaction failed")
}
func (r *Repository) ReclaimProcessing(ctx context.Context, batchNo string, batchType int, startedAt *gtime.Time) (claimed bool, err error) {
columns := dao.BatchMain.Columns()
model := r.db.Model(dao.BatchMain.Table()).Ctx(ctx).
Where(columns.BatchNo, batchNo).
Where(columns.BatchType, batchType).
Where(columns.BatchAsyncStatus, batchdomain.AsyncStatusProcessing)
if startedAt == nil || startedAt.IsZero() {
model = model.Where(columns.StartTime, nil)
} else {
model = model.Where(columns.StartTime, startedAt)
}
result, err := model.Data(do.BatchMain{
StartTime: gtime.Now(),
RunResult: batchdomain.RunResultRunning,
}).Update()
if err != nil {
return false, gerror.Wrap(err, "reclaim stale batch failed")
}
affected, err := result.RowsAffected()
if err != nil {
return false, gerror.Wrap(err, "read stale batch reclaim result failed")
}
return affected == 1, nil
}
func (r *Repository) MarkFailed(ctx context.Context, batchNo string, batchType int, reason string) (marked bool, err error) {
detailReason := truncateBatchDetailReason(reason)
err = r.db.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
mainColumns := dao.BatchMain.Columns()
result, updateErr := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
Where(mainColumns.BatchNo, batchNo).
Where(mainColumns.BatchType, batchType).
Where(mainColumns.BatchAsyncStatus, batchdomain.AsyncStatusProcessing).
Data(do.BatchMain{
BatchAsyncStatus: batchdomain.AsyncStatusFailed,
RunResult: batchdomain.RunResultFailed,
FailReason: reason,
EndTime: gtime.Now(),
}).
Update()
if updateErr != nil {
return gerror.Wrap(updateErr, "mark batch main failed failed")
}
affected, affectedErr := result.RowsAffected()
if affectedErr != nil {
return gerror.Wrap(affectedErr, "read batch failure status update result failed")
}
if affected == 0 {
return nil
}
detailColumns := dao.BatchDetail.Columns()
if _, updateErr = tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
Where(detailColumns.BatchNo, batchNo).
Where(detailColumns.BatchType, batchType).
Where(detailColumns.DetailStatus, batchdomain.DetailStatusRunning).
Data(do.BatchDetail{
DetailStatus: batchdomain.DetailStatusFailed,
ErrorReason: detailReason,
}).
Update(); updateErr != nil {
return gerror.Wrap(updateErr, "mark running batch details failed failed")
}
marked = true
return nil
})
return marked, gerror.Wrap(err, "mark batch failure transaction failed")
}
func truncateBatchDetailReason(reason string) string {
runes := []rune(strings.TrimSpace(reason))
if len(runes) <= batchDetailErrorReasonLimit {
return string(runes)
}
return string(runes[:batchDetailErrorReasonLimit])
}
// MarkSuccess 将指定批次从 Processing 标记为 Success,并结算明细状态。
func (r *Repository) MarkSuccess(ctx context.Context, batchNo string, batchType int, success, fail int, reason string) (marked bool, err error) {
err = r.db.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
mainColumns := dao.BatchMain.Columns()
result, updateErr := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
Where(mainColumns.BatchNo, batchNo).
Where(mainColumns.BatchType, batchType).
Where(mainColumns.BatchAsyncStatus, batchdomain.AsyncStatusProcessing).
Data(do.BatchMain{
BatchAsyncStatus: AsyncStatusSuccess,
RunResult: RunResultSuccess,
SuccessCount: success,
FailCount: fail,
EndTime: gtime.Now(),
}).
Update()
if updateErr != nil {
return gerror.Wrap(updateErr, "mark batch main success failed")
}
affected, affectedErr := result.RowsAffected()
if affectedErr != nil {
return gerror.Wrap(affectedErr, "read batch success status update result failed")
}
if affected == 0 {
return nil
}
detailColumns := dao.BatchDetail.Columns()
if _, updateErr = tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
Where(detailColumns.BatchNo, batchNo).
Where(detailColumns.BatchType, batchType).
Where(detailColumns.DetailStatus, batchdomain.DetailStatusRunning).
Data(do.BatchDetail{
DetailStatus: batchdomain.DetailStatusSuccess,
ErrorReason: reason,
}).
Update(); updateErr != nil {
return gerror.Wrap(updateErr, "mark running batch details success failed")
}
marked = true
return nil
})
return marked, gerror.Wrap(err, "mark batch success transaction failed")
}
@@ -0,0 +1,204 @@
package batch
import (
"context"
"strings"
"testing"
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
"github.com/gogf/gf/v2/database/gdb"
batchdomain "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
)
func TestMarkProcessingClaimsMainAndPendingDetails(t *testing.T) {
ctx := context.Background()
repository := newTestRepository(t, ctx)
insertBatch(t, ctx, repository.db, "B-103", batchdomain.Type103)
main, err := repository.FindMain(ctx, "B-103")
if err != nil || main == nil || main.BatchType != batchdomain.Type103 {
t.Fatalf("FindMain() = (%+v, %v), want batch type %d", main, err, batchdomain.Type103)
}
missing, err := repository.FindMain(ctx, "missing")
if err != nil || missing != nil {
t.Fatalf("FindMain(missing) = (%+v, %v), want nil, nil", missing, err)
}
claimed, err := repository.MarkProcessing(ctx, "B-103", batchdomain.Type103)
if err != nil {
t.Fatalf("MarkProcessing() error = %v", err)
}
if !claimed {
t.Fatal("MarkProcessing() claimed = false, want true")
}
assertStatus(t, ctx, repository.db, dao.BatchMain.Table(), dao.BatchMain.Columns().BatchAsyncStatus, batchdomain.AsyncStatusProcessing)
assertStatus(t, ctx, repository.db, dao.BatchMain.Table(), dao.BatchMain.Columns().RunResult, batchdomain.RunResultRunning)
main, err = repository.FindMain(ctx, "B-103")
if err != nil || main.StartTime == nil || main.StartTime.IsZero() {
t.Fatalf("processing batch start time = %v, error = %v; want current time", main.StartTime, err)
}
var details []entity.BatchDetail
if err = repository.db.Model(dao.BatchDetail.Table()).Ctx(ctx).
Order(dao.BatchDetail.Columns().Id).
Scan(&details); err != nil {
t.Fatalf("query detail statuses: %v", err)
}
if len(details) != 2 || details[0].DetailStatus != batchdomain.DetailStatusRunning || details[1].DetailStatus != 2 {
t.Fatalf("details = %+v, want statuses [1 2]", details)
}
claimed, err = repository.MarkProcessing(ctx, "B-103", batchdomain.Type103)
if err != nil {
t.Fatalf("second MarkProcessing() error = %v", err)
}
if claimed {
t.Fatal("second MarkProcessing() claimed = true, want false")
}
}
func TestMarkProcessingRollsBackMainWhenDetailUpdateFails(t *testing.T) {
ctx := context.Background()
repository := newTestRepository(t, ctx)
insertBatch(t, ctx, repository.db, "B-104", batchdomain.Type104)
if _, err := repository.db.Exec(ctx, "DROP TABLE batch_detail"); err != nil {
t.Fatalf("drop detail table: %v", err)
}
if claimed, err := repository.MarkProcessing(ctx, "B-104", batchdomain.Type104); err == nil || claimed {
t.Fatalf("MarkProcessing() = (%t, %v), want false and error", claimed, err)
}
assertStatus(t, ctx, repository.db, dao.BatchMain.Table(), dao.BatchMain.Columns().BatchAsyncStatus, batchdomain.AsyncStatusPending)
assertStatus(t, ctx, repository.db, dao.BatchMain.Table(), dao.BatchMain.Columns().RunResult, 0)
main, err := repository.FindMain(ctx, "B-104")
if err != nil || main.StartTime != nil {
t.Fatalf("rolled back batch start time = %v, error = %v; want nil", main.StartTime, err)
}
}
func TestMarkFailedFinalizesProcessingBatchAndDetails(t *testing.T) {
ctx := context.Background()
repository := newTestRepository(t, ctx)
insertBatch(t, ctx, repository.db, "B-105", batchdomain.Type103)
if claimed, err := repository.MarkProcessing(ctx, "B-105", batchdomain.Type103); err != nil || !claimed {
t.Fatalf("MarkProcessing() = (%t, %v), want true, nil", claimed, err)
}
marked, err := repository.MarkFailed(ctx, "B-105", batchdomain.Type103, "source file is unavailable")
if err != nil || !marked {
t.Fatalf("MarkFailed() = (%t, %v), want true, nil", marked, err)
}
assertStatus(t, ctx, repository.db, dao.BatchMain.Table(), dao.BatchMain.Columns().BatchAsyncStatus, batchdomain.AsyncStatusFailed)
assertStatus(t, ctx, repository.db, dao.BatchMain.Table(), dao.BatchMain.Columns().RunResult, batchdomain.RunResultFailed)
var main entity.BatchMain
if err = repository.db.Model(dao.BatchMain.Table()).Ctx(ctx).Where(dao.BatchMain.Columns().BatchNo, "B-105").Scan(&main); err != nil {
t.Fatalf("query failed batch main: %v", err)
}
if main.FailReason != "source file is unavailable" || main.EndTime == nil || main.EndTime.IsZero() {
t.Fatalf("failed batch main = %+v, want failure reason and end time", main)
}
var details []entity.BatchDetail
if err = repository.db.Model(dao.BatchDetail.Table()).Ctx(ctx).Order(dao.BatchDetail.Columns().Id).Scan(&details); err != nil {
t.Fatalf("query failed batch details: %v", err)
}
if len(details) != 2 || details[0].DetailStatus != batchdomain.DetailStatusFailed || details[0].ErrorReason != "source file is unavailable" || details[1].DetailStatus != 2 {
t.Fatalf("failed details = %+v, want statuses [3 2]", details)
}
marked, err = repository.MarkFailed(ctx, "B-105", batchdomain.Type103, "ignored")
if err != nil || marked {
t.Fatalf("second MarkFailed() = (%t, %v), want false, nil", marked, err)
}
}
func TestMarkFailedTruncatesDetailReason(t *testing.T) {
ctx := context.Background()
repository := newTestRepository(t, ctx)
insertBatch(t, ctx, repository.db, "B-106", batchdomain.Type103)
if claimed, err := repository.MarkProcessing(ctx, "B-106", batchdomain.Type103); err != nil || !claimed {
t.Fatalf("MarkProcessing() = (%t, %v), want true, nil", claimed, err)
}
reason := strings.Repeat("x", batchDetailErrorReasonLimit+20)
marked, err := repository.MarkFailed(ctx, "B-106", batchdomain.Type103, reason)
if err != nil || !marked {
t.Fatalf("MarkFailed() = (%t, %v), want true, nil", marked, err)
}
var main entity.BatchMain
if err = repository.db.Model(dao.BatchMain.Table()).Ctx(ctx).Where(dao.BatchMain.Columns().BatchNo, "B-106").Scan(&main); err != nil {
t.Fatalf("query failed batch main: %v", err)
}
if main.FailReason != reason {
t.Fatalf("main failure reason length = %d, want full length %d", len([]rune(main.FailReason)), len([]rune(reason)))
}
var details []entity.BatchDetail
if err = repository.db.Model(dao.BatchDetail.Table()).Ctx(ctx).
Where(dao.BatchDetail.Columns().BatchNo, "B-106").
Order(dao.BatchDetail.Columns().Id).
Scan(&details); err != nil {
t.Fatalf("query failed batch details: %v", err)
}
if len(details) != 2 || len([]rune(details[0].ErrorReason)) != batchDetailErrorReasonLimit || details[1].ErrorReason != "" {
t.Fatalf("detail failure reasons = %+v, want first truncated to %d and second unchanged", details, batchDetailErrorReasonLimit)
}
}
func newTestRepository(t *testing.T, ctx context.Context) *Repository {
t.Helper()
db, err := gdb.New(gdb.ConfigNode{
Type: "sqlite",
Link: "sqlite::@file(:memory:)",
})
if err != nil {
t.Fatalf("create sqlite database: %v", err)
}
statements := []string{
"CREATE TABLE batch_main (batch_no TEXT PRIMARY KEY, batch_type INTEGER NOT NULL, batch_async_status INTEGER NOT NULL, run_result INTEGER NOT NULL DEFAULT 0, fail_reason TEXT, start_time DATETIME, end_time DATETIME, source_channel TEXT NOT NULL DEFAULT 'legacy', execution_mode INTEGER NOT NULL DEFAULT 0)",
"CREATE TABLE batch_detail (id INTEGER PRIMARY KEY AUTOINCREMENT, batch_no TEXT NOT NULL, batch_type INTEGER NOT NULL, detail_status INTEGER NOT NULL, error_reason TEXT)",
}
for _, statement := range statements {
if _, err = db.Exec(ctx, statement); err != nil {
t.Fatalf("create test schema: %v", err)
}
}
t.Cleanup(func() { _ = db.Close(ctx) })
return &Repository{db: db}
}
func insertBatch(t *testing.T, ctx context.Context, db gdb.DB, batchNo string, batchType int) {
t.Helper()
if _, err := db.Model(dao.BatchMain.Table()).Ctx(ctx).Data(do.BatchMain{
BatchNo: batchNo,
BatchType: batchType,
BatchAsyncStatus: batchdomain.AsyncStatusPending,
}).Insert(); err != nil {
t.Fatalf("insert batch main: %v", err)
}
for _, status := range []int{batchdomain.DetailStatusPending, 2} {
if _, err := db.Model(dao.BatchDetail.Table()).Ctx(ctx).Data(do.BatchDetail{
BatchNo: batchNo,
BatchType: batchType,
DetailStatus: status,
}).Insert(); err != nil {
t.Fatalf("insert batch detail: %v", err)
}
}
}
func assertStatus(t *testing.T, ctx context.Context, db gdb.DB, table, column string, expected int) {
t.Helper()
actual, err := db.Model(table).Ctx(ctx).Value(column)
if err != nil {
t.Fatalf("query %s status: %v", table, err)
}
if actual.Int() != expected {
t.Fatalf("%s status = %d, want %d", table, actual.Int(), expected)
}
}