MiFaZhan
2bec257289
iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。 - Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件） - 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、 库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异， 不破坏「校验 MSISDN 格式」等测试断言） - 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23）， 移除写死的 git 凭据 - README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 已知：3 个测试包在源仓同一提交上同样失败（client/upstream、 service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。
515 lines
16 KiB
Go
515 lines
16 KiB
Go
package data
|
|
|
|
import (
|
|
"context"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/gogf/gf/v2/database/gdb"
|
|
"github.com/gogf/gf/v2/encoding/gjson"
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
"github.com/gogf/gf/v2/os/gtime"
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
|
|
data "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data"
|
|
)
|
|
|
|
type Repository struct {
|
|
db gdb.DB
|
|
resultFiles batchresult.Generator
|
|
}
|
|
|
|
func New() *Repository {
|
|
return &Repository{resultFiles: batchresult.New()}
|
|
}
|
|
|
|
func (r *Repository) database() gdb.DB {
|
|
if r.db != nil {
|
|
return r.db
|
|
}
|
|
return g.DB()
|
|
}
|
|
|
|
func (r *Repository) Load(ctx context.Context, batchNo string) (*data.Task, error) {
|
|
if batchNo == "" {
|
|
return nil, gerror.New("data service batch number is empty")
|
|
}
|
|
batch := entity.BatchMain{}
|
|
if err := r.database().Model(dao.BatchMain.Table()).Safe().Ctx(ctx).
|
|
Where(dao.BatchMain.Columns().BatchNo, batchNo).
|
|
WhereIn(dao.BatchMain.Columns().BatchType, []int{consts.BatchTypeDataCut, consts.BatchTypeDataStart, consts.BatchTypeDataStop}).
|
|
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
|
|
Scan(&batch); err != nil {
|
|
return nil, gerror.Wrap(err, "query data service batch")
|
|
}
|
|
if batch.Id == 0 {
|
|
return nil, gerror.New("processing data service batch does not exist")
|
|
}
|
|
operation, err := data.OperationByBatchType(batch.BatchType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var batchDetails []*entity.BatchDetail
|
|
if err = r.database().Model(dao.BatchDetail.Table()).Safe().Ctx(ctx).
|
|
Where(dao.BatchDetail.Columns().BatchNo, batchNo).
|
|
Where(dao.BatchDetail.Columns().BatchType, batch.BatchType).
|
|
Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
|
|
OrderAsc(dao.BatchDetail.Columns().RowNo).
|
|
Scan(&batchDetails); err != nil {
|
|
return nil, gerror.Wrap(err, "query running data service details")
|
|
}
|
|
if len(batchDetails) == 0 {
|
|
return nil, gerror.New("data service batch has no running details")
|
|
}
|
|
|
|
task := &data.Task{
|
|
Batch: data.Batch{
|
|
ID: batch.Id,
|
|
BatchNo: batch.BatchNo,
|
|
BatchType: batch.BatchType,
|
|
TenantID: batch.TenantId,
|
|
CreatedBy: batch.CreatedBy,
|
|
CreatedDept: batch.CreatedDept,
|
|
StartTime: batch.StartTime,
|
|
BatchInfo: batch.BatchInfo,
|
|
},
|
|
Operation: operation,
|
|
ApnName: apnName(batch.BatchInfo),
|
|
Details: make([]*data.Detail, 0, len(batchDetails)),
|
|
}
|
|
for _, batchDetail := range batchDetails {
|
|
if batchDetail == nil {
|
|
continue
|
|
}
|
|
detail := &data.Detail{
|
|
ID: batchDetail.Id,
|
|
RowNo: batchDetail.RowNo,
|
|
ICCID: strings.TrimSpace(batchDetail.ItemId),
|
|
Reason: strings.TrimSpace(batchDetail.ErrorReason),
|
|
}
|
|
if batchDetail.Snapshot != nil {
|
|
if value := strings.TrimSpace(batchDetail.Snapshot.Get("ICCID").String()); value != "" {
|
|
detail.ICCID = value
|
|
}
|
|
detail.InputMSISDN = strings.TrimSpace(batchDetail.Snapshot.Get("MSISDN").String())
|
|
detail.InputIMSI = strings.TrimSpace(batchDetail.Snapshot.Get("IMSI").String())
|
|
detail.MSISDN = detail.InputMSISDN
|
|
detail.IMSI = detail.InputIMSI
|
|
detail.Authorized = batchDetail.Snapshot.Get("authorized").Bool()
|
|
detail.ChanKey = strings.TrimSpace(batchDetail.Snapshot.Get("chanKey").String())
|
|
detail.TrafficCutoffCrossMonthHold = batchDetail.Snapshot.Get("trafficCutoffCrossMonthHold").Int()
|
|
}
|
|
task.Details = append(task.Details, detail)
|
|
}
|
|
if len(task.Details) == 0 {
|
|
return nil, gerror.New("data service batch has no readable running details")
|
|
}
|
|
if err = r.loadCards(ctx, task); err != nil {
|
|
return nil, err
|
|
}
|
|
return task, nil
|
|
}
|
|
|
|
func apnName(info *gjson.Json) string {
|
|
if info != nil {
|
|
if value := strings.TrimSpace(info.Get("apnName").String()); value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return consts.SimApnNameDefault
|
|
}
|
|
|
|
func (r *Repository) loadCards(ctx context.Context, task *data.Task) error {
|
|
iccids := make([]string, 0, len(task.Details))
|
|
for _, detail := range task.Details {
|
|
if detail != nil && detail.ICCID != "" {
|
|
iccids = append(iccids, detail.ICCID)
|
|
}
|
|
}
|
|
if len(iccids) == 0 {
|
|
return nil
|
|
}
|
|
var cards []*entity.TSimCardInfo
|
|
if err := r.database().Model(dao.TSimCardInfo.Table()).Safe().Ctx(ctx).
|
|
Where(dao.TSimCardInfo.Columns().TenantId, task.Batch.TenantID).
|
|
WhereIn(dao.TSimCardInfo.Columns().Iccid, iccids).
|
|
Scan(&cards); err != nil {
|
|
return gerror.Wrap(err, "query data service SIM cards")
|
|
}
|
|
byICCID := make(map[string]*entity.TSimCardInfo, len(cards))
|
|
for _, card := range cards {
|
|
if card != nil {
|
|
byICCID[strings.TrimSpace(card.Iccid)] = card
|
|
}
|
|
}
|
|
for _, detail := range task.Details {
|
|
card := byICCID[detail.ICCID]
|
|
if card == nil {
|
|
continue
|
|
}
|
|
detail.Card = &data.Card{
|
|
ID: card.Id,
|
|
ICCID: strings.TrimSpace(card.Iccid),
|
|
MSISDN: strings.TrimSpace(card.Msisdn),
|
|
IMSI: strings.TrimSpace(card.Imsi),
|
|
ChanKey: preferredDataServiceChanKey(dataAppBatchUsesSnapshotChannel(task.Batch.BatchInfo), detail.ChanKey, card.ChanKey),
|
|
DataServiceStatus: card.DataServiceStatus,
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func preferredDataServiceChanKey(useSnapshot bool, snapshotChanKey string, cardChanKey string) string {
|
|
if useSnapshot {
|
|
if value := strings.TrimSpace(snapshotChanKey); value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return strings.TrimSpace(cardChanKey)
|
|
}
|
|
|
|
func dataAppBatchUsesSnapshotChannel(info *gjson.Json) bool {
|
|
if info == nil {
|
|
return false
|
|
}
|
|
return strings.EqualFold(strings.TrimSpace(info.Get("source").String()), "app")
|
|
}
|
|
|
|
func (r *Repository) RefreshDataServiceState(ctx context.Context, task *data.Task) error {
|
|
if task == nil || len(task.Details) == 0 {
|
|
return nil
|
|
}
|
|
iccids := make([]string, 0, len(task.Details))
|
|
for _, detail := range task.Details {
|
|
if detail != nil && detail.ICCID != "" {
|
|
iccids = append(iccids, detail.ICCID)
|
|
}
|
|
}
|
|
if len(iccids) == 0 {
|
|
return nil
|
|
}
|
|
type row struct {
|
|
ICCID string `orm:"iccid"`
|
|
DataServiceStatus int `orm:"data_service_status"`
|
|
}
|
|
var cards []row
|
|
if err := r.database().Model(dao.TSimCardInfo.Table()).Safe().Ctx(ctx).
|
|
Fields(dao.TSimCardInfo.Columns().Iccid, dao.TSimCardInfo.Columns().DataServiceStatus).
|
|
Where(dao.TSimCardInfo.Columns().TenantId, task.Batch.TenantID).
|
|
WhereIn(dao.TSimCardInfo.Columns().Iccid, iccids).
|
|
Scan(&cards); err != nil {
|
|
return gerror.Wrap(err, "refresh data service card state")
|
|
}
|
|
byICCID := make(map[string]int, len(cards))
|
|
for _, card := range cards {
|
|
byICCID[strings.TrimSpace(card.ICCID)] = card.DataServiceStatus
|
|
}
|
|
for _, detail := range task.Details {
|
|
if detail == nil || detail.Card == nil {
|
|
continue
|
|
}
|
|
if status, ok := byICCID[detail.ICCID]; ok {
|
|
detail.Card.DataServiceStatus = status
|
|
detail.StatusBefore = status
|
|
detail.StatusAfter = status
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) SaveValidationFailure(ctx context.Context, task *data.Task) error {
|
|
return r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
for _, detail := range task.Details {
|
|
if detail == nil {
|
|
continue
|
|
}
|
|
result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
|
|
Where(dao.BatchDetail.Columns().Id, detail.ID).
|
|
Where(dao.BatchDetail.Columns().BatchNo, task.Batch.BatchNo).
|
|
Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
|
|
Data(do.BatchDetail{
|
|
DetailStatus: consts.BatchDetailStatusFailed,
|
|
ErrorReason: detail.Reason,
|
|
Snapshot: detail.Snapshot(task.Operation, task.ApnName),
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}).Update()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if affected != 1 {
|
|
return gerror.Newf("data service detail %d is no longer running", detail.ID)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (r *Repository) SaveResult(ctx context.Context, task *data.Task, detail *data.Detail, outcome data.Outcome) (data.Outcome, error) {
|
|
if detail == nil {
|
|
return data.Outcome{}, gerror.New("data service detail is nil")
|
|
}
|
|
resolved := outcome
|
|
err := r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
if resolved.Success {
|
|
if detail.Card == nil {
|
|
resolved.Success = false
|
|
resolved.Message = "SIM card record is missing"
|
|
} else {
|
|
result, err := tx.Ctx(ctx).Model(dao.TSimCardInfo.Table()).
|
|
Where(dao.TSimCardInfo.Columns().Id, detail.Card.ID).
|
|
Where(dao.TSimCardInfo.Columns().TenantId, task.Batch.TenantID).
|
|
Data(dataServiceUpdateData(task, detail)).Update()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if affected != 1 {
|
|
return gerror.Newf("data service SIM card %d is missing or changed", detail.Card.ID)
|
|
}
|
|
detail.StatusAfter = task.Operation.TargetStatus
|
|
}
|
|
}
|
|
detail.ApplyOutcome(resolved)
|
|
detailStatus := consts.BatchDetailStatusFailed
|
|
if resolved.Success {
|
|
detailStatus = consts.BatchDetailStatusSuccess
|
|
}
|
|
result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
|
|
Where(dao.BatchDetail.Columns().Id, detail.ID).
|
|
Where(dao.BatchDetail.Columns().BatchNo, task.Batch.BatchNo).
|
|
Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
|
|
Data(do.BatchDetail{
|
|
DetailStatus: detailStatus,
|
|
ErrorCode: strconv.Itoa(resolved.Code),
|
|
ErrorReason: detail.Reason,
|
|
Snapshot: detail.Snapshot(task.Operation, task.ApnName),
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}).Update()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if affected != 1 {
|
|
return gerror.Newf("data service detail %d is no longer running", detail.ID)
|
|
}
|
|
return nil
|
|
})
|
|
return resolved, err
|
|
}
|
|
|
|
func dataServiceUpdateData(task *data.Task, detail *data.Detail) do.TSimCardTrafficCutoffConfig {
|
|
now := gtime.Now()
|
|
result := do.TSimCardTrafficCutoffConfig{
|
|
DataServiceStatus: task.Operation.TargetStatus,
|
|
StatusSyncTime: now,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}
|
|
switch task.Batch.BatchType {
|
|
case consts.BatchTypeDataCut:
|
|
if detail != nil && detail.TrafficCutoffCrossMonthHold == 1 {
|
|
result.TrafficCutoffTriggeredAt = now
|
|
}
|
|
case consts.BatchTypeDataStart:
|
|
result.TrafficCutoffTriggeredAt = gdb.Raw("NULL")
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (r *Repository) Finalize(ctx context.Context, task *data.Task) error {
|
|
successCount := 0
|
|
for _, detail := range task.Details {
|
|
if detail != nil && detail.Reason == "" {
|
|
successCount++
|
|
}
|
|
}
|
|
failCount := len(task.Details) - successCount
|
|
batchAsyncStatus, runResult := batchResult(successCount, failCount)
|
|
if task.FailureReason == "" && failCount > 0 {
|
|
task.FailureReason = strconv.Itoa(failCount) + " SIM cards failed"
|
|
}
|
|
validCount := 0
|
|
if task.ValidationOK {
|
|
validCount = validDetailCount(task)
|
|
}
|
|
artifact, err := r.artifactGenerator().Create(ctx, artifactRequest(task))
|
|
if err != nil {
|
|
return gerror.Wrap(err, "generate data service result file")
|
|
}
|
|
now := gtime.Now()
|
|
err = r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
ossID, err := tx.Ctx(ctx).Model(dao.SysOss.Table()).Data(do.SysOss{
|
|
TenantId: task.Batch.TenantID,
|
|
FileName: artifact.FileName,
|
|
OriginalName: artifact.OriginalName,
|
|
FileSuffix: artifact.FileSuffix,
|
|
Path: artifact.Path,
|
|
Url: artifact.URL,
|
|
CreatedDept: task.Batch.CreatedDept,
|
|
CreatedBy: task.Batch.CreatedBy,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
Service: "local",
|
|
Md5: artifact.MD5,
|
|
FileSize: artifact.Size,
|
|
}).InsertAndGetId()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "insert data service result OSS")
|
|
}
|
|
downloadID, err := tx.Ctx(ctx).Model(dao.TResourceBatchDownload.Table()).Data(do.TResourceBatchDownload{
|
|
BatchNo: task.Batch.BatchNo,
|
|
TaskName: batchresult.TaskName(task.Batch.BatchType),
|
|
ResourceType: consts.ResourceTypeExcel,
|
|
ExportScene: task.Batch.BatchType,
|
|
TotalCount: len(task.Details),
|
|
SuccessCount: successCount,
|
|
FailCount: failCount,
|
|
DownloadStatus: consts.DownloadStatusGenerated,
|
|
OssId: ossID,
|
|
FileName: artifact.FileName,
|
|
FileExt: artifact.FileSuffix,
|
|
FileUrl: artifact.URL,
|
|
FilePath: artifact.Path,
|
|
FileSize: artifact.Size,
|
|
FileMd5: artifact.MD5,
|
|
DownloadCount: 0,
|
|
MaxDownloadCount: 0,
|
|
DeptId: task.Batch.CreatedDept,
|
|
StartTime: task.Batch.StartTime,
|
|
FinishTime: now,
|
|
TenantId: task.Batch.TenantID,
|
|
CreatedDept: task.Batch.CreatedDept,
|
|
CreatedBy: task.Batch.CreatedBy,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}).InsertAndGetId()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "insert data service result download")
|
|
}
|
|
result, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
|
Where(dao.BatchMain.Columns().BatchNo, task.Batch.BatchNo).
|
|
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
|
|
Data(do.BatchMain{
|
|
BatchAsyncStatus: batchAsyncStatus,
|
|
ValidCount: validCount,
|
|
SuccessCount: successCount,
|
|
FailCount: failCount,
|
|
RunResult: runResult,
|
|
FailReason: task.FailureReason,
|
|
BatchInfo: buildBatchInfo(task, ossID, downloadID),
|
|
EndTime: now,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}).Update()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if affected != 1 {
|
|
return gerror.New("data service batch is no longer processing")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
batchresult.RemoveLocalFile(artifact)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validDetailCount(task *data.Task) int {
|
|
if task == nil {
|
|
return 0
|
|
}
|
|
count := 0
|
|
for _, detail := range task.Details {
|
|
if detail != nil && detail.ValidationPassed {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func (r *Repository) artifactGenerator() batchresult.Generator {
|
|
if r.resultFiles != nil {
|
|
return r.resultFiles
|
|
}
|
|
return batchresult.New()
|
|
}
|
|
|
|
func artifactRequest(task *data.Task) batchresult.Request {
|
|
rows := make([]batchresult.Row, 0, len(task.Details))
|
|
for _, detail := range task.Details {
|
|
if detail == nil {
|
|
continue
|
|
}
|
|
rows = append(rows, batchresult.Row{
|
|
RowNo: detail.RowNo,
|
|
ICCID: detail.ICCID,
|
|
MSISDN: detail.MSISDN,
|
|
IMSI: detail.IMSI,
|
|
Operation: operationLabel(task.Operation),
|
|
ApnName: task.ApnName,
|
|
Success: detail.Reason == "",
|
|
Reason: detail.Reason,
|
|
UpstreamCode: detail.UpstreamCode,
|
|
OrderNum: detail.OrderNum,
|
|
})
|
|
}
|
|
return batchresult.Request{BatchNo: task.Batch.BatchNo, BatchType: task.Batch.BatchType, TenantID: task.Batch.TenantID, Rows: rows}
|
|
}
|
|
|
|
func operationLabel(operation data.Operation) string {
|
|
switch operation.BatchType {
|
|
case consts.BatchTypeDataCut:
|
|
return "达量断网"
|
|
case consts.BatchTypeDataStart:
|
|
return "开数据"
|
|
case consts.BatchTypeDataStop:
|
|
return "停数据"
|
|
default:
|
|
return operation.Name
|
|
}
|
|
}
|
|
|
|
func buildBatchInfo(task *data.Task, ossID, downloadID int64) *gjson.Json {
|
|
info := make(map[string]any)
|
|
if task.Batch.BatchInfo != nil {
|
|
for key, value := range task.Batch.BatchInfo.Map() {
|
|
info[key] = value
|
|
}
|
|
}
|
|
info["operation"] = task.Operation.Name
|
|
info["operType"] = task.Operation.ApnOperType
|
|
info["apnName"] = task.ApnName
|
|
info["resultFileOssId"] = ossID
|
|
info["resultDownloadId"] = downloadID
|
|
return gjson.New(info)
|
|
}
|
|
|
|
func batchResult(successCount, failCount int) (int, int) {
|
|
switch {
|
|
case failCount == 0:
|
|
return consts.BatchAsyncStatusSuccess, consts.BatchRunResultSuccess
|
|
case successCount == 0:
|
|
return consts.BatchAsyncStatusFailed, consts.BatchRunResultFailed
|
|
default:
|
|
return consts.BatchAsyncStatusPartialSuccess, consts.BatchRunResultPartialSuccess
|
|
}
|
|
}
|