iotlcm-public/iotlcm
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: 同步 v1.0.4 开源代码并脱敏

Browse Source
This commit is contained in:
MiFaZhan
2026-10-29 14:52:43 +08:00
parent 32e4316ee4
commit 6893e21f07
136 changed files with 9851 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/repository/data/repository.go
+138 -26
View File
@@ -4,6 +4,7 @@ import (
"context"
"strconv"
"strings"
"time"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/encoding/gjson"
@@ -13,6 +14,7 @@ import (
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
@@ -36,14 +38,30 @@ func (r *Repository) database() gdb.DB {
}
func (r *Repository) Load(ctx context.Context, batchNo string) (*data.Task, error) {
return r.loadTask(ctx, batchNo, false)
}
func (r *Repository) loadTask(ctx context.Context, batchNo string, allDetails bool) (*data.Task, error) {
return r.loadTaskFiltered(ctx, batchNo, allDetails, nil)
}
func (r *Repository) loadTaskDetails(ctx context.Context, batchNo string, ids []int64) (*data.Task, error) {
return r.loadTaskFiltered(ctx, batchNo, true, ids)
}
func (r *Repository) loadTaskFiltered(ctx context.Context, batchNo string, allDetails bool, ids []int64) (*data.Task, error) {
if batchNo == "" {
return nil, gerror.New("data service batch number is empty")
}
batch := entity.BatchMain{}
statuses := []int{consts.BatchAsyncStatusProcessing}
if allDetails {
statuses = append(statuses, consts.BatchAsyncStatusReview)
}
if err := r.database().Model(dao.BatchMain.Table()).Safe().Ctx(ctx).
Where(dao.BatchMain.Columns().BatchNo, batchNo).
WhereIn(dao.BatchMain.Columns().BatchType, []int{consts.BatchTypeDataCut, consts.BatchTypeDataStart, consts.BatchTypeDataStop}).
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
WhereIn(dao.BatchMain.Columns().BatchAsyncStatus, statuses).
Scan(&batch); err != nil {
return nil, gerror.Wrap(err, "query data service batch")
}
@@ -56,16 +74,18 @@ func (r *Repository) Load(ctx context.Context, batchNo string) (*data.Task, erro
}
var batchDetails []*entity.BatchDetail
if err = r.database().Model(dao.BatchDetail.Table()).Safe().Ctx(ctx).
detailQuery := r.database().Model(dao.BatchDetail.Table()).Safe().Ctx(ctx).
Where(dao.BatchDetail.Columns().BatchNo, batchNo).
Where(dao.BatchDetail.Columns().BatchType, batch.BatchType).
Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
OrderAsc(dao.BatchDetail.Columns().RowNo).
Scan(&batchDetails); err != nil {
return nil, gerror.Wrap(err, "query running data service details")
Where(dao.BatchDetail.Columns().BatchType, batch.BatchType)
if len(ids) > 0 {
detailQuery = detailQuery.WhereIn("id", ids)
}
if len(batchDetails) == 0 {
return nil, gerror.New("data service batch has no running details")
if !allDetails {
detailQuery = detailQuery.Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
Where("NOT EXISTS (SELECT 1 FROM data_operation_sub_detail assigned WHERE assigned.detail_id = batch_detail.id)")
}
if err = detailQuery.OrderAsc(dao.BatchDetail.Columns().RowNo).Scan(&batchDetails); err != nil {
return nil, gerror.Wrap(err, "query running data service details")
}
task := &data.Task{
@@ -88,10 +108,11 @@ func (r *Repository) Load(ctx context.Context, batchNo string) (*data.Task, erro
continue
}
detail := &data.Detail{
ID: batchDetail.Id,
RowNo: batchDetail.RowNo,
ICCID: strings.TrimSpace(batchDetail.ItemId),
Reason: strings.TrimSpace(batchDetail.ErrorReason),
ID: batchDetail.Id,
DetailStatus: batchDetail.DetailStatus,
RowNo: batchDetail.RowNo,
ICCID: strings.TrimSpace(batchDetail.ItemId),
Reason: strings.TrimSpace(batchDetail.ErrorReason),
}
if batchDetail.Snapshot != nil {
if value := strings.TrimSpace(batchDetail.Snapshot.Get("ICCID").String()); value != "" {
@@ -102,14 +123,17 @@ func (r *Repository) Load(ctx context.Context, batchNo string) (*data.Task, erro
detail.MSISDN = detail.InputMSISDN
detail.IMSI = detail.InputIMSI
detail.Authorized = batchDetail.Snapshot.Get("authorized").Bool()
detail.ValidationPassed = batchDetail.Snapshot.Get("validationPassed").Bool()
detail.ChanKey = strings.TrimSpace(batchDetail.Snapshot.Get("chanKey").String())
detail.TrafficCutoffCrossMonthHold = batchDetail.Snapshot.Get("trafficCutoffCrossMonthHold").Int()
detail.OrderNum = batchDetail.Snapshot.Get("orderNum").String()
detail.UpstreamCode = batchDetail.Snapshot.Get("upstreamCode").Int()
detail.UpstreamReason = batchDetail.Snapshot.Get("upstreamMessage").String()
detail.StatusBefore = batchDetail.Snapshot.Get("statusBefore").Int()
detail.StatusAfter = batchDetail.Snapshot.Get("statusAfter").Int()
}
task.Details = append(task.Details, detail)
}
if len(task.Details) == 0 {
return nil, gerror.New("data service batch has no readable running details")
}
if err = r.loadCards(ctx, task); err != nil {
return nil, err
}
@@ -229,6 +253,7 @@ func (r *Repository) SaveValidationFailure(ctx context.Context, task *data.Task)
if detail == nil {
continue
}
detail.DetailStatus = consts.BatchDetailStatusFailed
result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
Where(dao.BatchDetail.Columns().Id, detail.ID).
Where(dao.BatchDetail.Columns().BatchNo, task.Batch.BatchNo).
@@ -263,7 +288,7 @@ func (r *Repository) SaveResult(ctx context.Context, task *data.Task, detail *da
if resolved.Success {
if detail.Card == nil {
resolved.Success = false
resolved.Message = "SIM card record is missing"
resolved.Message = "卡记录不存在，未办理"
} else {
result, err := tx.Ctx(ctx).Model(dao.TSimCardInfo.Table()).
Where(dao.TSimCardInfo.Columns().Id, detail.Card.ID).
@@ -332,16 +357,37 @@ func dataServiceUpdateData(task *data.Task, detail *data.Detail) do.TSimCardTraf
}
func (r *Repository) Finalize(ctx context.Context, task *data.Task) error {
successCount := 0
successCount, unknownCount := 0, 0
for _, detail := range task.Details {
if detail != nil && detail.Reason == "" {
if detail != nil && detail.DetailStatus == consts.BatchDetailStatusUnknown {
unknownCount++
} else if detail != nil && detail.DetailStatus == consts.BatchDetailStatusSuccess {
successCount++
}
}
failCount := len(task.Details) - successCount
failCount := len(task.Details) - successCount - unknownCount
batchAsyncStatus, runResult := batchResult(successCount, failCount)
if task.FailureReason == "" && failCount > 0 {
task.FailureReason = strconv.Itoa(failCount) + " SIM cards failed"
if unknownCount > 0 {
batchAsyncStatus, runResult = consts.BatchAsyncStatusReview, consts.BatchRunResultReview
}
reasons := make([]string, 0, 3)
seenReasons := map[string]bool{}
for _, detail := range task.Details {
if detail != nil && detail.Reason != "" && !seenReasons[detail.Reason] && len(reasons) < 3 {
reasons = append(reasons, detail.Reason)
seenReasons[detail.Reason] = true
}
}
if len(task.Details) == 0 {
if task.FailureReason == "" {
task.FailureReason = "数据开停批次没有有效明细"
}
task.FailureReason = batchfields.FailureReason(task.FailureReason)
} else {
task.FailureReason = ""
if failCount > 0 || unknownCount > 0 {
task.FailureReason = batchfields.FailureReason("成功=" + strconv.Itoa(successCount) + "，明确失败=" + strconv.Itoa(failCount) + "，待核查=" + strconv.Itoa(unknownCount) + "；" + strings.Join(reasons, "；"))
}
}
validCount := 0
if task.ValidationOK {
@@ -353,6 +399,26 @@ func (r *Repository) Finalize(ctx context.Context, task *data.Task) error {
}
now := gtime.Now()
err = r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
var main struct {
Status int `orm:"batch_async_status"`
}
if err := r.subLock(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("batch_async_status").Where("batch_no", task.Batch.BatchNo)).Scan(&main); err != nil {
return err
}
if main.Status != consts.BatchAsyncStatusProcessing && !(ctx.Value(finalizeTokenKey{}) != nil && main.Status == consts.BatchAsyncStatusReview) {
return gerror.New("data service batch is no longer processing")
}
if token, ok := ctx.Value(finalizeTokenKey{}).(string); ok {
var lease struct {
Token string `orm:"finalize_token"`
}
if err := r.subLock(tx.Ctx(ctx).Model(parentTable).Fields("finalize_token").Where("batch_no", task.Batch.BatchNo).WhereGT("finalize_until", time.Now())).Scan(&lease); err != nil {
return err
}
if lease.Token != token {
return gerror.New("APN最终汇总租约已过期")
}
}
ossID, err := tx.Ctx(ctx).Model(dao.SysOss.Table()).Data(do.SysOss{
TenantId: task.Batch.TenantID,
FileName: artifact.FileName,
@@ -399,9 +465,13 @@ func (r *Repository) Finalize(ctx context.Context, task *data.Task) error {
if err != nil {
return gerror.Wrap(err, "insert data service result download")
}
allowedStatuses := []int{consts.BatchAsyncStatusProcessing}
if ctx.Value(finalizeTokenKey{}) != nil {
allowedStatuses = append(allowedStatuses, consts.BatchAsyncStatusReview)
}
result, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
Where(dao.BatchMain.Columns().BatchNo, task.Batch.BatchNo).
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
WhereIn(dao.BatchMain.Columns().BatchAsyncStatus, allowedStatuses).
Data(do.BatchMain{
BatchAsyncStatus: batchAsyncStatus,
ValidCount: validCount,
@@ -423,6 +493,10 @@ func (r *Repository) Finalize(ctx context.Context, task *data.Task) error {
if affected != 1 {
return gerror.New("data service batch is no longer processing")
}
if token, ok := ctx.Value(finalizeTokenKey{}).(string); ok {
_, err = tx.Ctx(ctx).Model(parentTable).Where("batch_no", task.Batch.BatchNo).Where("finalize_token", token).Data(g.Map{"finalize_token": gdb.Raw("NULL"), "finalize_until": gdb.Raw("NULL")}).Update()
return err
}
return nil
})
if err != nil {
@@ -432,6 +506,36 @@ func (r *Repository) Finalize(ctx context.Context, task *data.Task) error {
return nil
}
const subDetailOwnershipTable = "data_operation_sub_detail"
// markFrozenValidation 以“是否已被冻结进持久化子批”判定明细是否通过提交前校验。
// 只有通过校验的明细才会被冻结提交，因此冻结归属才是有效数的可信依据；
// authorized 只表示运营商授权范围，不能当作校验通过的证据。
func (r *Repository) markFrozenValidation(ctx context.Context, task *data.Task) error {
if task == nil || len(task.Details) == 0 {
return nil
}
var rows []struct {
DetailID int64 `orm:"detail_id"`
}
if err := r.database().Model(subDetailOwnershipTable).Ctx(ctx).
Fields("detail_id").Where("batch_no", task.Batch.BatchNo).Scan(&rows); err != nil {
return gerror.Wrap(err, "query frozen data service detail ownership")
}
frozen := make(map[int64]struct{}, len(rows))
for _, row := range rows {
frozen[row.DetailID] = struct{}{}
}
for _, detail := range task.Details {
if detail == nil {
continue
}
_, owned := frozen[detail.ID]
detail.ValidationPassed = owned
}
return nil
}
func validDetailCount(task *data.Task) int {
if task == nil {
return 0
@@ -460,12 +564,13 @@ func artifactRequest(task *data.Task) batchresult.Request {
}
rows = append(rows, batchresult.Row{
RowNo: detail.RowNo,
DetailStatus: detail.DetailStatus,
ICCID: detail.ICCID,
MSISDN: detail.MSISDN,
IMSI: detail.IMSI,
Operation: operationLabel(task.Operation),
ApnName: task.ApnName,
Success: detail.Reason == "",
Success: detail.DetailStatus == consts.BatchDetailStatusSuccess,
Reason: detail.Reason,
UpstreamCode: detail.UpstreamCode,
OrderNum: detail.OrderNum,
@@ -497,6 +602,13 @@ func buildBatchInfo(task *data.Task, ossID, downloadID int64) *gjson.Json {
info["operation"] = task.Operation.Name
info["operType"] = task.Operation.ApnOperType
info["apnName"] = task.ApnName
unknownCount := 0
for _, detail := range task.Details {
if detail != nil && detail.DetailStatus == consts.BatchDetailStatusUnknown {
unknownCount++
}
}
info["apnUnknownCount"] = unknownCount
info["resultFileOssId"] = ossID
info["resultDownloadId"] = downloadID
return gjson.New(info)
@@ -504,10 +616,10 @@ func buildBatchInfo(task *data.Task, ossID, downloadID int64) *gjson.Json {
func batchResult(successCount, failCount int) (int, int) {
switch {
case failCount == 0:
return consts.BatchAsyncStatusSuccess, consts.BatchRunResultSuccess
case successCount == 0:
return consts.BatchAsyncStatusFailed, consts.BatchRunResultFailed
case failCount == 0:
return consts.BatchAsyncStatusSuccess, consts.BatchRunResultSuccess
default:
return consts.BatchAsyncStatusPartialSuccess, consts.BatchRunResultPartialSuccess
}