Files
iotlcm/internal/repository/data/repository.go
T
2026-10-29 14:52:43 +08:00

627 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package data
import (
"context"
"strconv"
"strings"
"time"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/encoding/gjson"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
data "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data"
)
type Repository struct {
db gdb.DB
resultFiles batchresult.Generator
}
func New() *Repository {
return &Repository{resultFiles: batchresult.New()}
}
func (r *Repository) database() gdb.DB {
if r.db != nil {
return r.db
}
return g.DB()
}
func (r *Repository) Load(ctx context.Context, batchNo string) (*data.Task, error) {
return r.loadTask(ctx, batchNo, false)
}
func (r *Repository) loadTask(ctx context.Context, batchNo string, allDetails bool) (*data.Task, error) {
return r.loadTaskFiltered(ctx, batchNo, allDetails, nil)
}
func (r *Repository) loadTaskDetails(ctx context.Context, batchNo string, ids []int64) (*data.Task, error) {
return r.loadTaskFiltered(ctx, batchNo, true, ids)
}
func (r *Repository) loadTaskFiltered(ctx context.Context, batchNo string, allDetails bool, ids []int64) (*data.Task, error) {
if batchNo == "" {
return nil, gerror.New("data service batch number is empty")
}
batch := entity.BatchMain{}
statuses := []int{consts.BatchAsyncStatusProcessing}
if allDetails {
statuses = append(statuses, consts.BatchAsyncStatusReview)
}
if err := r.database().Model(dao.BatchMain.Table()).Safe().Ctx(ctx).
Where(dao.BatchMain.Columns().BatchNo, batchNo).
WhereIn(dao.BatchMain.Columns().BatchType, []int{consts.BatchTypeDataCut, consts.BatchTypeDataStart, consts.BatchTypeDataStop}).
WhereIn(dao.BatchMain.Columns().BatchAsyncStatus, statuses).
Scan(&batch); err != nil {
return nil, gerror.Wrap(err, "query data service batch")
}
if batch.Id == 0 {
return nil, gerror.New("processing data service batch does not exist")
}
operation, err := data.OperationByBatchType(batch.BatchType)
if err != nil {
return nil, err
}
var batchDetails []*entity.BatchDetail
detailQuery := r.database().Model(dao.BatchDetail.Table()).Safe().Ctx(ctx).
Where(dao.BatchDetail.Columns().BatchNo, batchNo).
Where(dao.BatchDetail.Columns().BatchType, batch.BatchType)
if len(ids) > 0 {
detailQuery = detailQuery.WhereIn("id", ids)
}
if !allDetails {
detailQuery = detailQuery.Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
Where("NOT EXISTS (SELECT 1 FROM data_operation_sub_detail assigned WHERE assigned.detail_id = batch_detail.id)")
}
if err = detailQuery.OrderAsc(dao.BatchDetail.Columns().RowNo).Scan(&batchDetails); err != nil {
return nil, gerror.Wrap(err, "query running data service details")
}
task := &data.Task{
Batch: data.Batch{
ID: batch.Id,
BatchNo: batch.BatchNo,
BatchType: batch.BatchType,
TenantID: batch.TenantId,
CreatedBy: batch.CreatedBy,
CreatedDept: batch.CreatedDept,
StartTime: batch.StartTime,
BatchInfo: batch.BatchInfo,
},
Operation: operation,
ApnName: apnName(batch.BatchInfo),
Details: make([]*data.Detail, 0, len(batchDetails)),
}
for _, batchDetail := range batchDetails {
if batchDetail == nil {
continue
}
detail := &data.Detail{
ID: batchDetail.Id,
DetailStatus: batchDetail.DetailStatus,
RowNo: batchDetail.RowNo,
ICCID: strings.TrimSpace(batchDetail.ItemId),
Reason: strings.TrimSpace(batchDetail.ErrorReason),
}
if batchDetail.Snapshot != nil {
if value := strings.TrimSpace(batchDetail.Snapshot.Get("ICCID").String()); value != "" {
detail.ICCID = value
}
detail.InputMSISDN = strings.TrimSpace(batchDetail.Snapshot.Get("MSISDN").String())
detail.InputIMSI = strings.TrimSpace(batchDetail.Snapshot.Get("IMSI").String())
detail.MSISDN = detail.InputMSISDN
detail.IMSI = detail.InputIMSI
detail.Authorized = batchDetail.Snapshot.Get("authorized").Bool()
detail.ValidationPassed = batchDetail.Snapshot.Get("validationPassed").Bool()
detail.ChanKey = strings.TrimSpace(batchDetail.Snapshot.Get("chanKey").String())
detail.TrafficCutoffCrossMonthHold = batchDetail.Snapshot.Get("trafficCutoffCrossMonthHold").Int()
detail.OrderNum = batchDetail.Snapshot.Get("orderNum").String()
detail.UpstreamCode = batchDetail.Snapshot.Get("upstreamCode").Int()
detail.UpstreamReason = batchDetail.Snapshot.Get("upstreamMessage").String()
detail.StatusBefore = batchDetail.Snapshot.Get("statusBefore").Int()
detail.StatusAfter = batchDetail.Snapshot.Get("statusAfter").Int()
}
task.Details = append(task.Details, detail)
}
if err = r.loadCards(ctx, task); err != nil {
return nil, err
}
return task, nil
}
func apnName(info *gjson.Json) string {
if info != nil {
if value := strings.TrimSpace(info.Get("apnName").String()); value != "" {
return value
}
}
return consts.SimApnNameDefault
}
func (r *Repository) loadCards(ctx context.Context, task *data.Task) error {
iccids := make([]string, 0, len(task.Details))
for _, detail := range task.Details {
if detail != nil && detail.ICCID != "" {
iccids = append(iccids, detail.ICCID)
}
}
if len(iccids) == 0 {
return nil
}
var cards []*entity.TSimCardInfo
if err := r.database().Model(dao.TSimCardInfo.Table()).Safe().Ctx(ctx).
Where(dao.TSimCardInfo.Columns().TenantId, task.Batch.TenantID).
WhereIn(dao.TSimCardInfo.Columns().Iccid, iccids).
Scan(&cards); err != nil {
return gerror.Wrap(err, "query data service SIM cards")
}
byICCID := make(map[string]*entity.TSimCardInfo, len(cards))
for _, card := range cards {
if card != nil {
byICCID[strings.TrimSpace(card.Iccid)] = card
}
}
for _, detail := range task.Details {
card := byICCID[detail.ICCID]
if card == nil {
continue
}
detail.Card = &data.Card{
ID: card.Id,
ICCID: strings.TrimSpace(card.Iccid),
MSISDN: strings.TrimSpace(card.Msisdn),
IMSI: strings.TrimSpace(card.Imsi),
ChanKey: preferredDataServiceChanKey(dataAppBatchUsesSnapshotChannel(task.Batch.BatchInfo), detail.ChanKey, card.ChanKey),
DataServiceStatus: card.DataServiceStatus,
}
}
return nil
}
func preferredDataServiceChanKey(useSnapshot bool, snapshotChanKey string, cardChanKey string) string {
if useSnapshot {
if value := strings.TrimSpace(snapshotChanKey); value != "" {
return value
}
}
return strings.TrimSpace(cardChanKey)
}
func dataAppBatchUsesSnapshotChannel(info *gjson.Json) bool {
if info == nil {
return false
}
return strings.EqualFold(strings.TrimSpace(info.Get("source").String()), "app")
}
func (r *Repository) RefreshDataServiceState(ctx context.Context, task *data.Task) error {
if task == nil || len(task.Details) == 0 {
return nil
}
iccids := make([]string, 0, len(task.Details))
for _, detail := range task.Details {
if detail != nil && detail.ICCID != "" {
iccids = append(iccids, detail.ICCID)
}
}
if len(iccids) == 0 {
return nil
}
type row struct {
ICCID string `orm:"iccid"`
DataServiceStatus int `orm:"data_service_status"`
}
var cards []row
if err := r.database().Model(dao.TSimCardInfo.Table()).Safe().Ctx(ctx).
Fields(dao.TSimCardInfo.Columns().Iccid, dao.TSimCardInfo.Columns().DataServiceStatus).
Where(dao.TSimCardInfo.Columns().TenantId, task.Batch.TenantID).
WhereIn(dao.TSimCardInfo.Columns().Iccid, iccids).
Scan(&cards); err != nil {
return gerror.Wrap(err, "refresh data service card state")
}
byICCID := make(map[string]int, len(cards))
for _, card := range cards {
byICCID[strings.TrimSpace(card.ICCID)] = card.DataServiceStatus
}
for _, detail := range task.Details {
if detail == nil || detail.Card == nil {
continue
}
if status, ok := byICCID[detail.ICCID]; ok {
detail.Card.DataServiceStatus = status
detail.StatusBefore = status
detail.StatusAfter = status
}
}
return nil
}
func (r *Repository) SaveValidationFailure(ctx context.Context, task *data.Task) error {
return r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
for _, detail := range task.Details {
if detail == nil {
continue
}
detail.DetailStatus = consts.BatchDetailStatusFailed
result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
Where(dao.BatchDetail.Columns().Id, detail.ID).
Where(dao.BatchDetail.Columns().BatchNo, task.Batch.BatchNo).
Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
Data(do.BatchDetail{
DetailStatus: consts.BatchDetailStatusFailed,
ErrorReason: detail.Reason,
Snapshot: detail.Snapshot(task.Operation, task.ApnName),
UpdatedBy: task.Batch.CreatedBy,
}).Update()
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != 1 {
return gerror.Newf("data service detail %d is no longer running", detail.ID)
}
}
return nil
})
}
func (r *Repository) SaveResult(ctx context.Context, task *data.Task, detail *data.Detail, outcome data.Outcome) (data.Outcome, error) {
if detail == nil {
return data.Outcome{}, gerror.New("data service detail is nil")
}
resolved := outcome
err := r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
if resolved.Success {
if detail.Card == nil {
resolved.Success = false
resolved.Message = "卡记录不存在,未办理"
} else {
result, err := tx.Ctx(ctx).Model(dao.TSimCardInfo.Table()).
Where(dao.TSimCardInfo.Columns().Id, detail.Card.ID).
Where(dao.TSimCardInfo.Columns().TenantId, task.Batch.TenantID).
Data(dataServiceUpdateData(task, detail)).Update()
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != 1 {
return gerror.Newf("data service SIM card %d is missing or changed", detail.Card.ID)
}
detail.StatusAfter = task.Operation.TargetStatus
}
}
detail.ApplyOutcome(resolved)
detailStatus := consts.BatchDetailStatusFailed
if resolved.Success {
detailStatus = consts.BatchDetailStatusSuccess
}
result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
Where(dao.BatchDetail.Columns().Id, detail.ID).
Where(dao.BatchDetail.Columns().BatchNo, task.Batch.BatchNo).
Where(dao.BatchDetail.Columns().DetailStatus, consts.BatchDetailStatusRunning).
Data(do.BatchDetail{
DetailStatus: detailStatus,
ErrorCode: strconv.Itoa(resolved.Code),
ErrorReason: detail.Reason,
Snapshot: detail.Snapshot(task.Operation, task.ApnName),
UpdatedBy: task.Batch.CreatedBy,
}).Update()
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != 1 {
return gerror.Newf("data service detail %d is no longer running", detail.ID)
}
return nil
})
return resolved, err
}
func dataServiceUpdateData(task *data.Task, detail *data.Detail) do.TSimCardTrafficCutoffConfig {
now := gtime.Now()
result := do.TSimCardTrafficCutoffConfig{
DataServiceStatus: task.Operation.TargetStatus,
StatusSyncTime: now,
UpdatedBy: task.Batch.CreatedBy,
}
switch task.Batch.BatchType {
case consts.BatchTypeDataCut:
if detail != nil && detail.TrafficCutoffCrossMonthHold == 1 {
result.TrafficCutoffTriggeredAt = now
}
case consts.BatchTypeDataStart:
result.TrafficCutoffTriggeredAt = gdb.Raw("NULL")
}
return result
}
func (r *Repository) Finalize(ctx context.Context, task *data.Task) error {
successCount, unknownCount := 0, 0
for _, detail := range task.Details {
if detail != nil && detail.DetailStatus == consts.BatchDetailStatusUnknown {
unknownCount++
} else if detail != nil && detail.DetailStatus == consts.BatchDetailStatusSuccess {
successCount++
}
}
failCount := len(task.Details) - successCount - unknownCount
batchAsyncStatus, runResult := batchResult(successCount, failCount)
if unknownCount > 0 {
batchAsyncStatus, runResult = consts.BatchAsyncStatusReview, consts.BatchRunResultReview
}
reasons := make([]string, 0, 3)
seenReasons := map[string]bool{}
for _, detail := range task.Details {
if detail != nil && detail.Reason != "" && !seenReasons[detail.Reason] && len(reasons) < 3 {
reasons = append(reasons, detail.Reason)
seenReasons[detail.Reason] = true
}
}
if len(task.Details) == 0 {
if task.FailureReason == "" {
task.FailureReason = "数据开停批次没有有效明细"
}
task.FailureReason = batchfields.FailureReason(task.FailureReason)
} else {
task.FailureReason = ""
if failCount > 0 || unknownCount > 0 {
task.FailureReason = batchfields.FailureReason("成功=" + strconv.Itoa(successCount) + ",明确失败=" + strconv.Itoa(failCount) + ",待核查=" + strconv.Itoa(unknownCount) + ";" + strings.Join(reasons, ";"))
}
}
validCount := 0
if task.ValidationOK {
validCount = validDetailCount(task)
}
artifact, err := r.artifactGenerator().Create(ctx, artifactRequest(task))
if err != nil {
return gerror.Wrap(err, "generate data service result file")
}
now := gtime.Now()
err = r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
var main struct {
Status int `orm:"batch_async_status"`
}
if err := r.subLock(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("batch_async_status").Where("batch_no", task.Batch.BatchNo)).Scan(&main); err != nil {
return err
}
if main.Status != consts.BatchAsyncStatusProcessing && !(ctx.Value(finalizeTokenKey{}) != nil && main.Status == consts.BatchAsyncStatusReview) {
return gerror.New("data service batch is no longer processing")
}
if token, ok := ctx.Value(finalizeTokenKey{}).(string); ok {
var lease struct {
Token string `orm:"finalize_token"`
}
if err := r.subLock(tx.Ctx(ctx).Model(parentTable).Fields("finalize_token").Where("batch_no", task.Batch.BatchNo).WhereGT("finalize_until", time.Now())).Scan(&lease); err != nil {
return err
}
if lease.Token != token {
return gerror.New("APN最终汇总租约已过期")
}
}
ossID, err := tx.Ctx(ctx).Model(dao.SysOss.Table()).Data(do.SysOss{
TenantId: task.Batch.TenantID,
FileName: artifact.FileName,
OriginalName: artifact.OriginalName,
FileSuffix: artifact.FileSuffix,
Path: artifact.Path,
Url: artifact.URL,
CreatedDept: task.Batch.CreatedDept,
CreatedBy: task.Batch.CreatedBy,
UpdatedBy: task.Batch.CreatedBy,
Service: "local",
Md5: artifact.MD5,
FileSize: artifact.Size,
}).InsertAndGetId()
if err != nil {
return gerror.Wrap(err, "insert data service result OSS")
}
downloadID, err := tx.Ctx(ctx).Model(dao.TResourceBatchDownload.Table()).Data(do.TResourceBatchDownload{
BatchNo: task.Batch.BatchNo,
TaskName: batchresult.TaskName(task.Batch.BatchType),
ResourceType: consts.ResourceTypeExcel,
ExportScene: task.Batch.BatchType,
TotalCount: len(task.Details),
SuccessCount: successCount,
FailCount: failCount,
DownloadStatus: consts.DownloadStatusGenerated,
OssId: ossID,
FileName: artifact.FileName,
FileExt: artifact.FileSuffix,
FileUrl: artifact.URL,
FilePath: artifact.Path,
FileSize: artifact.Size,
FileMd5: artifact.MD5,
DownloadCount: 0,
MaxDownloadCount: 0,
DeptId: task.Batch.CreatedDept,
StartTime: task.Batch.StartTime,
FinishTime: now,
TenantId: task.Batch.TenantID,
CreatedDept: task.Batch.CreatedDept,
CreatedBy: task.Batch.CreatedBy,
UpdatedBy: task.Batch.CreatedBy,
}).InsertAndGetId()
if err != nil {
return gerror.Wrap(err, "insert data service result download")
}
allowedStatuses := []int{consts.BatchAsyncStatusProcessing}
if ctx.Value(finalizeTokenKey{}) != nil {
allowedStatuses = append(allowedStatuses, consts.BatchAsyncStatusReview)
}
result, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
Where(dao.BatchMain.Columns().BatchNo, task.Batch.BatchNo).
WhereIn(dao.BatchMain.Columns().BatchAsyncStatus, allowedStatuses).
Data(do.BatchMain{
BatchAsyncStatus: batchAsyncStatus,
ValidCount: validCount,
SuccessCount: successCount,
FailCount: failCount,
RunResult: runResult,
FailReason: task.FailureReason,
BatchInfo: buildBatchInfo(task, ossID, downloadID),
EndTime: now,
UpdatedBy: task.Batch.CreatedBy,
}).Update()
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != 1 {
return gerror.New("data service batch is no longer processing")
}
if token, ok := ctx.Value(finalizeTokenKey{}).(string); ok {
_, err = tx.Ctx(ctx).Model(parentTable).Where("batch_no", task.Batch.BatchNo).Where("finalize_token", token).Data(g.Map{"finalize_token": gdb.Raw("NULL"), "finalize_until": gdb.Raw("NULL")}).Update()
return err
}
return nil
})
if err != nil {
batchresult.RemoveLocalFile(artifact)
return err
}
return nil
}
const subDetailOwnershipTable = "data_operation_sub_detail"
// markFrozenValidation 以“是否已被冻结进持久化子批”判定明细是否通过提交前校验。
// 只有通过校验的明细才会被冻结提交,因此冻结归属才是有效数的可信依据;
// authorized 只表示运营商授权范围,不能当作校验通过的证据。
func (r *Repository) markFrozenValidation(ctx context.Context, task *data.Task) error {
if task == nil || len(task.Details) == 0 {
return nil
}
var rows []struct {
DetailID int64 `orm:"detail_id"`
}
if err := r.database().Model(subDetailOwnershipTable).Ctx(ctx).
Fields("detail_id").Where("batch_no", task.Batch.BatchNo).Scan(&rows); err != nil {
return gerror.Wrap(err, "query frozen data service detail ownership")
}
frozen := make(map[int64]struct{}, len(rows))
for _, row := range rows {
frozen[row.DetailID] = struct{}{}
}
for _, detail := range task.Details {
if detail == nil {
continue
}
_, owned := frozen[detail.ID]
detail.ValidationPassed = owned
}
return nil
}
func validDetailCount(task *data.Task) int {
if task == nil {
return 0
}
count := 0
for _, detail := range task.Details {
if detail != nil && detail.ValidationPassed {
count++
}
}
return count
}
func (r *Repository) artifactGenerator() batchresult.Generator {
if r.resultFiles != nil {
return r.resultFiles
}
return batchresult.New()
}
func artifactRequest(task *data.Task) batchresult.Request {
rows := make([]batchresult.Row, 0, len(task.Details))
for _, detail := range task.Details {
if detail == nil {
continue
}
rows = append(rows, batchresult.Row{
RowNo: detail.RowNo,
DetailStatus: detail.DetailStatus,
ICCID: detail.ICCID,
MSISDN: detail.MSISDN,
IMSI: detail.IMSI,
Operation: operationLabel(task.Operation),
ApnName: task.ApnName,
Success: detail.DetailStatus == consts.BatchDetailStatusSuccess,
Reason: detail.Reason,
UpstreamCode: detail.UpstreamCode,
OrderNum: detail.OrderNum,
})
}
return batchresult.Request{BatchNo: task.Batch.BatchNo, BatchType: task.Batch.BatchType, TenantID: task.Batch.TenantID, Rows: rows}
}
func operationLabel(operation data.Operation) string {
switch operation.BatchType {
case consts.BatchTypeDataCut:
return "达量断网"
case consts.BatchTypeDataStart:
return "开数据"
case consts.BatchTypeDataStop:
return "停数据"
default:
return operation.Name
}
}
func buildBatchInfo(task *data.Task, ossID, downloadID int64) *gjson.Json {
info := make(map[string]any)
if task.Batch.BatchInfo != nil {
for key, value := range task.Batch.BatchInfo.Map() {
info[key] = value
}
}
info["operation"] = task.Operation.Name
info["operType"] = task.Operation.ApnOperType
info["apnName"] = task.ApnName
unknownCount := 0
for _, detail := range task.Details {
if detail != nil && detail.DetailStatus == consts.BatchDetailStatusUnknown {
unknownCount++
}
}
info["apnUnknownCount"] = unknownCount
info["resultFileOssId"] = ossID
info["resultDownloadId"] = downloadID
return gjson.New(info)
}
func batchResult(successCount, failCount int) (int, int) {
switch {
case successCount == 0:
return consts.BatchAsyncStatusFailed, consts.BatchRunResultFailed
case failCount == 0:
return consts.BatchAsyncStatusSuccess, consts.BatchRunResultSuccess
default:
return consts.BatchAsyncStatusPartialSuccess, consts.BatchRunResultPartialSuccess
}
}