package allocate import ( "bufio" "context" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata" "strconv" "strings" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/excel" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/iot/simcardbatch/common" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/do" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" ) const ( defaultAllocateChunkSize = 5000 defaultAllocateMaxFileSize = 50 << 20 minAllocateChunkSize = 100 maxAllocateChunkSize = 10000 allocateDetailInsertSize = 500 allocateParentJobRole = "allocate_parent" allocateChunkJobRole = "allocate_chunk" ) type allocateSplitState struct { ParentBatchNo string ChunkSize int ChunkCount int Total int Valid int Failed int } type allocateChunkStream func(context.Context, func([]batchimport.Row) error) (int, error) func allocateChunkSize(ctx context.Context) int { size := g.Cfg().MustGet(ctx, "batch.allocate.chunkSize", defaultAllocateChunkSize).Int() if size < minAllocateChunkSize { return minAllocateChunkSize } if size > maxAllocateChunkSize { return maxAllocateChunkSize } return size } func allocateMaxFileSize(ctx context.Context) int64 { size := g.Cfg().MustGet(ctx, "batch.allocate.maxFileSize", defaultAllocateMaxFileSize).Int64() if size <= 0 { return defaultAllocateMaxFileSize } return size } func createPendingSplit( ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, options batchimport.PendingOptions, parentBatchNo string, importOss *model.SysOssSaveContentModel, stream allocateChunkStream, ) (*PendingResult, error) { units, err := allocationunits.Snapshot(ctx, req.Operator.TenantId, req.Operator.DeptId, req.TargetDeptId) if err != nil { return nil, err } if opts, ok := req.BizParams.(*SubmitOptions); ok { opts.Units = units } if err := batchmetadata.ValidateCreation(req.SourceChannel, req.ExecutionMode); err != nil { return nil, err } if p == nil { return nil, gerror.New("allocate processor is required") } if strings.TrimSpace(parentBatchNo) == "" { return nil, gerror.New("parent batch number is required") } if stream == nil { return nil, gerror.New("allocate input stream is required") } chunkSize := allocateChunkSize(ctx) state := &allocateSplitState{ ParentBatchNo: parentBatchNo, ChunkSize: chunkSize, } if err := createAllocateParent(ctx, req, p, parentBatchNo, importOss, chunkSize); err != nil { return nil, err } if _, err := stream(ctx, func(rows []batchimport.Row) error { if len(rows) == 0 { return nil } if err := p.ValidateRows(ctx, rows); err != nil { return err } if options.PrepareRows != nil { if err := options.PrepareRows(ctx, rows); err != nil { return err } } if err := markAllocateCrossChunkDuplicates(ctx, parentBatchNo, req.Operator.TenantId, rows); err != nil { return err } state.ChunkCount++ result, err := createAllocateChunk(ctx, req, p, parentBatchNo, state.ChunkCount, rows, importOss, state) if err != nil { return err } return common.PublishBatchMessageStrict(ctx, result, consts.QueueBatchExecute, "allocate") }); err != nil { markAllocateParentFailed(ctx, parentBatchNo, err.Error()) return nil, err } if state.Total == 0 { err := gerror.New("allocate input has no data rows") markAllocateParentFailed(ctx, parentBatchNo, err.Error()) return nil, err } if err := markAllocateSplitCompleted(ctx, parentBatchNo, state); err != nil { markAllocateParentFailed(ctx, parentBatchNo, err.Error()) return nil, err } parentMessage := &batchimport.Result{ BatchNo: parentBatchNo, BatchType: consts.BatchTypeAllocate, } if err := common.PublishBatchMessageStrict(ctx, parentMessage, consts.QueueBatchExecute, "allocate_parent"); err != nil { markAllocateParentFailed(ctx, parentBatchNo, err.Error()) return nil, err } return &PendingResult{ BatchNo: parentBatchNo, BatchType: consts.BatchTypeAllocate, BatchAsyncStatus: consts.BatchAsyncStatusProcessing, Total: state.Total, ValidCount: state.Valid, FailCount: state.Failed, }, nil } func createAllocateParent(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, importOss *model.SysOssSaveContentModel, chunkSize int) error { result := &batchimport.Result{ BatchNo: parentBatchNo, BatchType: consts.BatchTypeAllocate, ImportFileOssId: ossIDFromContent(importOss), ImportFileUrl: urlFromContent(importOss), } info := p.BuildBatchInfo(ctx, req, result, importOss, nil, 0) info = withAllocateInfo(info, map[string]any{ "targetBatchType": consts.BatchTypeAllocate, "jobRole": allocateParentJobRole, "autoSplit": true, "chunkSize": chunkSize, "chunkCount": 0, "completedChunkCount": 0, "parsedCount": 0, }) now := gtime.Now() _, err := g.DB().Model(dao.BatchMain.Table()).Data(do.BatchMain{ BatchNo: parentBatchNo, OrderSn: "", BatchType: consts.BatchTypeAllocate, BatchAsyncStatus: consts.BatchAsyncStatusPending, BatchStatus: consts.BatchStatusNormal, StartTime: now, SourceType: req.SourceType, SourceChannel: batchmetadata.Channel(req.SourceChannel), ExecutionMode: req.ExecutionMode, SourceOssId: ossIDFromContent(importOss), IsCrossDept: 0, TargetDeptId: req.TargetDeptId, DeptId: req.Operator.DeptId, TotalCount: 0, ValidCount: 0, SuccessCount: 0, FailCount: 0, RunResult: consts.BatchRunResultPending, BatchInfo: info, TenantId: req.Operator.TenantId, CreatedDept: req.Operator.DeptId, CreatedBy: req.Operator.UserId, CreatedAt: now, UpdatedBy: req.Operator.UserId, UpdatedAt: now, }).Insert() if err != nil { return gerror.Wrap(err, "insert allocate parent batch failed") } return nil } func createAllocateChunk(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, chunkNo int, rows []batchimport.Row, importOss *model.SysOssSaveContentModel, state *allocateSplitState) (*batchimport.Result, error) { childBatchNo, err := batchimport.NextRandomNo(consts.BatchNoPrefixAllocate) if err != nil { return nil, err } validCount, failCount := countBatchRows(rows) result := &batchimport.Result{ ImportStatus: consts.ImportStatusSuccess, BatchNo: childBatchNo, BatchType: consts.BatchTypeAllocate, TotalRows: len(rows), ValidRows: validCount, ErrorRows: failCount, ImportFileOssId: ossIDFromContent(importOss), ImportFileUrl: urlFromContent(importOss), } childReq := req childReq.BatchNo = childBatchNo childReq.SourceBatchNo = parentBatchNo info := p.BuildBatchInfo(ctx, childReq, result, importOss, nil, 0) info = withAllocateInfo(info, map[string]any{ "targetBatchType": consts.BatchTypeAllocate, "jobRole": allocateChunkJobRole, "parentBatchNo": parentBatchNo, "chunkNo": chunkNo, "chunkSize": len(rows), }) now := gtime.Now() err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Data(do.BatchMain{ BatchNo: childBatchNo, OrderSn: "", BatchType: consts.BatchTypeAllocate, BatchAsyncStatus: consts.BatchAsyncStatusPending, BatchStatus: consts.BatchStatusNormal, SourceType: req.SourceType, SourceChannel: batchmetadata.Channel(req.SourceChannel), ExecutionMode: req.ExecutionMode, SourceBatchNo: parentBatchNo, SourceOssId: ossIDFromContent(importOss), IsCrossDept: 0, TargetDeptId: req.TargetDeptId, DeptId: req.Operator.DeptId, TotalCount: len(rows), ValidCount: validCount, SuccessCount: 0, FailCount: failCount, RunResult: consts.BatchRunResultPending, BatchInfo: info, TenantId: req.Operator.TenantId, CreatedDept: req.Operator.DeptId, CreatedBy: req.Operator.UserId, CreatedAt: now, UpdatedBy: req.Operator.UserId, UpdatedAt: now, }).Insert(); err != nil { return gerror.Wrap(err, "insert allocate child batch failed") } details := make([]do.BatchDetail, 0, len(rows)) for _, row := range rows { status := consts.BatchDetailStatusPending if len(row.GetErrorReasons()) > 0 { status = consts.BatchDetailStatusFailed } details = append(details, do.BatchDetail{ BatchNo: childBatchNo, BatchType: consts.BatchTypeAllocate, DetailStatus: status, BlankOutStatus: consts.BlankOutStatusNo, RowNo: row.GetRowNo(), ItemId: row.GetItemID(), ErrorReason: strings.Join(row.GetErrorReasons(), "; "), Snapshot: allocationunits.Merge(row.Snapshot(), sourceUnitInfo(info)), TenantId: req.Operator.TenantId, CreatedDept: req.Operator.DeptId, CreatedBy: req.Operator.UserId, CreatedAt: now, UpdatedBy: req.Operator.UserId, UpdatedAt: now, }) } for start := 0; start < len(details); start += allocateDetailInsertSize { end := start + allocateDetailInsertSize if end > len(details) { end = len(details) } if _, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Data(details[start:end]).Insert(); err != nil { return gerror.Wrapf(err, "insert allocate details failed, range:%d-%d", start+1, end) } } parent := &entity.BatchMain{} columns := dao.BatchMain.Columns() if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()). Where(columns.BatchNo, parentBatchNo). Where(columns.BatchType, consts.BatchTypeAllocate). LockUpdate().Scan(parent); err != nil { return gerror.Wrap(err, "load allocate parent batch failed") } if parent.Id == 0 { return gerror.New("allocate parent batch does not exist") } parentInfo := withAllocateInfo(parent.BatchInfo, map[string]any{ "chunkCount": chunkNo, "parsedCount": parent.TotalCount + len(rows), }) if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{ TotalCount: parent.TotalCount + len(rows), ValidCount: parent.ValidCount + validCount, FailCount: parent.FailCount + failCount, BatchInfo: parentInfo, UpdatedBy: req.Operator.UserId, UpdatedAt: now, }).Update(); err != nil { return gerror.Wrap(err, "update allocate parent split progress failed") } return nil }) if err != nil { return nil, err } if state != nil { state.Total += len(rows) state.Valid += validCount state.Failed += failCount } return result, nil } func markAllocateSplitCompleted(ctx context.Context, parentBatchNo string, state *allocateSplitState) error { columns := dao.BatchMain.Columns() parent := &entity.BatchMain{} if err := g.DB().Model(dao.BatchMain.Table()). Where(columns.BatchNo, parentBatchNo). Where(columns.BatchType, consts.BatchTypeAllocate). Scan(parent); err != nil { return err } if parent.Id == 0 { return gerror.New("allocate parent batch does not exist") } info := withAllocateInfo(parent.BatchInfo, map[string]any{ "splitCompleted": true, "chunkCount": state.ChunkCount, "completedChunkCount": 0, "parsedCount": state.Total, }) _, err := g.DB().Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{ BatchAsyncStatus: consts.BatchAsyncStatusProcessing, RunResult: consts.BatchRunResultRunning, BatchInfo: info, UpdatedAt: gtime.Now(), }).Update() return err } func markAllocateParentFailed(ctx context.Context, parentBatchNo, reason string) { if strings.TrimSpace(parentBatchNo) == "" { return } now := gtime.Now() columns := dao.BatchMain.Columns() _, _ = g.DB().Model(dao.BatchMain.Table()). Where(columns.BatchNo, parentBatchNo). Where(columns.BatchType, consts.BatchTypeAllocate). WhereIn(columns.BatchAsyncStatus, []int{consts.BatchAsyncStatusPending, consts.BatchAsyncStatusProcessing}). Data(do.BatchMain{ BatchAsyncStatus: consts.BatchAsyncStatusFailed, RunResult: consts.BatchRunResultFailed, FailReason: strings.TrimSpace(reason), EndTime: now, UpdatedAt: now, }).Update() } func countBatchRows(rows []batchimport.Row) (valid, failed int) { for _, row := range rows { if len(row.GetErrorReasons()) == 0 { valid++ } else { failed++ } } return } func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenantID string, rows []batchimport.Row) error { if strings.TrimSpace(parentBatchNo) == "" || len(rows) == 0 { return nil } columns := dao.BatchDetail.Columns() itemIDs := make([]string, 0, len(rows)) seen := make(map[string]struct{}, len(rows)) for _, item := range rows { row, ok := item.(*Row) if !ok || row == nil { continue } itemID := strings.TrimSpace(row.Iccid) if itemID == "" { continue } if _, exists := seen[itemID]; exists { continue } seen[itemID] = struct{}{} itemIDs = append(itemIDs, itemID) } if len(itemIDs) == 0 { return nil } var existing []struct { ItemId string `orm:"item_id"` } query := g.DB().Model(dao.BatchDetail.Table()). Fields(columns.ItemId). Where(columns.BatchType, consts.BatchTypeAllocate). Where(columns.TenantId, tenantID). WhereIn(columns.ItemId, itemIDs). Where("batch_no IN (SELECT batch_no FROM batch_main WHERE source_batch_no = ? AND batch_type = ? AND tenant_id = ?)", parentBatchNo, consts.BatchTypeAllocate, tenantID) if err := query.Scan(&existing); err != nil { return gerror.Wrap(err, "check duplicate allocate ICCIDs across chunks failed") } if len(existing) == 0 { return nil } duplicates := make(map[string]struct{}, len(existing)) for _, item := range existing { if itemID := strings.TrimSpace(item.ItemId); itemID != "" { duplicates[itemID] = struct{}{} } } for _, item := range rows { row, ok := item.(*Row) if !ok || row == nil { continue } if _, exists := duplicates[strings.TrimSpace(row.Iccid)]; exists { row.AddErrorReason("ICCID appears in an earlier allocation chunk") } } return nil } func withAllocateInfo(info *gjson.Json, values map[string]any) *gjson.Json { data := make(map[string]any) if info != nil { for key, value := range info.Map() { data[key] = value } } for key, value := range values { data[key] = value } return gjson.New(data) } func ossIDFromContent(content *model.SysOssSaveContentModel) int64 { if content == nil || content.SysOss == nil { return 0 } return content.OssId } func urlFromContent(content *model.SysOssSaveContentModel) string { if content == nil || content.SysOss == nil { return "" } return content.Url } func streamAllocateExcel(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) { return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) { return excel.StreamFile(ctx, filePath, excel.ParseOptions{ RequiredHeaders: []string{"ICCID"}, SingleSheet: true, }, handler) }) } func streamAllocateCSV(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) { return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) { return excel.StreamCSVFile(ctx, filePath, excel.ParseOptions{ RequiredHeaders: []string{"ICCID"}, }, handler) }) } func streamAllocateParsedFile(ctx context.Context, chunkSize int, emit func([]batchimport.Row) error, stream func(func(excel.ParsedRow) error) (int, error)) (int, error) { rows := make([]batchimport.Row, 0, chunkSize) total, err := stream(func(parsed excel.ParsedRow) error { rows = append(rows, rowFromExcel(parsed)) if len(rows) < chunkSize { return nil } if err := emit(rows); err != nil { return err } rows = make([]batchimport.Row, 0, chunkSize) return nil }) if err != nil { return total, err } if len(rows) > 0 { if err := emit(rows); err != nil { return total, err } } return total, nil } func streamAllocateInput(ctx context.Context, input string, targetDeptID int64, chunkSize int, emit func([]batchimport.Row) error) (int, error) { scanner := bufio.NewScanner(strings.NewReader(input)) scanner.Buffer(make([]byte, 64*1024), 1024*1024) rows := make([]batchimport.Row, 0, chunkSize) total := 0 rowNo := 0 for scanner.Scan() { rowNo++ if err := ctx.Err(); err != nil { return total, err } line := strings.TrimSpace(scanner.Text()) if line == "" { continue } row := &Row{RowNo: rowNo, TargetDeptId: targetDeptID} fields := strings.Fields(line) switch len(fields) { case 1: row.Iccid = fields[0] case 3: row.Iccid = fields[0] row.Msisdn = fields[1] row.Imsi = fields[2] default: row.AddErrorReason("invalid input format; expected ICCID or ICCID MSISDN IMSI") } rows = append(rows, row) total++ if len(rows) == chunkSize { if err := emit(rows); err != nil { return total, err } rows = make([]batchimport.Row, 0, chunkSize) } } if err := scanner.Err(); err != nil { return total, err } if total == 0 { return 0, gerror.New("allocate input has no data rows") } if len(rows) > 0 { if err := emit(rows); err != nil { return total, err } } return total, nil } func rowFromExcel(parsed excel.ParsedRow) *Row { row := &Row{ RowNo: parsed.RowNo, Iccid: excel.Field(parsed, "ICCID"), Msisdn: excel.Field(parsed, "MSISDN"), Imsi: excel.Field(parsed, "IMSI"), TargetDeptKey: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u7f16\u7801", "\u5ba2\u6237\u7f16\u7801", "TARGETDEPTKEY", "TARGET_CUSTOMER_CODE"), TargetDeptName: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u540d\u79f0", "\u5ba2\u6237\u540d\u79f0", "TARGETDEPTNAME", "TARGET_CUSTOMER_NAME"), Remark: firstExcelField(parsed, "\u5212\u62e8\u5907\u6ce8", "\u5907\u6ce8", "REMARK"), } if value := firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237ID", "\u76ee\u6807\u90e8\u95e8ID", "TARGETDEPTID", "TARGET_CUSTOMER_ID"); value != "" { if targetDeptID, err := parseTargetDeptID(value); err == nil { row.TargetDeptId = targetDeptID } else { row.AddErrorReason("invalid target customer id") } } return row } func parseTargetDeptID(value string) (int64, error) { parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) if err != nil || parsed <= 0 { return 0, gerror.New("invalid target customer id") } return parsed, nil } func sourceUnitInfo(info *gjson.Json) *gjson.Json { return gjson.New(map[string]any{"sourceDeptId": info.Get("sourceDeptId").Int64(), "sourceDeptKey": info.Get("sourceDeptKey").String(), "sourceDeptName": info.Get("sourceDeptName").String()}) }