chore: 并入 iotlcm-admin-svr 历史到 admin-svr/（原提交 49818c6）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,81 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/excel"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/text"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
)
|
||||
|
||||
func ParseExcelFileRows(ctx context.Context, filePath string) ([]batchimport.Row, error) {
|
||||
parsedRows, err := excel.ParseFile(ctx, filePath, excel.ParseOptions{
|
||||
RequiredHeaders: []string{"ICCID"},
|
||||
SingleSheet: true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]batchimport.Row, 0, len(parsedRows))
|
||||
for _, parsed := range parsedRows {
|
||||
row := &Row{
|
||||
RowNo: parsed.RowNo,
|
||||
Iccid: excel.Field(parsed, "ICCID"),
|
||||
Msisdn: excel.Field(parsed, "MSISDN"),
|
||||
Imsi: excel.Field(parsed, "IMSI"),
|
||||
TargetDeptKey: firstExcelField(parsed, "目标客户编码", "客户编码", "TARGETDEPTKEY", "TARGET_CUSTOMER_CODE"),
|
||||
TargetDeptName: firstExcelField(parsed, "目标客户名称", "客户名称", "TARGETDEPTNAME", "TARGET_CUSTOMER_NAME"),
|
||||
Remark: firstExcelField(parsed, "划拨备注", "备注", "REMARK"),
|
||||
}
|
||||
if value := firstExcelField(parsed, "目标客户ID", "目标部门ID", "TARGETDEPTID", "TARGET_CUSTOMER_ID"); value != "" {
|
||||
targetDeptID, parseErr := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
|
||||
if parseErr != nil || targetDeptID <= 0 {
|
||||
row.AddErrorReason("目标客户ID格式错误")
|
||||
} else {
|
||||
row.TargetDeptId = targetDeptID
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func ParseInputRows(ctx context.Context, input string, targetDeptID int64) ([]batchimport.Row, error) {
|
||||
lines, err := text.ParseWhitespaceFields(ctx, input)
|
||||
if err != nil {
|
||||
if errors.Is(err, text.ErrEmptyInput) {
|
||||
return nil, gerror.New("批量输入内容不能为空")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]batchimport.Row, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
row := &Row{RowNo: line.RowNo, TargetDeptId: targetDeptID}
|
||||
switch len(line.Fields) {
|
||||
case 1:
|
||||
row.Iccid = line.Fields[0]
|
||||
case 3:
|
||||
row.Iccid = line.Fields[0]
|
||||
row.Msisdn = line.Fields[1]
|
||||
row.Imsi = line.Fields[2]
|
||||
default:
|
||||
row.AddErrorReason("格式错误，应为 ICCID 或 ICCID MSISDN IMSI")
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func firstExcelField(row excel.ParsedRow, names ...string) string {
|
||||
for _, name := range names {
|
||||
if value := excel.Field(row, name); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
const (
|
||||
importDir = "sim-card-allocate/import"
|
||||
resultDir = "sim-card-allocate/result"
|
||||
maxFileSize = 50 << 20
|
||||
maxAllocateRows = 10000
|
||||
)
|
||||
|
||||
type Processor struct{}
|
||||
|
||||
func NewProcessor() *Processor {
|
||||
return &Processor{}
|
||||
}
|
||||
|
||||
func (p *Processor) Config() batchimport.ProcessorConfig {
|
||||
return batchimport.ProcessorConfig{
|
||||
Code: "sim_card_allocate",
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchNoPrefix: consts.BatchNoPrefixAllocate,
|
||||
TaskName: "卡划拨结果",
|
||||
ExportScene: consts.ResourceExportSceneAllocate,
|
||||
ImportDir: importDir,
|
||||
ResultDir: resultDir,
|
||||
MaxFileSize: maxFileSize,
|
||||
MaxRows: maxAllocateRows,
|
||||
TemplateHeaders: []string{"ICCID", "MSISDN", "IMSI", "目标客户名称"},
|
||||
RequiredHeaders: []string{"ICCID"},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) ParseRows(ctx context.Context, filePath string) ([]batchimport.Row, error) {
|
||||
return ParseExcelFileRows(ctx, filePath)
|
||||
}
|
||||
|
||||
func (p *Processor) ValidateRows(_ context.Context, rows []batchimport.Row) error {
|
||||
firstByIccid := make(map[string]*Row, len(rows))
|
||||
for _, item := range rows {
|
||||
row, ok := item.(*Row)
|
||||
if !ok || row == nil {
|
||||
continue
|
||||
}
|
||||
row.Iccid = strings.TrimSpace(row.Iccid)
|
||||
row.Msisdn = strings.TrimSpace(row.Msisdn)
|
||||
row.Imsi = strings.TrimSpace(row.Imsi)
|
||||
row.TargetDeptKey = strings.TrimSpace(row.TargetDeptKey)
|
||||
row.TargetDeptName = strings.TrimSpace(row.TargetDeptName)
|
||||
row.Remark = strings.TrimSpace(row.Remark)
|
||||
if row.Iccid == "" {
|
||||
row.AddErrorReason("ICCID不能为空")
|
||||
continue
|
||||
}
|
||||
if first, exists := firstByIccid[row.Iccid]; exists {
|
||||
row.AddErrorReason(fmt.Sprintf("ICCID与第%d行重复", first.RowNo))
|
||||
first.AddErrorReason("ICCID重复")
|
||||
continue
|
||||
}
|
||||
firstByIccid[row.Iccid] = row
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) BuildBatchInfo(_ context.Context, req batchimport.SubmitRequest, result *batchimport.Result, importOss *model.SysOssSaveContentModel, resultOss *model.SysOssSaveContentModel, downloadId int64) *gjson.Json {
|
||||
options := optionsFromParams(req.BizParams)
|
||||
info := g.Map{
|
||||
"operation": "sim_card_allocate",
|
||||
"configName": "卡划拨",
|
||||
"submitScene": submitScene(req.SourceType),
|
||||
"targetDeptId": options.TargetDeptId,
|
||||
"remark": strings.TrimSpace(options.Remark),
|
||||
"resultFile": g.Map{"status": "pending"},
|
||||
}
|
||||
if result != nil && result.ImportFileOssId > 0 {
|
||||
info["importFileOssId"] = result.ImportFileOssId
|
||||
} else if importOss != nil && importOss.OssId > 0 {
|
||||
info["importFileOssId"] = importOss.OssId
|
||||
}
|
||||
if resultOss != nil && resultOss.OssId > 0 {
|
||||
info["resultFileOssId"] = resultOss.OssId
|
||||
}
|
||||
if downloadId > 0 {
|
||||
info["resultDownloadId"] = downloadId
|
||||
}
|
||||
return allocationunits.Merge(gjson.New(info), options.Units)
|
||||
}
|
||||
|
||||
func optionsFromParams(params any) SubmitOptions {
|
||||
if options, ok := params.(*SubmitOptions); ok && options != nil {
|
||||
return *options
|
||||
}
|
||||
return SubmitOptions{}
|
||||
}
|
||||
|
||||
func submitScene(sourceType int) string {
|
||||
switch sourceType {
|
||||
case consts.BatchSourceTypeFile:
|
||||
return "excel"
|
||||
case consts.BatchSourceTypeBatchInput:
|
||||
return "input"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
func TestProcessorTemplateHeaders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
expected := []string{"ICCID", "MSISDN", "IMSI", "目标客户名称"}
|
||||
if !reflect.DeepEqual(NewProcessor().Config().TemplateHeaders, expected) {
|
||||
t.Fatalf("unexpected headers: %#v", NewProcessor().Config().TemplateHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExcelFileRowsCustomerFacingColumns(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
filePath := filepath.Join(t.TempDir(), "allocate.xlsx")
|
||||
f := excelize.NewFile()
|
||||
headers := []any{"ICCID", "MSISDN", "IMSI", "目标客户名称"}
|
||||
if err := f.SetSheetRow("Sheet1", "A1", &headers); err != nil {
|
||||
t.Fatalf("set headers failed: %v", err)
|
||||
}
|
||||
row := []any{
|
||||
"8986000000000000001",
|
||||
"13800008000",
|
||||
"460001234567890",
|
||||
"示例企业",
|
||||
}
|
||||
if err := f.SetSheetRow("Sheet1", "A2", &row); err != nil {
|
||||
t.Fatalf("set row failed: %v", err)
|
||||
}
|
||||
if err := f.SaveAs(filePath); err != nil {
|
||||
t.Fatalf("save excel failed: %v", err)
|
||||
}
|
||||
_ = f.Close()
|
||||
|
||||
rows, err := ParseExcelFileRows(context.Background(), filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("parse excel failed: %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("unexpected row count: %d", len(rows))
|
||||
}
|
||||
rowModel, ok := rows[0].(*Row)
|
||||
if !ok || rowModel == nil {
|
||||
t.Fatalf("unexpected row type: %#v", rows[0])
|
||||
}
|
||||
if rowModel.Iccid != "8986000000000000001" {
|
||||
t.Fatalf("unexpected iccid: %s", rowModel.Iccid)
|
||||
}
|
||||
if rowModel.Msisdn != "13800008000" {
|
||||
t.Fatalf("unexpected msisdn: %s", rowModel.Msisdn)
|
||||
}
|
||||
if rowModel.Imsi != "460001234567890" {
|
||||
t.Fatalf("unexpected imsi: %s", rowModel.Imsi)
|
||||
}
|
||||
if rowModel.TargetDeptName != "示例企业" {
|
||||
t.Fatalf("unexpected target dept name: %s", rowModel.TargetDeptName)
|
||||
}
|
||||
if rowModel.Remark != "" {
|
||||
t.Fatalf("unexpected remark: %s", rowModel.Remark)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessorConfigLimitsAllocationRows(t *testing.T) {
|
||||
config := NewProcessor().Config()
|
||||
if config.MaxRows != 10000 {
|
||||
t.Fatalf("MaxRows = %d, want 10000", config.MaxRows)
|
||||
}
|
||||
if config.MaxFileSize != 50<<20 {
|
||||
t.Fatalf("MaxFileSize = %d, want %d", config.MaxFileSize, 50<<20)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,661 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/excel"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/iot/simcardbatch/common"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/do"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultAllocateChunkSize = 5000
|
||||
defaultAllocateMaxFileSize = 50 << 20
|
||||
minAllocateChunkSize = 100
|
||||
maxAllocateChunkSize = 10000
|
||||
allocateDetailInsertSize = 500
|
||||
allocateParentJobRole = "allocate_parent"
|
||||
allocateChunkJobRole = "allocate_chunk"
|
||||
)
|
||||
|
||||
type allocateSplitState struct {
|
||||
ParentBatchNo string
|
||||
ChunkSize int
|
||||
ChunkCount int
|
||||
Total int
|
||||
Valid int
|
||||
Failed int
|
||||
}
|
||||
|
||||
type allocateChunkStream func(context.Context, func([]batchimport.Row) error) (int, error)
|
||||
|
||||
func allocateChunkSize(ctx context.Context) int {
|
||||
size := g.Cfg().MustGet(ctx, "batch.allocate.chunkSize", defaultAllocateChunkSize).Int()
|
||||
if size < minAllocateChunkSize {
|
||||
return minAllocateChunkSize
|
||||
}
|
||||
if size > maxAllocateChunkSize {
|
||||
return maxAllocateChunkSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func allocateMaxFileSize(ctx context.Context) int64 {
|
||||
size := g.Cfg().MustGet(ctx, "batch.allocate.maxFileSize", defaultAllocateMaxFileSize).Int64()
|
||||
if size <= 0 {
|
||||
return defaultAllocateMaxFileSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func createPendingSplit(
|
||||
ctx context.Context,
|
||||
req batchimport.SubmitRequest,
|
||||
p batchimport.PendingProcessor,
|
||||
options batchimport.PendingOptions,
|
||||
parentBatchNo string,
|
||||
importOss *model.SysOssSaveContentModel,
|
||||
stream allocateChunkStream,
|
||||
) (*PendingResult, error) {
|
||||
units, err := allocationunits.Snapshot(ctx, req.Operator.TenantId, req.Operator.DeptId, req.TargetDeptId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts, ok := req.BizParams.(*SubmitOptions); ok {
|
||||
opts.Units = units
|
||||
}
|
||||
|
||||
if err := batchmetadata.ValidateCreation(req.SourceChannel, req.ExecutionMode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p == nil {
|
||||
return nil, gerror.New("allocate processor is required")
|
||||
}
|
||||
if strings.TrimSpace(parentBatchNo) == "" {
|
||||
return nil, gerror.New("parent batch number is required")
|
||||
}
|
||||
if stream == nil {
|
||||
return nil, gerror.New("allocate input stream is required")
|
||||
}
|
||||
|
||||
chunkSize := allocateChunkSize(ctx)
|
||||
state := &allocateSplitState{
|
||||
ParentBatchNo: parentBatchNo,
|
||||
ChunkSize: chunkSize,
|
||||
}
|
||||
if err := createAllocateParent(ctx, req, p, parentBatchNo, importOss, chunkSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
children := make([]*batchimport.Result, 0)
|
||||
|
||||
if _, err := stream(ctx, func(rows []batchimport.Row) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := p.ValidateRows(ctx, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
if options.PrepareRows != nil {
|
||||
if err := options.PrepareRows(ctx, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
priorFailures, err := markAllocateCrossChunkDuplicates(ctx, parentBatchNo, req.Operator.TenantId, rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state.Valid -= priorFailures
|
||||
state.Failed += priorFailures
|
||||
|
||||
state.ChunkCount++
|
||||
result, err := createAllocateChunk(ctx, req, p, parentBatchNo, state.ChunkCount, rows, importOss, state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
children = append(children, result)
|
||||
return nil
|
||||
}); err != nil {
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
if state.Total == 0 {
|
||||
err := gerror.New("allocate input has no data rows")
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
if err := markAllocateSplitCompleted(ctx, parentBatchNo, state); err != nil {
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
for _, child := range children {
|
||||
if err := common.PublishBatchMessageStrict(ctx, child, consts.QueueBatchExecute, "allocate"); err != nil {
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
parentMessage := &batchimport.Result{
|
||||
BatchNo: parentBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
}
|
||||
if err := common.PublishBatchMessageStrict(ctx, parentMessage, consts.QueueBatchExecute, "allocate_parent"); err != nil {
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &PendingResult{
|
||||
BatchNo: parentBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
|
||||
Total: state.Total,
|
||||
ValidCount: state.Valid,
|
||||
FailCount: state.Failed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func createAllocateParent(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, importOss *model.SysOssSaveContentModel, chunkSize int) error {
|
||||
result := &batchimport.Result{
|
||||
BatchNo: parentBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
ImportFileOssId: ossIDFromContent(importOss),
|
||||
ImportFileUrl: urlFromContent(importOss),
|
||||
}
|
||||
info := p.BuildBatchInfo(ctx, req, result, importOss, nil, 0)
|
||||
info = withAllocateInfo(info, map[string]any{
|
||||
"targetBatchType": consts.BatchTypeAllocate,
|
||||
"jobRole": allocateParentJobRole,
|
||||
"autoSplit": true,
|
||||
"chunkSize": chunkSize,
|
||||
"chunkCount": 0,
|
||||
"completedChunkCount": 0,
|
||||
"parsedCount": 0,
|
||||
})
|
||||
now := gtime.Now()
|
||||
_, err := g.DB().Model(dao.BatchMain.Table()).Data(do.BatchMain{
|
||||
BatchNo: parentBatchNo,
|
||||
OrderSn: "",
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusPending,
|
||||
BatchStatus: consts.BatchStatusNormal,
|
||||
StartTime: now,
|
||||
SourceType: req.SourceType,
|
||||
SourceChannel: batchmetadata.Channel(req.SourceChannel),
|
||||
ExecutionMode: req.ExecutionMode,
|
||||
SourceOssId: ossIDFromContent(importOss),
|
||||
IsCrossDept: 0,
|
||||
TargetDeptId: req.TargetDeptId,
|
||||
DeptId: req.Operator.DeptId,
|
||||
TotalCount: 0,
|
||||
ValidCount: 0,
|
||||
SuccessCount: 0,
|
||||
FailCount: 0,
|
||||
RunResult: consts.BatchRunResultPending,
|
||||
BatchInfo: info,
|
||||
TenantId: req.Operator.TenantId,
|
||||
CreatedDept: req.Operator.DeptId,
|
||||
CreatedBy: req.Operator.UserId,
|
||||
CreatedAt: now,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
}).Insert()
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "insert allocate parent batch failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createAllocateChunk(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, chunkNo int, rows []batchimport.Row, importOss *model.SysOssSaveContentModel, state *allocateSplitState) (*batchimport.Result, error) {
|
||||
childBatchNo, err := batchimport.NextRandomNo(consts.BatchNoPrefixAllocate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
validCount, failCount := countBatchRows(rows)
|
||||
result := &batchimport.Result{
|
||||
ImportStatus: consts.ImportStatusSuccess,
|
||||
BatchNo: childBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
TotalRows: len(rows),
|
||||
ValidRows: validCount,
|
||||
ErrorRows: failCount,
|
||||
ImportFileOssId: ossIDFromContent(importOss),
|
||||
ImportFileUrl: urlFromContent(importOss),
|
||||
}
|
||||
childReq := req
|
||||
childReq.BatchNo = childBatchNo
|
||||
childReq.SourceBatchNo = parentBatchNo
|
||||
info := p.BuildBatchInfo(ctx, childReq, result, importOss, nil, 0)
|
||||
info = withAllocateInfo(info, map[string]any{
|
||||
"targetBatchType": consts.BatchTypeAllocate,
|
||||
"jobRole": allocateChunkJobRole,
|
||||
"parentBatchNo": parentBatchNo,
|
||||
"chunkNo": chunkNo,
|
||||
"chunkSize": len(rows),
|
||||
})
|
||||
now := gtime.Now()
|
||||
err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Data(do.BatchMain{
|
||||
BatchNo: childBatchNo,
|
||||
OrderSn: "",
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusPending,
|
||||
BatchStatus: consts.BatchStatusNormal,
|
||||
SourceType: req.SourceType,
|
||||
SourceChannel: batchmetadata.Channel(req.SourceChannel),
|
||||
ExecutionMode: req.ExecutionMode,
|
||||
SourceBatchNo: parentBatchNo,
|
||||
SourceOssId: ossIDFromContent(importOss),
|
||||
IsCrossDept: 0,
|
||||
TargetDeptId: req.TargetDeptId,
|
||||
DeptId: req.Operator.DeptId,
|
||||
TotalCount: len(rows),
|
||||
ValidCount: validCount,
|
||||
SuccessCount: 0,
|
||||
FailCount: failCount,
|
||||
RunResult: consts.BatchRunResultPending,
|
||||
BatchInfo: info,
|
||||
TenantId: req.Operator.TenantId,
|
||||
CreatedDept: req.Operator.DeptId,
|
||||
CreatedBy: req.Operator.UserId,
|
||||
CreatedAt: now,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
}).Insert(); err != nil {
|
||||
return gerror.Wrap(err, "insert allocate child batch failed")
|
||||
}
|
||||
|
||||
details := make([]do.BatchDetail, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
status := consts.BatchDetailStatusPending
|
||||
if len(row.GetErrorReasons()) > 0 {
|
||||
status = consts.BatchDetailStatusFailed
|
||||
}
|
||||
details = append(details, do.BatchDetail{
|
||||
BatchNo: childBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
DetailStatus: status,
|
||||
BlankOutStatus: consts.BlankOutStatusNo,
|
||||
RowNo: row.GetRowNo(),
|
||||
ItemId: row.GetItemID(),
|
||||
ErrorReason: strings.Join(row.GetErrorReasons(), "; "),
|
||||
Snapshot: allocationunits.Merge(row.Snapshot(), sourceUnitInfo(info)),
|
||||
TenantId: req.Operator.TenantId,
|
||||
CreatedDept: req.Operator.DeptId,
|
||||
CreatedBy: req.Operator.UserId,
|
||||
CreatedAt: now,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
for start := 0; start < len(details); start += allocateDetailInsertSize {
|
||||
end := start + allocateDetailInsertSize
|
||||
if end > len(details) {
|
||||
end = len(details)
|
||||
}
|
||||
if _, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Data(details[start:end]).Insert(); err != nil {
|
||||
return gerror.Wrapf(err, "insert allocate details failed, range:%d-%d", start+1, end)
|
||||
}
|
||||
}
|
||||
|
||||
parent := &entity.BatchMain{}
|
||||
columns := dao.BatchMain.Columns()
|
||||
if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
||||
Where(columns.BatchNo, parentBatchNo).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
LockUpdate().Scan(parent); err != nil {
|
||||
return gerror.Wrap(err, "load allocate parent batch failed")
|
||||
}
|
||||
if parent.Id == 0 {
|
||||
return gerror.New("allocate parent batch does not exist")
|
||||
}
|
||||
parentInfo := withAllocateInfo(parent.BatchInfo, map[string]any{
|
||||
"chunkCount": chunkNo,
|
||||
"parsedCount": parent.TotalCount + len(rows),
|
||||
})
|
||||
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
|
||||
TotalCount: parent.TotalCount + len(rows),
|
||||
ValidCount: parent.ValidCount + validCount,
|
||||
FailCount: parent.FailCount + failCount,
|
||||
BatchInfo: parentInfo,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
}).Update(); err != nil {
|
||||
return gerror.Wrap(err, "update allocate parent split progress failed")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if state != nil {
|
||||
state.Total += len(rows)
|
||||
state.Valid += validCount
|
||||
state.Failed += failCount
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func markAllocateSplitCompleted(ctx context.Context, parentBatchNo string, state *allocateSplitState) error {
|
||||
columns := dao.BatchMain.Columns()
|
||||
parent := &entity.BatchMain{}
|
||||
if err := g.DB().Model(dao.BatchMain.Table()).
|
||||
Where(columns.BatchNo, parentBatchNo).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
Scan(parent); err != nil {
|
||||
return err
|
||||
}
|
||||
if parent.Id == 0 {
|
||||
return gerror.New("allocate parent batch does not exist")
|
||||
}
|
||||
info := withAllocateInfo(parent.BatchInfo, map[string]any{
|
||||
"splitCompleted": true,
|
||||
"chunkCount": state.ChunkCount,
|
||||
"completedChunkCount": 0,
|
||||
"parsedCount": state.Total,
|
||||
})
|
||||
_, err := g.DB().Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
|
||||
RunResult: consts.BatchRunResultRunning,
|
||||
BatchInfo: info,
|
||||
UpdatedAt: gtime.Now(),
|
||||
}).Update()
|
||||
return err
|
||||
}
|
||||
|
||||
func markAllocateParentFailed(ctx context.Context, parentBatchNo, reason string) {
|
||||
if strings.TrimSpace(parentBatchNo) == "" {
|
||||
return
|
||||
}
|
||||
now := gtime.Now()
|
||||
columns := dao.BatchMain.Columns()
|
||||
_, _ = g.DB().Model(dao.BatchMain.Table()).
|
||||
Where(columns.BatchNo, parentBatchNo).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
WhereIn(columns.BatchAsyncStatus, []int{consts.BatchAsyncStatusPending, consts.BatchAsyncStatusProcessing}).
|
||||
Data(do.BatchMain{
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
||||
RunResult: consts.BatchRunResultFailed,
|
||||
FailReason: strings.TrimSpace(reason),
|
||||
EndTime: now,
|
||||
UpdatedAt: now,
|
||||
}).Update()
|
||||
}
|
||||
|
||||
func countBatchRows(rows []batchimport.Row) (valid, failed int) {
|
||||
for _, row := range rows {
|
||||
if len(row.GetErrorReasons()) == 0 {
|
||||
valid++
|
||||
} else {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenantID string, rows []batchimport.Row) (int, error) {
|
||||
if strings.TrimSpace(parentBatchNo) == "" || len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
columns := dao.BatchDetail.Columns()
|
||||
itemIDs := make([]string, 0, len(rows))
|
||||
seen := make(map[string]struct{}, len(rows))
|
||||
for _, item := range rows {
|
||||
row, ok := item.(*Row)
|
||||
if !ok || row == nil {
|
||||
continue
|
||||
}
|
||||
itemID := strings.TrimSpace(row.Iccid)
|
||||
if itemID == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[itemID]; exists {
|
||||
continue
|
||||
}
|
||||
seen[itemID] = struct{}{}
|
||||
itemIDs = append(itemIDs, itemID)
|
||||
}
|
||||
if len(itemIDs) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var existing []struct {
|
||||
Id int64 `orm:"id"`
|
||||
BatchNo string `orm:"batch_no"`
|
||||
ItemId string `orm:"item_id"`
|
||||
DetailStatus int `orm:"detail_status"`
|
||||
}
|
||||
query := g.DB().Model(dao.BatchDetail.Table()).
|
||||
Fields(columns.Id, columns.BatchNo, columns.ItemId, columns.DetailStatus).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
Where(columns.TenantId, tenantID).
|
||||
WhereIn(columns.ItemId, itemIDs).
|
||||
Where("batch_no IN (SELECT batch_no FROM batch_main WHERE source_batch_no = ? AND batch_type = ? AND tenant_id = ?)",
|
||||
parentBatchNo, consts.BatchTypeAllocate, tenantID)
|
||||
if err := query.Scan(&existing); err != nil {
|
||||
return 0, gerror.Wrap(err, "check duplicate allocate ICCIDs across chunks failed")
|
||||
}
|
||||
if len(existing) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
priorFailures := 0
|
||||
err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
for _, detail := range existing {
|
||||
if detail.DetailStatus != consts.BatchDetailStatusPending {
|
||||
continue
|
||||
}
|
||||
result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
|
||||
Where(columns.Id, detail.Id).
|
||||
Where(columns.TenantId, tenantID).
|
||||
Where(columns.DetailStatus, consts.BatchDetailStatusPending).
|
||||
Data(do.BatchDetail{DetailStatus: consts.BatchDetailStatusFailed, ErrorReason: "ICCID在划拨批次内重复"}).Update()
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "mark prior duplicate allocation detail failed")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected == 0 {
|
||||
continue
|
||||
}
|
||||
priorFailures++
|
||||
mainColumns := dao.BatchMain.Columns()
|
||||
for _, batchNo := range []string{detail.BatchNo, parentBatchNo} {
|
||||
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
||||
Where(mainColumns.BatchNo, batchNo).
|
||||
Where(mainColumns.TenantId, tenantID).
|
||||
Data(do.BatchMain{ValidCount: gdb.Raw("valid_count - 1"), FailCount: gdb.Raw("fail_count + 1")}).Update(); err != nil {
|
||||
return gerror.Wrap(err, "update cross-chunk duplicate counts failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
duplicates := make(map[string]struct{}, len(existing))
|
||||
for _, item := range existing {
|
||||
if itemID := strings.TrimSpace(item.ItemId); itemID != "" {
|
||||
duplicates[itemID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, item := range rows {
|
||||
row, ok := item.(*Row)
|
||||
if !ok || row == nil {
|
||||
continue
|
||||
}
|
||||
if _, exists := duplicates[strings.TrimSpace(row.Iccid)]; exists {
|
||||
row.AddErrorReason("ICCID在划拨批次内重复")
|
||||
}
|
||||
}
|
||||
return priorFailures, nil
|
||||
}
|
||||
|
||||
func withAllocateInfo(info *gjson.Json, values map[string]any) *gjson.Json {
|
||||
data := make(map[string]any)
|
||||
if info != nil {
|
||||
for key, value := range info.Map() {
|
||||
data[key] = value
|
||||
}
|
||||
}
|
||||
for key, value := range values {
|
||||
data[key] = value
|
||||
}
|
||||
return gjson.New(data)
|
||||
}
|
||||
|
||||
func ossIDFromContent(content *model.SysOssSaveContentModel) int64 {
|
||||
if content == nil || content.SysOss == nil {
|
||||
return 0
|
||||
}
|
||||
return content.OssId
|
||||
}
|
||||
|
||||
func urlFromContent(content *model.SysOssSaveContentModel) string {
|
||||
if content == nil || content.SysOss == nil {
|
||||
return ""
|
||||
}
|
||||
return content.Url
|
||||
}
|
||||
|
||||
func streamAllocateExcel(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
||||
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
|
||||
return excel.StreamFile(ctx, filePath, excel.ParseOptions{
|
||||
RequiredHeaders: []string{"ICCID"},
|
||||
SingleSheet: true,
|
||||
}, handler)
|
||||
})
|
||||
}
|
||||
|
||||
func streamAllocateCSV(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
||||
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
|
||||
return excel.StreamCSVFile(ctx, filePath, excel.ParseOptions{
|
||||
RequiredHeaders: []string{"ICCID"},
|
||||
}, handler)
|
||||
})
|
||||
}
|
||||
|
||||
func streamAllocateParsedFile(ctx context.Context, chunkSize int, emit func([]batchimport.Row) error, stream func(func(excel.ParsedRow) error) (int, error)) (int, error) {
|
||||
rows := make([]batchimport.Row, 0, chunkSize)
|
||||
total, err := stream(func(parsed excel.ParsedRow) error {
|
||||
rows = append(rows, rowFromExcel(parsed))
|
||||
if len(rows) < chunkSize {
|
||||
return nil
|
||||
}
|
||||
if err := emit(rows); err != nil {
|
||||
return err
|
||||
}
|
||||
rows = make([]batchimport.Row, 0, chunkSize)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
if err := emit(rows); err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func streamAllocateInput(ctx context.Context, input string, targetDeptID int64, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
||||
scanner := bufio.NewScanner(strings.NewReader(input))
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
rows := make([]batchimport.Row, 0, chunkSize)
|
||||
total := 0
|
||||
rowNo := 0
|
||||
for scanner.Scan() {
|
||||
rowNo++
|
||||
if err := ctx.Err(); err != nil {
|
||||
return total, err
|
||||
}
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
row := &Row{RowNo: rowNo, TargetDeptId: targetDeptID}
|
||||
fields := strings.Fields(line)
|
||||
switch len(fields) {
|
||||
case 1:
|
||||
row.Iccid = fields[0]
|
||||
case 3:
|
||||
row.Iccid = fields[0]
|
||||
row.Msisdn = fields[1]
|
||||
row.Imsi = fields[2]
|
||||
default:
|
||||
row.AddErrorReason("invalid input format; expected ICCID or ICCID MSISDN IMSI")
|
||||
}
|
||||
rows = append(rows, row)
|
||||
total++
|
||||
if len(rows) == chunkSize {
|
||||
if err := emit(rows); err != nil {
|
||||
return total, err
|
||||
}
|
||||
rows = make([]batchimport.Row, 0, chunkSize)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return total, err
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, gerror.New("allocate input has no data rows")
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
if err := emit(rows); err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func rowFromExcel(parsed excel.ParsedRow) *Row {
|
||||
row := &Row{
|
||||
RowNo: parsed.RowNo,
|
||||
Iccid: excel.Field(parsed, "ICCID"),
|
||||
Msisdn: excel.Field(parsed, "MSISDN"),
|
||||
Imsi: excel.Field(parsed, "IMSI"),
|
||||
TargetDeptKey: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u7f16\u7801", "\u5ba2\u6237\u7f16\u7801", "TARGETDEPTKEY", "TARGET_CUSTOMER_CODE"),
|
||||
TargetDeptName: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u540d\u79f0", "\u5ba2\u6237\u540d\u79f0", "TARGETDEPTNAME", "TARGET_CUSTOMER_NAME"),
|
||||
Remark: firstExcelField(parsed, "\u5212\u62e8\u5907\u6ce8", "\u5907\u6ce8", "REMARK"),
|
||||
}
|
||||
if value := firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237ID", "\u76ee\u6807\u90e8\u95e8ID", "TARGETDEPTID", "TARGET_CUSTOMER_ID"); value != "" {
|
||||
if targetDeptID, err := parseTargetDeptID(value); err == nil {
|
||||
row.TargetDeptId = targetDeptID
|
||||
} else {
|
||||
row.AddErrorReason("invalid target customer id")
|
||||
}
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func parseTargetDeptID(value string) (int64, error) {
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
|
||||
if err != nil || parsed <= 0 {
|
||||
return 0, gerror.New("invalid target customer id")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func sourceUnitInfo(info *gjson.Json) *gjson.Json {
|
||||
return gjson.New(map[string]any{"sourceDeptId": info.Get("sourceDeptId").Int64(), "sourceDeptKey": info.Get("sourceDeptKey").String(), "sourceDeptName": info.Get("sourceDeptName").String()})
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
||||
|
||||
"github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type allocateMetadataSQLiteDriver struct {
|
||||
*sqlite.Driver
|
||||
}
|
||||
|
||||
func (driver *allocateMetadataSQLiteDriver) New(core *gdb.Core, _ *gdb.ConfigNode) (gdb.DB, error) {
|
||||
return &allocateMetadataSQLiteDriver{Driver: &sqlite.Driver{Core: core}}, nil
|
||||
}
|
||||
|
||||
func (driver *allocateMetadataSQLiteDriver) DoSelect(ctx context.Context, link gdb.Link, query string, args ...any) (gdb.Result, error) {
|
||||
return driver.Core.DoSelect(ctx, link, strings.TrimSuffix(query, " FOR UPDATE"), args...)
|
||||
}
|
||||
|
||||
func TestAllocateSplitInheritsRequestMetadata(t *testing.T) {
|
||||
if err := gdb.Register("allocate_metadata_sqlite", &allocateMetadataSQLiteDriver{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "allocate_metadata_sqlite", Name: ":memory:", MaxOpenConnCount: 1}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = g.DB().Close(ctx) })
|
||||
if err := g.DB().GetCore().ClearTableFieldsAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for table, record := range map[string]any{"batch_main": entity.BatchMain{}, "batch_detail": entity.BatchDetail{}} {
|
||||
recordType := reflect.TypeOf(record)
|
||||
columns := make([]string, 0, recordType.NumField())
|
||||
for fieldIndex := 0; fieldIndex < recordType.NumField(); fieldIndex++ {
|
||||
field := recordType.Field(fieldIndex)
|
||||
column := field.Tag.Get("orm")
|
||||
if column == "id" {
|
||||
columns = append(columns, "id INTEGER PRIMARY KEY AUTOINCREMENT")
|
||||
} else {
|
||||
columns = append(columns, column+" TEXT")
|
||||
}
|
||||
}
|
||||
if _, err := g.DB().Exec(ctx, "CREATE TABLE "+table+" ("+strings.Join(columns, ",")+")"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := g.Model(table).Ctx(ctx).TableFields(table); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := g.DB().GetCore().GetTablesWithCache(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mode := range []int{batchmetadata.Single, batchmetadata.Bulk} {
|
||||
parentNo := "META-SPLIT-SINGLE"
|
||||
if mode == batchmetadata.Bulk {
|
||||
parentNo = "META-SPLIT-BULK"
|
||||
}
|
||||
req := batchimport.SubmitRequest{SourceChannel: batchmetadata.Console, ExecutionMode: mode, SourceType: consts.BatchSourceTypeBatchInput, Operator: batchimport.OperatorContext{TenantId: "A", UserId: 7}}
|
||||
processor := NewProcessor()
|
||||
if err := createAllocateParent(ctx, req, processor, parentNo, nil, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := []batchimport.Row{&Row{RowNo: 1, Iccid: "card1"}}
|
||||
result, err := createAllocateChunk(ctx, req, processor, parentNo, 1, rows, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, batchNo := range []string{parentNo, result.BatchNo} {
|
||||
var batch *entity.BatchMain
|
||||
if err := g.Model("batch_main").Ctx(ctx).Where("batch_no", batchNo).Scan(&batch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if batch == nil || batch.SourceChannel != req.SourceChannel || batch.ExecutionMode != uint(mode) || batch.SourceType != req.SourceType || batch.TotalCount != 1 {
|
||||
t.Fatalf("metadata not inherited: %+v", batch)
|
||||
}
|
||||
if batchNo != parentNo && batch.SourceBatchNo != parentNo {
|
||||
t.Fatalf("child lost parent association: %+v", batch)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/iot/simcardbatch/common"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/net/ghttp"
|
||||
)
|
||||
|
||||
func CreatePendingExcel(ctx context.Context, file *ghttp.UploadFile, in *model.TSimCardAllocateExcelSubmitParam) (*PendingResult, error) {
|
||||
if in == nil {
|
||||
in = &model.TSimCardAllocateExcelSubmitParam{}
|
||||
}
|
||||
if file == nil {
|
||||
return nil, gerror.New("upload file is required")
|
||||
}
|
||||
if err := service.SysOss().CheckType(ctx, consts.SysUploadFileTypeFile, file); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if file.Size > allocateMaxFileSize(ctx) {
|
||||
return nil, gerror.Newf("uploaded file exceeds the limit of %d MB", allocateMaxFileSize(ctx)>>20)
|
||||
}
|
||||
|
||||
options := SubmitOptions{
|
||||
TargetDeptId: in.TargetDeptId,
|
||||
Remark: strings.TrimSpace(in.Remark),
|
||||
}
|
||||
processor := NewProcessor()
|
||||
parentBatchNo, err := batchimport.NextBatchNoContext(ctx, processor.Config().BatchNoPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
extension := strings.ToLower(filepath.Ext(file.Filename))
|
||||
if extension == "" {
|
||||
extension = ".xlsx"
|
||||
}
|
||||
importOss, err := batchimport.SaveUpload(ctx, file, parentBatchNo+"-import"+extension, processor.Config().ImportDir)
|
||||
if err != nil {
|
||||
return nil, gerror.Wrap(err, "save allocate import file failed")
|
||||
}
|
||||
|
||||
req := batchimport.SubmitRequest{
|
||||
SourceChannel: batchmetadata.Console,
|
||||
ExecutionMode: batchmetadata.Bulk,
|
||||
BizParams: &options,
|
||||
SourceType: consts.BatchSourceTypeFile,
|
||||
TargetDeptId: options.TargetDeptId,
|
||||
Operator: common.CurrentOperator(ctx),
|
||||
BatchNo: parentBatchNo,
|
||||
}
|
||||
stream := func(streamCtx context.Context, emit func([]batchimport.Row) error) (int, error) {
|
||||
chunkSize := allocateChunkSize(streamCtx)
|
||||
if extension == ".csv" {
|
||||
return streamAllocateCSV(streamCtx, importOss.Path, chunkSize, emit)
|
||||
}
|
||||
return streamAllocateExcel(streamCtx, importOss.Path, chunkSize, emit)
|
||||
}
|
||||
return createPendingSplit(ctx, req, processor, prepareOptions(options), parentBatchNo, importOss, stream)
|
||||
}
|
||||
|
||||
func CreatePendingInput(ctx context.Context, in *model.TSimCardAllocatePrepareParam) (*PendingResult, error) {
|
||||
if in == nil {
|
||||
in = &model.TSimCardAllocatePrepareParam{}
|
||||
}
|
||||
if in.TargetDeptId <= 0 {
|
||||
return nil, gerror.New("target customer is required")
|
||||
}
|
||||
|
||||
options := SubmitOptions{TargetDeptId: in.TargetDeptId}
|
||||
processor := NewProcessor()
|
||||
parentBatchNo, err := batchimport.NextBatchNoContext(ctx, processor.Config().BatchNoPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req := batchimport.SubmitRequest{
|
||||
SourceChannel: batchmetadata.Console,
|
||||
ExecutionMode: batchmetadata.Bulk,
|
||||
BizParams: &options,
|
||||
SourceType: consts.BatchSourceTypeBatchInput,
|
||||
TargetDeptId: options.TargetDeptId,
|
||||
Operator: common.CurrentOperator(ctx),
|
||||
BatchNo: parentBatchNo,
|
||||
}
|
||||
stream := func(streamCtx context.Context, emit func([]batchimport.Row) error) (int, error) {
|
||||
return streamAllocateInput(streamCtx, in.Input, options.TargetDeptId, allocateChunkSize(streamCtx), emit)
|
||||
}
|
||||
return createPendingSplit(ctx, req, processor, prepareOptions(options), parentBatchNo, nil, stream)
|
||||
}
|
||||
|
||||
func prepareOptions(options SubmitOptions) batchimport.PendingOptions {
|
||||
return batchimport.PendingOptions{
|
||||
PrepareRows: func(ctx context.Context, rows []batchimport.Row) error {
|
||||
for _, item := range rows {
|
||||
row, ok := item.(*Row)
|
||||
if !ok || row == nil {
|
||||
return gerror.New("allocate row type is invalid")
|
||||
}
|
||||
if row.TargetDeptId <= 0 && options.TargetDeptId > 0 {
|
||||
row.TargetDeptId = options.TargetDeptId
|
||||
}
|
||||
if row.Remark == "" {
|
||||
row.Remark = options.Remark
|
||||
}
|
||||
if row.TargetDeptId <= 0 && row.TargetDeptKey == "" && row.TargetDeptName == "" {
|
||||
row.AddErrorReason("target customer is required")
|
||||
}
|
||||
}
|
||||
ids := []int64{}
|
||||
for _, item := range rows {
|
||||
if row, ok := item.(*Row); ok {
|
||||
ids = append(ids, row.TargetDeptId)
|
||||
}
|
||||
}
|
||||
units, err := allocationunits.Load(ctx, common.CurrentOperator(ctx).TenantId, ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range rows {
|
||||
if row, ok := item.(*Row); ok {
|
||||
if unit, exists := units[row.TargetDeptId]; exists {
|
||||
row.TargetDeptKey = unit.Key
|
||||
row.TargetDeptName = unit.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func pendingResultFromBatch(result *batchimport.Result) *PendingResult {
|
||||
res := &PendingResult{
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusPending,
|
||||
}
|
||||
if result == nil {
|
||||
return res
|
||||
}
|
||||
res.BatchNo = result.BatchNo
|
||||
res.BatchType = result.BatchType
|
||||
res.Total = result.TotalRows
|
||||
res.ValidCount = result.ValidRows
|
||||
res.FailCount = result.ErrorRows
|
||||
return res
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type Row struct {
|
||||
RowNo int
|
||||
Iccid string
|
||||
Msisdn string
|
||||
Imsi string
|
||||
TargetDeptId int64
|
||||
TargetDeptKey string
|
||||
TargetDeptName string
|
||||
Remark string
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (r *Row) GetRowNo() int {
|
||||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
return r.RowNo
|
||||
}
|
||||
|
||||
func (r *Row) GetItemID() string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
return r.Iccid
|
||||
}
|
||||
|
||||
func (r *Row) GetErrorReasons() []string {
|
||||
if r == nil || strings.TrimSpace(r.Reason) == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{strings.TrimSpace(r.Reason)}
|
||||
}
|
||||
|
||||
func (r *Row) AddErrorReason(reason string) {
|
||||
if r == nil || strings.TrimSpace(reason) == "" {
|
||||
return
|
||||
}
|
||||
if r.Reason != "" {
|
||||
r.Reason += "；"
|
||||
}
|
||||
r.Reason += strings.TrimSpace(reason)
|
||||
}
|
||||
|
||||
func (r *Row) Snapshot() *gjson.Json {
|
||||
if r == nil {
|
||||
return gjson.New(g.Map{})
|
||||
}
|
||||
return gjson.New(g.Map{
|
||||
"ICCID": strings.TrimSpace(r.Iccid),
|
||||
"MSISDN": strings.TrimSpace(r.Msisdn),
|
||||
"IMSI": strings.TrimSpace(r.Imsi),
|
||||
"targetDeptId": r.TargetDeptId,
|
||||
"targetDeptKey": strings.TrimSpace(r.TargetDeptKey),
|
||||
"targetDeptName": strings.TrimSpace(r.TargetDeptName),
|
||||
"remark": strings.TrimSpace(r.Remark),
|
||||
})
|
||||
}
|
||||
|
||||
type SubmitOptions struct {
|
||||
Units *gjson.Json
|
||||
TargetDeptId int64
|
||||
Remark string
|
||||
}
|
||||
|
||||
type PendingResult struct {
|
||||
BatchNo string
|
||||
BatchType int
|
||||
BatchAsyncStatus int
|
||||
Total int
|
||||
ValidCount int
|
||||
FailCount int
|
||||
}
|
||||
Reference in New Issue
Block a user