MiFaZhan
59eb34976a
iotlcm-admin-svr：管理后台服务。 - Go module：iotlcm → git.5iot.com/iotlcm-public/iotlcm-admin-svr（729 文件） - 凭据与地址脱敏：JWT signingKey（含代码里的 MustGet 默认值）、 MySQL DSN、Redis 口令、告警邮件发件人邮箱全部改为占位； 私网 IP → 文档地址段；库名 xiujie_admin → iotlcm - 真实手机号/邮箱 → 合法 11 位编造号 / example.com - 移除内部资产：CI 触发器、过时 DDL（sql/db_iotlcm.sql）、 私有依赖 go get 行、明文 registry 登录口令 - 启动 banner 统一 IOTLCM 方块字 + 社区惯例署名 - Dockerfile 改用公共基础镜像 - 拼错的 RADEME.md → README.md，README 重写 - 需要真实数据库的用例改为可用 IOTLCM_TEST_WITH_DB=1 显式启用 - 补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 数据库 schema 不随本仓发布（生产 DDL 已过时），README 已说明。
610 lines
19 KiB
Go
610 lines
19 KiB
Go
package allocate
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/excel"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/iot/simcardbatch/common"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/do"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
|
|
|
"github.com/gogf/gf/v2/database/gdb"
|
|
"github.com/gogf/gf/v2/encoding/gjson"
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
"github.com/gogf/gf/v2/os/gtime"
|
|
)
|
|
|
|
const (
|
|
defaultAllocateChunkSize = 5000
|
|
defaultAllocateMaxFileSize = 50 << 20
|
|
minAllocateChunkSize = 100
|
|
maxAllocateChunkSize = 10000
|
|
allocateDetailInsertSize = 500
|
|
allocateParentJobRole = "allocate_parent"
|
|
allocateChunkJobRole = "allocate_chunk"
|
|
)
|
|
|
|
type allocateSplitState struct {
|
|
ParentBatchNo string
|
|
ChunkSize int
|
|
ChunkCount int
|
|
Total int
|
|
Valid int
|
|
Failed int
|
|
}
|
|
|
|
type allocateChunkStream func(context.Context, func([]batchimport.Row) error) (int, error)
|
|
|
|
func allocateChunkSize(ctx context.Context) int {
|
|
size := g.Cfg().MustGet(ctx, "batch.allocate.chunkSize", defaultAllocateChunkSize).Int()
|
|
if size < minAllocateChunkSize {
|
|
return minAllocateChunkSize
|
|
}
|
|
if size > maxAllocateChunkSize {
|
|
return maxAllocateChunkSize
|
|
}
|
|
return size
|
|
}
|
|
|
|
func allocateMaxFileSize(ctx context.Context) int64 {
|
|
size := g.Cfg().MustGet(ctx, "batch.allocate.maxFileSize", defaultAllocateMaxFileSize).Int64()
|
|
if size <= 0 {
|
|
return defaultAllocateMaxFileSize
|
|
}
|
|
return size
|
|
}
|
|
|
|
func createPendingSplit(
|
|
ctx context.Context,
|
|
req batchimport.SubmitRequest,
|
|
p batchimport.PendingProcessor,
|
|
options batchimport.PendingOptions,
|
|
parentBatchNo string,
|
|
importOss *model.SysOssSaveContentModel,
|
|
stream allocateChunkStream,
|
|
) (*PendingResult, error) {
|
|
units, err := allocationunits.Snapshot(ctx, req.Operator.TenantId, req.Operator.DeptId, req.TargetDeptId)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if opts, ok := req.BizParams.(*SubmitOptions); ok {
|
|
opts.Units = units
|
|
}
|
|
|
|
if err := batchmetadata.ValidateCreation(req.SourceChannel, req.ExecutionMode); err != nil {
|
|
return nil, err
|
|
}
|
|
if p == nil {
|
|
return nil, gerror.New("allocate processor is required")
|
|
}
|
|
if strings.TrimSpace(parentBatchNo) == "" {
|
|
return nil, gerror.New("parent batch number is required")
|
|
}
|
|
if stream == nil {
|
|
return nil, gerror.New("allocate input stream is required")
|
|
}
|
|
|
|
chunkSize := allocateChunkSize(ctx)
|
|
state := &allocateSplitState{
|
|
ParentBatchNo: parentBatchNo,
|
|
ChunkSize: chunkSize,
|
|
}
|
|
if err := createAllocateParent(ctx, req, p, parentBatchNo, importOss, chunkSize); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if _, err := stream(ctx, func(rows []batchimport.Row) error {
|
|
if len(rows) == 0 {
|
|
return nil
|
|
}
|
|
if err := p.ValidateRows(ctx, rows); err != nil {
|
|
return err
|
|
}
|
|
if options.PrepareRows != nil {
|
|
if err := options.PrepareRows(ctx, rows); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := markAllocateCrossChunkDuplicates(ctx, parentBatchNo, req.Operator.TenantId, rows); err != nil {
|
|
return err
|
|
}
|
|
|
|
state.ChunkCount++
|
|
result, err := createAllocateChunk(ctx, req, p, parentBatchNo, state.ChunkCount, rows, importOss, state)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return common.PublishBatchMessageStrict(ctx, result, consts.QueueBatchExecute, "allocate")
|
|
}); err != nil {
|
|
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
|
return nil, err
|
|
}
|
|
if state.Total == 0 {
|
|
err := gerror.New("allocate input has no data rows")
|
|
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
|
return nil, err
|
|
}
|
|
if err := markAllocateSplitCompleted(ctx, parentBatchNo, state); err != nil {
|
|
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
|
return nil, err
|
|
}
|
|
|
|
parentMessage := &batchimport.Result{
|
|
BatchNo: parentBatchNo,
|
|
BatchType: consts.BatchTypeAllocate,
|
|
}
|
|
if err := common.PublishBatchMessageStrict(ctx, parentMessage, consts.QueueBatchExecute, "allocate_parent"); err != nil {
|
|
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
|
return nil, err
|
|
}
|
|
|
|
return &PendingResult{
|
|
BatchNo: parentBatchNo,
|
|
BatchType: consts.BatchTypeAllocate,
|
|
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
|
|
Total: state.Total,
|
|
ValidCount: state.Valid,
|
|
FailCount: state.Failed,
|
|
}, nil
|
|
}
|
|
|
|
func createAllocateParent(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, importOss *model.SysOssSaveContentModel, chunkSize int) error {
|
|
result := &batchimport.Result{
|
|
BatchNo: parentBatchNo,
|
|
BatchType: consts.BatchTypeAllocate,
|
|
ImportFileOssId: ossIDFromContent(importOss),
|
|
ImportFileUrl: urlFromContent(importOss),
|
|
}
|
|
info := p.BuildBatchInfo(ctx, req, result, importOss, nil, 0)
|
|
info = withAllocateInfo(info, map[string]any{
|
|
"targetBatchType": consts.BatchTypeAllocate,
|
|
"jobRole": allocateParentJobRole,
|
|
"autoSplit": true,
|
|
"chunkSize": chunkSize,
|
|
"chunkCount": 0,
|
|
"completedChunkCount": 0,
|
|
"parsedCount": 0,
|
|
})
|
|
now := gtime.Now()
|
|
_, err := g.DB().Model(dao.BatchMain.Table()).Data(do.BatchMain{
|
|
BatchNo: parentBatchNo,
|
|
OrderSn: "",
|
|
BatchType: consts.BatchTypeAllocate,
|
|
BatchAsyncStatus: consts.BatchAsyncStatusPending,
|
|
BatchStatus: consts.BatchStatusNormal,
|
|
StartTime: now,
|
|
SourceType: req.SourceType,
|
|
SourceChannel: batchmetadata.Channel(req.SourceChannel),
|
|
ExecutionMode: req.ExecutionMode,
|
|
SourceOssId: ossIDFromContent(importOss),
|
|
IsCrossDept: 0,
|
|
TargetDeptId: req.TargetDeptId,
|
|
DeptId: req.Operator.DeptId,
|
|
TotalCount: 0,
|
|
ValidCount: 0,
|
|
SuccessCount: 0,
|
|
FailCount: 0,
|
|
RunResult: consts.BatchRunResultPending,
|
|
BatchInfo: info,
|
|
TenantId: req.Operator.TenantId,
|
|
CreatedDept: req.Operator.DeptId,
|
|
CreatedBy: req.Operator.UserId,
|
|
CreatedAt: now,
|
|
UpdatedBy: req.Operator.UserId,
|
|
UpdatedAt: now,
|
|
}).Insert()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "insert allocate parent batch failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func createAllocateChunk(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, chunkNo int, rows []batchimport.Row, importOss *model.SysOssSaveContentModel, state *allocateSplitState) (*batchimport.Result, error) {
|
|
childBatchNo, err := batchimport.NextRandomNo(consts.BatchNoPrefixAllocate)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
validCount, failCount := countBatchRows(rows)
|
|
result := &batchimport.Result{
|
|
ImportStatus: consts.ImportStatusSuccess,
|
|
BatchNo: childBatchNo,
|
|
BatchType: consts.BatchTypeAllocate,
|
|
TotalRows: len(rows),
|
|
ValidRows: validCount,
|
|
ErrorRows: failCount,
|
|
ImportFileOssId: ossIDFromContent(importOss),
|
|
ImportFileUrl: urlFromContent(importOss),
|
|
}
|
|
childReq := req
|
|
childReq.BatchNo = childBatchNo
|
|
childReq.SourceBatchNo = parentBatchNo
|
|
info := p.BuildBatchInfo(ctx, childReq, result, importOss, nil, 0)
|
|
info = withAllocateInfo(info, map[string]any{
|
|
"targetBatchType": consts.BatchTypeAllocate,
|
|
"jobRole": allocateChunkJobRole,
|
|
"parentBatchNo": parentBatchNo,
|
|
"chunkNo": chunkNo,
|
|
"chunkSize": len(rows),
|
|
})
|
|
now := gtime.Now()
|
|
err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Data(do.BatchMain{
|
|
BatchNo: childBatchNo,
|
|
OrderSn: "",
|
|
BatchType: consts.BatchTypeAllocate,
|
|
BatchAsyncStatus: consts.BatchAsyncStatusPending,
|
|
BatchStatus: consts.BatchStatusNormal,
|
|
SourceType: req.SourceType,
|
|
SourceChannel: batchmetadata.Channel(req.SourceChannel),
|
|
ExecutionMode: req.ExecutionMode,
|
|
SourceBatchNo: parentBatchNo,
|
|
SourceOssId: ossIDFromContent(importOss),
|
|
IsCrossDept: 0,
|
|
TargetDeptId: req.TargetDeptId,
|
|
DeptId: req.Operator.DeptId,
|
|
TotalCount: len(rows),
|
|
ValidCount: validCount,
|
|
SuccessCount: 0,
|
|
FailCount: failCount,
|
|
RunResult: consts.BatchRunResultPending,
|
|
BatchInfo: info,
|
|
TenantId: req.Operator.TenantId,
|
|
CreatedDept: req.Operator.DeptId,
|
|
CreatedBy: req.Operator.UserId,
|
|
CreatedAt: now,
|
|
UpdatedBy: req.Operator.UserId,
|
|
UpdatedAt: now,
|
|
}).Insert(); err != nil {
|
|
return gerror.Wrap(err, "insert allocate child batch failed")
|
|
}
|
|
|
|
details := make([]do.BatchDetail, 0, len(rows))
|
|
for _, row := range rows {
|
|
status := consts.BatchDetailStatusPending
|
|
if len(row.GetErrorReasons()) > 0 {
|
|
status = consts.BatchDetailStatusFailed
|
|
}
|
|
details = append(details, do.BatchDetail{
|
|
BatchNo: childBatchNo,
|
|
BatchType: consts.BatchTypeAllocate,
|
|
DetailStatus: status,
|
|
BlankOutStatus: consts.BlankOutStatusNo,
|
|
RowNo: row.GetRowNo(),
|
|
ItemId: row.GetItemID(),
|
|
ErrorReason: strings.Join(row.GetErrorReasons(), "; "),
|
|
Snapshot: allocationunits.Merge(row.Snapshot(), sourceUnitInfo(info)),
|
|
TenantId: req.Operator.TenantId,
|
|
CreatedDept: req.Operator.DeptId,
|
|
CreatedBy: req.Operator.UserId,
|
|
CreatedAt: now,
|
|
UpdatedBy: req.Operator.UserId,
|
|
UpdatedAt: now,
|
|
})
|
|
}
|
|
for start := 0; start < len(details); start += allocateDetailInsertSize {
|
|
end := start + allocateDetailInsertSize
|
|
if end > len(details) {
|
|
end = len(details)
|
|
}
|
|
if _, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Data(details[start:end]).Insert(); err != nil {
|
|
return gerror.Wrapf(err, "insert allocate details failed, range:%d-%d", start+1, end)
|
|
}
|
|
}
|
|
|
|
parent := &entity.BatchMain{}
|
|
columns := dao.BatchMain.Columns()
|
|
if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
|
Where(columns.BatchNo, parentBatchNo).
|
|
Where(columns.BatchType, consts.BatchTypeAllocate).
|
|
LockUpdate().Scan(parent); err != nil {
|
|
return gerror.Wrap(err, "load allocate parent batch failed")
|
|
}
|
|
if parent.Id == 0 {
|
|
return gerror.New("allocate parent batch does not exist")
|
|
}
|
|
parentInfo := withAllocateInfo(parent.BatchInfo, map[string]any{
|
|
"chunkCount": chunkNo,
|
|
"parsedCount": parent.TotalCount + len(rows),
|
|
})
|
|
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
|
|
TotalCount: parent.TotalCount + len(rows),
|
|
ValidCount: parent.ValidCount + validCount,
|
|
FailCount: parent.FailCount + failCount,
|
|
BatchInfo: parentInfo,
|
|
UpdatedBy: req.Operator.UserId,
|
|
UpdatedAt: now,
|
|
}).Update(); err != nil {
|
|
return gerror.Wrap(err, "update allocate parent split progress failed")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if state != nil {
|
|
state.Total += len(rows)
|
|
state.Valid += validCount
|
|
state.Failed += failCount
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func markAllocateSplitCompleted(ctx context.Context, parentBatchNo string, state *allocateSplitState) error {
|
|
columns := dao.BatchMain.Columns()
|
|
parent := &entity.BatchMain{}
|
|
if err := g.DB().Model(dao.BatchMain.Table()).
|
|
Where(columns.BatchNo, parentBatchNo).
|
|
Where(columns.BatchType, consts.BatchTypeAllocate).
|
|
Scan(parent); err != nil {
|
|
return err
|
|
}
|
|
if parent.Id == 0 {
|
|
return gerror.New("allocate parent batch does not exist")
|
|
}
|
|
info := withAllocateInfo(parent.BatchInfo, map[string]any{
|
|
"splitCompleted": true,
|
|
"chunkCount": state.ChunkCount,
|
|
"completedChunkCount": 0,
|
|
"parsedCount": state.Total,
|
|
})
|
|
_, err := g.DB().Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
|
|
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
|
|
RunResult: consts.BatchRunResultRunning,
|
|
BatchInfo: info,
|
|
UpdatedAt: gtime.Now(),
|
|
}).Update()
|
|
return err
|
|
}
|
|
|
|
func markAllocateParentFailed(ctx context.Context, parentBatchNo, reason string) {
|
|
if strings.TrimSpace(parentBatchNo) == "" {
|
|
return
|
|
}
|
|
now := gtime.Now()
|
|
columns := dao.BatchMain.Columns()
|
|
_, _ = g.DB().Model(dao.BatchMain.Table()).
|
|
Where(columns.BatchNo, parentBatchNo).
|
|
Where(columns.BatchType, consts.BatchTypeAllocate).
|
|
WhereIn(columns.BatchAsyncStatus, []int{consts.BatchAsyncStatusPending, consts.BatchAsyncStatusProcessing}).
|
|
Data(do.BatchMain{
|
|
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
|
RunResult: consts.BatchRunResultFailed,
|
|
FailReason: strings.TrimSpace(reason),
|
|
EndTime: now,
|
|
UpdatedAt: now,
|
|
}).Update()
|
|
}
|
|
|
|
func countBatchRows(rows []batchimport.Row) (valid, failed int) {
|
|
for _, row := range rows {
|
|
if len(row.GetErrorReasons()) == 0 {
|
|
valid++
|
|
} else {
|
|
failed++
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenantID string, rows []batchimport.Row) error {
|
|
if strings.TrimSpace(parentBatchNo) == "" || len(rows) == 0 {
|
|
return nil
|
|
}
|
|
columns := dao.BatchDetail.Columns()
|
|
itemIDs := make([]string, 0, len(rows))
|
|
seen := make(map[string]struct{}, len(rows))
|
|
for _, item := range rows {
|
|
row, ok := item.(*Row)
|
|
if !ok || row == nil {
|
|
continue
|
|
}
|
|
itemID := strings.TrimSpace(row.Iccid)
|
|
if itemID == "" {
|
|
continue
|
|
}
|
|
if _, exists := seen[itemID]; exists {
|
|
continue
|
|
}
|
|
seen[itemID] = struct{}{}
|
|
itemIDs = append(itemIDs, itemID)
|
|
}
|
|
if len(itemIDs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var existing []struct {
|
|
ItemId string `orm:"item_id"`
|
|
}
|
|
query := g.DB().Model(dao.BatchDetail.Table()).
|
|
Fields(columns.ItemId).
|
|
Where(columns.BatchType, consts.BatchTypeAllocate).
|
|
Where(columns.TenantId, tenantID).
|
|
WhereIn(columns.ItemId, itemIDs).
|
|
Where("batch_no IN (SELECT batch_no FROM batch_main WHERE source_batch_no = ? AND batch_type = ? AND tenant_id = ?)",
|
|
parentBatchNo, consts.BatchTypeAllocate, tenantID)
|
|
if err := query.Scan(&existing); err != nil {
|
|
return gerror.Wrap(err, "check duplicate allocate ICCIDs across chunks failed")
|
|
}
|
|
if len(existing) == 0 {
|
|
return nil
|
|
}
|
|
|
|
duplicates := make(map[string]struct{}, len(existing))
|
|
for _, item := range existing {
|
|
if itemID := strings.TrimSpace(item.ItemId); itemID != "" {
|
|
duplicates[itemID] = struct{}{}
|
|
}
|
|
}
|
|
for _, item := range rows {
|
|
row, ok := item.(*Row)
|
|
if !ok || row == nil {
|
|
continue
|
|
}
|
|
if _, exists := duplicates[strings.TrimSpace(row.Iccid)]; exists {
|
|
row.AddErrorReason("ICCID appears in an earlier allocation chunk")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func withAllocateInfo(info *gjson.Json, values map[string]any) *gjson.Json {
|
|
data := make(map[string]any)
|
|
if info != nil {
|
|
for key, value := range info.Map() {
|
|
data[key] = value
|
|
}
|
|
}
|
|
for key, value := range values {
|
|
data[key] = value
|
|
}
|
|
return gjson.New(data)
|
|
}
|
|
|
|
func ossIDFromContent(content *model.SysOssSaveContentModel) int64 {
|
|
if content == nil || content.SysOss == nil {
|
|
return 0
|
|
}
|
|
return content.OssId
|
|
}
|
|
|
|
func urlFromContent(content *model.SysOssSaveContentModel) string {
|
|
if content == nil || content.SysOss == nil {
|
|
return ""
|
|
}
|
|
return content.Url
|
|
}
|
|
|
|
func streamAllocateExcel(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
|
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
|
|
return excel.StreamFile(ctx, filePath, excel.ParseOptions{
|
|
RequiredHeaders: []string{"ICCID"},
|
|
SingleSheet: true,
|
|
}, handler)
|
|
})
|
|
}
|
|
|
|
func streamAllocateCSV(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
|
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
|
|
return excel.StreamCSVFile(ctx, filePath, excel.ParseOptions{
|
|
RequiredHeaders: []string{"ICCID"},
|
|
}, handler)
|
|
})
|
|
}
|
|
|
|
func streamAllocateParsedFile(ctx context.Context, chunkSize int, emit func([]batchimport.Row) error, stream func(func(excel.ParsedRow) error) (int, error)) (int, error) {
|
|
rows := make([]batchimport.Row, 0, chunkSize)
|
|
total, err := stream(func(parsed excel.ParsedRow) error {
|
|
rows = append(rows, rowFromExcel(parsed))
|
|
if len(rows) < chunkSize {
|
|
return nil
|
|
}
|
|
if err := emit(rows); err != nil {
|
|
return err
|
|
}
|
|
rows = make([]batchimport.Row, 0, chunkSize)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return total, err
|
|
}
|
|
if len(rows) > 0 {
|
|
if err := emit(rows); err != nil {
|
|
return total, err
|
|
}
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
func streamAllocateInput(ctx context.Context, input string, targetDeptID int64, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
|
scanner := bufio.NewScanner(strings.NewReader(input))
|
|
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
|
rows := make([]batchimport.Row, 0, chunkSize)
|
|
total := 0
|
|
rowNo := 0
|
|
for scanner.Scan() {
|
|
rowNo++
|
|
if err := ctx.Err(); err != nil {
|
|
return total, err
|
|
}
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
row := &Row{RowNo: rowNo, TargetDeptId: targetDeptID}
|
|
fields := strings.Fields(line)
|
|
switch len(fields) {
|
|
case 1:
|
|
row.Iccid = fields[0]
|
|
case 3:
|
|
row.Iccid = fields[0]
|
|
row.Msisdn = fields[1]
|
|
row.Imsi = fields[2]
|
|
default:
|
|
row.AddErrorReason("invalid input format; expected ICCID or ICCID MSISDN IMSI")
|
|
}
|
|
rows = append(rows, row)
|
|
total++
|
|
if len(rows) == chunkSize {
|
|
if err := emit(rows); err != nil {
|
|
return total, err
|
|
}
|
|
rows = make([]batchimport.Row, 0, chunkSize)
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return total, err
|
|
}
|
|
if total == 0 {
|
|
return 0, gerror.New("allocate input has no data rows")
|
|
}
|
|
if len(rows) > 0 {
|
|
if err := emit(rows); err != nil {
|
|
return total, err
|
|
}
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
func rowFromExcel(parsed excel.ParsedRow) *Row {
|
|
row := &Row{
|
|
RowNo: parsed.RowNo,
|
|
Iccid: excel.Field(parsed, "ICCID"),
|
|
Msisdn: excel.Field(parsed, "MSISDN"),
|
|
Imsi: excel.Field(parsed, "IMSI"),
|
|
TargetDeptKey: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u7f16\u7801", "\u5ba2\u6237\u7f16\u7801", "TARGETDEPTKEY", "TARGET_CUSTOMER_CODE"),
|
|
TargetDeptName: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u540d\u79f0", "\u5ba2\u6237\u540d\u79f0", "TARGETDEPTNAME", "TARGET_CUSTOMER_NAME"),
|
|
Remark: firstExcelField(parsed, "\u5212\u62e8\u5907\u6ce8", "\u5907\u6ce8", "REMARK"),
|
|
}
|
|
if value := firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237ID", "\u76ee\u6807\u90e8\u95e8ID", "TARGETDEPTID", "TARGET_CUSTOMER_ID"); value != "" {
|
|
if targetDeptID, err := parseTargetDeptID(value); err == nil {
|
|
row.TargetDeptId = targetDeptID
|
|
} else {
|
|
row.AddErrorReason("invalid target customer id")
|
|
}
|
|
}
|
|
return row
|
|
}
|
|
|
|
func parseTargetDeptID(value string) (int64, error) {
|
|
parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
|
|
if err != nil || parsed <= 0 {
|
|
return 0, gerror.New("invalid target customer id")
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func sourceUnitInfo(info *gjson.Json) *gjson.Json {
|
|
return gjson.New(map[string]any{"sourceDeptId": info.Get("sourceDeptId").Int64(), "sourceDeptKey": info.Get("sourceDeptKey").String(), "sourceDeptName": info.Get("sourceDeptName").String()})
|
|
}
|