chore: 初始开源发布
iotlcm-admin-svr：管理后台服务。 - Go module：iotlcm → git.5iot.com/iotlcm-public/iotlcm-admin-svr（729 文件） - 凭据与地址脱敏：JWT signingKey（含代码里的 MustGet 默认值）、 MySQL DSN、Redis 口令、告警邮件发件人邮箱全部改为占位； 私网 IP → 文档地址段；库名 xiujie_admin → iotlcm - 真实手机号/邮箱 → 合法 11 位编造号 / example.com - 移除内部资产：CI 触发器、过时 DDL（sql/db_iotlcm.sql）、 私有依赖 go get 行、明文 registry 登录口令 - 启动 banner 统一 IOTLCM 方块字 + 社区惯例署名 - Dockerfile 改用公共基础镜像 - 拼错的 RADEME.md → README.md，README 重写 - 需要真实数据库的用例改为可用 IOTLCM_TEST_WITH_DB=1 显式启用 - 补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 数据库 schema 不随本仓发布（生产 DDL 已过时），README 已说明。
This commit is contained in:
MiFaZhan
@@ -0,0 +1,609 @@
|
||||
package allocate
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/excel"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/iot/simcardbatch/common"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/do"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultAllocateChunkSize = 5000
|
||||
defaultAllocateMaxFileSize = 50 << 20
|
||||
minAllocateChunkSize = 100
|
||||
maxAllocateChunkSize = 10000
|
||||
allocateDetailInsertSize = 500
|
||||
allocateParentJobRole = "allocate_parent"
|
||||
allocateChunkJobRole = "allocate_chunk"
|
||||
)
|
||||
|
||||
type allocateSplitState struct {
|
||||
ParentBatchNo string
|
||||
ChunkSize int
|
||||
ChunkCount int
|
||||
Total int
|
||||
Valid int
|
||||
Failed int
|
||||
}
|
||||
|
||||
type allocateChunkStream func(context.Context, func([]batchimport.Row) error) (int, error)
|
||||
|
||||
func allocateChunkSize(ctx context.Context) int {
|
||||
size := g.Cfg().MustGet(ctx, "batch.allocate.chunkSize", defaultAllocateChunkSize).Int()
|
||||
if size < minAllocateChunkSize {
|
||||
return minAllocateChunkSize
|
||||
}
|
||||
if size > maxAllocateChunkSize {
|
||||
return maxAllocateChunkSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func allocateMaxFileSize(ctx context.Context) int64 {
|
||||
size := g.Cfg().MustGet(ctx, "batch.allocate.maxFileSize", defaultAllocateMaxFileSize).Int64()
|
||||
if size <= 0 {
|
||||
return defaultAllocateMaxFileSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func createPendingSplit(
|
||||
ctx context.Context,
|
||||
req batchimport.SubmitRequest,
|
||||
p batchimport.PendingProcessor,
|
||||
options batchimport.PendingOptions,
|
||||
parentBatchNo string,
|
||||
importOss *model.SysOssSaveContentModel,
|
||||
stream allocateChunkStream,
|
||||
) (*PendingResult, error) {
|
||||
units, err := allocationunits.Snapshot(ctx, req.Operator.TenantId, req.Operator.DeptId, req.TargetDeptId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts, ok := req.BizParams.(*SubmitOptions); ok {
|
||||
opts.Units = units
|
||||
}
|
||||
|
||||
if err := batchmetadata.ValidateCreation(req.SourceChannel, req.ExecutionMode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p == nil {
|
||||
return nil, gerror.New("allocate processor is required")
|
||||
}
|
||||
if strings.TrimSpace(parentBatchNo) == "" {
|
||||
return nil, gerror.New("parent batch number is required")
|
||||
}
|
||||
if stream == nil {
|
||||
return nil, gerror.New("allocate input stream is required")
|
||||
}
|
||||
|
||||
chunkSize := allocateChunkSize(ctx)
|
||||
state := &allocateSplitState{
|
||||
ParentBatchNo: parentBatchNo,
|
||||
ChunkSize: chunkSize,
|
||||
}
|
||||
if err := createAllocateParent(ctx, req, p, parentBatchNo, importOss, chunkSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err := stream(ctx, func(rows []batchimport.Row) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := p.ValidateRows(ctx, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
if options.PrepareRows != nil {
|
||||
if err := options.PrepareRows(ctx, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := markAllocateCrossChunkDuplicates(ctx, parentBatchNo, req.Operator.TenantId, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
state.ChunkCount++
|
||||
result, err := createAllocateChunk(ctx, req, p, parentBatchNo, state.ChunkCount, rows, importOss, state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return common.PublishBatchMessageStrict(ctx, result, consts.QueueBatchExecute, "allocate")
|
||||
}); err != nil {
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
if state.Total == 0 {
|
||||
err := gerror.New("allocate input has no data rows")
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
if err := markAllocateSplitCompleted(ctx, parentBatchNo, state); err != nil {
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parentMessage := &batchimport.Result{
|
||||
BatchNo: parentBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
}
|
||||
if err := common.PublishBatchMessageStrict(ctx, parentMessage, consts.QueueBatchExecute, "allocate_parent"); err != nil {
|
||||
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &PendingResult{
|
||||
BatchNo: parentBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
|
||||
Total: state.Total,
|
||||
ValidCount: state.Valid,
|
||||
FailCount: state.Failed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func createAllocateParent(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, importOss *model.SysOssSaveContentModel, chunkSize int) error {
|
||||
result := &batchimport.Result{
|
||||
BatchNo: parentBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
ImportFileOssId: ossIDFromContent(importOss),
|
||||
ImportFileUrl: urlFromContent(importOss),
|
||||
}
|
||||
info := p.BuildBatchInfo(ctx, req, result, importOss, nil, 0)
|
||||
info = withAllocateInfo(info, map[string]any{
|
||||
"targetBatchType": consts.BatchTypeAllocate,
|
||||
"jobRole": allocateParentJobRole,
|
||||
"autoSplit": true,
|
||||
"chunkSize": chunkSize,
|
||||
"chunkCount": 0,
|
||||
"completedChunkCount": 0,
|
||||
"parsedCount": 0,
|
||||
})
|
||||
now := gtime.Now()
|
||||
_, err := g.DB().Model(dao.BatchMain.Table()).Data(do.BatchMain{
|
||||
BatchNo: parentBatchNo,
|
||||
OrderSn: "",
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusPending,
|
||||
BatchStatus: consts.BatchStatusNormal,
|
||||
StartTime: now,
|
||||
SourceType: req.SourceType,
|
||||
SourceChannel: batchmetadata.Channel(req.SourceChannel),
|
||||
ExecutionMode: req.ExecutionMode,
|
||||
SourceOssId: ossIDFromContent(importOss),
|
||||
IsCrossDept: 0,
|
||||
TargetDeptId: req.TargetDeptId,
|
||||
DeptId: req.Operator.DeptId,
|
||||
TotalCount: 0,
|
||||
ValidCount: 0,
|
||||
SuccessCount: 0,
|
||||
FailCount: 0,
|
||||
RunResult: consts.BatchRunResultPending,
|
||||
BatchInfo: info,
|
||||
TenantId: req.Operator.TenantId,
|
||||
CreatedDept: req.Operator.DeptId,
|
||||
CreatedBy: req.Operator.UserId,
|
||||
CreatedAt: now,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
}).Insert()
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "insert allocate parent batch failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createAllocateChunk(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, chunkNo int, rows []batchimport.Row, importOss *model.SysOssSaveContentModel, state *allocateSplitState) (*batchimport.Result, error) {
|
||||
childBatchNo, err := batchimport.NextRandomNo(consts.BatchNoPrefixAllocate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
validCount, failCount := countBatchRows(rows)
|
||||
result := &batchimport.Result{
|
||||
ImportStatus: consts.ImportStatusSuccess,
|
||||
BatchNo: childBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
TotalRows: len(rows),
|
||||
ValidRows: validCount,
|
||||
ErrorRows: failCount,
|
||||
ImportFileOssId: ossIDFromContent(importOss),
|
||||
ImportFileUrl: urlFromContent(importOss),
|
||||
}
|
||||
childReq := req
|
||||
childReq.BatchNo = childBatchNo
|
||||
childReq.SourceBatchNo = parentBatchNo
|
||||
info := p.BuildBatchInfo(ctx, childReq, result, importOss, nil, 0)
|
||||
info = withAllocateInfo(info, map[string]any{
|
||||
"targetBatchType": consts.BatchTypeAllocate,
|
||||
"jobRole": allocateChunkJobRole,
|
||||
"parentBatchNo": parentBatchNo,
|
||||
"chunkNo": chunkNo,
|
||||
"chunkSize": len(rows),
|
||||
})
|
||||
now := gtime.Now()
|
||||
err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Data(do.BatchMain{
|
||||
BatchNo: childBatchNo,
|
||||
OrderSn: "",
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusPending,
|
||||
BatchStatus: consts.BatchStatusNormal,
|
||||
SourceType: req.SourceType,
|
||||
SourceChannel: batchmetadata.Channel(req.SourceChannel),
|
||||
ExecutionMode: req.ExecutionMode,
|
||||
SourceBatchNo: parentBatchNo,
|
||||
SourceOssId: ossIDFromContent(importOss),
|
||||
IsCrossDept: 0,
|
||||
TargetDeptId: req.TargetDeptId,
|
||||
DeptId: req.Operator.DeptId,
|
||||
TotalCount: len(rows),
|
||||
ValidCount: validCount,
|
||||
SuccessCount: 0,
|
||||
FailCount: failCount,
|
||||
RunResult: consts.BatchRunResultPending,
|
||||
BatchInfo: info,
|
||||
TenantId: req.Operator.TenantId,
|
||||
CreatedDept: req.Operator.DeptId,
|
||||
CreatedBy: req.Operator.UserId,
|
||||
CreatedAt: now,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
}).Insert(); err != nil {
|
||||
return gerror.Wrap(err, "insert allocate child batch failed")
|
||||
}
|
||||
|
||||
details := make([]do.BatchDetail, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
status := consts.BatchDetailStatusPending
|
||||
if len(row.GetErrorReasons()) > 0 {
|
||||
status = consts.BatchDetailStatusFailed
|
||||
}
|
||||
details = append(details, do.BatchDetail{
|
||||
BatchNo: childBatchNo,
|
||||
BatchType: consts.BatchTypeAllocate,
|
||||
DetailStatus: status,
|
||||
BlankOutStatus: consts.BlankOutStatusNo,
|
||||
RowNo: row.GetRowNo(),
|
||||
ItemId: row.GetItemID(),
|
||||
ErrorReason: strings.Join(row.GetErrorReasons(), "; "),
|
||||
Snapshot: allocationunits.Merge(row.Snapshot(), sourceUnitInfo(info)),
|
||||
TenantId: req.Operator.TenantId,
|
||||
CreatedDept: req.Operator.DeptId,
|
||||
CreatedBy: req.Operator.UserId,
|
||||
CreatedAt: now,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
for start := 0; start < len(details); start += allocateDetailInsertSize {
|
||||
end := start + allocateDetailInsertSize
|
||||
if end > len(details) {
|
||||
end = len(details)
|
||||
}
|
||||
if _, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Data(details[start:end]).Insert(); err != nil {
|
||||
return gerror.Wrapf(err, "insert allocate details failed, range:%d-%d", start+1, end)
|
||||
}
|
||||
}
|
||||
|
||||
parent := &entity.BatchMain{}
|
||||
columns := dao.BatchMain.Columns()
|
||||
if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
||||
Where(columns.BatchNo, parentBatchNo).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
LockUpdate().Scan(parent); err != nil {
|
||||
return gerror.Wrap(err, "load allocate parent batch failed")
|
||||
}
|
||||
if parent.Id == 0 {
|
||||
return gerror.New("allocate parent batch does not exist")
|
||||
}
|
||||
parentInfo := withAllocateInfo(parent.BatchInfo, map[string]any{
|
||||
"chunkCount": chunkNo,
|
||||
"parsedCount": parent.TotalCount + len(rows),
|
||||
})
|
||||
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
|
||||
TotalCount: parent.TotalCount + len(rows),
|
||||
ValidCount: parent.ValidCount + validCount,
|
||||
FailCount: parent.FailCount + failCount,
|
||||
BatchInfo: parentInfo,
|
||||
UpdatedBy: req.Operator.UserId,
|
||||
UpdatedAt: now,
|
||||
}).Update(); err != nil {
|
||||
return gerror.Wrap(err, "update allocate parent split progress failed")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if state != nil {
|
||||
state.Total += len(rows)
|
||||
state.Valid += validCount
|
||||
state.Failed += failCount
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func markAllocateSplitCompleted(ctx context.Context, parentBatchNo string, state *allocateSplitState) error {
|
||||
columns := dao.BatchMain.Columns()
|
||||
parent := &entity.BatchMain{}
|
||||
if err := g.DB().Model(dao.BatchMain.Table()).
|
||||
Where(columns.BatchNo, parentBatchNo).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
Scan(parent); err != nil {
|
||||
return err
|
||||
}
|
||||
if parent.Id == 0 {
|
||||
return gerror.New("allocate parent batch does not exist")
|
||||
}
|
||||
info := withAllocateInfo(parent.BatchInfo, map[string]any{
|
||||
"splitCompleted": true,
|
||||
"chunkCount": state.ChunkCount,
|
||||
"completedChunkCount": 0,
|
||||
"parsedCount": state.Total,
|
||||
})
|
||||
_, err := g.DB().Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
|
||||
RunResult: consts.BatchRunResultRunning,
|
||||
BatchInfo: info,
|
||||
UpdatedAt: gtime.Now(),
|
||||
}).Update()
|
||||
return err
|
||||
}
|
||||
|
||||
func markAllocateParentFailed(ctx context.Context, parentBatchNo, reason string) {
|
||||
if strings.TrimSpace(parentBatchNo) == "" {
|
||||
return
|
||||
}
|
||||
now := gtime.Now()
|
||||
columns := dao.BatchMain.Columns()
|
||||
_, _ = g.DB().Model(dao.BatchMain.Table()).
|
||||
Where(columns.BatchNo, parentBatchNo).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
WhereIn(columns.BatchAsyncStatus, []int{consts.BatchAsyncStatusPending, consts.BatchAsyncStatusProcessing}).
|
||||
Data(do.BatchMain{
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
||||
RunResult: consts.BatchRunResultFailed,
|
||||
FailReason: strings.TrimSpace(reason),
|
||||
EndTime: now,
|
||||
UpdatedAt: now,
|
||||
}).Update()
|
||||
}
|
||||
|
||||
func countBatchRows(rows []batchimport.Row) (valid, failed int) {
|
||||
for _, row := range rows {
|
||||
if len(row.GetErrorReasons()) == 0 {
|
||||
valid++
|
||||
} else {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenantID string, rows []batchimport.Row) error {
|
||||
if strings.TrimSpace(parentBatchNo) == "" || len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
columns := dao.BatchDetail.Columns()
|
||||
itemIDs := make([]string, 0, len(rows))
|
||||
seen := make(map[string]struct{}, len(rows))
|
||||
for _, item := range rows {
|
||||
row, ok := item.(*Row)
|
||||
if !ok || row == nil {
|
||||
continue
|
||||
}
|
||||
itemID := strings.TrimSpace(row.Iccid)
|
||||
if itemID == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[itemID]; exists {
|
||||
continue
|
||||
}
|
||||
seen[itemID] = struct{}{}
|
||||
itemIDs = append(itemIDs, itemID)
|
||||
}
|
||||
if len(itemIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var existing []struct {
|
||||
ItemId string `orm:"item_id"`
|
||||
}
|
||||
query := g.DB().Model(dao.BatchDetail.Table()).
|
||||
Fields(columns.ItemId).
|
||||
Where(columns.BatchType, consts.BatchTypeAllocate).
|
||||
Where(columns.TenantId, tenantID).
|
||||
WhereIn(columns.ItemId, itemIDs).
|
||||
Where("batch_no IN (SELECT batch_no FROM batch_main WHERE source_batch_no = ? AND batch_type = ? AND tenant_id = ?)",
|
||||
parentBatchNo, consts.BatchTypeAllocate, tenantID)
|
||||
if err := query.Scan(&existing); err != nil {
|
||||
return gerror.Wrap(err, "check duplicate allocate ICCIDs across chunks failed")
|
||||
}
|
||||
if len(existing) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
duplicates := make(map[string]struct{}, len(existing))
|
||||
for _, item := range existing {
|
||||
if itemID := strings.TrimSpace(item.ItemId); itemID != "" {
|
||||
duplicates[itemID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, item := range rows {
|
||||
row, ok := item.(*Row)
|
||||
if !ok || row == nil {
|
||||
continue
|
||||
}
|
||||
if _, exists := duplicates[strings.TrimSpace(row.Iccid)]; exists {
|
||||
row.AddErrorReason("ICCID appears in an earlier allocation chunk")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func withAllocateInfo(info *gjson.Json, values map[string]any) *gjson.Json {
|
||||
data := make(map[string]any)
|
||||
if info != nil {
|
||||
for key, value := range info.Map() {
|
||||
data[key] = value
|
||||
}
|
||||
}
|
||||
for key, value := range values {
|
||||
data[key] = value
|
||||
}
|
||||
return gjson.New(data)
|
||||
}
|
||||
|
||||
func ossIDFromContent(content *model.SysOssSaveContentModel) int64 {
|
||||
if content == nil || content.SysOss == nil {
|
||||
return 0
|
||||
}
|
||||
return content.OssId
|
||||
}
|
||||
|
||||
func urlFromContent(content *model.SysOssSaveContentModel) string {
|
||||
if content == nil || content.SysOss == nil {
|
||||
return ""
|
||||
}
|
||||
return content.Url
|
||||
}
|
||||
|
||||
func streamAllocateExcel(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
||||
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
|
||||
return excel.StreamFile(ctx, filePath, excel.ParseOptions{
|
||||
RequiredHeaders: []string{"ICCID"},
|
||||
SingleSheet: true,
|
||||
}, handler)
|
||||
})
|
||||
}
|
||||
|
||||
func streamAllocateCSV(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
||||
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
|
||||
return excel.StreamCSVFile(ctx, filePath, excel.ParseOptions{
|
||||
RequiredHeaders: []string{"ICCID"},
|
||||
}, handler)
|
||||
})
|
||||
}
|
||||
|
||||
func streamAllocateParsedFile(ctx context.Context, chunkSize int, emit func([]batchimport.Row) error, stream func(func(excel.ParsedRow) error) (int, error)) (int, error) {
|
||||
rows := make([]batchimport.Row, 0, chunkSize)
|
||||
total, err := stream(func(parsed excel.ParsedRow) error {
|
||||
rows = append(rows, rowFromExcel(parsed))
|
||||
if len(rows) < chunkSize {
|
||||
return nil
|
||||
}
|
||||
if err := emit(rows); err != nil {
|
||||
return err
|
||||
}
|
||||
rows = make([]batchimport.Row, 0, chunkSize)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
if err := emit(rows); err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func streamAllocateInput(ctx context.Context, input string, targetDeptID int64, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
|
||||
scanner := bufio.NewScanner(strings.NewReader(input))
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
rows := make([]batchimport.Row, 0, chunkSize)
|
||||
total := 0
|
||||
rowNo := 0
|
||||
for scanner.Scan() {
|
||||
rowNo++
|
||||
if err := ctx.Err(); err != nil {
|
||||
return total, err
|
||||
}
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
row := &Row{RowNo: rowNo, TargetDeptId: targetDeptID}
|
||||
fields := strings.Fields(line)
|
||||
switch len(fields) {
|
||||
case 1:
|
||||
row.Iccid = fields[0]
|
||||
case 3:
|
||||
row.Iccid = fields[0]
|
||||
row.Msisdn = fields[1]
|
||||
row.Imsi = fields[2]
|
||||
default:
|
||||
row.AddErrorReason("invalid input format; expected ICCID or ICCID MSISDN IMSI")
|
||||
}
|
||||
rows = append(rows, row)
|
||||
total++
|
||||
if len(rows) == chunkSize {
|
||||
if err := emit(rows); err != nil {
|
||||
return total, err
|
||||
}
|
||||
rows = make([]batchimport.Row, 0, chunkSize)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return total, err
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, gerror.New("allocate input has no data rows")
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
if err := emit(rows); err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func rowFromExcel(parsed excel.ParsedRow) *Row {
|
||||
row := &Row{
|
||||
RowNo: parsed.RowNo,
|
||||
Iccid: excel.Field(parsed, "ICCID"),
|
||||
Msisdn: excel.Field(parsed, "MSISDN"),
|
||||
Imsi: excel.Field(parsed, "IMSI"),
|
||||
TargetDeptKey: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u7f16\u7801", "\u5ba2\u6237\u7f16\u7801", "TARGETDEPTKEY", "TARGET_CUSTOMER_CODE"),
|
||||
TargetDeptName: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u540d\u79f0", "\u5ba2\u6237\u540d\u79f0", "TARGETDEPTNAME", "TARGET_CUSTOMER_NAME"),
|
||||
Remark: firstExcelField(parsed, "\u5212\u62e8\u5907\u6ce8", "\u5907\u6ce8", "REMARK"),
|
||||
}
|
||||
if value := firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237ID", "\u76ee\u6807\u90e8\u95e8ID", "TARGETDEPTID", "TARGET_CUSTOMER_ID"); value != "" {
|
||||
if targetDeptID, err := parseTargetDeptID(value); err == nil {
|
||||
row.TargetDeptId = targetDeptID
|
||||
} else {
|
||||
row.AddErrorReason("invalid target customer id")
|
||||
}
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func parseTargetDeptID(value string) (int64, error) {
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
|
||||
if err != nil || parsed <= 0 {
|
||||
return 0, gerror.New("invalid target customer id")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func sourceUnitInfo(info *gjson.Json) *gjson.Json {
|
||||
return gjson.New(map[string]any{"sourceDeptId": info.Get("sourceDeptId").Int64(), "sourceDeptKey": info.Get("sourceDeptKey").String(), "sourceDeptName": info.Get("sourceDeptName").String()})
|
||||
}
|
||||
Reference in New Issue
Block a user