chore: 初始开源发布

iotlcm-admin-svr:管理后台服务。

- Go module:iotlcm → git.5iot.com/iotlcm-public/iotlcm-admin-svr(729 文件)
- 凭据与地址脱敏:JWT signingKey(含代码里的 MustGet 默认值)、
  MySQL DSN、Redis 口令、告警邮件发件人邮箱全部改为占位;
  私网 IP → 文档地址段;库名 xiujie_admin → iotlcm
- 真实手机号/邮箱 → 合法 11 位编造号 / example.com
- 移除内部资产:CI 触发器、过时 DDL(sql/db_iotlcm.sql)、
  私有依赖 go get 行、明文 registry 登录口令
- 启动 banner 统一 IOTLCM 方块字 + 社区惯例署名
- Dockerfile 改用公共基础镜像
- 拼错的 RADEME.md → README.md,README 重写
- 需要真实数据库的用例改为可用 IOTLCM_TEST_WITH_DB=1 显式启用
- 补 Apache-2.0 LICENSE,LF 行尾 + .gitattributes

数据库 schema 不随本仓发布(生产 DDL 已过时),README 已说明。
This commit is contained in:
MiFaZhan
2026-10-23 14:46:05 +08:00
commit 59eb34976a
1132 changed files with 138197 additions and 0 deletions
@@ -0,0 +1,81 @@
package allocate
import (
"context"
"errors"
"strconv"
"strings"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/excel"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/text"
"github.com/gogf/gf/v2/errors/gerror"
)
func ParseExcelFileRows(ctx context.Context, filePath string) ([]batchimport.Row, error) {
parsedRows, err := excel.ParseFile(ctx, filePath, excel.ParseOptions{
RequiredHeaders: []string{"ICCID"},
SingleSheet: true,
})
if err != nil {
return nil, err
}
rows := make([]batchimport.Row, 0, len(parsedRows))
for _, parsed := range parsedRows {
row := &Row{
RowNo: parsed.RowNo,
Iccid: excel.Field(parsed, "ICCID"),
Msisdn: excel.Field(parsed, "MSISDN"),
Imsi: excel.Field(parsed, "IMSI"),
TargetDeptKey: firstExcelField(parsed, "目标客户编码", "客户编码", "TARGETDEPTKEY", "TARGET_CUSTOMER_CODE"),
TargetDeptName: firstExcelField(parsed, "目标客户名称", "客户名称", "TARGETDEPTNAME", "TARGET_CUSTOMER_NAME"),
Remark: firstExcelField(parsed, "划拨备注", "备注", "REMARK"),
}
if value := firstExcelField(parsed, "目标客户ID", "目标部门ID", "TARGETDEPTID", "TARGET_CUSTOMER_ID"); value != "" {
targetDeptID, parseErr := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
if parseErr != nil || targetDeptID <= 0 {
row.AddErrorReason("目标客户ID格式错误")
} else {
row.TargetDeptId = targetDeptID
}
}
rows = append(rows, row)
}
return rows, nil
}
func ParseInputRows(ctx context.Context, input string, targetDeptID int64) ([]batchimport.Row, error) {
lines, err := text.ParseWhitespaceFields(ctx, input)
if err != nil {
if errors.Is(err, text.ErrEmptyInput) {
return nil, gerror.New("批量输入内容不能为空")
}
return nil, err
}
rows := make([]batchimport.Row, 0, len(lines))
for _, line := range lines {
row := &Row{RowNo: line.RowNo, TargetDeptId: targetDeptID}
switch len(line.Fields) {
case 1:
row.Iccid = line.Fields[0]
case 3:
row.Iccid = line.Fields[0]
row.Msisdn = line.Fields[1]
row.Imsi = line.Fields[2]
default:
row.AddErrorReason("格式错误,应为 ICCID 或 ICCID MSISDN IMSI")
}
rows = append(rows, row)
}
return rows, nil
}
func firstExcelField(row excel.ParsedRow, names ...string) string {
for _, name := range names {
if value := excel.Field(row, name); value != "" {
return value
}
}
return ""
}
@@ -0,0 +1,117 @@
package allocate
import (
"context"
"fmt"
"strings"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"github.com/gogf/gf/v2/encoding/gjson"
"github.com/gogf/gf/v2/frame/g"
)
const (
importDir = "sim-card-allocate/import"
resultDir = "sim-card-allocate/result"
maxFileSize = 50 << 20
maxAllocateRows = 10000
)
type Processor struct{}
func NewProcessor() *Processor {
return &Processor{}
}
func (p *Processor) Config() batchimport.ProcessorConfig {
return batchimport.ProcessorConfig{
Code: "sim_card_allocate",
BatchType: consts.BatchTypeAllocate,
BatchNoPrefix: consts.BatchNoPrefixAllocate,
TaskName: "卡划拨结果",
ExportScene: consts.ResourceExportSceneAllocate,
ImportDir: importDir,
ResultDir: resultDir,
MaxFileSize: maxFileSize,
MaxRows: maxAllocateRows,
TemplateHeaders: []string{"ICCID", "MSISDN", "IMSI", "目标客户名称"},
RequiredHeaders: []string{"ICCID"},
}
}
func (p *Processor) ParseRows(ctx context.Context, filePath string) ([]batchimport.Row, error) {
return ParseExcelFileRows(ctx, filePath)
}
func (p *Processor) ValidateRows(_ context.Context, rows []batchimport.Row) error {
firstByIccid := make(map[string]*Row, len(rows))
for _, item := range rows {
row, ok := item.(*Row)
if !ok || row == nil {
continue
}
row.Iccid = strings.TrimSpace(row.Iccid)
row.Msisdn = strings.TrimSpace(row.Msisdn)
row.Imsi = strings.TrimSpace(row.Imsi)
row.TargetDeptKey = strings.TrimSpace(row.TargetDeptKey)
row.TargetDeptName = strings.TrimSpace(row.TargetDeptName)
row.Remark = strings.TrimSpace(row.Remark)
if row.Iccid == "" {
row.AddErrorReason("ICCID不能为空")
continue
}
if first, exists := firstByIccid[row.Iccid]; exists {
row.AddErrorReason(fmt.Sprintf("ICCID与第%d行重复", first.RowNo))
first.AddErrorReason("ICCID重复")
continue
}
firstByIccid[row.Iccid] = row
}
return nil
}
func (p *Processor) BuildBatchInfo(_ context.Context, req batchimport.SubmitRequest, result *batchimport.Result, importOss *model.SysOssSaveContentModel, resultOss *model.SysOssSaveContentModel, downloadId int64) *gjson.Json {
options := optionsFromParams(req.BizParams)
info := g.Map{
"operation": "sim_card_allocate",
"configName": "卡划拨",
"submitScene": submitScene(req.SourceType),
"targetDeptId": options.TargetDeptId,
"remark": strings.TrimSpace(options.Remark),
"resultFile": g.Map{"status": "pending"},
}
if result != nil && result.ImportFileOssId > 0 {
info["importFileOssId"] = result.ImportFileOssId
} else if importOss != nil && importOss.OssId > 0 {
info["importFileOssId"] = importOss.OssId
}
if resultOss != nil && resultOss.OssId > 0 {
info["resultFileOssId"] = resultOss.OssId
}
if downloadId > 0 {
info["resultDownloadId"] = downloadId
}
return allocationunits.Merge(gjson.New(info), options.Units)
}
func optionsFromParams(params any) SubmitOptions {
if options, ok := params.(*SubmitOptions); ok && options != nil {
return *options
}
return SubmitOptions{}
}
func submitScene(sourceType int) string {
switch sourceType {
case consts.BatchSourceTypeFile:
return "excel"
case consts.BatchSourceTypeBatchInput:
return "input"
default:
return ""
}
}
@@ -0,0 +1,609 @@
package allocate
import (
"bufio"
"context"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
"strconv"
"strings"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchinput/excel"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/iot/simcardbatch/common"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/encoding/gjson"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
const (
defaultAllocateChunkSize = 5000
defaultAllocateMaxFileSize = 50 << 20
minAllocateChunkSize = 100
maxAllocateChunkSize = 10000
allocateDetailInsertSize = 500
allocateParentJobRole = "allocate_parent"
allocateChunkJobRole = "allocate_chunk"
)
type allocateSplitState struct {
ParentBatchNo string
ChunkSize int
ChunkCount int
Total int
Valid int
Failed int
}
type allocateChunkStream func(context.Context, func([]batchimport.Row) error) (int, error)
func allocateChunkSize(ctx context.Context) int {
size := g.Cfg().MustGet(ctx, "batch.allocate.chunkSize", defaultAllocateChunkSize).Int()
if size < minAllocateChunkSize {
return minAllocateChunkSize
}
if size > maxAllocateChunkSize {
return maxAllocateChunkSize
}
return size
}
func allocateMaxFileSize(ctx context.Context) int64 {
size := g.Cfg().MustGet(ctx, "batch.allocate.maxFileSize", defaultAllocateMaxFileSize).Int64()
if size <= 0 {
return defaultAllocateMaxFileSize
}
return size
}
func createPendingSplit(
ctx context.Context,
req batchimport.SubmitRequest,
p batchimport.PendingProcessor,
options batchimport.PendingOptions,
parentBatchNo string,
importOss *model.SysOssSaveContentModel,
stream allocateChunkStream,
) (*PendingResult, error) {
units, err := allocationunits.Snapshot(ctx, req.Operator.TenantId, req.Operator.DeptId, req.TargetDeptId)
if err != nil {
return nil, err
}
if opts, ok := req.BizParams.(*SubmitOptions); ok {
opts.Units = units
}
if err := batchmetadata.ValidateCreation(req.SourceChannel, req.ExecutionMode); err != nil {
return nil, err
}
if p == nil {
return nil, gerror.New("allocate processor is required")
}
if strings.TrimSpace(parentBatchNo) == "" {
return nil, gerror.New("parent batch number is required")
}
if stream == nil {
return nil, gerror.New("allocate input stream is required")
}
chunkSize := allocateChunkSize(ctx)
state := &allocateSplitState{
ParentBatchNo: parentBatchNo,
ChunkSize: chunkSize,
}
if err := createAllocateParent(ctx, req, p, parentBatchNo, importOss, chunkSize); err != nil {
return nil, err
}
if _, err := stream(ctx, func(rows []batchimport.Row) error {
if len(rows) == 0 {
return nil
}
if err := p.ValidateRows(ctx, rows); err != nil {
return err
}
if options.PrepareRows != nil {
if err := options.PrepareRows(ctx, rows); err != nil {
return err
}
}
if err := markAllocateCrossChunkDuplicates(ctx, parentBatchNo, req.Operator.TenantId, rows); err != nil {
return err
}
state.ChunkCount++
result, err := createAllocateChunk(ctx, req, p, parentBatchNo, state.ChunkCount, rows, importOss, state)
if err != nil {
return err
}
return common.PublishBatchMessageStrict(ctx, result, consts.QueueBatchExecute, "allocate")
}); err != nil {
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
return nil, err
}
if state.Total == 0 {
err := gerror.New("allocate input has no data rows")
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
return nil, err
}
if err := markAllocateSplitCompleted(ctx, parentBatchNo, state); err != nil {
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
return nil, err
}
parentMessage := &batchimport.Result{
BatchNo: parentBatchNo,
BatchType: consts.BatchTypeAllocate,
}
if err := common.PublishBatchMessageStrict(ctx, parentMessage, consts.QueueBatchExecute, "allocate_parent"); err != nil {
markAllocateParentFailed(ctx, parentBatchNo, err.Error())
return nil, err
}
return &PendingResult{
BatchNo: parentBatchNo,
BatchType: consts.BatchTypeAllocate,
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
Total: state.Total,
ValidCount: state.Valid,
FailCount: state.Failed,
}, nil
}
func createAllocateParent(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, importOss *model.SysOssSaveContentModel, chunkSize int) error {
result := &batchimport.Result{
BatchNo: parentBatchNo,
BatchType: consts.BatchTypeAllocate,
ImportFileOssId: ossIDFromContent(importOss),
ImportFileUrl: urlFromContent(importOss),
}
info := p.BuildBatchInfo(ctx, req, result, importOss, nil, 0)
info = withAllocateInfo(info, map[string]any{
"targetBatchType": consts.BatchTypeAllocate,
"jobRole": allocateParentJobRole,
"autoSplit": true,
"chunkSize": chunkSize,
"chunkCount": 0,
"completedChunkCount": 0,
"parsedCount": 0,
})
now := gtime.Now()
_, err := g.DB().Model(dao.BatchMain.Table()).Data(do.BatchMain{
BatchNo: parentBatchNo,
OrderSn: "",
BatchType: consts.BatchTypeAllocate,
BatchAsyncStatus: consts.BatchAsyncStatusPending,
BatchStatus: consts.BatchStatusNormal,
StartTime: now,
SourceType: req.SourceType,
SourceChannel: batchmetadata.Channel(req.SourceChannel),
ExecutionMode: req.ExecutionMode,
SourceOssId: ossIDFromContent(importOss),
IsCrossDept: 0,
TargetDeptId: req.TargetDeptId,
DeptId: req.Operator.DeptId,
TotalCount: 0,
ValidCount: 0,
SuccessCount: 0,
FailCount: 0,
RunResult: consts.BatchRunResultPending,
BatchInfo: info,
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
CreatedAt: now,
UpdatedBy: req.Operator.UserId,
UpdatedAt: now,
}).Insert()
if err != nil {
return gerror.Wrap(err, "insert allocate parent batch failed")
}
return nil
}
func createAllocateChunk(ctx context.Context, req batchimport.SubmitRequest, p batchimport.PendingProcessor, parentBatchNo string, chunkNo int, rows []batchimport.Row, importOss *model.SysOssSaveContentModel, state *allocateSplitState) (*batchimport.Result, error) {
childBatchNo, err := batchimport.NextRandomNo(consts.BatchNoPrefixAllocate)
if err != nil {
return nil, err
}
validCount, failCount := countBatchRows(rows)
result := &batchimport.Result{
ImportStatus: consts.ImportStatusSuccess,
BatchNo: childBatchNo,
BatchType: consts.BatchTypeAllocate,
TotalRows: len(rows),
ValidRows: validCount,
ErrorRows: failCount,
ImportFileOssId: ossIDFromContent(importOss),
ImportFileUrl: urlFromContent(importOss),
}
childReq := req
childReq.BatchNo = childBatchNo
childReq.SourceBatchNo = parentBatchNo
info := p.BuildBatchInfo(ctx, childReq, result, importOss, nil, 0)
info = withAllocateInfo(info, map[string]any{
"targetBatchType": consts.BatchTypeAllocate,
"jobRole": allocateChunkJobRole,
"parentBatchNo": parentBatchNo,
"chunkNo": chunkNo,
"chunkSize": len(rows),
})
now := gtime.Now()
err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Data(do.BatchMain{
BatchNo: childBatchNo,
OrderSn: "",
BatchType: consts.BatchTypeAllocate,
BatchAsyncStatus: consts.BatchAsyncStatusPending,
BatchStatus: consts.BatchStatusNormal,
SourceType: req.SourceType,
SourceChannel: batchmetadata.Channel(req.SourceChannel),
ExecutionMode: req.ExecutionMode,
SourceBatchNo: parentBatchNo,
SourceOssId: ossIDFromContent(importOss),
IsCrossDept: 0,
TargetDeptId: req.TargetDeptId,
DeptId: req.Operator.DeptId,
TotalCount: len(rows),
ValidCount: validCount,
SuccessCount: 0,
FailCount: failCount,
RunResult: consts.BatchRunResultPending,
BatchInfo: info,
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
CreatedAt: now,
UpdatedBy: req.Operator.UserId,
UpdatedAt: now,
}).Insert(); err != nil {
return gerror.Wrap(err, "insert allocate child batch failed")
}
details := make([]do.BatchDetail, 0, len(rows))
for _, row := range rows {
status := consts.BatchDetailStatusPending
if len(row.GetErrorReasons()) > 0 {
status = consts.BatchDetailStatusFailed
}
details = append(details, do.BatchDetail{
BatchNo: childBatchNo,
BatchType: consts.BatchTypeAllocate,
DetailStatus: status,
BlankOutStatus: consts.BlankOutStatusNo,
RowNo: row.GetRowNo(),
ItemId: row.GetItemID(),
ErrorReason: strings.Join(row.GetErrorReasons(), "; "),
Snapshot: allocationunits.Merge(row.Snapshot(), sourceUnitInfo(info)),
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
CreatedAt: now,
UpdatedBy: req.Operator.UserId,
UpdatedAt: now,
})
}
for start := 0; start < len(details); start += allocateDetailInsertSize {
end := start + allocateDetailInsertSize
if end > len(details) {
end = len(details)
}
if _, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Data(details[start:end]).Insert(); err != nil {
return gerror.Wrapf(err, "insert allocate details failed, range:%d-%d", start+1, end)
}
}
parent := &entity.BatchMain{}
columns := dao.BatchMain.Columns()
if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
Where(columns.BatchNo, parentBatchNo).
Where(columns.BatchType, consts.BatchTypeAllocate).
LockUpdate().Scan(parent); err != nil {
return gerror.Wrap(err, "load allocate parent batch failed")
}
if parent.Id == 0 {
return gerror.New("allocate parent batch does not exist")
}
parentInfo := withAllocateInfo(parent.BatchInfo, map[string]any{
"chunkCount": chunkNo,
"parsedCount": parent.TotalCount + len(rows),
})
if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
TotalCount: parent.TotalCount + len(rows),
ValidCount: parent.ValidCount + validCount,
FailCount: parent.FailCount + failCount,
BatchInfo: parentInfo,
UpdatedBy: req.Operator.UserId,
UpdatedAt: now,
}).Update(); err != nil {
return gerror.Wrap(err, "update allocate parent split progress failed")
}
return nil
})
if err != nil {
return nil, err
}
if state != nil {
state.Total += len(rows)
state.Valid += validCount
state.Failed += failCount
}
return result, nil
}
func markAllocateSplitCompleted(ctx context.Context, parentBatchNo string, state *allocateSplitState) error {
columns := dao.BatchMain.Columns()
parent := &entity.BatchMain{}
if err := g.DB().Model(dao.BatchMain.Table()).
Where(columns.BatchNo, parentBatchNo).
Where(columns.BatchType, consts.BatchTypeAllocate).
Scan(parent); err != nil {
return err
}
if parent.Id == 0 {
return gerror.New("allocate parent batch does not exist")
}
info := withAllocateInfo(parent.BatchInfo, map[string]any{
"splitCompleted": true,
"chunkCount": state.ChunkCount,
"completedChunkCount": 0,
"parsedCount": state.Total,
})
_, err := g.DB().Model(dao.BatchMain.Table()).Where(columns.Id, parent.Id).Data(do.BatchMain{
BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
RunResult: consts.BatchRunResultRunning,
BatchInfo: info,
UpdatedAt: gtime.Now(),
}).Update()
return err
}
func markAllocateParentFailed(ctx context.Context, parentBatchNo, reason string) {
if strings.TrimSpace(parentBatchNo) == "" {
return
}
now := gtime.Now()
columns := dao.BatchMain.Columns()
_, _ = g.DB().Model(dao.BatchMain.Table()).
Where(columns.BatchNo, parentBatchNo).
Where(columns.BatchType, consts.BatchTypeAllocate).
WhereIn(columns.BatchAsyncStatus, []int{consts.BatchAsyncStatusPending, consts.BatchAsyncStatusProcessing}).
Data(do.BatchMain{
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
RunResult: consts.BatchRunResultFailed,
FailReason: strings.TrimSpace(reason),
EndTime: now,
UpdatedAt: now,
}).Update()
}
func countBatchRows(rows []batchimport.Row) (valid, failed int) {
for _, row := range rows {
if len(row.GetErrorReasons()) == 0 {
valid++
} else {
failed++
}
}
return
}
func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenantID string, rows []batchimport.Row) error {
if strings.TrimSpace(parentBatchNo) == "" || len(rows) == 0 {
return nil
}
columns := dao.BatchDetail.Columns()
itemIDs := make([]string, 0, len(rows))
seen := make(map[string]struct{}, len(rows))
for _, item := range rows {
row, ok := item.(*Row)
if !ok || row == nil {
continue
}
itemID := strings.TrimSpace(row.Iccid)
if itemID == "" {
continue
}
if _, exists := seen[itemID]; exists {
continue
}
seen[itemID] = struct{}{}
itemIDs = append(itemIDs, itemID)
}
if len(itemIDs) == 0 {
return nil
}
var existing []struct {
ItemId string `orm:"item_id"`
}
query := g.DB().Model(dao.BatchDetail.Table()).
Fields(columns.ItemId).
Where(columns.BatchType, consts.BatchTypeAllocate).
Where(columns.TenantId, tenantID).
WhereIn(columns.ItemId, itemIDs).
Where("batch_no IN (SELECT batch_no FROM batch_main WHERE source_batch_no = ? AND batch_type = ? AND tenant_id = ?)",
parentBatchNo, consts.BatchTypeAllocate, tenantID)
if err := query.Scan(&existing); err != nil {
return gerror.Wrap(err, "check duplicate allocate ICCIDs across chunks failed")
}
if len(existing) == 0 {
return nil
}
duplicates := make(map[string]struct{}, len(existing))
for _, item := range existing {
if itemID := strings.TrimSpace(item.ItemId); itemID != "" {
duplicates[itemID] = struct{}{}
}
}
for _, item := range rows {
row, ok := item.(*Row)
if !ok || row == nil {
continue
}
if _, exists := duplicates[strings.TrimSpace(row.Iccid)]; exists {
row.AddErrorReason("ICCID appears in an earlier allocation chunk")
}
}
return nil
}
func withAllocateInfo(info *gjson.Json, values map[string]any) *gjson.Json {
data := make(map[string]any)
if info != nil {
for key, value := range info.Map() {
data[key] = value
}
}
for key, value := range values {
data[key] = value
}
return gjson.New(data)
}
func ossIDFromContent(content *model.SysOssSaveContentModel) int64 {
if content == nil || content.SysOss == nil {
return 0
}
return content.OssId
}
func urlFromContent(content *model.SysOssSaveContentModel) string {
if content == nil || content.SysOss == nil {
return ""
}
return content.Url
}
func streamAllocateExcel(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
return excel.StreamFile(ctx, filePath, excel.ParseOptions{
RequiredHeaders: []string{"ICCID"},
SingleSheet: true,
}, handler)
})
}
func streamAllocateCSV(ctx context.Context, filePath string, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
return streamAllocateParsedFile(ctx, chunkSize, emit, func(handler func(excel.ParsedRow) error) (int, error) {
return excel.StreamCSVFile(ctx, filePath, excel.ParseOptions{
RequiredHeaders: []string{"ICCID"},
}, handler)
})
}
func streamAllocateParsedFile(ctx context.Context, chunkSize int, emit func([]batchimport.Row) error, stream func(func(excel.ParsedRow) error) (int, error)) (int, error) {
rows := make([]batchimport.Row, 0, chunkSize)
total, err := stream(func(parsed excel.ParsedRow) error {
rows = append(rows, rowFromExcel(parsed))
if len(rows) < chunkSize {
return nil
}
if err := emit(rows); err != nil {
return err
}
rows = make([]batchimport.Row, 0, chunkSize)
return nil
})
if err != nil {
return total, err
}
if len(rows) > 0 {
if err := emit(rows); err != nil {
return total, err
}
}
return total, nil
}
func streamAllocateInput(ctx context.Context, input string, targetDeptID int64, chunkSize int, emit func([]batchimport.Row) error) (int, error) {
scanner := bufio.NewScanner(strings.NewReader(input))
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
rows := make([]batchimport.Row, 0, chunkSize)
total := 0
rowNo := 0
for scanner.Scan() {
rowNo++
if err := ctx.Err(); err != nil {
return total, err
}
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
row := &Row{RowNo: rowNo, TargetDeptId: targetDeptID}
fields := strings.Fields(line)
switch len(fields) {
case 1:
row.Iccid = fields[0]
case 3:
row.Iccid = fields[0]
row.Msisdn = fields[1]
row.Imsi = fields[2]
default:
row.AddErrorReason("invalid input format; expected ICCID or ICCID MSISDN IMSI")
}
rows = append(rows, row)
total++
if len(rows) == chunkSize {
if err := emit(rows); err != nil {
return total, err
}
rows = make([]batchimport.Row, 0, chunkSize)
}
}
if err := scanner.Err(); err != nil {
return total, err
}
if total == 0 {
return 0, gerror.New("allocate input has no data rows")
}
if len(rows) > 0 {
if err := emit(rows); err != nil {
return total, err
}
}
return total, nil
}
func rowFromExcel(parsed excel.ParsedRow) *Row {
row := &Row{
RowNo: parsed.RowNo,
Iccid: excel.Field(parsed, "ICCID"),
Msisdn: excel.Field(parsed, "MSISDN"),
Imsi: excel.Field(parsed, "IMSI"),
TargetDeptKey: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u7f16\u7801", "\u5ba2\u6237\u7f16\u7801", "TARGETDEPTKEY", "TARGET_CUSTOMER_CODE"),
TargetDeptName: firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237\u540d\u79f0", "\u5ba2\u6237\u540d\u79f0", "TARGETDEPTNAME", "TARGET_CUSTOMER_NAME"),
Remark: firstExcelField(parsed, "\u5212\u62e8\u5907\u6ce8", "\u5907\u6ce8", "REMARK"),
}
if value := firstExcelField(parsed, "\u76ee\u6807\u5ba2\u6237ID", "\u76ee\u6807\u90e8\u95e8ID", "TARGETDEPTID", "TARGET_CUSTOMER_ID"); value != "" {
if targetDeptID, err := parseTargetDeptID(value); err == nil {
row.TargetDeptId = targetDeptID
} else {
row.AddErrorReason("invalid target customer id")
}
}
return row
}
func parseTargetDeptID(value string) (int64, error) {
parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
if err != nil || parsed <= 0 {
return 0, gerror.New("invalid target customer id")
}
return parsed, nil
}
func sourceUnitInfo(info *gjson.Json) *gjson.Json {
return gjson.New(map[string]any{"sourceDeptId": info.Get("sourceDeptId").Int64(), "sourceDeptKey": info.Get("sourceDeptKey").String(), "sourceDeptName": info.Get("sourceDeptName").String()})
}
@@ -0,0 +1,155 @@
package allocate
import (
"context"
"path/filepath"
"strings"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/allocationunits"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/iot/simcardbatch/common"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
)
func CreatePendingExcel(ctx context.Context, file *ghttp.UploadFile, in *model.TSimCardAllocateExcelSubmitParam) (*PendingResult, error) {
if in == nil {
in = &model.TSimCardAllocateExcelSubmitParam{}
}
if file == nil {
return nil, gerror.New("upload file is required")
}
if err := service.SysOss().CheckType(ctx, consts.SysUploadFileTypeFile, file); err != nil {
return nil, err
}
if file.Size > allocateMaxFileSize(ctx) {
return nil, gerror.Newf("uploaded file exceeds the limit of %d MB", allocateMaxFileSize(ctx)>>20)
}
options := SubmitOptions{
TargetDeptId: in.TargetDeptId,
Remark: strings.TrimSpace(in.Remark),
}
processor := NewProcessor()
parentBatchNo, err := batchimport.NextBatchNoContext(ctx, processor.Config().BatchNoPrefix)
if err != nil {
return nil, err
}
extension := strings.ToLower(filepath.Ext(file.Filename))
if extension == "" {
extension = ".xlsx"
}
importOss, err := batchimport.SaveUpload(ctx, file, parentBatchNo+"-import"+extension, processor.Config().ImportDir)
if err != nil {
return nil, gerror.Wrap(err, "save allocate import file failed")
}
req := batchimport.SubmitRequest{
SourceChannel: batchmetadata.Console,
ExecutionMode: batchmetadata.Bulk,
BizParams: &options,
SourceType: consts.BatchSourceTypeFile,
TargetDeptId: options.TargetDeptId,
Operator: common.CurrentOperator(ctx),
BatchNo: parentBatchNo,
}
stream := func(streamCtx context.Context, emit func([]batchimport.Row) error) (int, error) {
chunkSize := allocateChunkSize(streamCtx)
if extension == ".csv" {
return streamAllocateCSV(streamCtx, importOss.Path, chunkSize, emit)
}
return streamAllocateExcel(streamCtx, importOss.Path, chunkSize, emit)
}
return createPendingSplit(ctx, req, processor, prepareOptions(options), parentBatchNo, importOss, stream)
}
func CreatePendingInput(ctx context.Context, in *model.TSimCardAllocatePrepareParam) (*PendingResult, error) {
if in == nil {
in = &model.TSimCardAllocatePrepareParam{}
}
if in.TargetDeptId <= 0 {
return nil, gerror.New("target customer is required")
}
options := SubmitOptions{TargetDeptId: in.TargetDeptId}
processor := NewProcessor()
parentBatchNo, err := batchimport.NextBatchNoContext(ctx, processor.Config().BatchNoPrefix)
if err != nil {
return nil, err
}
req := batchimport.SubmitRequest{
SourceChannel: batchmetadata.Console,
ExecutionMode: batchmetadata.Bulk,
BizParams: &options,
SourceType: consts.BatchSourceTypeBatchInput,
TargetDeptId: options.TargetDeptId,
Operator: common.CurrentOperator(ctx),
BatchNo: parentBatchNo,
}
stream := func(streamCtx context.Context, emit func([]batchimport.Row) error) (int, error) {
return streamAllocateInput(streamCtx, in.Input, options.TargetDeptId, allocateChunkSize(streamCtx), emit)
}
return createPendingSplit(ctx, req, processor, prepareOptions(options), parentBatchNo, nil, stream)
}
func prepareOptions(options SubmitOptions) batchimport.PendingOptions {
return batchimport.PendingOptions{
PrepareRows: func(ctx context.Context, rows []batchimport.Row) error {
for _, item := range rows {
row, ok := item.(*Row)
if !ok || row == nil {
return gerror.New("allocate row type is invalid")
}
if row.TargetDeptId <= 0 && options.TargetDeptId > 0 {
row.TargetDeptId = options.TargetDeptId
}
if row.Remark == "" {
row.Remark = options.Remark
}
if row.TargetDeptId <= 0 && row.TargetDeptKey == "" && row.TargetDeptName == "" {
row.AddErrorReason("target customer is required")
}
}
ids := []int64{}
for _, item := range rows {
if row, ok := item.(*Row); ok {
ids = append(ids, row.TargetDeptId)
}
}
units, err := allocationunits.Load(ctx, common.CurrentOperator(ctx).TenantId, ids)
if err != nil {
return err
}
for _, item := range rows {
if row, ok := item.(*Row); ok {
if unit, exists := units[row.TargetDeptId]; exists {
row.TargetDeptKey = unit.Key
row.TargetDeptName = unit.Name
}
}
}
return nil
},
}
}
func pendingResultFromBatch(result *batchimport.Result) *PendingResult {
res := &PendingResult{
BatchType: consts.BatchTypeAllocate,
BatchAsyncStatus: consts.BatchAsyncStatusPending,
}
if result == nil {
return res
}
res.BatchNo = result.BatchNo
res.BatchType = result.BatchType
res.Total = result.TotalRows
res.ValidCount = result.ValidRows
res.FailCount = result.ErrorRows
return res
}
@@ -0,0 +1,81 @@
package allocate
import (
"strings"
"github.com/gogf/gf/v2/encoding/gjson"
"github.com/gogf/gf/v2/frame/g"
)
type Row struct {
RowNo int
Iccid string
Msisdn string
Imsi string
TargetDeptId int64
TargetDeptKey string
TargetDeptName string
Remark string
Reason string
}
func (r *Row) GetRowNo() int {
if r == nil {
return 0
}
return r.RowNo
}
func (r *Row) GetItemID() string {
if r == nil {
return ""
}
return r.Iccid
}
func (r *Row) GetErrorReasons() []string {
if r == nil || strings.TrimSpace(r.Reason) == "" {
return nil
}
return []string{strings.TrimSpace(r.Reason)}
}
func (r *Row) AddErrorReason(reason string) {
if r == nil || strings.TrimSpace(reason) == "" {
return
}
if r.Reason != "" {
r.Reason += ";"
}
r.Reason += strings.TrimSpace(reason)
}
func (r *Row) Snapshot() *gjson.Json {
if r == nil {
return gjson.New(g.Map{})
}
return gjson.New(g.Map{
"ICCID": strings.TrimSpace(r.Iccid),
"MSISDN": strings.TrimSpace(r.Msisdn),
"IMSI": strings.TrimSpace(r.Imsi),
"targetDeptId": r.TargetDeptId,
"targetDeptKey": strings.TrimSpace(r.TargetDeptKey),
"targetDeptName": strings.TrimSpace(r.TargetDeptName),
"remark": strings.TrimSpace(r.Remark),
})
}
type SubmitOptions struct {
Units *gjson.Json
TargetDeptId int64
Remark string
}
type PendingResult struct {
BatchNo string
BatchType int
BatchAsyncStatus int
Total int
ValidCount int
FailCount int
}