MiFaZhan
915 lines
29 KiB
Go
915 lines
29 KiB
Go
package inbound
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gogf/gf/v2/database/gdb"
|
|
"github.com/gogf/gf/v2/encoding/gjson"
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
"github.com/gogf/gf/v2/os/gtime"
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
|
|
inbound "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/inbound"
|
|
)
|
|
|
|
const stockTable = "t_sim_card_stock"
|
|
const (
|
|
inboundStockQueryBatchSizeConfigKey = "batch.inbound.stockQueryBatchSize"
|
|
inboundWriteBatchSizeConfigKey = "batch.inbound.writeBatchSize"
|
|
defaultInboundStockQueryBatchSize = 1000
|
|
defaultInboundWriteBatchSize = 1000
|
|
)
|
|
|
|
type Repository struct {
|
|
db gdb.DB
|
|
resultFiles batchresult.Generator
|
|
}
|
|
|
|
type stockDO struct {
|
|
g.Meta `orm:"table:t_sim_card_stock, do:true"`
|
|
BatchNo any `orm:"batch_no"`
|
|
ICCID any `orm:"iccid"`
|
|
MSISDN any `orm:"msisdn"`
|
|
IMSI any `orm:"imsi"`
|
|
IMEI any `orm:"imei"`
|
|
ChanKey any `orm:"chan_key"`
|
|
TenantID any `orm:"tenant_id"`
|
|
CreatedDept any `orm:"created_dept"`
|
|
CreatedBy any `orm:"created_by"`
|
|
UpdatedBy any `orm:"updated_by"`
|
|
}
|
|
|
|
func New() *Repository {
|
|
return &Repository{db: dao.BatchMain.DB(), resultFiles: batchresult.New()}
|
|
}
|
|
|
|
func (r *Repository) database() gdb.DB {
|
|
if r.db != nil {
|
|
return r.db
|
|
}
|
|
return g.DB()
|
|
}
|
|
|
|
func (r *Repository) Load(ctx context.Context, batchNo string) (*inbound.Task, error) {
|
|
if batchNo == "" {
|
|
return nil, gerror.New("inbound batch number is empty")
|
|
}
|
|
var batch entity.BatchMain
|
|
if err := r.database().Model(dao.BatchMain.Table()).Safe().Ctx(ctx).
|
|
Where(dao.BatchMain.Columns().BatchNo, batchNo).
|
|
Where(dao.BatchMain.Columns().BatchType, consts.BatchTypeInbound).
|
|
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
|
|
Scan(&batch); err != nil {
|
|
return nil, gerror.Wrap(err, "query inbound batch")
|
|
}
|
|
if batch.Id == 0 {
|
|
return nil, gerror.New("processing inbound batch does not exist")
|
|
}
|
|
task := &inbound.Task{
|
|
Batch: inbound.Batch{
|
|
ID: batch.Id,
|
|
BatchNo: batch.BatchNo,
|
|
BatchType: batch.BatchType,
|
|
SourceType: batch.SourceType,
|
|
TenantID: batch.TenantId,
|
|
CreatedDept: batch.CreatedDept,
|
|
CreatedBy: batch.CreatedBy,
|
|
StartTime: batch.StartTime,
|
|
BatchInfo: batch.BatchInfo,
|
|
},
|
|
}
|
|
task.Source = parseSource(batch.BatchInfo, batch.SourceOssId, batch.SourceType)
|
|
task.Options = parseOptions(batch.BatchInfo)
|
|
return task, nil
|
|
}
|
|
|
|
func (r *Repository) ReadOssFile(ctx context.Context, ossID int64) ([]byte, string, error) {
|
|
if ossID <= 0 {
|
|
return nil, "", gerror.New("oss id is empty")
|
|
}
|
|
var oss entity.SysOss
|
|
if err := r.database().Model(dao.SysOss.Table()).Safe().Ctx(ctx).
|
|
Where(dao.SysOss.Columns().OssId, ossID).
|
|
Scan(&oss); err != nil {
|
|
return nil, "", gerror.Wrap(err, "query OSS file")
|
|
}
|
|
if oss.OssId == 0 {
|
|
return nil, "", gerror.Newf("OSS file %d does not exist", ossID)
|
|
}
|
|
if oss.Service != "local" {
|
|
return nil, "", gerror.Newf("OSS file %d is not a local file: service=%q", ossID, oss.Service)
|
|
}
|
|
uploadRoot := g.Cfg().MustGet(ctx, "system.upload.local.path").String()
|
|
localPath, attempts, err := resolveReadableLocalOssPath(uploadRoot, oss.TenantId, oss.Path)
|
|
if err != nil {
|
|
return nil, fileDisplayName(oss, ""), gerror.Wrapf(err, "resolve OSS file %d local path from %q, attempted paths: %s", ossID, oss.Path, strings.Join(attempts, "; "))
|
|
}
|
|
content, err := os.ReadFile(localPath)
|
|
if err != nil {
|
|
return nil, fileDisplayName(oss, localPath), gerror.Wrapf(err, "OSS file %d local path is unreadable: original path=%q, resolved path=%q", ossID, oss.Path, localPath)
|
|
}
|
|
return content, fileDisplayName(oss, localPath), nil
|
|
}
|
|
|
|
func resolveLocalOssPath(uploadRoot, tenantID, storedPath string) (string, error) {
|
|
uploadRoot = strings.TrimSpace(uploadRoot)
|
|
if uploadRoot == "" {
|
|
return "", gerror.New("system.upload.local.path is empty")
|
|
}
|
|
tenantID = strings.TrimSpace(tenantID)
|
|
if tenantID == "" {
|
|
tenantID = "default"
|
|
}
|
|
storedPath = strings.TrimSpace(storedPath)
|
|
if storedPath == "" {
|
|
return "", gerror.New("OSS path is empty")
|
|
}
|
|
|
|
objectSegments, err := ossObjectSegments(tenantID, storedPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
objectPath := filepath.Join(objectSegments...)
|
|
localPath := filepath.Join(uploadRoot, objectPath)
|
|
relativePath, err := filepath.Rel(uploadRoot, localPath)
|
|
if err != nil {
|
|
return "", gerror.Wrap(err, "resolve local OSS path")
|
|
}
|
|
if relativePath == ".." || strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) {
|
|
return "", gerror.New("resolved OSS path escapes upload root")
|
|
}
|
|
return localPath, nil
|
|
}
|
|
|
|
func resolveReadableLocalOssPath(uploadRoot, tenantID, storedPath string) (string, []string, error) {
|
|
var attempts []string
|
|
var firstErr error
|
|
for _, root := range uploadRootCandidates(uploadRoot) {
|
|
localPath, err := resolveLocalOssPath(root, tenantID, storedPath)
|
|
if err != nil {
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
continue
|
|
}
|
|
attempts = appendUniqueString(attempts, localPath)
|
|
if isReadableFile(localPath) {
|
|
return localPath, attempts, nil
|
|
}
|
|
}
|
|
|
|
if _, err := ossObjectSegments(strings.TrimSpace(defaultTenant(tenantID)), storedPath); err != nil {
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
} else {
|
|
for _, candidate := range storedPathCandidates(storedPath) {
|
|
attempts = appendUniqueString(attempts, candidate)
|
|
if isReadableFile(candidate) {
|
|
return candidate, attempts, nil
|
|
}
|
|
}
|
|
}
|
|
if firstErr != nil && len(attempts) == 0 {
|
|
return "", attempts, firstErr
|
|
}
|
|
return "", attempts, gerror.New("OSS local file does not exist")
|
|
}
|
|
|
|
func uploadRootCandidates(uploadRoot string) []string {
|
|
var candidates []string
|
|
for _, root := range []string{
|
|
os.Getenv("IOTLCM_WORKER_UPLOAD_ROOT"),
|
|
os.Getenv("IOTLCM_UPLOAD_ROOT"),
|
|
uploadRoot,
|
|
} {
|
|
root = strings.TrimSpace(root)
|
|
if root == "" {
|
|
continue
|
|
}
|
|
candidates = appendUploadRootCandidate(candidates, root)
|
|
}
|
|
return candidates
|
|
}
|
|
|
|
func appendUploadRootCandidate(candidates []string, root string) []string {
|
|
for _, candidate := range uploadRootEquivalentCandidates(root) {
|
|
if filepath.IsAbs(candidate) {
|
|
candidates = appendUniqueString(candidates, candidate)
|
|
} else {
|
|
for _, base := range localBaseDirs() {
|
|
candidates = appendUniqueString(candidates, filepath.Join(base, candidate))
|
|
}
|
|
candidates = appendUniqueString(candidates, candidate)
|
|
}
|
|
}
|
|
return candidates
|
|
}
|
|
|
|
func uploadRootEquivalentCandidates(root string) []string {
|
|
root = strings.TrimSpace(root)
|
|
if root == "" {
|
|
return nil
|
|
}
|
|
candidates := []string{root}
|
|
normalized := filepath.ToSlash(filepath.Clean(root))
|
|
if normalized == "resource/upload" || strings.HasSuffix(normalized, "/resource/upload") {
|
|
prefix := strings.TrimSuffix(normalized, "resource/upload")
|
|
candidates = appendUniqueString(candidates, filepath.FromSlash(prefix+"upload"))
|
|
}
|
|
return candidates
|
|
}
|
|
|
|
func storedPathCandidates(storedPath string) []string {
|
|
storedPath = strings.TrimSpace(storedPath)
|
|
if storedPath == "" {
|
|
return nil
|
|
}
|
|
var candidates []string
|
|
candidates = appendUniqueString(candidates, storedPath)
|
|
if !filepath.IsAbs(storedPath) {
|
|
for _, base := range localBaseDirs() {
|
|
candidates = appendUniqueString(candidates, filepath.Join(base, storedPath))
|
|
}
|
|
}
|
|
return candidates
|
|
}
|
|
|
|
func localBaseDirs() []string {
|
|
var dirs []string
|
|
if wd, err := os.Getwd(); err == nil && strings.TrimSpace(wd) != "" {
|
|
dirs = appendUniqueString(dirs, wd)
|
|
}
|
|
if exe, err := os.Executable(); err == nil && strings.TrimSpace(exe) != "" {
|
|
dirs = appendUniqueString(dirs, filepath.Dir(exe))
|
|
}
|
|
return dirs
|
|
}
|
|
|
|
func ossObjectSegments(tenantID, storedPath string) ([]string, error) {
|
|
tenantID = defaultTenant(tenantID)
|
|
segments := strings.FieldsFunc(strings.ReplaceAll(storedPath, "\\", "/"), func(r rune) bool {
|
|
return r == '/'
|
|
})
|
|
for index, segment := range segments {
|
|
if segment != tenantID {
|
|
continue
|
|
}
|
|
objectSegments := segments[index:]
|
|
for _, objectSegment := range objectSegments {
|
|
if objectSegment == "." || objectSegment == ".." {
|
|
return nil, gerror.Newf("invalid OSS object path segment %q", objectSegment)
|
|
}
|
|
}
|
|
return objectSegments, nil
|
|
}
|
|
return nil, gerror.Newf("OSS path does not contain tenant %q", tenantID)
|
|
}
|
|
|
|
func defaultTenant(tenantID string) string {
|
|
tenantID = strings.TrimSpace(tenantID)
|
|
if tenantID == "" {
|
|
return "default"
|
|
}
|
|
return tenantID
|
|
}
|
|
|
|
func isReadableFile(path string) bool {
|
|
info, err := os.Stat(path)
|
|
return err == nil && !info.IsDir()
|
|
}
|
|
|
|
func appendUniqueString(values []string, value string) []string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return values
|
|
}
|
|
for _, existing := range values {
|
|
if existing == value {
|
|
return values
|
|
}
|
|
}
|
|
return append(values, value)
|
|
}
|
|
|
|
func uniqueStrings(values []string) []string {
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
uniq := make([]string, 0, len(values))
|
|
seen := make(map[string]struct{}, len(values))
|
|
for _, value := range values {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[value]; ok {
|
|
continue
|
|
}
|
|
seen[value] = struct{}{}
|
|
uniq = append(uniq, value)
|
|
}
|
|
return uniq
|
|
}
|
|
|
|
func (r *Repository) FindExistingStocks(ctx context.Context, _ string, iccids, msisdns, imsis []string) ([]inbound.Stock, error) {
|
|
if len(iccids) == 0 && len(msisdns) == 0 && len(imsis) == 0 {
|
|
return nil, nil
|
|
}
|
|
type queryResult struct {
|
|
stocks []inbound.Stock
|
|
err error
|
|
}
|
|
results := make(chan queryResult, 3)
|
|
queries := 0
|
|
if len(iccids) > 0 {
|
|
queries++
|
|
go func() {
|
|
stocks, err := r.findExistingStocksByField(ctx, "iccid", "ICCID", iccids)
|
|
results <- queryResult{stocks: stocks, err: err}
|
|
}()
|
|
}
|
|
if len(msisdns) > 0 {
|
|
queries++
|
|
go func() {
|
|
stocks, err := r.findExistingStocksByField(ctx, "msisdn", "MSISDN", msisdns)
|
|
results <- queryResult{stocks: stocks, err: err}
|
|
}()
|
|
}
|
|
if len(imsis) > 0 {
|
|
queries++
|
|
go func() {
|
|
stocks, err := r.findExistingStocksByField(ctx, "imsi", "IMSI", imsis)
|
|
results <- queryResult{stocks: stocks, err: err}
|
|
}()
|
|
}
|
|
stocks := make([]inbound.Stock, 0)
|
|
for i := 0; i < queries; i++ {
|
|
result := <-results
|
|
if result.err != nil {
|
|
return nil, result.err
|
|
}
|
|
stocks = append(stocks, result.stocks...)
|
|
}
|
|
return stocks, nil
|
|
}
|
|
|
|
func (r *Repository) stockQuery(ctx context.Context) *gdb.Model {
|
|
return r.database().Model(stockTable).Safe().Ctx(ctx)
|
|
}
|
|
|
|
func (r *Repository) findExistingStocksByField(ctx context.Context, field, label string, values []string) ([]inbound.Stock, error) {
|
|
uniq := uniqueStrings(values)
|
|
if len(uniq) == 0 {
|
|
return nil, nil
|
|
}
|
|
batchSize := inboundStockQueryBatchSize(ctx)
|
|
stocks := make([]inbound.Stock, 0)
|
|
for _, chunk := range chunkStrings(uniq, batchSize) {
|
|
var chunkStocks []inbound.Stock
|
|
model := r.stockQuery(ctx).WhereIn(field, chunk)
|
|
if err := model.Fields("iccid", "msisdn", "imsi").Scan(&chunkStocks); err != nil {
|
|
return nil, gerror.Wrapf(err, "query inbound stock %s duplicates", label)
|
|
}
|
|
stocks = append(stocks, chunkStocks...)
|
|
}
|
|
return stocks, nil
|
|
}
|
|
|
|
func (r *Repository) SaveValidationFailure(ctx context.Context, task *inbound.Task) error {
|
|
err := r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
batchInfo := buildInboundBatchInfo(task)
|
|
task.Batch.BatchInfo = batchInfo
|
|
details := buildBatchDetails(task, consts.BatchDetailStatusFailed)
|
|
failCount := 0
|
|
detailIndex := 0
|
|
for _, row := range task.Rows {
|
|
if row == nil {
|
|
continue
|
|
}
|
|
if inbound.IsCascadeFailure(row) {
|
|
details[detailIndex].DetailStatus = consts.BatchDetailStatusNotExecuted
|
|
} else {
|
|
failCount++
|
|
}
|
|
detailIndex++
|
|
}
|
|
if err := insertChunkedRows(ctx, tx, dao.BatchDetail.Table(), "inbound detail", details, inboundWriteBatchSize(ctx)); err != nil {
|
|
return err
|
|
}
|
|
return updateBatchMain(ctx, tx, task, do.BatchMain{
|
|
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
|
TotalCount: len(task.Rows),
|
|
ValidCount: task.ValidCount,
|
|
SuccessCount: 0,
|
|
FailCount: failCount,
|
|
RunResult: consts.BatchRunResultFailed,
|
|
FailReason: batchfields.FailureReason(task.FailureReason),
|
|
BatchInfo: batchInfo,
|
|
EndTime: gtime.Now(),
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
})
|
|
})
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
g.Log().Errorf(ctx, "save inbound validation details failed, batchNo:%s, err:%s", task.Batch.BatchNo, batchfields.FailureReason(err.Error()))
|
|
finalizationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
defer cancel()
|
|
fallbackErr := r.database().Transaction(finalizationCtx, func(ctx context.Context, tx gdb.TX) error {
|
|
return updateBatchMain(ctx, tx, task, do.BatchMain{
|
|
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
|
TotalCount: len(task.Rows),
|
|
ValidCount: task.ValidCount,
|
|
SuccessCount: 0,
|
|
FailCount: len(task.Rows),
|
|
RunResult: consts.BatchRunResultFailed,
|
|
FailReason: batchfields.FailureReason("错误明细保存失败，本次整批未入库"),
|
|
BatchInfo: task.Batch.BatchInfo,
|
|
EndTime: gtime.Now(),
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
})
|
|
})
|
|
if fallbackErr != nil {
|
|
return gerror.Wrapf(fallbackErr, "save inbound failure details failed: %v; finalize batch failed", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) SaveSuccess(ctx context.Context, task *inbound.Task) error {
|
|
return r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
batchInfo := buildInboundBatchInfo(task)
|
|
task.Batch.BatchInfo = batchInfo
|
|
stocks := make([]stockDO, 0, len(task.Rows))
|
|
details := make([]do.BatchDetail, 0, len(task.Rows))
|
|
for _, row := range task.Rows {
|
|
if row == nil {
|
|
continue
|
|
}
|
|
stocks = append(stocks, stockDO{
|
|
BatchNo: task.Batch.BatchNo,
|
|
ICCID: row.ICCID,
|
|
MSISDN: nullableString(row.MSISDN),
|
|
IMSI: nullableString(row.IMSI),
|
|
IMEI: nullableString(row.IMEI),
|
|
ChanKey: row.ChanKey,
|
|
TenantID: task.Batch.TenantID,
|
|
CreatedDept: task.Batch.CreatedDept,
|
|
CreatedBy: task.Batch.CreatedBy,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
})
|
|
details = append(details, batchDetailRow(task, row, consts.BatchDetailStatusSuccess))
|
|
}
|
|
if err := insertChunkedRows(ctx, tx, stockTable, "inbound stock", stocks, inboundWriteBatchSize(ctx)); err != nil {
|
|
return err
|
|
}
|
|
if err := insertChunkedRows(ctx, tx, dao.BatchDetail.Table(), "inbound detail", details, inboundWriteBatchSize(ctx)); err != nil {
|
|
return err
|
|
}
|
|
return updateBatchMain(ctx, tx, task, do.BatchMain{
|
|
BatchAsyncStatus: consts.BatchAsyncStatusSuccess,
|
|
TotalCount: len(task.Rows),
|
|
ValidCount: len(task.Rows),
|
|
SuccessCount: len(task.Rows),
|
|
FailCount: 0,
|
|
RunResult: consts.BatchRunResultSuccess,
|
|
FailReason: "",
|
|
BatchInfo: batchInfo,
|
|
EndTime: gtime.Now(),
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
})
|
|
})
|
|
}
|
|
|
|
func (r *Repository) SaveBusinessFailure(ctx context.Context, task *inbound.Task, reason string) error {
|
|
task.ValidCount = countRowsWithoutReason(task.Rows)
|
|
for _, row := range task.Rows {
|
|
if row != nil {
|
|
row.Reason = reason
|
|
}
|
|
}
|
|
task.FailureReason = reason
|
|
return r.SaveValidationFailure(ctx, task)
|
|
}
|
|
|
|
func buildBatchDetails(task *inbound.Task, status int) []do.BatchDetail {
|
|
details := make([]do.BatchDetail, 0, len(task.Rows))
|
|
for _, row := range task.Rows {
|
|
if row == nil {
|
|
continue
|
|
}
|
|
details = append(details, batchDetailRow(task, row, status))
|
|
}
|
|
return details
|
|
}
|
|
|
|
func batchDetailRow(task *inbound.Task, row *inbound.Row, status int) do.BatchDetail {
|
|
return do.BatchDetail{
|
|
BatchNo: task.Batch.BatchNo,
|
|
BatchType: task.Batch.BatchType,
|
|
DetailStatus: status,
|
|
BlankOutStatus: 1,
|
|
RowNo: row.RowNo,
|
|
ItemId: batchfields.ItemID(row.ICCID),
|
|
ErrorReason: batchfields.DetailReason(row.Reason),
|
|
Snapshot: rowSnapshot(row),
|
|
TenantId: task.Batch.TenantID,
|
|
CreatedDept: task.Batch.CreatedDept,
|
|
CreatedBy: task.Batch.CreatedBy,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}
|
|
}
|
|
|
|
func (r *Repository) SaveResultFile(ctx context.Context, task *inbound.Task, request batchresult.Request) error {
|
|
artifact, err := r.artifactGenerator().Create(ctx, request)
|
|
if err != nil {
|
|
return r.updateResultFileStatus(ctx, task, "failed", 0, 0, err.Error())
|
|
}
|
|
var (
|
|
ossID int64
|
|
downloadID int64
|
|
)
|
|
err = r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
var insertErr error
|
|
ossID, insertErr = tx.Ctx(ctx).Model(dao.SysOss.Table()).Data(do.SysOss{
|
|
TenantId: task.Batch.TenantID,
|
|
FileName: artifact.FileName,
|
|
OriginalName: artifact.OriginalName,
|
|
FileSuffix: artifact.FileSuffix,
|
|
Path: artifact.Path,
|
|
Url: artifact.URL,
|
|
CreatedDept: task.Batch.CreatedDept,
|
|
CreatedBy: task.Batch.CreatedBy,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
Service: "local",
|
|
Md5: artifact.MD5,
|
|
FileSize: artifact.Size,
|
|
}).InsertAndGetId()
|
|
if insertErr != nil {
|
|
return gerror.Wrap(insertErr, "insert inbound result OSS")
|
|
}
|
|
downloadID, insertErr = tx.Ctx(ctx).Model(dao.TResourceBatchDownload.Table()).Data(do.TResourceBatchDownload{
|
|
BatchNo: task.Batch.BatchNo,
|
|
TaskName: batchresult.TaskName(task.Batch.BatchType),
|
|
ResourceType: consts.ResourceTypeExcel,
|
|
ExportScene: task.Batch.BatchType,
|
|
TotalCount: len(task.Rows),
|
|
SuccessCount: successRows(task.Rows),
|
|
FailCount: failedRows(task.Rows),
|
|
DownloadStatus: consts.DownloadStatusGenerated,
|
|
OssId: ossID,
|
|
FileName: artifact.FileName,
|
|
FileExt: artifact.FileSuffix,
|
|
FileUrl: artifact.URL,
|
|
FilePath: artifact.Path,
|
|
FileSize: artifact.Size,
|
|
FileMd5: artifact.MD5,
|
|
DownloadCount: 0,
|
|
MaxDownloadCount: 0,
|
|
DeptId: task.Batch.CreatedDept,
|
|
StartTime: task.Batch.StartTime,
|
|
FinishTime: gtime.Now(),
|
|
TenantId: task.Batch.TenantID,
|
|
CreatedDept: task.Batch.CreatedDept,
|
|
CreatedBy: task.Batch.CreatedBy,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}).InsertAndGetId()
|
|
if insertErr != nil {
|
|
return gerror.Wrap(insertErr, "insert inbound result download")
|
|
}
|
|
return updateBatchInfoOnly(ctx, tx, task, buildResultBatchInfo(task, "success", ossID, downloadID, ""), ossID)
|
|
})
|
|
if err != nil {
|
|
batchresult.RemoveLocalFile(artifact)
|
|
return r.updateResultFileStatus(ctx, task, "failed", 0, 0, err.Error())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) artifactGenerator() batchresult.Generator {
|
|
if r.resultFiles != nil {
|
|
return r.resultFiles
|
|
}
|
|
return batchresult.New()
|
|
}
|
|
|
|
func (r *Repository) updateResultFileStatus(ctx context.Context, task *inbound.Task, status string, ossID, downloadID int64, reason string) error {
|
|
return r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
|
return updateBatchInfoOnly(ctx, tx, task, buildResultBatchInfo(task, status, ossID, downloadID, reason), ossID)
|
|
})
|
|
}
|
|
|
|
func insertChunkedRows[T any](ctx context.Context, tx gdb.TX, table, label string, rows []T, batchSize int) error {
|
|
if len(rows) == 0 {
|
|
return nil
|
|
}
|
|
if batchSize <= 0 {
|
|
batchSize = defaultInboundWriteBatchSize
|
|
}
|
|
for start := 0; start < len(rows); start += batchSize {
|
|
end := start + batchSize
|
|
if end > len(rows) {
|
|
end = len(rows)
|
|
}
|
|
if _, err := tx.Ctx(ctx).Model(table).Data(rows[start:end]).Insert(); err != nil {
|
|
return gerror.Wrapf(err, "insert %s chunk", label)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func chunkStrings(values []string, batchSize int) [][]string {
|
|
if batchSize <= 0 {
|
|
batchSize = defaultInboundStockQueryBatchSize
|
|
}
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
chunks := make([][]string, 0, (len(values)+batchSize-1)/batchSize)
|
|
for start := 0; start < len(values); start += batchSize {
|
|
end := start + batchSize
|
|
if end > len(values) {
|
|
end = len(values)
|
|
}
|
|
chunks = append(chunks, values[start:end])
|
|
}
|
|
return chunks
|
|
}
|
|
|
|
func inboundStockQueryBatchSize(ctx context.Context) int {
|
|
value := g.Cfg().MustGet(ctx, inboundStockQueryBatchSizeConfigKey, defaultInboundStockQueryBatchSize).Int()
|
|
if value <= 0 {
|
|
return defaultInboundStockQueryBatchSize
|
|
}
|
|
return value
|
|
}
|
|
|
|
func inboundWriteBatchSize(ctx context.Context) int {
|
|
value := g.Cfg().MustGet(ctx, inboundWriteBatchSizeConfigKey, defaultInboundWriteBatchSize).Int()
|
|
if value <= 0 {
|
|
return defaultInboundWriteBatchSize
|
|
}
|
|
return value
|
|
}
|
|
|
|
func updateBatchMain(ctx context.Context, tx gdb.TX, task *inbound.Task, data do.BatchMain) error {
|
|
result, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
|
Where(dao.BatchMain.Columns().BatchNo, task.Batch.BatchNo).
|
|
Where(dao.BatchMain.Columns().BatchType, task.Batch.BatchType).
|
|
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
|
|
Data(data).
|
|
Update()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "update inbound batch main")
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "read inbound batch main update result")
|
|
}
|
|
if affected != 1 {
|
|
return gerror.New("inbound batch is no longer processing")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func updateBatchInfoOnly(ctx context.Context, tx gdb.TX, task *inbound.Task, info *gjson.Json, resultOssID int64) error {
|
|
result, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
|
Where(dao.BatchMain.Columns().BatchNo, task.Batch.BatchNo).
|
|
Where(dao.BatchMain.Columns().BatchType, task.Batch.BatchType).
|
|
Data(do.BatchMain{
|
|
ResultOssId: workerOssIDValue(resultOssID),
|
|
BatchInfo: info,
|
|
UpdatedBy: task.Batch.CreatedBy,
|
|
}).
|
|
Update()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "update inbound result file status")
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "read inbound result file update result")
|
|
}
|
|
if affected != 1 {
|
|
return gerror.New("inbound batch disappeared while saving result file")
|
|
}
|
|
task.Batch.BatchInfo = info
|
|
return nil
|
|
}
|
|
|
|
func workerOssIDValue(ossID int64) any {
|
|
if ossID <= 0 {
|
|
return nil
|
|
}
|
|
return ossID
|
|
}
|
|
|
|
func parseSource(info *gjson.Json, sourceOssID int64, sourceType int) inbound.Source {
|
|
source := inbound.Source{
|
|
ImportFileOssID: jsonInt64(info, "source.importFileOssId", "importFileOssId"),
|
|
InputText: jsonString(info, "source.inputText", "inputText"),
|
|
InputFileOssID: jsonInt64(info, "source.inputFileOssId", "inputFileOssId"),
|
|
Range: inbound.Range{
|
|
RangeType: jsonString(info, "source.range.rangeType", "range.rangeType", "rangeType"),
|
|
StartICCID: jsonString(info, "source.range.startIccid", "range.startIccid", "startIccid"),
|
|
EndICCID: jsonString(info, "source.range.endIccid", "range.endIccid", "endIccid"),
|
|
StartMSISDN: jsonString(info, "source.range.startMsisdn", "range.startMsisdn", "startMsisdn"),
|
|
EndMSISDN: jsonString(info, "source.range.endMsisdn", "range.endMsisdn", "endMsisdn"),
|
|
StartIMSI: jsonString(info, "source.range.startImsi", "range.startImsi", "startImsi"),
|
|
EndIMSI: jsonString(info, "source.range.endImsi", "range.endImsi", "endImsi"),
|
|
},
|
|
}
|
|
if sourceOssID > 0 {
|
|
switch sourceType {
|
|
case consts.BatchSourceTypeBatchInput:
|
|
source.InputFileOssID = sourceOssID
|
|
default:
|
|
source.ImportFileOssID = sourceOssID
|
|
}
|
|
}
|
|
return source
|
|
}
|
|
|
|
func parseOptions(info *gjson.Json) inbound.Options {
|
|
return inbound.Options{
|
|
ChanKey: jsonString(info, "inboundOptions.chanKey", "chanKey"),
|
|
ValidateUpstreamBasicInfo: jsonString(info, "inboundOptions.validateUpstreamBasicInfo", "validateUpstreamBasicInfo"),
|
|
CardType: jsonString(info, "inboundOptions.cardType", "cardType"),
|
|
UpPkgCode: jsonString(info, "inboundOptions.upPkgCode", "upPkgCode"),
|
|
SuppliersCode: jsonString(info, "inboundOptions.suppliersCode", "suppliersCode"),
|
|
UpstreamActivationTime: jsonString(info, "inboundOptions.upstreamActivationTime", "upstreamActivationTime"),
|
|
InsertToPlugEnabled: jsonString(info, "inboundOptions.insertToPlugEnabled", "insertToPlugEnabled"),
|
|
CarrierTrafficCutoffEnabled: jsonString(info, "inboundOptions.carrierTrafficCutoffEnabled", "carrierTrafficCutoffEnabled"),
|
|
AgreementExpiryDate: jsonString(info, "inboundOptions.agreementExpiryDate", "agreementExpiryDate"),
|
|
HasTestPeriod: jsonString(info, "inboundOptions.hasTestPeriod", "hasTestPeriod"),
|
|
TestPeriodDays: jsonString(info, "inboundOptions.testPeriodDays", "testPeriodDays"),
|
|
TestPeriodLatestExpiryDate: jsonString(info, "inboundOptions.testPeriodLatestExpiryDate", "testPeriodLatestExpiryDate"),
|
|
HasSilentPeriod: jsonString(info, "inboundOptions.hasSilentPeriod", "hasSilentPeriod"),
|
|
SilentPeriodDays: jsonString(info, "inboundOptions.silentPeriodDays", "silentPeriodDays"),
|
|
SilentPeriodLatestExpiryDate: jsonString(info, "inboundOptions.silentPeriodLatestExpiryDate", "silentPeriodLatestExpiryDate"),
|
|
}
|
|
}
|
|
|
|
func jsonString(info *gjson.Json, paths ...string) string {
|
|
if info == nil {
|
|
return ""
|
|
}
|
|
for _, path := range paths {
|
|
value := strings.TrimSpace(info.Get(path).String())
|
|
if value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func jsonInt64(info *gjson.Json, paths ...string) int64 {
|
|
if info == nil {
|
|
return 0
|
|
}
|
|
for _, path := range paths {
|
|
value := info.Get(path).Int64()
|
|
if value > 0 {
|
|
return value
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func rowSnapshot(row *inbound.Row) *gjson.Json {
|
|
raw := map[string]any{}
|
|
for key, value := range row.Raw {
|
|
raw[key] = value
|
|
}
|
|
return gjson.New(map[string]any{
|
|
"rowNo": row.RowNo,
|
|
"inputCode": row.InputCode,
|
|
"ICCID": row.ICCID,
|
|
"MSISDN": row.MSISDN,
|
|
"IMSI": row.IMSI,
|
|
"IMEI": row.IMEI,
|
|
"chanKey": row.ChanKey,
|
|
"cardType": row.CardType,
|
|
"upPkgCode": row.UpPkgCode,
|
|
"suppliersCode": row.SuppliersCode,
|
|
"upstreamActivationTime": row.UpstreamActivationTime,
|
|
"insertToPlugEnabled": row.InsertToPlugEnabled,
|
|
"carrierTrafficCutoffEnabled": row.CarrierTrafficCutoffEnabled,
|
|
"agreementExpiryDate": row.AgreementExpiryDate,
|
|
"hasTestPeriod": row.HasTestPeriod,
|
|
"testPeriodDays": row.TestPeriodDays,
|
|
"testPeriodLatestExpiryDate": row.TestPeriodLatestExpiryDate,
|
|
"hasSilentPeriod": row.HasSilentPeriod,
|
|
"silentPeriodDays": row.SilentPeriodDays,
|
|
"silentPeriodLatestExpiryDate": row.SilentPeriodLatestExpiryDate,
|
|
"rawFields": raw,
|
|
})
|
|
}
|
|
|
|
func buildInboundBatchInfo(task *inbound.Task) *gjson.Json {
|
|
if task == nil {
|
|
return gjson.New(map[string]any{})
|
|
}
|
|
info := make(map[string]any)
|
|
if task != nil && task.Batch.BatchInfo != nil {
|
|
for key, value := range task.Batch.BatchInfo.Map() {
|
|
info[key] = value
|
|
}
|
|
}
|
|
info["inboundOptions"] = map[string]any{
|
|
"chanKey": task.Options.ChanKey,
|
|
"validateUpstreamBasicInfo": task.Options.ValidateUpstreamBasicInfo,
|
|
"cardType": task.Options.CardType,
|
|
"upPkgCode": task.Options.UpPkgCode,
|
|
"suppliersCode": task.Options.SuppliersCode,
|
|
"upstreamActivationTime": task.Options.UpstreamActivationTime,
|
|
"insertToPlugEnabled": task.Options.InsertToPlugEnabled,
|
|
"carrierTrafficCutoffEnabled": task.Options.CarrierTrafficCutoffEnabled,
|
|
"agreementExpiryDate": task.Options.AgreementExpiryDate,
|
|
"hasTestPeriod": task.Options.HasTestPeriod,
|
|
"testPeriodDays": task.Options.TestPeriodDays,
|
|
"testPeriodLatestExpiryDate": task.Options.TestPeriodLatestExpiryDate,
|
|
"hasSilentPeriod": task.Options.HasSilentPeriod,
|
|
"silentPeriodDays": task.Options.SilentPeriodDays,
|
|
"silentPeriodLatestExpiryDate": task.Options.SilentPeriodLatestExpiryDate,
|
|
}
|
|
return gjson.New(info)
|
|
}
|
|
|
|
func buildResultBatchInfo(task *inbound.Task, status string, ossID, downloadID int64, reason string) *gjson.Json {
|
|
info := make(map[string]any)
|
|
if task.Batch.BatchInfo != nil {
|
|
for key, value := range task.Batch.BatchInfo.Map() {
|
|
info[key] = value
|
|
}
|
|
}
|
|
resultFile := map[string]any{"status": status}
|
|
if ossID > 0 {
|
|
resultFile["ossId"] = ossID
|
|
}
|
|
if downloadID > 0 {
|
|
resultFile["downloadId"] = downloadID
|
|
}
|
|
if reason != "" {
|
|
resultFile["error"] = reason
|
|
}
|
|
info["resultFile"] = resultFile
|
|
return gjson.New(info)
|
|
}
|
|
|
|
func fileDisplayName(oss entity.SysOss, fallback string) string {
|
|
for _, value := range []string{oss.OriginalName, oss.FileName, fallback} {
|
|
if strings.TrimSpace(value) != "" {
|
|
return strings.TrimSpace(value)
|
|
}
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func nullableString(value string) any {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func countRowsWithoutReason(rows []*inbound.Row) int {
|
|
count := 0
|
|
for _, row := range rows {
|
|
if row != nil && row.Reason == "" {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func successRows(rows []*inbound.Row) int {
|
|
count := 0
|
|
for _, row := range rows {
|
|
if row != nil && row.Reason == "" {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func failedRows(rows []*inbound.Row) int {
|
|
count := 0
|
|
for _, row := range rows {
|
|
if row != nil && row.Reason != "" && !inbound.IsCascadeFailure(row) {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|