chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,914 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
|
||||
inbound "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/inbound"
|
||||
)
|
||||
|
||||
const stockTable = "t_sim_card_stock"
|
||||
const (
|
||||
inboundStockQueryBatchSizeConfigKey = "batch.inbound.stockQueryBatchSize"
|
||||
inboundWriteBatchSizeConfigKey = "batch.inbound.writeBatchSize"
|
||||
defaultInboundStockQueryBatchSize = 1000
|
||||
defaultInboundWriteBatchSize = 1000
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
db gdb.DB
|
||||
resultFiles batchresult.Generator
|
||||
}
|
||||
|
||||
type stockDO struct {
|
||||
g.Meta `orm:"table:t_sim_card_stock, do:true"`
|
||||
BatchNo any `orm:"batch_no"`
|
||||
ICCID any `orm:"iccid"`
|
||||
MSISDN any `orm:"msisdn"`
|
||||
IMSI any `orm:"imsi"`
|
||||
IMEI any `orm:"imei"`
|
||||
ChanKey any `orm:"chan_key"`
|
||||
TenantID any `orm:"tenant_id"`
|
||||
CreatedDept any `orm:"created_dept"`
|
||||
CreatedBy any `orm:"created_by"`
|
||||
UpdatedBy any `orm:"updated_by"`
|
||||
}
|
||||
|
||||
func New() *Repository {
|
||||
return &Repository{db: dao.BatchMain.DB(), resultFiles: batchresult.New()}
|
||||
}
|
||||
|
||||
func (r *Repository) database() gdb.DB {
|
||||
if r.db != nil {
|
||||
return r.db
|
||||
}
|
||||
return g.DB()
|
||||
}
|
||||
|
||||
func (r *Repository) Load(ctx context.Context, batchNo string) (*inbound.Task, error) {
|
||||
if batchNo == "" {
|
||||
return nil, gerror.New("inbound batch number is empty")
|
||||
}
|
||||
var batch entity.BatchMain
|
||||
if err := r.database().Model(dao.BatchMain.Table()).Safe().Ctx(ctx).
|
||||
Where(dao.BatchMain.Columns().BatchNo, batchNo).
|
||||
Where(dao.BatchMain.Columns().BatchType, consts.BatchTypeInbound).
|
||||
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
|
||||
Scan(&batch); err != nil {
|
||||
return nil, gerror.Wrap(err, "query inbound batch")
|
||||
}
|
||||
if batch.Id == 0 {
|
||||
return nil, gerror.New("processing inbound batch does not exist")
|
||||
}
|
||||
task := &inbound.Task{
|
||||
Batch: inbound.Batch{
|
||||
ID: batch.Id,
|
||||
BatchNo: batch.BatchNo,
|
||||
BatchType: batch.BatchType,
|
||||
SourceType: batch.SourceType,
|
||||
TenantID: batch.TenantId,
|
||||
CreatedDept: batch.CreatedDept,
|
||||
CreatedBy: batch.CreatedBy,
|
||||
StartTime: batch.StartTime,
|
||||
BatchInfo: batch.BatchInfo,
|
||||
},
|
||||
}
|
||||
task.Source = parseSource(batch.BatchInfo, batch.SourceOssId, batch.SourceType)
|
||||
task.Options = parseOptions(batch.BatchInfo)
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func (r *Repository) ReadOssFile(ctx context.Context, ossID int64) ([]byte, string, error) {
|
||||
if ossID <= 0 {
|
||||
return nil, "", gerror.New("oss id is empty")
|
||||
}
|
||||
var oss entity.SysOss
|
||||
if err := r.database().Model(dao.SysOss.Table()).Safe().Ctx(ctx).
|
||||
Where(dao.SysOss.Columns().OssId, ossID).
|
||||
Scan(&oss); err != nil {
|
||||
return nil, "", gerror.Wrap(err, "query OSS file")
|
||||
}
|
||||
if oss.OssId == 0 {
|
||||
return nil, "", gerror.Newf("OSS file %d does not exist", ossID)
|
||||
}
|
||||
if oss.Service != "local" {
|
||||
return nil, "", gerror.Newf("OSS file %d is not a local file: service=%q", ossID, oss.Service)
|
||||
}
|
||||
uploadRoot := g.Cfg().MustGet(ctx, "system.upload.local.path").String()
|
||||
localPath, attempts, err := resolveReadableLocalOssPath(uploadRoot, oss.TenantId, oss.Path)
|
||||
if err != nil {
|
||||
return nil, fileDisplayName(oss, ""), gerror.Wrapf(err, "resolve OSS file %d local path from %q, attempted paths: %s", ossID, oss.Path, strings.Join(attempts, "; "))
|
||||
}
|
||||
content, err := os.ReadFile(localPath)
|
||||
if err != nil {
|
||||
return nil, fileDisplayName(oss, localPath), gerror.Wrapf(err, "OSS file %d local path is unreadable: original path=%q, resolved path=%q", ossID, oss.Path, localPath)
|
||||
}
|
||||
return content, fileDisplayName(oss, localPath), nil
|
||||
}
|
||||
|
||||
func resolveLocalOssPath(uploadRoot, tenantID, storedPath string) (string, error) {
|
||||
uploadRoot = strings.TrimSpace(uploadRoot)
|
||||
if uploadRoot == "" {
|
||||
return "", gerror.New("system.upload.local.path is empty")
|
||||
}
|
||||
tenantID = strings.TrimSpace(tenantID)
|
||||
if tenantID == "" {
|
||||
tenantID = "default"
|
||||
}
|
||||
storedPath = strings.TrimSpace(storedPath)
|
||||
if storedPath == "" {
|
||||
return "", gerror.New("OSS path is empty")
|
||||
}
|
||||
|
||||
objectSegments, err := ossObjectSegments(tenantID, storedPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
objectPath := filepath.Join(objectSegments...)
|
||||
localPath := filepath.Join(uploadRoot, objectPath)
|
||||
relativePath, err := filepath.Rel(uploadRoot, localPath)
|
||||
if err != nil {
|
||||
return "", gerror.Wrap(err, "resolve local OSS path")
|
||||
}
|
||||
if relativePath == ".." || strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) {
|
||||
return "", gerror.New("resolved OSS path escapes upload root")
|
||||
}
|
||||
return localPath, nil
|
||||
}
|
||||
|
||||
func resolveReadableLocalOssPath(uploadRoot, tenantID, storedPath string) (string, []string, error) {
|
||||
var attempts []string
|
||||
var firstErr error
|
||||
for _, root := range uploadRootCandidates(uploadRoot) {
|
||||
localPath, err := resolveLocalOssPath(root, tenantID, storedPath)
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
attempts = appendUniqueString(attempts, localPath)
|
||||
if isReadableFile(localPath) {
|
||||
return localPath, attempts, nil
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := ossObjectSegments(strings.TrimSpace(defaultTenant(tenantID)), storedPath); err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
} else {
|
||||
for _, candidate := range storedPathCandidates(storedPath) {
|
||||
attempts = appendUniqueString(attempts, candidate)
|
||||
if isReadableFile(candidate) {
|
||||
return candidate, attempts, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if firstErr != nil && len(attempts) == 0 {
|
||||
return "", attempts, firstErr
|
||||
}
|
||||
return "", attempts, gerror.New("OSS local file does not exist")
|
||||
}
|
||||
|
||||
func uploadRootCandidates(uploadRoot string) []string {
|
||||
var candidates []string
|
||||
for _, root := range []string{
|
||||
os.Getenv("IOTLCM_WORKER_UPLOAD_ROOT"),
|
||||
os.Getenv("IOTLCM_UPLOAD_ROOT"),
|
||||
uploadRoot,
|
||||
} {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
continue
|
||||
}
|
||||
candidates = appendUploadRootCandidate(candidates, root)
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func appendUploadRootCandidate(candidates []string, root string) []string {
|
||||
for _, candidate := range uploadRootEquivalentCandidates(root) {
|
||||
if filepath.IsAbs(candidate) {
|
||||
candidates = appendUniqueString(candidates, candidate)
|
||||
} else {
|
||||
for _, base := range localBaseDirs() {
|
||||
candidates = appendUniqueString(candidates, filepath.Join(base, candidate))
|
||||
}
|
||||
candidates = appendUniqueString(candidates, candidate)
|
||||
}
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func uploadRootEquivalentCandidates(root string) []string {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return nil
|
||||
}
|
||||
candidates := []string{root}
|
||||
normalized := filepath.ToSlash(filepath.Clean(root))
|
||||
if normalized == "resource/upload" || strings.HasSuffix(normalized, "/resource/upload") {
|
||||
prefix := strings.TrimSuffix(normalized, "resource/upload")
|
||||
candidates = appendUniqueString(candidates, filepath.FromSlash(prefix+"upload"))
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func storedPathCandidates(storedPath string) []string {
|
||||
storedPath = strings.TrimSpace(storedPath)
|
||||
if storedPath == "" {
|
||||
return nil
|
||||
}
|
||||
var candidates []string
|
||||
candidates = appendUniqueString(candidates, storedPath)
|
||||
if !filepath.IsAbs(storedPath) {
|
||||
for _, base := range localBaseDirs() {
|
||||
candidates = appendUniqueString(candidates, filepath.Join(base, storedPath))
|
||||
}
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func localBaseDirs() []string {
|
||||
var dirs []string
|
||||
if wd, err := os.Getwd(); err == nil && strings.TrimSpace(wd) != "" {
|
||||
dirs = appendUniqueString(dirs, wd)
|
||||
}
|
||||
if exe, err := os.Executable(); err == nil && strings.TrimSpace(exe) != "" {
|
||||
dirs = appendUniqueString(dirs, filepath.Dir(exe))
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
func ossObjectSegments(tenantID, storedPath string) ([]string, error) {
|
||||
tenantID = defaultTenant(tenantID)
|
||||
segments := strings.FieldsFunc(strings.ReplaceAll(storedPath, "\\", "/"), func(r rune) bool {
|
||||
return r == '/'
|
||||
})
|
||||
for index, segment := range segments {
|
||||
if segment != tenantID {
|
||||
continue
|
||||
}
|
||||
objectSegments := segments[index:]
|
||||
for _, objectSegment := range objectSegments {
|
||||
if objectSegment == "." || objectSegment == ".." {
|
||||
return nil, gerror.Newf("invalid OSS object path segment %q", objectSegment)
|
||||
}
|
||||
}
|
||||
return objectSegments, nil
|
||||
}
|
||||
return nil, gerror.Newf("OSS path does not contain tenant %q", tenantID)
|
||||
}
|
||||
|
||||
func defaultTenant(tenantID string) string {
|
||||
tenantID = strings.TrimSpace(tenantID)
|
||||
if tenantID == "" {
|
||||
return "default"
|
||||
}
|
||||
return tenantID
|
||||
}
|
||||
|
||||
func isReadableFile(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && !info.IsDir()
|
||||
}
|
||||
|
||||
func appendUniqueString(values []string, value string) []string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return values
|
||||
}
|
||||
for _, existing := range values {
|
||||
if existing == value {
|
||||
return values
|
||||
}
|
||||
}
|
||||
return append(values, value)
|
||||
}
|
||||
|
||||
func uniqueStrings(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
uniq := make([]string, 0, len(values))
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
uniq = append(uniq, value)
|
||||
}
|
||||
return uniq
|
||||
}
|
||||
|
||||
func (r *Repository) FindExistingStocks(ctx context.Context, _ string, iccids, msisdns, imsis []string) ([]inbound.Stock, error) {
|
||||
if len(iccids) == 0 && len(msisdns) == 0 && len(imsis) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
type queryResult struct {
|
||||
stocks []inbound.Stock
|
||||
err error
|
||||
}
|
||||
results := make(chan queryResult, 3)
|
||||
queries := 0
|
||||
if len(iccids) > 0 {
|
||||
queries++
|
||||
go func() {
|
||||
stocks, err := r.findExistingStocksByField(ctx, "iccid", "ICCID", iccids)
|
||||
results <- queryResult{stocks: stocks, err: err}
|
||||
}()
|
||||
}
|
||||
if len(msisdns) > 0 {
|
||||
queries++
|
||||
go func() {
|
||||
stocks, err := r.findExistingStocksByField(ctx, "msisdn", "MSISDN", msisdns)
|
||||
results <- queryResult{stocks: stocks, err: err}
|
||||
}()
|
||||
}
|
||||
if len(imsis) > 0 {
|
||||
queries++
|
||||
go func() {
|
||||
stocks, err := r.findExistingStocksByField(ctx, "imsi", "IMSI", imsis)
|
||||
results <- queryResult{stocks: stocks, err: err}
|
||||
}()
|
||||
}
|
||||
stocks := make([]inbound.Stock, 0)
|
||||
for i := 0; i < queries; i++ {
|
||||
result := <-results
|
||||
if result.err != nil {
|
||||
return nil, result.err
|
||||
}
|
||||
stocks = append(stocks, result.stocks...)
|
||||
}
|
||||
return stocks, nil
|
||||
}
|
||||
|
||||
func (r *Repository) stockQuery(ctx context.Context) *gdb.Model {
|
||||
return r.database().Model(stockTable).Safe().Ctx(ctx)
|
||||
}
|
||||
|
||||
func (r *Repository) findExistingStocksByField(ctx context.Context, field, label string, values []string) ([]inbound.Stock, error) {
|
||||
uniq := uniqueStrings(values)
|
||||
if len(uniq) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
batchSize := inboundStockQueryBatchSize(ctx)
|
||||
stocks := make([]inbound.Stock, 0)
|
||||
for _, chunk := range chunkStrings(uniq, batchSize) {
|
||||
var chunkStocks []inbound.Stock
|
||||
model := r.stockQuery(ctx).WhereIn(field, chunk)
|
||||
if err := model.Fields("iccid", "msisdn", "imsi").Scan(&chunkStocks); err != nil {
|
||||
return nil, gerror.Wrapf(err, "query inbound stock %s duplicates", label)
|
||||
}
|
||||
stocks = append(stocks, chunkStocks...)
|
||||
}
|
||||
return stocks, nil
|
||||
}
|
||||
|
||||
func (r *Repository) SaveValidationFailure(ctx context.Context, task *inbound.Task) error {
|
||||
err := r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
batchInfo := buildInboundBatchInfo(task)
|
||||
task.Batch.BatchInfo = batchInfo
|
||||
details := buildBatchDetails(task, consts.BatchDetailStatusFailed)
|
||||
failCount := 0
|
||||
detailIndex := 0
|
||||
for _, row := range task.Rows {
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
if inbound.IsCascadeFailure(row) {
|
||||
details[detailIndex].DetailStatus = consts.BatchDetailStatusNotExecuted
|
||||
} else {
|
||||
failCount++
|
||||
}
|
||||
detailIndex++
|
||||
}
|
||||
if err := insertChunkedRows(ctx, tx, dao.BatchDetail.Table(), "inbound detail", details, inboundWriteBatchSize(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
return updateBatchMain(ctx, tx, task, do.BatchMain{
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
||||
TotalCount: len(task.Rows),
|
||||
ValidCount: task.ValidCount,
|
||||
SuccessCount: 0,
|
||||
FailCount: failCount,
|
||||
RunResult: consts.BatchRunResultFailed,
|
||||
FailReason: batchfields.FailureReason(task.FailureReason),
|
||||
BatchInfo: batchInfo,
|
||||
EndTime: gtime.Now(),
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
})
|
||||
})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
g.Log().Errorf(ctx, "save inbound validation details failed, batchNo:%s, err:%s", task.Batch.BatchNo, batchfields.FailureReason(err.Error()))
|
||||
finalizationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
fallbackErr := r.database().Transaction(finalizationCtx, func(ctx context.Context, tx gdb.TX) error {
|
||||
return updateBatchMain(ctx, tx, task, do.BatchMain{
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
||||
TotalCount: len(task.Rows),
|
||||
ValidCount: task.ValidCount,
|
||||
SuccessCount: 0,
|
||||
FailCount: len(task.Rows),
|
||||
RunResult: consts.BatchRunResultFailed,
|
||||
FailReason: batchfields.FailureReason("错误明细保存失败，本次整批未入库"),
|
||||
BatchInfo: task.Batch.BatchInfo,
|
||||
EndTime: gtime.Now(),
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
})
|
||||
})
|
||||
if fallbackErr != nil {
|
||||
return gerror.Wrapf(fallbackErr, "save inbound failure details failed: %v; finalize batch failed", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Repository) SaveSuccess(ctx context.Context, task *inbound.Task) error {
|
||||
return r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
batchInfo := buildInboundBatchInfo(task)
|
||||
task.Batch.BatchInfo = batchInfo
|
||||
stocks := make([]stockDO, 0, len(task.Rows))
|
||||
details := make([]do.BatchDetail, 0, len(task.Rows))
|
||||
for _, row := range task.Rows {
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
stocks = append(stocks, stockDO{
|
||||
BatchNo: task.Batch.BatchNo,
|
||||
ICCID: row.ICCID,
|
||||
MSISDN: nullableString(row.MSISDN),
|
||||
IMSI: nullableString(row.IMSI),
|
||||
IMEI: nullableString(row.IMEI),
|
||||
ChanKey: row.ChanKey,
|
||||
TenantID: task.Batch.TenantID,
|
||||
CreatedDept: task.Batch.CreatedDept,
|
||||
CreatedBy: task.Batch.CreatedBy,
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
})
|
||||
details = append(details, batchDetailRow(task, row, consts.BatchDetailStatusSuccess))
|
||||
}
|
||||
if err := insertChunkedRows(ctx, tx, stockTable, "inbound stock", stocks, inboundWriteBatchSize(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := insertChunkedRows(ctx, tx, dao.BatchDetail.Table(), "inbound detail", details, inboundWriteBatchSize(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
return updateBatchMain(ctx, tx, task, do.BatchMain{
|
||||
BatchAsyncStatus: consts.BatchAsyncStatusSuccess,
|
||||
TotalCount: len(task.Rows),
|
||||
ValidCount: len(task.Rows),
|
||||
SuccessCount: len(task.Rows),
|
||||
FailCount: 0,
|
||||
RunResult: consts.BatchRunResultSuccess,
|
||||
FailReason: "",
|
||||
BatchInfo: batchInfo,
|
||||
EndTime: gtime.Now(),
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Repository) SaveBusinessFailure(ctx context.Context, task *inbound.Task, reason string) error {
|
||||
task.ValidCount = countRowsWithoutReason(task.Rows)
|
||||
for _, row := range task.Rows {
|
||||
if row != nil {
|
||||
row.Reason = reason
|
||||
}
|
||||
}
|
||||
task.FailureReason = reason
|
||||
return r.SaveValidationFailure(ctx, task)
|
||||
}
|
||||
|
||||
func buildBatchDetails(task *inbound.Task, status int) []do.BatchDetail {
|
||||
details := make([]do.BatchDetail, 0, len(task.Rows))
|
||||
for _, row := range task.Rows {
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
details = append(details, batchDetailRow(task, row, status))
|
||||
}
|
||||
return details
|
||||
}
|
||||
|
||||
func batchDetailRow(task *inbound.Task, row *inbound.Row, status int) do.BatchDetail {
|
||||
return do.BatchDetail{
|
||||
BatchNo: task.Batch.BatchNo,
|
||||
BatchType: task.Batch.BatchType,
|
||||
DetailStatus: status,
|
||||
BlankOutStatus: 1,
|
||||
RowNo: row.RowNo,
|
||||
ItemId: batchfields.ItemID(row.ICCID),
|
||||
ErrorReason: batchfields.DetailReason(row.Reason),
|
||||
Snapshot: rowSnapshot(row),
|
||||
TenantId: task.Batch.TenantID,
|
||||
CreatedDept: task.Batch.CreatedDept,
|
||||
CreatedBy: task.Batch.CreatedBy,
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Repository) SaveResultFile(ctx context.Context, task *inbound.Task, request batchresult.Request) error {
|
||||
artifact, err := r.artifactGenerator().Create(ctx, request)
|
||||
if err != nil {
|
||||
return r.updateResultFileStatus(ctx, task, "failed", 0, 0, err.Error())
|
||||
}
|
||||
var (
|
||||
ossID int64
|
||||
downloadID int64
|
||||
)
|
||||
err = r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
var insertErr error
|
||||
ossID, insertErr = tx.Ctx(ctx).Model(dao.SysOss.Table()).Data(do.SysOss{
|
||||
TenantId: task.Batch.TenantID,
|
||||
FileName: artifact.FileName,
|
||||
OriginalName: artifact.OriginalName,
|
||||
FileSuffix: artifact.FileSuffix,
|
||||
Path: artifact.Path,
|
||||
Url: artifact.URL,
|
||||
CreatedDept: task.Batch.CreatedDept,
|
||||
CreatedBy: task.Batch.CreatedBy,
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
Service: "local",
|
||||
Md5: artifact.MD5,
|
||||
FileSize: artifact.Size,
|
||||
}).InsertAndGetId()
|
||||
if insertErr != nil {
|
||||
return gerror.Wrap(insertErr, "insert inbound result OSS")
|
||||
}
|
||||
downloadID, insertErr = tx.Ctx(ctx).Model(dao.TResourceBatchDownload.Table()).Data(do.TResourceBatchDownload{
|
||||
BatchNo: task.Batch.BatchNo,
|
||||
TaskName: batchresult.TaskName(task.Batch.BatchType),
|
||||
ResourceType: consts.ResourceTypeExcel,
|
||||
ExportScene: task.Batch.BatchType,
|
||||
TotalCount: len(task.Rows),
|
||||
SuccessCount: successRows(task.Rows),
|
||||
FailCount: failedRows(task.Rows),
|
||||
DownloadStatus: consts.DownloadStatusGenerated,
|
||||
OssId: ossID,
|
||||
FileName: artifact.FileName,
|
||||
FileExt: artifact.FileSuffix,
|
||||
FileUrl: artifact.URL,
|
||||
FilePath: artifact.Path,
|
||||
FileSize: artifact.Size,
|
||||
FileMd5: artifact.MD5,
|
||||
DownloadCount: 0,
|
||||
MaxDownloadCount: 0,
|
||||
DeptId: task.Batch.CreatedDept,
|
||||
StartTime: task.Batch.StartTime,
|
||||
FinishTime: gtime.Now(),
|
||||
TenantId: task.Batch.TenantID,
|
||||
CreatedDept: task.Batch.CreatedDept,
|
||||
CreatedBy: task.Batch.CreatedBy,
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
}).InsertAndGetId()
|
||||
if insertErr != nil {
|
||||
return gerror.Wrap(insertErr, "insert inbound result download")
|
||||
}
|
||||
return updateBatchInfoOnly(ctx, tx, task, buildResultBatchInfo(task, "success", ossID, downloadID, ""), ossID)
|
||||
})
|
||||
if err != nil {
|
||||
batchresult.RemoveLocalFile(artifact)
|
||||
return r.updateResultFileStatus(ctx, task, "failed", 0, 0, err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Repository) artifactGenerator() batchresult.Generator {
|
||||
if r.resultFiles != nil {
|
||||
return r.resultFiles
|
||||
}
|
||||
return batchresult.New()
|
||||
}
|
||||
|
||||
func (r *Repository) updateResultFileStatus(ctx context.Context, task *inbound.Task, status string, ossID, downloadID int64, reason string) error {
|
||||
return r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
return updateBatchInfoOnly(ctx, tx, task, buildResultBatchInfo(task, status, ossID, downloadID, reason), ossID)
|
||||
})
|
||||
}
|
||||
|
||||
func insertChunkedRows[T any](ctx context.Context, tx gdb.TX, table, label string, rows []T, batchSize int) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
batchSize = defaultInboundWriteBatchSize
|
||||
}
|
||||
for start := 0; start < len(rows); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
if _, err := tx.Ctx(ctx).Model(table).Data(rows[start:end]).Insert(); err != nil {
|
||||
return gerror.Wrapf(err, "insert %s chunk", label)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func chunkStrings(values []string, batchSize int) [][]string {
|
||||
if batchSize <= 0 {
|
||||
batchSize = defaultInboundStockQueryBatchSize
|
||||
}
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
chunks := make([][]string, 0, (len(values)+batchSize-1)/batchSize)
|
||||
for start := 0; start < len(values); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(values) {
|
||||
end = len(values)
|
||||
}
|
||||
chunks = append(chunks, values[start:end])
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
func inboundStockQueryBatchSize(ctx context.Context) int {
|
||||
value := g.Cfg().MustGet(ctx, inboundStockQueryBatchSizeConfigKey, defaultInboundStockQueryBatchSize).Int()
|
||||
if value <= 0 {
|
||||
return defaultInboundStockQueryBatchSize
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func inboundWriteBatchSize(ctx context.Context) int {
|
||||
value := g.Cfg().MustGet(ctx, inboundWriteBatchSizeConfigKey, defaultInboundWriteBatchSize).Int()
|
||||
if value <= 0 {
|
||||
return defaultInboundWriteBatchSize
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func updateBatchMain(ctx context.Context, tx gdb.TX, task *inbound.Task, data do.BatchMain) error {
|
||||
result, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
||||
Where(dao.BatchMain.Columns().BatchNo, task.Batch.BatchNo).
|
||||
Where(dao.BatchMain.Columns().BatchType, task.Batch.BatchType).
|
||||
Where(dao.BatchMain.Columns().BatchAsyncStatus, consts.BatchAsyncStatusProcessing).
|
||||
Data(data).
|
||||
Update()
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "update inbound batch main")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "read inbound batch main update result")
|
||||
}
|
||||
if affected != 1 {
|
||||
return gerror.New("inbound batch is no longer processing")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateBatchInfoOnly(ctx context.Context, tx gdb.TX, task *inbound.Task, info *gjson.Json, resultOssID int64) error {
|
||||
result, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
|
||||
Where(dao.BatchMain.Columns().BatchNo, task.Batch.BatchNo).
|
||||
Where(dao.BatchMain.Columns().BatchType, task.Batch.BatchType).
|
||||
Data(do.BatchMain{
|
||||
ResultOssId: workerOssIDValue(resultOssID),
|
||||
BatchInfo: info,
|
||||
UpdatedBy: task.Batch.CreatedBy,
|
||||
}).
|
||||
Update()
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "update inbound result file status")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "read inbound result file update result")
|
||||
}
|
||||
if affected != 1 {
|
||||
return gerror.New("inbound batch disappeared while saving result file")
|
||||
}
|
||||
task.Batch.BatchInfo = info
|
||||
return nil
|
||||
}
|
||||
|
||||
func workerOssIDValue(ossID int64) any {
|
||||
if ossID <= 0 {
|
||||
return nil
|
||||
}
|
||||
return ossID
|
||||
}
|
||||
|
||||
func parseSource(info *gjson.Json, sourceOssID int64, sourceType int) inbound.Source {
|
||||
source := inbound.Source{
|
||||
ImportFileOssID: jsonInt64(info, "source.importFileOssId", "importFileOssId"),
|
||||
InputText: jsonString(info, "source.inputText", "inputText"),
|
||||
InputFileOssID: jsonInt64(info, "source.inputFileOssId", "inputFileOssId"),
|
||||
Range: inbound.Range{
|
||||
RangeType: jsonString(info, "source.range.rangeType", "range.rangeType", "rangeType"),
|
||||
StartICCID: jsonString(info, "source.range.startIccid", "range.startIccid", "startIccid"),
|
||||
EndICCID: jsonString(info, "source.range.endIccid", "range.endIccid", "endIccid"),
|
||||
StartMSISDN: jsonString(info, "source.range.startMsisdn", "range.startMsisdn", "startMsisdn"),
|
||||
EndMSISDN: jsonString(info, "source.range.endMsisdn", "range.endMsisdn", "endMsisdn"),
|
||||
StartIMSI: jsonString(info, "source.range.startImsi", "range.startImsi", "startImsi"),
|
||||
EndIMSI: jsonString(info, "source.range.endImsi", "range.endImsi", "endImsi"),
|
||||
},
|
||||
}
|
||||
if sourceOssID > 0 {
|
||||
switch sourceType {
|
||||
case consts.BatchSourceTypeBatchInput:
|
||||
source.InputFileOssID = sourceOssID
|
||||
default:
|
||||
source.ImportFileOssID = sourceOssID
|
||||
}
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
func parseOptions(info *gjson.Json) inbound.Options {
|
||||
return inbound.Options{
|
||||
ChanKey: jsonString(info, "inboundOptions.chanKey", "chanKey"),
|
||||
ValidateUpstreamBasicInfo: jsonString(info, "inboundOptions.validateUpstreamBasicInfo", "validateUpstreamBasicInfo"),
|
||||
CardType: jsonString(info, "inboundOptions.cardType", "cardType"),
|
||||
UpPkgCode: jsonString(info, "inboundOptions.upPkgCode", "upPkgCode"),
|
||||
SuppliersCode: jsonString(info, "inboundOptions.suppliersCode", "suppliersCode"),
|
||||
UpstreamActivationTime: jsonString(info, "inboundOptions.upstreamActivationTime", "upstreamActivationTime"),
|
||||
InsertToPlugEnabled: jsonString(info, "inboundOptions.insertToPlugEnabled", "insertToPlugEnabled"),
|
||||
CarrierTrafficCutoffEnabled: jsonString(info, "inboundOptions.carrierTrafficCutoffEnabled", "carrierTrafficCutoffEnabled"),
|
||||
AgreementExpiryDate: jsonString(info, "inboundOptions.agreementExpiryDate", "agreementExpiryDate"),
|
||||
HasTestPeriod: jsonString(info, "inboundOptions.hasTestPeriod", "hasTestPeriod"),
|
||||
TestPeriodDays: jsonString(info, "inboundOptions.testPeriodDays", "testPeriodDays"),
|
||||
TestPeriodLatestExpiryDate: jsonString(info, "inboundOptions.testPeriodLatestExpiryDate", "testPeriodLatestExpiryDate"),
|
||||
HasSilentPeriod: jsonString(info, "inboundOptions.hasSilentPeriod", "hasSilentPeriod"),
|
||||
SilentPeriodDays: jsonString(info, "inboundOptions.silentPeriodDays", "silentPeriodDays"),
|
||||
SilentPeriodLatestExpiryDate: jsonString(info, "inboundOptions.silentPeriodLatestExpiryDate", "silentPeriodLatestExpiryDate"),
|
||||
}
|
||||
}
|
||||
|
||||
func jsonString(info *gjson.Json, paths ...string) string {
|
||||
if info == nil {
|
||||
return ""
|
||||
}
|
||||
for _, path := range paths {
|
||||
value := strings.TrimSpace(info.Get(path).String())
|
||||
if value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func jsonInt64(info *gjson.Json, paths ...string) int64 {
|
||||
if info == nil {
|
||||
return 0
|
||||
}
|
||||
for _, path := range paths {
|
||||
value := info.Get(path).Int64()
|
||||
if value > 0 {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func rowSnapshot(row *inbound.Row) *gjson.Json {
|
||||
raw := map[string]any{}
|
||||
for key, value := range row.Raw {
|
||||
raw[key] = value
|
||||
}
|
||||
return gjson.New(map[string]any{
|
||||
"rowNo": row.RowNo,
|
||||
"inputCode": row.InputCode,
|
||||
"ICCID": row.ICCID,
|
||||
"MSISDN": row.MSISDN,
|
||||
"IMSI": row.IMSI,
|
||||
"IMEI": row.IMEI,
|
||||
"chanKey": row.ChanKey,
|
||||
"cardType": row.CardType,
|
||||
"upPkgCode": row.UpPkgCode,
|
||||
"suppliersCode": row.SuppliersCode,
|
||||
"upstreamActivationTime": row.UpstreamActivationTime,
|
||||
"insertToPlugEnabled": row.InsertToPlugEnabled,
|
||||
"carrierTrafficCutoffEnabled": row.CarrierTrafficCutoffEnabled,
|
||||
"agreementExpiryDate": row.AgreementExpiryDate,
|
||||
"hasTestPeriod": row.HasTestPeriod,
|
||||
"testPeriodDays": row.TestPeriodDays,
|
||||
"testPeriodLatestExpiryDate": row.TestPeriodLatestExpiryDate,
|
||||
"hasSilentPeriod": row.HasSilentPeriod,
|
||||
"silentPeriodDays": row.SilentPeriodDays,
|
||||
"silentPeriodLatestExpiryDate": row.SilentPeriodLatestExpiryDate,
|
||||
"rawFields": raw,
|
||||
})
|
||||
}
|
||||
|
||||
func buildInboundBatchInfo(task *inbound.Task) *gjson.Json {
|
||||
if task == nil {
|
||||
return gjson.New(map[string]any{})
|
||||
}
|
||||
info := make(map[string]any)
|
||||
if task != nil && task.Batch.BatchInfo != nil {
|
||||
for key, value := range task.Batch.BatchInfo.Map() {
|
||||
info[key] = value
|
||||
}
|
||||
}
|
||||
info["inboundOptions"] = map[string]any{
|
||||
"chanKey": task.Options.ChanKey,
|
||||
"validateUpstreamBasicInfo": task.Options.ValidateUpstreamBasicInfo,
|
||||
"cardType": task.Options.CardType,
|
||||
"upPkgCode": task.Options.UpPkgCode,
|
||||
"suppliersCode": task.Options.SuppliersCode,
|
||||
"upstreamActivationTime": task.Options.UpstreamActivationTime,
|
||||
"insertToPlugEnabled": task.Options.InsertToPlugEnabled,
|
||||
"carrierTrafficCutoffEnabled": task.Options.CarrierTrafficCutoffEnabled,
|
||||
"agreementExpiryDate": task.Options.AgreementExpiryDate,
|
||||
"hasTestPeriod": task.Options.HasTestPeriod,
|
||||
"testPeriodDays": task.Options.TestPeriodDays,
|
||||
"testPeriodLatestExpiryDate": task.Options.TestPeriodLatestExpiryDate,
|
||||
"hasSilentPeriod": task.Options.HasSilentPeriod,
|
||||
"silentPeriodDays": task.Options.SilentPeriodDays,
|
||||
"silentPeriodLatestExpiryDate": task.Options.SilentPeriodLatestExpiryDate,
|
||||
}
|
||||
return gjson.New(info)
|
||||
}
|
||||
|
||||
func buildResultBatchInfo(task *inbound.Task, status string, ossID, downloadID int64, reason string) *gjson.Json {
|
||||
info := make(map[string]any)
|
||||
if task.Batch.BatchInfo != nil {
|
||||
for key, value := range task.Batch.BatchInfo.Map() {
|
||||
info[key] = value
|
||||
}
|
||||
}
|
||||
resultFile := map[string]any{"status": status}
|
||||
if ossID > 0 {
|
||||
resultFile["ossId"] = ossID
|
||||
}
|
||||
if downloadID > 0 {
|
||||
resultFile["downloadId"] = downloadID
|
||||
}
|
||||
if reason != "" {
|
||||
resultFile["error"] = reason
|
||||
}
|
||||
info["resultFile"] = resultFile
|
||||
return gjson.New(info)
|
||||
}
|
||||
|
||||
func fileDisplayName(oss entity.SysOss, fallback string) string {
|
||||
for _, value := range []string{oss.OriginalName, oss.FileName, fallback} {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func nullableString(value string) any {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func countRowsWithoutReason(rows []*inbound.Row) int {
|
||||
count := 0
|
||||
for _, row := range rows {
|
||||
if row != nil && row.Reason == "" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func successRows(rows []*inbound.Row) int {
|
||||
count := 0
|
||||
for _, row := range rows {
|
||||
if row != nil && row.Reason == "" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func failedRows(rows []*inbound.Row) int {
|
||||
count := 0
|
||||
for _, row := range rows {
|
||||
if row != nil && row.Reason != "" && !inbound.IsCascadeFailure(row) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
|
||||
serviceinbound "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/inbound"
|
||||
)
|
||||
|
||||
func TestBatchDetailRowClampsInvalidIdentifiersAndReasons(t *testing.T) {
|
||||
task := &serviceinbound.Task{Batch: serviceinbound.Batch{BatchNo: "INB-TEST", BatchType: 103}}
|
||||
row := &serviceinbound.Row{RowNo: 2, ICCID: strings.Repeat("9", 100), Reason: strings.Repeat("错", 1000)}
|
||||
detail := batchDetailRow(task, row, consts.BatchDetailStatusFailed)
|
||||
if detail.ItemId != "" || len([]rune(detail.ErrorReason.(string))) != batchfields.DetailReasonLimit {
|
||||
t.Fatalf("invalid detail: itemID=%v, reason=%v", detail.ItemId, detail.ErrorReason)
|
||||
}
|
||||
if row.ICCID != strings.Repeat("9", 100) || row.Reason != strings.Repeat("错", 1000) {
|
||||
t.Fatal("source row was mutated; full value must remain available to result export")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveValidationFailureHandlesOversizedDetailsAndFallback(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
oldAdapter := g.Cfg().GetAdapter()
|
||||
adapter, err := gcfg.NewAdapterContent("batch:\n inbound:\n writeBatchSize: 1000\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
defer g.Cfg().SetAdapter(oldAdapter)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
constraint string
|
||||
wantFallback bool
|
||||
}{
|
||||
{name: "oversized invalid row is saved", constraint: "CHECK(length(item_id) <= 64 AND length(error_reason) <= 512)"},
|
||||
{name: "detail write fails", constraint: "CHECK(error_reason = 'allowed')", wantFallback: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
db, err := gdb.New(gdb.ConfigNode{Type: "sqlite", Link: "sqlite::@file(:memory:)", MaxOpenConnCount: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close(ctx) })
|
||||
statements := []string{
|
||||
"CREATE TABLE batch_main (batch_no TEXT PRIMARY KEY, batch_type INTEGER, batch_async_status INTEGER, total_count INTEGER, valid_count INTEGER, success_count INTEGER, fail_count INTEGER, run_result INTEGER, fail_reason TEXT, batch_info TEXT, end_time DATETIME, updated_by INTEGER)",
|
||||
"CREATE TABLE batch_detail (id INTEGER PRIMARY KEY AUTOINCREMENT, batch_no TEXT, batch_type INTEGER, detail_status INTEGER, blank_out_status INTEGER, row_no INTEGER, item_id TEXT, error_reason TEXT " + test.constraint + ", snapshot TEXT, tenant_id TEXT, created_dept INTEGER, created_by INTEGER, updated_by INTEGER)",
|
||||
"INSERT INTO batch_main (batch_no, batch_type, batch_async_status) VALUES ('INB-TEST', 103, 2)",
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := db.Exec(ctx, statement); err != nil {
|
||||
t.Fatalf("fixture failed: %v", err)
|
||||
}
|
||||
}
|
||||
task := &serviceinbound.Task{
|
||||
Batch: serviceinbound.Batch{BatchNo: "INB-TEST", BatchType: 103},
|
||||
Rows: []*serviceinbound.Row{{RowNo: 2, ICCID: strings.Repeat("9", 100), Reason: strings.Repeat("错", 1000)}},
|
||||
FailureReason: "输入格式不正确",
|
||||
}
|
||||
if err := (&Repository{db: db}).SaveValidationFailure(ctx, task); err != nil {
|
||||
t.Fatalf("failed batch must be finalized: %v", err)
|
||||
}
|
||||
status, err := db.Model("batch_main").Ctx(ctx).Where("batch_no", "INB-TEST").Value("batch_async_status")
|
||||
if err != nil || status.Int() != consts.BatchAsyncStatusFailed {
|
||||
t.Fatalf("status = %v, err = %v", status, err)
|
||||
}
|
||||
reason, err := db.Model("batch_main").Ctx(ctx).Where("batch_no", "INB-TEST").Value("fail_reason")
|
||||
if err != nil || strings.Contains(reason.String(), "错误明细保存失败") != test.wantFallback {
|
||||
t.Fatalf("fallback reason = %v, err = %v", reason, err)
|
||||
}
|
||||
detailCount, err := db.Model("batch_detail").Ctx(ctx).Where("batch_no", "INB-TEST").Count()
|
||||
wantDetails := 1
|
||||
if test.wantFallback {
|
||||
wantDetails = 0
|
||||
}
|
||||
if err != nil || detailCount != wantDetails {
|
||||
t.Fatalf("details = %d, want %d, err = %v", detailCount, wantDetails, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLocalOssPath(t *testing.T) {
|
||||
uploadRoot := t.TempDir()
|
||||
expectedPath := filepath.Join(uploadRoot, "000000", "user", "file", "20260721", "source.xlsx")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
tenantID string
|
||||
storedPath string
|
||||
wantPath string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "relative Admin path",
|
||||
tenantID: "000000",
|
||||
storedPath: "resource/upload/000000/user/file/20260721/source.xlsx",
|
||||
wantPath: expectedPath,
|
||||
},
|
||||
{
|
||||
name: "absolute Admin path",
|
||||
tenantID: "000000",
|
||||
storedPath: "/app/upload/000000/user/file/20260721/source.xlsx",
|
||||
wantPath: expectedPath,
|
||||
},
|
||||
{
|
||||
name: "default tenant",
|
||||
tenantID: "",
|
||||
storedPath: "resource/upload/default/user/file/20260721/source.xlsx",
|
||||
wantPath: filepath.Join(uploadRoot, "default", "user", "file", "20260721", "source.xlsx"),
|
||||
},
|
||||
{
|
||||
name: "tenant is missing",
|
||||
tenantID: "000000",
|
||||
storedPath: "resource/upload/000001/user/file/20260721/source.xlsx",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "path traversal is rejected",
|
||||
tenantID: "000000",
|
||||
storedPath: "resource/upload/000000/../secret.xlsx",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotPath, err := resolveLocalOssPath(uploadRoot, tt.tenantID, tt.storedPath)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("resolveLocalOssPath() error = %v", err)
|
||||
}
|
||||
if gotPath != tt.wantPath {
|
||||
t.Fatalf("resolveLocalOssPath() = %q, want %q", gotPath, tt.wantPath)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLocalOssPathReturnsReadableFile(t *testing.T) {
|
||||
uploadRoot := t.TempDir()
|
||||
storedPath := "resource/upload/000000/user/file/20260721/source.xlsx"
|
||||
localPath, err := resolveLocalOssPath(uploadRoot, "000000", storedPath)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveLocalOssPath() error = %v", err)
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
t.Fatalf("create directory: %v", err)
|
||||
}
|
||||
if err = os.WriteFile(localPath, []byte("inbound source"), 0o600); err != nil {
|
||||
t.Fatalf("write source file: %v", err)
|
||||
}
|
||||
content, err := os.ReadFile(localPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read source file: %v", err)
|
||||
}
|
||||
if string(content) != "inbound source" {
|
||||
t.Fatalf("unexpected source content: %q", content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRootCandidatesIncludeMountedUploadDirectory(t *testing.T) {
|
||||
wd := t.TempDir()
|
||||
oldWd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("get working directory: %v", err)
|
||||
}
|
||||
if err = os.Chdir(wd); err != nil {
|
||||
t.Fatalf("change working directory: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = os.Chdir(oldWd)
|
||||
}()
|
||||
|
||||
candidates := uploadRootCandidates("resource/upload")
|
||||
want := filepath.Join(wd, "upload")
|
||||
for _, candidate := range candidates {
|
||||
if filepath.Clean(candidate) == filepath.Clean(want) {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("uploadRootCandidates() = %#v, want %q", candidates, want)
|
||||
}
|
||||
|
||||
func TestResolveReadableLocalOssPathUsesMountedUploadDirectory(t *testing.T) {
|
||||
wd := t.TempDir()
|
||||
oldWd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("get working directory: %v", err)
|
||||
}
|
||||
if err = os.Chdir(wd); err != nil {
|
||||
t.Fatalf("change working directory: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = os.Chdir(oldWd)
|
||||
}()
|
||||
|
||||
storedPath := "resource/upload/000000/user/file/20260813/sim-card-inbound/import/source.xlsx"
|
||||
mountedPath := filepath.Join(wd, "upload", "000000", "user", "file", "20260813", "sim-card-inbound", "import", "source.xlsx")
|
||||
if err = os.MkdirAll(filepath.Dir(mountedPath), 0o755); err != nil {
|
||||
t.Fatalf("create mounted upload directory: %v", err)
|
||||
}
|
||||
if err = os.WriteFile(mountedPath, []byte("xlsx"), 0o600); err != nil {
|
||||
t.Fatalf("write mounted upload file: %v", err)
|
||||
}
|
||||
|
||||
gotPath, attempts, err := resolveReadableLocalOssPath("resource/upload", "000000", storedPath)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveReadableLocalOssPath() error = %v, attempts = %#v", err, attempts)
|
||||
}
|
||||
if gotPath != mountedPath {
|
||||
t.Fatalf("resolveReadableLocalOssPath() = %q, want %q, attempts = %#v", gotPath, mountedPath, attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildResultBatchInfoUsesGroupedResultFile(t *testing.T) {
|
||||
task := &serviceinbound.Task{
|
||||
Batch: serviceinbound.Batch{
|
||||
BatchInfo: gjson.New(map[string]any{
|
||||
"source": map[string]any{"inputText": "8986"},
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
info := buildResultBatchInfo(task, "failed", 0, 0, "disk full")
|
||||
if info.Get("resultFile.status").String() != "failed" {
|
||||
t.Fatalf("unexpected result file status: %s", info.Get("resultFile.status").String())
|
||||
}
|
||||
if info.Get("resultFile.error").String() != "disk full" {
|
||||
t.Fatalf("unexpected result file error: %s", info.Get("resultFile.error").String())
|
||||
}
|
||||
if info.Get("resultFileOssId").Int64() != 0 || info.Get("resultDownloadId").Int64() != 0 {
|
||||
t.Fatalf("result file metadata should stay grouped: %s", info.MustToJsonString())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAndBuildInboundOptionsPreserveUpstreamValidationFlag(t *testing.T) {
|
||||
info := gjson.New(map[string]any{
|
||||
"inboundOptions": map[string]any{
|
||||
"chanKey": "chan-a",
|
||||
"validateUpstreamBasicInfo": 1,
|
||||
},
|
||||
})
|
||||
|
||||
options := parseOptions(info)
|
||||
if options.ValidateUpstreamBasicInfo != "1" {
|
||||
t.Fatalf("ValidateUpstreamBasicInfo = %q, want 1", options.ValidateUpstreamBasicInfo)
|
||||
}
|
||||
|
||||
task := &serviceinbound.Task{
|
||||
Batch: serviceinbound.Batch{BatchInfo: info},
|
||||
Options: options,
|
||||
}
|
||||
built := buildInboundBatchInfo(task)
|
||||
if built.Get("inboundOptions.validateUpstreamBasicInfo").String() != "1" {
|
||||
t.Fatalf("validate option not preserved: %s", built.MustToJsonString())
|
||||
}
|
||||
if built.Get("inboundOptions.chanKey").String() != "chan-a" {
|
||||
t.Fatalf("chanKey not preserved: %s", built.MustToJsonString())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSourcePrefersSourceOssId(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sourceType int
|
||||
wantImportID int64
|
||||
wantInputID int64
|
||||
}{
|
||||
{name: "file source", sourceType: consts.BatchSourceTypeFile, wantImportID: 88, wantInputID: 22},
|
||||
{name: "input source", sourceType: consts.BatchSourceTypeBatchInput, wantImportID: 11, wantInputID: 88},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
info := gjson.New(map[string]any{
|
||||
"source": map[string]any{
|
||||
"importFileOssId": int64(11),
|
||||
"inputFileOssId": int64(22),
|
||||
},
|
||||
})
|
||||
source := parseSource(info, 88, tt.sourceType)
|
||||
if source.ImportFileOssID != tt.wantImportID {
|
||||
t.Fatalf("ImportFileOssID = %d, want %d", source.ImportFileOssID, tt.wantImportID)
|
||||
}
|
||||
if source.InputFileOssID != tt.wantInputID {
|
||||
t.Fatalf("InputFileOssID = %d, want %d", source.InputFileOssID, tt.wantInputID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user