iotlcm-public/iotlcm
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: 同步 v1.0.4 开源代码并脱敏

Browse Source
This commit is contained in:
MiFaZhan
2026-10-29 14:52:43 +08:00
parent 32e4316ee4
commit 6893e21f07
136 changed files with 9851 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/repository/inbound/repository.go
+55 -6
View File
@@ -5,6 +5,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/encoding/gjson"
@@ -14,6 +15,7 @@ import (
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
@@ -385,10 +387,23 @@ func (r *Repository) findExistingStocksByField(ctx context.Context, field, label
}
func (r *Repository) SaveValidationFailure(ctx context.Context, task *inbound.Task) error {
return r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
err := r.database().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
batchInfo := buildInboundBatchInfo(task)
task.Batch.BatchInfo = batchInfo
details := buildBatchDetails(task, consts.BatchDetailStatusFailed)
failCount := 0
detailIndex := 0
for _, row := range task.Rows {
if row == nil {
continue
}
if inbound.IsCascadeFailure(row) {
details[detailIndex].DetailStatus = consts.BatchDetailStatusNotExecuted
} else {
failCount++
}
detailIndex++
}
if err := insertChunkedRows(ctx, tx, dao.BatchDetail.Table(), "inbound detail", details, inboundWriteBatchSize(ctx)); err != nil {
return err
}
@@ -397,14 +412,38 @@ func (r *Repository) SaveValidationFailure(ctx context.Context, task *inbound.Ta
TotalCount: len(task.Rows),
ValidCount: task.ValidCount,
SuccessCount: 0,
FailCount: len(task.Rows),
FailCount: failCount,
RunResult: consts.BatchRunResultFailed,
FailReason: task.FailureReason,
FailReason: batchfields.FailureReason(task.FailureReason),
BatchInfo: batchInfo,
EndTime: gtime.Now(),
UpdatedBy: task.Batch.CreatedBy,
})
})
if err == nil {
return nil
}
g.Log().Errorf(ctx, "save inbound validation details failed, batchNo:%s, err:%s", task.Batch.BatchNo, batchfields.FailureReason(err.Error()))
finalizationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
fallbackErr := r.database().Transaction(finalizationCtx, func(ctx context.Context, tx gdb.TX) error {
return updateBatchMain(ctx, tx, task, do.BatchMain{
BatchAsyncStatus: consts.BatchAsyncStatusFailed,
TotalCount: len(task.Rows),
ValidCount: task.ValidCount,
SuccessCount: 0,
FailCount: len(task.Rows),
RunResult: consts.BatchRunResultFailed,
FailReason: batchfields.FailureReason("错误明细保存失败，本次整批未入库"),
BatchInfo: task.Batch.BatchInfo,
EndTime: gtime.Now(),
UpdatedBy: task.Batch.CreatedBy,
})
})
if fallbackErr != nil {
return gerror.Wrapf(fallbackErr, "save inbound failure details failed: %v; finalize batch failed", err)
}
return nil
}
func (r *Repository) SaveSuccess(ctx context.Context, task *inbound.Task) error {
@@ -481,8 +520,8 @@ func batchDetailRow(task *inbound.Task, row *inbound.Row, status int) do.BatchDe
DetailStatus: status,
BlankOutStatus: 1,
RowNo: row.RowNo,
ItemId: row.ICCID,
ErrorReason: row.Reason,
ItemId: batchfields.ItemID(row.ICCID),
ErrorReason: batchfields.DetailReason(row.Reason),
Snapshot: rowSnapshot(row),
TenantId: task.Batch.TenantID,
CreatedDept: task.Batch.CreatedDept,
@@ -526,7 +565,7 @@ func (r *Repository) SaveResultFile(ctx context.Context, task *inbound.Task, req
ExportScene: task.Batch.BatchType,
TotalCount: len(task.Rows),
SuccessCount: successRows(task.Rows),
FailCount: len(task.Rows) - successRows(task.Rows),
FailCount: failedRows(task.Rows),
DownloadStatus: consts.DownloadStatusGenerated,
OssId: ossID,
FileName: artifact.FileName,
@@ -863,3 +902,13 @@ func successRows(rows []*inbound.Row) int {
}
return count
}
func failedRows(rows []*inbound.Row) int {
count := 0
for _, row := range rows {
if row != nil && row.Reason != "" && !inbound.IsCascadeFailure(row) {
count++
}
}
return count
}
internal/repository/inbound/repository_test.go
+83 -3
View File
@@ -1,16 +1,96 @@
package inbound
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
serviceinbound "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/inbound"
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/encoding/gjson"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gcfg"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
serviceinbound "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/inbound"
)
func TestBatchDetailRowClampsInvalidIdentifiersAndReasons(t *testing.T) {
task := &serviceinbound.Task{Batch: serviceinbound.Batch{BatchNo: "INB-TEST", BatchType: 103}}
row := &serviceinbound.Row{RowNo: 2, ICCID: strings.Repeat("9", 100), Reason: strings.Repeat("错", 1000)}
detail := batchDetailRow(task, row, consts.BatchDetailStatusFailed)
if detail.ItemId != "" || len([]rune(detail.ErrorReason.(string))) != batchfields.DetailReasonLimit {
t.Fatalf("invalid detail: itemID=%v, reason=%v", detail.ItemId, detail.ErrorReason)
}
if row.ICCID != strings.Repeat("9", 100) || row.Reason != strings.Repeat("错", 1000) {
t.Fatal("source row was mutated; full value must remain available to result export")
}
}
func TestSaveValidationFailureHandlesOversizedDetailsAndFallback(t *testing.T) {
ctx := context.Background()
oldAdapter := g.Cfg().GetAdapter()
adapter, err := gcfg.NewAdapterContent("batch:\n inbound:\n writeBatchSize: 1000\n")
if err != nil {
t.Fatal(err)
}
g.Cfg().SetAdapter(adapter)
defer g.Cfg().SetAdapter(oldAdapter)
for _, test := range []struct {
name string
constraint string
wantFallback bool
}{
{name: "oversized invalid row is saved", constraint: "CHECK(length(item_id) <= 64 AND length(error_reason) <= 512)"},
{name: "detail write fails", constraint: "CHECK(error_reason = 'allowed')", wantFallback: true},
} {
t.Run(test.name, func(t *testing.T) {
db, err := gdb.New(gdb.ConfigNode{Type: "sqlite", Link: "sqlite::@file(:memory:)", MaxOpenConnCount: 1})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close(ctx) })
statements := []string{
"CREATE TABLE batch_main (batch_no TEXT PRIMARY KEY, batch_type INTEGER, batch_async_status INTEGER, total_count INTEGER, valid_count INTEGER, success_count INTEGER, fail_count INTEGER, run_result INTEGER, fail_reason TEXT, batch_info TEXT, end_time DATETIME, updated_by INTEGER)",
"CREATE TABLE batch_detail (id INTEGER PRIMARY KEY AUTOINCREMENT, batch_no TEXT, batch_type INTEGER, detail_status INTEGER, blank_out_status INTEGER, row_no INTEGER, item_id TEXT, error_reason TEXT " + test.constraint + ", snapshot TEXT, tenant_id TEXT, created_dept INTEGER, created_by INTEGER, updated_by INTEGER)",
"INSERT INTO batch_main (batch_no, batch_type, batch_async_status) VALUES ('INB-TEST', 103, 2)",
}
for _, statement := range statements {
if _, err := db.Exec(ctx, statement); err != nil {
t.Fatalf("fixture failed: %v", err)
}
}
task := &serviceinbound.Task{
Batch: serviceinbound.Batch{BatchNo: "INB-TEST", BatchType: 103},
Rows: []*serviceinbound.Row{{RowNo: 2, ICCID: strings.Repeat("9", 100), Reason: strings.Repeat("错", 1000)}},
FailureReason: "输入格式不正确",
}
if err := (&Repository{db: db}).SaveValidationFailure(ctx, task); err != nil {
t.Fatalf("failed batch must be finalized: %v", err)
}
status, err := db.Model("batch_main").Ctx(ctx).Where("batch_no", "INB-TEST").Value("batch_async_status")
if err != nil || status.Int() != consts.BatchAsyncStatusFailed {
t.Fatalf("status = %v, err = %v", status, err)
}
reason, err := db.Model("batch_main").Ctx(ctx).Where("batch_no", "INB-TEST").Value("fail_reason")
if err != nil || strings.Contains(reason.String(), "错误明细保存失败") != test.wantFallback {
t.Fatalf("fallback reason = %v, err = %v", reason, err)
}
detailCount, err := db.Model("batch_detail").Ctx(ctx).Where("batch_no", "INB-TEST").Count()
wantDetails := 1
if test.wantFallback {
wantDetails = 0
}
if err != nil || detailCount != wantDetails {
t.Fatalf("details = %d, want %d, err = %v", detailCount, wantDetails, err)
}
})
}
}
func TestResolveLocalOssPath(t *testing.T) {
uploadRoot := t.TempDir()
expectedPath := filepath.Join(uploadRoot, "000000", "user", "file", "20260721", "source.xlsx")