sync: 同步 v1.0.4 并脱敏配置与镜像

server/internal/logic/iot/simcardbatch/allocate/processor_test.go

@@ -0,0 +1,80 @@
+package allocate
+
+import (
+	"context"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"github.com/xuri/excelize/v2"
+)
+
+func TestProcessorTemplateHeaders(t *testing.T) {
+	t.Parallel()
+
+	expected := []string{"ICCID", "MSISDN", "IMSI", "目标客户名称"}
+	if !reflect.DeepEqual(NewProcessor().Config().TemplateHeaders, expected) {
+		t.Fatalf("unexpected headers: %#v", NewProcessor().Config().TemplateHeaders)
+	}
+}
+
+func TestParseExcelFileRowsCustomerFacingColumns(t *testing.T) {
+	t.Parallel()
+
+	filePath := filepath.Join(t.TempDir(), "allocate.xlsx")
+	f := excelize.NewFile()
+	headers := []any{"ICCID", "MSISDN", "IMSI", "目标客户名称"}
+	if err := f.SetSheetRow("Sheet1", "A1", &headers); err != nil {
+		t.Fatalf("set headers failed: %v", err)
+	}
+	row := []any{
+		"8986000000000000001",
+		"13800008000",
+		"460001234567890",
+		"示例企业",
+	}
+	if err := f.SetSheetRow("Sheet1", "A2", &row); err != nil {
+		t.Fatalf("set row failed: %v", err)
+	}
+	if err := f.SaveAs(filePath); err != nil {
+		t.Fatalf("save excel failed: %v", err)
+	}
+	_ = f.Close()
+
+	rows, err := ParseExcelFileRows(context.Background(), filePath)
+	if err != nil {
+		t.Fatalf("parse excel failed: %v", err)
+	}
+	if len(rows) != 1 {
+		t.Fatalf("unexpected row count: %d", len(rows))
+	}
+	rowModel, ok := rows[0].(*Row)
+	if !ok || rowModel == nil {
+		t.Fatalf("unexpected row type: %#v", rows[0])
+	}
+	if rowModel.Iccid != "8986000000000000001" {
+		t.Fatalf("unexpected iccid: %s", rowModel.Iccid)
+	}
+	if rowModel.Msisdn != "13800008000" {
+		t.Fatalf("unexpected msisdn: %s", rowModel.Msisdn)
+	}
+	if rowModel.Imsi != "460001234567890" {
+		t.Fatalf("unexpected imsi: %s", rowModel.Imsi)
+	}
+	if rowModel.TargetDeptName != "示例企业" {
+		t.Fatalf("unexpected target dept name: %s", rowModel.TargetDeptName)
+	}
+	if rowModel.Remark != "" {
+		t.Fatalf("unexpected remark: %s", rowModel.Remark)
+	}
+}
+
+func TestProcessorConfigLimitsAllocationRows(t *testing.T) {
+	config := NewProcessor().Config()
+	if config.MaxRows != 10000 {
+		t.Fatalf("MaxRows = %d, want 10000", config.MaxRows)
+	}
+	if config.MaxFileSize != 50<<20 {
+		t.Fatalf("MaxFileSize = %d, want %d", config.MaxFileSize, 50<<20)
+	}
+}

server/internal/logic/iot/simcardbatch/allocate/splitter.go

@@ -102,6 +102,7 @@ func createPendingSplit(
 	if err := createAllocateParent(ctx, req, p, parentBatchNo, importOss, chunkSize); err != nil {
 		return nil, err
 	}
+	children := make([]*batchimport.Result, 0)
 
 	if _, err := stream(ctx, func(rows []batchimport.Row) error {
 		if len(rows) == 0 {
@@ -115,16 +116,20 @@ func createPendingSplit(
 				return err
 			}
 		}
-		if err := markAllocateCrossChunkDuplicates(ctx, parentBatchNo, req.Operator.TenantId, rows); err != nil {
+		priorFailures, err := markAllocateCrossChunkDuplicates(ctx, parentBatchNo, req.Operator.TenantId, rows)
+		if err != nil {
 			return err
 		}
+		state.Valid -= priorFailures
+		state.Failed += priorFailures
 
 		state.ChunkCount++
 		result, err := createAllocateChunk(ctx, req, p, parentBatchNo, state.ChunkCount, rows, importOss, state)
 		if err != nil {
 			return err
 		}
-		return common.PublishBatchMessageStrict(ctx, result, consts.QueueBatchExecute, "allocate")
+		children = append(children, result)
+		return nil
 	}); err != nil {
 		markAllocateParentFailed(ctx, parentBatchNo, err.Error())
 		return nil, err
@@ -138,6 +143,12 @@ func createPendingSplit(
 		markAllocateParentFailed(ctx, parentBatchNo, err.Error())
 		return nil, err
 	}
+	for _, child := range children {
+		if err := common.PublishBatchMessageStrict(ctx, child, consts.QueueBatchExecute, "allocate"); err != nil {
+			markAllocateParentFailed(ctx, parentBatchNo, err.Error())
+			return nil, err
+		}
+	}
 
 	parentMessage := &batchimport.Result{
 		BatchNo:   parentBatchNo,
@@ -397,9 +408,9 @@ func countBatchRows(rows []batchimport.Row) (valid, failed int) {
 	return
 }
 
-func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenantID string, rows []batchimport.Row) error {
+func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenantID string, rows []batchimport.Row) (int, error) {
 	if strings.TrimSpace(parentBatchNo) == "" || len(rows) == 0 {
-		return nil
+		return 0, nil
 	}
 	columns := dao.BatchDetail.Columns()
 	itemIDs := make([]string, 0, len(rows))
@@ -420,24 +431,65 @@ func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenant
 		itemIDs = append(itemIDs, itemID)
 	}
 	if len(itemIDs) == 0 {
-		return nil
+		return 0, nil
 	}
 
 	var existing []struct {
-		ItemId string `orm:"item_id"`
+		Id           int64  `orm:"id"`
+		BatchNo      string `orm:"batch_no"`
+		ItemId       string `orm:"item_id"`
+		DetailStatus int    `orm:"detail_status"`
 	}
 	query := g.DB().Model(dao.BatchDetail.Table()).
-		Fields(columns.ItemId).
+		Fields(columns.Id, columns.BatchNo, columns.ItemId, columns.DetailStatus).
 		Where(columns.BatchType, consts.BatchTypeAllocate).
 		Where(columns.TenantId, tenantID).
 		WhereIn(columns.ItemId, itemIDs).
 		Where("batch_no IN (SELECT batch_no FROM batch_main WHERE source_batch_no = ? AND batch_type = ? AND tenant_id = ?)",
 			parentBatchNo, consts.BatchTypeAllocate, tenantID)
 	if err := query.Scan(&existing); err != nil {
-		return gerror.Wrap(err, "check duplicate allocate ICCIDs across chunks failed")
+		return 0, gerror.Wrap(err, "check duplicate allocate ICCIDs across chunks failed")
 	}
 	if len(existing) == 0 {
+		return 0, nil
+	}
+
+	priorFailures := 0
+	err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
+		for _, detail := range existing {
+			if detail.DetailStatus != consts.BatchDetailStatusPending {
+				continue
+			}
+			result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).
+				Where(columns.Id, detail.Id).
+				Where(columns.TenantId, tenantID).
+				Where(columns.DetailStatus, consts.BatchDetailStatusPending).
+				Data(do.BatchDetail{DetailStatus: consts.BatchDetailStatusFailed, ErrorReason: "ICCID在划拨批次内重复"}).Update()
+			if err != nil {
+				return gerror.Wrap(err, "mark prior duplicate allocation detail failed")
+			}
+			affected, err := result.RowsAffected()
+			if err != nil {
+				return err
+			}
+			if affected == 0 {
+				continue
+			}
+			priorFailures++
+			mainColumns := dao.BatchMain.Columns()
+			for _, batchNo := range []string{detail.BatchNo, parentBatchNo} {
+				if _, err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).
+					Where(mainColumns.BatchNo, batchNo).
+					Where(mainColumns.TenantId, tenantID).
+					Data(do.BatchMain{ValidCount: gdb.Raw("valid_count - 1"), FailCount: gdb.Raw("fail_count + 1")}).Update(); err != nil {
+					return gerror.Wrap(err, "update cross-chunk duplicate counts failed")
+				}
+			}
+		}
 		return nil
+	})
+	if err != nil {
+		return 0, err
 	}
 
 	duplicates := make(map[string]struct{}, len(existing))
@@ -452,10 +504,10 @@ func markAllocateCrossChunkDuplicates(ctx context.Context, parentBatchNo, tenant
 			continue
 		}
 		if _, exists := duplicates[strings.TrimSpace(row.Iccid)]; exists {
-			row.AddErrorReason("ICCID appears in an earlier allocation chunk")
+			row.AddErrorReason("ICCID在划拨批次内重复")
 		}
 	}
-	return nil
+	return priorFailures, nil
 }
 
 func withAllocateInfo(info *gjson.Json, values map[string]any) *gjson.Json {

server/internal/logic/iot/simcardbatch/allocate/splitter_metadata_test.go

@@ -0,0 +1,93 @@
+package allocate
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchimport"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/batchmetadata"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
+
+	"github.com/gogf/gf/contrib/drivers/sqlite/v2"
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+type allocateMetadataSQLiteDriver struct {
+	*sqlite.Driver
+}
+
+func (driver *allocateMetadataSQLiteDriver) New(core *gdb.Core, _ *gdb.ConfigNode) (gdb.DB, error) {
+	return &allocateMetadataSQLiteDriver{Driver: &sqlite.Driver{Core: core}}, nil
+}
+
+func (driver *allocateMetadataSQLiteDriver) DoSelect(ctx context.Context, link gdb.Link, query string, args ...any) (gdb.Result, error) {
+	return driver.Core.DoSelect(ctx, link, strings.TrimSuffix(query, " FOR UPDATE"), args...)
+}
+
+func TestAllocateSplitInheritsRequestMetadata(t *testing.T) {
+	if err := gdb.Register("allocate_metadata_sqlite", &allocateMetadataSQLiteDriver{}); err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "allocate_metadata_sqlite", Name: ":memory:", MaxOpenConnCount: 1}}}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = g.DB().Close(ctx) })
+	if err := g.DB().GetCore().ClearTableFieldsAll(ctx); err != nil {
+		t.Fatal(err)
+	}
+	for table, record := range map[string]any{"batch_main": entity.BatchMain{}, "batch_detail": entity.BatchDetail{}} {
+		recordType := reflect.TypeOf(record)
+		columns := make([]string, 0, recordType.NumField())
+		for fieldIndex := 0; fieldIndex < recordType.NumField(); fieldIndex++ {
+			field := recordType.Field(fieldIndex)
+			column := field.Tag.Get("orm")
+			if column == "id" {
+				columns = append(columns, "id INTEGER PRIMARY KEY AUTOINCREMENT")
+			} else {
+				columns = append(columns, column+" TEXT")
+			}
+		}
+		if _, err := g.DB().Exec(ctx, "CREATE TABLE "+table+" ("+strings.Join(columns, ",")+")"); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := g.Model(table).Ctx(ctx).TableFields(table); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := g.DB().GetCore().GetTablesWithCache(); err != nil {
+		t.Fatal(err)
+	}
+	for _, mode := range []int{batchmetadata.Single, batchmetadata.Bulk} {
+		parentNo := "META-SPLIT-SINGLE"
+		if mode == batchmetadata.Bulk {
+			parentNo = "META-SPLIT-BULK"
+		}
+		req := batchimport.SubmitRequest{SourceChannel: batchmetadata.Console, ExecutionMode: mode, SourceType: consts.BatchSourceTypeBatchInput, Operator: batchimport.OperatorContext{TenantId: "A", UserId: 7}}
+		processor := NewProcessor()
+		if err := createAllocateParent(ctx, req, processor, parentNo, nil, 100); err != nil {
+			t.Fatal(err)
+		}
+		rows := []batchimport.Row{&Row{RowNo: 1, Iccid: "card1"}}
+		result, err := createAllocateChunk(ctx, req, processor, parentNo, 1, rows, nil, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, batchNo := range []string{parentNo, result.BatchNo} {
+			var batch *entity.BatchMain
+			if err := g.Model("batch_main").Ctx(ctx).Where("batch_no", batchNo).Scan(&batch); err != nil {
+				t.Fatal(err)
+			}
+			if batch == nil || batch.SourceChannel != req.SourceChannel || batch.ExecutionMode != uint(mode) || batch.SourceType != req.SourceType || batch.TotalCount != 1 {
+				t.Fatalf("metadata not inherited: %+v", batch)
+			}
+			if batchNo != parentNo && batch.SourceBatchNo != parentNo {
+				t.Fatalf("child lost parent association: %+v", batch)
+			}
+		}
+	}
+}