sync: 同步 v1.0.4 开源代码并脱敏

2026-10-29 14:52:43 +08:00
parent 32e4316ee4
commit 6893e21f07
136 changed files with 9851 additions and 436 deletions
@@ -0,0 +1,232 @@
+package data
+
+import (
+	"context"
+	"database/sql"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/os/gtime"
+
+	mysql "github.com/go-sql-driver/mysql"
+	_ "github.com/gogf/gf/contrib/drivers/mysql/v2"
+	"github.com/gogf/gf/v2/database/gdb"
+	"gopkg.in/yaml.v3"
+)
+
+// 仅显式启用时使用已核验的本机专用测试库；认证始终仅在内存中处理。
+func testSubBatchMySQL(t *testing.T) gdb.DB {
+	t.Helper()
+	if os.Getenv("APN_WORKER_MYSQL_TEST") != "1" {
+		t.Fatal("未显式启用隔离MySQL测试")
+	}
+	_, file, _, _ := runtime.Caller(0)
+	root := filepath.Clean(filepath.Join(filepath.Dir(file), "../../../.."))
+	raw, err := os.ReadFile(filepath.Join(root, "iotlcm-sms/deploy/config.yaml"))
+	if err != nil {
+		t.Fatal("读取本机认证配置失败，未打印凭据")
+	}
+	var config map[string]any
+	if err := yaml.Unmarshal(raw, &config); err != nil {
+		t.Fatal("解析本机认证配置失败，未打印内容")
+	}
+	var dsn string
+	var visit func(map[string]any)
+	visit = func(m map[string]any) {
+		for key, value := range m {
+			if key == "sceneDbDsn" {
+				if text, ok := value.(string); ok {
+					dsn = text
+				}
+			}
+			if nested, ok := value.(map[string]any); ok {
+				visit(nested)
+			}
+		}
+	}
+	visit(config)
+	parsed, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		t.Fatal("本机DSN格式不正确，未打印内容")
+	}
+	if parsed.Net != "tcp" || (parsed.Addr != "127.0.0.1:3306" && parsed.Addr != "localhost:3306") {
+		t.Fatal("认证配置并非允许的本机3306，拒绝连接")
+	}
+	const database = "iotdb_apn_worker_test_20261015"
+	parsed.DBName = database
+	connection, err := sql.Open("mysql", parsed.FormatDSN())
+	if err != nil {
+		t.Fatal("创建隔离连接失败")
+	}
+	defer connection.Close()
+	ctx := context.Background()
+	var uuid, datadir, name, version string
+	var port int
+	if err := connection.QueryRowContext(ctx, "SELECT @@server_uuid,@@datadir,@@port,DATABASE(),VERSION()").Scan(&uuid, &datadir, &port, &name, &version); err != nil {
+		t.Fatal("只读核验本机实例身份失败")
+	}
+	normalized := strings.TrimRight(strings.ReplaceAll(datadir, "\\", "/"), "/")
+	if uuid != "4fa545af-ab36-11f1-be02-040e3c9de361" || port != 3306 || name != database || !strings.EqualFold(normalized, "D:/MySQL/MySQL Server 8.0/Data") {
+		t.Fatal("实例身份不匹配，拒绝任何写入")
+	}
+	t.Logf("隔离MySQL身份：version=%s uuid=%s port=%d database=%s", version, uuid, port, name)
+	db, err := gdb.New(gdb.ConfigNode{Type: "mysql", Host: "127.0.0.1", Port: "3306", User: parsed.User, Pass: parsed.Passwd, Name: database, Charset: "utf8mb4", Timezone: "Asia/Shanghai", Extra: "parseTime=true"})
+	if err != nil {
+		t.Fatal("创建GoFrame隔离连接失败，未打印认证")
+	}
+	// 仅重建此专用库中的固定fixture表，不读取或修改任何业务库。
+	for _, table := range []string{"data_operation_sub_detail", "data_operation_sub_batch", "data_operation_card_claim", "data_operation_parent", "batch_detail", "batch_main", "t_sim_card_info", "sys_oss", "t_resource_batch_download"} {
+		if _, err := db.Exec(ctx, "DROP TABLE IF EXISTS `"+database+"`.`"+table+"`"); err != nil {
+			t.Fatalf("重建专用fixture失败：%s", table)
+		}
+	}
+	migration, err := os.ReadFile(filepath.Join(root, "iotlcm-database/SQL/20261015_data_operation_sub_batch.sql"))
+	if err != nil {
+		t.Fatal("读取worker迁移失败")
+	}
+	for _, statement := range strings.Split(string(migration), ";") {
+		statement = strings.TrimSpace(statement)
+		if statement == "" {
+			continue
+		}
+		if _, err := db.Exec(ctx, statement); err != nil {
+			t.Fatalf("执行专用库worker迁移失败：%v", err)
+		}
+	}
+	return db
+}
+
+func TestSubBatchMySQLMigrationRepeatPreservesAcceptedIntent(t *testing.T) {
+	if os.Getenv("APN_WORKER_MYSQL_TEST") != "1" {
+		t.Skip("仅隔离MySQL验证真实迁移重复执行")
+	}
+	repo, ctx := testSubBatchDB(t)
+	b := testSub("S-1", "migration-accepted", "a", []int64{1}, []string{"fake-A"})
+	if err := repo.PrepareSubBatch(ctx, b); err != nil {
+		t.Fatal(err)
+	}
+	claimed, err := repo.ClaimSubBatch(ctx, b.OperationID, "owner", time.Now())
+	if err != nil || claimed == nil {
+		t.Fatalf("领取: %+v %v", claimed, err)
+	}
+	if err := repo.UpdateSubBatch(ctx, *claimed, "accepted", "preserved-job", "已受理", time.Now(), 2); err != nil {
+		t.Fatal(err)
+	}
+	query, err := repo.ClaimSubBatch(ctx, b.OperationID, "preserved-query-owner", time.Now().Add(time.Second))
+	if err != nil || query == nil {
+		t.Fatalf("领取在途查询租约: %+v %v", query, err)
+	}
+	before, err := repo.database().GetOne(ctx, "SELECT request_data,detail_ids,job_id,poll_count,submitted_at,status,lease_token,lease_until,next_check_at FROM data_operation_sub_batch WHERE operation_id=?", b.OperationID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if before["status"].String() != "accepted" || before["job_id"].String() != "preserved-job" || before["poll_count"].Int() != 3 || before["lease_token"].String() != "preserved-query-owner" || before["lease_until"].IsNil() {
+		t.Fatalf("未实际构造accepted在途租约: %v", before)
+	}
+	_, file, _, _ := runtime.Caller(0)
+	path := filepath.Join(filepath.Dir(file), "../../../../iotlcm-database/SQL/20261015_data_operation_sub_batch.sql")
+	migration, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for repeat := 0; repeat < 2; repeat++ {
+		for _, statement := range strings.Split(string(migration), ";") {
+			if strings.TrimSpace(statement) == "" {
+				continue
+			}
+			if _, err := repo.database().Exec(ctx, statement); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	after, err := repo.database().GetOne(ctx, "SELECT request_data,detail_ids,job_id,poll_count,submitted_at,status,lease_token,lease_until,next_check_at FROM data_operation_sub_batch WHERE operation_id=?", b.OperationID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for key, value := range before {
+		if value.String() != after[key].String() {
+			t.Fatalf("重复迁移改变受理字段: %s", key)
+		}
+	}
+	n, err := repo.database().Model("data_operation_sub_detail").Ctx(ctx).Where("operation_id", b.OperationID).Count()
+	if err != nil || n != 1 {
+		t.Fatalf("重复迁移丢明细归属: %d %v", n, err)
+	}
+	n, err = repo.database().Model(claimTable).Ctx(ctx).Where("active_operation_id", b.OperationID).Count()
+	if err != nil || n != 1 {
+		t.Fatalf("重复迁移丢active: %d %v", n, err)
+	}
+}
+
+func TestSubBatchMySQLDifferentDatabaseDefaultCollation(t *testing.T) {
+	if os.Getenv("APN_WORKER_MYSQL_TEST") != "1" {
+		t.Skip("仅隔离MySQL验证不同默认排序规则")
+	}
+	repo, ctx := testSubBatchDB(t)
+	value, err := repo.database().GetValue(ctx, "SELECT DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME=DATABASE()")
+	if err != nil || value.String() != "utf8mb4_0900_ai_ci" {
+		t.Fatalf("未实际覆盖不同DB默认排序规则: %v %v", value, err)
+	}
+	rows, err := repo.database().GetAll(ctx, "SELECT TABLE_NAME,COLLATION_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND COLUMN_NAME='batch_no' AND TABLE_NAME IN ('batch_main','batch_detail','data_operation_parent','data_operation_sub_batch','data_operation_sub_detail')")
+	if err != nil || len(rows) != 5 {
+		t.Fatalf("关联列元数据: %v %v", rows, err)
+	}
+	for _, row := range rows {
+		if row["COLLATION_NAME"].String() != "utf8mb4_general_ci" {
+			t.Fatalf("关联列未匹配旧基线: %v", rows)
+		}
+	}
+	b := testSub("S-1", "collation-real-join", "a", []int64{1}, []string{"fake-A"})
+	if err := repo.PrepareSubBatch(ctx, b); err != nil {
+		t.Fatal(err)
+	}
+	if ids, err := repo.ListDueSubBatches(ctx, time.Now(), 128); err != nil || len(ids) != 1 {
+		t.Fatalf("跨新旧表due查询: %v %v", ids, err)
+	}
+	if ids, err := repo.ListRecoverableBatches(ctx, 128); err != nil {
+		t.Fatalf("跨新旧表恢复查询: %v %v", ids, err)
+	}
+	if got, err := repo.ClaimSubBatch(ctx, b.OperationID, "collation-owner", time.Now()); err != nil || got == nil {
+		t.Fatalf("跨新旧表前序锁定JOIN: %+v %v", got, err)
+	}
+	indexRows, err := repo.database().GetAll(ctx, "EXPLAIN SELECT STRAIGHT_JOIN s.operation_id FROM batch_main m FORCE INDEX(PRIMARY) JOIN data_operation_sub_detail sd ON sd.batch_no=m.batch_no AND sd.iccid='fake-A' JOIN data_operation_sub_batch s ON s.operation_id=sd.operation_id JOIN batch_detail d ON d.id=sd.detail_id WHERE m.tenant_id='tenant' AND m.id<11 AND d.detail_status=1 AND s.status IN ('prepared','submitting','accepted','uncertain') LIMIT 1 FOR UPDATE OF sd,s,d")
+	if err != nil || len(indexRows) < 2 || indexRows[0]["table"].String() != "m" || indexRows[1]["table"].String() != "sd" || indexRows[1]["key"].String() != "uk_data_sub_batch_card" {
+		t.Fatalf("前序锁定查询未按父批→同卡映射驱动: %v %v", indexRows, err)
+	}
+	t.Logf("前序EXPLAIN首表=%s key=%s；第二表=%s key=%s", indexRows[0]["table"].String(), indexRows[0]["key"].String(), indexRows[1]["table"].String(), indexRows[1]["key"].String())
+	t.Logf("DB默认=%s，五张新旧表batch_no均general_ci，真实due/recovery/claim JOIN成功", value.String())
+}
+
+func logMySQLLease(t *testing.T, repo *Repository, batchID int64, token string) {
+	t.Helper()
+	if os.Getenv("APN_WORKER_MYSQL_TEST") != "1" {
+		return
+	}
+	ctx := context.Background()
+	raw, err := repo.database().GetOne(ctx, "SELECT CAST(lease_until AS CHAR) AS raw_lease, CAST(NOW(6) AS CHAR) AS db_now, CAST(UTC_TIMESTAMP(6) AS CHAR) AS db_utc, @@session.time_zone AS session_zone FROM data_operation_sub_batch WHERE id=?", batchID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var row subRow
+	if err := repo.database().Model(subTable).Ctx(ctx).Where("id", batchID).Scan(&row); err != nil {
+		t.Fatal(err)
+	}
+	var alternate struct {
+		LeaseUntil *gtime.Time `orm:"lease_until"`
+	}
+	if err := repo.database().Model(subTable).Ctx(ctx).Where("id", batchID).Scan(&alternate); err != nil {
+		t.Fatal(err)
+	}
+	now := time.Now()
+	t.Logf("租约原值 raw=%s dbNow=%s dbUTC=%s sessionZone=%s tokenMatch=%t status=%s nil=%t", raw["raw_lease"].String(), raw["db_now"].String(), raw["db_utc"].String(), raw["session_zone"].String(), row.LeaseToken == token, row.Status, row.LeaseUntil == nil)
+	if row.LeaseUntil != nil {
+		t.Logf("time扫描=%s location=%s unixNano=%d; now=%s location=%s unixNano=%d", row.LeaseUntil.Format(time.RFC3339Nano), row.LeaseUntil.Location(), row.LeaseUntil.UnixNano(), now.Format(time.RFC3339Nano), now.Location(), now.UnixNano())
+	}
+	if alternate.LeaseUntil != nil {
+		t.Logf("gtime扫描=%s location=%s unixNano=%d", alternate.LeaseUntil.Format("c"), alternate.LeaseUntil.Time.Location(), alternate.LeaseUntil.UnixNano())
+	}
+}