package rabbitmq import ( "context" "errors" "path/filepath" "testing" _ "github.com/gogf/gf/contrib/drivers/sqlite/v2" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch" mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity" batchrepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/batch" datarepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/data" dataservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data" ) type durableBatchRepo struct{ failed int } func (*durableBatchRepo) FindMain(context.Context, string) (*entity.BatchMain, error) { return &entity.BatchMain{BatchNo: "APN-RECOVER", BatchType: 109, BatchAsyncStatus: batch.AsyncStatusPending}, nil } func (*durableBatchRepo) MarkProcessing(context.Context, string, int) (bool, error) { return true, nil } func (r *durableBatchRepo) MarkFailed(context.Context, string, int, string) (bool, error) { r.failed++ return true, nil } func (*durableBatchRepo) MarkSuccess(context.Context, string, int, int, int, string) (bool, error) { return true, nil } type durableBatchExecutor struct { calls, protections int retryable bool protectionErr error } func (e *durableBatchExecutor) Execute(context.Context, string) error { e.calls++ if e.retryable { return batch.Retryable(errors.New("准备中断")) } return errors.New("本地执行中断") } func (e *durableBatchExecutor) ProtectExecutionFailure(ctx context.Context, _ string, _ int, _ string) (bool, error) { e.protections++ if ctx.Err() != nil { return true, ctx.Err() } return true, e.protectionErr } func TestDurableAPNConsumerNeverFallsThroughToGenericFailure(t *testing.T) { repo := &durableBatchRepo{} executor := &durableBatchExecutor{} dispatcher := batch.NewDispatcher(nil) dispatcher.Register(109, executor) consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher)} if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109}); err != nil { t.Fatal(err) } if executor.calls != 1 || executor.protections != 1 || repo.failed != 0 { t.Fatalf("消费者普通错误越过durable保护: calls=%d protections=%d failed=%d", executor.calls, executor.protections, repo.failed) } } func TestDurableAPNConsumerRetryExhaustionProtectsBeforePublish(t *testing.T) { repo := &durableBatchRepo{} dbDown := errors.New("模拟保护数据库不可用") executor := &durableBatchExecutor{retryable: true, protectionErr: dbDown} dispatcher := batch.NewDispatcher(nil) dispatcher.Register(109, executor) consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher), batchExecuteMaxRetries: 0} ctx := mq.WithDeliveryMetadata(context.Background(), mq.DeliveryMetadata{Headers: map[string]any{batchRetryCountHeader: 3}}) err := consumers.handleBatchExecute(ctx, batch.Message{BatchNo: "APN-RECOVER", BatchType: 109}) // 保护失败必须在消息发布前返回,测试不会连接真实RabbitMQ。 if !errors.Is(err, dbDown) || executor.calls != 1 || executor.protections != 1 || repo.failed != 0 { t.Fatalf("重试耗尽错误终结或重发: err=%v calls=%d protections=%d failed=%d", err, executor.calls, executor.protections, repo.failed) } executor.protectionErr = nil if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109, FailureReason: "重试耗尽"}); err != nil { t.Fatal(err) } if executor.calls != 1 || executor.protections != 2 || repo.failed != 0 { t.Fatal("失败重投重新执行或MarkFailed") } } // reentryWriteCounter 同时覆盖同步写、异步提交与结果查询,重入准备期间均不得调用。 type reentryWriteCounter struct{ syncWrites, asyncWrites, queries int } func (u *reentryWriteCounter) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) { u.syncWrites++ return nil, nil } func (u *reentryWriteCounter) SubmitSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) { u.asyncWrites++ return nil, nil } func (u *reentryWriteCounter) QuerySimApnFunctionResult(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) { u.queries++ return nil, nil } func TestAPNProcessingRabbitMQReentryRequiresDurableParent(t *testing.T) { ctx := context.Background() if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "sqlite", Link: "sqlite::@file(" + filepath.ToSlash(filepath.Join(t.TempDir(), "apn-reentry.db")) + ")"}}}); err != nil { t.Fatal(err) } db := g.DB() t.Cleanup(func() { if err := db.Close(ctx); err != nil { t.Error(err) } }) for _, sql := range []string{ "CREATE TABLE batch_main (id INTEGER PRIMARY KEY,batch_no TEXT UNIQUE,batch_type INTEGER,batch_async_status INTEGER,tenant_id TEXT,start_time DATETIME,batch_info TEXT,run_result INTEGER DEFAULT 0,valid_count INTEGER DEFAULT 0,success_count INTEGER DEFAULT 0,fail_count INTEGER DEFAULT 0,fail_reason TEXT,end_time DATETIME,updated_at DATETIME,created_by INTEGER,created_dept INTEGER,updated_by INTEGER)", "CREATE TABLE batch_detail (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,row_no INTEGER,item_id TEXT,detail_status INTEGER,error_reason TEXT,snapshot TEXT,updated_by INTEGER)", "CREATE TABLE t_sim_card_info (id INTEGER PRIMARY KEY AUTOINCREMENT,tenant_id TEXT,iccid TEXT,msisdn TEXT,imsi TEXT,chan_key TEXT,data_service_status INTEGER)", "CREATE TABLE data_operation_parent (batch_no TEXT PRIMARY KEY,ready INTEGER DEFAULT 0,finalize_token TEXT,finalize_until DATETIME,recovery_attempt_at DATETIME,last_reason TEXT)", "CREATE TABLE data_operation_sub_batch (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,chan_key TEXT,operation_id TEXT UNIQUE,detail_ids TEXT,request_data TEXT,job_id TEXT,status TEXT,poll_count INTEGER DEFAULT 0,next_check_at DATETIME,submitted_at DATETIME,lease_token TEXT,lease_until DATETIME,created_at DATETIME DEFAULT CURRENT_TIMESTAMP,last_reason TEXT)", "CREATE TABLE data_operation_sub_detail (detail_id INTEGER PRIMARY KEY,operation_id TEXT,batch_no TEXT,iccid TEXT,UNIQUE(batch_no,iccid))", "CREATE TABLE data_operation_card_claim (tenant_id TEXT,iccid TEXT,batch_id INTEGER,operation_id TEXT,active_batch_id INTEGER DEFAULT 0,active_operation_id TEXT DEFAULT '',PRIMARY KEY(tenant_id,iccid))", } { if _, err := db.Exec(ctx, sql); err != nil { t.Fatalf("初始化本机 SQLite fixture: %v", err) } } for _, tc := range []struct { name string old, retryable bool }{ {name: "legacy-ordinary", old: true}, {name: "legacy-retryable", old: true, retryable: true}, {name: "durable-ordinary"}, {name: "durable-retryable", retryable: true}, } { t.Run(tc.name, func(t *testing.T) { status := batch.AsyncStatusPending if tc.old { status = batch.AsyncStatusProcessing } // 使用 NULL 旧首发时间以确保 ReclaimProcessing 的 WHERE start_time IS NULL 可以命中。 if _, err := db.Exec(ctx, "INSERT INTO batch_main(batch_no,batch_type,batch_async_status,tenant_id) VALUES (?,109,?,'tenant')", tc.name, status); err != nil { t.Fatal(err) } for i := 0; i < 10; i++ { iccid := tc.name + "-card-" + string(rune('0'+i)) detailStatus := batch.DetailStatusPending if tc.old { detailStatus = batch.DetailStatusRunning } if _, err := db.Exec(ctx, "INSERT INTO batch_detail(batch_no,batch_type,row_no,item_id,detail_status,snapshot) VALUES (?,109,?,?,?,?)", tc.name, i+1, iccid, detailStatus, `{"authorized":true}`); err != nil { t.Fatal(err) } if _, err := db.Exec(ctx, "INSERT INTO t_sim_card_info(tenant_id,iccid,msisdn,imsi,chan_key,data_service_status) VALUES ('tenant',?,?,?,'mock-apn',0)", iccid, "130000000"+string(rune('0'+i)), "460000000"+string(rune('0'+i))); err != nil { t.Fatal(err) } } mainRepo := batchrepo.New() if tc.old && tc.retryable { // 先证实同一 fixture 的旧批确实可被重领,再复位空首发时间。 claimed, err := mainRepo.ReclaimProcessing(ctx, tc.name, 109, nil) if err != nil || !claimed { t.Fatalf("重领 CAS 必须可命中,避免假阳性: claimed=%t err=%v", claimed, err) } if _, err := db.Exec(ctx, "UPDATE batch_main SET start_time=NULL WHERE batch_no=?", tc.name); err != nil { t.Fatal(err) } } if !tc.old { claimed, err := mainRepo.MarkProcessing(ctx, tc.name, 109) if err != nil || !claimed { t.Fatalf("新批需同事务领取 Processing 与 ready=0 父凭据: claimed=%t err=%v", claimed, err) } } parent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=0", tc.name) if err != nil || parent.Int() != 1 && !tc.old || parent.Int() != 0 && tc.old { t.Fatalf("前置凭据不符: old=%t ready0=%d err=%v", tc.old, parent.Int(), err) } before, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name) if err != nil { t.Fatal(err) } upstream := &reentryWriteCounter{} dispatcher := batch.NewDispatcher(nil) dispatcher.Register(109, dataservice.NewExecutor(datarepo.New(), upstream)) consumer := &Consumers{batch: batch.NewRuntime(mainRepo, dispatcher)} err = consumer.handleBatchExecute(ctx, batch.Message{BatchNo: tc.name, BatchType: 109, Retryable: tc.retryable}) if err != nil { t.Errorf("重入应安全跳过旧批或准备新批,不能进入消息重发路径: %v", err) } if upstream.syncWrites != 0 || upstream.asyncWrites != 0 || upstream.queries != 0 { t.Errorf("重入意外调用上游: sync=%d submit=%d query=%d", upstream.syncWrites, upstream.asyncWrites, upstream.queries) } main, err := mainRepo.FindMain(ctx, tc.name) if err != nil || main == nil || main.BatchAsyncStatus != batch.AsyncStatusProcessing { t.Fatalf("批状态被意外终结: main=%+v err=%v", main, err) } remaining, err := db.GetValue(ctx, "SELECT COUNT(*) FROM batch_detail WHERE batch_no=? AND detail_status=1", tc.name) if err != nil || remaining.Int() != 10 { t.Fatalf("运行中明细不能被更改: count=%d err=%v", remaining.Int(), err) } intent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_batch WHERE batch_no=? AND status='prepared'", tc.name) if err != nil { t.Fatal(err) } if tc.old { if intent.Int() != 0 { t.Errorf("无凭据历史批不应产生新提交意图: %d", intent.Int()) } assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name) if err != nil || assigned.Int() != 0 { t.Errorf("历史批不可冻结明细: count=%d err=%v", assigned.Int(), err) } claims, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_card_claim WHERE operation_id!='' AND iccid LIKE ?", tc.name+"-%") if err != nil || claims.Int() != 0 { t.Errorf("历史批不可建立卡级提交占位: count=%d err=%v", claims.Int(), err) } parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=?", tc.name) if err != nil || parent.Int() != 0 { t.Errorf("历史批不可补造父凭据: count=%d err=%v", parent.Int(), err) } after, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name) if err != nil || after.String() != before.String() { t.Errorf("旧批重入不可重领取: before=%s after=%s err=%v", before.String(), after.String(), err) } } else { if intent.Int() != 1 { t.Errorf("新批 ready=0 未正确恢复成 10 卡一个冻结意图: %d", intent.Int()) } parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=1", tc.name) if err != nil || parent.Int() != 1 { t.Errorf("新批冻结意图与 ready=1 未原子提交: count=%d err=%v", parent.Int(), err) } assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name) if err != nil || assigned.Int() != 10 { t.Errorf("新批冻结明细必须覆盖全部 10 卡: count=%d err=%v", assigned.Int(), err) } } }) } }