|
|
|
@@ -0,0 +1,247 @@
|
|
|
|
|
package rabbitmq
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"errors"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
|
|
|
|
"github.com/gogf/gf/v2/database/gdb"
|
|
|
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
|
|
|
|
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
|
|
|
|
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
|
|
|
|
batchrepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/batch"
|
|
|
|
|
datarepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/data"
|
|
|
|
|
dataservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type durableBatchRepo struct{ failed int }
|
|
|
|
|
|
|
|
|
|
func (*durableBatchRepo) FindMain(context.Context, string) (*entity.BatchMain, error) {
|
|
|
|
|
return &entity.BatchMain{BatchNo: "APN-RECOVER", BatchType: 109, BatchAsyncStatus: batch.AsyncStatusPending}, nil
|
|
|
|
|
}
|
|
|
|
|
func (*durableBatchRepo) MarkProcessing(context.Context, string, int) (bool, error) { return true, nil }
|
|
|
|
|
func (r *durableBatchRepo) MarkFailed(context.Context, string, int, string) (bool, error) {
|
|
|
|
|
r.failed++
|
|
|
|
|
return true, nil
|
|
|
|
|
}
|
|
|
|
|
func (*durableBatchRepo) MarkSuccess(context.Context, string, int, int, int, string) (bool, error) {
|
|
|
|
|
return true, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type durableBatchExecutor struct {
|
|
|
|
|
calls, protections int
|
|
|
|
|
retryable bool
|
|
|
|
|
protectionErr error
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *durableBatchExecutor) Execute(context.Context, string) error {
|
|
|
|
|
e.calls++
|
|
|
|
|
if e.retryable {
|
|
|
|
|
return batch.Retryable(errors.New("准备中断"))
|
|
|
|
|
}
|
|
|
|
|
return errors.New("本地执行中断")
|
|
|
|
|
}
|
|
|
|
|
func (e *durableBatchExecutor) ProtectExecutionFailure(ctx context.Context, _ string, _ int, _ string) (bool, error) {
|
|
|
|
|
e.protections++
|
|
|
|
|
if ctx.Err() != nil {
|
|
|
|
|
return true, ctx.Err()
|
|
|
|
|
}
|
|
|
|
|
return true, e.protectionErr
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestDurableAPNConsumerNeverFallsThroughToGenericFailure(t *testing.T) {
|
|
|
|
|
repo := &durableBatchRepo{}
|
|
|
|
|
executor := &durableBatchExecutor{}
|
|
|
|
|
dispatcher := batch.NewDispatcher(nil)
|
|
|
|
|
dispatcher.Register(109, executor)
|
|
|
|
|
consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher)}
|
|
|
|
|
if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if executor.calls != 1 || executor.protections != 1 || repo.failed != 0 {
|
|
|
|
|
t.Fatalf("消费者普通错误越过durable保护: calls=%d protections=%d failed=%d", executor.calls, executor.protections, repo.failed)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestDurableAPNConsumerRetryExhaustionProtectsBeforePublish(t *testing.T) {
|
|
|
|
|
repo := &durableBatchRepo{}
|
|
|
|
|
dbDown := errors.New("模拟保护数据库不可用")
|
|
|
|
|
executor := &durableBatchExecutor{retryable: true, protectionErr: dbDown}
|
|
|
|
|
dispatcher := batch.NewDispatcher(nil)
|
|
|
|
|
dispatcher.Register(109, executor)
|
|
|
|
|
consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher), batchExecuteMaxRetries: 0}
|
|
|
|
|
ctx := mq.WithDeliveryMetadata(context.Background(), mq.DeliveryMetadata{Headers: map[string]any{batchRetryCountHeader: 3}})
|
|
|
|
|
err := consumers.handleBatchExecute(ctx, batch.Message{BatchNo: "APN-RECOVER", BatchType: 109})
|
|
|
|
|
// 保护失败必须在消息发布前返回，测试不会连接真实RabbitMQ。
|
|
|
|
|
if !errors.Is(err, dbDown) || executor.calls != 1 || executor.protections != 1 || repo.failed != 0 {
|
|
|
|
|
t.Fatalf("重试耗尽错误终结或重发: err=%v calls=%d protections=%d failed=%d", err, executor.calls, executor.protections, repo.failed)
|
|
|
|
|
}
|
|
|
|
|
executor.protectionErr = nil
|
|
|
|
|
if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109, FailureReason: "重试耗尽"}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if executor.calls != 1 || executor.protections != 2 || repo.failed != 0 {
|
|
|
|
|
t.Fatal("失败重投重新执行或MarkFailed")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// reentryWriteCounter 同时覆盖同步写、异步提交与结果查询，重入准备期间均不得调用。
|
|
|
|
|
type reentryWriteCounter struct{ syncWrites, asyncWrites, queries int }
|
|
|
|
|
|
|
|
|
|
func (u *reentryWriteCounter) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
|
|
|
|
u.syncWrites++
|
|
|
|
|
return nil, nil
|
|
|
|
|
}
|
|
|
|
|
func (u *reentryWriteCounter) SubmitSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
|
|
|
|
u.asyncWrites++
|
|
|
|
|
return nil, nil
|
|
|
|
|
}
|
|
|
|
|
func (u *reentryWriteCounter) QuerySimApnFunctionResult(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
|
|
|
|
u.queries++
|
|
|
|
|
return nil, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestAPNProcessingRabbitMQReentryRequiresDurableParent(t *testing.T) {
|
|
|
|
|
ctx := context.Background()
|
|
|
|
|
if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "sqlite", Link: "sqlite::@file(" + filepath.ToSlash(filepath.Join(t.TempDir(), "apn-reentry.db")) + ")"}}}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
db := g.DB()
|
|
|
|
|
t.Cleanup(func() {
|
|
|
|
|
if err := db.Close(ctx); err != nil {
|
|
|
|
|
t.Error(err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
for _, sql := range []string{
|
|
|
|
|
"CREATE TABLE batch_main (id INTEGER PRIMARY KEY,batch_no TEXT UNIQUE,batch_type INTEGER,batch_async_status INTEGER,tenant_id TEXT,start_time DATETIME,batch_info TEXT,run_result INTEGER DEFAULT 0,valid_count INTEGER DEFAULT 0,success_count INTEGER DEFAULT 0,fail_count INTEGER DEFAULT 0,fail_reason TEXT,end_time DATETIME,updated_at DATETIME,created_by INTEGER,created_dept INTEGER,updated_by INTEGER)",
|
|
|
|
|
"CREATE TABLE batch_detail (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,row_no INTEGER,item_id TEXT,detail_status INTEGER,error_reason TEXT,snapshot TEXT,updated_by INTEGER)",
|
|
|
|
|
"CREATE TABLE t_sim_card_info (id INTEGER PRIMARY KEY AUTOINCREMENT,tenant_id TEXT,iccid TEXT,msisdn TEXT,imsi TEXT,chan_key TEXT,data_service_status INTEGER)",
|
|
|
|
|
"CREATE TABLE data_operation_parent (batch_no TEXT PRIMARY KEY,ready INTEGER DEFAULT 0,finalize_token TEXT,finalize_until DATETIME,recovery_attempt_at DATETIME,last_reason TEXT)",
|
|
|
|
|
"CREATE TABLE data_operation_sub_batch (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,chan_key TEXT,operation_id TEXT UNIQUE,detail_ids TEXT,request_data TEXT,job_id TEXT,status TEXT,poll_count INTEGER DEFAULT 0,next_check_at DATETIME,submitted_at DATETIME,lease_token TEXT,lease_until DATETIME,created_at DATETIME DEFAULT CURRENT_TIMESTAMP,last_reason TEXT)",
|
|
|
|
|
"CREATE TABLE data_operation_sub_detail (detail_id INTEGER PRIMARY KEY,operation_id TEXT,batch_no TEXT,iccid TEXT,UNIQUE(batch_no,iccid))",
|
|
|
|
|
"CREATE TABLE data_operation_card_claim (tenant_id TEXT,iccid TEXT,batch_id INTEGER,operation_id TEXT,active_batch_id INTEGER DEFAULT 0,active_operation_id TEXT DEFAULT '',PRIMARY KEY(tenant_id,iccid))",
|
|
|
|
|
} {
|
|
|
|
|
if _, err := db.Exec(ctx, sql); err != nil {
|
|
|
|
|
t.Fatalf("初始化本机 SQLite fixture: %v", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for _, tc := range []struct {
|
|
|
|
|
name string
|
|
|
|
|
old, retryable bool
|
|
|
|
|
}{
|
|
|
|
|
{name: "legacy-ordinary", old: true},
|
|
|
|
|
{name: "legacy-retryable", old: true, retryable: true},
|
|
|
|
|
{name: "durable-ordinary"},
|
|
|
|
|
{name: "durable-retryable", retryable: true},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
|
|
|
status := batch.AsyncStatusPending
|
|
|
|
|
if tc.old {
|
|
|
|
|
status = batch.AsyncStatusProcessing
|
|
|
|
|
}
|
|
|
|
|
// 使用 NULL 旧首发时间以确保 ReclaimProcessing 的 WHERE start_time IS NULL 可以命中。
|
|
|
|
|
if _, err := db.Exec(ctx, "INSERT INTO batch_main(batch_no,batch_type,batch_async_status,tenant_id) VALUES (?,109,?,'tenant')", tc.name, status); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
for i := 0; i < 10; i++ {
|
|
|
|
|
iccid := tc.name + "-card-" + string(rune('0'+i))
|
|
|
|
|
detailStatus := batch.DetailStatusPending
|
|
|
|
|
if tc.old {
|
|
|
|
|
detailStatus = batch.DetailStatusRunning
|
|
|
|
|
}
|
|
|
|
|
if _, err := db.Exec(ctx, "INSERT INTO batch_detail(batch_no,batch_type,row_no,item_id,detail_status,snapshot) VALUES (?,109,?,?,?,?)", tc.name, i+1, iccid, detailStatus, `{"authorized":true}`); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if _, err := db.Exec(ctx, "INSERT INTO t_sim_card_info(tenant_id,iccid,msisdn,imsi,chan_key,data_service_status) VALUES ('tenant',?,?,?,'mock-apn',0)", iccid, "130000000"+string(rune('0'+i)), "460000000"+string(rune('0'+i))); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
mainRepo := batchrepo.New()
|
|
|
|
|
if tc.old && tc.retryable {
|
|
|
|
|
// 先证实同一 fixture 的旧批确实可被重领，再复位空首发时间。
|
|
|
|
|
claimed, err := mainRepo.ReclaimProcessing(ctx, tc.name, 109, nil)
|
|
|
|
|
if err != nil || !claimed {
|
|
|
|
|
t.Fatalf("重领 CAS 必须可命中，避免假阳性: claimed=%t err=%v", claimed, err)
|
|
|
|
|
}
|
|
|
|
|
if _, err := db.Exec(ctx, "UPDATE batch_main SET start_time=NULL WHERE batch_no=?", tc.name); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if !tc.old {
|
|
|
|
|
claimed, err := mainRepo.MarkProcessing(ctx, tc.name, 109)
|
|
|
|
|
if err != nil || !claimed {
|
|
|
|
|
t.Fatalf("新批需同事务领取 Processing 与 ready=0 父凭据: claimed=%t err=%v", claimed, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
parent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=0", tc.name)
|
|
|
|
|
if err != nil || parent.Int() != 1 && !tc.old || parent.Int() != 0 && tc.old {
|
|
|
|
|
t.Fatalf("前置凭据不符: old=%t ready0=%d err=%v", tc.old, parent.Int(), err)
|
|
|
|
|
}
|
|
|
|
|
before, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
upstream := &reentryWriteCounter{}
|
|
|
|
|
dispatcher := batch.NewDispatcher(nil)
|
|
|
|
|
dispatcher.Register(109, dataservice.NewExecutor(datarepo.New(), upstream))
|
|
|
|
|
consumer := &Consumers{batch: batch.NewRuntime(mainRepo, dispatcher)}
|
|
|
|
|
err = consumer.handleBatchExecute(ctx, batch.Message{BatchNo: tc.name, BatchType: 109, Retryable: tc.retryable})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Errorf("重入应安全跳过旧批或准备新批，不能进入消息重发路径: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if upstream.syncWrites != 0 || upstream.asyncWrites != 0 || upstream.queries != 0 {
|
|
|
|
|
t.Errorf("重入意外调用上游: sync=%d submit=%d query=%d", upstream.syncWrites, upstream.asyncWrites, upstream.queries)
|
|
|
|
|
}
|
|
|
|
|
main, err := mainRepo.FindMain(ctx, tc.name)
|
|
|
|
|
if err != nil || main == nil || main.BatchAsyncStatus != batch.AsyncStatusProcessing {
|
|
|
|
|
t.Fatalf("批状态被意外终结: main=%+v err=%v", main, err)
|
|
|
|
|
}
|
|
|
|
|
remaining, err := db.GetValue(ctx, "SELECT COUNT(*) FROM batch_detail WHERE batch_no=? AND detail_status=1", tc.name)
|
|
|
|
|
if err != nil || remaining.Int() != 10 {
|
|
|
|
|
t.Fatalf("运行中明细不能被更改: count=%d err=%v", remaining.Int(), err)
|
|
|
|
|
}
|
|
|
|
|
intent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_batch WHERE batch_no=? AND status='prepared'", tc.name)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if tc.old {
|
|
|
|
|
if intent.Int() != 0 {
|
|
|
|
|
t.Errorf("无凭据历史批不应产生新提交意图: %d", intent.Int())
|
|
|
|
|
}
|
|
|
|
|
assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name)
|
|
|
|
|
if err != nil || assigned.Int() != 0 {
|
|
|
|
|
t.Errorf("历史批不可冻结明细: count=%d err=%v", assigned.Int(), err)
|
|
|
|
|
}
|
|
|
|
|
claims, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_card_claim WHERE operation_id!='' AND iccid LIKE ?", tc.name+"-%")
|
|
|
|
|
if err != nil || claims.Int() != 0 {
|
|
|
|
|
t.Errorf("历史批不可建立卡级提交占位: count=%d err=%v", claims.Int(), err)
|
|
|
|
|
}
|
|
|
|
|
parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=?", tc.name)
|
|
|
|
|
if err != nil || parent.Int() != 0 {
|
|
|
|
|
t.Errorf("历史批不可补造父凭据: count=%d err=%v", parent.Int(), err)
|
|
|
|
|
}
|
|
|
|
|
after, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name)
|
|
|
|
|
if err != nil || after.String() != before.String() {
|
|
|
|
|
t.Errorf("旧批重入不可重领取: before=%s after=%s err=%v", before.String(), after.String(), err)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if intent.Int() != 1 {
|
|
|
|
|
t.Errorf("新批 ready=0 未正确恢复成 10 卡一个冻结意图: %d", intent.Int())
|
|
|
|
|
}
|
|
|
|
|
parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=1", tc.name)
|
|
|
|
|
if err != nil || parent.Int() != 1 {
|
|
|
|
|
t.Errorf("新批冻结意图与 ready=1 未原子提交: count=%d err=%v", parent.Int(), err)
|
|
|
|
|
}
|
|
|
|
|
assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name)
|
|
|
|
|
if err != nil || assigned.Int() != 10 {
|
|
|
|
|
t.Errorf("新批冻结明细必须覆盖全部 10 卡: count=%d err=%v", assigned.Int(), err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
MiFaZhan