chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）

2026-11-02 13:28:03 +08:00
parent 6de2e503cc 96f0f4eacd
commit 2923c78755
422 changed files with 80993 additions and 0 deletions
@@ -0,0 +1,650 @@
+package data
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gcfg"
+	upstreamclient "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/client/upstream"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+)
+
+func TestDurableAPNResultScheduleIsThreeMinutesAndThreeRetries(t *testing.T) {
+	if maxResultQueries != 4 {
+		t.Fatalf("首次加三次重试应为四次查询: got=%d", maxResultQueries)
+	}
+	base := time.Date(2026, 10, 15, 8, 0, 0, 0, time.UTC)
+	for i, minute := range []int{3, 6, 9, 12} {
+		got := nextResultCheck(SubBatch{SubmittedAt: base, PollCount: i})
+		want := base.Add(time.Duration(minute) * time.Minute)
+		if !got.Equal(want) {
+			t.Fatalf("第%d次查询: got=%v want=%v", i+1, got, want)
+		}
+	}
+}
+
+type durableFake struct {
+	mu                   sync.Mutex
+	batches              map[string]SubBatch
+	next                 map[string]time.Time
+	outcomes             map[string][]Outcome
+	preparedBeforeSubmit int
+}
+
+func newDurableFake() *durableFake {
+	return &durableFake{batches: map[string]SubBatch{}, next: map[string]time.Time{}, outcomes: map[string][]Outcome{}}
+}
+func (f *durableFake) HasDurableParent(context.Context, string) (bool, error) { return true, nil }
+func (f *durableFake) PrepareSubBatch(_ context.Context, b SubBatch) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if _, ok := f.batches[b.OperationID]; !ok {
+		b.ID = int64(len(f.batches) + 1)
+		f.batches[b.OperationID] = b
+	}
+	return nil
+}
+func (f *durableFake) ResumeOrPrepare(ctx context.Context, _ string, p func(context.Context) error) error {
+	return p(ctx)
+}
+func (f *durableFake) GetSubBatch(_ context.Context, id string) (*SubBatch, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	b := f.batches[id]
+	b.NextCheckAt = f.next[id]
+	return &b, nil
+}
+func (f *durableFake) DelaySubBatchPrecheck(_ context.Context, snapshot SubBatch, next time.Time, _ string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	b := f.batches[snapshot.OperationID]
+	if b.Status == snapshot.Status && b.LeaseToken == snapshot.LeaseToken && f.next[snapshot.OperationID].Equal(snapshot.NextCheckAt) {
+		f.next[snapshot.OperationID] = next
+	}
+	return nil
+}
+func (f *durableFake) ReleaseUnsentSubBatch(_ context.Context, b SubBatch, _ string) error {
+	old := f.batches[b.OperationID]
+	old.Status = b.Status
+	old.LeaseToken = ""
+	if b.Status == "prepared" {
+		old.SubmittedAt = time.Time{}
+	} else {
+		old.PollCount--
+	}
+	f.batches[b.OperationID] = old
+	return nil
+}
+func (f *durableFake) SubBatchChannel(_ context.Context, id string) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.batches[id].ChanKey, nil
+}
+func (f *durableFake) ProtectExecutionFailure(context.Context, string, string) error { return nil }
+func (f *durableFake) SaveSubBatchProgress(_ context.Context, b SubBatch, results []Outcome) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	old := f.outcomes[b.OperationID]
+	if old == nil {
+		old = make([]Outcome, len(results))
+		for i := range old {
+			old[i].Pending = true
+		}
+	}
+	for i, result := range results {
+		if old[i].Pending && !result.Pending {
+			old[i] = result
+		}
+	}
+	confirmed := false
+	for _, result := range old {
+		confirmed = confirmed || !result.Pending
+	}
+	if confirmed {
+		f.outcomes[b.OperationID] = old
+	}
+	return nil
+}
+func (f *durableFake) ClaimSubBatch(_ context.Context, id, token string, now time.Time) (*SubBatch, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	b, ok := f.batches[id]
+	if !ok || b.LeaseToken != "" || b.Status == "done" || b.Status == "review" || f.next[id].After(now) {
+		return nil, nil
+	}
+	out := b
+	b.LeaseToken = token
+	out.LeaseToken = token
+	if b.Status == "prepared" {
+		b.Status = "submitting"
+		b.SubmittedAt = now
+		out.SubmittedAt = now
+	} else {
+		b.PollCount++
+		out.PollCount = b.PollCount
+	}
+	f.batches[id] = b
+	return &out, nil
+}
+func (f *durableFake) ListDueSubBatches(_ context.Context, now time.Time, limit int) ([]string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var ids []string
+	for id, b := range f.batches {
+		if b.Status != "done" && b.Status != "review" && !f.next[id].After(now) {
+			ids = append(ids, id)
+		}
+	}
+	sort.Slice(ids, func(i, j int) bool {
+		a, b := f.next[ids[i]], f.next[ids[j]]
+		if a.Equal(b) {
+			return ids[i] < ids[j]
+		}
+		return a.Before(b)
+	})
+	if len(ids) > limit {
+		ids = ids[:limit]
+	}
+	return ids, nil
+}
+func (f *durableFake) UpdateSubBatch(_ context.Context, b SubBatch, status, job, reason string, next time.Time, count int) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	old := f.batches[b.OperationID]
+	if old.LeaseToken != b.LeaseToken {
+		return errors.New("lease lost")
+	}
+	old.Status = status
+	old.JobID = job
+	old.PollCount = count
+	old.LeaseToken = ""
+	f.batches[b.OperationID] = old
+	f.next[b.OperationID] = next
+	return nil
+}
+func (f *durableFake) CompleteSubBatch(_ context.Context, b SubBatch, results []Outcome) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	old := f.batches[b.OperationID]
+	if old.LeaseToken != b.LeaseToken {
+		return errors.New("lease lost")
+	}
+	old.Status = "done"
+	old.LeaseToken = ""
+	f.batches[b.OperationID] = old
+	f.outcomes[b.OperationID] = results
+	return nil
+}
+func (f *durableFake) FinalizeReadyBatch(context.Context, string) error              { return nil }
+func (f *durableFake) ListRecoverableBatches(context.Context, int) ([]string, error) { return nil, nil }
+
+type asyncFake struct {
+	submitCalls int
+	queryCalls  int
+	submit      func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error)
+	query       func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error)
+}
+
+func (f *asyncFake) SubmitSimApnFunction(_ context.Context, in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+	f.submitCalls++
+	return f.submit(in)
+}
+func (f *asyncFake) QuerySimApnFunctionResult(_ context.Context, in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+	f.queryCalls++
+	return f.query(in)
+}
+func fixtureSub() SubBatch {
+	op, _ := OperationByBatchType(consts.BatchTypeDataStop)
+	return newSubBatch(&Task{Batch: Batch{BatchNo: "P-1", BatchType: consts.BatchTypeDataStop}, Operation: op, ApnName: "CMIOT"}, []*Detail{{ID: 1, ICCID: "iccid-1", MSISDN: "1001", ChanKey: "a"}, {ID: 2, ICCID: "iccid-2", MSISDN: "1002", ChanKey: "a"}})
+}
+func pendingResponse(in *model.SimCardApnChangeInput, code int) []model.SimBaseRes[model.SimCardApnChangeOutputItem] {
+	var out []model.SimBaseRes[model.SimCardApnChangeOutputItem]
+	for _, key := range in.SimKeys {
+		out = append(out, model.SimBaseRes[model.SimCardApnChangeOutputItem]{SimKey: key, Code: code, Message: "结果暂不可查", ResData: model.SimCardApnChangeOutputItem{OrderNum: "job-1", Pending: true}})
+	}
+	return out
+}
+
+func TestDurableAPNRestartDoesNotSubmitAcceptedAgain(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{submit: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		if repo.batches[sub.OperationID].Status != "submitting" {
+			t.Fatal("提交前未保存意图")
+		}
+		return pendingResponse(in, 0), nil
+	}, query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		if in.ReqData.QueryJobID != "job-1" {
+			t.Fatalf("查询丢失jobId: %+v", in.ReqData)
+		}
+		out := pendingResponse(in, 0)
+		out[0].ResData.Pending = false
+		out[1].ResData.Pending = false
+		out[1].Code = 12047
+		out[1].Message = "明确失败"
+		return out, nil
+	}}
+	e := &executor{}
+	before := time.Now()
+	if err := e.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if repo.next[sub.OperationID].Before(before.Add(firstResultDelay)) || repo.next[sub.OperationID].After(time.Now().Add(firstResultDelay)) {
+		t.Fatalf("首查未按firstResultDelay排程: %v", repo.next[sub.OperationID])
+	}
+	// A second process must respect next_check_at and never issue a second write.
+	restarted := &executor{}
+	if err := restarted.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if upstream.queryCalls != 0 {
+		t.Fatal("过早查询")
+	}
+	repo.next[sub.OperationID] = time.Time{}
+	if err := restarted.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if upstream.submitCalls != 1 || upstream.queryCalls != 1 {
+		t.Fatalf("submit=%d query=%d", upstream.submitCalls, upstream.queryCalls)
+	}
+	outcomes := repo.outcomes[sub.OperationID]
+	if len(outcomes) != 2 || !outcomes[0].Success || outcomes[1].Success {
+		t.Fatalf("部分失败未逐卡保留: %+v", outcomes)
+	}
+}
+
+func TestDurableAPNTimeoutRecoversMappingWithoutResubmission(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		return nil, context.DeadlineExceeded
+	}, query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		if !in.ReqData.RecoverOnly || in.ReqData.QueryJobID != "" || in.ReqData.OperationID != sub.OperationID {
+			t.Fatalf("不是按operationId恢复: %+v", in.ReqData)
+		}
+		return pendingResponse(in, 0), nil
+	}}
+	e := &executor{}
+	if err := e.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if len(repo.outcomes) != 0 {
+		t.Fatal("超时不能标记办理失败")
+	}
+	repo.next[sub.OperationID] = time.Time{}
+	if err := e.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if upstream.submitCalls != 1 || upstream.queryCalls != 1 || repo.batches[sub.OperationID].JobID != "job-1" {
+		t.Fatalf("受理恢复错误: %+v", repo.batches[sub.OperationID])
+	}
+}
+
+func TestDurableAPNPendingNonzeroStopsAfterQueryWindowWithoutFailure(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	sub.Status = "accepted"
+	sub.JobID = "job-1"
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		return pendingResponse(in, 500), nil
+	}}
+	for i := 1; i <= maxResultQueries; i++ {
+		repo.next[sub.OperationID] = time.Time{}
+		if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+			t.Fatal(err)
+		}
+		b := repo.batches[sub.OperationID]
+		if b.PollCount != i {
+			t.Fatalf("poll count=%d want %d", b.PollCount, i)
+		}
+		if i < maxResultQueries && b.Status != "accepted" {
+			t.Fatalf("pending误终态: %+v", b)
+		}
+	}
+	if repo.batches[sub.OperationID].Status != "review" || len(repo.outcomes) != 0 {
+		t.Fatal("补查窗口耗尽后未确认不得写办理失败")
+	}
+}
+
+func TestDurableAPNMixedResultsPersistAcrossMissingAndReview(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{submit: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		out := pendingResponse(in, 0)
+		out[0].ResData.Pending = false
+		out[0].ResData.OrderNum = "single-order"
+		return out, nil
+	}, query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		return nil, nil
+	}}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if got := repo.outcomes[sub.OperationID]; len(got) != 2 || !got[0].Success || !got[1].Pending {
+		t.Fatalf("混合确认结果未立即保存: %+v", got)
+	}
+	base := repo.batches[sub.OperationID].SubmittedAt
+	for i := 1; i <= maxResultQueries; i++ {
+		repo.next[sub.OperationID] = time.Time{}
+		if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+			t.Fatal(err)
+		}
+		b := repo.batches[sub.OperationID]
+		if b.PollCount != i || !b.SubmittedAt.Equal(base) {
+			t.Fatalf("重启丢失持久时点/次数: %+v", b)
+		}
+		if i < maxResultQueries && !repo.next[sub.OperationID].Equal(base.Add(firstResultDelay+time.Duration(i)*furtherResultDelay)) {
+			t.Fatalf("排程未按submitted_at: %v", repo.next[sub.OperationID])
+		}
+	}
+	if !repo.outcomes[sub.OperationID][0].Success || repo.batches[sub.OperationID].Status != "review" || upstream.submitCalls != 1 {
+		t.Fatal("后续缺卡覆盖终态或重发")
+	}
+}
+
+func TestDurableAPNMappingRecoveryKeepsConsumedPollCount(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	sub.Status = "uncertain"
+	sub.PollCount = 2
+	sub.SubmittedAt = time.Now().Add(-8 * time.Minute)
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		if !in.ReqData.RecoverOnly {
+			t.Fatal("恢复应为只读")
+		}
+		return pendingResponse(in, 0), nil
+	}}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	b := repo.batches[sub.OperationID]
+	if b.JobID != "job-1" || b.PollCount != 3 || !repo.next[sub.OperationID].Equal(sub.SubmittedAt.Add(firstResultDelay+time.Duration(b.PollCount)*furtherResultDelay)) {
+		t.Fatalf("恢复jobId清零或重排: %+v next=%v", b, repo.next[sub.OperationID])
+	}
+}
+
+func TestDurableAPNExplicitUncertainAndMissingNeverFailCard(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{submit: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		return []model.SimBaseRes[model.SimCardApnChangeOutputItem]{{SimKey: in.SimKeys[0], Code: 9999, Message: "代理未确认", ResData: model.SimCardApnChangeOutputItem{Uncertain: true}}}, nil
+	}}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if len(repo.outcomes) != 0 || repo.batches[sub.OperationID].Status != "uncertain" {
+		t.Fatal("显式未知或缺卡不能转明确失败")
+	}
+}
+
+type capabilityFake struct {
+	*asyncFake
+	checkErr error
+}
+
+func (f *capabilityFake) CheckSimApnAsyncCapability(context.Context, *model.SimCardApnChangeInput) error {
+	return f.checkErr
+}
+
+type requestNotSent struct{}
+
+func (requestNotSent) Error() string           { return "能力预检明确拒绝" }
+func (requestNotSent) APNRequestNotSent() bool { return true }
+
+func TestDurableAPNCapabilityRefusalDoesNotClaimOrSpendQueries(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &capabilityFake{asyncFake: &asyncFake{}, checkErr: errors.New("门禁关闭")}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err == nil {
+		t.Fatal("应拒绝预检")
+	}
+	b := repo.batches[sub.OperationID]
+	if b.Status != "prepared" || b.LeaseToken != "" || !b.SubmittedAt.IsZero() || b.PollCount != 0 || upstream.submitCalls != 0 {
+		t.Fatalf("预检拒绝却已领取或提交: %+v", b)
+	}
+	upstream.checkErr = nil
+	upstream.submit = func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		return pendingResponse(in, 0), nil
+	}
+	if upstream.submitCalls != 0 || !repo.next[sub.OperationID].After(time.Now()) {
+		t.Fatal("只读预检拒绝未持久退避")
+	}
+	repo.next[sub.OperationID] = time.Time{}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if upstream.submitCalls != 1 || repo.batches[sub.OperationID].Status != "accepted" {
+		t.Fatal("门禁打开且退避到期后不能恢复首发")
+	}
+}
+
+func TestDurableAPNCapabilityRefusalDoesNotBlockReadOnlyQueries(t *testing.T) {
+	for _, tt := range []struct {
+		name        string
+		status      string
+		jobID       string
+		recoverOnly bool
+	}{
+		{name: "accepted_with_job_id", status: "accepted", jobID: "job-1"},
+		{name: "submitting", status: "submitting", recoverOnly: true},
+		{name: "uncertain", status: "uncertain", recoverOnly: true},
+		{name: "accepted_without_job_id", status: "accepted", recoverOnly: true},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			repo := newDurableFake()
+			sub := fixtureSub()
+			sub.Status = tt.status
+			sub.JobID = tt.jobID
+			sub.SubmittedAt = time.Now().Add(-firstResultDelay)
+			if err := repo.PrepareSubBatch(ctx, sub); err != nil {
+				t.Fatal(err)
+			}
+			upstream := &capabilityFake{
+				checkErr: errors.New("门禁关闭"),
+				asyncFake: &asyncFake{
+					submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+						t.Fatal("只读补查不得重新提交")
+						return nil, nil
+					},
+					query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+						if in.ReqData.OperationID != sub.OperationID || in.ReqData.QueryJobID != tt.jobID || in.ReqData.RecoverOnly != tt.recoverOnly {
+							t.Fatalf("只读查询参数错误: %+v", in.ReqData)
+						}
+						return pendingResponse(in, 0), nil
+					},
+				},
+			}
+			if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+				t.Fatalf("只读查询被提交门禁阻断: %v", err)
+			}
+			if upstream.queryCalls != 1 || upstream.submitCalls != 0 {
+				t.Fatalf("应仅查询一次: query=%d submit=%d", upstream.queryCalls, upstream.submitCalls)
+			}
+			b := repo.batches[sub.OperationID]
+			if b.Status != "accepted" || b.JobID != "job-1" || b.PollCount != 1 || b.LeaseToken != "" {
+				t.Fatalf("只读查询结果未持久化: %+v", b)
+			}
+		})
+	}
+}
+
+func TestDurableAPNOnlyTypedUnsentCanReturnPrepared(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		return nil, fmt.Errorf("预检竞态: %w", requestNotSent{})
+	}}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	b := repo.batches[sub.OperationID]
+	if b.Status != "prepared" || !b.SubmittedAt.IsZero() || b.PollCount != 0 {
+		t.Fatalf("确定未发送应安全回退: %+v", b)
+	}
+	upstream.submit = func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		return nil, context.DeadlineExceeded
+	}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if repo.batches[sub.OperationID].Status != "uncertain" {
+		t.Fatal("普通网络错误被误作确定未发送")
+	}
+}
+
+func TestDurableAPNFinishedFailure9999IsTerminal(t *testing.T) {
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	sub.Status = "accepted"
+	sub.JobID = "failed-job"
+	sub.SubmittedAt = time.Now().Add(-time.Minute)
+	_ = repo.PrepareSubBatch(ctx, sub)
+	upstream := &asyncFake{query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
+		var out []model.SimBaseRes[model.SimCardApnChangeOutputItem]
+		for _, key := range in.SimKeys {
+			out = append(out, model.SimBaseRes[model.SimCardApnChangeOutputItem]{SimKey: key, Code: 9999, Message: "任务已完成且明确失败"})
+		}
+		return out, nil
+	}}
+	if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if repo.batches[sub.OperationID].Status != "done" || len(repo.outcomes[sub.OperationID]) != 2 {
+		t.Fatal("明确9999被误判待查")
+	}
+	for _, out := range repo.outcomes[sub.OperationID] {
+		if out.Pending || out.Success || out.Code != 9999 {
+			t.Fatalf("明确失败终态错误: %+v", out)
+		}
+	}
+}
+
+func TestDurableAPNRealClientSecondProbeRejectsThenSubmitsOnce(t *testing.T) {
+	oldAdapter := g.Cfg().GetAdapter()
+	defer g.Cfg().SetAdapter(oldAdapter)
+	ctx := context.Background()
+	repo := newDurableFake()
+	sub := fixtureSub()
+	_ = repo.PrepareSubBatch(ctx, sub)
+	probes, submits := 0, 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var in model.SimCardApnChangeInput
+		if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
+			t.Error(err)
+			http.Error(w, "bad", 400)
+			return
+		}
+		if r.URL.Path == "/sim/query/sim-apn-capability" {
+			probes++
+			if probes == 2 {
+				http.Error(w, "第二次探测拒绝", 404)
+				return
+			}
+			type capability struct {
+				ProtocolVersion int `json:"protocolVersion"`
+			}
+			out := make([]model.SimBaseRes[capability], 0, len(in.SimKeys))
+			for _, key := range in.SimKeys {
+				out = append(out, model.SimBaseRes[capability]{SimKey: key, ResData: capability{ProtocolVersion: 1}})
+			}
+			_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": out})
+			return
+		}
+		submits++
+		out := pendingResponse(&in, 0)
+		for i := range out {
+			out[i].ResData.Pending = false
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": out})
+	}))
+	defer server.Close()
+	adapter, err := gcfg.NewAdapterContent(fmt.Sprintf(`{"upstream":{"proxy":{"address":%q,"timeout":"1s","apnAsyncProtocolVersion":1}}}`, strings.TrimPrefix(server.URL, "http://")))
+	if err != nil {
+		t.Fatal(err)
+	}
+	g.Cfg().SetAdapter(adapter)
+	client := upstreamclient.New()
+	if err := (&executor{}).processSubBatch(ctx, repo, client, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	b := repo.batches[sub.OperationID]
+	if probes != 2 || submits != 0 || b.Status != "prepared" || !b.SubmittedAt.IsZero() || b.LeaseToken != "" {
+		t.Fatalf("双probe竞态烧掉意图: probes=%d submits=%d sub=%+v", probes, submits, b)
+	}
+	if err := (&executor{}).processSubBatch(ctx, repo, client, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if submits != 1 || repo.batches[sub.OperationID].Status != "done" {
+		t.Fatalf("门禁恢复未唯一首发: submits=%d sub=%+v", submits, repo.batches[sub.OperationID])
+	}
+	if err := (&executor{}).processSubBatch(ctx, repo, client, sub.OperationID); err != nil {
+		t.Fatal(err)
+	}
+	if submits != 1 {
+		t.Fatal("完成后重复首发")
+	}
+}
+
+func TestAsyncSubBatchStableAndChannelIsolated(t *testing.T) {
+	op, _ := OperationByBatchType(consts.BatchTypeDataStart)
+	task := &Task{Batch: Batch{BatchNo: "P-2", BatchType: consts.BatchTypeDataStart}, Operation: op}
+	groups := map[string][]*Detail{"a": {}, "b": {}}
+	for i := 0; i < 203; i++ {
+		key := "a"
+		if i >= 201 {
+			key = "b"
+		}
+		groups[key] = append(groups[key], &Detail{ID: int64(i + 1), ICCID: fmt.Sprint(i), MSISDN: fmt.Sprint(i), ChanKey: key})
+	}
+	total := 0
+	for key, details := range groups {
+		for _, chunk := range splitAsyncDetails(details) {
+			if len(chunk) > 100 {
+				t.Fatal("超过100卡")
+			}
+			first := newSubBatch(task, chunk)
+			second := newSubBatch(task, chunk)
+			if first.OperationID != second.OperationID {
+				t.Fatal("operationId不稳定")
+			}
+			for _, sim := range first.Request.SimKeys {
+				if sim.ChanKey != key {
+					t.Fatal("跨通道子批")
+				}
+			}
+			total++
+		}
+	}
+	if total != 5 {
+		t.Fatalf("sub batches=%d want5", total)
+	}
+}