chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,431 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/google/uuid"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
// 首查 +3min，此后每 3min 重试 3 次（+3/+6/+9/+12min）。
|
||||
// 2026-10-15 潍坊、濮阳实测：10/30/100 卡共12笔首次可见终态约 65~225s；
|
||||
// 15s 采样只能给出完成区间，不能据此承诺所有高峰时段的完成上限。
|
||||
// 偶发超长任务不增加自动重试；超窗记 review，保留受理号和原因供手动查询。
|
||||
// 未确认不得记办理失败，也不得重新提交办理请求。
|
||||
const (
|
||||
subBatchLease = 5 * time.Minute
|
||||
firstResultDelay = 3 * time.Minute
|
||||
furtherResultDelay = 3 * time.Minute
|
||||
maxResultQueries = 4
|
||||
)
|
||||
|
||||
func newSubBatch(task *Task, details []*Detail) SubBatch {
|
||||
keys := make([]*model.SimKey, 0, len(details))
|
||||
ids := make([]int64, 0, len(details))
|
||||
for _, detail := range details {
|
||||
ids = append(ids, detail.ID)
|
||||
keys = append(keys, &model.SimKey{ChanKey: detail.ChanKey, ICCID: detail.ICCID, MSISDN: detail.MSISDN, IMSI: detail.IMSI})
|
||||
}
|
||||
raw, _ := json.Marshal(struct {
|
||||
BatchNo string
|
||||
BatchType int
|
||||
IDs []int64
|
||||
}{task.Batch.BatchNo, task.Batch.BatchType, ids})
|
||||
digest := sha256.Sum256(raw)
|
||||
id := hex.EncodeToString(digest[:])
|
||||
input := model.SimCardApnChangeInput{SimKeys: keys, ReqData: model.SimCardApnChangeInputItem{OperType: task.Operation.ApnOperType, ApnName: task.ApnName, OperationID: id, SubmitOnly: true}}
|
||||
return SubBatch{BatchNo: task.Batch.BatchNo, BatchType: task.Batch.BatchType, ChanKey: details[0].ChanKey, OperationID: id, DetailIDs: ids, Request: input, Status: "prepared"}
|
||||
}
|
||||
|
||||
func splitAsyncDetails(details []*Detail) [][]*Detail {
|
||||
withMSISDN := make([]*Detail, 0, len(details))
|
||||
batches := make([][]*Detail, 0)
|
||||
for _, detail := range details {
|
||||
if strings.TrimSpace(detail.MSISDN) == "" {
|
||||
batches = append(batches, []*Detail{detail})
|
||||
} else {
|
||||
withMSISDN = append(withMSISDN, detail)
|
||||
}
|
||||
}
|
||||
for _, chunk := range splitDetails(withMSISDN, consts.BatchSubBatchSize) {
|
||||
if len(chunk) < 10 {
|
||||
for _, detail := range chunk {
|
||||
batches = append(batches, []*Detail{detail})
|
||||
}
|
||||
} else {
|
||||
batches = append(batches, chunk)
|
||||
}
|
||||
}
|
||||
return batches
|
||||
}
|
||||
|
||||
// prepareAndSubmit persists every intent before a single write leaves the process.
|
||||
func (e *executor) prepareAndSubmit(ctx context.Context, task *Task, repo SubBatchRepository, upstream AsyncUpstream) error {
|
||||
groups := make(map[string][]*Detail)
|
||||
for _, detail := range task.Details {
|
||||
if detail != nil && detail.Reason == "" && (detail.DetailStatus == 0 || detail.DetailStatus == consts.BatchDetailStatusRunning) {
|
||||
groups[detail.ChanKey] = append(groups[detail.ChanKey], detail)
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
for _, slice := range splitAsyncDetails(groups[key]) {
|
||||
sub := newSubBatch(task, slice)
|
||||
if err := repo.PrepareSubBatch(ctx, sub); err != nil {
|
||||
return fmt.Errorf("保存APN提交意图失败（未提交运营商）: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// processSubBatch only submits while prepared. Any interrupted submission is recovered by operationId, never repeated.
|
||||
func (e *executor) processSubBatch(ctx context.Context, repo SubBatchRepository, upstream AsyncUpstream, operationID string) error {
|
||||
channel, err := repo.SubBatchChannel(ctx, operationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
value, _ := e.channelSlots.LoadOrStore(channel, make(chan struct{}, consts.BatchSubBatchConcurrency))
|
||||
slots := value.(chan struct{})
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
defer func() { <-slots }()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
token := uuid.NewString()
|
||||
callCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
|
||||
defer cancel()
|
||||
if checker, ok := upstream.(interface {
|
||||
CheckSimApnAsyncCapability(context.Context, *model.SimCardApnChangeInput) error
|
||||
}); ok {
|
||||
snapshot, err := repo.GetSubBatch(callCtx, operationID)
|
||||
if err != nil || snapshot == nil {
|
||||
return err
|
||||
}
|
||||
if snapshot.Status == "done" || snapshot.Status == "review" {
|
||||
return nil
|
||||
}
|
||||
if snapshot.Status == "prepared" {
|
||||
if err := checker.CheckSimApnAsyncCapability(callCtx, &snapshot.Request); err != nil {
|
||||
persistCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if delayErr := repo.DelaySubBatchPrecheck(persistCtx, *snapshot, time.Now().Add(30*time.Second), err.Error()); delayErr != nil {
|
||||
return fmt.Errorf("保存APN预检调度退避失败: %w", delayErr)
|
||||
}
|
||||
return fmt.Errorf("APN提交预检未就绪，未领取提交意图: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
sub, err := repo.ClaimSubBatch(ctx, operationID, token, time.Now())
|
||||
if err != nil || sub == nil {
|
||||
return err
|
||||
}
|
||||
if sub.Status == "prepared" {
|
||||
response, submitErr := upstream.SubmitSimApnFunction(callCtx, &sub.Request)
|
||||
if submitErr != nil {
|
||||
if isAPNRequestNotSent(submitErr) {
|
||||
return releaseUnsent(ctx, repo, *sub, submitErr)
|
||||
}
|
||||
return e.rememberUncertain(ctx, repo, *sub, "提交结果未确认（网络超时或连接中断）: "+submitErr.Error())
|
||||
}
|
||||
return e.handleResponse(ctx, repo, *sub, response, false)
|
||||
}
|
||||
if sub.Status == "submitting" || sub.Status == "uncertain" || (sub.Status == "accepted" && sub.JobID == "") {
|
||||
sub.Request.ReqData.RecoverOnly = true
|
||||
sub.Request.ReqData.QueryJobID = ""
|
||||
response, queryErr := upstream.QuerySimApnFunctionResult(callCtx, &sub.Request)
|
||||
if queryErr != nil {
|
||||
if isAPNRequestNotSent(queryErr) {
|
||||
return releaseUnsent(ctx, repo, *sub, queryErr)
|
||||
}
|
||||
return e.rememberUncertain(ctx, repo, *sub, "受理映射恢复失败（不重发）: "+queryErr.Error())
|
||||
}
|
||||
return e.handleResponse(ctx, repo, *sub, response, false)
|
||||
}
|
||||
sub.Request.ReqData.QueryJobID = sub.JobID
|
||||
sub.Request.ReqData.RecoverOnly = false
|
||||
response, queryErr := upstream.QuerySimApnFunctionResult(callCtx, &sub.Request)
|
||||
if queryErr != nil {
|
||||
if isAPNRequestNotSent(queryErr) {
|
||||
return releaseUnsent(ctx, repo, *sub, queryErr)
|
||||
}
|
||||
return e.queryUnavailable(ctx, repo, *sub, "查询请求超时或不可用: "+queryErr.Error())
|
||||
}
|
||||
return e.handleResponse(ctx, repo, *sub, response, true)
|
||||
}
|
||||
|
||||
// QueryReview仅执行一次只读补查，不重新开启自动轮询或提交办理。
|
||||
func (e *executor) QueryReview(ctx context.Context, operationID string) error {
|
||||
operationID = strings.TrimSpace(operationID)
|
||||
if operationID == "" {
|
||||
return fmt.Errorf("operationId不能为空")
|
||||
}
|
||||
repo, ok := e.repository.(SubBatchRepository)
|
||||
if !ok {
|
||||
return fmt.Errorf("未配置APN持久化仓储")
|
||||
}
|
||||
reviewRepo, ok := e.repository.(ReviewRepository)
|
||||
if !ok {
|
||||
return fmt.Errorf("未配置APN待核查查询仓储")
|
||||
}
|
||||
upstream, ok := e.upstream.(AsyncUpstream)
|
||||
if !ok {
|
||||
return fmt.Errorf("未配置APN只读补查客户端")
|
||||
}
|
||||
sub, err := reviewRepo.ClaimReviewSubBatch(ctx, operationID, uuid.NewString(), time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sub == nil {
|
||||
snapshot, err := repo.GetSubBatch(ctx, operationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if snapshot != nil && snapshot.Status == "done" {
|
||||
return repo.FinalizeReadyBatch(ctx, snapshot.BatchNo)
|
||||
}
|
||||
return fmt.Errorf("子批不处于可查询的待核查状态，或已有查询/汇总正在运行")
|
||||
}
|
||||
callCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
|
||||
defer cancel()
|
||||
sub.Request.ReqData.SubmitOnly = false
|
||||
sub.Request.ReqData.QueryJobID = sub.JobID
|
||||
sub.Request.ReqData.RecoverOnly = sub.JobID == ""
|
||||
items, queryErr := upstream.QuerySimApnFunctionResult(callCtx, &sub.Request)
|
||||
if queryErr != nil {
|
||||
if err := e.deferResult(ctx, repo, *sub, "review", "手动补查未确认："+queryErr.Error()); err != nil {
|
||||
return err
|
||||
}
|
||||
return queryErr
|
||||
}
|
||||
return e.handleResponse(ctx, repo, *sub, items, true)
|
||||
}
|
||||
|
||||
// QueryReviewBatch 是"手动同步上游结果"入口：只对父批下已冻结的待核查子批
|
||||
// 做一次只读补查，然后尝试汇总父批。
|
||||
// 不重发办理请求、不清零自动补查计数、不重新开启轮询调度。
|
||||
func (e *executor) QueryReviewBatch(ctx context.Context, batchNo string) (int, error) {
|
||||
batchNo = strings.TrimSpace(batchNo)
|
||||
if batchNo == "" {
|
||||
return 0, fmt.Errorf("批次编号不能为空")
|
||||
}
|
||||
repo, ok := e.repository.(ReviewBatchRepository)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("未配置APN待核查批量同步仓储")
|
||||
}
|
||||
if _, ok := e.upstream.(AsyncUpstream); !ok {
|
||||
return 0, fmt.Errorf("未配置APN只读补查客户端")
|
||||
}
|
||||
ids, err := repo.ListReviewSubBatches(ctx, batchNo)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
queried := 0
|
||||
var firstErr error
|
||||
for _, id := range ids {
|
||||
if err := e.QueryReview(ctx, id); err != nil {
|
||||
g.Log().Warningf(ctx, "手动同步待核查子批未完成 batchNo=%s operationId=%s error=%v", batchNo, id, err)
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("operationId=%s: %w", id, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
queried++
|
||||
}
|
||||
// 已确认终态的子批会在这里收敛父批；仍为 review 的子批继续留在待核查。
|
||||
if err := repo.FinalizeReadyBatch(ctx, batchNo); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
g.Log().Infof(ctx, "手动同步APN上游结果 batchNo=%s 待核查子批=%d 本次已同步=%d", batchNo, len(ids), queried)
|
||||
if firstErr != nil {
|
||||
return queried, firstErr
|
||||
}
|
||||
return queried, nil
|
||||
}
|
||||
|
||||
func isAPNRequestNotSent(err error) bool {
|
||||
var notSent interface{ APNRequestNotSent() bool }
|
||||
return errors.As(err, ¬Sent) && notSent.APNRequestNotSent()
|
||||
}
|
||||
func releaseUnsent(ctx context.Context, repo SubBatchRepository, sub SubBatch, cause error) error {
|
||||
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
return repo.ReleaseUnsentSubBatch(persistCtx, sub, cause.Error())
|
||||
}
|
||||
|
||||
func nextResultCheck(sub SubBatch) time.Time {
|
||||
base := sub.SubmittedAt
|
||||
if base.IsZero() {
|
||||
base = time.Now()
|
||||
}
|
||||
return base.Add(firstResultDelay + time.Duration(sub.PollCount)*furtherResultDelay)
|
||||
}
|
||||
|
||||
func (e *executor) rememberUncertain(ctx context.Context, repo SubBatchRepository, sub SubBatch, reason string) error {
|
||||
return e.deferResult(ctx, repo, sub, "uncertain", reason)
|
||||
}
|
||||
|
||||
func (e *executor) queryUnavailable(ctx context.Context, repo SubBatchRepository, sub SubBatch, reason string) error {
|
||||
return e.deferResult(ctx, repo, sub, "accepted", reason)
|
||||
}
|
||||
|
||||
func (e *executor) deferResult(ctx context.Context, repo SubBatchRepository, sub SubBatch, state, reason string) error {
|
||||
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
g.Log().Warningf(persistCtx, "APN补查 operationId=%s jobId=%s pollCount=%d reason=%s", sub.OperationID, sub.JobID, sub.PollCount, reason)
|
||||
if sub.Status == "review_query" || sub.PollCount >= maxResultQueries {
|
||||
if err := repo.UpdateSubBatch(persistCtx, sub, "review", sub.JobID, "补查窗口结束，办理结果未知："+reason, time.Time{}, sub.PollCount); err != nil {
|
||||
return err
|
||||
}
|
||||
return repo.FinalizeReadyBatch(persistCtx, sub.BatchNo)
|
||||
}
|
||||
return repo.UpdateSubBatch(persistCtx, sub, state, sub.JobID, reason, nextResultCheck(sub), sub.PollCount)
|
||||
}
|
||||
|
||||
func (e *executor) handleResponse(ctx context.Context, repo SubBatchRepository, sub SubBatch, items []model.SimBaseRes[model.SimCardApnChangeOutputItem], queried bool) error {
|
||||
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||||
defer cancel()
|
||||
outcomes := make([]Outcome, len(sub.Request.SimKeys))
|
||||
used := make([]bool, len(items))
|
||||
pending := false
|
||||
reason := "未返回该卡结果，办理未确认"
|
||||
for i, key := range sub.Request.SimKeys {
|
||||
matched := -1
|
||||
for j, item := range items {
|
||||
if !used[j] && sameSim(&Detail{ICCID: key.ICCID, MSISDN: key.MSISDN, IMSI: key.IMSI}, item.SimKey) {
|
||||
matched = j
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched < 0 {
|
||||
outcomes[i] = Outcome{Pending: true, Message: reason}
|
||||
pending = true
|
||||
continue
|
||||
}
|
||||
used[matched] = true
|
||||
item := items[matched]
|
||||
if item.ResData.Pending || item.ResData.Uncertain {
|
||||
pending = true
|
||||
outcomes[i] = Outcome{Pending: true, Code: item.Code, Message: item.Message}
|
||||
if strings.TrimSpace(item.Message) != "" {
|
||||
reason = item.Message
|
||||
}
|
||||
// Only Pending identifies a batch job: a synchronous per-card order is never queried as a job.
|
||||
if item.ResData.Pending && strings.TrimSpace(item.ResData.OrderNum) != "" {
|
||||
sub.JobID = strings.TrimSpace(item.ResData.OrderNum)
|
||||
}
|
||||
continue
|
||||
}
|
||||
outcomes[i] = Outcome{Success: item.Code == 0, Code: item.Code, Message: item.Message, OrderNum: item.ResData.OrderNum}
|
||||
}
|
||||
if pending {
|
||||
// Confirmed cards are committed now. Later missing/pending responses cannot overwrite their terminal rows.
|
||||
if err := repo.SaveSubBatchProgress(persistCtx, sub, outcomes); err != nil {
|
||||
return err
|
||||
}
|
||||
state := "accepted"
|
||||
if sub.JobID == "" {
|
||||
state = "uncertain"
|
||||
reason += "（缺少jobId，仅按operationId恢复，不重发）"
|
||||
}
|
||||
return e.deferResult(persistCtx, repo, sub, state, reason)
|
||||
}
|
||||
if err := repo.CompleteSubBatch(persistCtx, sub, outcomes); err != nil {
|
||||
return err
|
||||
}
|
||||
return repo.FinalizeReadyBatch(persistCtx, sub.BatchNo)
|
||||
}
|
||||
|
||||
func mapOutcomesFromKeys(keys []*model.SimKey, response []model.SimBaseRes[model.SimCardApnChangeOutputItem]) []Outcome {
|
||||
details := make([]*Detail, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
details = append(details, &Detail{ICCID: key.ICCID, MSISDN: key.MSISDN, IMSI: key.IMSI})
|
||||
}
|
||||
return mapOutcomes(details, response)
|
||||
}
|
||||
|
||||
// RunReconciler uses a fixed global worker bound; waiting for a job never retains a worker slot.
|
||||
func (e *executor) RunReconciler(ctx context.Context) error {
|
||||
repo, ok := e.repository.(SubBatchRepository)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
upstream, ok := e.upstream.(AsyncUpstream)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
jobs := make(chan string, 64)
|
||||
var inFlight sync.Map
|
||||
var workers sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
workers.Add(1)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case id := <-jobs:
|
||||
if err := e.processSubBatch(ctx, repo, upstream, id); err != nil {
|
||||
g.Log().Errorf(ctx, "APN恢复失败 operationId=%s error=%v", id, err)
|
||||
}
|
||||
inFlight.Delete(id)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
defer workers.Wait()
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
batches, err := repo.ListRecoverableBatches(ctx, 32)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "读取APN恢复父批失败: %v", err)
|
||||
} else {
|
||||
for _, batchNo := range batches {
|
||||
if err := e.Execute(ctx, batchNo); err != nil {
|
||||
g.Log().Errorf(ctx, "恢复APN父批失败 batchNo=%s error=%v", batchNo, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
ids, err := repo.ListDueSubBatches(ctx, time.Now(), 128)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "读取APN补查子批失败: %v", err)
|
||||
} else {
|
||||
for _, id := range ids {
|
||||
if _, loaded := inFlight.LoadOrStore(id, true); loaded {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case jobs <- id:
|
||||
case <-ctx.Done():
|
||||
inFlight.Delete(id)
|
||||
return nil
|
||||
default:
|
||||
inFlight.Delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
type fairnessRepository struct{ *durableFake }
|
||||
|
||||
func (f *fairnessRepository) Load(context.Context, string) (*Task, error) {
|
||||
return nil, errors.New("公平性测试不应准备父批")
|
||||
}
|
||||
func (f *fairnessRepository) SaveValidationFailure(context.Context, *Task) error { return nil }
|
||||
func (f *fairnessRepository) SaveResult(context.Context, *Task, *Detail, Outcome) (Outcome, error) {
|
||||
return Outcome{}, nil
|
||||
}
|
||||
func (f *fairnessRepository) Finalize(context.Context, *Task) error { return nil }
|
||||
|
||||
type fairnessUpstream struct {
|
||||
asyncFake
|
||||
rejected atomic.Int32
|
||||
}
|
||||
|
||||
func (f *fairnessUpstream) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return nil, errors.New("调度不得走同步办理")
|
||||
}
|
||||
|
||||
func (f *fairnessUpstream) CheckSimApnAsyncCapability(_ context.Context, in *model.SimCardApnChangeInput) error {
|
||||
if in.SimKeys[0].ChanKey == "removed" {
|
||||
f.rejected.Add(1)
|
||||
return errors.New("通道已移除，无protocolVersion")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDurableAPNPrecheckBackoffAllowsHealthy129thTask(t *testing.T) {
|
||||
repo := newDurableFake()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
for i := 0; i < 129; i++ {
|
||||
sub := fixtureSub()
|
||||
sub.OperationID = fmt.Sprintf("fair-%03d", i)
|
||||
sub.Request.ReqData.OperationID = sub.OperationID
|
||||
sub.ChanKey = "removed"
|
||||
if i == 128 {
|
||||
sub.ChanKey = "healthy"
|
||||
}
|
||||
for _, key := range sub.Request.SimKeys {
|
||||
key.ChanKey = sub.ChanKey
|
||||
}
|
||||
if err := repo.PrepareSubBatch(ctx, sub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
completed := make(chan struct{}, 1)
|
||||
upstream := &fairnessUpstream{}
|
||||
upstream.submit = func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if in.SimKeys[0].ChanKey != "healthy" {
|
||||
t.Error("拒绝通道发出办理")
|
||||
}
|
||||
out := pendingResponse(in, 0)
|
||||
for i := range out {
|
||||
out[i].ResData.Pending = false
|
||||
}
|
||||
completed <- struct{}{}
|
||||
return out, nil
|
||||
}
|
||||
e := &executor{repository: &fairnessRepository{repo}, upstream: upstream}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.RunReconciler(ctx) }()
|
||||
select {
|
||||
case <-completed:
|
||||
case <-time.After(15 * time.Second):
|
||||
cancel()
|
||||
<-done
|
||||
t.Fatal("前128个预检拒绝使第129健康任务饥饿")
|
||||
}
|
||||
// 健康提交可能早于已入队的拒绝任务完成；等待退避持久化后再取消调度器。
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
timeout := time.NewTimer(15 * time.Second)
|
||||
defer timeout.Stop()
|
||||
for {
|
||||
repo.mu.Lock()
|
||||
delayed := 0
|
||||
for id, sub := range repo.batches {
|
||||
if sub.ChanKey == "removed" && !repo.next[id].IsZero() {
|
||||
delayed++
|
||||
}
|
||||
}
|
||||
repo.mu.Unlock()
|
||||
if delayed == 128 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ticker.C:
|
||||
case <-timeout.C:
|
||||
cancel()
|
||||
<-done
|
||||
t.Fatalf("拒绝任务退避未全部持久化: delayed=%d rejected=%d", delayed, upstream.rejected.Load())
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.submitCalls != 1 || upstream.rejected.Load() != 128 {
|
||||
t.Fatalf("submit=%d rejected=%d", upstream.submitCalls, upstream.rejected.Load())
|
||||
}
|
||||
repo.mu.Lock()
|
||||
defer repo.mu.Unlock()
|
||||
for id, sub := range repo.batches {
|
||||
if sub.ChanKey == "healthy" {
|
||||
continue
|
||||
}
|
||||
if sub.Status != "prepared" || sub.PollCount != 0 || !sub.SubmittedAt.IsZero() || sub.LeaseToken != "" || repo.next[id].IsZero() {
|
||||
t.Fatalf("只读退避改变办理意图: %+v", sub)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
// reviewBatchFake 复用 durableFake 的持久化语义，只补手动同步需要的仓储能力。
|
||||
type reviewBatchFake struct {
|
||||
*durableFake
|
||||
candidates []string
|
||||
finalized []string
|
||||
}
|
||||
|
||||
func newReviewBatchFake(candidates ...string) *reviewBatchFake {
|
||||
return &reviewBatchFake{durableFake: newDurableFake(), candidates: candidates}
|
||||
}
|
||||
|
||||
func (f *reviewBatchFake) ListReviewSubBatches(_ context.Context, _ string) ([]string, error) {
|
||||
return append([]string(nil), f.candidates...), nil
|
||||
}
|
||||
|
||||
// 手动同步不触碰校验/终态写入路径，这里只补齐 Repository 契约。
|
||||
func (f *reviewBatchFake) Load(context.Context, string) (*Task, error) { return &Task{}, nil }
|
||||
func (f *reviewBatchFake) SaveValidationFailure(context.Context, *Task) error { return nil }
|
||||
func (f *reviewBatchFake) SaveResult(context.Context, *Task, *Detail, Outcome) (Outcome, error) {
|
||||
return Outcome{}, nil
|
||||
}
|
||||
func (f *reviewBatchFake) Finalize(context.Context, *Task) error { return nil }
|
||||
|
||||
// manualSyncUpstream 补上同步办理入口，确保手动同步一旦误走同步路径立即暴露。
|
||||
type manualSyncUpstream struct {
|
||||
*asyncFake
|
||||
}
|
||||
|
||||
func (u *manualSyncUpstream) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return nil, errors.New("手动同步不得调用同步办理入口")
|
||||
}
|
||||
|
||||
func (f *reviewBatchFake) FinalizeReadyBatch(_ context.Context, batchNo string) error {
|
||||
f.finalized = append(f.finalized, batchNo)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *reviewBatchFake) ClaimReviewSubBatch(_ context.Context, operationID, token string, now time.Time) (*SubBatch, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
sub, ok := f.batches[operationID]
|
||||
if !ok || (sub.Status != "review" && sub.Status != "review_query") {
|
||||
return nil, nil
|
||||
}
|
||||
sub.Status = "review_query"
|
||||
sub.LeaseToken = token
|
||||
sub.LeaseUntil = now.Add(subBatchLease)
|
||||
f.batches[operationID] = sub
|
||||
out := sub
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// 手动同步只用既有受理号只读补查：不重发办理、不清零自动补查计数。
|
||||
func TestManualSyncQueriesFrozenIntentsWithoutResubmitting(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
sub := fixtureSub()
|
||||
repo := newReviewBatchFake(sub.OperationID)
|
||||
sub.Status = "review"
|
||||
sub.JobID = "job-1"
|
||||
sub.PollCount = 4
|
||||
repo.batches[sub.OperationID] = sub
|
||||
|
||||
upstream := &manualSyncUpstream{asyncFake: &asyncFake{
|
||||
submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
t.Fatal("手动同步不得重发办理")
|
||||
return nil, nil
|
||||
},
|
||||
query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if in.ReqData.QueryJobID != "job-1" {
|
||||
t.Fatalf("手动同步必须按既有 jobId 查询: %+v", in.ReqData)
|
||||
}
|
||||
out := make([]model.SimBaseRes[model.SimCardApnChangeOutputItem], 0, len(in.SimKeys))
|
||||
for _, key := range in.SimKeys {
|
||||
out = append(out, model.SimBaseRes[model.SimCardApnChangeOutputItem]{SimKey: key, Code: 0, ResData: model.SimCardApnChangeOutputItem{OrderNum: "job-1"}})
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
}}
|
||||
executor := &executor{repository: repo, upstream: upstream}
|
||||
|
||||
queried, err := executor.QueryReviewBatch(ctx, " P-1 ")
|
||||
if err != nil {
|
||||
t.Fatalf("QueryReviewBatch() error = %v", err)
|
||||
}
|
||||
if queried != 1 {
|
||||
t.Fatalf("queried = %d, want 1", queried)
|
||||
}
|
||||
if upstream.submitCalls != 0 {
|
||||
t.Fatalf("submitCalls = %d, want 0", upstream.submitCalls)
|
||||
}
|
||||
got := repo.batches[sub.OperationID]
|
||||
if got.Status != "done" {
|
||||
t.Fatalf("状态 = %s, want done", got.Status)
|
||||
}
|
||||
if got.PollCount != 4 {
|
||||
t.Fatalf("pollCount = %d, want 4（手动同步不得清零自动补查计数）", got.PollCount)
|
||||
}
|
||||
// 子批收敛与批量入口都会尝试汇总父批，重复汇总是幂等的。
|
||||
if len(repo.finalized) == 0 || repo.finalized[0] != "P-1" {
|
||||
t.Fatalf("父批未汇总: %+v", repo.finalized)
|
||||
}
|
||||
if len(repo.outcomes[sub.OperationID]) != len(sub.DetailIDs) {
|
||||
t.Fatalf("逐卡结果未写回: %+v", repo.outcomes[sub.OperationID])
|
||||
}
|
||||
}
|
||||
|
||||
// 上游仍无结论时明细保持待核查，且可以再次手动同步。
|
||||
func TestManualSyncKeepsUnknownWhenUpstreamStillPending(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
sub := fixtureSub()
|
||||
repo := newReviewBatchFake(sub.OperationID)
|
||||
sub.Status = "review"
|
||||
repo.batches[sub.OperationID] = sub
|
||||
|
||||
upstream := &manualSyncUpstream{asyncFake: &asyncFake{
|
||||
submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
t.Fatal("手动同步不得重发办理")
|
||||
return nil, nil
|
||||
},
|
||||
query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return pendingResponse(in, 14008), nil
|
||||
},
|
||||
}}
|
||||
executor := &executor{repository: repo, upstream: upstream}
|
||||
|
||||
if _, err := executor.QueryReviewBatch(ctx, "P-1"); err != nil {
|
||||
t.Fatalf("QueryReviewBatch() error = %v", err)
|
||||
}
|
||||
got := repo.batches[sub.OperationID]
|
||||
if got.Status != "review" {
|
||||
t.Fatalf("状态 = %s, want review（未确认不得记失败）", got.Status)
|
||||
}
|
||||
if got.JobID != "job-1" {
|
||||
t.Fatalf("jobId = %s, want job-1", got.JobID)
|
||||
}
|
||||
if got.PollCount != 0 {
|
||||
t.Fatalf("pollCount = %d, want 0（手动同步不得占用自动补查次数）", got.PollCount)
|
||||
}
|
||||
// 再次同步仍可用，说明没有被自动窗口锁死。
|
||||
if _, err := executor.QueryReviewBatch(ctx, "P-1"); err != nil {
|
||||
t.Fatalf("第二次 QueryReviewBatch() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 单个子批补查失败不影响其余子批，并把失败如实返回给调用方。
|
||||
func TestManualSyncReportsPerIntentFailureAndContinues(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
first := fixtureSub()
|
||||
first.Status = "review"
|
||||
first.JobID = "job-1"
|
||||
second := fixtureSub()
|
||||
second.OperationID = "intent-2"
|
||||
second.Status = "review"
|
||||
repo := newReviewBatchFake(first.OperationID, second.OperationID)
|
||||
repo.batches[first.OperationID] = first
|
||||
repo.batches[second.OperationID] = second
|
||||
|
||||
upstream := &manualSyncUpstream{asyncFake: &asyncFake{
|
||||
submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
t.Fatal("手动同步不得重发办理")
|
||||
return nil, nil
|
||||
},
|
||||
query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if in.ReqData.RecoverOnly {
|
||||
return nil, errors.New("上游查询不可用")
|
||||
}
|
||||
return pendingResponse(in, 0), nil
|
||||
},
|
||||
}}
|
||||
executor := &executor{repository: repo, upstream: upstream}
|
||||
|
||||
queried, err := executor.QueryReviewBatch(ctx, "P-1")
|
||||
if err == nil {
|
||||
t.Fatal("QueryReviewBatch() error = nil, want error")
|
||||
}
|
||||
if queried != 1 {
|
||||
t.Fatalf("queried = %d, want 1（成功一个，失败一个）", queried)
|
||||
}
|
||||
if upstream.submitCalls != 0 {
|
||||
t.Fatalf("submitCalls = %d, want 0", upstream.submitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// 空批次号直接拒绝，避免误触发全量扫描。
|
||||
func TestManualSyncRejectsBlankBatchNo(t *testing.T) {
|
||||
executor := &executor{repository: newReviewBatchFake(), upstream: &manualSyncUpstream{asyncFake: &asyncFake{}}}
|
||||
if _, err := executor.QueryReviewBatch(context.Background(), " "); err == nil {
|
||||
t.Fatal("QueryReviewBatch() blank batchNo error = nil, want error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,650 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
upstreamclient "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/client/upstream"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func TestDurableAPNResultScheduleIsThreeMinutesAndThreeRetries(t *testing.T) {
|
||||
if maxResultQueries != 4 {
|
||||
t.Fatalf("首次加三次重试应为四次查询: got=%d", maxResultQueries)
|
||||
}
|
||||
base := time.Date(2026, 10, 15, 8, 0, 0, 0, time.UTC)
|
||||
for i, minute := range []int{3, 6, 9, 12} {
|
||||
got := nextResultCheck(SubBatch{SubmittedAt: base, PollCount: i})
|
||||
want := base.Add(time.Duration(minute) * time.Minute)
|
||||
if !got.Equal(want) {
|
||||
t.Fatalf("第%d次查询: got=%v want=%v", i+1, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type durableFake struct {
|
||||
mu sync.Mutex
|
||||
batches map[string]SubBatch
|
||||
next map[string]time.Time
|
||||
outcomes map[string][]Outcome
|
||||
preparedBeforeSubmit int
|
||||
}
|
||||
|
||||
func newDurableFake() *durableFake {
|
||||
return &durableFake{batches: map[string]SubBatch{}, next: map[string]time.Time{}, outcomes: map[string][]Outcome{}}
|
||||
}
|
||||
func (f *durableFake) HasDurableParent(context.Context, string) (bool, error) { return true, nil }
|
||||
func (f *durableFake) PrepareSubBatch(_ context.Context, b SubBatch) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.batches[b.OperationID]; !ok {
|
||||
b.ID = int64(len(f.batches) + 1)
|
||||
f.batches[b.OperationID] = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *durableFake) ResumeOrPrepare(ctx context.Context, _ string, p func(context.Context) error) error {
|
||||
return p(ctx)
|
||||
}
|
||||
func (f *durableFake) GetSubBatch(_ context.Context, id string) (*SubBatch, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
b := f.batches[id]
|
||||
b.NextCheckAt = f.next[id]
|
||||
return &b, nil
|
||||
}
|
||||
func (f *durableFake) DelaySubBatchPrecheck(_ context.Context, snapshot SubBatch, next time.Time, _ string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
b := f.batches[snapshot.OperationID]
|
||||
if b.Status == snapshot.Status && b.LeaseToken == snapshot.LeaseToken && f.next[snapshot.OperationID].Equal(snapshot.NextCheckAt) {
|
||||
f.next[snapshot.OperationID] = next
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *durableFake) ReleaseUnsentSubBatch(_ context.Context, b SubBatch, _ string) error {
|
||||
old := f.batches[b.OperationID]
|
||||
old.Status = b.Status
|
||||
old.LeaseToken = ""
|
||||
if b.Status == "prepared" {
|
||||
old.SubmittedAt = time.Time{}
|
||||
} else {
|
||||
old.PollCount--
|
||||
}
|
||||
f.batches[b.OperationID] = old
|
||||
return nil
|
||||
}
|
||||
func (f *durableFake) SubBatchChannel(_ context.Context, id string) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.batches[id].ChanKey, nil
|
||||
}
|
||||
func (f *durableFake) ProtectExecutionFailure(context.Context, string, string) error { return nil }
|
||||
func (f *durableFake) SaveSubBatchProgress(_ context.Context, b SubBatch, results []Outcome) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
old := f.outcomes[b.OperationID]
|
||||
if old == nil {
|
||||
old = make([]Outcome, len(results))
|
||||
for i := range old {
|
||||
old[i].Pending = true
|
||||
}
|
||||
}
|
||||
for i, result := range results {
|
||||
if old[i].Pending && !result.Pending {
|
||||
old[i] = result
|
||||
}
|
||||
}
|
||||
confirmed := false
|
||||
for _, result := range old {
|
||||
confirmed = confirmed || !result.Pending
|
||||
}
|
||||
if confirmed {
|
||||
f.outcomes[b.OperationID] = old
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *durableFake) ClaimSubBatch(_ context.Context, id, token string, now time.Time) (*SubBatch, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
b, ok := f.batches[id]
|
||||
if !ok || b.LeaseToken != "" || b.Status == "done" || b.Status == "review" || f.next[id].After(now) {
|
||||
return nil, nil
|
||||
}
|
||||
out := b
|
||||
b.LeaseToken = token
|
||||
out.LeaseToken = token
|
||||
if b.Status == "prepared" {
|
||||
b.Status = "submitting"
|
||||
b.SubmittedAt = now
|
||||
out.SubmittedAt = now
|
||||
} else {
|
||||
b.PollCount++
|
||||
out.PollCount = b.PollCount
|
||||
}
|
||||
f.batches[id] = b
|
||||
return &out, nil
|
||||
}
|
||||
func (f *durableFake) ListDueSubBatches(_ context.Context, now time.Time, limit int) ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var ids []string
|
||||
for id, b := range f.batches {
|
||||
if b.Status != "done" && b.Status != "review" && !f.next[id].After(now) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool {
|
||||
a, b := f.next[ids[i]], f.next[ids[j]]
|
||||
if a.Equal(b) {
|
||||
return ids[i] < ids[j]
|
||||
}
|
||||
return a.Before(b)
|
||||
})
|
||||
if len(ids) > limit {
|
||||
ids = ids[:limit]
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
func (f *durableFake) UpdateSubBatch(_ context.Context, b SubBatch, status, job, reason string, next time.Time, count int) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
old := f.batches[b.OperationID]
|
||||
if old.LeaseToken != b.LeaseToken {
|
||||
return errors.New("lease lost")
|
||||
}
|
||||
old.Status = status
|
||||
old.JobID = job
|
||||
old.PollCount = count
|
||||
old.LeaseToken = ""
|
||||
f.batches[b.OperationID] = old
|
||||
f.next[b.OperationID] = next
|
||||
return nil
|
||||
}
|
||||
func (f *durableFake) CompleteSubBatch(_ context.Context, b SubBatch, results []Outcome) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
old := f.batches[b.OperationID]
|
||||
if old.LeaseToken != b.LeaseToken {
|
||||
return errors.New("lease lost")
|
||||
}
|
||||
old.Status = "done"
|
||||
old.LeaseToken = ""
|
||||
f.batches[b.OperationID] = old
|
||||
f.outcomes[b.OperationID] = results
|
||||
return nil
|
||||
}
|
||||
func (f *durableFake) FinalizeReadyBatch(context.Context, string) error { return nil }
|
||||
func (f *durableFake) ListRecoverableBatches(context.Context, int) ([]string, error) { return nil, nil }
|
||||
|
||||
type asyncFake struct {
|
||||
submitCalls int
|
||||
queryCalls int
|
||||
submit func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error)
|
||||
query func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error)
|
||||
}
|
||||
|
||||
func (f *asyncFake) SubmitSimApnFunction(_ context.Context, in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
f.submitCalls++
|
||||
return f.submit(in)
|
||||
}
|
||||
func (f *asyncFake) QuerySimApnFunctionResult(_ context.Context, in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
f.queryCalls++
|
||||
return f.query(in)
|
||||
}
|
||||
func fixtureSub() SubBatch {
|
||||
op, _ := OperationByBatchType(consts.BatchTypeDataStop)
|
||||
return newSubBatch(&Task{Batch: Batch{BatchNo: "P-1", BatchType: consts.BatchTypeDataStop}, Operation: op, ApnName: "CMIOT"}, []*Detail{{ID: 1, ICCID: "iccid-1", MSISDN: "1001", ChanKey: "a"}, {ID: 2, ICCID: "iccid-2", MSISDN: "1002", ChanKey: "a"}})
|
||||
}
|
||||
func pendingResponse(in *model.SimCardApnChangeInput, code int) []model.SimBaseRes[model.SimCardApnChangeOutputItem] {
|
||||
var out []model.SimBaseRes[model.SimCardApnChangeOutputItem]
|
||||
for _, key := range in.SimKeys {
|
||||
out = append(out, model.SimBaseRes[model.SimCardApnChangeOutputItem]{SimKey: key, Code: code, Message: "结果暂不可查", ResData: model.SimCardApnChangeOutputItem{OrderNum: "job-1", Pending: true}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDurableAPNRestartDoesNotSubmitAcceptedAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{submit: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if repo.batches[sub.OperationID].Status != "submitting" {
|
||||
t.Fatal("提交前未保存意图")
|
||||
}
|
||||
return pendingResponse(in, 0), nil
|
||||
}, query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if in.ReqData.QueryJobID != "job-1" {
|
||||
t.Fatalf("查询丢失jobId: %+v", in.ReqData)
|
||||
}
|
||||
out := pendingResponse(in, 0)
|
||||
out[0].ResData.Pending = false
|
||||
out[1].ResData.Pending = false
|
||||
out[1].Code = 12047
|
||||
out[1].Message = "明确失败"
|
||||
return out, nil
|
||||
}}
|
||||
e := &executor{}
|
||||
before := time.Now()
|
||||
if err := e.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if repo.next[sub.OperationID].Before(before.Add(firstResultDelay)) || repo.next[sub.OperationID].After(time.Now().Add(firstResultDelay)) {
|
||||
t.Fatalf("首查未按firstResultDelay排程: %v", repo.next[sub.OperationID])
|
||||
}
|
||||
// A second process must respect next_check_at and never issue a second write.
|
||||
restarted := &executor{}
|
||||
if err := restarted.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.queryCalls != 0 {
|
||||
t.Fatal("过早查询")
|
||||
}
|
||||
repo.next[sub.OperationID] = time.Time{}
|
||||
if err := restarted.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.submitCalls != 1 || upstream.queryCalls != 1 {
|
||||
t.Fatalf("submit=%d query=%d", upstream.submitCalls, upstream.queryCalls)
|
||||
}
|
||||
outcomes := repo.outcomes[sub.OperationID]
|
||||
if len(outcomes) != 2 || !outcomes[0].Success || outcomes[1].Success {
|
||||
t.Fatalf("部分失败未逐卡保留: %+v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNTimeoutRecoversMappingWithoutResubmission(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return nil, context.DeadlineExceeded
|
||||
}, query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if !in.ReqData.RecoverOnly || in.ReqData.QueryJobID != "" || in.ReqData.OperationID != sub.OperationID {
|
||||
t.Fatalf("不是按operationId恢复: %+v", in.ReqData)
|
||||
}
|
||||
return pendingResponse(in, 0), nil
|
||||
}}
|
||||
e := &executor{}
|
||||
if err := e.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(repo.outcomes) != 0 {
|
||||
t.Fatal("超时不能标记办理失败")
|
||||
}
|
||||
repo.next[sub.OperationID] = time.Time{}
|
||||
if err := e.processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.submitCalls != 1 || upstream.queryCalls != 1 || repo.batches[sub.OperationID].JobID != "job-1" {
|
||||
t.Fatalf("受理恢复错误: %+v", repo.batches[sub.OperationID])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNPendingNonzeroStopsAfterQueryWindowWithoutFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
sub.Status = "accepted"
|
||||
sub.JobID = "job-1"
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return pendingResponse(in, 500), nil
|
||||
}}
|
||||
for i := 1; i <= maxResultQueries; i++ {
|
||||
repo.next[sub.OperationID] = time.Time{}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := repo.batches[sub.OperationID]
|
||||
if b.PollCount != i {
|
||||
t.Fatalf("poll count=%d want %d", b.PollCount, i)
|
||||
}
|
||||
if i < maxResultQueries && b.Status != "accepted" {
|
||||
t.Fatalf("pending误终态: %+v", b)
|
||||
}
|
||||
}
|
||||
if repo.batches[sub.OperationID].Status != "review" || len(repo.outcomes) != 0 {
|
||||
t.Fatal("补查窗口耗尽后未确认不得写办理失败")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNMixedResultsPersistAcrossMissingAndReview(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{submit: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
out := pendingResponse(in, 0)
|
||||
out[0].ResData.Pending = false
|
||||
out[0].ResData.OrderNum = "single-order"
|
||||
return out, nil
|
||||
}, query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return nil, nil
|
||||
}}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := repo.outcomes[sub.OperationID]; len(got) != 2 || !got[0].Success || !got[1].Pending {
|
||||
t.Fatalf("混合确认结果未立即保存: %+v", got)
|
||||
}
|
||||
base := repo.batches[sub.OperationID].SubmittedAt
|
||||
for i := 1; i <= maxResultQueries; i++ {
|
||||
repo.next[sub.OperationID] = time.Time{}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := repo.batches[sub.OperationID]
|
||||
if b.PollCount != i || !b.SubmittedAt.Equal(base) {
|
||||
t.Fatalf("重启丢失持久时点/次数: %+v", b)
|
||||
}
|
||||
if i < maxResultQueries && !repo.next[sub.OperationID].Equal(base.Add(firstResultDelay+time.Duration(i)*furtherResultDelay)) {
|
||||
t.Fatalf("排程未按submitted_at: %v", repo.next[sub.OperationID])
|
||||
}
|
||||
}
|
||||
if !repo.outcomes[sub.OperationID][0].Success || repo.batches[sub.OperationID].Status != "review" || upstream.submitCalls != 1 {
|
||||
t.Fatal("后续缺卡覆盖终态或重发")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNMappingRecoveryKeepsConsumedPollCount(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
sub.Status = "uncertain"
|
||||
sub.PollCount = 2
|
||||
sub.SubmittedAt = time.Now().Add(-8 * time.Minute)
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if !in.ReqData.RecoverOnly {
|
||||
t.Fatal("恢复应为只读")
|
||||
}
|
||||
return pendingResponse(in, 0), nil
|
||||
}}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := repo.batches[sub.OperationID]
|
||||
if b.JobID != "job-1" || b.PollCount != 3 || !repo.next[sub.OperationID].Equal(sub.SubmittedAt.Add(firstResultDelay+time.Duration(b.PollCount)*furtherResultDelay)) {
|
||||
t.Fatalf("恢复jobId清零或重排: %+v next=%v", b, repo.next[sub.OperationID])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNExplicitUncertainAndMissingNeverFailCard(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{submit: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return []model.SimBaseRes[model.SimCardApnChangeOutputItem]{{SimKey: in.SimKeys[0], Code: 9999, Message: "代理未确认", ResData: model.SimCardApnChangeOutputItem{Uncertain: true}}}, nil
|
||||
}}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(repo.outcomes) != 0 || repo.batches[sub.OperationID].Status != "uncertain" {
|
||||
t.Fatal("显式未知或缺卡不能转明确失败")
|
||||
}
|
||||
}
|
||||
|
||||
type capabilityFake struct {
|
||||
*asyncFake
|
||||
checkErr error
|
||||
}
|
||||
|
||||
func (f *capabilityFake) CheckSimApnAsyncCapability(context.Context, *model.SimCardApnChangeInput) error {
|
||||
return f.checkErr
|
||||
}
|
||||
|
||||
type requestNotSent struct{}
|
||||
|
||||
func (requestNotSent) Error() string { return "能力预检明确拒绝" }
|
||||
func (requestNotSent) APNRequestNotSent() bool { return true }
|
||||
|
||||
func TestDurableAPNCapabilityRefusalDoesNotClaimOrSpendQueries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &capabilityFake{asyncFake: &asyncFake{}, checkErr: errors.New("门禁关闭")}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err == nil {
|
||||
t.Fatal("应拒绝预检")
|
||||
}
|
||||
b := repo.batches[sub.OperationID]
|
||||
if b.Status != "prepared" || b.LeaseToken != "" || !b.SubmittedAt.IsZero() || b.PollCount != 0 || upstream.submitCalls != 0 {
|
||||
t.Fatalf("预检拒绝却已领取或提交: %+v", b)
|
||||
}
|
||||
upstream.checkErr = nil
|
||||
upstream.submit = func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return pendingResponse(in, 0), nil
|
||||
}
|
||||
if upstream.submitCalls != 0 || !repo.next[sub.OperationID].After(time.Now()) {
|
||||
t.Fatal("只读预检拒绝未持久退避")
|
||||
}
|
||||
repo.next[sub.OperationID] = time.Time{}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.submitCalls != 1 || repo.batches[sub.OperationID].Status != "accepted" {
|
||||
t.Fatal("门禁打开且退避到期后不能恢复首发")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNCapabilityRefusalDoesNotBlockReadOnlyQueries(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
status string
|
||||
jobID string
|
||||
recoverOnly bool
|
||||
}{
|
||||
{name: "accepted_with_job_id", status: "accepted", jobID: "job-1"},
|
||||
{name: "submitting", status: "submitting", recoverOnly: true},
|
||||
{name: "uncertain", status: "uncertain", recoverOnly: true},
|
||||
{name: "accepted_without_job_id", status: "accepted", recoverOnly: true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
sub.Status = tt.status
|
||||
sub.JobID = tt.jobID
|
||||
sub.SubmittedAt = time.Now().Add(-firstResultDelay)
|
||||
if err := repo.PrepareSubBatch(ctx, sub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upstream := &capabilityFake{
|
||||
checkErr: errors.New("门禁关闭"),
|
||||
asyncFake: &asyncFake{
|
||||
submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
t.Fatal("只读补查不得重新提交")
|
||||
return nil, nil
|
||||
},
|
||||
query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
if in.ReqData.OperationID != sub.OperationID || in.ReqData.QueryJobID != tt.jobID || in.ReqData.RecoverOnly != tt.recoverOnly {
|
||||
t.Fatalf("只读查询参数错误: %+v", in.ReqData)
|
||||
}
|
||||
return pendingResponse(in, 0), nil
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatalf("只读查询被提交门禁阻断: %v", err)
|
||||
}
|
||||
if upstream.queryCalls != 1 || upstream.submitCalls != 0 {
|
||||
t.Fatalf("应仅查询一次: query=%d submit=%d", upstream.queryCalls, upstream.submitCalls)
|
||||
}
|
||||
b := repo.batches[sub.OperationID]
|
||||
if b.Status != "accepted" || b.JobID != "job-1" || b.PollCount != 1 || b.LeaseToken != "" {
|
||||
t.Fatalf("只读查询结果未持久化: %+v", b)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNOnlyTypedUnsentCanReturnPrepared(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{submit: func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return nil, fmt.Errorf("预检竞态: %w", requestNotSent{})
|
||||
}}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := repo.batches[sub.OperationID]
|
||||
if b.Status != "prepared" || !b.SubmittedAt.IsZero() || b.PollCount != 0 {
|
||||
t.Fatalf("确定未发送应安全回退: %+v", b)
|
||||
}
|
||||
upstream.submit = func(*model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
return nil, context.DeadlineExceeded
|
||||
}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if repo.batches[sub.OperationID].Status != "uncertain" {
|
||||
t.Fatal("普通网络错误被误作确定未发送")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNFinishedFailure9999IsTerminal(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
sub.Status = "accepted"
|
||||
sub.JobID = "failed-job"
|
||||
sub.SubmittedAt = time.Now().Add(-time.Minute)
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
upstream := &asyncFake{query: func(in *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
var out []model.SimBaseRes[model.SimCardApnChangeOutputItem]
|
||||
for _, key := range in.SimKeys {
|
||||
out = append(out, model.SimBaseRes[model.SimCardApnChangeOutputItem]{SimKey: key, Code: 9999, Message: "任务已完成且明确失败"})
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, upstream, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if repo.batches[sub.OperationID].Status != "done" || len(repo.outcomes[sub.OperationID]) != 2 {
|
||||
t.Fatal("明确9999被误判待查")
|
||||
}
|
||||
for _, out := range repo.outcomes[sub.OperationID] {
|
||||
if out.Pending || out.Success || out.Code != 9999 {
|
||||
t.Fatalf("明确失败终态错误: %+v", out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNRealClientSecondProbeRejectsThenSubmitsOnce(t *testing.T) {
|
||||
oldAdapter := g.Cfg().GetAdapter()
|
||||
defer g.Cfg().SetAdapter(oldAdapter)
|
||||
ctx := context.Background()
|
||||
repo := newDurableFake()
|
||||
sub := fixtureSub()
|
||||
_ = repo.PrepareSubBatch(ctx, sub)
|
||||
probes, submits := 0, 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var in model.SimCardApnChangeInput
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
t.Error(err)
|
||||
http.Error(w, "bad", 400)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/sim/query/sim-apn-capability" {
|
||||
probes++
|
||||
if probes == 2 {
|
||||
http.Error(w, "第二次探测拒绝", 404)
|
||||
return
|
||||
}
|
||||
type capability struct {
|
||||
ProtocolVersion int `json:"protocolVersion"`
|
||||
}
|
||||
out := make([]model.SimBaseRes[capability], 0, len(in.SimKeys))
|
||||
for _, key := range in.SimKeys {
|
||||
out = append(out, model.SimBaseRes[capability]{SimKey: key, ResData: capability{ProtocolVersion: 1}})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": out})
|
||||
return
|
||||
}
|
||||
submits++
|
||||
out := pendingResponse(&in, 0)
|
||||
for i := range out {
|
||||
out[i].ResData.Pending = false
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": out})
|
||||
}))
|
||||
defer server.Close()
|
||||
adapter, err := gcfg.NewAdapterContent(fmt.Sprintf(`{"upstream":{"proxy":{"address":%q,"timeout":"1s","apnAsyncProtocolVersion":1}}}`, strings.TrimPrefix(server.URL, "http://")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
client := upstreamclient.New()
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, client, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := repo.batches[sub.OperationID]
|
||||
if probes != 2 || submits != 0 || b.Status != "prepared" || !b.SubmittedAt.IsZero() || b.LeaseToken != "" {
|
||||
t.Fatalf("双probe竞态烧掉意图: probes=%d submits=%d sub=%+v", probes, submits, b)
|
||||
}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, client, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if submits != 1 || repo.batches[sub.OperationID].Status != "done" {
|
||||
t.Fatalf("门禁恢复未唯一首发: submits=%d sub=%+v", submits, repo.batches[sub.OperationID])
|
||||
}
|
||||
if err := (&executor{}).processSubBatch(ctx, repo, client, sub.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if submits != 1 {
|
||||
t.Fatal("完成后重复首发")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsyncSubBatchStableAndChannelIsolated(t *testing.T) {
|
||||
op, _ := OperationByBatchType(consts.BatchTypeDataStart)
|
||||
task := &Task{Batch: Batch{BatchNo: "P-2", BatchType: consts.BatchTypeDataStart}, Operation: op}
|
||||
groups := map[string][]*Detail{"a": {}, "b": {}}
|
||||
for i := 0; i < 203; i++ {
|
||||
key := "a"
|
||||
if i >= 201 {
|
||||
key = "b"
|
||||
}
|
||||
groups[key] = append(groups[key], &Detail{ID: int64(i + 1), ICCID: fmt.Sprint(i), MSISDN: fmt.Sprint(i), ChanKey: key})
|
||||
}
|
||||
total := 0
|
||||
for key, details := range groups {
|
||||
for _, chunk := range splitAsyncDetails(details) {
|
||||
if len(chunk) > 100 {
|
||||
t.Fatal("超过100卡")
|
||||
}
|
||||
first := newSubBatch(task, chunk)
|
||||
second := newSubBatch(task, chunk)
|
||||
if first.OperationID != second.OperationID {
|
||||
t.Fatal("operationId不稳定")
|
||||
}
|
||||
for _, sim := range first.Request.SimKeys {
|
||||
if sim.ChanKey != key {
|
||||
t.Fatal("跨通道子批")
|
||||
}
|
||||
}
|
||||
total++
|
||||
}
|
||||
}
|
||||
if total != 5 {
|
||||
t.Fatalf("sub batches=%d want5", total)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
type executor struct {
|
||||
repository Repository
|
||||
upstream Upstream
|
||||
channelSlots sync.Map
|
||||
}
|
||||
|
||||
func (e *executor) ClaimsProcessingReentry() bool { return true }
|
||||
func (e *executor) CanExecuteProcessing(ctx context.Context, main *entity.BatchMain) (bool, error) {
|
||||
repo, durable := e.repository.(SubBatchRepository)
|
||||
_, async := e.upstream.(AsyncUpstream)
|
||||
if !durable || !async {
|
||||
return false, nil
|
||||
}
|
||||
return repo.HasDurableParent(ctx, strings.TrimSpace(main.BatchNo))
|
||||
}
|
||||
func (e *executor) ProtectExecutionFailure(ctx context.Context, batchNo string, batchType int, reason string) (bool, error) {
|
||||
repo, durable := e.repository.(SubBatchRepository)
|
||||
_, async := e.upstream.(AsyncUpstream)
|
||||
if !durable || !async {
|
||||
return false, nil
|
||||
}
|
||||
return true, repo.ProtectExecutionFailure(ctx, batchNo, reason)
|
||||
}
|
||||
|
||||
func NewExecutor(repository Repository, upstream Upstream) Executor {
|
||||
return &executor{repository: repository, upstream: upstream}
|
||||
}
|
||||
|
||||
func (e *executor) Execute(ctx context.Context, batchNo string) error {
|
||||
if durable, ok := e.repository.(SubBatchRepository); ok {
|
||||
if _, ok := e.upstream.(AsyncUpstream); ok {
|
||||
// Earlier child intents were durably frozen: a re-delivered parent cannot re-split running rows.
|
||||
return batch.Retryable(durable.ResumeOrPrepare(ctx, strings.TrimSpace(batchNo), func(prepareCtx context.Context) error {
|
||||
return e.executeFresh(prepareCtx, batchNo)
|
||||
}))
|
||||
}
|
||||
}
|
||||
return e.executeFresh(ctx, batchNo)
|
||||
}
|
||||
|
||||
func (e *executor) executeFresh(ctx context.Context, batchNo string) error {
|
||||
task, err := e.repository.Load(ctx, strings.TrimSpace(batchNo))
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "load data service batch")
|
||||
}
|
||||
if task.Operation.BatchType == consts.BatchTypeDataCut {
|
||||
return e.executeTrafficCutoff(ctx, task)
|
||||
}
|
||||
if !validateTask(task) {
|
||||
if err = e.repository.SaveValidationFailure(ctx, task); err != nil {
|
||||
return gerror.Wrap(err, "save data service validation failure")
|
||||
}
|
||||
if _, durable := e.repository.(SubBatchRepository); durable {
|
||||
if _, async := e.upstream.(AsyncUpstream); async {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return gerror.Wrap(e.repository.Finalize(ctx, task), "finalize data service validation failure")
|
||||
}
|
||||
|
||||
return e.executeValidated(ctx, task)
|
||||
}
|
||||
|
||||
func (e *executor) executeTrafficCutoff(ctx context.Context, task *Task) error {
|
||||
if !validateTrafficCutoffTask(task) {
|
||||
if err := e.repository.SaveValidationFailure(ctx, task); err != nil {
|
||||
return gerror.Wrap(err, "save traffic cutoff validation failure")
|
||||
}
|
||||
if _, durable := e.repository.(SubBatchRepository); durable {
|
||||
if _, async := e.upstream.(AsyncUpstream); async {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return gerror.Wrap(e.repository.Finalize(ctx, task), "finalize traffic cutoff validation failure")
|
||||
}
|
||||
if err := e.refreshDataServiceState(ctx, task); err != nil {
|
||||
return gerror.Wrap(err, "refresh data service state")
|
||||
}
|
||||
if err := e.saveInvalidTrafficCutoffRows(ctx, task); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, durable := e.repository.(SubBatchRepository); durable {
|
||||
if _, async := e.upstream.(AsyncUpstream); async {
|
||||
// 本地disabled不能证明前序开通已终结；持久化路径统一经过卡级提交屏障。
|
||||
return e.executeValidated(ctx, task)
|
||||
}
|
||||
}
|
||||
if err := e.saveIdempotentCutoffSkips(ctx, task); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.executeValidated(ctx, task)
|
||||
}
|
||||
|
||||
func (e *executor) executeValidated(ctx context.Context, task *Task) error {
|
||||
if durable, ok := e.repository.(SubBatchRepository); ok {
|
||||
if async, ok := e.upstream.(AsyncUpstream); ok {
|
||||
seen := map[string]bool{}
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil || detail.Reason != "" || (detail.DetailStatus != consts.BatchDetailStatusPending && detail.DetailStatus != consts.BatchDetailStatusRunning) {
|
||||
continue
|
||||
}
|
||||
if seen[detail.ICCID] {
|
||||
resolved, err := e.repository.SaveResult(ctx, task, detail, Outcome{Message: "重复ICCID，未提交上游"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
continue
|
||||
}
|
||||
seen[detail.ICCID] = true
|
||||
}
|
||||
return e.prepareAndSubmit(ctx, task, durable, async)
|
||||
}
|
||||
}
|
||||
groups := groupByChanKey(task.Operation, task.Details)
|
||||
keys := make([]string, 0, len(groups))
|
||||
for chanKey := range groups {
|
||||
keys = append(keys, chanKey)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, chanKey := range keys {
|
||||
if err := e.changeGroup(ctx, task, groups[chanKey]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return gerror.Wrap(e.repository.Finalize(ctx, task), "finalize data service batch")
|
||||
}
|
||||
|
||||
type dataServiceStateRefresher interface {
|
||||
RefreshDataServiceState(ctx context.Context, task *Task) error
|
||||
}
|
||||
|
||||
func (e *executor) refreshDataServiceState(ctx context.Context, task *Task) error {
|
||||
refresher, ok := e.repository.(dataServiceStateRefresher)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return refresher.RefreshDataServiceState(ctx, task)
|
||||
}
|
||||
|
||||
func (e *executor) saveIdempotentCutoffSkips(ctx context.Context, task *Task) error {
|
||||
if task == nil || task.Operation.BatchType != consts.BatchTypeDataCut {
|
||||
return nil
|
||||
}
|
||||
for _, detail := range task.Details {
|
||||
if !skipUpstreamForIdempotentCutoff(task.Operation, detail) {
|
||||
continue
|
||||
}
|
||||
resolved, saveErr := e.repository.SaveResult(ctx, task, detail, Outcome{
|
||||
Success: true,
|
||||
Message: "数据服务已关闭，未发起上游请求",
|
||||
})
|
||||
if saveErr != nil {
|
||||
return gerror.Wrapf(saveErr, "save data service detail %d idempotent skip result", detail.ID)
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateTask(task *Task) bool {
|
||||
if task == nil || len(task.Details) == 0 {
|
||||
if task != nil && task.FailureReason == "" {
|
||||
task.FailureReason = "数据开停批次没有有效明细"
|
||||
}
|
||||
return false
|
||||
}
|
||||
hasFailure := false
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil {
|
||||
hasFailure = true
|
||||
continue
|
||||
}
|
||||
validateDetail(task.Operation, detail)
|
||||
if detail.Reason != "" {
|
||||
hasFailure = true
|
||||
}
|
||||
}
|
||||
if !hasFailure {
|
||||
task.ValidationOK = true
|
||||
for _, detail := range task.Details {
|
||||
if detail != nil {
|
||||
detail.ValidationPassed = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, detail := range task.Details {
|
||||
if detail != nil && detail.Reason == "" {
|
||||
detail.Reason = "同批次存在校验失败卡，本卡未请求上游"
|
||||
}
|
||||
}
|
||||
task.FailureReason = "批次校验未通过，未发起上游请求"
|
||||
return false
|
||||
}
|
||||
|
||||
func validateTrafficCutoffTask(task *Task) bool {
|
||||
if task == nil || len(task.Details) == 0 {
|
||||
if task != nil {
|
||||
task.FailureReason = "流量断网批次无有效明细"
|
||||
}
|
||||
return false
|
||||
}
|
||||
hasSuccess := false
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil {
|
||||
continue
|
||||
}
|
||||
validateDetail(task.Operation, detail)
|
||||
detail.ValidationPassed = detail.Reason == ""
|
||||
if detail.ValidationPassed {
|
||||
hasSuccess = true
|
||||
}
|
||||
}
|
||||
if hasSuccess {
|
||||
task.ValidationOK = true
|
||||
if failed := countFailedDetails(task); failed > 0 {
|
||||
task.FailureReason = strconv.Itoa(failed) + " 张卡未通过校验"
|
||||
} else {
|
||||
task.FailureReason = ""
|
||||
}
|
||||
return true
|
||||
}
|
||||
task.FailureReason = "流量断网批次校验未通过，未发起上游请求"
|
||||
return false
|
||||
}
|
||||
|
||||
func countFailedDetails(task *Task) int {
|
||||
if task == nil {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil || detail.Reason != "" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func validateDetail(_ Operation, detail *Detail) {
|
||||
if detail.Reason != "" {
|
||||
return
|
||||
}
|
||||
if detail.Card == nil {
|
||||
detail.Reason = "ICCID不存在或不属于当前租户"
|
||||
return
|
||||
}
|
||||
detail.MSISDN = strings.TrimSpace(detail.Card.MSISDN)
|
||||
detail.IMSI = strings.TrimSpace(detail.Card.IMSI)
|
||||
detail.ChanKey = strings.TrimSpace(detail.Card.ChanKey)
|
||||
detail.StatusBefore = detail.Card.DataServiceStatus
|
||||
detail.StatusAfter = detail.Card.DataServiceStatus
|
||||
|
||||
if !detail.Authorized {
|
||||
detail.Reason = "ICCID不在当前操作员授权范围内"
|
||||
return
|
||||
}
|
||||
var reasons []string
|
||||
if detail.ChanKey == "" {
|
||||
reasons = append(reasons, "ICCID缺少通道标识")
|
||||
}
|
||||
if detail.InputMSISDN != "" && detail.InputMSISDN != detail.MSISDN {
|
||||
reasons = append(reasons, "MSISDN与卡档案不一致")
|
||||
}
|
||||
if detail.InputIMSI != "" && detail.InputIMSI != detail.IMSI {
|
||||
reasons = append(reasons, "IMSI与卡档案不一致")
|
||||
}
|
||||
detail.Reason = strings.Join(reasons, "；")
|
||||
}
|
||||
|
||||
func groupByChanKey(operation Operation, details []*Detail) map[string][]*Detail {
|
||||
groups := make(map[string][]*Detail)
|
||||
for _, detail := range details {
|
||||
if detail == nil || detail.Reason != "" {
|
||||
continue
|
||||
}
|
||||
if operation.BatchType == consts.BatchTypeDataCut && (detail == nil || !detail.ValidationPassed || detail.Reason != "") {
|
||||
continue
|
||||
}
|
||||
if skipUpstreamForIdempotentCutoff(operation, detail) {
|
||||
continue
|
||||
}
|
||||
groups[detail.ChanKey] = append(groups[detail.ChanKey], detail)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func (e *executor) saveInvalidTrafficCutoffRows(ctx context.Context, task *Task) error {
|
||||
if task == nil {
|
||||
return nil
|
||||
}
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil || detail.ValidationPassed || detail.Reason == "" {
|
||||
continue
|
||||
}
|
||||
resolved, saveErr := e.repository.SaveResult(ctx, task, detail, Outcome{
|
||||
Message: detail.Reason,
|
||||
})
|
||||
if saveErr != nil {
|
||||
return gerror.Wrapf(saveErr, "save traffic cutoff detail %d validation result", detail.ID)
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func skipUpstreamForIdempotentCutoff(operation Operation, detail *Detail) bool {
|
||||
return operation.BatchType == consts.BatchTypeDataCut &&
|
||||
detail != nil &&
|
||||
detail.ValidationPassed &&
|
||||
detail.StatusBefore == consts.SimDataServiceStatusDisabled &&
|
||||
detail.StatusAfter == consts.SimDataServiceStatusDisabled
|
||||
}
|
||||
|
||||
// changeGroup 把同一通道下的明细切成子批次并发调用上游，并在**每个子批次拿到结果后立即落库该子批的明细**。
|
||||
//
|
||||
// 不再等所有子批次跑完才统一写库：大批量（如 2 万张 = 200 个子批）执行时，执行日志可以按子批
|
||||
// 实时反映进度，而不是在整个批次结束前始终全部停留在"执行中"。子批之间相互隔离——某个子批
|
||||
// 落库失败只记录错误、不影响其他子批继续办理与落库。
|
||||
func (e *executor) changeGroup(ctx context.Context, task *Task, details []*Detail) error {
|
||||
subBatches := splitDetails(details, consts.BatchSubBatchSize)
|
||||
|
||||
sem := make(chan struct{}, consts.BatchSubBatchConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
|
||||
for _, sub := range subBatches {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(batch []*Detail) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
if err := e.changeAndSaveSubBatch(ctx, task, batch); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}(sub)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// changeAndSaveSubBatch 处理单个子批次：构建请求、调用上游，然后逐条落库本子批的明细。
|
||||
// 明细结果只在所属子批次落库，天然做到"查到一个子批就落库一个子批"。
|
||||
func (e *executor) changeAndSaveSubBatch(ctx context.Context, task *Task, details []*Detail) error {
|
||||
for index, outcome := range e.changeSubBatch(ctx, task, details) {
|
||||
detail := details[index]
|
||||
resolved, saveErr := e.repository.SaveResult(ctx, task, detail, outcome)
|
||||
if saveErr != nil {
|
||||
return gerror.Wrapf(saveErr, "save data service detail %d result", detail.ID)
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// changeSubBatch 处理单个子批次：构建请求并调用上游，返回与该子批次明细等长的 Outcome 切片。
|
||||
func (e *executor) changeSubBatch(ctx context.Context, task *Task, details []*Detail) []Outcome {
|
||||
input := &model.SimCardApnChangeInput{
|
||||
SimKeys: make([]*model.SimKey, 0, len(details)),
|
||||
ReqData: model.SimCardApnChangeInputItem{OperType: task.Operation.ApnOperType, ApnName: task.ApnName},
|
||||
}
|
||||
for _, detail := range details {
|
||||
input.SimKeys = append(input.SimKeys, &model.SimKey{ChanKey: detail.ChanKey, ICCID: detail.ICCID, MSISDN: detail.MSISDN, IMSI: detail.IMSI})
|
||||
}
|
||||
response, err := e.upstream.ChangeSimApnFunction(ctx, input)
|
||||
if err != nil {
|
||||
outcomes := make([]Outcome, len(details))
|
||||
for index := range outcomes {
|
||||
outcomes[index].Message = "请求上游失败：" + err.Error()
|
||||
}
|
||||
return outcomes
|
||||
}
|
||||
return mapOutcomes(details, response)
|
||||
}
|
||||
|
||||
// splitDetails 按固定大小把明细切成有序的子批次；空输入返回空切片。
|
||||
func splitDetails(details []*Detail, size int) [][]*Detail {
|
||||
if size <= 0 {
|
||||
size = 1
|
||||
}
|
||||
batches := make([][]*Detail, 0, (len(details)+size-1)/size)
|
||||
for i := 0; i < len(details); i += size {
|
||||
end := i + size
|
||||
if end > len(details) {
|
||||
end = len(details)
|
||||
}
|
||||
batches = append(batches, details[i:end])
|
||||
}
|
||||
return batches
|
||||
}
|
||||
|
||||
func mapOutcomes(details []*Detail, response []model.SimBaseRes[model.SimCardApnChangeOutputItem]) []Outcome {
|
||||
outcomes := make([]Outcome, len(details))
|
||||
used := make([]bool, len(response))
|
||||
for detailIndex, detail := range details {
|
||||
matched := -1
|
||||
for responseIndex, item := range response {
|
||||
if !used[responseIndex] && sameSim(detail, item.SimKey) {
|
||||
matched = responseIndex
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched < 0 && detailIndex < len(response) && !used[detailIndex] && response[detailIndex].SimKey == nil {
|
||||
matched = detailIndex
|
||||
}
|
||||
if matched < 0 {
|
||||
outcomes[detailIndex] = Outcome{Message: "上游未返回该卡处理结果"}
|
||||
continue
|
||||
}
|
||||
used[matched] = true
|
||||
item := response[matched]
|
||||
message := strings.TrimSpace(item.Message)
|
||||
if item.Code != 0 && message == "" {
|
||||
message = "上游处理失败"
|
||||
}
|
||||
outcomes[detailIndex] = Outcome{Success: item.Code == 0, Code: item.Code, Message: message, OrderNum: strings.TrimSpace(item.ResData.OrderNum)}
|
||||
}
|
||||
return outcomes
|
||||
}
|
||||
|
||||
func sameSim(detail *Detail, key *model.SimKey) bool {
|
||||
if detail == nil || key == nil {
|
||||
return false
|
||||
}
|
||||
if value := strings.TrimSpace(key.ICCID); value != "" {
|
||||
return value == detail.ICCID
|
||||
}
|
||||
if value := strings.TrimSpace(key.MSISDN); value != "" {
|
||||
return value == detail.MSISDN
|
||||
}
|
||||
if value := strings.TrimSpace(key.IMSI); value != "" {
|
||||
return value == detail.IMSI
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func OperationByBatchType(batchType int) (Operation, error) {
|
||||
switch batchType {
|
||||
case consts.BatchTypeDataStart:
|
||||
return Operation{BatchType: batchType, Name: "dataStart", ApnOperType: consts.SimApnOperTypeOpen, TargetStatus: consts.SimDataServiceStatusEnabled}, nil
|
||||
case consts.BatchTypeDataCut:
|
||||
return Operation{BatchType: batchType, Name: "trafficCutoff", ApnOperType: consts.SimApnOperTypeStop, TargetStatus: consts.SimDataServiceStatusDisabled}, nil
|
||||
case consts.BatchTypeDataStop:
|
||||
return Operation{BatchType: batchType, Name: "dataStop", ApnOperType: consts.SimApnOperTypeStop, TargetStatus: consts.SimDataServiceStatusDisabled}, nil
|
||||
default:
|
||||
return Operation{}, gerror.Newf("unsupported data service batch type %d", batchType)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
func TestAccuracyInvalidDuplicateKeepsValidTrafficCutoffRow(t *testing.T) {
|
||||
for _, invalidFirst := range []bool{true, false} {
|
||||
for _, invalidInput := range []string{"MSISDN", "IMSI", "authorization", "existing"} {
|
||||
t.Run(fmt.Sprintf("%s/invalidFirst=%v", invalidInput, invalidFirst), func(t *testing.T) {
|
||||
operation, err := OperationByBatchType(107)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
card := &Card{ID: 10, ICCID: "same-card", MSISDN: "1001", IMSI: "2001", ChanKey: "channel", DataServiceStatus: 1}
|
||||
invalid := &Detail{ID: 1, ICCID: card.ICCID, Authorized: true, Card: card}
|
||||
var wantReason string
|
||||
switch invalidInput {
|
||||
case "MSISDN":
|
||||
invalid.InputMSISDN = "wrong"
|
||||
wantReason = "MSISDN与卡档案不一致"
|
||||
case "IMSI":
|
||||
invalid.InputIMSI = "wrong"
|
||||
wantReason = "IMSI与卡档案不一致"
|
||||
case "authorization":
|
||||
invalid.Authorized = false
|
||||
wantReason = "ICCID不在当前操作员授权范围内"
|
||||
case "existing":
|
||||
wantReason = "导入行格式校验失败"
|
||||
invalid.Reason = wantReason
|
||||
}
|
||||
valid := &Detail{ID: 2, ICCID: card.ICCID, Authorized: true, Card: card}
|
||||
otherInvalid := &Detail{ID: 3, ICCID: card.ICCID, Reason: "该行独立校验失败"}
|
||||
duplicate := &Detail{ID: 4, ICCID: card.ICCID, Authorized: true, Card: card}
|
||||
details := []*Detail{invalid, valid, otherInvalid, duplicate}
|
||||
if !invalidFirst {
|
||||
details[0], details[1] = details[1], details[0]
|
||||
}
|
||||
task := &Task{Batch: Batch{BatchNo: "ACCURACY-CUT", BatchType: 107}, Operation: operation, ApnName: "CMIOT", Details: details}
|
||||
base := &accuracyRepository{task: task, saved: map[int64]Outcome{}}
|
||||
repo := &accuracyDurableRepository{accuracyRepository: base}
|
||||
if err := NewExecutor(repo, &accuracyNoUpstream{t: t}).Execute(context.Background(), task.Batch.BatchNo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(repo.prepared) != 1 || len(repo.prepared[0].DetailIDs) != 1 || repo.prepared[0].DetailIDs[0] != valid.ID {
|
||||
t.Fatalf("有效行未独立冻结: %+v", repo.prepared)
|
||||
}
|
||||
if valid.Reason != "" || valid.DetailStatus == consts.BatchDetailStatusFailed {
|
||||
t.Fatalf("有效行被无效重复行阻断: %+v", valid)
|
||||
}
|
||||
for _, check := range []struct {
|
||||
detail *Detail
|
||||
reason string
|
||||
}{{invalid, wantReason}, {otherInvalid, "该行独立校验失败"}, {duplicate, "重复ICCID，未提交上游"}} {
|
||||
outcome, ok := base.saved[check.detail.ID]
|
||||
if !ok || outcome.Success || outcome.Message != check.reason || check.detail.Reason != check.reason {
|
||||
t.Fatalf("逐行原因被改写: id=%d detail=%+v outcome=%+v", check.detail.ID, check.detail, outcome)
|
||||
}
|
||||
}
|
||||
if repo.prepared[0].Request.ReqData.OperType != "1" || operation.TargetStatus != 0 {
|
||||
t.Fatalf("107开停方向倒置: %+v", repo.prepared[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccuracyFrozenRequestsKeepAPNDirection(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
batchType int
|
||||
operType string
|
||||
target int
|
||||
}{{107, "1", 0}, {109, "0", 1}, {110, "1", 0}} {
|
||||
t.Run(fmt.Sprint(tc.batchType), func(t *testing.T) {
|
||||
operation, err := OperationByBatchType(tc.batchType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
card := &Card{ID: 10, ICCID: "card", MSISDN: "1001", ChanKey: "channel", DataServiceStatus: 1 - tc.target}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "ACCURACY-DIRECTION", BatchType: tc.batchType}, Operation: operation, ApnName: "CMIOT",
|
||||
Details: []*Detail{{ID: 1, ICCID: card.ICCID, Authorized: true, Card: card}, {ID: 2, ICCID: card.ICCID, Authorized: true, Card: card}},
|
||||
}
|
||||
repo := &accuracyDurableRepository{accuracyRepository: &accuracyRepository{task: task, saved: map[int64]Outcome{}}}
|
||||
if err := NewExecutor(repo, &accuracyNoUpstream{t: t}).Execute(context.Background(), task.Batch.BatchNo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(repo.prepared) != 1 || len(repo.prepared[0].DetailIDs) != 1 || repo.prepared[0].DetailIDs[0] != 1 {
|
||||
t.Fatalf("同ICCID未只保留首个有效行: %+v", repo.prepared)
|
||||
}
|
||||
if operation.TargetStatus != tc.target || repo.prepared[0].Request.ReqData.OperType != tc.operType {
|
||||
t.Fatalf("批次%d开停方向错误: operation=%+v request=%+v", tc.batchType, operation, repo.prepared[0].Request)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccuracyEmptyBatchKeepsValidationReason(t *testing.T) {
|
||||
for _, batchType := range []int{107, 109, 110} {
|
||||
t.Run(fmt.Sprint(batchType), func(t *testing.T) {
|
||||
operation, err := OperationByBatchType(batchType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{Batch: Batch{BatchNo: "ACCURACY-EMPTY", BatchType: batchType}, Operation: operation}
|
||||
repo := &accuracyRepository{task: task}
|
||||
if err := NewExecutor(repo, &accuracyNoUpstream{t: t}).Execute(context.Background(), task.Batch.BatchNo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !repo.validationSaved || !repo.finalized || task.ValidationOK || task.FailureReason == "" {
|
||||
t.Fatalf("空明细校验失败原因缺失: task=%+v saved=%v finalized=%v", task, repo.validationSaved, repo.finalized)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type accuracyRepository struct {
|
||||
task *Task
|
||||
saved map[int64]Outcome
|
||||
validationSaved bool
|
||||
finalized bool
|
||||
}
|
||||
|
||||
func (r *accuracyRepository) Load(context.Context, string) (*Task, error) { return r.task, nil }
|
||||
func (r *accuracyRepository) SaveValidationFailure(context.Context, *Task) error {
|
||||
r.validationSaved = true
|
||||
return nil
|
||||
}
|
||||
func (r *accuracyRepository) SaveResult(_ context.Context, _ *Task, detail *Detail, outcome Outcome) (Outcome, error) {
|
||||
r.saved[detail.ID] = outcome
|
||||
return outcome, nil
|
||||
}
|
||||
func (r *accuracyRepository) Finalize(context.Context, *Task) error {
|
||||
r.finalized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
type accuracyDurableRepository struct {
|
||||
*accuracyRepository
|
||||
SubBatchRepository
|
||||
prepared []SubBatch
|
||||
}
|
||||
|
||||
func (r *accuracyDurableRepository) ResumeOrPrepare(ctx context.Context, _ string, prepare func(context.Context) error) error {
|
||||
return prepare(ctx)
|
||||
}
|
||||
func (r *accuracyDurableRepository) PrepareSubBatch(_ context.Context, sub SubBatch) error {
|
||||
r.prepared = append(r.prepared, sub)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAccuracyCanExecuteProcessingRequiresExistingParent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
parent bool
|
||||
readErr error
|
||||
wantOK bool
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "旧Processing无凭据不可接管"},
|
||||
{name: "新批次ready0已持久化凭据", parent: true, wantOK: true},
|
||||
{name: "缺少迁移查询失败安全拒绝", readErr: errors.New("no such table: data_operation_parent"), wantErr: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := &accuracyParentRepository{accuracyDurableRepository: &accuracyDurableRepository{accuracyRepository: &accuracyRepository{}}, present: tc.parent, readErr: tc.readErr}
|
||||
e := NewExecutor(repo, &accuracyNoUpstream{t: t}).(*executor)
|
||||
ok, err := e.CanExecuteProcessing(ctx, &entity.BatchMain{BatchNo: " APN-EXISTING "})
|
||||
if ok != tc.wantOK || (err != nil) != tc.wantErr || repo.lookups != 1 || repo.batchNo != "APN-EXISTING" {
|
||||
t.Fatalf("parent gate ok=%v err=%v lookups=%d batchNo=%q", ok, err, repo.lookups, repo.batchNo)
|
||||
}
|
||||
})
|
||||
}
|
||||
base := &accuracyRepository{}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
repo Repository
|
||||
upstream Upstream
|
||||
}{
|
||||
{name: "缺少持久仓储", repo: base, upstream: &accuracyNoUpstream{t: t}},
|
||||
{name: "仅同步上游", repo: &accuracyParentRepository{accuracyDurableRepository: &accuracyDurableRepository{accuracyRepository: base}}, upstream: &accuracySyncOnlyUpstream{}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
e := NewExecutor(tc.repo, tc.upstream).(*executor)
|
||||
ok, err := e.CanExecuteProcessing(ctx, &entity.BatchMain{BatchNo: "APN-EXISTING"})
|
||||
if ok || err != nil {
|
||||
t.Fatalf("不支持持久异步不得接管Processing: ok=%v err=%v", ok, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccuracyDurableRepositoryWithSyncUpstreamFinalizesValidationFailure(t *testing.T) {
|
||||
for _, batchType := range []int{107, 109, 110} {
|
||||
t.Run(fmt.Sprint(batchType), func(t *testing.T) {
|
||||
operation, err := OperationByBatchType(batchType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{Batch: Batch{BatchNo: "SYNC-FAILED", BatchType: batchType}, Operation: operation}
|
||||
base := &accuracyRepository{task: task}
|
||||
repo := &accuracyDurableRepository{accuracyRepository: base}
|
||||
if err := NewExecutor(repo, &accuracySyncOnlyUpstream{}).Execute(context.Background(), task.Batch.BatchNo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !base.validationSaved || !base.finalized || task.FailureReason == "" {
|
||||
t.Fatalf("同步兜底遗漏校验失败父批: saved=%v finalized=%v task=%+v", base.validationSaved, base.finalized, task)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type accuracyParentRepository struct {
|
||||
*accuracyDurableRepository
|
||||
present bool
|
||||
readErr error
|
||||
lookups int
|
||||
batchNo string
|
||||
}
|
||||
|
||||
func (r *accuracyParentRepository) HasDurableParent(_ context.Context, batchNo string) (bool, error) {
|
||||
r.lookups++
|
||||
r.batchNo = batchNo
|
||||
return r.present, r.readErr
|
||||
}
|
||||
|
||||
type accuracySyncOnlyUpstream struct{}
|
||||
|
||||
func (*accuracySyncOnlyUpstream) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
panic("校验失败不得发运营商请求")
|
||||
}
|
||||
|
||||
type accuracyNoUpstream struct{ t *testing.T }
|
||||
|
||||
func (u *accuracyNoUpstream) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.t.Fatal("该回归不得调用同步上游")
|
||||
return nil, nil
|
||||
}
|
||||
func (u *accuracyNoUpstream) SubmitSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.t.Fatal("该回归只冻结意图，不得提交上游")
|
||||
return nil, nil
|
||||
}
|
||||
func (u *accuracyNoUpstream) QuerySimApnFunctionResult(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.t.Fatal("该回归不得查询上游")
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,543 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func TestExecuteDataStartSuccess(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "D-1"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusDisabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{response: []model.SimBaseRes[model.SimCardApnChangeOutputItem]{
|
||||
{Code: 0, Message: "ok", SimKey: &model.SimKey{ICCID: "iccid-1"}, ResData: model.SimCardApnChangeOutputItem{OrderNum: "order-1"}},
|
||||
}}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "D-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1", upstream.calls)
|
||||
}
|
||||
if upstream.input.ReqData.OperType != consts.SimApnOperTypeOpen || upstream.input.ReqData.ApnName != consts.SimApnNameDefault {
|
||||
t.Fatalf("reqData = %+v", upstream.input.ReqData)
|
||||
}
|
||||
if len(repo.saved) != 1 || !repo.saved[0].Success || repo.saved[0].OrderNum != "order-1" {
|
||||
t.Fatalf("saved = %+v", repo.saved)
|
||||
}
|
||||
if task.Details[0].StatusBefore != consts.SimDataServiceStatusDisabled {
|
||||
t.Fatalf("StatusBefore = %d", task.Details[0].StatusBefore)
|
||||
}
|
||||
if !repo.finalized {
|
||||
t.Fatal("batch was not finalized")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteDataStopSuccess(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataStop)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "D-2"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{response: []model.SimBaseRes[model.SimCardApnChangeOutputItem]{
|
||||
{Code: 0, Message: "ok", SimKey: &model.SimKey{ICCID: "iccid-1"}},
|
||||
}}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "D-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 1 || upstream.input.ReqData.OperType != consts.SimApnOperTypeStop {
|
||||
t.Fatalf("upstream calls = %d, operType = %s", upstream.calls, upstream.input.ReqData.OperType)
|
||||
}
|
||||
if len(repo.saved) != 1 || !repo.saved[0].Success {
|
||||
t.Fatalf("saved = %+v", repo.saved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTrafficCutoffSuccess(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataCut)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "DCT-1"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{response: []model.SimBaseRes[model.SimCardApnChangeOutputItem]{
|
||||
{Code: 0, Message: "ok", SimKey: &model.SimKey{ICCID: "iccid-1"}},
|
||||
}}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "DCT-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 1 || upstream.input.ReqData.OperType != consts.SimApnOperTypeStop {
|
||||
t.Fatalf("upstream calls = %d, operType = %s", upstream.calls, upstream.input.ReqData.OperType)
|
||||
}
|
||||
if len(repo.saved) != 1 || !repo.saved[0].Success {
|
||||
t.Fatalf("saved = %+v", repo.saved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTrafficCutoffRefreshesCurrentStatusBeforeUpstream(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataCut)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "DCT-3"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{
|
||||
task: task,
|
||||
refresh: func(task *Task) {
|
||||
if len(task.Details) == 0 || task.Details[0] == nil || task.Details[0].Card == nil {
|
||||
return
|
||||
}
|
||||
task.Details[0].Card.DataServiceStatus = consts.SimDataServiceStatusDisabled
|
||||
task.Details[0].StatusBefore = consts.SimDataServiceStatusDisabled
|
||||
task.Details[0].StatusAfter = consts.SimDataServiceStatusDisabled
|
||||
},
|
||||
}
|
||||
upstream := &fakeUpstream{responseError: errors.New("must not be called")}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "DCT-3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 0 {
|
||||
t.Fatalf("upstream calls = %d, want 0", upstream.calls)
|
||||
}
|
||||
if len(repo.saved) != 1 || !repo.saved[0].Success {
|
||||
t.Fatalf("saved = %+v", repo.saved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTrafficCutoffSkipsAlreadyDisabledCard(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataCut)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "DCT-2"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusDisabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{responseError: errors.New("must not be called")}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "DCT-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 0 {
|
||||
t.Fatalf("upstream calls = %d, want 0", upstream.calls)
|
||||
}
|
||||
if len(repo.saved) != 1 || !repo.saved[0].Success {
|
||||
t.Fatalf("saved = %+v", repo.saved)
|
||||
}
|
||||
if task.Details[0].Reason != "" {
|
||||
t.Fatalf("already disabled cutoff should be settled as success, reason = %q", task.Details[0].Reason)
|
||||
}
|
||||
if !repo.finalized {
|
||||
t.Fatal("batch was not finalized")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTrafficCutoffValidationFailureAllowsValidRows(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataCut)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "DCT-4"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: false, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
{ID: 2, ICCID: "iccid-2", Authorized: true, Card: &Card{ID: 12, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{response: []model.SimBaseRes[model.SimCardApnChangeOutputItem]{
|
||||
{Code: 0, Message: "ok", SimKey: &model.SimKey{ICCID: "iccid-2"}},
|
||||
}}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "DCT-4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1", upstream.calls)
|
||||
}
|
||||
if len(upstream.input.SimKeys) != 1 || upstream.input.SimKeys[0].ICCID != "iccid-2" {
|
||||
t.Fatalf("upstream input = %+v", upstream.input.SimKeys)
|
||||
}
|
||||
if len(repo.saved) != 2 {
|
||||
t.Fatalf("saved count = %d, want 2; saved = %+v", len(repo.saved), repo.saved)
|
||||
}
|
||||
if repo.saved[0].Success || repo.saved[0].Message != "ICCID不在当前操作员授权范围内" {
|
||||
t.Fatalf("invalid row result = %+v", repo.saved[0])
|
||||
}
|
||||
if !repo.saved[1].Success {
|
||||
t.Fatalf("valid row result = %+v", repo.saved[1])
|
||||
}
|
||||
if task.Details[0].ValidationPassed || !task.Details[1].ValidationPassed {
|
||||
t.Fatalf("validation flags = (%v, %v)", task.Details[0].ValidationPassed, task.Details[1].ValidationPassed)
|
||||
}
|
||||
if !repo.finalized {
|
||||
t.Fatal("batch was not finalized")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSameLocalStatusStillCallsUpstream(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "D-3"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{response: []model.SimBaseRes[model.SimCardApnChangeOutputItem]{
|
||||
{Code: 0, Message: "ok", SimKey: &model.SimKey{ICCID: "iccid-1"}},
|
||||
}}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "D-3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1 even when local status already matches", upstream.calls)
|
||||
}
|
||||
if task.Details[0].Reason != "" {
|
||||
t.Fatalf("same local status must not be a validation blocker, reason = %q", task.Details[0].Reason)
|
||||
}
|
||||
if len(repo.saved) != 1 || !repo.saved[0].Success {
|
||||
t.Fatalf("saved = %+v", repo.saved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteValidationFailureSkipsUpstream(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataStop)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "D-4"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
{ID: 2, ICCID: "iccid-2", Authorized: false, Card: &Card{ID: 12, ChanKey: "channel", DataServiceStatus: consts.SimDataServiceStatusEnabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{responseError: errors.New("must not be called")}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "D-4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 0 {
|
||||
t.Fatalf("upstream calls = %d, want 0", upstream.calls)
|
||||
}
|
||||
if !repo.validationSaved || !repo.finalized {
|
||||
t.Fatal("validation failure was not persisted and finalized")
|
||||
}
|
||||
if task.Details[0].Reason == "" || task.Details[1].Reason == "" {
|
||||
t.Fatal("every detail must receive a validation failure reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUpstreamPartialSuccess(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{
|
||||
Batch: Batch{BatchNo: "D-5"},
|
||||
Operation: operation,
|
||||
ApnName: consts.SimApnNameDefault,
|
||||
Details: []*Detail{
|
||||
{ID: 1, ICCID: "iccid-1", MSISDN: "msisdn-1", IMSI: "imsi-1", Authorized: true, Card: &Card{ID: 11, ChanKey: "channel", MSISDN: "msisdn-1", IMSI: "imsi-1", DataServiceStatus: consts.SimDataServiceStatusDisabled}},
|
||||
{ID: 2, ICCID: "iccid-2", MSISDN: "msisdn-2", IMSI: "imsi-2", Authorized: true, Card: &Card{ID: 12, ChanKey: "channel", MSISDN: "msisdn-2", IMSI: "imsi-2", DataServiceStatus: consts.SimDataServiceStatusDisabled}},
|
||||
},
|
||||
}
|
||||
repo := &fakeRepository{task: task}
|
||||
upstream := &fakeUpstream{response: []model.SimBaseRes[model.SimCardApnChangeOutputItem]{
|
||||
{Code: 0, Message: "ok", SimKey: nil, ResData: model.SimCardApnChangeOutputItem{OrderNum: "order-1"}},
|
||||
{Code: 9001, Message: "denied", SimKey: &model.SimKey{ICCID: "iccid-2"}, ResData: model.SimCardApnChangeOutputItem{OrderNum: "order-2"}},
|
||||
}}
|
||||
|
||||
if err = NewExecutor(repo, upstream).Execute(context.Background(), "D-5"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(repo.saved) != 2 || !repo.saved[0].Success || repo.saved[0].OrderNum != "order-1" {
|
||||
t.Fatalf("first result = %+v", repo.saved)
|
||||
}
|
||||
if repo.saved[1].Success || repo.saved[1].Code != 9001 || repo.saved[1].Message != "denied" || repo.saved[1].OrderNum != "order-2" {
|
||||
t.Fatalf("second result = %+v", repo.saved[1])
|
||||
}
|
||||
if !repo.finalized {
|
||||
t.Fatal("batch was not finalized")
|
||||
}
|
||||
}
|
||||
|
||||
// 大批量场景下，每个子批次一拿到结果就立即落库，不必等所有子批次跑完。
|
||||
func TestExecuteSavesEachSubBatchAsSoonAsItSettles(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 201 张 = 3 个子批（100 / 100 / 1）；并发度为 5，三个子批会同时发起。
|
||||
details := make([]*Detail, 0, 201)
|
||||
for i := 0; i < 201; i++ {
|
||||
details = append(details, &Detail{
|
||||
ID: int64(i + 1),
|
||||
ICCID: fmt.Sprintf("iccid-%03d", i),
|
||||
Authorized: true,
|
||||
Card: &Card{
|
||||
ID: int64(1000 + i),
|
||||
ChanKey: "channel",
|
||||
DataServiceStatus: consts.SimDataServiceStatusDisabled,
|
||||
},
|
||||
})
|
||||
}
|
||||
task := &Task{Batch: Batch{BatchNo: "D-6"}, Operation: operation, ApnName: consts.SimApnNameDefault, Details: details}
|
||||
|
||||
repo := &fakeRepository{task: task}
|
||||
release := make(chan struct{})
|
||||
upstream := &subBatchUpstream{blockICCID: "iccid-100", block: release}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- NewExecutor(repo, upstream).Execute(context.Background(), "D-6") }()
|
||||
|
||||
// 第 1、3 个子批共 101 条应先落库，此时第 2 个子批仍阻塞在上游。
|
||||
waitForSavedCount(t, repo, 101)
|
||||
if repo.isFinalized() {
|
||||
t.Fatal("batch must not be finalized while one sub batch is still running")
|
||||
}
|
||||
|
||||
close(release)
|
||||
if err = <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := repo.savedCount(); got != 201 {
|
||||
t.Fatalf("saved count = %d, want 201", got)
|
||||
}
|
||||
if !repo.isFinalized() {
|
||||
t.Fatal("batch was not finalized")
|
||||
}
|
||||
if calls := atomic.LoadInt32(&upstream.calls); calls != 3 {
|
||||
t.Fatalf("upstream calls = %d, want 3", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForSavedCount(t *testing.T, repo *fakeRepository, want int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if repo.savedCount() >= want {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("saved count = %d, want at least %d", repo.savedCount(), want)
|
||||
}
|
||||
|
||||
func TestOperationByBatchTypeMapping(t *testing.T) {
|
||||
start, err := OperationByBatchType(consts.BatchTypeDataStart)
|
||||
if err != nil || start.Name != "dataStart" || start.ApnOperType != consts.SimApnOperTypeOpen || start.TargetStatus != consts.SimDataServiceStatusEnabled {
|
||||
t.Fatalf("start operation = %+v, err = %v", start, err)
|
||||
}
|
||||
stop, err := OperationByBatchType(consts.BatchTypeDataStop)
|
||||
if err != nil || stop.Name != "dataStop" || stop.ApnOperType != consts.SimApnOperTypeStop || stop.TargetStatus != consts.SimDataServiceStatusDisabled {
|
||||
t.Fatalf("stop operation = %+v, err = %v", stop, err)
|
||||
}
|
||||
cut, err := OperationByBatchType(consts.BatchTypeDataCut)
|
||||
if err != nil || cut.Name != "trafficCutoff" || cut.ApnOperType != consts.SimApnOperTypeStop || cut.TargetStatus != consts.SimDataServiceStatusDisabled {
|
||||
t.Fatalf("cut operation = %+v, err = %v", cut, err)
|
||||
}
|
||||
if _, err = OperationByBatchType(consts.BatchTypeSimCardStart); err == nil {
|
||||
t.Fatal("lifecycle batch type must not map to a data service operation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetailSnapshotIncludesOperationContext(t *testing.T) {
|
||||
operation, err := OperationByBatchType(consts.BatchTypeDataStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail := &Detail{
|
||||
ID: 1,
|
||||
ICCID: "iccid-1",
|
||||
ChanKey: "channel",
|
||||
StatusBefore: consts.SimDataServiceStatusDisabled,
|
||||
StatusAfter: consts.SimDataServiceStatusEnabled,
|
||||
OrderNum: "order-1",
|
||||
UpstreamCode: 0,
|
||||
UpstreamReason: "ok",
|
||||
}
|
||||
snapshot := detail.Snapshot(operation, consts.SimApnNameDefault)
|
||||
if snapshot.Get("operation").String() != "dataStart" ||
|
||||
snapshot.Get("operType").String() != consts.SimApnOperTypeOpen ||
|
||||
snapshot.Get("apnName").String() != consts.SimApnNameDefault ||
|
||||
snapshot.Get("statusBefore").Int() != consts.SimDataServiceStatusDisabled ||
|
||||
snapshot.Get("statusAfter").Int() != consts.SimDataServiceStatusEnabled ||
|
||||
snapshot.Get("orderNum").String() != "order-1" {
|
||||
t.Fatalf("snapshot = %s", snapshot.MustToJsonString())
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRepository 会被多个子批次 goroutine 并发写入，因此内部状态统一用锁保护。
|
||||
type fakeRepository struct {
|
||||
task *Task
|
||||
refresh func(*Task)
|
||||
|
||||
mu sync.Mutex
|
||||
validationSaved bool
|
||||
finalized bool
|
||||
saved []Outcome
|
||||
}
|
||||
|
||||
func (r *fakeRepository) Load(context.Context, string) (*Task, error) { return r.task, nil }
|
||||
|
||||
func (r *fakeRepository) RefreshDataServiceState(_ context.Context, task *Task) error {
|
||||
if r.refresh != nil {
|
||||
r.refresh(task)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeRepository) SaveValidationFailure(context.Context, *Task) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.validationSaved = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeRepository) SaveResult(_ context.Context, _ *Task, detail *Detail, outcome Outcome) (Outcome, error) {
|
||||
detail.ApplyOutcome(outcome)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.saved = append(r.saved, outcome)
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
func (r *fakeRepository) Finalize(context.Context, *Task) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.finalized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeRepository) savedCount() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return len(r.saved)
|
||||
}
|
||||
|
||||
func (r *fakeRepository) isFinalized() bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.finalized
|
||||
}
|
||||
|
||||
// subBatchUpstream 按子批次返回成功结果，并可在指定子批（以其首张 ICCID 标识）上阻塞，
|
||||
// 用于验证"某个子批还没结束时，其他子批的结果已经落库"。
|
||||
type subBatchUpstream struct {
|
||||
calls int32
|
||||
blockICCID string
|
||||
block chan struct{}
|
||||
}
|
||||
|
||||
func (u *subBatchUpstream) ChangeSimApnFunction(ctx context.Context, input *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
atomic.AddInt32(&u.calls, 1)
|
||||
if u.block != nil && len(input.SimKeys) > 0 && strings.TrimSpace(input.SimKeys[0].ICCID) == u.blockICCID {
|
||||
select {
|
||||
case <-u.block:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
response := make([]model.SimBaseRes[model.SimCardApnChangeOutputItem], 0, len(input.SimKeys))
|
||||
for _, key := range input.SimKeys {
|
||||
response = append(response, model.SimBaseRes[model.SimCardApnChangeOutputItem]{
|
||||
Code: 0,
|
||||
Message: "ok",
|
||||
SimKey: &model.SimKey{ICCID: key.ICCID},
|
||||
})
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
type fakeUpstream struct {
|
||||
calls int
|
||||
input *model.SimCardApnChangeInput
|
||||
response []model.SimBaseRes[model.SimCardApnChangeOutputItem]
|
||||
responseError error
|
||||
}
|
||||
|
||||
func (u *fakeUpstream) ChangeSimApnFunction(_ context.Context, input *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.calls++
|
||||
u.input = input
|
||||
return u.response, u.responseError
|
||||
}
|
||||
|
||||
func TestValidateDetailCollectsIndependentDiagnostics(t *testing.T) {
|
||||
detail := &Detail{ICCID: "card", Authorized: true, InputMSISDN: "wrong-phone", InputIMSI: "wrong-imsi", Card: &Card{MSISDN: "phone", IMSI: "imsi"}}
|
||||
validateDetail(Operation{}, detail)
|
||||
want := "ICCID缺少通道标识；MSISDN与卡档案不一致；IMSI与卡档案不一致"
|
||||
if detail.Reason != want {
|
||||
t.Fatalf("独立校验原因丢失: got=%q want=%q", detail.Reason, want)
|
||||
}
|
||||
unauthorized := &Detail{ICCID: "card", InputMSISDN: "wrong", Card: &Card{MSISDN: "phone"}}
|
||||
validateDetail(Operation{}, unauthorized)
|
||||
if unauthorized.Reason != "ICCID不在当前操作员授权范围内" {
|
||||
t.Fatalf("越权卡不应返回其他数据差异: %q", unauthorized.Reason)
|
||||
}
|
||||
existing := &Detail{Reason: "原始输入格式错误", Authorized: true, Card: &Card{}}
|
||||
validateDetail(Operation{}, existing)
|
||||
if existing.Reason != "原始输入格式错误" {
|
||||
t.Fatal("原始原因被覆盖")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
type Operation struct {
|
||||
BatchType int
|
||||
Name string
|
||||
ApnOperType string
|
||||
TargetStatus int
|
||||
}
|
||||
|
||||
type Batch struct {
|
||||
ID int64
|
||||
BatchNo string
|
||||
BatchType int
|
||||
TenantID string
|
||||
CreatedBy int64
|
||||
CreatedDept int64
|
||||
StartTime *gtime.Time
|
||||
BatchInfo *gjson.Json
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
DataServiceStatus int
|
||||
}
|
||||
|
||||
type Detail struct {
|
||||
ID int64
|
||||
DetailStatus int
|
||||
RowNo int
|
||||
ICCID string
|
||||
InputMSISDN string
|
||||
InputIMSI string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
Authorized bool
|
||||
ChanKey string
|
||||
StatusBefore int
|
||||
StatusAfter int
|
||||
TrafficCutoffCrossMonthHold int
|
||||
Reason string
|
||||
ValidationPassed bool
|
||||
OrderNum string
|
||||
UpstreamCode int
|
||||
UpstreamReason string
|
||||
Card *Card
|
||||
}
|
||||
|
||||
type Outcome struct {
|
||||
Pending bool
|
||||
Success bool
|
||||
Code int
|
||||
Message string
|
||||
OrderNum string
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
Batch Batch
|
||||
Operation Operation
|
||||
ApnName string
|
||||
Details []*Detail
|
||||
ValidationOK bool
|
||||
FailureReason string
|
||||
}
|
||||
|
||||
func (d *Detail) ApplyOutcome(outcome Outcome) {
|
||||
d.OrderNum = strings.TrimSpace(outcome.OrderNum)
|
||||
if outcome.Success {
|
||||
d.DetailStatus = 2
|
||||
} else {
|
||||
d.DetailStatus = 3
|
||||
}
|
||||
d.UpstreamCode = outcome.Code
|
||||
d.UpstreamReason = strings.TrimSpace(outcome.Message)
|
||||
if outcome.Success {
|
||||
d.Reason = ""
|
||||
return
|
||||
}
|
||||
if d.UpstreamReason == "" {
|
||||
d.UpstreamReason = "上游处理失败"
|
||||
}
|
||||
d.Reason = d.UpstreamReason
|
||||
}
|
||||
|
||||
func (d *Detail) Snapshot(operation Operation, apnName string) *gjson.Json {
|
||||
if d == nil {
|
||||
return gjson.New(nil)
|
||||
}
|
||||
return gjson.New(detailSnapshot{
|
||||
ICCID: d.ICCID,
|
||||
InputMSISDN: d.InputMSISDN,
|
||||
InputIMSI: d.InputIMSI,
|
||||
MSISDN: d.MSISDN,
|
||||
IMSI: d.IMSI,
|
||||
Authorized: d.Authorized,
|
||||
ChanKey: d.ChanKey,
|
||||
Operation: operation.Name,
|
||||
OperType: operation.ApnOperType,
|
||||
ApnName: apnName,
|
||||
StatusBefore: d.StatusBefore,
|
||||
StatusAfter: d.StatusAfter,
|
||||
TrafficCutoffCrossMonthHold: d.TrafficCutoffCrossMonthHold,
|
||||
OrderNum: d.OrderNum,
|
||||
UpstreamCode: d.UpstreamCode,
|
||||
UpstreamMessage: d.UpstreamReason,
|
||||
})
|
||||
}
|
||||
|
||||
type detailSnapshot struct {
|
||||
ICCID string `json:"ICCID"`
|
||||
InputMSISDN string `json:"inputMsisdn"`
|
||||
InputIMSI string `json:"inputImsi"`
|
||||
MSISDN string `json:"MSISDN"`
|
||||
IMSI string `json:"IMSI"`
|
||||
Authorized bool `json:"authorized"`
|
||||
ChanKey string `json:"chanKey"`
|
||||
Operation string `json:"operation"`
|
||||
OperType string `json:"operType"`
|
||||
ApnName string `json:"apnName"`
|
||||
StatusBefore int `json:"statusBefore"`
|
||||
StatusAfter int `json:"statusAfter"`
|
||||
TrafficCutoffCrossMonthHold int `json:"trafficCutoffCrossMonthHold,omitempty"`
|
||||
OrderNum string `json:"orderNum"`
|
||||
UpstreamCode int `json:"upstreamCode"`
|
||||
UpstreamMessage string `json:"upstreamMessage"`
|
||||
}
|
||||
|
||||
type Repository interface {
|
||||
Load(ctx context.Context, batchNo string) (*Task, error)
|
||||
SaveValidationFailure(ctx context.Context, task *Task) error
|
||||
SaveResult(ctx context.Context, task *Task, detail *Detail, outcome Outcome) (Outcome, error)
|
||||
Finalize(ctx context.Context, task *Task) error
|
||||
}
|
||||
|
||||
// SubBatch is a durable, channel-isolated intent; Request is frozen before submission.
|
||||
type SubBatch struct {
|
||||
ID int64
|
||||
BatchNo string
|
||||
BatchType int
|
||||
ChanKey string
|
||||
OperationID string
|
||||
DetailIDs []int64
|
||||
Request model.SimCardApnChangeInput
|
||||
JobID string
|
||||
Status string
|
||||
PollCount int
|
||||
SubmittedAt time.Time
|
||||
LeaseToken string
|
||||
LeaseUntil time.Time
|
||||
NextCheckAt time.Time
|
||||
}
|
||||
|
||||
// SubBatchRepository keeps the synchronous legacy tests separate from durable scheduling.
|
||||
type SubBatchRepository interface {
|
||||
HasDurableParent(ctx context.Context, batchNo string) (bool, error)
|
||||
PrepareSubBatch(ctx context.Context, batch SubBatch) error
|
||||
ResumeOrPrepare(ctx context.Context, batchNo string, prepare func(context.Context) error) error
|
||||
ClaimSubBatch(ctx context.Context, operationID, token string, now time.Time) (*SubBatch, error)
|
||||
ListDueSubBatches(ctx context.Context, now time.Time, limit int) ([]string, error)
|
||||
GetSubBatch(ctx context.Context, operationID string) (*SubBatch, error)
|
||||
DelaySubBatchPrecheck(ctx context.Context, snapshot SubBatch, next time.Time, reason string) error
|
||||
ReleaseUnsentSubBatch(ctx context.Context, batch SubBatch, reason string) error
|
||||
SubBatchChannel(ctx context.Context, operationID string) (string, error)
|
||||
UpdateSubBatch(ctx context.Context, batch SubBatch, status, jobID, reason string, next time.Time, pollCount int) error
|
||||
CompleteSubBatch(ctx context.Context, batch SubBatch, outcomes []Outcome) error
|
||||
SaveSubBatchProgress(ctx context.Context, batch SubBatch, outcomes []Outcome) error
|
||||
ProtectExecutionFailure(ctx context.Context, batchNo, reason string) error
|
||||
FinalizeReadyBatch(ctx context.Context, batchNo string) error
|
||||
ListRecoverableBatches(ctx context.Context, limit int) ([]string, error)
|
||||
}
|
||||
|
||||
type Upstream interface {
|
||||
ChangeSimApnFunction(ctx context.Context, input *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error)
|
||||
}
|
||||
|
||||
// AsyncUpstream provides an explicit submit/recovery/query protocol: never resend a write request.
|
||||
type AsyncUpstream interface {
|
||||
SubmitSimApnFunction(ctx context.Context, input *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error)
|
||||
QuerySimApnFunctionResult(ctx context.Context, input *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error)
|
||||
}
|
||||
|
||||
type ReviewRepository interface {
|
||||
ClaimReviewSubBatch(ctx context.Context, operationID, token string, now time.Time) (*SubBatch, error)
|
||||
}
|
||||
|
||||
// ReviewBatchRepository 支撑"手动同步上游结果"：按父批找出待核查子批并只读补查，
|
||||
// 不重发办理、不重置自动补查计数。
|
||||
type ReviewBatchRepository interface {
|
||||
ListReviewSubBatches(ctx context.Context, batchNo string) ([]string, error)
|
||||
FinalizeReadyBatch(ctx context.Context, batchNo string) error
|
||||
}
|
||||
|
||||
type Executor interface {
|
||||
Execute(ctx context.Context, batchNo string) error
|
||||
QueryReview(ctx context.Context, operationID string) error
|
||||
QueryReviewBatch(ctx context.Context, batchNo string) (int, error)
|
||||
}
|
||||
Reference in New Issue
Block a user