MiFaZhan
432 lines
15 KiB
Go
432 lines
15 KiB
Go
package data
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/google/uuid"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
|
)
|
|
|
|
// 首查 +3min，此后每 3min 重试 3 次（+3/+6/+9/+12min）。
|
|
// 2026-10-15 潍坊、濮阳实测：10/30/100 卡共12笔首次可见终态约 65~225s；
|
|
// 15s 采样只能给出完成区间，不能据此承诺所有高峰时段的完成上限。
|
|
// 偶发超长任务不增加自动重试；超窗记 review，保留受理号和原因供手动查询。
|
|
// 未确认不得记办理失败，也不得重新提交办理请求。
|
|
const (
|
|
subBatchLease = 5 * time.Minute
|
|
firstResultDelay = 3 * time.Minute
|
|
furtherResultDelay = 3 * time.Minute
|
|
maxResultQueries = 4
|
|
)
|
|
|
|
func newSubBatch(task *Task, details []*Detail) SubBatch {
|
|
keys := make([]*model.SimKey, 0, len(details))
|
|
ids := make([]int64, 0, len(details))
|
|
for _, detail := range details {
|
|
ids = append(ids, detail.ID)
|
|
keys = append(keys, &model.SimKey{ChanKey: detail.ChanKey, ICCID: detail.ICCID, MSISDN: detail.MSISDN, IMSI: detail.IMSI})
|
|
}
|
|
raw, _ := json.Marshal(struct {
|
|
BatchNo string
|
|
BatchType int
|
|
IDs []int64
|
|
}{task.Batch.BatchNo, task.Batch.BatchType, ids})
|
|
digest := sha256.Sum256(raw)
|
|
id := hex.EncodeToString(digest[:])
|
|
input := model.SimCardApnChangeInput{SimKeys: keys, ReqData: model.SimCardApnChangeInputItem{OperType: task.Operation.ApnOperType, ApnName: task.ApnName, OperationID: id, SubmitOnly: true}}
|
|
return SubBatch{BatchNo: task.Batch.BatchNo, BatchType: task.Batch.BatchType, ChanKey: details[0].ChanKey, OperationID: id, DetailIDs: ids, Request: input, Status: "prepared"}
|
|
}
|
|
|
|
func splitAsyncDetails(details []*Detail) [][]*Detail {
|
|
withMSISDN := make([]*Detail, 0, len(details))
|
|
batches := make([][]*Detail, 0)
|
|
for _, detail := range details {
|
|
if strings.TrimSpace(detail.MSISDN) == "" {
|
|
batches = append(batches, []*Detail{detail})
|
|
} else {
|
|
withMSISDN = append(withMSISDN, detail)
|
|
}
|
|
}
|
|
for _, chunk := range splitDetails(withMSISDN, consts.BatchSubBatchSize) {
|
|
if len(chunk) < 10 {
|
|
for _, detail := range chunk {
|
|
batches = append(batches, []*Detail{detail})
|
|
}
|
|
} else {
|
|
batches = append(batches, chunk)
|
|
}
|
|
}
|
|
return batches
|
|
}
|
|
|
|
// prepareAndSubmit persists every intent before a single write leaves the process.
|
|
func (e *executor) prepareAndSubmit(ctx context.Context, task *Task, repo SubBatchRepository, upstream AsyncUpstream) error {
|
|
groups := make(map[string][]*Detail)
|
|
for _, detail := range task.Details {
|
|
if detail != nil && detail.Reason == "" && (detail.DetailStatus == 0 || detail.DetailStatus == consts.BatchDetailStatusRunning) {
|
|
groups[detail.ChanKey] = append(groups[detail.ChanKey], detail)
|
|
}
|
|
}
|
|
keys := make([]string, 0, len(groups))
|
|
for key := range groups {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, key := range keys {
|
|
for _, slice := range splitAsyncDetails(groups[key]) {
|
|
sub := newSubBatch(task, slice)
|
|
if err := repo.PrepareSubBatch(ctx, sub); err != nil {
|
|
return fmt.Errorf("保存APN提交意图失败（未提交运营商）: %w", err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// processSubBatch only submits while prepared. Any interrupted submission is recovered by operationId, never repeated.
|
|
func (e *executor) processSubBatch(ctx context.Context, repo SubBatchRepository, upstream AsyncUpstream, operationID string) error {
|
|
channel, err := repo.SubBatchChannel(ctx, operationID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
value, _ := e.channelSlots.LoadOrStore(channel, make(chan struct{}, consts.BatchSubBatchConcurrency))
|
|
slots := value.(chan struct{})
|
|
select {
|
|
case slots <- struct{}{}:
|
|
defer func() { <-slots }()
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
token := uuid.NewString()
|
|
callCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
|
|
defer cancel()
|
|
if checker, ok := upstream.(interface {
|
|
CheckSimApnAsyncCapability(context.Context, *model.SimCardApnChangeInput) error
|
|
}); ok {
|
|
snapshot, err := repo.GetSubBatch(callCtx, operationID)
|
|
if err != nil || snapshot == nil {
|
|
return err
|
|
}
|
|
if snapshot.Status == "done" || snapshot.Status == "review" {
|
|
return nil
|
|
}
|
|
if snapshot.Status == "prepared" {
|
|
if err := checker.CheckSimApnAsyncCapability(callCtx, &snapshot.Request); err != nil {
|
|
persistCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer stop()
|
|
if delayErr := repo.DelaySubBatchPrecheck(persistCtx, *snapshot, time.Now().Add(30*time.Second), err.Error()); delayErr != nil {
|
|
return fmt.Errorf("保存APN预检调度退避失败: %w", delayErr)
|
|
}
|
|
return fmt.Errorf("APN提交预检未就绪，未领取提交意图: %w", err)
|
|
}
|
|
}
|
|
}
|
|
sub, err := repo.ClaimSubBatch(ctx, operationID, token, time.Now())
|
|
if err != nil || sub == nil {
|
|
return err
|
|
}
|
|
if sub.Status == "prepared" {
|
|
response, submitErr := upstream.SubmitSimApnFunction(callCtx, &sub.Request)
|
|
if submitErr != nil {
|
|
if isAPNRequestNotSent(submitErr) {
|
|
return releaseUnsent(ctx, repo, *sub, submitErr)
|
|
}
|
|
return e.rememberUncertain(ctx, repo, *sub, "提交结果未确认（网络超时或连接中断）: "+submitErr.Error())
|
|
}
|
|
return e.handleResponse(ctx, repo, *sub, response, false)
|
|
}
|
|
if sub.Status == "submitting" || sub.Status == "uncertain" || (sub.Status == "accepted" && sub.JobID == "") {
|
|
sub.Request.ReqData.RecoverOnly = true
|
|
sub.Request.ReqData.QueryJobID = ""
|
|
response, queryErr := upstream.QuerySimApnFunctionResult(callCtx, &sub.Request)
|
|
if queryErr != nil {
|
|
if isAPNRequestNotSent(queryErr) {
|
|
return releaseUnsent(ctx, repo, *sub, queryErr)
|
|
}
|
|
return e.rememberUncertain(ctx, repo, *sub, "受理映射恢复失败（不重发）: "+queryErr.Error())
|
|
}
|
|
return e.handleResponse(ctx, repo, *sub, response, false)
|
|
}
|
|
sub.Request.ReqData.QueryJobID = sub.JobID
|
|
sub.Request.ReqData.RecoverOnly = false
|
|
response, queryErr := upstream.QuerySimApnFunctionResult(callCtx, &sub.Request)
|
|
if queryErr != nil {
|
|
if isAPNRequestNotSent(queryErr) {
|
|
return releaseUnsent(ctx, repo, *sub, queryErr)
|
|
}
|
|
return e.queryUnavailable(ctx, repo, *sub, "查询请求超时或不可用: "+queryErr.Error())
|
|
}
|
|
return e.handleResponse(ctx, repo, *sub, response, true)
|
|
}
|
|
|
|
// QueryReview仅执行一次只读补查，不重新开启自动轮询或提交办理。
|
|
func (e *executor) QueryReview(ctx context.Context, operationID string) error {
|
|
operationID = strings.TrimSpace(operationID)
|
|
if operationID == "" {
|
|
return fmt.Errorf("operationId不能为空")
|
|
}
|
|
repo, ok := e.repository.(SubBatchRepository)
|
|
if !ok {
|
|
return fmt.Errorf("未配置APN持久化仓储")
|
|
}
|
|
reviewRepo, ok := e.repository.(ReviewRepository)
|
|
if !ok {
|
|
return fmt.Errorf("未配置APN待核查查询仓储")
|
|
}
|
|
upstream, ok := e.upstream.(AsyncUpstream)
|
|
if !ok {
|
|
return fmt.Errorf("未配置APN只读补查客户端")
|
|
}
|
|
sub, err := reviewRepo.ClaimReviewSubBatch(ctx, operationID, uuid.NewString(), time.Now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if sub == nil {
|
|
snapshot, err := repo.GetSubBatch(ctx, operationID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if snapshot != nil && snapshot.Status == "done" {
|
|
return repo.FinalizeReadyBatch(ctx, snapshot.BatchNo)
|
|
}
|
|
return fmt.Errorf("子批不处于可查询的待核查状态，或已有查询/汇总正在运行")
|
|
}
|
|
callCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
|
|
defer cancel()
|
|
sub.Request.ReqData.SubmitOnly = false
|
|
sub.Request.ReqData.QueryJobID = sub.JobID
|
|
sub.Request.ReqData.RecoverOnly = sub.JobID == ""
|
|
items, queryErr := upstream.QuerySimApnFunctionResult(callCtx, &sub.Request)
|
|
if queryErr != nil {
|
|
if err := e.deferResult(ctx, repo, *sub, "review", "手动补查未确认："+queryErr.Error()); err != nil {
|
|
return err
|
|
}
|
|
return queryErr
|
|
}
|
|
return e.handleResponse(ctx, repo, *sub, items, true)
|
|
}
|
|
|
|
// QueryReviewBatch 是"手动同步上游结果"入口：只对父批下已冻结的待核查子批
|
|
// 做一次只读补查，然后尝试汇总父批。
|
|
// 不重发办理请求、不清零自动补查计数、不重新开启轮询调度。
|
|
func (e *executor) QueryReviewBatch(ctx context.Context, batchNo string) (int, error) {
|
|
batchNo = strings.TrimSpace(batchNo)
|
|
if batchNo == "" {
|
|
return 0, fmt.Errorf("批次编号不能为空")
|
|
}
|
|
repo, ok := e.repository.(ReviewBatchRepository)
|
|
if !ok {
|
|
return 0, fmt.Errorf("未配置APN待核查批量同步仓储")
|
|
}
|
|
if _, ok := e.upstream.(AsyncUpstream); !ok {
|
|
return 0, fmt.Errorf("未配置APN只读补查客户端")
|
|
}
|
|
ids, err := repo.ListReviewSubBatches(ctx, batchNo)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
queried := 0
|
|
var firstErr error
|
|
for _, id := range ids {
|
|
if err := e.QueryReview(ctx, id); err != nil {
|
|
g.Log().Warningf(ctx, "手动同步待核查子批未完成 batchNo=%s operationId=%s error=%v", batchNo, id, err)
|
|
if firstErr == nil {
|
|
firstErr = fmt.Errorf("operationId=%s: %w", id, err)
|
|
}
|
|
continue
|
|
}
|
|
queried++
|
|
}
|
|
// 已确认终态的子批会在这里收敛父批；仍为 review 的子批继续留在待核查。
|
|
if err := repo.FinalizeReadyBatch(ctx, batchNo); err != nil && firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
g.Log().Infof(ctx, "手动同步APN上游结果 batchNo=%s 待核查子批=%d 本次已同步=%d", batchNo, len(ids), queried)
|
|
if firstErr != nil {
|
|
return queried, firstErr
|
|
}
|
|
return queried, nil
|
|
}
|
|
|
|
func isAPNRequestNotSent(err error) bool {
|
|
var notSent interface{ APNRequestNotSent() bool }
|
|
return errors.As(err, ¬Sent) && notSent.APNRequestNotSent()
|
|
}
|
|
func releaseUnsent(ctx context.Context, repo SubBatchRepository, sub SubBatch, cause error) error {
|
|
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
return repo.ReleaseUnsentSubBatch(persistCtx, sub, cause.Error())
|
|
}
|
|
|
|
func nextResultCheck(sub SubBatch) time.Time {
|
|
base := sub.SubmittedAt
|
|
if base.IsZero() {
|
|
base = time.Now()
|
|
}
|
|
return base.Add(firstResultDelay + time.Duration(sub.PollCount)*furtherResultDelay)
|
|
}
|
|
|
|
func (e *executor) rememberUncertain(ctx context.Context, repo SubBatchRepository, sub SubBatch, reason string) error {
|
|
return e.deferResult(ctx, repo, sub, "uncertain", reason)
|
|
}
|
|
|
|
func (e *executor) queryUnavailable(ctx context.Context, repo SubBatchRepository, sub SubBatch, reason string) error {
|
|
return e.deferResult(ctx, repo, sub, "accepted", reason)
|
|
}
|
|
|
|
func (e *executor) deferResult(ctx context.Context, repo SubBatchRepository, sub SubBatch, state, reason string) error {
|
|
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
g.Log().Warningf(persistCtx, "APN补查 operationId=%s jobId=%s pollCount=%d reason=%s", sub.OperationID, sub.JobID, sub.PollCount, reason)
|
|
if sub.Status == "review_query" || sub.PollCount >= maxResultQueries {
|
|
if err := repo.UpdateSubBatch(persistCtx, sub, "review", sub.JobID, "补查窗口结束，办理结果未知："+reason, time.Time{}, sub.PollCount); err != nil {
|
|
return err
|
|
}
|
|
return repo.FinalizeReadyBatch(persistCtx, sub.BatchNo)
|
|
}
|
|
return repo.UpdateSubBatch(persistCtx, sub, state, sub.JobID, reason, nextResultCheck(sub), sub.PollCount)
|
|
}
|
|
|
|
func (e *executor) handleResponse(ctx context.Context, repo SubBatchRepository, sub SubBatch, items []model.SimBaseRes[model.SimCardApnChangeOutputItem], queried bool) error {
|
|
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
|
defer cancel()
|
|
outcomes := make([]Outcome, len(sub.Request.SimKeys))
|
|
used := make([]bool, len(items))
|
|
pending := false
|
|
reason := "未返回该卡结果，办理未确认"
|
|
for i, key := range sub.Request.SimKeys {
|
|
matched := -1
|
|
for j, item := range items {
|
|
if !used[j] && sameSim(&Detail{ICCID: key.ICCID, MSISDN: key.MSISDN, IMSI: key.IMSI}, item.SimKey) {
|
|
matched = j
|
|
break
|
|
}
|
|
}
|
|
if matched < 0 {
|
|
outcomes[i] = Outcome{Pending: true, Message: reason}
|
|
pending = true
|
|
continue
|
|
}
|
|
used[matched] = true
|
|
item := items[matched]
|
|
if item.ResData.Pending || item.ResData.Uncertain {
|
|
pending = true
|
|
outcomes[i] = Outcome{Pending: true, Code: item.Code, Message: item.Message}
|
|
if strings.TrimSpace(item.Message) != "" {
|
|
reason = item.Message
|
|
}
|
|
// Only Pending identifies a batch job: a synchronous per-card order is never queried as a job.
|
|
if item.ResData.Pending && strings.TrimSpace(item.ResData.OrderNum) != "" {
|
|
sub.JobID = strings.TrimSpace(item.ResData.OrderNum)
|
|
}
|
|
continue
|
|
}
|
|
outcomes[i] = Outcome{Success: item.Code == 0, Code: item.Code, Message: item.Message, OrderNum: item.ResData.OrderNum}
|
|
}
|
|
if pending {
|
|
// Confirmed cards are committed now. Later missing/pending responses cannot overwrite their terminal rows.
|
|
if err := repo.SaveSubBatchProgress(persistCtx, sub, outcomes); err != nil {
|
|
return err
|
|
}
|
|
state := "accepted"
|
|
if sub.JobID == "" {
|
|
state = "uncertain"
|
|
reason += "（缺少jobId，仅按operationId恢复，不重发）"
|
|
}
|
|
return e.deferResult(persistCtx, repo, sub, state, reason)
|
|
}
|
|
if err := repo.CompleteSubBatch(persistCtx, sub, outcomes); err != nil {
|
|
return err
|
|
}
|
|
return repo.FinalizeReadyBatch(persistCtx, sub.BatchNo)
|
|
}
|
|
|
|
func mapOutcomesFromKeys(keys []*model.SimKey, response []model.SimBaseRes[model.SimCardApnChangeOutputItem]) []Outcome {
|
|
details := make([]*Detail, 0, len(keys))
|
|
for _, key := range keys {
|
|
details = append(details, &Detail{ICCID: key.ICCID, MSISDN: key.MSISDN, IMSI: key.IMSI})
|
|
}
|
|
return mapOutcomes(details, response)
|
|
}
|
|
|
|
// RunReconciler uses a fixed global worker bound; waiting for a job never retains a worker slot.
|
|
func (e *executor) RunReconciler(ctx context.Context) error {
|
|
repo, ok := e.repository.(SubBatchRepository)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
upstream, ok := e.upstream.(AsyncUpstream)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
jobs := make(chan string, 64)
|
|
var inFlight sync.Map
|
|
var workers sync.WaitGroup
|
|
for i := 0; i < 20; i++ {
|
|
workers.Add(1)
|
|
go func() {
|
|
defer workers.Done()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case id := <-jobs:
|
|
if err := e.processSubBatch(ctx, repo, upstream, id); err != nil {
|
|
g.Log().Errorf(ctx, "APN恢复失败 operationId=%s error=%v", id, err)
|
|
}
|
|
inFlight.Delete(id)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
defer workers.Wait()
|
|
ticker := time.NewTicker(5 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
batches, err := repo.ListRecoverableBatches(ctx, 32)
|
|
if err != nil {
|
|
g.Log().Errorf(ctx, "读取APN恢复父批失败: %v", err)
|
|
} else {
|
|
for _, batchNo := range batches {
|
|
if err := e.Execute(ctx, batchNo); err != nil {
|
|
g.Log().Errorf(ctx, "恢复APN父批失败 batchNo=%s error=%v", batchNo, err)
|
|
}
|
|
}
|
|
}
|
|
ids, err := repo.ListDueSubBatches(ctx, time.Now(), 128)
|
|
if err != nil {
|
|
g.Log().Errorf(ctx, "读取APN补查子批失败: %v", err)
|
|
} else {
|
|
for _, id := range ids {
|
|
if _, loaded := inFlight.LoadOrStore(id, true); loaded {
|
|
continue
|
|
}
|
|
select {
|
|
case jobs <- id:
|
|
case <-ctx.Done():
|
|
inFlight.Delete(id)
|
|
return nil
|
|
default:
|
|
inFlight.Delete(id)
|
|
}
|
|
}
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|