package data import ( "context" "database/sql" "encoding/json" "errors" "fmt" "reflect" "sort" "strings" "time" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "github.com/google/uuid" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model" data "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data" ) const subTable = "data_operation_sub_batch" const claimTable = "data_operation_card_claim" const parentTable = "data_operation_parent" const subLease = 5 * time.Minute type finalizeTokenKey struct{} type preparingKey struct{} func (r *Repository) subLock(q *gdb.Model) *gdb.Model { if strings.EqualFold(r.database().GetConfig().Type, "sqlite") { return q } return q.LockUpdate() } func (r *Repository) lockSubCards(ctx context.Context, tx gdb.TX, tenantID string, keys []*model.SimKey) ([]cardClaim, error) { keys = append([]*model.SimKey(nil), keys...) sort.Slice(keys, func(i, j int) bool { return keys[i].ICCID < keys[j].ICCID }) claims := make([]cardClaim, len(keys)) for i, key := range keys { if err := r.subLock(tx.Ctx(ctx).Model(claimTable).Where("tenant_id", tenantID).Where("iccid", key.ICCID)).Scan(&claims[i]); err != nil { return nil, err } } return claims, nil } func noRows(err error) error { if errors.Is(err, sql.ErrNoRows) { return nil } return err } func (r *Repository) ignoreInsert(ctx context.Context, tx gdb.TX, table string, values g.Map) error { _, err := tx.Ctx(ctx).Model(table).Data(values).InsertIgnore() return err } func (r *Repository) durableTx(ctx context.Context, fn func(context.Context, gdb.TX) error) error { var err error for attempt := 0; attempt < 3; attempt++ { err = r.database().Transaction(ctx, fn) if err == nil { return nil } text := err.Error() if !strings.Contains(text, "1213") && !strings.Contains(text, "1205") && !strings.Contains(text, "database is locked") { return err } if ctx.Err() != nil { return ctx.Err() } } return err } type subRow struct { ID int64 `orm:"id"` BatchNo string `orm:"batch_no"` BatchType int `orm:"batch_type"` ChanKey string `orm:"chan_key"` OperationID string `orm:"operation_id"` DetailIDs string `orm:"detail_ids"` RequestData string `orm:"request_data"` JobID string `orm:"job_id"` Status string `orm:"status"` PollCount int `orm:"poll_count"` SubmittedAt *time.Time `orm:"submitted_at"` NextCheckAt *time.Time `orm:"next_check_at"` LeaseToken string `orm:"lease_token"` LeaseUntil *time.Time `orm:"lease_until"` CreatedAt *time.Time `orm:"created_at"` } type cardClaim struct { BatchID int64 `orm:"batch_id"` OperationID string `orm:"operation_id"` ActiveBatchID int64 `orm:"active_batch_id"` ActiveOperationID string `orm:"active_operation_id"` } func decodeSub(row subRow) (data.SubBatch, error) { out := data.SubBatch{ID: row.ID, BatchNo: row.BatchNo, BatchType: row.BatchType, ChanKey: row.ChanKey, OperationID: row.OperationID, JobID: row.JobID, Status: row.Status, PollCount: row.PollCount, LeaseToken: row.LeaseToken} if row.SubmittedAt != nil { out.SubmittedAt = *row.SubmittedAt } if row.NextCheckAt != nil { out.NextCheckAt = *row.NextCheckAt } if row.LeaseUntil != nil { out.LeaseUntil = *row.LeaseUntil } if err := json.Unmarshal([]byte(row.DetailIDs), &out.DetailIDs); err != nil { return out, err } if err := json.Unmarshal([]byte(row.RequestData), &out.Request); err != nil { return out, err } if len(out.DetailIDs) == 0 || len(out.DetailIDs) != len(out.Request.SimKeys) { return out, fmt.Errorf("APN冻结请求与明细数不匹配: %s", row.OperationID) } return out, nil } func (r *Repository) HasDurableParent(ctx context.Context, batchNo string) (bool, error) { n, err := r.database().Model(parentTable).Ctx(ctx).Where("batch_no", batchNo).Count() return n == 1, err } // ResumeOrPrepare只恢复由Pending领取事务写入的凭据,绝不接管无映射的历史同步批。 func (r *Repository) ResumeOrPrepare(ctx context.Context, batchNo string, prepare func(context.Context) error) error { if ctx.Value(preparingKey{}) != nil { return prepare(ctx) } err := r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { var main struct { ID int64 `orm:"id"` Status int `orm:"batch_async_status"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("id,batch_async_status").Where("batch_no", batchNo)).Scan(&main)); err != nil { return err } if main.ID == 0 || main.Status != consts.BatchAsyncStatusProcessing { return nil } var parent *struct { Ready int `orm:"ready"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(parentTable).Where("batch_no", batchNo)).Scan(&parent)); err != nil { return err } if parent == nil { return fmt.Errorf("历史APN批次缺少持久化凭据,须人工核查,禁止自动提交: %s", batchNo) } if parent.Ready == 1 { return nil } if err := prepare(context.WithValue(ctx, preparingKey{}, true)); err != nil { return err } _, err := tx.Ctx(ctx).Model(parentTable).Where("batch_no", batchNo).Data(g.Map{"ready": 1}).Update() return err }) if err != nil { return err } return r.FinalizeReadyBatch(ctx, batchNo) } func (r *Repository) PrepareSubBatch(ctx context.Context, batch data.SubBatch) error { return r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { var main struct { ID int64 `orm:"id"` TenantID string `orm:"tenant_id"` } if err := r.subLock(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("id,tenant_id").Where("batch_no", batch.BatchNo).Where("batch_async_status", consts.BatchAsyncStatusProcessing)).Scan(&main); err != nil { return err } if main.ID == 0 || len(batch.DetailIDs) == 0 || len(batch.DetailIDs) > 100 || len(batch.DetailIDs) != len(batch.Request.SimKeys) { return fmt.Errorf("APN准备参数或父批状态不合法") } keys := append([]*model.SimKey(nil), batch.Request.SimKeys...) for _, key := range keys { if key == nil { return fmt.Errorf("APN卡标识为空") } } sort.Slice(keys, func(i, j int) bool { return keys[i].ICCID < keys[j].ICCID }) seen := map[string]bool{} for _, key := range keys { if key == nil || key.ICCID == "" || key.ChanKey != batch.ChanKey || seen[key.ICCID] { return fmt.Errorf("APN子批卡重复或跨通道") } seen[key.ICCID] = true if err := r.ignoreInsert(ctx, tx, claimTable, g.Map{"tenant_id": main.TenantID, "iccid": key.ICCID, "batch_id": 0, "operation_id": "", "active_batch_id": 0, "active_operation_id": ""}); err != nil { return err } var claim cardClaim if err := r.subLock(tx.Ctx(ctx).Model(claimTable).Where("tenant_id", main.TenantID).Where("iccid", key.ICCID)).Scan(&claim); err != nil { return err } if claim.BatchID < main.ID { if _, err := tx.Ctx(ctx).Model(claimTable).Where("tenant_id", main.TenantID).Where("iccid", key.ICCID).WhereLT("batch_id", main.ID).Data(g.Map{"batch_id": main.ID, "operation_id": batch.OperationID}).Update(); err != nil { return err } } else if claim.BatchID == main.ID && claim.OperationID != batch.OperationID { return fmt.Errorf("同批ICCID重复,禁止二次提交: %s", key.ICCID) } } var old subRow if err := noRows(r.subLock(tx.Ctx(ctx).Model(subTable).Where("operation_id", batch.OperationID)).Scan(&old)); err != nil { return err } if old.ID != 0 { stored, err := decodeSub(old) if err != nil { return err } if stored.BatchNo != batch.BatchNo || stored.ChanKey != batch.ChanKey || !reflect.DeepEqual(stored.Request, batch.Request) || !reflect.DeepEqual(stored.DetailIDs, batch.DetailIDs) { return fmt.Errorf("operationId绑定了不同请求: %s", batch.OperationID) } return nil } ids, err := json.Marshal(batch.DetailIDs) if err != nil { return err } request, err := json.Marshal(batch.Request) if err != nil { return err } if _, err = tx.Ctx(ctx).Model(subTable).Data(g.Map{"batch_no": batch.BatchNo, "batch_type": batch.BatchType, "chan_key": batch.ChanKey, "operation_id": batch.OperationID, "detail_ids": string(ids), "request_data": string(request), "status": "prepared", "next_check_at": time.Now()}).Insert(); err != nil { return err } for i, detailID := range batch.DetailIDs { if _, err := tx.Ctx(ctx).Model("data_operation_sub_detail").Data(g.Map{"detail_id": detailID, "operation_id": batch.OperationID, "batch_no": batch.BatchNo, "iccid": batch.Request.SimKeys[i].ICCID}).Insert(); err != nil { return fmt.Errorf("明细已归属其他冻结意图: %w", err) } } return nil }) } func (r *Repository) GetSubBatch(ctx context.Context, operationID string) (*data.SubBatch, error) { var row subRow if err := noRows(r.database().Model(subTable).Ctx(ctx).Where("operation_id", operationID).Scan(&row)); err != nil { return nil, err } if row.ID == 0 { return nil, nil } sub, err := decodeSub(row) return &sub, err } func (r *Repository) DelaySubBatchPrecheck(ctx context.Context, snapshot data.SubBatch, next time.Time, reason string) error { if snapshot.Status == "done" || snapshot.Status == "review" { return nil } q := r.database().Model(subTable).Ctx(ctx).Where("operation_id", snapshot.OperationID).Where("status", snapshot.Status).Where("COALESCE(lease_token,'')=?", snapshot.LeaseToken).Where("lease_until IS NULL OR lease_until<=?", time.Now()) if snapshot.NextCheckAt.IsZero() { q = q.WhereNull("next_check_at") } else { q = q.Where("next_check_at", snapshot.NextCheckAt) } if snapshot.LeaseUntil.IsZero() { q = q.WhereNull("lease_until") } else { q = q.Where("lease_until", snapshot.LeaseUntil) } _, err := q.Data(g.Map{"next_check_at": next, "last_reason": truncateReason("只读预检未就绪:"+reason, 1024)}).Update() return err } func (r *Repository) ReleaseUnsentSubBatch(ctx context.Context, batch data.SubBatch, reason string) error { return r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { var owner struct { TenantID string `orm:"tenant_id"` } if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("tenant_id").Where("batch_no", batch.BatchNo).Scan(&owner); err != nil { return err } if _, err := r.lockSubCards(ctx, tx, owner.TenantID, batch.Request.SimKeys); err != nil { return err } var row subRow if err := r.subLock(tx.Ctx(ctx).Model(subTable).Where("id", batch.ID)).Scan(&row); err != nil { return err } if row.LeaseToken != batch.LeaseToken || row.LeaseUntil == nil || !row.LeaseUntil.After(time.Now()) { return fmt.Errorf("未发送回退租约失效") } changes := g.Map{"status": batch.Status, "last_reason": truncateReason("确定未发送:"+reason, 1024), "next_check_at": time.Now().Add(15 * time.Second), "lease_token": gdb.Raw("NULL"), "lease_until": gdb.Raw("NULL")} if batch.Status == "prepared" { changes["submitted_at"] = gdb.Raw("NULL") var main struct { TenantID string `orm:"tenant_id"` } if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("tenant_id").Where("batch_no", batch.BatchNo).Scan(&main); err != nil { return err } keys := append([]*model.SimKey(nil), batch.Request.SimKeys...) sort.Slice(keys, func(i, j int) bool { return keys[i].ICCID < keys[j].ICCID }) for _, key := range keys { if _, err := tx.Ctx(ctx).Model(claimTable).Where("tenant_id", main.TenantID).Where("iccid", key.ICCID).Where("active_operation_id", batch.OperationID).Data(g.Map{"active_operation_id": "", "active_batch_id": 0}).Update(); err != nil { return err } } } else if batch.PollCount > 0 { changes["poll_count"] = batch.PollCount - 1 } _, err := tx.Ctx(ctx).Model(subTable).Where("id", batch.ID).Where("lease_token", batch.LeaseToken).Data(changes).Update() return err }) } func (r *Repository) SubBatchChannel(ctx context.Context, operationID string) (string, error) { row, err := r.database().GetOne(ctx, "SELECT chan_key FROM data_operation_sub_batch WHERE operation_id=?", operationID) if err != nil { return "", err } if len(row) == 0 { return "", fmt.Errorf("APN子批不存在") } return row["chan_key"].String(), nil } func (r *Repository) ListDueSubBatches(ctx context.Context, now time.Time, limit int) ([]string, error) { var rows []struct { OperationID string `orm:"operation_id"` } err := r.database().Model(subTable).Ctx(ctx).Fields("operation_id").WhereIn("status", []string{"prepared", "submitting", "accepted", "uncertain"}).WhereLTE("next_check_at", now).Where("lease_until IS NULL OR lease_until <= ?", now).Where("EXISTS (SELECT 1 FROM data_operation_parent p WHERE p.batch_no=data_operation_sub_batch.batch_no AND p.ready=1)").OrderAsc("next_check_at").Limit(limit).Scan(&rows) ids := make([]string, 0, len(rows)) for _, row := range rows { ids = append(ids, row.OperationID) } return ids, err } func (r *Repository) ClaimSubBatch(ctx context.Context, operationID, token string, now time.Time) (*data.SubBatch, error) { var out *data.SubBatch err := r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { out = nil var hint subRow if err := noRows(tx.Ctx(ctx).Model(subTable).Where("operation_id", operationID).Scan(&hint)); err != nil { return err } if hint.ID == 0 { return nil } var parent struct { Ready int `orm:"ready"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(parentTable).Where("batch_no", hint.BatchNo)).Scan(&parent)); err != nil { return err } if parent.Ready != 1 { return nil } var main struct { ID int64 `orm:"id"` TenantID string `orm:"tenant_id"` Status int `orm:"batch_async_status"` } if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("id,tenant_id,batch_async_status").Where("batch_no", hint.BatchNo).Scan(&main); err != nil { return err } if main.Status != consts.BatchAsyncStatusProcessing { return nil } frozen, err := decodeSub(hint) if err != nil { return err } claims, err := r.lockSubCards(ctx, tx, main.TenantID, frozen.Request.SimKeys) if err != nil { return err } var row subRow if err := r.subLock(tx.Ctx(ctx).Model(subTable).Where("id", hint.ID)).Scan(&row); err != nil { return err } if row.Status == "done" || row.Status == "review" || row.Status == "review_query" || (row.NextCheckAt != nil && row.NextCheckAt.After(now)) || (row.LeaseUntil != nil && row.LeaseUntil.After(now)) { return nil } sub, err := decodeSub(row) if err != nil { return err } if row.Status == "prepared" { keys := append([]*model.SimKey(nil), sub.Request.SimKeys...) sort.Slice(keys, func(i, j int) bool { return keys[i].ICCID < keys[j].ICCID }) for _, claim := range claims { if claim.ActiveOperationID != "" && claim.ActiveOperationID != operationID { var previous subRow if err := noRows(tx.Ctx(ctx).Model(subTable).Fields("status").Where("operation_id", claim.ActiveOperationID).Scan(&previous)); err != nil { return err } if previous.Status == "review" || (row.CreatedAt != nil && !row.CreatedAt.Add(15*time.Minute).After(now)) { reason := "未提交:同卡前序办理仍在途或结果未知,须人工核对前序后重新发起" if _, err := tx.Ctx(ctx).Model(subTable).Where("id", row.ID).Data(g.Map{"status": "review", "last_reason": reason, "next_check_at": gdb.Raw("NULL")}).Update(); err != nil { return err } _, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).WhereIn("id", sub.DetailIDs).Where("detail_status", consts.BatchDetailStatusRunning).Data(g.Map{"detail_status": consts.BatchDetailStatusUnknown, "error_code": "NOT_SUBMITTED", "error_reason": reason}).Update() return err } _, err := tx.Ctx(ctx).Model(subTable).Where("id", row.ID).Data(g.Map{"last_reason": "同卡前序办理在途或结果未知,等待核对后才能提交", "next_check_at": now.Add(15 * time.Second)}).Update() return err } } // 父批创建ID决定已冻结意图的队序;不能用可能交错的冻结ID替代。 for _, key := range keys { query := "SELECT s.operation_id FROM batch_main m JOIN data_operation_sub_detail sd ON sd.batch_no=m.batch_no AND sd.iccid=? JOIN data_operation_sub_batch s ON s.operation_id=sd.operation_id JOIN batch_detail d ON d.id=sd.detail_id WHERE m.tenant_id=? AND m.id= 4 { if _, err := tx.Ctx(ctx).Model(subTable).Where("id", row.ID).Data(g.Map{"status": "review", "last_reason": "补查窗口耗尽,结果未知,禁止重发", "next_check_at": gdb.Raw("NULL")}).Update(); err != nil { return err } _, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).WhereIn("id", sub.DetailIDs).Where("detail_status", consts.BatchDetailStatusRunning).Data(g.Map{"detail_status": consts.BatchDetailStatusUnknown, "error_code": "UNKNOWN", "error_reason": "补查窗口耗尽,结果未知,禁止重发"}).Update() return err } changes := g.Map{"lease_token": token, "lease_until": now.Add(subLease), "next_check_at": now.Add(subLease)} if row.Status == "prepared" { changes["status"] = "submitting" changes["submitted_at"] = now sub.SubmittedAt = now } else { changes["poll_count"] = row.PollCount + 1 sub.PollCount++ } result, err := tx.Ctx(ctx).Model(subTable).Where("id", row.ID).Where("status", row.Status).Where("lease_until IS NULL OR lease_until <= ?", now).Data(changes).Update() if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("APN租约CAS失败") } sub.LeaseToken = token out = &sub return nil }) return out, err } func (r *Repository) ClaimReviewSubBatch(ctx context.Context, operationID, token string, now time.Time) (*data.SubBatch, error) { var out *data.SubBatch err := r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { out = nil var hint subRow if err := noRows(tx.Ctx(ctx).Model(subTable).Where("operation_id", operationID).Scan(&hint)); err != nil { return err } if hint.ID == 0 { return nil } var main struct { Status int `orm:"batch_async_status"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("batch_async_status").Where("batch_no", hint.BatchNo)).Scan(&main)); err != nil { return err } if main.Status != consts.BatchAsyncStatusReview && main.Status != consts.BatchAsyncStatusProcessing { return nil } var parent struct { Ready int `orm:"ready"` FinalizeUntil *time.Time `orm:"finalize_until"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(parentTable).Where("batch_no", hint.BatchNo)).Scan(&parent)); err != nil { return err } if parent.Ready != 1 || (parent.FinalizeUntil != nil && parent.FinalizeUntil.After(now)) { return nil } var row subRow if err := r.subLock(tx.Ctx(ctx).Model(subTable).Where("id", hint.ID)).Scan(&row); err != nil { return err } if (row.Status != "review" && row.Status != "review_query") || row.SubmittedAt == nil || (row.LeaseUntil != nil && row.LeaseUntil.After(now)) { return nil } sub, err := decodeSub(row) if err != nil { return err } result, err := tx.Ctx(ctx).Model(subTable).Where("id", row.ID).Where("status", row.Status).Where("lease_until IS NULL OR lease_until <= ?", now).Data(g.Map{"status": "review_query", "lease_token": token, "lease_until": now.Add(subLease)}).Update() if err != nil { return err } n, err := result.RowsAffected() if err != nil || n != 1 { return err } sub.Status = "review_query" sub.LeaseToken = token sub.LeaseUntil = now.Add(subLease) out = &sub return nil }) return out, err } // ListReviewSubBatches 只列出该父批下已冻结、仍待核查的子批. // 手动同步据此按既有 operationId 查询上游,不重发办理,也不重开自动补查窗口。 func (r *Repository) ListReviewSubBatches(ctx context.Context, batchNo string) ([]string, error) { var rows []struct { OperationID string `orm:"operation_id"` } if err := r.database().Model(subTable).Ctx(ctx). Fields("operation_id"). Where("batch_no", batchNo). WhereIn("status", []string{"review", "review_query"}). Where("lease_until IS NULL OR lease_until <= ?", time.Now()). Order("id"). Scan(&rows); err != nil { return nil, err } out := make([]string, 0, len(rows)) for _, row := range rows { if id := strings.TrimSpace(row.OperationID); id != "" { out = append(out, id) } } return out, nil } func (r *Repository) UpdateSubBatch(ctx context.Context, batch data.SubBatch, status, jobID, reason string, next time.Time, pollCount int) error { return r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { var nextAt any = gdb.Raw("NULL") if !next.IsZero() { nextAt = next } result, err := tx.Ctx(ctx).Model(subTable).Where("id", batch.ID).Where("lease_token", batch.LeaseToken).WhereGT("lease_until", time.Now()).WhereIn("status", []string{"submitting", "accepted", "uncertain", "review_query"}).Data(g.Map{"status": status, "job_id": jobID, "last_reason": truncateReason(reason, 1024), "next_check_at": nextAt, "poll_count": pollCount, "lease_token": gdb.Raw("NULL"), "lease_until": gdb.Raw("NULL")}).Update() if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("APN租约已失效: %s", batch.OperationID) } if status == "review" { if _, err = tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Where("batch_no", batch.BatchNo).WhereIn("id", batch.DetailIDs).WhereIn("detail_status", []int{consts.BatchDetailStatusRunning, consts.BatchDetailStatusUnknown}).Data(g.Map{"detail_status": consts.BatchDetailStatusUnknown, "error_code": "UNKNOWN", "error_reason": batchfields.DetailReason(reason)}).Update(); err != nil { return err } } if status != "review" && status != "done" { return nil } // 手动补查把 review_query 子批改成终态时父批可能已是待核查,需重新纳入汇总。 return r.releaseReviewParentWhenSettled(ctx, tx, batch.BatchNo) }) } func truncateReason(value string, n int) string { runes := []rune(value) if len(runes) > n { return string(runes[:n]) } return value } func (r *Repository) SaveSubBatchProgress(ctx context.Context, batch data.SubBatch, outcomes []data.Outcome) error { return r.saveProgress(ctx, batch, outcomes, false) } func (r *Repository) CompleteSubBatch(ctx context.Context, batch data.SubBatch, outcomes []data.Outcome) error { return r.saveProgress(ctx, batch, outcomes, true) } func (r *Repository) saveProgress(ctx context.Context, batch data.SubBatch, outcomes []data.Outcome, complete bool) error { if len(outcomes) != len(batch.DetailIDs) { return fmt.Errorf("APN明细结果数量不符") } return r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { var owner struct { TenantID string `orm:"tenant_id"` } if err := tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("tenant_id").Where("batch_no", batch.BatchNo).Scan(&owner); err != nil { return err } if _, err := r.lockSubCards(ctx, tx, owner.TenantID, batch.Request.SimKeys); err != nil { return err } var current subRow if err := r.subLock(tx.Ctx(ctx).Model(subTable).Where("id", batch.ID)).Scan(¤t); err != nil { return err } if current.LeaseToken != batch.LeaseToken || current.LeaseUntil == nil || !current.LeaseUntil.After(time.Now()) || current.Status == "done" || current.Status == "review" { return fmt.Errorf("APN进度租约失效") } if batch.JobID != "" { if _, err := tx.Ctx(ctx).Model(subTable).Where("id", batch.ID).Where("lease_token", batch.LeaseToken).Data(g.Map{"job_id": batch.JobID}).Update(); err != nil { return err } } task, err := r.loadTaskDetails(ctx, batch.BatchNo, batch.DetailIDs) if err != nil { return err } byID := map[int64]*data.Detail{} for _, detail := range task.Details { byID[detail.ID] = detail } order := make([]int, len(batch.DetailIDs)) for i := range order { order[i] = i } sort.Slice(order, func(i, j int) bool { return batch.Request.SimKeys[order[i]].ICCID < batch.Request.SimKeys[order[j]].ICCID }) for _, i := range order { outcome := outcomes[i] if outcome.Pending { continue } detail := byID[batch.DetailIDs[i]] if detail == nil { return fmt.Errorf("缺少APN明细") } if detail.DetailStatus != consts.BatchDetailStatusRunning && !(batch.Status == "review_query" && detail.DetailStatus == consts.BatchDetailStatusUnknown) { continue } var claim cardClaim if err := r.subLock(tx.Ctx(ctx).Model(claimTable).Where("tenant_id", task.Batch.TenantID).Where("iccid", detail.ICCID)).Scan(&claim); err != nil { return err } previousStatus := detail.DetailStatus detail.ApplyOutcome(outcome) detail.Reason = batchfields.DetailReason(detail.Reason) if detail.Card != nil { detail.StatusBefore = detail.Card.DataServiceStatus detail.StatusAfter = detail.StatusBefore } status := consts.BatchDetailStatusFailed if outcome.Success { status = consts.BatchDetailStatusSuccess detail.StatusAfter = task.Operation.TargetStatus } if outcome.Success && detail.Card != nil && claim.ActiveOperationID == batch.OperationID { if _, err := tx.Ctx(ctx).Model(dao.TSimCardInfo.Table()).Where("id", detail.Card.ID).Where("tenant_id", task.Batch.TenantID).Data(dataServiceUpdateData(task, detail)).Update(); err != nil { return err } } result, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Where("id", detail.ID).Where("detail_status", previousStatus).Data(g.Map{"detail_status": status, "error_code": fmt.Sprint(outcome.Code), "error_reason": detail.Reason, "snapshot": detail.Snapshot(task.Operation, task.ApnName).String(), "updated_by": task.Batch.CreatedBy}).Update() if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("APN明细终态CAS失败") } if _, err := tx.Ctx(ctx).Model(claimTable).Where("tenant_id", task.Batch.TenantID).Where("iccid", detail.ICCID).Where("active_operation_id", batch.OperationID).Data(g.Map{"active_operation_id": "", "active_batch_id": 0}).Update(); err != nil { return err } } if complete { n, err := tx.Ctx(ctx).Model(dao.BatchDetail.Table()).WhereIn("id", batch.DetailIDs).WhereIn("detail_status", []int{consts.BatchDetailStatusRunning, consts.BatchDetailStatusUnknown}).Count() if err != nil { return err } if n != 0 { return fmt.Errorf("APN子批仍有未确认明细,不能完成") } if _, err = tx.Ctx(ctx).Model(subTable).Where("id", batch.ID).Where("lease_token", batch.LeaseToken).Data(g.Map{"status": "done", "job_id": batch.JobID, "next_check_at": gdb.Raw("NULL"), "lease_token": gdb.Raw("NULL"), "lease_until": gdb.Raw("NULL")}).Update(); err != nil { return err } return r.releaseReviewParentWhenSettled(ctx, tx, batch.BatchNo) } return nil }) } // releaseReviewParentWhenSettled 在同一事务里:当子批转入终态、且该父批已无未终结子批时, // 把父批从“待核查”退回“处理中”,交给既有恢复扫描重新触发汇总。 // 修复“子批与明细都已终态,但汇总失败或进程退出后父批永久挂在待核查”的死角。 // 只在确有终态转移处调用,因此合法的待核查批次不会被反复捞起空转。 // 状态读取不加锁,最后用带状态的 CAS 更新收口,避免在已持有卡/子批锁的事务里再锁父批造成锁序反转。 func (r *Repository) releaseReviewParentWhenSettled(ctx context.Context, tx gdb.TX, batchNo string) error { var main struct { Status int `orm:"batch_async_status"` } if err := noRows(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("batch_async_status").Where("batch_no", batchNo).Scan(&main)); err != nil { return err } if main.Status != consts.BatchAsyncStatusReview { return nil } // 只有已完成准备(ready=1)的父批才允许回到处理中,避免恢复扫描重跑准备阶段产生重复提交。 var parent struct { Ready int `orm:"ready"` } if err := noRows(tx.Ctx(ctx).Model(parentTable).Fields("ready").Where("batch_no", batchNo).Scan(&parent)); err != nil { return err } if parent.Ready != 1 { return nil } // 待核查父批的子批只可能是终态或在途手动补查:在途补查不阻塞退回, // 因为 FinalizeReadyBatch 会再校验一次;不依赖一致读快照,避免并发补查互相看不见对方的终态。 pending, err := tx.Ctx(ctx).Model(subTable).Where("batch_no", batchNo).WhereNotIn("status", []string{"done", "review", "review_query"}).Count() if err != nil || pending != 0 { return err } _, err = tx.Ctx(ctx).Model(dao.BatchMain.Table()).Where("batch_no", batchNo).Where("batch_async_status", consts.BatchAsyncStatusReview).Data(g.Map{"batch_async_status": consts.BatchAsyncStatusProcessing}).Update() return err } func (r *Repository) FinalizeReadyBatch(ctx context.Context, batchNo string) error { token := uuid.NewString() claimed := false err := r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { claimed = false var main struct { Status int `orm:"batch_async_status"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("batch_async_status").Where("batch_no", batchNo)).Scan(&main)); err != nil { return err } if main.Status != consts.BatchAsyncStatusProcessing && main.Status != consts.BatchAsyncStatusReview { return nil } var parent struct { Ready int `orm:"ready"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(parentTable).Where("batch_no", batchNo)).Scan(&parent)); err != nil { return err } if parent.Ready != 1 { return nil } n, err := tx.Ctx(ctx).Model(subTable).Where("batch_no", batchNo).WhereNotIn("status", []string{"done", "review"}).Count() if err != nil || n != 0 { return err } n, err = tx.Ctx(ctx).Model(dao.BatchDetail.Table()).Where("batch_no", batchNo).WhereIn("detail_status", []int{consts.BatchDetailStatusPending, consts.BatchDetailStatusRunning}).Count() if err != nil || n != 0 { return err } n, err = tx.Ctx(ctx).Model(dao.BatchMain.Table()).Where("batch_no", batchNo).WhereIn("batch_async_status", []int{consts.BatchAsyncStatusProcessing, consts.BatchAsyncStatusReview}).Count() if err != nil || n == 0 { return err } result, err := tx.Ctx(ctx).Model(parentTable).Where("batch_no", batchNo).Where("finalize_until IS NULL OR finalize_until <= ?", time.Now()).Data(g.Map{"finalize_token": token, "finalize_until": time.Now().Add(subLease)}).Update() if err != nil { return err } count, err := result.RowsAffected() claimed = count == 1 return err }) if err != nil || !claimed { return err } finalCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 4*time.Minute) defer cancel() task, err := r.loadTask(finalCtx, batchNo, true) if err == nil { // 冻结归属决定有效数:只有通过校验的明细才会进入子批。 if err = r.markFrozenValidation(finalCtx, task); err == nil { task.ValidationOK = true err = r.Finalize(context.WithValue(finalCtx, finalizeTokenKey{}, token), task) } } if err != nil { _, _ = r.database().Model(parentTable).Ctx(finalCtx).Where("batch_no", batchNo).Where("finalize_token", token).Data(g.Map{"finalize_token": gdb.Raw("NULL"), "finalize_until": gdb.Raw("NULL"), "last_reason": truncateReason(err.Error(), 1024)}).Update() } return err } func (r *Repository) ListRecoverableBatches(ctx context.Context, limit int) ([]string, error) { var rows []struct { BatchNo string `orm:"batch_no"` } // 未准备父批与可最终化父批轮转领取;在途父批交给子任务扫描,不挤占准备恢复窗口。 err := r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { rows = nil q := tx.Ctx(ctx).Model(dao.BatchMain.Table()+" m").InnerJoin(parentTable+" p", "p.batch_no=m.batch_no").Fields("m.batch_no").WhereIn("m.batch_type", []int{107, 109, 110}).Where("m.batch_async_status", consts.BatchAsyncStatusProcessing).Where("m.start_time IS NULL OR m.start_time <= ?", time.Now().Add(-30*time.Second)).Where("p.ready=0 OR (NOT EXISTS (SELECT 1 FROM data_operation_sub_batch s WHERE s.batch_no=m.batch_no AND s.status NOT IN ('done','review')) AND NOT EXISTS (SELECT 1 FROM batch_detail d WHERE d.batch_no=m.batch_no AND d.detail_status IN (0,1)))").Order("COALESCE(p.recovery_attempt_at,'1970-01-01'),m.id").Limit(limit) if err := q.Scan(&rows); err != nil { return err } for _, row := range rows { if _, err := tx.Ctx(ctx).Model(parentTable).Where("batch_no", row.BatchNo).Data(g.Map{"recovery_attempt_at": time.Now()}).Update(); err != nil { return err } } return nil }) out := make([]string, 0, len(rows)) for _, row := range rows { out = append(out, row.BatchNo) } return out, err } func (r *Repository) ProtectExecutionFailure(ctx context.Context, batchNo, reason string) error { return r.durableTx(ctx, func(ctx context.Context, tx gdb.TX) error { var main struct { Status int `orm:"batch_async_status"` } if err := noRows(r.subLock(tx.Ctx(ctx).Model(dao.BatchMain.Table()).Fields("batch_async_status").Where("batch_no", batchNo)).Scan(&main)); err != nil { return err } if main.Status != consts.BatchAsyncStatusProcessing { return nil } _, err := tx.Ctx(ctx).Model(parentTable).Where("batch_no", batchNo).Data(g.Map{"last_reason": truncateReason("执行中断,交持久化调度恢复:"+reason, 1024)}).Update() return err }) }