chore: 初始开源发布
iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。 - Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件） - 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、 库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异， 不破坏「校验 MSISDN 格式」等测试断言） - 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23）， 移除写死的 git 凭据 - README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 已知：3 个测试包在源仓同一提交上同样失败（client/upstream、 service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。
This commit is contained in:
MiFaZhan
@@ -0,0 +1,398 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
type executor struct {
|
||||
repository Repository
|
||||
upstream Upstream
|
||||
}
|
||||
|
||||
func NewExecutor(repository Repository, upstream Upstream) Executor {
|
||||
return &executor{repository: repository, upstream: upstream}
|
||||
}
|
||||
|
||||
func (e *executor) Execute(ctx context.Context, batchNo string) error {
|
||||
task, err := e.repository.Load(ctx, strings.TrimSpace(batchNo))
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "load data service batch")
|
||||
}
|
||||
if task.Operation.BatchType == consts.BatchTypeDataCut {
|
||||
return e.executeTrafficCutoff(ctx, task)
|
||||
}
|
||||
if !validateTask(task) {
|
||||
if err = e.repository.SaveValidationFailure(ctx, task); err != nil {
|
||||
return gerror.Wrap(err, "save data service validation failure")
|
||||
}
|
||||
return gerror.Wrap(e.repository.Finalize(ctx, task), "finalize data service validation failure")
|
||||
}
|
||||
|
||||
return e.executeValidated(ctx, task)
|
||||
}
|
||||
|
||||
func (e *executor) executeTrafficCutoff(ctx context.Context, task *Task) error {
|
||||
if !validateTrafficCutoffTask(task) {
|
||||
if err := e.repository.SaveValidationFailure(ctx, task); err != nil {
|
||||
return gerror.Wrap(err, "save traffic cutoff validation failure")
|
||||
}
|
||||
return gerror.Wrap(e.repository.Finalize(ctx, task), "finalize traffic cutoff validation failure")
|
||||
}
|
||||
if err := e.refreshDataServiceState(ctx, task); err != nil {
|
||||
return gerror.Wrap(err, "refresh data service state")
|
||||
}
|
||||
if err := e.saveInvalidTrafficCutoffRows(ctx, task); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.saveIdempotentCutoffSkips(ctx, task); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.executeValidated(ctx, task)
|
||||
}
|
||||
|
||||
func (e *executor) executeValidated(ctx context.Context, task *Task) error {
|
||||
groups := groupByChanKey(task.Operation, task.Details)
|
||||
keys := make([]string, 0, len(groups))
|
||||
for chanKey := range groups {
|
||||
keys = append(keys, chanKey)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, chanKey := range keys {
|
||||
if err := e.changeGroup(ctx, task, groups[chanKey]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return gerror.Wrap(e.repository.Finalize(ctx, task), "finalize data service batch")
|
||||
}
|
||||
|
||||
type dataServiceStateRefresher interface {
|
||||
RefreshDataServiceState(ctx context.Context, task *Task) error
|
||||
}
|
||||
|
||||
func (e *executor) refreshDataServiceState(ctx context.Context, task *Task) error {
|
||||
refresher, ok := e.repository.(dataServiceStateRefresher)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return refresher.RefreshDataServiceState(ctx, task)
|
||||
}
|
||||
|
||||
func (e *executor) saveIdempotentCutoffSkips(ctx context.Context, task *Task) error {
|
||||
if task == nil || task.Operation.BatchType != consts.BatchTypeDataCut {
|
||||
return nil
|
||||
}
|
||||
for _, detail := range task.Details {
|
||||
if !skipUpstreamForIdempotentCutoff(task.Operation, detail) {
|
||||
continue
|
||||
}
|
||||
resolved, saveErr := e.repository.SaveResult(ctx, task, detail, Outcome{
|
||||
Success: true,
|
||||
Message: "data service is already disabled; upstream request was skipped",
|
||||
})
|
||||
if saveErr != nil {
|
||||
return gerror.Wrapf(saveErr, "save data service detail %d idempotent skip result", detail.ID)
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateTask(task *Task) bool {
|
||||
if task == nil || len(task.Details) == 0 {
|
||||
return false
|
||||
}
|
||||
hasFailure := false
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil {
|
||||
hasFailure = true
|
||||
continue
|
||||
}
|
||||
validateDetail(task.Operation, detail)
|
||||
if detail.Reason != "" {
|
||||
hasFailure = true
|
||||
}
|
||||
}
|
||||
if !hasFailure {
|
||||
task.ValidationOK = true
|
||||
for _, detail := range task.Details {
|
||||
if detail != nil {
|
||||
detail.ValidationPassed = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, detail := range task.Details {
|
||||
if detail != nil && detail.Reason == "" {
|
||||
detail.Reason = "same batch contains validation failure; upstream request was skipped"
|
||||
}
|
||||
}
|
||||
task.FailureReason = "batch validation failed; upstream request was skipped"
|
||||
return false
|
||||
}
|
||||
|
||||
func validateTrafficCutoffTask(task *Task) bool {
|
||||
if task == nil || len(task.Details) == 0 {
|
||||
if task != nil {
|
||||
task.FailureReason = "traffic cutoff batch has no valid details"
|
||||
}
|
||||
return false
|
||||
}
|
||||
hasSuccess := false
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil {
|
||||
continue
|
||||
}
|
||||
validateDetail(task.Operation, detail)
|
||||
detail.ValidationPassed = detail.Reason == ""
|
||||
if detail.ValidationPassed {
|
||||
hasSuccess = true
|
||||
}
|
||||
}
|
||||
if hasSuccess {
|
||||
task.ValidationOK = true
|
||||
if failed := countFailedDetails(task); failed > 0 {
|
||||
task.FailureReason = strconv.Itoa(failed) + " SIM cards failed validation"
|
||||
} else {
|
||||
task.FailureReason = ""
|
||||
}
|
||||
return true
|
||||
}
|
||||
task.FailureReason = "traffic cutoff batch validation failed; upstream request was skipped"
|
||||
return false
|
||||
}
|
||||
|
||||
func countFailedDetails(task *Task) int {
|
||||
if task == nil {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil || detail.Reason != "" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func validateDetail(_ Operation, detail *Detail) {
|
||||
if detail.Reason != "" {
|
||||
return
|
||||
}
|
||||
if detail.Card == nil {
|
||||
detail.Reason = "ICCID does not exist in SIM card records"
|
||||
return
|
||||
}
|
||||
detail.MSISDN = strings.TrimSpace(detail.Card.MSISDN)
|
||||
detail.IMSI = strings.TrimSpace(detail.Card.IMSI)
|
||||
detail.ChanKey = strings.TrimSpace(detail.Card.ChanKey)
|
||||
detail.StatusBefore = detail.Card.DataServiceStatus
|
||||
detail.StatusAfter = detail.Card.DataServiceStatus
|
||||
|
||||
if !detail.Authorized {
|
||||
detail.Reason = "ICCID is outside the current operator authorization scope"
|
||||
return
|
||||
}
|
||||
if detail.ChanKey == "" {
|
||||
detail.Reason = "ICCID is missing chanKey"
|
||||
return
|
||||
}
|
||||
if detail.InputMSISDN != "" && detail.InputMSISDN != detail.MSISDN {
|
||||
detail.Reason = "MSISDN does not match the SIM card record"
|
||||
return
|
||||
}
|
||||
if detail.InputIMSI != "" && detail.InputIMSI != detail.IMSI {
|
||||
detail.Reason = "IMSI does not match the SIM card record"
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func groupByChanKey(operation Operation, details []*Detail) map[string][]*Detail {
|
||||
groups := make(map[string][]*Detail)
|
||||
for _, detail := range details {
|
||||
if operation.BatchType == consts.BatchTypeDataCut && (detail == nil || !detail.ValidationPassed || detail.Reason != "") {
|
||||
continue
|
||||
}
|
||||
if skipUpstreamForIdempotentCutoff(operation, detail) {
|
||||
continue
|
||||
}
|
||||
groups[detail.ChanKey] = append(groups[detail.ChanKey], detail)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func (e *executor) saveInvalidTrafficCutoffRows(ctx context.Context, task *Task) error {
|
||||
if task == nil {
|
||||
return nil
|
||||
}
|
||||
for _, detail := range task.Details {
|
||||
if detail == nil || detail.ValidationPassed || detail.Reason == "" {
|
||||
continue
|
||||
}
|
||||
resolved, saveErr := e.repository.SaveResult(ctx, task, detail, Outcome{
|
||||
Message: detail.Reason,
|
||||
})
|
||||
if saveErr != nil {
|
||||
return gerror.Wrapf(saveErr, "save traffic cutoff detail %d validation result", detail.ID)
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func skipUpstreamForIdempotentCutoff(operation Operation, detail *Detail) bool {
|
||||
return operation.BatchType == consts.BatchTypeDataCut &&
|
||||
detail != nil &&
|
||||
detail.ValidationPassed &&
|
||||
detail.StatusBefore == consts.SimDataServiceStatusDisabled &&
|
||||
detail.StatusAfter == consts.SimDataServiceStatusDisabled
|
||||
}
|
||||
|
||||
// changeGroup 把同一通道下的明细切成子批次并发调用上游，并在**每个子批次拿到结果后立即落库该子批的明细**。
|
||||
//
|
||||
// 不再等所有子批次跑完才统一写库：大批量（如 2 万张 = 200 个子批）执行时，执行日志可以按子批
|
||||
// 实时反映进度，而不是在整个批次结束前始终全部停留在"执行中"。子批之间相互隔离——某个子批
|
||||
// 落库失败只记录错误、不影响其他子批继续办理与落库。
|
||||
func (e *executor) changeGroup(ctx context.Context, task *Task, details []*Detail) error {
|
||||
subBatches := splitDetails(details, consts.BatchSubBatchSize)
|
||||
|
||||
sem := make(chan struct{}, consts.BatchSubBatchConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
|
||||
for _, sub := range subBatches {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(batch []*Detail) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
if err := e.changeAndSaveSubBatch(ctx, task, batch); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}(sub)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// changeAndSaveSubBatch 处理单个子批次：构建请求、调用上游，然后逐条落库本子批的明细。
|
||||
// 明细结果只在所属子批次落库，天然做到"查到一个子批就落库一个子批"。
|
||||
func (e *executor) changeAndSaveSubBatch(ctx context.Context, task *Task, details []*Detail) error {
|
||||
for index, outcome := range e.changeSubBatch(ctx, task, details) {
|
||||
detail := details[index]
|
||||
resolved, saveErr := e.repository.SaveResult(ctx, task, detail, outcome)
|
||||
if saveErr != nil {
|
||||
return gerror.Wrapf(saveErr, "save data service detail %d result", detail.ID)
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// changeSubBatch 处理单个子批次：构建请求并调用上游，返回与该子批次明细等长的 Outcome 切片。
|
||||
func (e *executor) changeSubBatch(ctx context.Context, task *Task, details []*Detail) []Outcome {
|
||||
input := &model.SimCardApnChangeInput{
|
||||
SimKeys: make([]*model.SimKey, 0, len(details)),
|
||||
ReqData: model.SimCardApnChangeInputItem{OperType: task.Operation.ApnOperType, ApnName: task.ApnName},
|
||||
}
|
||||
for _, detail := range details {
|
||||
input.SimKeys = append(input.SimKeys, &model.SimKey{ChanKey: detail.ChanKey, ICCID: detail.ICCID, MSISDN: detail.MSISDN, IMSI: detail.IMSI})
|
||||
}
|
||||
response, err := e.upstream.ChangeSimApnFunction(ctx, input)
|
||||
if err != nil {
|
||||
outcomes := make([]Outcome, len(details))
|
||||
for index := range outcomes {
|
||||
outcomes[index].Message = "upstream request failed: " + err.Error()
|
||||
}
|
||||
return outcomes
|
||||
}
|
||||
return mapOutcomes(details, response)
|
||||
}
|
||||
|
||||
// splitDetails 按固定大小把明细切成有序的子批次；空输入返回空切片。
|
||||
func splitDetails(details []*Detail, size int) [][]*Detail {
|
||||
if size <= 0 {
|
||||
size = 1
|
||||
}
|
||||
batches := make([][]*Detail, 0, (len(details)+size-1)/size)
|
||||
for i := 0; i < len(details); i += size {
|
||||
end := i + size
|
||||
if end > len(details) {
|
||||
end = len(details)
|
||||
}
|
||||
batches = append(batches, details[i:end])
|
||||
}
|
||||
return batches
|
||||
}
|
||||
|
||||
func mapOutcomes(details []*Detail, response []model.SimBaseRes[model.SimCardApnChangeOutputItem]) []Outcome {
|
||||
outcomes := make([]Outcome, len(details))
|
||||
used := make([]bool, len(response))
|
||||
for detailIndex, detail := range details {
|
||||
matched := -1
|
||||
for responseIndex, item := range response {
|
||||
if !used[responseIndex] && sameSim(detail, item.SimKey) {
|
||||
matched = responseIndex
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched < 0 && detailIndex < len(response) && !used[detailIndex] && response[detailIndex].SimKey == nil {
|
||||
matched = detailIndex
|
||||
}
|
||||
if matched < 0 {
|
||||
outcomes[detailIndex] = Outcome{Message: "upstream did not return a result for this SIM card"}
|
||||
continue
|
||||
}
|
||||
used[matched] = true
|
||||
item := response[matched]
|
||||
message := strings.TrimSpace(item.Message)
|
||||
if item.Code != 0 && message == "" {
|
||||
message = "upstream processing failed"
|
||||
}
|
||||
outcomes[detailIndex] = Outcome{Success: item.Code == 0, Code: item.Code, Message: message, OrderNum: strings.TrimSpace(item.ResData.OrderNum)}
|
||||
}
|
||||
return outcomes
|
||||
}
|
||||
|
||||
func sameSim(detail *Detail, key *model.SimKey) bool {
|
||||
if detail == nil || key == nil {
|
||||
return false
|
||||
}
|
||||
if value := strings.TrimSpace(key.ICCID); value != "" {
|
||||
return value == detail.ICCID
|
||||
}
|
||||
if value := strings.TrimSpace(key.MSISDN); value != "" {
|
||||
return value == detail.MSISDN
|
||||
}
|
||||
if value := strings.TrimSpace(key.IMSI); value != "" {
|
||||
return value == detail.IMSI
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func OperationByBatchType(batchType int) (Operation, error) {
|
||||
switch batchType {
|
||||
case consts.BatchTypeDataStart:
|
||||
return Operation{BatchType: batchType, Name: "dataStart", ApnOperType: consts.SimApnOperTypeOpen, TargetStatus: consts.SimDataServiceStatusEnabled}, nil
|
||||
case consts.BatchTypeDataCut:
|
||||
return Operation{BatchType: batchType, Name: "trafficCutoff", ApnOperType: consts.SimApnOperTypeStop, TargetStatus: consts.SimDataServiceStatusDisabled}, nil
|
||||
case consts.BatchTypeDataStop:
|
||||
return Operation{BatchType: batchType, Name: "dataStop", ApnOperType: consts.SimApnOperTypeStop, TargetStatus: consts.SimDataServiceStatusDisabled}, nil
|
||||
default:
|
||||
return Operation{}, gerror.Newf("unsupported data service batch type %d", batchType)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user