package lifecycle import ( "context" "encoding/json" "sort" "strings" "time" "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts" mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity" ) const ( restartStageStop = "stop" restartStageResume = "resume" ) type RestartRepository interface { FindRestartMain(ctx context.Context, batchNo string) (*entity.BatchMain, error) LoadRestartStop(ctx context.Context, batchNo string) (*Task, error) LoadRestartResume(ctx context.Context, batchNo string) (*Task, error) SaveRestartValidationFailure(ctx context.Context, task *Task, stage string) error SaveRestartResult(ctx context.Context, task *Task, detail *Detail, outcome Outcome, stage string) (Outcome, error) WaitRestartResume(ctx context.Context, task *Task, resumeAt *gtime.Time) error ClaimRestartResume(ctx context.Context, batchNo string, now *gtime.Time) (bool, error) FinalizeRestart(ctx context.Context, batchNo string) error } type RestartPublisher func(ctx context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error type restartExecutor struct { repository RestartRepository upstream Upstream publishDelayed RestartPublisher now func() *gtime.Time } func NewRestartExecutor(repository RestartRepository, upstream Upstream) Executor { return NewRestartExecutorWithPublisher(repository, upstream, nil) } func NewRestartExecutorWithPublisher(repository RestartRepository, upstream Upstream, publisher RestartPublisher) Executor { if publisher == nil { publisher = publishRestartDelayed } return &restartExecutor{ repository: repository, upstream: upstream, publishDelayed: publisher, now: gtime.Now, } } func publishRestartDelayed(ctx context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error { body, err := json.Marshal(data) if err != nil { return gerror.Wrap(err, "marshal restart resume message") } publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() return mq.PublishDelayedBytesWithHeaders(publishCtx, delayQueue, targetQueue, delay, body, nil) } func (e *restartExecutor) CanExecuteProcessing(ctx context.Context, main *entity.BatchMain) (bool, error) { if main == nil || main.BatchType != consts.BatchTypeSimCardRestart || main.BatchAsyncStatus != consts.BatchAsyncStatusProcessing { return false, nil } if restartPhase(main.BatchInfo) != consts.RestartPhaseWaitingResume { return false, nil } resumeAt := restartResumeAt(main.BatchInfo) if resumeAt == nil { g.Log().Warningf(ctx, "restart batch waiting without resumeAt, batchNo:%s", main.BatchNo) return false, nil } now := e.currentTime() if resumeAt.After(now) { delay := resumeAt.Sub(now) if delay < time.Millisecond { delay = time.Millisecond } message := batch.Message{BatchNo: main.BatchNo, BatchType: main.BatchType} if err := e.publishDelayed(ctx, consts.QueueBatchRestartResumeDelay, consts.QueueBatchExecute, delay, message); err != nil { return false, gerror.Wrap(err, "reschedule early restart resume message") } g.Log().Infof(ctx, "restart batch resume message rescheduled before resumeAt, batchNo:%s, resumeAt:%s, delay:%s", main.BatchNo, resumeAt.String(), delay) return false, nil } return true, nil } func (e *restartExecutor) ClaimsProcessingReentry() bool { return true } func (e *restartExecutor) Execute(ctx context.Context, batchNo string) error { batchNo = strings.TrimSpace(batchNo) if batchNo == "" { return gerror.New("restart batch number is empty") } main, err := e.repository.FindRestartMain(ctx, batchNo) if err != nil { return gerror.Wrap(err, "load restart batch main") } if main == nil { return gerror.New("restart batch does not exist") } if restartPhase(main.BatchInfo) == consts.RestartPhaseWaitingResume { claimed, claimErr := e.repository.ClaimRestartResume(ctx, batchNo, e.currentTime()) if claimErr != nil { return gerror.Wrap(claimErr, "claim restart resume phase") } if !claimed { g.Log().Infof(ctx, "restart resume message ignored after CAS lost, batchNo:%s", batchNo) return nil } return e.executeResume(ctx, batchNo) } return e.executeStop(ctx, batchNo) } func (e *restartExecutor) executeStop(ctx context.Context, batchNo string) error { task, err := e.repository.LoadRestartStop(ctx, batchNo) if err != nil { return gerror.Wrap(err, "load restart stop phase") } if !validateTask(task) { if err = e.repository.SaveRestartValidationFailure(ctx, task, restartStageStop); err != nil { return gerror.Wrap(err, "save restart stop validation failure") } return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart stop validation failure") } successCount, err := e.changeRestartGroups(ctx, task, restartStageStop) if err != nil { return err } if successCount == 0 { return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart with no stop success") } delay := restartDelay(task.Batch.BatchInfo) resumeAt := e.currentTime().Add(delay) if err = e.repository.WaitRestartResume(ctx, task, resumeAt); err != nil { return gerror.Wrap(err, "mark restart waiting resume") } message := batch.Message{BatchNo: task.Batch.BatchNo, BatchType: consts.BatchTypeSimCardRestart} if err = e.publishDelayed(ctx, consts.QueueBatchRestartResumeDelay, consts.QueueBatchExecute, delay, message); err != nil { return gerror.Wrap(err, "publish restart resume delayed message") } return nil } func (e *restartExecutor) executeResume(ctx context.Context, batchNo string) error { task, err := e.repository.LoadRestartResume(ctx, batchNo) if err != nil { return gerror.Wrap(err, "load restart resume phase") } if !validateTask(task) { if err = e.repository.SaveRestartValidationFailure(ctx, task, restartStageResume); err != nil { return gerror.Wrap(err, "save restart resume validation failure") } return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart resume validation failure") } if _, err = e.changeRestartGroups(ctx, task, restartStageResume); err != nil { return err } return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart batch") } func (e *restartExecutor) changeRestartGroups(ctx context.Context, task *Task, stage string) (int, error) { groups := groupByChanKey(task.Details) keys := make([]string, 0, len(groups)) for chanKey := range groups { keys = append(keys, chanKey) } sort.Strings(keys) successCount := 0 for _, chanKey := range keys { outcomes := (&executor{upstream: e.upstream}).changeGroup(ctx, task, groups[chanKey]) for index, outcome := range outcomes { detail := groups[chanKey][index] resolved, saveErr := e.repository.SaveRestartResult(ctx, task, detail, outcome, stage) if saveErr != nil { return successCount, gerror.Wrapf(saveErr, "save restart %s detail %d result", stage, detail.ID) } detail.ApplyOutcome(resolved) if resolved.Success { successCount++ } } } return successCount, nil } func (e *restartExecutor) currentTime() *gtime.Time { if e.now != nil { return e.now() } return gtime.Now() } func restartDelay(info *gjson.Json) time.Duration { seconds := 300 if info != nil { if configured := info.Get("resumeDelaySeconds").Int(); configured > 0 { seconds = configured } } return time.Duration(seconds) * time.Second } func restartPhase(info *gjson.Json) string { if info == nil { return "" } return strings.TrimSpace(info.Get("restart.phase").String()) } func restartResumeAt(info *gjson.Json) *gtime.Time { if info == nil { return nil } value := strings.TrimSpace(info.Get("restart.resumeAt").String()) if value == "" { return nil } parsed := gtime.NewFromStr(value) if parsed == nil || parsed.IsZero() { return nil } return parsed }