chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,242 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
const (
|
||||
restartStageStop = "stop"
|
||||
restartStageResume = "resume"
|
||||
)
|
||||
|
||||
type RestartRepository interface {
|
||||
FindRestartMain(ctx context.Context, batchNo string) (*entity.BatchMain, error)
|
||||
LoadRestartStop(ctx context.Context, batchNo string) (*Task, error)
|
||||
LoadRestartResume(ctx context.Context, batchNo string) (*Task, error)
|
||||
SaveRestartValidationFailure(ctx context.Context, task *Task, stage string) error
|
||||
SaveRestartResult(ctx context.Context, task *Task, detail *Detail, outcome Outcome, stage string) (Outcome, error)
|
||||
WaitRestartResume(ctx context.Context, task *Task, resumeAt *gtime.Time) error
|
||||
ClaimRestartResume(ctx context.Context, batchNo string, now *gtime.Time) (bool, error)
|
||||
FinalizeRestart(ctx context.Context, batchNo string) error
|
||||
}
|
||||
|
||||
type RestartPublisher func(ctx context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error
|
||||
|
||||
type restartExecutor struct {
|
||||
repository RestartRepository
|
||||
upstream Upstream
|
||||
publishDelayed RestartPublisher
|
||||
now func() *gtime.Time
|
||||
}
|
||||
|
||||
func NewRestartExecutor(repository RestartRepository, upstream Upstream) Executor {
|
||||
return NewRestartExecutorWithPublisher(repository, upstream, nil)
|
||||
}
|
||||
|
||||
func NewRestartExecutorWithPublisher(repository RestartRepository, upstream Upstream, publisher RestartPublisher) Executor {
|
||||
if publisher == nil {
|
||||
publisher = publishRestartDelayed
|
||||
}
|
||||
return &restartExecutor{
|
||||
repository: repository,
|
||||
upstream: upstream,
|
||||
publishDelayed: publisher,
|
||||
now: gtime.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func publishRestartDelayed(ctx context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error {
|
||||
body, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "marshal restart resume message")
|
||||
}
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
return mq.PublishDelayedBytesWithHeaders(publishCtx, delayQueue, targetQueue, delay, body, nil)
|
||||
}
|
||||
|
||||
func (e *restartExecutor) CanExecuteProcessing(ctx context.Context, main *entity.BatchMain) (bool, error) {
|
||||
if main == nil || main.BatchType != consts.BatchTypeSimCardRestart || main.BatchAsyncStatus != consts.BatchAsyncStatusProcessing {
|
||||
return false, nil
|
||||
}
|
||||
if restartPhase(main.BatchInfo) != consts.RestartPhaseWaitingResume {
|
||||
return false, nil
|
||||
}
|
||||
resumeAt := restartResumeAt(main.BatchInfo)
|
||||
if resumeAt == nil {
|
||||
g.Log().Warningf(ctx, "restart batch waiting without resumeAt, batchNo:%s", main.BatchNo)
|
||||
return false, nil
|
||||
}
|
||||
now := e.currentTime()
|
||||
if resumeAt.After(now) {
|
||||
delay := resumeAt.Sub(now)
|
||||
if delay < time.Millisecond {
|
||||
delay = time.Millisecond
|
||||
}
|
||||
message := batch.Message{BatchNo: main.BatchNo, BatchType: main.BatchType}
|
||||
if err := e.publishDelayed(ctx, consts.QueueBatchRestartResumeDelay, consts.QueueBatchExecute, delay, message); err != nil {
|
||||
return false, gerror.Wrap(err, "reschedule early restart resume message")
|
||||
}
|
||||
g.Log().Infof(ctx, "restart batch resume message rescheduled before resumeAt, batchNo:%s, resumeAt:%s, delay:%s", main.BatchNo, resumeAt.String(), delay)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (e *restartExecutor) ClaimsProcessingReentry() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *restartExecutor) Execute(ctx context.Context, batchNo string) error {
|
||||
batchNo = strings.TrimSpace(batchNo)
|
||||
if batchNo == "" {
|
||||
return gerror.New("restart batch number is empty")
|
||||
}
|
||||
main, err := e.repository.FindRestartMain(ctx, batchNo)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "load restart batch main")
|
||||
}
|
||||
if main == nil {
|
||||
return gerror.New("restart batch does not exist")
|
||||
}
|
||||
if restartPhase(main.BatchInfo) == consts.RestartPhaseWaitingResume {
|
||||
claimed, claimErr := e.repository.ClaimRestartResume(ctx, batchNo, e.currentTime())
|
||||
if claimErr != nil {
|
||||
return gerror.Wrap(claimErr, "claim restart resume phase")
|
||||
}
|
||||
if !claimed {
|
||||
g.Log().Infof(ctx, "restart resume message ignored after CAS lost, batchNo:%s", batchNo)
|
||||
return nil
|
||||
}
|
||||
return e.executeResume(ctx, batchNo)
|
||||
}
|
||||
return e.executeStop(ctx, batchNo)
|
||||
}
|
||||
|
||||
func (e *restartExecutor) executeStop(ctx context.Context, batchNo string) error {
|
||||
task, err := e.repository.LoadRestartStop(ctx, batchNo)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "load restart stop phase")
|
||||
}
|
||||
if !validateTask(task) {
|
||||
if err = e.repository.SaveRestartValidationFailure(ctx, task, restartStageStop); err != nil {
|
||||
return gerror.Wrap(err, "save restart stop validation failure")
|
||||
}
|
||||
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart stop validation failure")
|
||||
}
|
||||
|
||||
successCount, err := e.changeRestartGroups(ctx, task, restartStageStop)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if successCount == 0 {
|
||||
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart with no stop success")
|
||||
}
|
||||
|
||||
delay := restartDelay(task.Batch.BatchInfo)
|
||||
resumeAt := e.currentTime().Add(delay)
|
||||
if err = e.repository.WaitRestartResume(ctx, task, resumeAt); err != nil {
|
||||
return gerror.Wrap(err, "mark restart waiting resume")
|
||||
}
|
||||
message := batch.Message{BatchNo: task.Batch.BatchNo, BatchType: consts.BatchTypeSimCardRestart}
|
||||
if err = e.publishDelayed(ctx, consts.QueueBatchRestartResumeDelay, consts.QueueBatchExecute, delay, message); err != nil {
|
||||
return gerror.Wrap(err, "publish restart resume delayed message")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *restartExecutor) executeResume(ctx context.Context, batchNo string) error {
|
||||
task, err := e.repository.LoadRestartResume(ctx, batchNo)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "load restart resume phase")
|
||||
}
|
||||
if !validateTask(task) {
|
||||
if err = e.repository.SaveRestartValidationFailure(ctx, task, restartStageResume); err != nil {
|
||||
return gerror.Wrap(err, "save restart resume validation failure")
|
||||
}
|
||||
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart resume validation failure")
|
||||
}
|
||||
if _, err = e.changeRestartGroups(ctx, task, restartStageResume); err != nil {
|
||||
return err
|
||||
}
|
||||
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart batch")
|
||||
}
|
||||
|
||||
func (e *restartExecutor) changeRestartGroups(ctx context.Context, task *Task, stage string) (int, error) {
|
||||
groups := groupByChanKey(task.Details)
|
||||
keys := make([]string, 0, len(groups))
|
||||
for chanKey := range groups {
|
||||
keys = append(keys, chanKey)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
successCount := 0
|
||||
for _, chanKey := range keys {
|
||||
outcomes := (&executor{upstream: e.upstream}).changeGroup(ctx, task, groups[chanKey])
|
||||
for index, outcome := range outcomes {
|
||||
detail := groups[chanKey][index]
|
||||
resolved, saveErr := e.repository.SaveRestartResult(ctx, task, detail, outcome, stage)
|
||||
if saveErr != nil {
|
||||
return successCount, gerror.Wrapf(saveErr, "save restart %s detail %d result", stage, detail.ID)
|
||||
}
|
||||
detail.ApplyOutcome(resolved)
|
||||
if resolved.Success {
|
||||
successCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
return successCount, nil
|
||||
}
|
||||
|
||||
func (e *restartExecutor) currentTime() *gtime.Time {
|
||||
if e.now != nil {
|
||||
return e.now()
|
||||
}
|
||||
return gtime.Now()
|
||||
}
|
||||
|
||||
func restartDelay(info *gjson.Json) time.Duration {
|
||||
seconds := 300
|
||||
if info != nil {
|
||||
if configured := info.Get("resumeDelaySeconds").Int(); configured > 0 {
|
||||
seconds = configured
|
||||
}
|
||||
}
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
func restartPhase(info *gjson.Json) string {
|
||||
if info == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(info.Get("restart.phase").String())
|
||||
}
|
||||
|
||||
func restartResumeAt(info *gjson.Json) *gtime.Time {
|
||||
if info == nil {
|
||||
return nil
|
||||
}
|
||||
value := strings.TrimSpace(info.Get("restart.resumeAt").String())
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
parsed := gtime.NewFromStr(value)
|
||||
if parsed == nil || parsed.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
Reference in New Issue
Block a user