MiFaZhan
243 lines
8.0 KiB
Go
243 lines
8.0 KiB
Go
package lifecycle
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gogf/gf/v2/encoding/gjson"
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
"github.com/gogf/gf/v2/os/gtime"
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
|
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
|
)
|
|
|
|
const (
|
|
restartStageStop = "stop"
|
|
restartStageResume = "resume"
|
|
)
|
|
|
|
type RestartRepository interface {
|
|
FindRestartMain(ctx context.Context, batchNo string) (*entity.BatchMain, error)
|
|
LoadRestartStop(ctx context.Context, batchNo string) (*Task, error)
|
|
LoadRestartResume(ctx context.Context, batchNo string) (*Task, error)
|
|
SaveRestartValidationFailure(ctx context.Context, task *Task, stage string) error
|
|
SaveRestartResult(ctx context.Context, task *Task, detail *Detail, outcome Outcome, stage string) (Outcome, error)
|
|
WaitRestartResume(ctx context.Context, task *Task, resumeAt *gtime.Time) error
|
|
ClaimRestartResume(ctx context.Context, batchNo string, now *gtime.Time) (bool, error)
|
|
FinalizeRestart(ctx context.Context, batchNo string) error
|
|
}
|
|
|
|
type RestartPublisher func(ctx context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error
|
|
|
|
type restartExecutor struct {
|
|
repository RestartRepository
|
|
upstream Upstream
|
|
publishDelayed RestartPublisher
|
|
now func() *gtime.Time
|
|
}
|
|
|
|
func NewRestartExecutor(repository RestartRepository, upstream Upstream) Executor {
|
|
return NewRestartExecutorWithPublisher(repository, upstream, nil)
|
|
}
|
|
|
|
func NewRestartExecutorWithPublisher(repository RestartRepository, upstream Upstream, publisher RestartPublisher) Executor {
|
|
if publisher == nil {
|
|
publisher = publishRestartDelayed
|
|
}
|
|
return &restartExecutor{
|
|
repository: repository,
|
|
upstream: upstream,
|
|
publishDelayed: publisher,
|
|
now: gtime.Now,
|
|
}
|
|
}
|
|
|
|
func publishRestartDelayed(ctx context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error {
|
|
body, err := json.Marshal(data)
|
|
if err != nil {
|
|
return gerror.Wrap(err, "marshal restart resume message")
|
|
}
|
|
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
return mq.PublishDelayedBytesWithHeaders(publishCtx, delayQueue, targetQueue, delay, body, nil)
|
|
}
|
|
|
|
func (e *restartExecutor) CanExecuteProcessing(ctx context.Context, main *entity.BatchMain) (bool, error) {
|
|
if main == nil || main.BatchType != consts.BatchTypeSimCardRestart || main.BatchAsyncStatus != consts.BatchAsyncStatusProcessing {
|
|
return false, nil
|
|
}
|
|
if restartPhase(main.BatchInfo) != consts.RestartPhaseWaitingResume {
|
|
return false, nil
|
|
}
|
|
resumeAt := restartResumeAt(main.BatchInfo)
|
|
if resumeAt == nil {
|
|
g.Log().Warningf(ctx, "restart batch waiting without resumeAt, batchNo:%s", main.BatchNo)
|
|
return false, nil
|
|
}
|
|
now := e.currentTime()
|
|
if resumeAt.After(now) {
|
|
delay := resumeAt.Sub(now)
|
|
if delay < time.Millisecond {
|
|
delay = time.Millisecond
|
|
}
|
|
message := batch.Message{BatchNo: main.BatchNo, BatchType: main.BatchType}
|
|
if err := e.publishDelayed(ctx, consts.QueueBatchRestartResumeDelay, consts.QueueBatchExecute, delay, message); err != nil {
|
|
return false, gerror.Wrap(err, "reschedule early restart resume message")
|
|
}
|
|
g.Log().Infof(ctx, "restart batch resume message rescheduled before resumeAt, batchNo:%s, resumeAt:%s, delay:%s", main.BatchNo, resumeAt.String(), delay)
|
|
return false, nil
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (e *restartExecutor) ClaimsProcessingReentry() bool {
|
|
return true
|
|
}
|
|
|
|
func (e *restartExecutor) Execute(ctx context.Context, batchNo string) error {
|
|
batchNo = strings.TrimSpace(batchNo)
|
|
if batchNo == "" {
|
|
return gerror.New("restart batch number is empty")
|
|
}
|
|
main, err := e.repository.FindRestartMain(ctx, batchNo)
|
|
if err != nil {
|
|
return gerror.Wrap(err, "load restart batch main")
|
|
}
|
|
if main == nil {
|
|
return gerror.New("restart batch does not exist")
|
|
}
|
|
if restartPhase(main.BatchInfo) == consts.RestartPhaseWaitingResume {
|
|
claimed, claimErr := e.repository.ClaimRestartResume(ctx, batchNo, e.currentTime())
|
|
if claimErr != nil {
|
|
return gerror.Wrap(claimErr, "claim restart resume phase")
|
|
}
|
|
if !claimed {
|
|
g.Log().Infof(ctx, "restart resume message ignored after CAS lost, batchNo:%s", batchNo)
|
|
return nil
|
|
}
|
|
return e.executeResume(ctx, batchNo)
|
|
}
|
|
return e.executeStop(ctx, batchNo)
|
|
}
|
|
|
|
func (e *restartExecutor) executeStop(ctx context.Context, batchNo string) error {
|
|
task, err := e.repository.LoadRestartStop(ctx, batchNo)
|
|
if err != nil {
|
|
return gerror.Wrap(err, "load restart stop phase")
|
|
}
|
|
if !validateTask(task) {
|
|
if err = e.repository.SaveRestartValidationFailure(ctx, task, restartStageStop); err != nil {
|
|
return gerror.Wrap(err, "save restart stop validation failure")
|
|
}
|
|
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart stop validation failure")
|
|
}
|
|
|
|
successCount, err := e.changeRestartGroups(ctx, task, restartStageStop)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if successCount == 0 {
|
|
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart with no stop success")
|
|
}
|
|
|
|
delay := restartDelay(task.Batch.BatchInfo)
|
|
resumeAt := e.currentTime().Add(delay)
|
|
if err = e.repository.WaitRestartResume(ctx, task, resumeAt); err != nil {
|
|
return gerror.Wrap(err, "mark restart waiting resume")
|
|
}
|
|
message := batch.Message{BatchNo: task.Batch.BatchNo, BatchType: consts.BatchTypeSimCardRestart}
|
|
if err = e.publishDelayed(ctx, consts.QueueBatchRestartResumeDelay, consts.QueueBatchExecute, delay, message); err != nil {
|
|
return gerror.Wrap(err, "publish restart resume delayed message")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (e *restartExecutor) executeResume(ctx context.Context, batchNo string) error {
|
|
task, err := e.repository.LoadRestartResume(ctx, batchNo)
|
|
if err != nil {
|
|
return gerror.Wrap(err, "load restart resume phase")
|
|
}
|
|
if !validateTask(task) {
|
|
if err = e.repository.SaveRestartValidationFailure(ctx, task, restartStageResume); err != nil {
|
|
return gerror.Wrap(err, "save restart resume validation failure")
|
|
}
|
|
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart resume validation failure")
|
|
}
|
|
if _, err = e.changeRestartGroups(ctx, task, restartStageResume); err != nil {
|
|
return err
|
|
}
|
|
return gerror.Wrap(e.repository.FinalizeRestart(ctx, batchNo), "finalize restart batch")
|
|
}
|
|
|
|
func (e *restartExecutor) changeRestartGroups(ctx context.Context, task *Task, stage string) (int, error) {
|
|
groups := groupByChanKey(task.Details)
|
|
keys := make([]string, 0, len(groups))
|
|
for chanKey := range groups {
|
|
keys = append(keys, chanKey)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
successCount := 0
|
|
for _, chanKey := range keys {
|
|
outcomes := (&executor{upstream: e.upstream}).changeGroup(ctx, task, groups[chanKey])
|
|
for index, outcome := range outcomes {
|
|
detail := groups[chanKey][index]
|
|
resolved, saveErr := e.repository.SaveRestartResult(ctx, task, detail, outcome, stage)
|
|
if saveErr != nil {
|
|
return successCount, gerror.Wrapf(saveErr, "save restart %s detail %d result", stage, detail.ID)
|
|
}
|
|
detail.ApplyOutcome(resolved)
|
|
if resolved.Success {
|
|
successCount++
|
|
}
|
|
}
|
|
}
|
|
return successCount, nil
|
|
}
|
|
|
|
func (e *restartExecutor) currentTime() *gtime.Time {
|
|
if e.now != nil {
|
|
return e.now()
|
|
}
|
|
return gtime.Now()
|
|
}
|
|
|
|
func restartDelay(info *gjson.Json) time.Duration {
|
|
seconds := 300
|
|
if info != nil {
|
|
if configured := info.Get("resumeDelaySeconds").Int(); configured > 0 {
|
|
seconds = configured
|
|
}
|
|
}
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
|
|
func restartPhase(info *gjson.Json) string {
|
|
if info == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(info.Get("restart.phase").String())
|
|
}
|
|
|
|
func restartResumeAt(info *gjson.Json) *gtime.Time {
|
|
if info == nil {
|
|
return nil
|
|
}
|
|
value := strings.TrimSpace(info.Get("restart.resumeAt").String())
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
parsed := gtime.NewFromStr(value)
|
|
if parsed == nil || parsed.IsZero() {
|
|
return nil
|
|
}
|
|
return parsed
|
|
}
|