chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,157 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
const (
|
||||
Type3 = 3
|
||||
Type4 = 4
|
||||
Type11 = 11
|
||||
Type12 = 12
|
||||
Type40 = 40
|
||||
Type41 = 41
|
||||
Type101 = 101
|
||||
Type102 = 102
|
||||
Type103 = 103
|
||||
Type104 = 104
|
||||
Type105 = 105
|
||||
Type106 = 106
|
||||
Type107 = 107
|
||||
Type108 = 108
|
||||
Type109 = 109
|
||||
Type110 = 110
|
||||
|
||||
AsyncStatusPending = 1
|
||||
AsyncStatusProcessing = 2
|
||||
AsyncStatusSuccess = 3
|
||||
AsyncStatusFailed = 4
|
||||
RunResultRunning = 1
|
||||
RunResultSuccess = 2
|
||||
RunResultFailed = 3
|
||||
|
||||
DetailStatusPending = 0
|
||||
DetailStatusRunning = 1
|
||||
DetailStatusSuccess = 2
|
||||
DetailStatusFailed = 3
|
||||
)
|
||||
|
||||
type Message struct {
|
||||
BatchNo string `json:"batchNo"`
|
||||
BatchType int `json:"batchType"`
|
||||
FailureReason string `json:"failureReason,omitempty"`
|
||||
Retryable bool `json:"retryable,omitempty"`
|
||||
}
|
||||
|
||||
// RetryableError marks an executor failure that may succeed after delayed redelivery.
|
||||
// The batch remains in processing until retries are exhausted and finalized.
|
||||
type RetryableError struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *RetryableError) Error() string {
|
||||
if e == nil || e.Err == nil {
|
||||
return "retryable batch execution error"
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *RetryableError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Err
|
||||
}
|
||||
|
||||
func Retryable(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if IsRetryable(err) {
|
||||
return err
|
||||
}
|
||||
return &RetryableError{Err: err}
|
||||
}
|
||||
|
||||
func IsRetryable(err error) bool {
|
||||
var retryableErr *RetryableError
|
||||
return errors.As(err, &retryableErr)
|
||||
}
|
||||
|
||||
// FailureMarkError 表示执行已结束，但失败终态尚未成功写入数据库。
|
||||
// 消费者重投这类消息时只能补写失败终态，不能再次执行具体业务。
|
||||
type FailureMarkError struct {
|
||||
Reason string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *FailureMarkError) Error() string {
|
||||
return "mark batch failed after executor error: " + e.Reason + ": " + e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *FailureMarkError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
type Executor interface {
|
||||
Execute(ctx context.Context, batchNo string) error
|
||||
}
|
||||
|
||||
type ProcessingExecutor interface {
|
||||
CanExecuteProcessing(ctx context.Context, main *entity.BatchMain) (bool, error)
|
||||
}
|
||||
|
||||
// SelfClaimingProcessingExecutor marks executors that perform their own
|
||||
// processing-state CAS before executing a re-entry phase.
|
||||
type SelfClaimingProcessingExecutor interface {
|
||||
ClaimsProcessingReentry() bool
|
||||
}
|
||||
|
||||
// DurableFailureHandler takes ownership before any generic MarkFailed fallback.
|
||||
type DurableFailureHandler interface {
|
||||
ProtectExecutionFailure(ctx context.Context, batchNo string, batchType int, reason string) (bool, error)
|
||||
}
|
||||
|
||||
type ExecutionFailureHandler interface {
|
||||
HandleExecutionFailure(ctx context.Context, batchNo string, batchType int, reason string) error
|
||||
}
|
||||
|
||||
type ProcessingClaimer interface {
|
||||
ReclaimProcessing(ctx context.Context, batchNo string, batchType int, startedAt *gtime.Time) (bool, error)
|
||||
}
|
||||
|
||||
type Dispatcher struct {
|
||||
executors map[int]Executor
|
||||
}
|
||||
|
||||
func NewDispatcher(executor Executor) *Dispatcher {
|
||||
return &Dispatcher{executors: map[int]Executor{
|
||||
Type103: executor,
|
||||
Type104: executor,
|
||||
}}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) Register(batchType int, executor Executor) {
|
||||
if d == nil || executor == nil {
|
||||
return
|
||||
}
|
||||
d.executors[batchType] = executor
|
||||
}
|
||||
|
||||
func (d *Dispatcher) Executor(batchType int) (Executor, bool) {
|
||||
if d == nil {
|
||||
return nil, false
|
||||
}
|
||||
executor, ok := d.executors[batchType]
|
||||
return executor, ok && executor != nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) Has(batchType int) bool {
|
||||
_, ok := d.Executor(batchType)
|
||||
return ok
|
||||
}
|
||||
@@ -0,0 +1,574 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
type executorStub struct {
|
||||
calls int
|
||||
batchNo string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *executorStub) Execute(_ context.Context, batchNo string) error {
|
||||
s.calls++
|
||||
s.batchNo = batchNo
|
||||
return s.err
|
||||
}
|
||||
|
||||
type processingExecutorStub struct {
|
||||
executorStub
|
||||
canCalls int
|
||||
main *entity.BatchMain
|
||||
can bool
|
||||
canErr error
|
||||
selfClaim bool
|
||||
}
|
||||
|
||||
type blockingExecutorStub struct {
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *blockingExecutorStub) Execute(ctx context.Context, _ string) error {
|
||||
s.calls++
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (s *processingExecutorStub) CanExecuteProcessing(_ context.Context, main *entity.BatchMain) (bool, error) {
|
||||
s.canCalls++
|
||||
s.main = main
|
||||
return s.can, s.canErr
|
||||
}
|
||||
|
||||
func (s *processingExecutorStub) ClaimsProcessingReentry() bool {
|
||||
return s.selfClaim
|
||||
}
|
||||
|
||||
type repositoryStub struct {
|
||||
mains []*entity.BatchMain
|
||||
findErr error
|
||||
markErr error
|
||||
claimed bool
|
||||
findCalls int
|
||||
markCalls int
|
||||
markBatchNo string
|
||||
markType int
|
||||
failErr error
|
||||
failCalls int
|
||||
failBatchNo string
|
||||
failType int
|
||||
failReason string
|
||||
successErr error
|
||||
successCalls int
|
||||
successBatchNo string
|
||||
successType int
|
||||
successCount int
|
||||
successFails int
|
||||
successReason string
|
||||
reclaimErr error
|
||||
reclaimSet bool
|
||||
reclaimed bool
|
||||
reclaimCalls int
|
||||
reclaimBatchNo string
|
||||
reclaimType int
|
||||
}
|
||||
|
||||
func (s *repositoryStub) FindMain(context.Context, string) (*entity.BatchMain, error) {
|
||||
s.findCalls++
|
||||
if s.findErr != nil {
|
||||
return nil, s.findErr
|
||||
}
|
||||
if len(s.mains) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
index := s.findCalls - 1
|
||||
if index >= len(s.mains) {
|
||||
index = len(s.mains) - 1
|
||||
}
|
||||
return s.mains[index], nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) MarkProcessing(_ context.Context, batchNo string, batchType int) (bool, error) {
|
||||
s.markCalls++
|
||||
s.markBatchNo = batchNo
|
||||
s.markType = batchType
|
||||
return s.claimed, s.markErr
|
||||
}
|
||||
|
||||
func (s *repositoryStub) MarkFailed(_ context.Context, batchNo string, batchType int, reason string) (bool, error) {
|
||||
s.failCalls++
|
||||
s.failBatchNo = batchNo
|
||||
s.failType = batchType
|
||||
s.failReason = reason
|
||||
return s.failErr == nil, s.failErr
|
||||
}
|
||||
|
||||
func (s *repositoryStub) MarkSuccess(_ context.Context, batchNo string, batchType, success, fail int, reason string) (bool, error) {
|
||||
s.successCalls++
|
||||
s.successBatchNo = batchNo
|
||||
s.successType = batchType
|
||||
s.successCount = success
|
||||
s.successFails = fail
|
||||
s.successReason = reason
|
||||
return s.successErr == nil, s.successErr
|
||||
}
|
||||
|
||||
func (s *repositoryStub) ReclaimProcessing(_ context.Context, batchNo string, batchType int, _ *gtime.Time) (bool, error) {
|
||||
s.reclaimCalls++
|
||||
s.reclaimBatchNo = batchNo
|
||||
s.reclaimType = batchType
|
||||
if !s.reclaimSet {
|
||||
return s.reclaimErr == nil, s.reclaimErr
|
||||
}
|
||||
return s.reclaimed, s.reclaimErr
|
||||
}
|
||||
|
||||
type durableExecutorStub struct {
|
||||
execute func(context.Context) error
|
||||
protectErr error
|
||||
protectCalls int
|
||||
}
|
||||
|
||||
func (s *durableExecutorStub) Execute(ctx context.Context, _ string) error {
|
||||
if s.execute != nil {
|
||||
return s.execute(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *durableExecutorStub) ProtectExecutionFailure(ctx context.Context, _ string, _ int, _ string) (bool, error) {
|
||||
s.protectCalls++
|
||||
if ctx.Err() != nil {
|
||||
return true, ctx.Err()
|
||||
}
|
||||
return true, s.protectErr
|
||||
}
|
||||
|
||||
func TestRuntimeDurableOwnsEveryFailurePath(t *testing.T) {
|
||||
for _, kind := range []string{"error", "deadline-error", "deadline-nil", "failure-reason", "retry-exhausted", "protect-db-down"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
repo := &repositoryStub{mains: []*entity.BatchMain{{BatchNo: "DURABLE", BatchType: Type109, BatchAsyncStatus: AsyncStatusPending}}, claimed: true}
|
||||
exe := &durableExecutorStub{}
|
||||
msg := Message{BatchNo: "DURABLE", BatchType: Type109}
|
||||
switch kind {
|
||||
case "error":
|
||||
exe.execute = func(context.Context) error { return errors.New("执行中断") }
|
||||
case "deadline-error":
|
||||
exe.execute = func(ctx context.Context) error { <-ctx.Done(); return ctx.Err() }
|
||||
case "deadline-nil":
|
||||
exe.execute = func(ctx context.Context) error { <-ctx.Done(); return nil }
|
||||
case "failure-reason", "retry-exhausted":
|
||||
msg.FailureReason = "重试耗尽或失败重投"
|
||||
repo.mains[0].BatchAsyncStatus = AsyncStatusProcessing
|
||||
case "protect-db-down":
|
||||
exe.execute = func(context.Context) error { return errors.New("执行中断") }
|
||||
exe.protectErr = errors.New("数据库不可用")
|
||||
}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type109, exe)
|
||||
err := NewRuntime(repo, dispatcher, WithExecutionTimeout(time.Millisecond)).Handle(context.Background(), msg)
|
||||
if kind == "protect-db-down" {
|
||||
var failure *FailureMarkError
|
||||
if !errors.As(err, &failure) {
|
||||
t.Fatalf("保护失败应可重投: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exe.protectCalls != 1 || repo.failCalls != 0 {
|
||||
t.Fatalf("通用MarkFailed越过durable保护: protect=%d failed=%d", exe.protectCalls, repo.failCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageJSONContract(t *testing.T) {
|
||||
body, err := json.Marshal(Message{BatchNo: "BATCH-1", BatchType: Type103})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal message: %v", err)
|
||||
}
|
||||
const expected = `{"batchNo":"BATCH-1","batchType":103}`
|
||||
if string(body) != expected {
|
||||
t.Fatalf("message JSON = %s, want %s", body, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatcherSupportsLifecycleTypesAndExplicitRegistration(t *testing.T) {
|
||||
executor := &executorStub{}
|
||||
dispatcher := NewDispatcher(executor)
|
||||
for _, batchType := range []int{Type103, Type104} {
|
||||
actual, ok := dispatcher.Executor(batchType)
|
||||
if !ok || actual != executor {
|
||||
t.Fatalf("batch type %d was not registered with shared executor", batchType)
|
||||
}
|
||||
}
|
||||
if _, ok := dispatcher.Executor(102); ok {
|
||||
t.Fatal("unexpected executor for unsupported batch type")
|
||||
}
|
||||
|
||||
inbound := &executorStub{}
|
||||
dispatcher.Register(1, inbound)
|
||||
actual, ok := dispatcher.Executor(1)
|
||||
if !ok || actual != inbound {
|
||||
t.Fatal("registered inbound executor was not returned")
|
||||
}
|
||||
|
||||
dataExecutor := &executorStub{}
|
||||
dispatcher.Register(Type107, dataExecutor)
|
||||
dispatcher.Register(Type109, dataExecutor)
|
||||
dispatcher.Register(Type110, dataExecutor)
|
||||
for _, batchType := range []int{Type107, Type109, Type110} {
|
||||
actual, ok = dispatcher.Executor(batchType)
|
||||
if !ok || actual != dataExecutor {
|
||||
t.Fatalf("batch type %d was not registered with the data service executor", batchType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeAcknowledgesInvalidAndUnclaimableMessages(t *testing.T) {
|
||||
g.Log().SetWriter(io.Discard)
|
||||
tests := []struct {
|
||||
name string
|
||||
message Message
|
||||
mains []*entity.BatchMain
|
||||
claimed bool
|
||||
wantFind int
|
||||
wantMarked int
|
||||
}{
|
||||
{name: "empty batch number", message: Message{BatchType: Type103}},
|
||||
{name: "blank batch number", message: Message{BatchNo: " ", BatchType: Type103}},
|
||||
{name: "unsupported type", message: Message{BatchNo: "B-1", BatchType: 102}},
|
||||
{name: "not found", message: Message{BatchNo: "B-1", BatchType: Type103}, wantFind: 1},
|
||||
{name: "type mismatch", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}}, wantFind: 1},
|
||||
{name: "processing", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusProcessing}}, wantFind: 1},
|
||||
{name: "terminal", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: 3}}, wantFind: 1},
|
||||
{name: "CAS lost", message: Message{BatchNo: "B-1", BatchType: Type103}, mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusPending}, {BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusProcessing}}, wantFind: 2, wantMarked: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
repository := &repositoryStub{mains: tt.mains, claimed: tt.claimed}
|
||||
executor := &executorStub{}
|
||||
runtime := NewRuntime(repository, NewDispatcher(executor))
|
||||
if err := runtime.Handle(context.Background(), tt.message); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if repository.findCalls != tt.wantFind || repository.markCalls != tt.wantMarked {
|
||||
t.Fatalf("repository calls: find=%d mark=%d, want find=%d mark=%d", repository.findCalls, repository.markCalls, tt.wantFind, tt.wantMarked)
|
||||
}
|
||||
if executor.calls != 0 {
|
||||
t.Fatalf("executor calls = %d, want 0", executor.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeExecutesRegisteredDataServiceBatch(t *testing.T) {
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-109", BatchType: Type109, BatchAsyncStatus: AsyncStatusPending}},
|
||||
claimed: true,
|
||||
}
|
||||
dataExecutor := &executorStub{}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type109, dataExecutor)
|
||||
dispatcher.Register(Type110, dataExecutor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-109", BatchType: Type109}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if dataExecutor.calls != 1 || dataExecutor.batchNo != "B-109" {
|
||||
t.Fatalf("data executor was not invoked: %+v", dataExecutor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeExecutesTrafficCutoffConfigWithDedicatedExecutor(t *testing.T) {
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-41", BatchType: Type41, BatchAsyncStatus: AsyncStatusPending}},
|
||||
claimed: true,
|
||||
}
|
||||
dataExecutor := &executorStub{err: errors.New("data executor must not be called")}
|
||||
trafficCutoffConfigExecutor := &executorStub{}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type107, dataExecutor)
|
||||
dispatcher.Register(Type41, trafficCutoffConfigExecutor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-41", BatchType: Type41}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if trafficCutoffConfigExecutor.calls != 1 || trafficCutoffConfigExecutor.batchNo != "B-41" {
|
||||
t.Fatalf("traffic cutoff config executor was not invoked: %+v", trafficCutoffConfigExecutor)
|
||||
}
|
||||
if dataExecutor.calls != 0 {
|
||||
t.Fatalf("data executor calls = %d, want 0", dataExecutor.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeIgnoresFinalizedTrafficCutoffConfigDuplicateMessage(t *testing.T) {
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-41-DONE", BatchType: Type41, BatchAsyncStatus: AsyncStatusSuccess}},
|
||||
}
|
||||
trafficCutoffConfigExecutor := &executorStub{err: errors.New("traffic cutoff config executor must not be called")}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type41, trafficCutoffConfigExecutor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-41-DONE", BatchType: Type41}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if trafficCutoffConfigExecutor.calls != 0 {
|
||||
t.Fatalf("traffic cutoff config executor calls = %d, want 0", trafficCutoffConfigExecutor.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeMarksExecutorFailureAsFailed(t *testing.T) {
|
||||
expectedErr := errors.New("execute failed")
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-104", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}},
|
||||
claimed: true,
|
||||
}
|
||||
executor := &executorStub{err: expectedErr}
|
||||
runtime := NewRuntime(repository, NewDispatcher(executor))
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-104", BatchType: Type104}); err != nil {
|
||||
t.Fatalf("Handle() error = %v, want nil", err)
|
||||
}
|
||||
if repository.markCalls != 1 || repository.markBatchNo != "B-104" || repository.markType != Type104 {
|
||||
t.Fatalf("unexpected claim call: %+v", repository)
|
||||
}
|
||||
if executor.calls != 1 || executor.batchNo != "B-104" {
|
||||
t.Fatalf("unexpected executor call: %+v", executor)
|
||||
}
|
||||
if repository.failCalls != 1 || repository.failBatchNo != "B-104" || repository.failType != Type104 || repository.failReason != expectedErr.Error() {
|
||||
t.Fatalf("unexpected failure mark call: %+v", repository)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeReturnsRetryableExecutorFailureWithoutFinalizing(t *testing.T) {
|
||||
sourceErr := errors.New("source file is temporarily unavailable")
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "INB-RETRY", BatchType: 1, BatchAsyncStatus: AsyncStatusPending}},
|
||||
claimed: true,
|
||||
}
|
||||
executor := &executorStub{err: Retryable(sourceErr)}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(1, executor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
err := runtime.Handle(context.Background(), Message{BatchNo: "INB-RETRY", BatchType: 1})
|
||||
if !errors.Is(err, sourceErr) || !IsRetryable(err) {
|
||||
t.Fatalf("Handle() error = %v, want retryable wrapper of %v", err, sourceErr)
|
||||
}
|
||||
if repository.failCalls != 0 {
|
||||
t.Fatalf("MarkFailed calls = %d, want 0", repository.failCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeReclaimsRetryableProcessingBatch(t *testing.T) {
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "INB-RETRY", BatchType: 1, BatchAsyncStatus: AsyncStatusProcessing}},
|
||||
reclaimSet: true,
|
||||
reclaimed: true,
|
||||
}
|
||||
executor := &executorStub{}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(1, executor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "INB-RETRY", BatchType: 1, Retryable: true}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if repository.reclaimCalls != 1 || repository.reclaimBatchNo != "INB-RETRY" || repository.reclaimType != 1 {
|
||||
t.Fatalf("unexpected reclaim call: %+v", repository)
|
||||
}
|
||||
if executor.calls != 1 {
|
||||
t.Fatalf("executor calls = %d, want 1", executor.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeMarksTimeoutAsFailed(t *testing.T) {
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-TIMEOUT", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}},
|
||||
claimed: true,
|
||||
}
|
||||
executor := &blockingExecutorStub{}
|
||||
runtime := NewRuntime(
|
||||
repository,
|
||||
NewDispatcher(executor),
|
||||
WithExecutionTimeout(time.Millisecond),
|
||||
WithFinalizationTimeout(time.Second),
|
||||
)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-TIMEOUT", BatchType: Type104}); err != nil {
|
||||
t.Fatalf("Handle() error = %v, want nil", err)
|
||||
}
|
||||
if executor.calls != 1 {
|
||||
t.Fatalf("executor calls = %d, want 1", executor.calls)
|
||||
}
|
||||
if repository.failCalls != 1 || !strings.Contains(repository.failReason, "timed out after 1ms") {
|
||||
t.Fatalf("unexpected timeout failure mark: %+v", repository)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeReturnsFailureMarkErrors(t *testing.T) {
|
||||
executeErr := errors.New("execute failed")
|
||||
markErr := errors.New("mark failed")
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-104", BatchType: Type104, BatchAsyncStatus: AsyncStatusPending}},
|
||||
claimed: true,
|
||||
failErr: markErr,
|
||||
}
|
||||
runtime := NewRuntime(repository, NewDispatcher(&executorStub{err: executeErr}))
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-104", BatchType: Type104}); !errors.Is(err, markErr) {
|
||||
t.Fatalf("Handle() error = %v, want %v", err, markErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeCompensatesFailureMarkWithoutReexecutingBatch(t *testing.T) {
|
||||
markErr := errors.New("mark failed")
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-RETRY", BatchType: Type104, BatchAsyncStatus: AsyncStatusProcessing}},
|
||||
failErr: markErr,
|
||||
}
|
||||
executor := &executorStub{err: errors.New("must not execute again")}
|
||||
runtime := NewRuntime(repository, NewDispatcher(executor))
|
||||
|
||||
err := runtime.Handle(context.Background(), Message{
|
||||
BatchNo: "B-RETRY",
|
||||
BatchType: Type104,
|
||||
FailureReason: "original executor failure",
|
||||
})
|
||||
if !errors.Is(err, markErr) {
|
||||
t.Fatalf("Handle() error = %v, want %v", err, markErr)
|
||||
}
|
||||
if executor.calls != 0 {
|
||||
t.Fatalf("executor calls = %d, want 0 for failure finalization retry", executor.calls)
|
||||
}
|
||||
if repository.markCalls != 0 {
|
||||
t.Fatalf("MarkProcessing calls = %d, want 0 for failure finalization retry", repository.markCalls)
|
||||
}
|
||||
if repository.failCalls != 1 || repository.failReason != "original executor failure" {
|
||||
t.Fatalf("unexpected failure finalization call: %+v", repository)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeAllowsSelfClaimingProcessingReEntryWithoutRefreshingStartTime(t *testing.T) {
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "RST-1", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
|
||||
}
|
||||
executor := &processingExecutorStub{can: true, selfClaim: true}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type105, executor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-1", BatchType: Type105}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if executor.canCalls != 1 || executor.main == nil || executor.main.BatchNo != "RST-1" {
|
||||
t.Fatalf("processing check not called with main: %+v", executor)
|
||||
}
|
||||
if repository.markCalls != 0 {
|
||||
t.Fatalf("MarkProcessing calls = %d, want 0", repository.markCalls)
|
||||
}
|
||||
if repository.reclaimCalls != 0 {
|
||||
t.Fatalf("ReclaimProcessing calls = %d, want 0 for self-claiming executor", repository.reclaimCalls)
|
||||
}
|
||||
if executor.calls != 1 || executor.batchNo != "RST-1" {
|
||||
t.Fatalf("executor calls = %+v", executor.executorStub)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeMarksProcessingReEntryFailureAsFailed(t *testing.T) {
|
||||
expectedErr := errors.New("resume failed")
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "RST-1", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
|
||||
}
|
||||
executor := &processingExecutorStub{can: true}
|
||||
executor.err = expectedErr
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type105, executor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-1", BatchType: Type105}); err != nil {
|
||||
t.Fatalf("Handle() error = %v, want nil", err)
|
||||
}
|
||||
if repository.markCalls != 0 {
|
||||
t.Fatalf("MarkProcessing calls = %d, want 0", repository.markCalls)
|
||||
}
|
||||
if repository.failCalls != 1 || repository.failBatchNo != "RST-1" || repository.failType != Type105 || repository.failReason != expectedErr.Error() {
|
||||
t.Fatalf("unexpected failure mark call: %+v", repository)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeIgnoresProcessingWhenOptInExecutorDeclines(t *testing.T) {
|
||||
repository := &repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "RST-2", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
|
||||
}
|
||||
executor := &processingExecutorStub{can: false}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type105, executor)
|
||||
runtime := NewRuntime(repository, dispatcher)
|
||||
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-2", BatchType: Type105}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if executor.canCalls != 1 {
|
||||
t.Fatalf("processing check calls = %d, want 1", executor.canCalls)
|
||||
}
|
||||
if executor.calls != 0 {
|
||||
t.Fatalf("executor calls = %d, want 0", executor.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeReturnsDatabaseErrors(t *testing.T) {
|
||||
findErr := errors.New("find failed")
|
||||
runtime := NewRuntime(&repositoryStub{findErr: findErr}, NewDispatcher(&executorStub{}))
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-1", BatchType: Type103}); !errors.Is(err, findErr) {
|
||||
t.Fatalf("find error = %v, want %v", err, findErr)
|
||||
}
|
||||
|
||||
markErr := errors.New("mark failed")
|
||||
runtime = NewRuntime(&repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "B-1", BatchType: Type103, BatchAsyncStatus: AsyncStatusPending}},
|
||||
markErr: markErr,
|
||||
}, NewDispatcher(&executorStub{}))
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "B-1", BatchType: Type103}); !errors.Is(err, markErr) {
|
||||
t.Fatalf("mark error = %v, want %v", err, markErr)
|
||||
}
|
||||
|
||||
checkErr := errors.New("check failed")
|
||||
processing := &processingExecutorStub{canErr: checkErr}
|
||||
dispatcher := NewDispatcher(&executorStub{})
|
||||
dispatcher.Register(Type105, processing)
|
||||
runtime = NewRuntime(&repositoryStub{
|
||||
mains: []*entity.BatchMain{{BatchNo: "RST-3", BatchType: Type105, BatchAsyncStatus: AsyncStatusProcessing}},
|
||||
}, dispatcher)
|
||||
if err := runtime.Handle(context.Background(), Message{BatchNo: "RST-3", BatchType: Type105}); !errors.Is(err, checkErr) {
|
||||
t.Fatalf("processing check error = %v, want %v", err, checkErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageHasOnlyContractFields(t *testing.T) {
|
||||
typeOfMessage := reflect.TypeOf(Message{})
|
||||
if typeOfMessage.NumField() != 4 {
|
||||
t.Fatalf("Message has %d fields, want 4", typeOfMessage.NumField())
|
||||
}
|
||||
if field, ok := typeOfMessage.FieldByName("FailureReason"); !ok || field.Tag.Get("json") != "failureReason,omitempty" {
|
||||
t.Fatalf("FailureReason JSON contract is missing or incorrect")
|
||||
}
|
||||
if field, ok := typeOfMessage.FieldByName("Retryable"); !ok || field.Tag.Get("json") != "retryable,omitempty" {
|
||||
t.Fatalf("Retryable JSON contract is missing or incorrect")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package batch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/batchfields"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
FindMain(ctx context.Context, batchNo string) (*entity.BatchMain, error)
|
||||
MarkProcessing(ctx context.Context, batchNo string, batchType int) (bool, error)
|
||||
MarkFailed(ctx context.Context, batchNo string, batchType int, reason string) (bool, error)
|
||||
MarkSuccess(ctx context.Context, batchNo string, batchType int, success, fail int, reason string) (bool, error)
|
||||
}
|
||||
|
||||
type Runtime struct {
|
||||
repository Repository
|
||||
dispatcher *Dispatcher
|
||||
executionTimeout time.Duration
|
||||
finalizationTimeout time.Duration
|
||||
}
|
||||
|
||||
func NewRuntime(repository Repository, dispatcher *Dispatcher, options ...RuntimeOption) *Runtime {
|
||||
runtime := &Runtime{
|
||||
repository: repository,
|
||||
dispatcher: dispatcher,
|
||||
executionTimeout: 0,
|
||||
finalizationTimeout: 10 * time.Second,
|
||||
}
|
||||
for _, option := range options {
|
||||
option(runtime)
|
||||
}
|
||||
return runtime
|
||||
}
|
||||
|
||||
type RuntimeOption func(*Runtime)
|
||||
|
||||
func WithExecutionTimeout(timeout time.Duration) RuntimeOption {
|
||||
return func(runtime *Runtime) {
|
||||
if timeout >= 0 {
|
||||
runtime.executionTimeout = timeout
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithFinalizationTimeout(timeout time.Duration) RuntimeOption {
|
||||
return func(runtime *Runtime) {
|
||||
if timeout > 0 {
|
||||
runtime.finalizationTimeout = timeout
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runtime) Handle(ctx context.Context, message Message) error {
|
||||
if strings.TrimSpace(message.BatchNo) == "" {
|
||||
g.Log().Warning(ctx, "batch message dropped, empty batchNo")
|
||||
return nil
|
||||
}
|
||||
executor, ok := r.dispatcher.Executor(message.BatchType)
|
||||
if !ok {
|
||||
g.Log().Warningf(ctx, "batch message dropped, unsupported batchType:%d, batchNo:%s", message.BatchType, message.BatchNo)
|
||||
return nil
|
||||
}
|
||||
|
||||
main, err := r.repository.FindMain(ctx, message.BatchNo)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "load batch main failed")
|
||||
}
|
||||
if main == nil {
|
||||
g.Log().Warningf(ctx, "batch message ignored, batch not found, batchNo:%s", message.BatchNo)
|
||||
return nil
|
||||
}
|
||||
if main.BatchType != message.BatchType {
|
||||
g.Log().Warningf(ctx, "batch message ignored, type mismatch, batchNo:%s, messageType:%d, storedType:%d", message.BatchNo, message.BatchType, main.BatchType)
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(message.FailureReason) != "" {
|
||||
return r.markExecutionFailed(ctx, message, executor, message.FailureReason)
|
||||
}
|
||||
if main.BatchAsyncStatus != AsyncStatusPending {
|
||||
if main.BatchAsyncStatus == AsyncStatusProcessing {
|
||||
if processingExecutor, ok := executor.(ProcessingExecutor); ok {
|
||||
canExecute, checkErr := processingExecutor.CanExecuteProcessing(ctx, main)
|
||||
if checkErr != nil {
|
||||
return gerror.Wrap(checkErr, "check processing batch re-entry failed")
|
||||
}
|
||||
if canExecute {
|
||||
if selfClaimer, selfClaims := executor.(SelfClaimingProcessingExecutor); selfClaims && selfClaimer.ClaimsProcessingReentry() {
|
||||
return r.execute(ctx, message, executor)
|
||||
}
|
||||
return r.reclaimAndExecute(ctx, message, executor, main)
|
||||
}
|
||||
} else if message.Retryable {
|
||||
return r.reclaimAndExecute(ctx, message, executor, main)
|
||||
}
|
||||
}
|
||||
logUnclaimable(ctx, main)
|
||||
return nil
|
||||
}
|
||||
|
||||
claimed, err := r.repository.MarkProcessing(ctx, message.BatchNo, message.BatchType)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "mark batch processing failed")
|
||||
}
|
||||
if !claimed {
|
||||
current, findErr := r.repository.FindMain(ctx, message.BatchNo)
|
||||
if findErr != nil {
|
||||
return gerror.Wrap(findErr, "reload batch main after CAS failed")
|
||||
}
|
||||
if current == nil {
|
||||
g.Log().Warningf(ctx, "batch CAS lost, batch not found, batchNo:%s", message.BatchNo)
|
||||
return nil
|
||||
}
|
||||
if current.BatchType != message.BatchType {
|
||||
g.Log().Warningf(ctx, "batch CAS lost, type mismatch, batchNo:%s, messageType:%d, storedType:%d", message.BatchNo, message.BatchType, current.BatchType)
|
||||
return nil
|
||||
}
|
||||
logUnclaimable(ctx, current)
|
||||
return nil
|
||||
}
|
||||
|
||||
g.Log().Infof(ctx, "batch claimed, batchNo:%s, batchType:%d", message.BatchNo, message.BatchType)
|
||||
return r.execute(ctx, message, executor)
|
||||
}
|
||||
|
||||
func (r *Runtime) reclaimAndExecute(ctx context.Context, message Message, executor Executor, main *entity.BatchMain) error {
|
||||
if claimer, claimSupported := r.repository.(ProcessingClaimer); claimSupported {
|
||||
reclaimed, claimErr := claimer.ReclaimProcessing(ctx, message.BatchNo, message.BatchType, main.StartTime)
|
||||
if claimErr != nil {
|
||||
return gerror.Wrap(claimErr, "claim stale processing batch failed")
|
||||
}
|
||||
if !reclaimed {
|
||||
g.Log().Infof(ctx, "batch re-entry claim lost, batchNo:%s", message.BatchNo)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return r.execute(ctx, message, executor)
|
||||
}
|
||||
|
||||
func (r *Runtime) execute(ctx context.Context, message Message, executor Executor) error {
|
||||
executionCtx := ctx
|
||||
var cancel context.CancelFunc = func() {}
|
||||
if r.executionTimeout > 0 {
|
||||
executionCtx, cancel = context.WithTimeout(ctx, r.executionTimeout)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
if err := executor.Execute(executionCtx, message.BatchNo); err != nil {
|
||||
if IsRetryable(err) {
|
||||
return err
|
||||
}
|
||||
reason := err.Error()
|
||||
if r.executionTimeout > 0 && executionCtx.Err() == context.DeadlineExceeded {
|
||||
reason = gerror.Newf("batch execution timed out after %s", r.executionTimeout).Error()
|
||||
}
|
||||
return r.markExecutionFailed(ctx, message, executor, reason)
|
||||
}
|
||||
if r.executionTimeout > 0 && executionCtx.Err() == context.DeadlineExceeded {
|
||||
return r.markExecutionFailed(ctx, message, executor, gerror.Newf("batch execution timed out after %s", r.executionTimeout).Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runtime) markExecutionFailed(ctx context.Context, message Message, executor Executor, reason string) error {
|
||||
reason = batchfields.FailureReason(reason)
|
||||
finalizationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), r.finalizationTimeout)
|
||||
defer cancel()
|
||||
|
||||
if handler, ok := executor.(DurableFailureHandler); ok {
|
||||
handled, err := handler.ProtectExecutionFailure(finalizationCtx, message.BatchNo, message.BatchType, reason)
|
||||
if err != nil {
|
||||
return &FailureMarkError{Reason: reason, Err: err}
|
||||
}
|
||||
if handled {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
failed, markErr := r.repository.MarkFailed(finalizationCtx, message.BatchNo, message.BatchType, reason)
|
||||
if markErr != nil {
|
||||
return &FailureMarkError{Reason: reason, Err: markErr}
|
||||
}
|
||||
if handler, ok := executor.(ExecutionFailureHandler); ok {
|
||||
if err := handler.HandleExecutionFailure(finalizationCtx, message.BatchNo, message.BatchType, reason); err != nil {
|
||||
return &FailureMarkError{Reason: reason, Err: err}
|
||||
}
|
||||
}
|
||||
if !failed {
|
||||
g.Log().Warningf(ctx, "batch execution failed but status was already finalized, batchNo:%s, reason:%s", message.BatchNo, reason)
|
||||
return nil
|
||||
}
|
||||
g.Log().Errorf(ctx, "batch execution failed and was marked failed, batchNo:%s, reason:%s", message.BatchNo, reason)
|
||||
return nil
|
||||
}
|
||||
|
||||
func logUnclaimable(ctx context.Context, main *entity.BatchMain) {
|
||||
if main.BatchAsyncStatus == AsyncStatusProcessing {
|
||||
g.Log().Infof(ctx, "batch message ignored, batch already processing, batchNo:%s", main.BatchNo)
|
||||
return
|
||||
}
|
||||
g.Log().Infof(ctx, "batch message ignored, batch is terminal or not pending, batchNo:%s, status:%d", main.BatchNo, main.BatchAsyncStatus)
|
||||
}
|
||||
Reference in New Issue
Block a user