chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,179 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
zsetconsumer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consumer/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
monthlyusagerepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/monthlyusage"
|
||||
)
|
||||
|
||||
// errCodeTelecomChannelLimited is the carrier error code that means the whole
|
||||
// channel is throttled or disabled and must stop for this run.
|
||||
const errCodeTelecomChannelLimited = "S00004"
|
||||
|
||||
// The gateway uses code 9999 without an errCode for transport, HTTP, and
|
||||
// response-decoding failures. These failures are safe to retry from the task
|
||||
// lease; a populated errCode is treated as a business result.
|
||||
const chanGatewayInternalErrorCode = 9999
|
||||
|
||||
// tokenExpiredCode is the carrier code for an expired/invalid upstream token.
|
||||
// The gateway already invalidates and refreshes the token once on this code;
|
||||
// when it still reaches the worker, retrying in a later round lets a refreshed
|
||||
// token succeed instead of parking the card as a terminal business failure.
|
||||
const tokenExpiredCode = 12021
|
||||
|
||||
const (
|
||||
mobileBusyFirstCode = 13000
|
||||
mobileBusyLastCode = 13007
|
||||
mobileBusyClass = "mobile_busy"
|
||||
)
|
||||
|
||||
// Mobile usage templates are identified by the template returned with the
|
||||
// channel metadata. The channel name and error message are not stable
|
||||
// carrier identifiers.
|
||||
//
|
||||
// The set is derived from the OneLink registration table in the upstream
|
||||
// proxy (iotlcm-upstream-proxy/internal/logic/upstreamimpl/sim_yd.go
|
||||
// init()): every key below is registered to the China Mobile OneLink
|
||||
// sSimYD implementation, so numeric codes 13000-13007 emitted by any of
|
||||
// them carry OneLink semantics and are safe to retry across rounds.
|
||||
// Keep this list in sync with that registration table; templates that are
|
||||
// not registered there (including unknown future keys) stay "generic" and
|
||||
// are never classified as mobile.
|
||||
var mobileUsageTemplates = map[string]struct{}{
|
||||
"sim_yd_v5": {},
|
||||
"sim_yd_mock_v5": {},
|
||||
"sim_sdtywl_v1": {},
|
||||
"sim_onelink_tengyu_v1": {},
|
||||
"sim_onelink_xuzhou_mobile_v1": {},
|
||||
"sim_onelink_lelian_v1": {},
|
||||
"sim_onelink_qianxun_v1": {},
|
||||
"sim_onelink_henan_xinjing_v1": {},
|
||||
"sim_henan_xinjing_direct_v1": {},
|
||||
"sim_onelink_zibo_v1": {},
|
||||
"sim_tibiot_v1": {},
|
||||
"sim_qianxun_v3": {},
|
||||
"sim_yunda_v1": {},
|
||||
"sim_mingqian_v3": {},
|
||||
"sim_daihe_v1": {},
|
||||
"sim_onelink_daihe_v1": {},
|
||||
"sim_onelink_daliuliang_v1": {},
|
||||
"sim__daliuliang_v1": {},
|
||||
"sim_onelink_mobile_public_v1": {},
|
||||
"sim_onelink_kedang_v1": {},
|
||||
"sim_onelink_beixin_v1": {},
|
||||
"lanshang_8000": {},
|
||||
"lanshang_8003": {},
|
||||
}
|
||||
|
||||
const telecomUsageTemplate = "sim_dx_v1"
|
||||
|
||||
func usageCarrierForTemplate(tempKey string) string {
|
||||
tempKey = strings.ToLower(strings.TrimSpace(tempKey))
|
||||
if model.IsTelecomTemplate(tempKey) {
|
||||
return "telecom"
|
||||
}
|
||||
if _, ok := mobileUsageTemplates[tempKey]; ok {
|
||||
return "mobile"
|
||||
}
|
||||
return "generic"
|
||||
}
|
||||
|
||||
// classifyUsageFailure maps a gateway item to the queue semantics owned by
|
||||
// this service. Carrier identity is resolved from the template first; error
|
||||
// text and channel names are deliberately excluded from the decision.
|
||||
func classifyUsageFailure(tempKey string, code int, errCode string) (errorClass string, retryable, blockChannel bool) {
|
||||
errorClass = "business"
|
||||
carrier := usageCarrierForTemplate(tempKey)
|
||||
switch {
|
||||
case carrier == "telecom" && strings.EqualFold(strings.TrimSpace(errCode), errCodeTelecomChannelLimited):
|
||||
return "channel_limited", false, true
|
||||
case carrier == "mobile" && code >= mobileBusyFirstCode && code <= mobileBusyLastCode:
|
||||
return mobileBusyClass, true, false
|
||||
case code == chanGatewayInternalErrorCode && strings.TrimSpace(errCode) == "":
|
||||
return "network", true, false
|
||||
case code == tokenExpiredCode:
|
||||
return "token_expired", true, false
|
||||
default:
|
||||
return errorClass, false, false
|
||||
}
|
||||
}
|
||||
|
||||
type UsageClient interface {
|
||||
QuerySimDataUsage(context.Context, *model.SimDataUsageInput) ([]model.SimBaseRes[model.SimDataUsageItem], error)
|
||||
}
|
||||
|
||||
type Handler struct {
|
||||
client UsageClient
|
||||
repo UsageRepository
|
||||
}
|
||||
|
||||
type UsageRepository interface {
|
||||
ApplyBatch(context.Context, []monthlyusagerepository.UsageResult) error
|
||||
}
|
||||
|
||||
func NewHandler(client UsageClient, repo UsageRepository) (*Handler, error) {
|
||||
if client == nil || repo == nil {
|
||||
return nil, fmt.Errorf("monthly usage handler dependencies are incomplete")
|
||||
}
|
||||
return &Handler{client: client, repo: repo}, nil
|
||||
}
|
||||
|
||||
func (h *Handler) Query(ctx context.Context, task queue.Task) zsetconsumer.Result {
|
||||
result := zsetconsumer.Result{Task: task}
|
||||
output, err := h.client.QuerySimDataUsage(ctx, &model.SimDataUsageInput{
|
||||
SimKeys: []*model.SimKey{{
|
||||
ChanKey: task.ChanKey,
|
||||
ICCID: task.ICCID,
|
||||
IMSI: task.IMSI,
|
||||
MSISDN: task.MSISDN,
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
result.ErrorClass = "network"
|
||||
result.Retryable = true
|
||||
result.Message = err.Error()
|
||||
return result
|
||||
}
|
||||
if len(output) == 0 || output[0].SimKey == nil {
|
||||
result.ErrorClass = "protocol"
|
||||
result.Retryable = true
|
||||
result.Message = "chan returned empty current usage result"
|
||||
return result
|
||||
}
|
||||
item := output[0]
|
||||
if task.ICCID != "" && item.SimKey.ICCID != task.ICCID {
|
||||
result.ErrorClass = "protocol"
|
||||
result.Retryable = true
|
||||
result.Message = "chan returned mismatched card identifier"
|
||||
return result
|
||||
}
|
||||
result.Code = item.Code
|
||||
result.Message = item.Message
|
||||
result.ErrCode = item.ErrCode
|
||||
if item.Code != 0 {
|
||||
result.ErrorClass, result.Retryable, result.BlockChannel = classifyUsageFailure(task.TempKey, item.Code, item.ErrCode)
|
||||
return result
|
||||
}
|
||||
result.Success = true
|
||||
result.UsageMB = item.ResData.DataUsage
|
||||
return result
|
||||
}
|
||||
|
||||
func (h *Handler) Apply(ctx context.Context, results []zsetconsumer.Result) error {
|
||||
updates := make([]monthlyusagerepository.UsageResult, 0, len(results))
|
||||
for _, result := range results {
|
||||
if !result.Success {
|
||||
continue
|
||||
}
|
||||
updates = append(updates, monthlyusagerepository.UsageResult{
|
||||
CardID: result.Task.CardID,
|
||||
UsageMB: result.UsageMB,
|
||||
})
|
||||
}
|
||||
return h.repo.ApplyBatch(ctx, updates)
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
zsetconsumer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consumer/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
monthlyusagerepository "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/monthlyusage"
|
||||
)
|
||||
|
||||
type usageClientStub struct {
|
||||
output []model.SimBaseRes[model.SimDataUsageItem]
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *usageClientStub) QuerySimDataUsage(context.Context, *model.SimDataUsageInput) ([]model.SimBaseRes[model.SimDataUsageItem], error) {
|
||||
return s.output, s.err
|
||||
}
|
||||
|
||||
type usageRepositoryStub struct {
|
||||
calls int
|
||||
updates []monthlyusagerepository.UsageResult
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *usageRepositoryStub) ApplyBatch(_ context.Context, updates []monthlyusagerepository.UsageResult) error {
|
||||
s.calls++
|
||||
s.updates = append(s.updates, updates...)
|
||||
return s.err
|
||||
}
|
||||
|
||||
func TestQueryClassifiesCurrentUsageResults(t *testing.T) {
|
||||
task := queue.Task{TaskID: "1", ChanKey: "telecom-a", TempKey: telecomUsageTemplate, CardID: 1, ICCID: "89860001"}
|
||||
tests := []struct {
|
||||
name string
|
||||
client *usageClientStub
|
||||
success bool
|
||||
errorClass string
|
||||
retryable bool
|
||||
blockChannel bool
|
||||
}{
|
||||
{
|
||||
name: "success",
|
||||
client: &usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{
|
||||
SimKey: &model.SimKey{ICCID: task.ICCID},
|
||||
ResData: model.SimDataUsageItem{DataUsage: 12.5},
|
||||
}}},
|
||||
success: true,
|
||||
},
|
||||
{
|
||||
name: "mobile busy is terminal business failure",
|
||||
client: &usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{SimKey: &model.SimKey{ICCID: task.ICCID}, Code: 13000, Message: "busy"}}},
|
||||
errorClass: "business",
|
||||
},
|
||||
{
|
||||
name: "gateway internal failure is retryable",
|
||||
client: &usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{SimKey: &model.SimKey{ICCID: task.ICCID}, Code: 9999, Message: "upstream timeout"}}},
|
||||
errorClass: "network",
|
||||
retryable: true,
|
||||
},
|
||||
{
|
||||
name: "gateway business error code remains terminal",
|
||||
client: &usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{SimKey: &model.SimKey{ICCID: task.ICCID}, Code: 9999, ErrCode: "9999", Message: "invalid request"}}},
|
||||
errorClass: "business",
|
||||
},
|
||||
{
|
||||
name: "telecom threshold blocks channel via errCode",
|
||||
client: &usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{SimKey: &model.SimKey{ICCID: task.ICCID}, Code: 9999, Message: "流量查询失败", ErrCode: "S00004"}}},
|
||||
errorClass: "channel_limited",
|
||||
blockChannel: true,
|
||||
},
|
||||
{
|
||||
name: "message text alone does not block channel",
|
||||
client: &usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{SimKey: &model.SimKey{ICCID: task.ICCID}, Code: 13000, Message: "carrier S00004 mentioned in text"}}},
|
||||
errorClass: "business",
|
||||
},
|
||||
{
|
||||
name: "network failure is terminal for run",
|
||||
client: &usageClientStub{err: errors.New("connection reset")},
|
||||
errorClass: "network",
|
||||
retryable: true,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
handler, err := NewHandler(test.client, &usageRepositoryStub{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := handler.Query(context.Background(), task)
|
||||
if result.Success != test.success || result.ErrorClass != test.errorClass || result.Retryable != test.retryable || result.BlockChannel != test.blockChannel {
|
||||
t.Fatalf("result = %+v", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMobileTemplatesClassifyBusyForRoundRetry(t *testing.T) {
|
||||
for template := range mobileUsageTemplates {
|
||||
for code := mobileBusyFirstCode; code <= mobileBusyLastCode; code++ {
|
||||
t.Run(fmt.Sprintf("%s/code-%d", template, code), func(t *testing.T) {
|
||||
handler, err := NewHandler(&usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{
|
||||
SimKey: &model.SimKey{ICCID: "89860001"}, Code: code, Message: "system busy",
|
||||
}}}, &usageRepositoryStub{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := handler.Query(context.Background(), queue.Task{TaskID: "busy", TempKey: template, ICCID: "89860001"})
|
||||
if result.ErrorClass != mobileBusyClass || !result.Retryable || result.BlockChannel {
|
||||
t.Fatalf("template %q code %d result = %+v", template, code, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLanshangSharedTokenTemplatesRetryAcrossRounds(t *testing.T) {
|
||||
for _, template := range []string{"lanshang_8000", "lanshang_8003"} {
|
||||
for _, code := range []int{13000, 13001} {
|
||||
t.Run(fmt.Sprintf("%s/code-%d", template, code), func(t *testing.T) {
|
||||
errorClass, retryable, blockChannel := classifyUsageFailure(template, code, "")
|
||||
if errorClass != mobileBusyClass || !retryable || blockChannel {
|
||||
t.Fatalf("template %q code %d = (%q, %t, %t)", template, code, errorClass, retryable, blockChannel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMobileBusyRangeBoundariesAndPermanentCodes(t *testing.T) {
|
||||
template := "sim_onelink_kedang_v1"
|
||||
tests := []struct {
|
||||
name string
|
||||
code int
|
||||
errCode string
|
||||
mobileBusy bool
|
||||
retryable bool
|
||||
}{
|
||||
{name: "13007 is last busy code", code: 13007, mobileBusy: true, retryable: true},
|
||||
{name: "13008 is outside busy range", code: 13008},
|
||||
{name: "12021 token expiry is retryable but not busy", code: 12021, retryable: true},
|
||||
{name: "11011 permanent failure is not busy", code: 11011},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
errorClass, retryable, _ := classifyUsageFailure(template, test.code, test.errCode)
|
||||
if test.mobileBusy {
|
||||
if errorClass != mobileBusyClass || !retryable {
|
||||
t.Fatalf("code %d = (%q, %t)", test.code, errorClass, retryable)
|
||||
}
|
||||
return
|
||||
}
|
||||
if errorClass == mobileBusyClass {
|
||||
t.Fatalf("code %d = (%q, %t)", test.code, errorClass, retryable)
|
||||
}
|
||||
if retryable != test.retryable {
|
||||
t.Fatalf("code %d retryable = %t, want %t (errorClass=%q)", test.code, retryable, test.retryable, errorClass)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelecomTemplateIsNeverMobileBusy(t *testing.T) {
|
||||
for code := mobileBusyFirstCode; code <= mobileBusyLastCode; code++ {
|
||||
errorClass, retryable, blockChannel := classifyUsageFailure(telecomUsageTemplate, code, "")
|
||||
if errorClass == mobileBusyClass || retryable || blockChannel {
|
||||
t.Fatalf("telecom code %d = (%q, %t, %t)", code, errorClass, retryable, blockChannel)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownTemplateDefaultsToGenericNotMobile(t *testing.T) {
|
||||
if carrier := usageCarrierForTemplate("sim_dx_v2_future"); carrier != "generic" {
|
||||
t.Fatalf("unknown template carrier = %q", carrier)
|
||||
}
|
||||
errorClass, retryable, _ := classifyUsageFailure("sim_dx_v2_future", 13000, "")
|
||||
if errorClass == mobileBusyClass || retryable {
|
||||
t.Fatalf("unknown template 13000 = (%q, %t)", errorClass, retryable)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelecomV3ChannelLimitClassification(t *testing.T) {
|
||||
errorClass, retryable, blockChannel := classifyUsageFailure(" SIM_DX_V3 ", 9999, "S00004")
|
||||
if errorClass != "channel_limited" || retryable || !blockChannel {
|
||||
t.Fatalf("v3 channel limit = (%q, %t, %t)", errorClass, retryable, blockChannel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonTelecomS00004DoesNotBlockChannel(t *testing.T) {
|
||||
handler, err := NewHandler(&usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{
|
||||
SimKey: &model.SimKey{ICCID: "89860001"}, Code: 9999, ErrCode: errCodeTelecomChannelLimited,
|
||||
}}}, &usageRepositoryStub{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := handler.Query(context.Background(), queue.Task{TaskID: "mobile", TempKey: "sim_yd_v5", ICCID: "89860001"})
|
||||
if result.ErrorClass != "business" || result.BlockChannel {
|
||||
t.Fatalf("result = %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyWritesSuccessfulResultsInOneBatch(t *testing.T) {
|
||||
repository := &usageRepositoryStub{}
|
||||
handler, err := NewHandler(&usageClientStub{}, repository)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
results := []zsetconsumer.Result{
|
||||
{Task: queue.Task{CardID: 1}, Success: true, UsageMB: 1.5},
|
||||
{Task: queue.Task{CardID: 2}, ErrorClass: "business", Code: 13000},
|
||||
{Task: queue.Task{CardID: 3}, Success: true, UsageMB: 3.5},
|
||||
}
|
||||
if err = handler.Apply(context.Background(), results); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if repository.calls != 1 || len(repository.updates) != 2 {
|
||||
t.Fatalf("repository calls = %d, updates = %+v", repository.calls, repository.updates)
|
||||
}
|
||||
if repository.updates[0].CardID != 1 || repository.updates[1].CardID != 3 {
|
||||
t.Fatalf("updates = %+v", repository.updates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPreservesNetworkFailureMessage(t *testing.T) {
|
||||
handler, err := NewHandler(&usageClientStub{err: errors.New("connection refused")}, &usageRepositoryStub{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := handler.Query(context.Background(), queue.Task{ICCID: "89860001"})
|
||||
if result.ErrorClass != "network" || !strings.Contains(result.Message, "connection refused") {
|
||||
t.Fatalf("result = %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuerySkipsIccidCheckWhenTaskHasNoIccid(t *testing.T) {
|
||||
handler, err := NewHandler(&usageClientStub{output: []model.SimBaseRes[model.SimDataUsageItem]{{
|
||||
SimKey: &model.SimKey{ICCID: ""},
|
||||
ResData: model.SimDataUsageItem{DataUsage: 7.5},
|
||||
}}}, &usageRepositoryStub{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := handler.Query(context.Background(), queue.Task{TaskID: "1", ChanKey: "telecom-a", CardID: 1, MSISDN: "14900000001"})
|
||||
if !result.Success || result.UsageMB != 7.5 {
|
||||
t.Fatalf("result = %+v", result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package monthlyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
zsetconsumer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consumer/zset"
|
||||
fullsyncarchive "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/archive"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
|
||||
fullsyncruntime "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/runtime"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type runUsageClient struct{}
|
||||
|
||||
func (runUsageClient) QuerySimDataUsage(_ context.Context, input *model.SimDataUsageInput) ([]model.SimBaseRes[model.SimDataUsageItem], error) {
|
||||
iccid := input.SimKeys[0].ICCID
|
||||
switch iccid {
|
||||
case "retry":
|
||||
return nil, errors.New("temporary network failure")
|
||||
case "business":
|
||||
return []model.SimBaseRes[model.SimDataUsageItem]{{SimKey: &model.SimKey{ICCID: iccid}, Code: 11011, Message: "business failure"}}, nil
|
||||
default:
|
||||
return []model.SimBaseRes[model.SimDataUsageItem]{{SimKey: &model.SimKey{ICCID: iccid}, ResData: model.SimDataUsageItem{DataUsage: 12.5}}}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonthlyRunRetryConvergesAndArchives(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
ctx := context.Background()
|
||||
store, err := zset.New(redis, "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.EnableArchiveIndex(true)
|
||||
const runID = "monthly-retry-archive"
|
||||
if err := store.CreateRun(ctx, zset.RunMeta{RunID: runID, Operation: "current_month_usage", QueryMonth: "2026-09"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks := []zset.Task{
|
||||
{RunID: runID, TaskID: "retry", CardID: 1, ChanKey: "channel", TempKey: "sim_yd_v5", ICCID: "retry"},
|
||||
{RunID: runID, TaskID: "business", CardID: 2, ChanKey: "channel", TempKey: "sim_yd_v5", ICCID: "business"},
|
||||
{RunID: runID, TaskID: "success", CardID: 3, ChanKey: "channel", TempKey: "sim_yd_v5", ICCID: "success"},
|
||||
}
|
||||
if n, err := store.EnqueueRunBatch(ctx, runID, "channel", tasks, time.Now()); err != nil || n != len(tasks) {
|
||||
t.Fatalf("enqueue = %d, err = %v", n, err)
|
||||
}
|
||||
if err := store.MarkProducerDone(ctx, runID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := &usageRepositoryStub{}
|
||||
handler, err := NewHandler(runUsageClient{}, repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer, err := zsetconsumer.New(store, handler, zsetconsumer.Config{
|
||||
RunQueueEnabled: true,
|
||||
RoundRetryEnabled: true,
|
||||
MaxRounds: 2,
|
||||
ShouldRetryNextRound: func(result zsetconsumer.Result) bool {
|
||||
return !result.Success && result.Retryable
|
||||
},
|
||||
PollInterval: 500 * time.Millisecond,
|
||||
ClaimBatch: 3,
|
||||
TemplateQPS: map[string]float64{"sim_yd_v5": 1000000},
|
||||
LeaseTTL: time.Minute,
|
||||
TaskTimeout: 30 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- consumer.Run(runCtx) }()
|
||||
defer func() {
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Errorf("consumer stopped: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
var meta zset.RunMeta
|
||||
for time.Now().Before(deadline) {
|
||||
meta, err = store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.NextRoundCount == 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if meta.CurrentRound != 1 || meta.ActiveCount != 0 || meta.InflightCount != 0 || meta.NextRoundCount != 1 || meta.SuccessCount != 1 || meta.FailedCount != 1 || meta.Status != "running" {
|
||||
t.Fatalf("first round did not defer only retryable failure: %+v", meta)
|
||||
}
|
||||
failedKey, _ := store.FailedKey(runID)
|
||||
if count, err := redis.Do(ctx, "HLEN", failedKey); err != nil || count.Int() != 1 {
|
||||
t.Fatalf("first round final failures = %v, err = %v", count, err)
|
||||
}
|
||||
for time.Now().Before(deadline) {
|
||||
meta, err = store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.Status == "completed_with_failures" {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if meta.Status != "completed_with_failures" || meta.CurrentRound != 2 || meta.SuccessCount != 1 || meta.FailedCount != 2 || meta.ActiveCount != 0 || meta.NextRoundCount != 0 || meta.CompletedAt.IsZero() {
|
||||
t.Fatalf("run did not converge at max rounds: %+v", meta)
|
||||
}
|
||||
if len(repo.updates) != 1 || repo.updates[0].CardID != 3 {
|
||||
t.Fatalf("successful usage update changed: %+v", repo.updates)
|
||||
}
|
||||
failed, err := redis.Do(ctx, "HGET", failedKey, "retry")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var record zset.FailureRecord
|
||||
if err := json.Unmarshal([]byte(failed.String()), &record); err != nil || record.Round != 2 || record.FailureType != "network" {
|
||||
t.Fatalf("exhausted failure = %+v, err = %v", record, err)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
runtimeRepo, err := fullsyncruntime.New(store, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page, err := runtimeRepo.ListArchiveWork(ctx, contract.PageRequest{Limit: 10})
|
||||
if err != nil || len(page.Items) != 1 || page.Items[0].RunID != runID {
|
||||
t.Fatalf("pending archive = %+v, err = %v", page, err)
|
||||
}
|
||||
service := fullsyncarchive.New(root, runtimeRepo, runtimeRepo, runtimeRepo)
|
||||
if err := service.RunOnce(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(root, "run-"+runID+".json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var archive contract.RunArchive
|
||||
if err := json.Unmarshal(data, &archive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if archive.Run.Status != contract.RunCompletedWithFailures || archive.Run.SuccessCount != 1 || archive.Run.FailedCount != 2 || len(archive.Rounds) != 2 {
|
||||
t.Fatalf("archived run summary = %+v", archive)
|
||||
}
|
||||
if first, last := archive.Rounds[0], archive.Rounds[1]; first.Round != 1 || first.EnqueuedCount != 3 || first.SuccessCount != 1 || first.FailedCount != 1 || first.NextRoundCount != 1 || last.Round != 2 || last.EnqueuedCount != 1 || last.FailedCount != 1 || last.NextRoundCount != 0 {
|
||||
t.Fatalf("archived round stats = %+v", archive.Rounds)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user