chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,103 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
// defaultBalanceRechargeMaxRetry 是充值超时消息的默认最大重试次数。
|
||||
// 支付宝凭据未配置等永久性失败会让 CheckRechargeTimeout 一直返回错误，
|
||||
// 必须设置上限，否则消息会无限重投并刷屏日志。
|
||||
const defaultBalanceRechargeMaxRetry = 100
|
||||
|
||||
// 包级变量便于测试注入发布行为，与 trafficUsagePublish 保持一致。
|
||||
var balanceRechargePublish = mq.Publish
|
||||
var balanceRechargePublishDelayed = mq.PublishDelayed
|
||||
|
||||
func (c *Consumers) initBalanceRechargeTimeout(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "balance recharge timeout consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if c.balance == nil {
|
||||
return gerror.New("balance recharge handler is not configured")
|
||||
}
|
||||
retryDelay, err := getBalanceRechargeRetryDelay(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maxRetry, err := getBalanceRechargeMaxRetry(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.balanceRechargeRetryDelay = retryDelay
|
||||
c.balanceRechargeMaxRetry = maxRetry
|
||||
if err := mq.DeclareQueue(ctx, consts.QueueBalanceRechargeDeadLetter); err != nil {
|
||||
return gerror.Wrap(err, "declare balance recharge dead-letter queue failed")
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.BalanceRechargeTimeoutMessage](
|
||||
ctx,
|
||||
consts.QueueBalanceRechargeTimeout,
|
||||
"iotlcm-balance-recharge-timeout",
|
||||
c.handleBalanceRechargeTimeout,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "balance recharge timeout consumer started, queue:%s, retryDelay:%s, maxRetry:%d", consts.QueueBalanceRechargeTimeout, retryDelay, maxRetry)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleBalanceRechargeTimeout(ctx context.Context, message model.BalanceRechargeTimeoutMessage) error {
|
||||
if message.OutTradeNo == "" {
|
||||
g.Log().Errorf(ctx, "balance recharge timeout message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
if c.balance == nil {
|
||||
return gerror.New("balance recharge handler is not configured")
|
||||
}
|
||||
if err := c.balance.CheckRechargeTimeout(ctx, message.OutTradeNo); err != nil {
|
||||
message.RetryCount++
|
||||
if shouldDeadLetterBalanceRecharge(message.RetryCount, c.balanceRechargeMaxRetry) {
|
||||
if publishErr := balanceRechargePublish(ctx, consts.QueueBalanceRechargeDeadLetter, message); publishErr != nil {
|
||||
return gerror.Wrap(publishErr, "publish balance recharge dead-letter message failed")
|
||||
}
|
||||
g.Log().Errorf(ctx, "balance recharge timeout retries exhausted and message sent to dead-letter queue, outTradeNo:%s, retryCount:%d, maxRetry:%d, err:%+v", message.OutTradeNo, message.RetryCount, c.balanceRechargeMaxRetry, err)
|
||||
return nil
|
||||
}
|
||||
if retryErr := balanceRechargePublishDelayed(ctx, consts.QueueBalanceRechargeDelay, consts.QueueBalanceRechargeTimeout, c.balanceRechargeRetryDelay, message); retryErr != nil {
|
||||
return gerror.Wrap(retryErr, "republish balance recharge timeout message failed")
|
||||
}
|
||||
g.Log().Warningf(ctx, "balance recharge timeout query failed and will retry, outTradeNo:%s, retryCount:%d, err:%+v", message.OutTradeNo, message.RetryCount, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shouldDeadLetterBalanceRecharge 判断本次失败后是否放弃重试。
|
||||
// retryCount 是自增后的重试次数，因此 maxRetry 表示最多重投多少次。
|
||||
func shouldDeadLetterBalanceRecharge(retryCount, maxRetry int) bool {
|
||||
return retryCount > maxRetry
|
||||
}
|
||||
|
||||
func getBalanceRechargeRetryDelay(ctx context.Context) (time.Duration, error) {
|
||||
value := g.Cfg().MustGet(ctx, "rabbitmq.balanceRecharge.retryDelay", "1m").String()
|
||||
delay, err := time.ParseDuration(value)
|
||||
if err != nil || delay <= 0 {
|
||||
return 0, gerror.Newf("balance recharge retry delay is invalid: %q", value)
|
||||
}
|
||||
return delay, nil
|
||||
}
|
||||
|
||||
func getBalanceRechargeMaxRetry(ctx context.Context) (int, error) {
|
||||
maxRetry := g.Cfg().MustGet(ctx, "rabbitmq.balanceRecharge.maxRetry", defaultBalanceRechargeMaxRetry).Int()
|
||||
if maxRetry <= 0 {
|
||||
return 0, gerror.Newf("balance recharge max retry is invalid: %d", maxRetry)
|
||||
}
|
||||
return maxRetry, nil
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func stubBalanceRechargePublish(t *testing.T) {
|
||||
t.Helper()
|
||||
originalPublish, originalDelayed := balanceRechargePublish, balanceRechargePublishDelayed
|
||||
t.Cleanup(func() {
|
||||
balanceRechargePublish, balanceRechargePublishDelayed = originalPublish, originalDelayed
|
||||
})
|
||||
balanceRechargePublish = func(context.Context, string, any) error { return nil }
|
||||
balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error { return nil }
|
||||
}
|
||||
|
||||
func setBalanceRechargeTestConfig(t *testing.T, content string) {
|
||||
t.Helper()
|
||||
oldAdapter := g.Cfg().GetAdapter()
|
||||
adapter, err := gcfg.NewAdapterContent(content)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
t.Cleanup(func() { g.Cfg().SetAdapter(oldAdapter) })
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeoutRepublishesDelayedBelowMaxRetry(t *testing.T) {
|
||||
stubBalanceRechargePublish(t)
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
|
||||
consumers.balanceRechargeRetryDelay = time.Minute
|
||||
consumers.balanceRechargeMaxRetry = 3
|
||||
|
||||
delayedCalls := 0
|
||||
balanceRechargePublishDelayed = func(_ context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error {
|
||||
delayedCalls++
|
||||
if delayQueue != consts.QueueBalanceRechargeDelay || targetQueue != consts.QueueBalanceRechargeTimeout || delay != time.Minute {
|
||||
t.Fatalf("unexpected delayed publish: %s -> %s, delay:%s", delayQueue, targetQueue, delay)
|
||||
}
|
||||
message, ok := data.(model.BalanceRechargeTimeoutMessage)
|
||||
if !ok || message.OutTradeNo != "ALI-1" || message.RetryCount != 3 {
|
||||
t.Fatalf("unexpected retry payload: %+v", data)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
deadLetterCalls := 0
|
||||
balanceRechargePublish = func(_ context.Context, queue string, _ any) error {
|
||||
deadLetterCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
// 自增后达到上限仍未超过，应继续重投延迟队列。
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 2}); err != nil {
|
||||
t.Fatalf("retry below cap returned error: %v", err)
|
||||
}
|
||||
if delayedCalls != 1 || deadLetterCalls != 0 {
|
||||
t.Fatalf("delayedCalls=%d deadLetterCalls=%d, want 1 and 0", delayedCalls, deadLetterCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeoutDeadLettersAboveMaxRetry(t *testing.T) {
|
||||
stubBalanceRechargePublish(t)
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
|
||||
consumers.balanceRechargeRetryDelay = time.Minute
|
||||
consumers.balanceRechargeMaxRetry = 3
|
||||
|
||||
delayedCalls := 0
|
||||
balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error {
|
||||
delayedCalls++
|
||||
return nil
|
||||
}
|
||||
deadLetterCalls := 0
|
||||
balanceRechargePublish = func(_ context.Context, queue string, data any) error {
|
||||
deadLetterCalls++
|
||||
if queue != consts.QueueBalanceRechargeDeadLetter {
|
||||
t.Fatalf("dead letter queue = %s, want %s", queue, consts.QueueBalanceRechargeDeadLetter)
|
||||
}
|
||||
message, ok := data.(model.BalanceRechargeTimeoutMessage)
|
||||
if !ok || message.OutTradeNo != "ALI-1" || message.RetryCount != 4 {
|
||||
t.Fatalf("unexpected dead-letter payload: %+v", data)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 3}); err != nil {
|
||||
t.Fatalf("dead-letter path returned error: %v", err)
|
||||
}
|
||||
if delayedCalls != 0 || deadLetterCalls != 1 {
|
||||
t.Fatalf("delayedCalls=%d deadLetterCalls=%d, want 0 and 1", delayedCalls, deadLetterCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeoutReturnsPublishErrors(t *testing.T) {
|
||||
stubBalanceRechargePublish(t)
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
|
||||
consumers.balanceRechargeRetryDelay = time.Minute
|
||||
consumers.balanceRechargeMaxRetry = 3
|
||||
|
||||
expected := errors.New("broker unavailable")
|
||||
balanceRechargePublish = func(context.Context, string, any) error { return expected }
|
||||
balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error { return expected }
|
||||
|
||||
// 重投失败必须向上返回错误，让消费框架 Nack 重新入队，不能静默丢失消息。
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 0}); !errors.Is(err, expected) {
|
||||
t.Fatalf("delayed publish failure = %v, want %v", err, expected)
|
||||
}
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 3}); !errors.Is(err, expected) {
|
||||
t.Fatalf("dead-letter publish failure = %v, want %v", err, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBalanceRechargeMaxRetry(t *testing.T) {
|
||||
t.Run("defaults when unconfigured", func(t *testing.T) {
|
||||
setBalanceRechargeTestConfig(t, "rabbitmq:\n balanceRecharge:\n retryDelay: \"1m\"\n")
|
||||
maxRetry, err := getBalanceRechargeMaxRetry(context.Background())
|
||||
if err != nil || maxRetry != defaultBalanceRechargeMaxRetry {
|
||||
t.Fatalf("maxRetry = %d (err=%v), want %d", maxRetry, err, defaultBalanceRechargeMaxRetry)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reads configured value", func(t *testing.T) {
|
||||
setBalanceRechargeTestConfig(t, "rabbitmq:\n balanceRecharge:\n maxRetry: 7\n")
|
||||
maxRetry, err := getBalanceRechargeMaxRetry(context.Background())
|
||||
if err != nil || maxRetry != 7 {
|
||||
t.Fatalf("maxRetry = %d (err=%v), want 7", maxRetry, err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{name: "zero", value: "0"},
|
||||
{name: "negative", value: "-1"},
|
||||
{name: "non-numeric", value: "\"abc\""},
|
||||
} {
|
||||
t.Run("rejects "+test.name, func(t *testing.T) {
|
||||
setBalanceRechargeTestConfig(t, "rabbitmq:\n balanceRecharge:\n maxRetry: "+test.value+"\n")
|
||||
if _, err := getBalanceRechargeMaxRetry(context.Background()); err == nil {
|
||||
t.Fatalf("maxRetry=%s was accepted, want error", test.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
)
|
||||
|
||||
const batchRetryCountHeader = "x-iotlcm-retry-count"
|
||||
|
||||
func (c *Consumers) initBatchExecute(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "batch execute consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if c.batch == nil {
|
||||
return gerror.New("batch execute handler is not configured")
|
||||
}
|
||||
retryDelay, maxRetries, err := getBatchExecuteRetryConfig(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.batchExecuteRetryDelay = retryDelay
|
||||
c.batchExecuteMaxRetries = maxRetries
|
||||
if err = mq.DeclareQueue(ctx, consts.QueueBatchExecuteDeadLetter); err != nil {
|
||||
return gerror.Wrap(err, "declare batch execute dead-letter queue failed")
|
||||
}
|
||||
if err = mq.ConsumeJSON[batch.Message](
|
||||
ctx,
|
||||
consts.QueueBatchExecute,
|
||||
"iotlcm-batch-execute",
|
||||
c.handleBatchExecute,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "batch execute consumer started, queue:%s, retryDelay:%s, maxRetries:%d", consts.QueueBatchExecute, retryDelay, maxRetries)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleBatchExecute(ctx context.Context, message batch.Message) error {
|
||||
if c.batch == nil {
|
||||
return gerror.New("batch execute handler is not configured")
|
||||
}
|
||||
g.Log().Infof(ctx, "batch execute message received, batchNo:%s, batchType:%d", message.BatchNo, message.BatchType)
|
||||
if err := c.batch.Handle(ctx, message); err != nil {
|
||||
if _, delivered := mq.DeliveryMetadataFromContext(ctx); !delivered {
|
||||
return err
|
||||
}
|
||||
if batch.IsRetryable(err) {
|
||||
message.Retryable = true
|
||||
}
|
||||
var failureMarkErr *batch.FailureMarkError
|
||||
if errors.As(err, &failureMarkErr) {
|
||||
message.FailureReason = failureMarkErr.Reason
|
||||
}
|
||||
return c.retryOrDeadLetterBatchMessage(ctx, message, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) retryOrDeadLetterBatchMessage(ctx context.Context, message batch.Message, handlerErr error) error {
|
||||
metadata, _ := mq.DeliveryMetadataFromContext(ctx)
|
||||
retryCount := batchRetryCount(metadata.Headers)
|
||||
headers := amqp.Table{batchRetryCountHeader: retryCount + 1}
|
||||
if shouldDeadLetterBatch(retryCount, c.batchExecuteMaxRetries) {
|
||||
if message.Retryable {
|
||||
finalizationMessage := message
|
||||
finalizationMessage.Retryable = false
|
||||
finalizationMessage.FailureReason = handlerErr.Error()
|
||||
if err := c.batch.Handle(ctx, finalizationMessage); err != nil {
|
||||
return gerror.Wrap(err, "finalize retryable batch failure before dead-lettering")
|
||||
}
|
||||
message.FailureReason = handlerErr.Error()
|
||||
}
|
||||
deadLetterBody, marshalErr := json.Marshal(batchDeadLetterMessage{
|
||||
Message: message,
|
||||
RetryCount: retryCount,
|
||||
Error: handlerErr.Error(),
|
||||
FailedAt: time.Now().Format(time.RFC3339),
|
||||
})
|
||||
if marshalErr != nil {
|
||||
return gerror.Wrap(marshalErr, "marshal batch execute dead-letter message failed")
|
||||
}
|
||||
if err := mq.PublishBytesWithHeaders(ctx, consts.QueueBatchExecuteDeadLetter, deadLetterBody, nil); err != nil {
|
||||
return gerror.Wrap(err, "publish batch execute dead-letter message failed")
|
||||
}
|
||||
g.Log().Errorf(ctx, "batch execute message sent to dead-letter queue, batchNo:%s, batchType:%d, retryCount:%d, err:%+v", message.BatchNo, message.BatchType, retryCount, handlerErr)
|
||||
return nil
|
||||
}
|
||||
body, marshalErr := json.Marshal(message)
|
||||
if marshalErr != nil {
|
||||
return gerror.Wrap(marshalErr, "marshal batch execute retry message failed")
|
||||
}
|
||||
if err := mq.PublishDelayedBytesWithHeaders(
|
||||
ctx,
|
||||
consts.QueueBatchExecuteRetry,
|
||||
consts.QueueBatchExecute,
|
||||
c.batchExecuteRetryDelay,
|
||||
body,
|
||||
headers,
|
||||
); err != nil {
|
||||
return gerror.Wrap(err, "republish batch execute retry message failed")
|
||||
}
|
||||
g.Log().Warningf(ctx, "batch execute message scheduled for retry, batchNo:%s, batchType:%d, retryCount:%d, delay:%s, err:%+v", message.BatchNo, message.BatchType, retryCount+1, c.batchExecuteRetryDelay, handlerErr)
|
||||
return nil
|
||||
}
|
||||
|
||||
type batchDeadLetterMessage struct {
|
||||
Message batch.Message `json:"message"`
|
||||
RetryCount int `json:"retryCount"`
|
||||
Error string `json:"error"`
|
||||
FailedAt string `json:"failedAt"`
|
||||
}
|
||||
|
||||
func shouldDeadLetterBatch(retryCount, maxRetries int) bool {
|
||||
return retryCount >= maxRetries
|
||||
}
|
||||
|
||||
func batchRetryCount(headers map[string]any) int {
|
||||
if headers == nil {
|
||||
return 0
|
||||
}
|
||||
value, ok := headers[batchRetryCountHeader]
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
switch count := value.(type) {
|
||||
case int:
|
||||
return max(count, 0)
|
||||
case int8:
|
||||
return max(int(count), 0)
|
||||
case int16:
|
||||
return max(int(count), 0)
|
||||
case int32:
|
||||
return max(int(count), 0)
|
||||
case int64:
|
||||
return max(int(count), 0)
|
||||
case uint8:
|
||||
return int(count)
|
||||
case uint16:
|
||||
return int(count)
|
||||
case uint32:
|
||||
return int(count)
|
||||
case uint64:
|
||||
if count > uint64(^uint(0)>>1) {
|
||||
return int(^uint(0) >> 1)
|
||||
}
|
||||
return int(count)
|
||||
case string:
|
||||
parsed, err := strconv.Atoi(count)
|
||||
if err == nil {
|
||||
return max(parsed, 0)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func getBatchExecuteRetryConfig(ctx context.Context) (time.Duration, int, error) {
|
||||
delay := g.Cfg().MustGet(ctx, "rabbitmq.batchExecute.retryDelay", "30s").Duration()
|
||||
if delay <= 0 {
|
||||
return 0, 0, gerror.New("batch execute retry delay must be positive")
|
||||
}
|
||||
maxRetries := g.Cfg().MustGet(ctx, "rabbitmq.batchExecute.maxRetries", 3).Int()
|
||||
if maxRetries < 0 {
|
||||
return 0, 0, gerror.New("batch execute max retries cannot be negative")
|
||||
}
|
||||
return delay, maxRetries, nil
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
batchrepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/batch"
|
||||
datarepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/data"
|
||||
dataservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data"
|
||||
)
|
||||
|
||||
type durableBatchRepo struct{ failed int }
|
||||
|
||||
func (*durableBatchRepo) FindMain(context.Context, string) (*entity.BatchMain, error) {
|
||||
return &entity.BatchMain{BatchNo: "APN-RECOVER", BatchType: 109, BatchAsyncStatus: batch.AsyncStatusPending}, nil
|
||||
}
|
||||
func (*durableBatchRepo) MarkProcessing(context.Context, string, int) (bool, error) { return true, nil }
|
||||
func (r *durableBatchRepo) MarkFailed(context.Context, string, int, string) (bool, error) {
|
||||
r.failed++
|
||||
return true, nil
|
||||
}
|
||||
func (*durableBatchRepo) MarkSuccess(context.Context, string, int, int, int, string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type durableBatchExecutor struct {
|
||||
calls, protections int
|
||||
retryable bool
|
||||
protectionErr error
|
||||
}
|
||||
|
||||
func (e *durableBatchExecutor) Execute(context.Context, string) error {
|
||||
e.calls++
|
||||
if e.retryable {
|
||||
return batch.Retryable(errors.New("准备中断"))
|
||||
}
|
||||
return errors.New("本地执行中断")
|
||||
}
|
||||
func (e *durableBatchExecutor) ProtectExecutionFailure(ctx context.Context, _ string, _ int, _ string) (bool, error) {
|
||||
e.protections++
|
||||
if ctx.Err() != nil {
|
||||
return true, ctx.Err()
|
||||
}
|
||||
return true, e.protectionErr
|
||||
}
|
||||
|
||||
func TestDurableAPNConsumerNeverFallsThroughToGenericFailure(t *testing.T) {
|
||||
repo := &durableBatchRepo{}
|
||||
executor := &durableBatchExecutor{}
|
||||
dispatcher := batch.NewDispatcher(nil)
|
||||
dispatcher.Register(109, executor)
|
||||
consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher)}
|
||||
if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if executor.calls != 1 || executor.protections != 1 || repo.failed != 0 {
|
||||
t.Fatalf("消费者普通错误越过durable保护: calls=%d protections=%d failed=%d", executor.calls, executor.protections, repo.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNConsumerRetryExhaustionProtectsBeforePublish(t *testing.T) {
|
||||
repo := &durableBatchRepo{}
|
||||
dbDown := errors.New("模拟保护数据库不可用")
|
||||
executor := &durableBatchExecutor{retryable: true, protectionErr: dbDown}
|
||||
dispatcher := batch.NewDispatcher(nil)
|
||||
dispatcher.Register(109, executor)
|
||||
consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher), batchExecuteMaxRetries: 0}
|
||||
ctx := mq.WithDeliveryMetadata(context.Background(), mq.DeliveryMetadata{Headers: map[string]any{batchRetryCountHeader: 3}})
|
||||
err := consumers.handleBatchExecute(ctx, batch.Message{BatchNo: "APN-RECOVER", BatchType: 109})
|
||||
// 保护失败必须在消息发布前返回，测试不会连接真实RabbitMQ。
|
||||
if !errors.Is(err, dbDown) || executor.calls != 1 || executor.protections != 1 || repo.failed != 0 {
|
||||
t.Fatalf("重试耗尽错误终结或重发: err=%v calls=%d protections=%d failed=%d", err, executor.calls, executor.protections, repo.failed)
|
||||
}
|
||||
executor.protectionErr = nil
|
||||
if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109, FailureReason: "重试耗尽"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if executor.calls != 1 || executor.protections != 2 || repo.failed != 0 {
|
||||
t.Fatal("失败重投重新执行或MarkFailed")
|
||||
}
|
||||
}
|
||||
|
||||
// reentryWriteCounter 同时覆盖同步写、异步提交与结果查询，重入准备期间均不得调用。
|
||||
type reentryWriteCounter struct{ syncWrites, asyncWrites, queries int }
|
||||
|
||||
func (u *reentryWriteCounter) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.syncWrites++
|
||||
return nil, nil
|
||||
}
|
||||
func (u *reentryWriteCounter) SubmitSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.asyncWrites++
|
||||
return nil, nil
|
||||
}
|
||||
func (u *reentryWriteCounter) QuerySimApnFunctionResult(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.queries++
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestAPNProcessingRabbitMQReentryRequiresDurableParent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "sqlite", Link: "sqlite::@file(" + filepath.ToSlash(filepath.Join(t.TempDir(), "apn-reentry.db")) + ")"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db := g.DB()
|
||||
t.Cleanup(func() {
|
||||
if err := db.Close(ctx); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
for _, sql := range []string{
|
||||
"CREATE TABLE batch_main (id INTEGER PRIMARY KEY,batch_no TEXT UNIQUE,batch_type INTEGER,batch_async_status INTEGER,tenant_id TEXT,start_time DATETIME,batch_info TEXT,run_result INTEGER DEFAULT 0,valid_count INTEGER DEFAULT 0,success_count INTEGER DEFAULT 0,fail_count INTEGER DEFAULT 0,fail_reason TEXT,end_time DATETIME,updated_at DATETIME,created_by INTEGER,created_dept INTEGER,updated_by INTEGER)",
|
||||
"CREATE TABLE batch_detail (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,row_no INTEGER,item_id TEXT,detail_status INTEGER,error_reason TEXT,snapshot TEXT,updated_by INTEGER)",
|
||||
"CREATE TABLE t_sim_card_info (id INTEGER PRIMARY KEY AUTOINCREMENT,tenant_id TEXT,iccid TEXT,msisdn TEXT,imsi TEXT,chan_key TEXT,data_service_status INTEGER)",
|
||||
"CREATE TABLE data_operation_parent (batch_no TEXT PRIMARY KEY,ready INTEGER DEFAULT 0,finalize_token TEXT,finalize_until DATETIME,recovery_attempt_at DATETIME,last_reason TEXT)",
|
||||
"CREATE TABLE data_operation_sub_batch (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,chan_key TEXT,operation_id TEXT UNIQUE,detail_ids TEXT,request_data TEXT,job_id TEXT,status TEXT,poll_count INTEGER DEFAULT 0,next_check_at DATETIME,submitted_at DATETIME,lease_token TEXT,lease_until DATETIME,created_at DATETIME DEFAULT CURRENT_TIMESTAMP,last_reason TEXT)",
|
||||
"CREATE TABLE data_operation_sub_detail (detail_id INTEGER PRIMARY KEY,operation_id TEXT,batch_no TEXT,iccid TEXT,UNIQUE(batch_no,iccid))",
|
||||
"CREATE TABLE data_operation_card_claim (tenant_id TEXT,iccid TEXT,batch_id INTEGER,operation_id TEXT,active_batch_id INTEGER DEFAULT 0,active_operation_id TEXT DEFAULT '',PRIMARY KEY(tenant_id,iccid))",
|
||||
} {
|
||||
if _, err := db.Exec(ctx, sql); err != nil {
|
||||
t.Fatalf("初始化本机 SQLite fixture: %v", err)
|
||||
}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
old, retryable bool
|
||||
}{
|
||||
{name: "legacy-ordinary", old: true},
|
||||
{name: "legacy-retryable", old: true, retryable: true},
|
||||
{name: "durable-ordinary"},
|
||||
{name: "durable-retryable", retryable: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
status := batch.AsyncStatusPending
|
||||
if tc.old {
|
||||
status = batch.AsyncStatusProcessing
|
||||
}
|
||||
// 使用 NULL 旧首发时间以确保 ReclaimProcessing 的 WHERE start_time IS NULL 可以命中。
|
||||
if _, err := db.Exec(ctx, "INSERT INTO batch_main(batch_no,batch_type,batch_async_status,tenant_id) VALUES (?,109,?,'tenant')", tc.name, status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
iccid := tc.name + "-card-" + string(rune('0'+i))
|
||||
detailStatus := batch.DetailStatusPending
|
||||
if tc.old {
|
||||
detailStatus = batch.DetailStatusRunning
|
||||
}
|
||||
if _, err := db.Exec(ctx, "INSERT INTO batch_detail(batch_no,batch_type,row_no,item_id,detail_status,snapshot) VALUES (?,109,?,?,?,?)", tc.name, i+1, iccid, detailStatus, `{"authorized":true}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(ctx, "INSERT INTO t_sim_card_info(tenant_id,iccid,msisdn,imsi,chan_key,data_service_status) VALUES ('tenant',?,?,?,'mock-apn',0)", iccid, "130000000"+string(rune('0'+i)), "460000000"+string(rune('0'+i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mainRepo := batchrepo.New()
|
||||
if tc.old && tc.retryable {
|
||||
// 先证实同一 fixture 的旧批确实可被重领，再复位空首发时间。
|
||||
claimed, err := mainRepo.ReclaimProcessing(ctx, tc.name, 109, nil)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("重领 CAS 必须可命中，避免假阳性: claimed=%t err=%v", claimed, err)
|
||||
}
|
||||
if _, err := db.Exec(ctx, "UPDATE batch_main SET start_time=NULL WHERE batch_no=?", tc.name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !tc.old {
|
||||
claimed, err := mainRepo.MarkProcessing(ctx, tc.name, 109)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("新批需同事务领取 Processing 与 ready=0 父凭据: claimed=%t err=%v", claimed, err)
|
||||
}
|
||||
}
|
||||
parent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=0", tc.name)
|
||||
if err != nil || parent.Int() != 1 && !tc.old || parent.Int() != 0 && tc.old {
|
||||
t.Fatalf("前置凭据不符: old=%t ready0=%d err=%v", tc.old, parent.Int(), err)
|
||||
}
|
||||
before, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upstream := &reentryWriteCounter{}
|
||||
dispatcher := batch.NewDispatcher(nil)
|
||||
dispatcher.Register(109, dataservice.NewExecutor(datarepo.New(), upstream))
|
||||
consumer := &Consumers{batch: batch.NewRuntime(mainRepo, dispatcher)}
|
||||
err = consumer.handleBatchExecute(ctx, batch.Message{BatchNo: tc.name, BatchType: 109, Retryable: tc.retryable})
|
||||
if err != nil {
|
||||
t.Errorf("重入应安全跳过旧批或准备新批，不能进入消息重发路径: %v", err)
|
||||
}
|
||||
if upstream.syncWrites != 0 || upstream.asyncWrites != 0 || upstream.queries != 0 {
|
||||
t.Errorf("重入意外调用上游: sync=%d submit=%d query=%d", upstream.syncWrites, upstream.asyncWrites, upstream.queries)
|
||||
}
|
||||
main, err := mainRepo.FindMain(ctx, tc.name)
|
||||
if err != nil || main == nil || main.BatchAsyncStatus != batch.AsyncStatusProcessing {
|
||||
t.Fatalf("批状态被意外终结: main=%+v err=%v", main, err)
|
||||
}
|
||||
remaining, err := db.GetValue(ctx, "SELECT COUNT(*) FROM batch_detail WHERE batch_no=? AND detail_status=1", tc.name)
|
||||
if err != nil || remaining.Int() != 10 {
|
||||
t.Fatalf("运行中明细不能被更改: count=%d err=%v", remaining.Int(), err)
|
||||
}
|
||||
intent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_batch WHERE batch_no=? AND status='prepared'", tc.name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.old {
|
||||
if intent.Int() != 0 {
|
||||
t.Errorf("无凭据历史批不应产生新提交意图: %d", intent.Int())
|
||||
}
|
||||
assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name)
|
||||
if err != nil || assigned.Int() != 0 {
|
||||
t.Errorf("历史批不可冻结明细: count=%d err=%v", assigned.Int(), err)
|
||||
}
|
||||
claims, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_card_claim WHERE operation_id!='' AND iccid LIKE ?", tc.name+"-%")
|
||||
if err != nil || claims.Int() != 0 {
|
||||
t.Errorf("历史批不可建立卡级提交占位: count=%d err=%v", claims.Int(), err)
|
||||
}
|
||||
parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=?", tc.name)
|
||||
if err != nil || parent.Int() != 0 {
|
||||
t.Errorf("历史批不可补造父凭据: count=%d err=%v", parent.Int(), err)
|
||||
}
|
||||
after, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name)
|
||||
if err != nil || after.String() != before.String() {
|
||||
t.Errorf("旧批重入不可重领取: before=%s after=%s err=%v", before.String(), after.String(), err)
|
||||
}
|
||||
} else {
|
||||
if intent.Int() != 1 {
|
||||
t.Errorf("新批 ready=0 未正确恢复成 10 卡一个冻结意图: %d", intent.Int())
|
||||
}
|
||||
parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=1", tc.name)
|
||||
if err != nil || parent.Int() != 1 {
|
||||
t.Errorf("新批冻结意图与 ready=1 未原子提交: count=%d err=%v", parent.Int(), err)
|
||||
}
|
||||
assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name)
|
||||
if err != nil || assigned.Int() != 10 {
|
||||
t.Errorf("新批冻结明细必须覆盖全部 10 卡: count=%d err=%v", assigned.Int(), err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package rabbitmq
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestShouldDeadLetterBatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
retryCount int
|
||||
maxRetries int
|
||||
want bool
|
||||
}{
|
||||
{name: "first attempt retries", retryCount: 0, maxRetries: 3, want: false},
|
||||
{name: "second attempt retries", retryCount: 1, maxRetries: 3, want: false},
|
||||
{name: "last configured retry retries", retryCount: 2, maxRetries: 3, want: false},
|
||||
{name: "next failure dead letters", retryCount: 3, maxRetries: 3, want: true},
|
||||
{name: "zero retries dead letters immediately", retryCount: 0, maxRetries: 0, want: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if actual := shouldDeadLetterBatch(tt.retryCount, tt.maxRetries); actual != tt.want {
|
||||
t.Fatalf("shouldDeadLetterBatch(%d, %d) = %t, want %t", tt.retryCount, tt.maxRetries, actual, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchRetryCount(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headers map[string]any
|
||||
want int
|
||||
}{
|
||||
{name: "missing", want: 0},
|
||||
{name: "int", headers: map[string]any{batchRetryCountHeader: 2}, want: 2},
|
||||
{name: "int64", headers: map[string]any{batchRetryCountHeader: int64(3)}, want: 3},
|
||||
{name: "uint", headers: map[string]any{batchRetryCountHeader: uint16(4)}, want: 4},
|
||||
{name: "string", headers: map[string]any{batchRetryCountHeader: "5"}, want: 5},
|
||||
{name: "negative", headers: map[string]any{batchRetryCountHeader: -1}, want: 0},
|
||||
{name: "invalid", headers: map[string]any{batchRetryCountHeader: "not-a-number"}, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if actual := batchRetryCount(tt.headers); actual != tt.want {
|
||||
t.Fatalf("batchRetryCount() = %d, want %d", actual, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
const channelRefreshConsumer = "iotlcm-worker-channel-refresh"
|
||||
|
||||
type channelRefreshConfig struct {
|
||||
Address string `json:"address"`
|
||||
Timeout time.Duration `json:"timeout"`
|
||||
}
|
||||
|
||||
type channelRefreshResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (c *Consumers) initChannelRefresh(ctx context.Context) error {
|
||||
if !rabbitmq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "channel refresh consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := rabbitmq.ConsumeJSON[model.ChannelRefreshMessage](
|
||||
ctx,
|
||||
consts.QueueChannelRefresh,
|
||||
channelRefreshConsumer,
|
||||
c.handleChannelRefresh,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "channel refresh consumer started, queue:%s", consts.QueueChannelRefresh)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleChannelRefresh(ctx context.Context, message model.ChannelRefreshMessage) error {
|
||||
switch strings.ToLower(strings.TrimSpace(message.Type)) {
|
||||
case "channel":
|
||||
if strings.TrimSpace(message.ChanKey) == "" {
|
||||
g.Log().Errorf(ctx, "channel refresh message dropped, chanKey is empty")
|
||||
return nil
|
||||
}
|
||||
if err := refreshChannelWithRetry(ctx, message.ChanKey); err != nil {
|
||||
g.Log().Errorf(ctx, "channel refresh failed, chanKey=%s, err=%+v", message.ChanKey, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
case "template":
|
||||
if strings.TrimSpace(message.TempKey) == "" {
|
||||
g.Log().Errorf(ctx, "channel refresh message dropped, tempKey is empty")
|
||||
return nil
|
||||
}
|
||||
return refreshTemplateChannels(ctx, message.TempKey)
|
||||
case "channel_rename":
|
||||
oldChanKey := strings.TrimSpace(message.OldChanKey)
|
||||
newChanKey := strings.TrimSpace(message.NewChanKey)
|
||||
if oldChanKey == "" || newChanKey == "" || oldChanKey == newChanKey {
|
||||
g.Log().Errorf(ctx, "channel rename message dropped, oldChanKey=%q newChanKey=%q", oldChanKey, newChanKey)
|
||||
return nil
|
||||
}
|
||||
if err := renameChannelWithRetry(ctx, oldChanKey, newChanKey); err != nil {
|
||||
// 返回 error 让消费框架 Nack 并重新入队：rename 各步骤幂等，
|
||||
// 重复投递安全；静默 Ack 会导致旧通道实例永久残留，只能重启恢复。
|
||||
g.Log().Errorf(ctx, "channel rename failed after retries, will requeue, oldChanKey=%s newChanKey=%s err=%+v", oldChanKey, newChanKey, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
g.Log().Errorf(ctx, "channel refresh message dropped, unsupported type=%s", message.Type)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func renameChannelWithRetry(ctx context.Context, oldChanKey, newChanKey string) error {
|
||||
var err error
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
err = renameChannel(ctx, oldChanKey, newChanKey)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if attempt == 3 {
|
||||
break
|
||||
}
|
||||
delay := time.Second
|
||||
if attempt == 2 {
|
||||
delay = 3 * time.Second
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
return gerror.Wrapf(err, "channel rename failed after %d attempts, oldChanKey=%s newChanKey=%s", 3, oldChanKey, newChanKey)
|
||||
}
|
||||
|
||||
// renameChannel is ordered so that the old runtime remains available until
|
||||
// both upstreams have built the new instance. Card data and ICCID routes are
|
||||
// intentionally left untouched for operations staff to migrate manually.
|
||||
func renameChannel(ctx context.Context, oldChanKey, newChanKey string) error {
|
||||
if err := refreshChannel(ctx, newChanKey); err != nil {
|
||||
return gerror.Wrap(err, "refresh new channel failed")
|
||||
}
|
||||
if err := removeChannelTarget(ctx, "upstream.gateway", oldChanKey); err != nil {
|
||||
return gerror.Wrap(err, "remove old gateway channel failed")
|
||||
}
|
||||
if err := removeChannelTarget(ctx, "upstream.proxy", oldChanKey); err != nil {
|
||||
return gerror.Wrap(err, "remove old proxy channel failed")
|
||||
}
|
||||
g.Log().Infof(ctx, "channel instance rebuilt after rename, oldChanKey=%s newChanKey=%s", oldChanKey, newChanKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
func refreshTemplateChannels(ctx context.Context, tempKey string) error {
|
||||
var rows []struct {
|
||||
ChanKey string `json:"chanKey"`
|
||||
}
|
||||
if err := g.DB().Model("t_sim_channels").Ctx(ctx).
|
||||
Fields("chan_key").
|
||||
Where("temp_key", strings.TrimSpace(tempKey)).
|
||||
Scan(&rows); err != nil {
|
||||
return gerror.Wrap(err, "query template channels failed")
|
||||
}
|
||||
|
||||
var firstErr error
|
||||
for _, row := range rows {
|
||||
if err := refreshChannelWithRetry(ctx, row.ChanKey); err != nil {
|
||||
g.Log().Errorf(ctx, "template channel refresh failed, tempKey=%s, chanKey=%s, err=%+v", tempKey, row.ChanKey, err)
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func refreshChannelWithRetry(ctx context.Context, chanKey string) error {
|
||||
var err error
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
err = refreshChannel(ctx, chanKey)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if attempt == 3 {
|
||||
break
|
||||
}
|
||||
delay := time.Second
|
||||
if attempt == 2 {
|
||||
delay = 3 * time.Second
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return gerror.Wrapf(err, "channel refresh canceled after %d attempts, chanKey=%s", attempt, chanKey)
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
return gerror.Wrapf(err, "channel refresh failed after %d attempts, chanKey=%s", 3, chanKey)
|
||||
}
|
||||
|
||||
func refreshChannel(ctx context.Context, chanKey string) error {
|
||||
chanKey = strings.TrimSpace(chanKey)
|
||||
if chanKey == "" {
|
||||
return gerror.New("chanKey is required")
|
||||
}
|
||||
if err := refreshChannelTarget(ctx, "upstream.gateway", chanKey); err != nil {
|
||||
return gerror.Wrap(err, "refresh gateway channel failed")
|
||||
}
|
||||
if err := refreshChannelTarget(ctx, "upstream.proxy", chanKey); err != nil {
|
||||
return gerror.Wrap(err, "refresh proxy channel failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func refreshChannelTarget(ctx context.Context, configKey, chanKey string) error {
|
||||
return channelControlTarget(ctx, configKey, "/internal/channel/refresh", chanKey)
|
||||
}
|
||||
|
||||
func removeChannelTarget(ctx context.Context, configKey, chanKey string) error {
|
||||
return channelControlTarget(ctx, configKey, "/internal/channel/remove", chanKey)
|
||||
}
|
||||
|
||||
func channelControlTarget(ctx context.Context, configKey, path, chanKey string) error {
|
||||
configValue, err := g.Cfg().Get(ctx, configKey)
|
||||
if err != nil {
|
||||
return gerror.Wrapf(err, "read upstream configuration failed: %s", configKey)
|
||||
}
|
||||
if configValue == nil || configValue.IsNil() {
|
||||
return gerror.Newf("upstream configuration is missing: %s", configKey)
|
||||
}
|
||||
config := channelRefreshConfig{}
|
||||
if err = configValue.Scan(&config); err != nil {
|
||||
return gerror.Wrapf(err, "parse upstream configuration failed: %s", configKey)
|
||||
}
|
||||
address := strings.TrimSpace(config.Address)
|
||||
if address == "" {
|
||||
return gerror.Newf("upstream address is empty: %s", configKey)
|
||||
}
|
||||
if config.Timeout <= 0 {
|
||||
config.Timeout = 10 * time.Second
|
||||
}
|
||||
requestCtx, cancel := context.WithTimeout(ctx, config.Timeout)
|
||||
defer cancel()
|
||||
|
||||
client := g.Client()
|
||||
client.SetTimeout(config.Timeout)
|
||||
controlURL := fmt.Sprintf("http://%s%s?chanKey=%s", address, path, url.QueryEscape(chanKey))
|
||||
response, err := client.Post(requestCtx, controlURL, nil)
|
||||
if err != nil {
|
||||
return gerror.Wrapf(err, "request upstream channel control failed: target=%s path=%s", configKey, path)
|
||||
}
|
||||
defer response.Close()
|
||||
|
||||
output := channelRefreshResponse{}
|
||||
statusCode := 0
|
||||
if response.Response != nil {
|
||||
statusCode = response.StatusCode
|
||||
}
|
||||
if err = json.Unmarshal(response.ReadAll(), &output); err != nil {
|
||||
return gerror.Wrapf(err, "parse upstream channel control response failed: target=%s path=%s status=%d", configKey, path, statusCode)
|
||||
}
|
||||
if output.Code != 0 {
|
||||
if output.Message == "" {
|
||||
output.Message = "upstream channel control failed"
|
||||
}
|
||||
return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d message=%s", configKey, path, statusCode, output.Message)
|
||||
}
|
||||
if statusCode < 200 || statusCode >= 300 {
|
||||
return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d", configKey, path, statusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func setChannelRefreshTestConfig(t *testing.T, gateway, proxy string) {
|
||||
t.Helper()
|
||||
oldAdapter := g.Cfg().GetAdapter()
|
||||
adapter, err := gcfg.NewAdapterContent(fmt.Sprintf(`
|
||||
upstream:
|
||||
gateway:
|
||||
address: %q
|
||||
timeout: 1s
|
||||
proxy:
|
||||
address: %q
|
||||
timeout: 1s
|
||||
`, gateway, proxy))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
t.Cleanup(func() { g.Cfg().SetAdapter(oldAdapter) })
|
||||
}
|
||||
|
||||
func testServerAddress(server *httptest.Server) string {
|
||||
return strings.TrimPrefix(server.URL, "http://")
|
||||
}
|
||||
|
||||
func TestRefreshChannelCallsGatewayAndProxy(t *testing.T) {
|
||||
var gatewayCalls atomic.Int32
|
||||
var proxyCalls atomic.Int32
|
||||
newServer := func(counter *atomic.Int32) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
counter.Add(1)
|
||||
if request.URL.Path != "/internal/channel/refresh" || request.URL.Query().Get("chanKey") != "channel-a" {
|
||||
t.Fatalf("unexpected request: %s", request.URL.String())
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success"}`))
|
||||
}))
|
||||
}
|
||||
gateway := newServer(&gatewayCalls)
|
||||
defer gateway.Close()
|
||||
proxy := newServer(&proxyCalls)
|
||||
defer proxy.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), testServerAddress(proxy))
|
||||
|
||||
if err := refreshChannel(context.Background(), "channel-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gatewayCalls.Load() != 1 || proxyCalls.Load() != 1 {
|
||||
t.Fatalf("gateway calls=%d proxy calls=%d", gatewayCalls.Load(), proxyCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameChannelOnlyRebuildsRuntimeInstances(t *testing.T) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
)
|
||||
newServer := func(target string) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
mu.Lock()
|
||||
events = append(events, target+":"+request.URL.Path+":"+request.URL.Query().Get("chanKey"))
|
||||
mu.Unlock()
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success"}`))
|
||||
}))
|
||||
}
|
||||
gateway := newServer("gateway")
|
||||
defer gateway.Close()
|
||||
proxy := newServer("proxy")
|
||||
defer proxy.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), testServerAddress(proxy))
|
||||
|
||||
if err := renameChannel(context.Background(), "channel-old", "channel-new"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"gateway:/internal/channel/refresh:channel-new",
|
||||
"proxy:/internal/channel/refresh:channel-new",
|
||||
"gateway:/internal/channel/remove:channel-old",
|
||||
"proxy:/internal/channel/remove:channel-old",
|
||||
}
|
||||
if strings.Join(events, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("rename events = %q, want %q", events, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelControlErrorIncludesTargetStatusAndMessage(t *testing.T) {
|
||||
gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
_, _ = w.Write([]byte(`{"code":50,"message":"carrier unavailable"}`))
|
||||
}))
|
||||
defer gateway.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), "127.0.0.1:1")
|
||||
|
||||
err := refreshChannelTarget(context.Background(), "upstream.gateway", "channel-a")
|
||||
if err == nil {
|
||||
t.Fatal("expected target error")
|
||||
}
|
||||
message := err.Error()
|
||||
for _, part := range []string{"target=upstream.gateway", "status=502", "carrier unavailable"} {
|
||||
if !strings.Contains(message, part) {
|
||||
t.Fatalf("error %q does not contain %q", message, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleChannelRefreshReturnsError(t *testing.T) {
|
||||
gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"code":50,"message":"refresh rejected"}`))
|
||||
}))
|
||||
defer gateway.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), "127.0.0.1:1")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
consumer := &Consumers{}
|
||||
if err := consumer.handleChannelRefresh(ctx, model.ChannelRefreshMessage{Type: "channel", ChanKey: "channel-a"}); err == nil {
|
||||
t.Fatal("refresh failure was acknowledged as success")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
type TrafficUsageHandler interface {
|
||||
RetryDaily(context.Context, model.TrafficUsageDailyRetryMessage) error
|
||||
RetryMonthly(context.Context, model.TrafficUsageMonthlyRetryMessage) error
|
||||
}
|
||||
|
||||
type WorkOrderHandler interface {
|
||||
AutoCloseResolved(context.Context, model.WorkOrderAutoCloseMessage) error
|
||||
}
|
||||
|
||||
type BalanceRechargeHandler interface {
|
||||
CheckRechargeTimeout(context.Context, string) error
|
||||
}
|
||||
|
||||
type BatchHandler interface {
|
||||
Handle(context.Context, batch.Message) error
|
||||
}
|
||||
|
||||
type PackageCustomerSyncHandler interface {
|
||||
SyncDeptPackages(context.Context, model.PackageCustomerSyncMessage) error
|
||||
}
|
||||
|
||||
type TrafficUsageMonthlyStartHandler interface {
|
||||
Produce(context.Context, model.TrafficUsageMonthlyStartMessage) error
|
||||
}
|
||||
|
||||
type TrafficUsageDailyStartHandler interface {
|
||||
Produce(context.Context, model.TrafficUsageDailyStartMessage) error
|
||||
}
|
||||
|
||||
type TrafficUsageMonthlyHistoryStartHandler interface {
|
||||
Produce(context.Context, model.TrafficUsageMonthlyHistoryStartMessage) error
|
||||
}
|
||||
|
||||
type SimStatusQueueStartHandler interface {
|
||||
Produce(context.Context, model.SimStatusQueueStartMessage) error
|
||||
}
|
||||
|
||||
type SimStatusQueueTaskHandler interface {
|
||||
EnqueueCard(context.Context, model.SimStatusQueueTaskMessage) error
|
||||
}
|
||||
|
||||
type ImeiQueueStartHandler interface {
|
||||
Produce(context.Context, model.ImeiQueueStartMessage) error
|
||||
}
|
||||
|
||||
// DataReviewQueryHandler 执行"手动同步上游结果"：只读补查既有 operationId。
|
||||
type DataReviewQueryHandler interface {
|
||||
QueryReviewBatch(ctx context.Context, batchNo string) (int, error)
|
||||
}
|
||||
|
||||
type Consumers struct {
|
||||
trafficUsage TrafficUsageHandler
|
||||
workOrder WorkOrderHandler
|
||||
balance BalanceRechargeHandler
|
||||
balanceRechargeRetryDelay time.Duration
|
||||
balanceRechargeMaxRetry int
|
||||
batch BatchHandler
|
||||
batchExecuteRetryDelay time.Duration
|
||||
batchExecuteMaxRetries int
|
||||
packageCustomerSync PackageCustomerSyncHandler
|
||||
trafficUsageMonthlyStart TrafficUsageMonthlyStartHandler
|
||||
trafficUsageDailyStart TrafficUsageDailyStartHandler
|
||||
trafficUsageMonthlyHistoryStart TrafficUsageMonthlyHistoryStartHandler
|
||||
simStatusQueueStart SimStatusQueueStartHandler
|
||||
simStatusQueueTask SimStatusQueueTaskHandler
|
||||
imeiQueueStart ImeiQueueStartHandler
|
||||
dataReview DataReviewQueryHandler
|
||||
}
|
||||
|
||||
// WithDataReviewQuery adds the manual "sync upstream result" consumer without
|
||||
// changing any existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithDataReviewQuery(handler DataReviewQueryHandler) *Consumers {
|
||||
c.dataReview = handler
|
||||
return c
|
||||
}
|
||||
|
||||
// WithDailyUsageStart adds the previous-day usage producer without changing any
|
||||
// existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithDailyUsageStart(start TrafficUsageDailyStartHandler) *Consumers {
|
||||
c.trafficUsageDailyStart = start
|
||||
return c
|
||||
}
|
||||
|
||||
// WithMonthlyHistoryStart adds the previous-month usage history producer without
|
||||
// changing any existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithMonthlyHistoryStart(start TrafficUsageMonthlyHistoryStartHandler) *Consumers {
|
||||
c.trafficUsageMonthlyHistoryStart = start
|
||||
return c
|
||||
}
|
||||
|
||||
// WithSimStatusQueue adds the independent status queue handlers without
|
||||
// changing any existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithSimStatusQueue(start SimStatusQueueStartHandler, task SimStatusQueueTaskHandler) *Consumers {
|
||||
c.simStatusQueueStart = start
|
||||
c.simStatusQueueTask = task
|
||||
return c
|
||||
}
|
||||
|
||||
// WithImeiQueue adds the independent IMEI queue handler without changing any
|
||||
// existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithImeiQueue(start ImeiQueueStartHandler) *Consumers {
|
||||
c.imeiQueueStart = start
|
||||
return c
|
||||
}
|
||||
|
||||
func New(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, batchHandlers ...BatchHandler) *Consumers {
|
||||
consumers := &Consumers{
|
||||
trafficUsage: trafficUsage,
|
||||
workOrder: workOrder,
|
||||
balance: balance,
|
||||
}
|
||||
if len(batchHandlers) > 0 {
|
||||
consumers.batch = batchHandlers[0]
|
||||
}
|
||||
return consumers
|
||||
}
|
||||
|
||||
func NewWithPackageCustomerSync(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, packageCustomerSync PackageCustomerSyncHandler, batchHandlers ...BatchHandler) *Consumers {
|
||||
consumers := New(trafficUsage, workOrder, balance, batchHandlers...)
|
||||
consumers.packageCustomerSync = packageCustomerSync
|
||||
return consumers
|
||||
}
|
||||
|
||||
func NewWithPackageCustomerSyncAndMonthlyUsage(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, packageCustomerSync PackageCustomerSyncHandler, monthlyStart TrafficUsageMonthlyStartHandler, batchHandlers ...BatchHandler) *Consumers {
|
||||
consumers := NewWithPackageCustomerSync(trafficUsage, workOrder, balance, packageCustomerSync, batchHandlers...)
|
||||
consumers.trafficUsageMonthlyStart = monthlyStart
|
||||
return consumers
|
||||
}
|
||||
|
||||
func (c *Consumers) Start(ctx context.Context) error {
|
||||
initializers := c.initializers(g.Cfg().MustGet(ctx, "workerDiagnostic.enabled", false).Bool())
|
||||
for _, initialize := range initializers {
|
||||
if err := initialize(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) initializers(workerDiagnosticEnabled bool) []func(context.Context) error {
|
||||
initializers := []func(context.Context) error{
|
||||
c.initTrafficUsageDailyRetry,
|
||||
c.initTrafficUsageMonthlyRetry,
|
||||
c.initWorkOrderAutoClose,
|
||||
c.initBalanceRechargeTimeout,
|
||||
c.initBatchExecute,
|
||||
c.initChannelRefresh,
|
||||
c.initDataReviewQuery,
|
||||
}
|
||||
if c.packageCustomerSync != nil {
|
||||
initializers = append(initializers, c.initPackageCustomerSync)
|
||||
}
|
||||
if c.trafficUsageMonthlyStart != nil {
|
||||
initializers = append(initializers, c.initTrafficUsageMonthlyStart)
|
||||
}
|
||||
if c.trafficUsageDailyStart != nil {
|
||||
initializers = append(initializers, c.initTrafficUsageDailyStart)
|
||||
}
|
||||
if c.trafficUsageMonthlyHistoryStart != nil {
|
||||
initializers = append(initializers, c.initTrafficUsageMonthlyHistoryStart)
|
||||
}
|
||||
if c.simStatusQueueStart != nil && c.simStatusQueueTask != nil {
|
||||
initializers = append(initializers, c.initSimStatusQueueStart, c.initSimStatusQueueTask)
|
||||
}
|
||||
if c.imeiQueueStart != nil {
|
||||
initializers = append(initializers, c.initImeiQueueStart)
|
||||
}
|
||||
if workerDiagnosticEnabled {
|
||||
initializers = append(initializers, c.initWorkerDiagnostic)
|
||||
}
|
||||
return initializers
|
||||
}
|
||||
|
||||
func (c *Consumers) Stop(ctx context.Context) error {
|
||||
return mq.Close(ctx)
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
dailyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
|
||||
monthlyhistoryproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
|
||||
packagecustomerservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/packagecustomer"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
g.Log().SetWriter(io.Discard)
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
type trafficUsageHandlerStub struct {
|
||||
dailyCalls int
|
||||
monthlyCalls int
|
||||
dailyMessage model.TrafficUsageDailyRetryMessage
|
||||
monthMessage model.TrafficUsageMonthlyRetryMessage
|
||||
dailyErr error
|
||||
monthlyErr error
|
||||
}
|
||||
|
||||
func (s *trafficUsageHandlerStub) RetryDaily(_ context.Context, message model.TrafficUsageDailyRetryMessage) error {
|
||||
s.dailyCalls++
|
||||
s.dailyMessage = message
|
||||
return s.dailyErr
|
||||
}
|
||||
|
||||
func (s *trafficUsageHandlerStub) RetryMonthly(_ context.Context, message model.TrafficUsageMonthlyRetryMessage) error {
|
||||
s.monthlyCalls++
|
||||
s.monthMessage = message
|
||||
return s.monthlyErr
|
||||
}
|
||||
|
||||
type workOrderHandlerStub struct {
|
||||
calls int
|
||||
message model.WorkOrderAutoCloseMessage
|
||||
err error
|
||||
}
|
||||
|
||||
type balanceRechargeHandlerStub struct {
|
||||
calls int
|
||||
outTradeNo string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *balanceRechargeHandlerStub) CheckRechargeTimeout(_ context.Context, outTradeNo string) error {
|
||||
s.calls++
|
||||
s.outTradeNo = outTradeNo
|
||||
return s.err
|
||||
}
|
||||
|
||||
type batchHandlerStub struct {
|
||||
calls int
|
||||
message batch.Message
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *batchHandlerStub) Handle(_ context.Context, message batch.Message) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
type packageCustomerSyncHandlerStub struct {
|
||||
calls int
|
||||
message model.PackageCustomerSyncMessage
|
||||
err error
|
||||
}
|
||||
|
||||
type dailyStartHandlerStub struct {
|
||||
calls int
|
||||
message model.TrafficUsageDailyStartMessage
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *dailyStartHandlerStub) Produce(_ context.Context, message model.TrafficUsageDailyStartMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
type monthlyHistoryStartHandlerStub struct {
|
||||
calls int
|
||||
message model.TrafficUsageMonthlyHistoryStartMessage
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *monthlyHistoryStartHandlerStub) Produce(_ context.Context, message model.TrafficUsageMonthlyHistoryStartMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
func (s *packageCustomerSyncHandlerStub) SyncDeptPackages(_ context.Context, message model.PackageCustomerSyncMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
func (s *workOrderHandlerStub) AutoCloseResolved(_ context.Context, message model.WorkOrderAutoCloseMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageDailyRetry(t *testing.T) {
|
||||
stubTrafficPublish(t)
|
||||
t.Run("invalid messages are dead lettered", func(t *testing.T) {
|
||||
messages := []model.TrafficUsageDailyRetryMessage{
|
||||
{},
|
||||
{SimKey: &model.SimKey{}, Card: model.TrafficUsageSimCard{Iccid: "card"}},
|
||||
{SimKey: &model.SimKey{ICCID: "sim"}},
|
||||
}
|
||||
for _, message := range messages {
|
||||
handler := &trafficUsageHandlerStub{}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageDailyRetry(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.dailyCalls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.dailyCalls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expectedErr := errors.New("daily failed")
|
||||
expected := model.TrafficUsageDailyRetryMessage{
|
||||
SimKey: &model.SimKey{ICCID: "sim"},
|
||||
Card: model.TrafficUsageSimCard{Iccid: "sim", PackageCode: "package"},
|
||||
QueryDate: "20260714",
|
||||
}
|
||||
handler := &trafficUsageHandlerStub{dailyErr: expectedErr}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageDailyRetry(context.Background(), expected); err != nil {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.dailyCalls != 1 || !reflect.DeepEqual(handler.dailyMessage, expected) {
|
||||
t.Fatalf("daily message was not delegated exactly once: %+v", handler.dailyMessage)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageMonthlyRetry(t *testing.T) {
|
||||
stubTrafficPublish(t)
|
||||
t.Run("invalid messages are dead lettered", func(t *testing.T) {
|
||||
messages := []model.TrafficUsageMonthlyRetryMessage{
|
||||
{},
|
||||
{SimKey: &model.SimKey{}, Card: model.TrafficUsageSimCard{Iccid: "card"}},
|
||||
{SimKey: &model.SimKey{ICCID: "sim"}},
|
||||
}
|
||||
for _, message := range messages {
|
||||
handler := &trafficUsageHandlerStub{}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageMonthlyRetry(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.monthlyCalls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.monthlyCalls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expectedErr := errors.New("monthly failed")
|
||||
expected := model.TrafficUsageMonthlyRetryMessage{
|
||||
SimKey: &model.SimKey{ICCID: "sim"},
|
||||
Card: model.TrafficUsageSimCard{Iccid: "sim", PackageCode: "package"},
|
||||
QueryDate: "202607",
|
||||
}
|
||||
handler := &trafficUsageHandlerStub{monthlyErr: expectedErr}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageMonthlyRetry(context.Background(), expected); err != nil {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.monthlyCalls != 1 || !reflect.DeepEqual(handler.monthMessage, expected) {
|
||||
t.Fatalf("monthly message was not delegated exactly once: %+v", handler.monthMessage)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func stubTrafficPublish(t *testing.T) {
|
||||
t.Helper()
|
||||
original, delayed := trafficUsagePublish, trafficUsagePublishDelayed
|
||||
t.Cleanup(func() { trafficUsagePublish, trafficUsagePublishDelayed = original, delayed })
|
||||
trafficUsagePublish = func(context.Context, string, []byte, amqp.Table) error { return nil }
|
||||
trafficUsagePublishDelayed = func(context.Context, string, string, time.Duration, []byte, amqp.Table) error { return nil }
|
||||
}
|
||||
|
||||
func TestHandleWorkOrderAutoClose(t *testing.T) {
|
||||
handler := &workOrderHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, handler, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleWorkOrderAutoClose(context.Background(), model.WorkOrderAutoCloseMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.calls)
|
||||
}
|
||||
|
||||
expectedErr := errors.New("work order failed")
|
||||
expected := model.WorkOrderAutoCloseMessage{WoNo: "WO-1", ResolveTimeTs: 1}
|
||||
handler.err = expectedErr
|
||||
if err := consumers.handleWorkOrderAutoClose(context.Background(), expected); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("work order message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeout(t *testing.T) {
|
||||
handler := &balanceRechargeHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, handler)
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.calls)
|
||||
}
|
||||
|
||||
message := model.BalanceRechargeTimeoutMessage{OutTradeNo: "BAL-1"}
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), message); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || handler.outTradeNo != message.OutTradeNo {
|
||||
t.Fatalf("balance recharge message was not delegated exactly once: %+v", handler)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBatchExecute(t *testing.T) {
|
||||
expectedErr := errors.New("batch failed")
|
||||
expected := batch.Message{BatchNo: "BATCH-1", BatchType: batch.Type103}
|
||||
handler := &batchHandlerStub{err: expectedErr}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
|
||||
if err := consumers.handleBatchExecute(context.Background(), expected); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("batch message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePackageCustomerSync(t *testing.T) {
|
||||
t.Run("invalid message is dropped", func(t *testing.T) {
|
||||
handler := &packageCustomerSyncHandlerStub{}
|
||||
consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
if err := consumers.handlePackageCustomerSync(context.Background(), model.PackageCustomerSyncMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing dept is dropped", func(t *testing.T) {
|
||||
expected := model.PackageCustomerSyncMessage{TenantId: "000000", DeptId: 164, Trigger: "package_lifecycle", SourceBatchNo: "898604017"}
|
||||
handler := &packageCustomerSyncHandlerStub{err: packagecustomerservice.ErrDeptNotFound}
|
||||
consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
if err := consumers.handlePackageCustomerSync(context.Background(), expected); err != nil {
|
||||
t.Fatalf("missing dept returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("package customer sync message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("other errors are retried", func(t *testing.T) {
|
||||
expectedErr := errors.New("database failed")
|
||||
expected := model.PackageCustomerSyncMessage{TenantId: "000000", DeptId: 117}
|
||||
handler := &packageCustomerSyncHandlerStub{err: expectedErr}
|
||||
consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
if err := consumers.handlePackageCustomerSync(context.Background(), expected); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestHandleWorkerDiagnostic(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
|
||||
if err := consumers.handleWorkerDiagnostic(context.Background(), model.WorkerDiagnosticMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
|
||||
message := model.WorkerDiagnosticMessage{
|
||||
TraceID: "worker-test-1",
|
||||
SentAt: 1784030000000,
|
||||
Message: "worker ping",
|
||||
}
|
||||
if err := consumers.handleWorkerDiagnostic(context.Background(), message); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDiagnosticMessageJSONContract(t *testing.T) {
|
||||
message := model.WorkerDiagnosticMessage{
|
||||
TraceID: "worker-test-1",
|
||||
SentAt: 1784030000000,
|
||||
Message: "worker ping",
|
||||
}
|
||||
body, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal diagnostic message: %v", err)
|
||||
}
|
||||
const expected = `{"traceId":"worker-test-1","sentAt":1784030000000,"message":"worker ping"}`
|
||||
if string(body) != expected {
|
||||
t.Fatalf("unexpected diagnostic JSON: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelRefreshMessageJSONContract(t *testing.T) {
|
||||
message := model.ChannelRefreshMessage{
|
||||
Type: "channel_rename",
|
||||
OldChanKey: "old_channel",
|
||||
NewChanKey: "new_channel",
|
||||
}
|
||||
body, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal channel rename message: %v", err)
|
||||
}
|
||||
const expected = `{"type":"channel_rename","oldChanKey":"old_channel","newChanKey":"new_channel"}`
|
||||
if string(body) != expected {
|
||||
t.Fatalf("unexpected channel rename JSON: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleChannelRefreshDropsInvalidRename(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
for _, message := range []model.ChannelRefreshMessage{
|
||||
{Type: "channel_rename"},
|
||||
{Type: "channel_rename", OldChanKey: "old_channel"},
|
||||
{Type: "channel_rename", NewChanKey: "new_channel"},
|
||||
{Type: "channel_rename", OldChanKey: "same", NewChanKey: "same"},
|
||||
} {
|
||||
if err := consumers.handleChannelRefresh(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid channel rename returned error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleChannelRefreshRenameFailureReturnsError(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
message := model.ChannelRefreshMessage{
|
||||
Type: "channel_rename",
|
||||
OldChanKey: "old_channel",
|
||||
NewChanKey: "new_channel",
|
||||
}
|
||||
// ctx 已取消时 renameChannelWithRetry 会在首次重试等待处返回 ctx.Err()，
|
||||
// 消费者必须把错误向上抛出，让框架 Nack 重新入队而不是 Ack 丢弃。
|
||||
if err := consumers.handleChannelRefresh(ctx, message); err == nil {
|
||||
t.Fatalf("expected failed channel rename to return error for Nack+requeue")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageDailyStart(t *testing.T) {
|
||||
t.Run("invalid message is dropped", func(t *testing.T) {
|
||||
for _, message := range []model.TrafficUsageDailyStartMessage{
|
||||
{},
|
||||
{RunID: "run-1"},
|
||||
{QueryDate: "20261009"},
|
||||
} {
|
||||
handler := &dailyStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called producer %d times", handler.calls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expected := model.TrafficUsageDailyStartMessage{
|
||||
RunID: "daily-usage-20261009-abc",
|
||||
Operation: model.OperationPreviousDayUsageFullSync,
|
||||
QueryDate: "20261009",
|
||||
TriggeredAt: "2026-10-10T06:00:00+08:00",
|
||||
Scope: &model.SyncScope{Channels: []string{"217"}},
|
||||
}
|
||||
handler := &dailyStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), expected); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("daily start message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an already running producer is acked", func(t *testing.T) {
|
||||
handler := &dailyStartHandlerStub{err: dailyusageproducer.ErrAlreadyRunning}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); err != nil {
|
||||
t.Fatalf("ErrAlreadyRunning must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a finished run is acked instead of being resurrected", func(t *testing.T) {
|
||||
handler := &dailyStartHandlerStub{err: dailyusageproducer.ErrRunFinished}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); err != nil {
|
||||
t.Fatalf("ErrRunFinished must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("other errors are surfaced for requeue", func(t *testing.T) {
|
||||
expectedErr := errors.New("redis unavailable")
|
||||
handler := &dailyStartHandlerStub{err: expectedErr}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected the producer error to propagate, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestDailyUsageStartConsumerIsOptIn pins the assembly contract: without an
|
||||
// explicit WithDailyUsageStart the initializer list must be unchanged, so a
|
||||
// disabled daily chain cannot alter existing deployments.
|
||||
func TestDailyUsageStartConsumerIsOptIn(t *testing.T) {
|
||||
base := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
withDaily := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(&dailyStartHandlerStub{})
|
||||
if got, want := len(withDaily.initializers(false)), len(base.initializers(false))+1; got != want {
|
||||
t.Fatalf("initializers = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageMonthlyHistoryStart(t *testing.T) {
|
||||
t.Run("invalid message is dropped", func(t *testing.T) {
|
||||
for _, message := range []model.TrafficUsageMonthlyHistoryStartMessage{
|
||||
{},
|
||||
{RunID: "run-1"},
|
||||
{QueryMonth: "202609"},
|
||||
} {
|
||||
handler := &monthlyHistoryStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called producer %d times", handler.calls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expected := model.TrafficUsageMonthlyHistoryStartMessage{
|
||||
RunID: "monthly-history-202609-abc",
|
||||
Operation: model.OperationPreviousMonthUsageFullSync,
|
||||
QueryMonth: "202609",
|
||||
TriggeredAt: "2026-10-01T06:00:00+08:00",
|
||||
Scope: &model.SyncScope{Channels: []string{"217"}},
|
||||
}
|
||||
handler := &monthlyHistoryStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), expected); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("monthly history start message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an already running producer is acked", func(t *testing.T) {
|
||||
handler := &monthlyHistoryStartHandlerStub{err: monthlyhistoryproducer.ErrAlreadyRunning}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); err != nil {
|
||||
t.Fatalf("ErrAlreadyRunning must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a finished run is acked instead of being resurrected", func(t *testing.T) {
|
||||
handler := &monthlyHistoryStartHandlerStub{err: monthlyhistoryproducer.ErrRunFinished}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); err != nil {
|
||||
t.Fatalf("ErrRunFinished must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("other errors are surfaced for requeue", func(t *testing.T) {
|
||||
expectedErr := errors.New("redis unavailable")
|
||||
handler := &monthlyHistoryStartHandlerStub{err: expectedErr}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected the producer error to propagate, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMonthlyHistoryStartConsumerIsOptIn mirrors the daily chain: a disabled
|
||||
// monthly-history chain must not add an initializer.
|
||||
func TestMonthlyHistoryStartConsumerIsOptIn(t *testing.T) {
|
||||
base := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
withHistory := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(&monthlyHistoryStartHandlerStub{})
|
||||
if got, want := len(withHistory.initializers(false)), len(base.initializers(false))+1; got != want {
|
||||
t.Fatalf("initializers = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsumerInitializersRespectWorkerDiagnosticSwitch(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
// 手动同步上游结果（initDataReviewQuery）随业务消费者常驻，无处理器时自跳过。
|
||||
if got := len(consumers.initializers(false)); got != 7 {
|
||||
t.Fatalf("diagnostic disabled: expected 7 business consumers, got %d", got)
|
||||
}
|
||||
if got := len(consumers.initializers(true)); got != 8 {
|
||||
t.Fatalf("diagnostic enabled: expected 8 consumers, got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initDataReviewQuery(ctx context.Context) error {
|
||||
if c.dataReview == nil {
|
||||
return nil
|
||||
}
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "data review query consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.DataReviewQueryMessage](
|
||||
ctx,
|
||||
consts.QueueDataReviewQuery,
|
||||
"iotlcm-data-review-query",
|
||||
c.handleDataReviewQuery,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "data review query consumer started, queue:%s", consts.QueueDataReviewQuery)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleDataReviewQuery(ctx context.Context, message model.DataReviewQueryMessage) error {
|
||||
if c.dataReview == nil || message.BatchNo == "" {
|
||||
g.Log().Errorf(ctx, "data review query message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
queried, err := c.dataReview.QueryReviewBatch(ctx, message.BatchNo)
|
||||
if err != nil {
|
||||
// 手动同步失败只记日志，不重投：重投也不会改变上游结果，且可能放大查询压力。
|
||||
g.Log().Errorf(ctx, "手动同步APN上游结果失败 batchNo=%s 已同步=%d error=%v", message.BatchNo, queried, err)
|
||||
return nil
|
||||
}
|
||||
g.Log().Infof(ctx, "手动同步APN上游结果完成 batchNo=%s 已同步=%d", message.BatchNo, queried)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initWorkerDiagnostic(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "worker diagnostic consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.WorkerDiagnosticMessage](
|
||||
ctx,
|
||||
consts.QueueWorkerDiagnostic,
|
||||
"iotlcm-worker-diagnostic",
|
||||
c.handleWorkerDiagnostic,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "worker diagnostic consumer started, queue:%s", consts.QueueWorkerDiagnostic)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleWorkerDiagnostic(ctx context.Context, message model.WorkerDiagnosticMessage) error {
|
||||
if message.TraceID == "" {
|
||||
g.Log().Errorf(ctx, "worker diagnostic message dropped, invalid traceId, message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
g.Log().Infof(
|
||||
ctx,
|
||||
"worker diagnostic message consumed, traceId:%s, sentAt:%d, message:%s",
|
||||
message.TraceID,
|
||||
message.SentAt,
|
||||
message.Message,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
imeiproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/imei"
|
||||
)
|
||||
|
||||
func (c *Consumers) initImeiQueueStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "imei queue start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.ImeiQueueStartMessage](ctx, consts.QueueImeiQueueStart, "iotlcm-imei-queue-start", c.handleImeiQueueStart); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "imei queue start consumer started, queue:%s", consts.QueueImeiQueueStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleImeiQueueStart(ctx context.Context, message model.ImeiQueueStartMessage) error {
|
||||
if message.RunID == "" {
|
||||
g.Log().Errorf(ctx, "imei queue start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.imeiQueueStart.Produce(ctx, message)
|
||||
if errors.Is(err, imeiproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "imei queue start skipped because a run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, imeiproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "imei queue start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
packagecustomerservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/packagecustomer"
|
||||
)
|
||||
|
||||
func (c *Consumers) initPackageCustomerSync(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "package customer sync consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if c.packageCustomerSync == nil {
|
||||
return gerror.New("package customer sync handler is not configured")
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.PackageCustomerSyncMessage](
|
||||
ctx,
|
||||
consts.QueuePackageCustomerSync,
|
||||
"iotlcm-package-customer-sync",
|
||||
c.handlePackageCustomerSync,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "package customer sync consumer started, queue:%s", consts.QueuePackageCustomerSync)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handlePackageCustomerSync(ctx context.Context, message model.PackageCustomerSyncMessage) error {
|
||||
if c.packageCustomerSync == nil {
|
||||
return gerror.New("package customer sync handler is not configured")
|
||||
}
|
||||
if message.TenantId == "" || message.DeptId <= 0 {
|
||||
g.Log().Errorf(ctx, "package customer sync message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
if err := c.packageCustomerSync.SyncDeptPackages(ctx, message); err != nil {
|
||||
if errors.Is(err, packagecustomerservice.ErrDeptNotFound) {
|
||||
g.Log().Warningf(ctx, "package customer sync message dropped, dept not found, tenantId:%s, deptId:%d, trigger:%s, sourceBatchNo:%s",
|
||||
message.TenantId, message.DeptId, message.Trigger, message.SourceBatchNo)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
simstatusproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/simstatus"
|
||||
)
|
||||
|
||||
func (c *Consumers) initSimStatusQueueStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "sim status queue start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.SimStatusQueueStartMessage](ctx, consts.QueueSimStatusQueueStart, "iotlcm-sim-status-queue-start", c.handleSimStatusQueueStart); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "sim status queue start consumer started, queue:%s", consts.QueueSimStatusQueueStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleSimStatusQueueStart(ctx context.Context, message model.SimStatusQueueStartMessage) error {
|
||||
if message.RunID == "" {
|
||||
g.Log().Errorf(ctx, "sim status queue start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.simStatusQueueStart.Produce(ctx, message)
|
||||
if errors.Is(err, simstatusproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "sim status queue start skipped because a run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, simstatusproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "sim status queue start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initSimStatusQueueTask(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "sim status queue task consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.SimStatusQueueTaskMessage](ctx, consts.QueueSimStatusQueueTask, "iotlcm-sim-status-queue-task", c.handleSimStatusQueueTask); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "sim status queue task consumer started, queue:%s", consts.QueueSimStatusQueueTask)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleSimStatusQueueTask(ctx context.Context, message model.SimStatusQueueTaskMessage) error {
|
||||
if message.CardID <= 0 || message.ICCID == "" || message.ChanKey == "" || message.TempKey == "" {
|
||||
g.Log().Errorf(ctx, "sim status queue task dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
return c.simStatusQueueTask.EnqueueCard(ctx, message)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageDailyRetry(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage daily retry consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageDailyRetryMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageDailyRetry,
|
||||
"iotlcm-traffic-usage-daily",
|
||||
c.handleTrafficUsageDailyRetry,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage daily retry consumer started, queue:%s", consts.QueueTrafficUsageDailyRetry)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageDailyRetry(ctx context.Context, message model.TrafficUsageDailyRetryMessage) error {
|
||||
err := validateTrafficUsageIdentity(message.SimKey, message.Card)
|
||||
if err == nil {
|
||||
err = c.trafficUsage.RetryDaily(ctx, message)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
message.Attempt++
|
||||
message.LastError = err.Error()
|
||||
permanent := trafficUsageFailureSnapshot(err, &message.UsageMb)
|
||||
return publishTrafficUsageFailure(ctx, consts.QueueTrafficUsageDailyRetry, message, message.Attempt, permanent)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
dailyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageDailyStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage daily start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageDailyStartMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageDailyStart,
|
||||
"iotlcm-traffic-usage-daily-start",
|
||||
c.handleTrafficUsageDailyStart,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage daily start consumer started, queue:%s", consts.QueueTrafficUsageDailyStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageDailyStart(ctx context.Context, message model.TrafficUsageDailyStartMessage) error {
|
||||
if message.RunID == "" || message.QueryDate == "" {
|
||||
g.Log().Errorf(ctx, "traffic usage daily start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.trafficUsageDailyStart.Produce(ctx, message)
|
||||
if errors.Is(err, dailyusageproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "traffic usage daily start skipped because previous run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, dailyusageproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "traffic usage daily start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageMonthlyRetry(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage monthly retry consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageMonthlyRetryMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageMonthlyRetry,
|
||||
"iotlcm-traffic-usage-monthly",
|
||||
c.handleTrafficUsageMonthlyRetry,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage monthly retry consumer started, queue:%s", consts.QueueTrafficUsageMonthlyRetry)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageMonthlyRetry(ctx context.Context, message model.TrafficUsageMonthlyRetryMessage) error {
|
||||
err := validateTrafficUsageIdentity(message.SimKey, message.Card)
|
||||
if err == nil {
|
||||
err = c.trafficUsage.RetryMonthly(ctx, message)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
message.Attempt++
|
||||
message.LastError = err.Error()
|
||||
permanent := trafficUsageFailureSnapshot(err, &message.UsageMb)
|
||||
return publishTrafficUsageFailure(ctx, consts.QueueTrafficUsageMonthlyRetry, message, message.Attempt, permanent)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
monthlyhistoryproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageMonthlyHistoryStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage monthly history start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageMonthlyHistoryStartMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageMonthlyHistoryStart,
|
||||
"iotlcm-traffic-usage-monthly-history-start",
|
||||
c.handleTrafficUsageMonthlyHistoryStart,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage monthly history start consumer started, queue:%s", consts.QueueTrafficUsageMonthlyHistoryStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageMonthlyHistoryStart(ctx context.Context, message model.TrafficUsageMonthlyHistoryStartMessage) error {
|
||||
if message.RunID == "" || message.QueryMonth == "" {
|
||||
g.Log().Errorf(ctx, "traffic usage monthly history start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.trafficUsageMonthlyHistoryStart.Produce(ctx, message)
|
||||
if errors.Is(err, monthlyhistoryproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly history start skipped because previous run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, monthlyhistoryproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly history start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
monthlyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusage"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageMonthlyStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage monthly start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageMonthlyStartMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageMonthlyStart,
|
||||
"iotlcm-traffic-usage-monthly-start",
|
||||
c.handleTrafficUsageMonthlyStart,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage monthly start consumer started, queue:%s", consts.QueueTrafficUsageMonthlyStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageMonthlyStart(ctx context.Context, message model.TrafficUsageMonthlyStartMessage) error {
|
||||
if message.RunID == "" || message.QueryMonth == "" {
|
||||
g.Log().Errorf(ctx, "traffic usage monthly start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.trafficUsageMonthlyStart.Produce(ctx, message)
|
||||
if errors.Is(err, monthlyusageproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly start skipped because previous run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, monthlyusageproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficusage"
|
||||
)
|
||||
|
||||
var trafficUsagePublish = mq.PublishBytesWithHeaders
|
||||
var trafficUsagePublishDelayed = mq.PublishDelayedBytesWithHeaders
|
||||
|
||||
func validateTrafficUsageIdentity(simKey *model.SimKey, card model.TrafficUsageSimCard) error {
|
||||
if simKey == nil || card.Iccid == "" || simKey.ICCID != card.Iccid {
|
||||
return &trafficusage.Failure{Cause: gerror.New("invalid traffic usage card identity"), Permanent: true}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func trafficUsageFailureSnapshot(err error, usage **float64) bool {
|
||||
var failure *trafficusage.Failure
|
||||
if errors.As(err, &failure) {
|
||||
if failure.UsageMb != nil {
|
||||
*usage = failure.UsageMb
|
||||
}
|
||||
return failure.Permanent
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func publishTrafficUsageFailure(ctx context.Context, queue string, message any, attempt int, permanent bool) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
body, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if permanent || attempt >= 10 {
|
||||
g.Log().Errorf(ctx, "traffic usage retry exhausted or invalid, queue:%s, attempt:%d, payload:%s", queue, attempt, body)
|
||||
err = trafficUsagePublish(ctx, queue+".dlq", body, nil)
|
||||
return waitTrafficUsagePublishFailure(ctx, err)
|
||||
}
|
||||
delay := 30 * time.Second
|
||||
if attempt >= 3 {
|
||||
delay = 2 * time.Minute
|
||||
}
|
||||
if attempt >= 6 {
|
||||
delay = 5 * time.Minute
|
||||
}
|
||||
err = trafficUsagePublishDelayed(ctx, queue+".delay."+delay.String(), queue, delay, body, nil)
|
||||
return waitTrafficUsagePublishFailure(ctx, err)
|
||||
}
|
||||
|
||||
func waitTrafficUsagePublishFailure(ctx context.Context, err error) error {
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficusage"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTrafficUsageRetryRetainsPayload(t *testing.T) {
|
||||
stubTrafficPublish(t)
|
||||
usage := 12.5
|
||||
handler := &trafficUsageHandlerStub{dailyErr: &trafficusage.Failure{Cause: errors.New("database down"), UsageMb: &usage}}
|
||||
consumer := New(handler, nil, nil)
|
||||
message := model.TrafficUsageDailyRetryMessage{SimKey: &model.SimKey{ICCID: "card"}, Card: model.TrafficUsageSimCard{Iccid: "card"}, QueryDate: "20260831"}
|
||||
calls := 0
|
||||
trafficUsagePublishDelayed = func(_ context.Context, queue, target string, delay time.Duration, body []byte, _ amqp.Table) error {
|
||||
calls++
|
||||
var retry model.TrafficUsageDailyRetryMessage
|
||||
if err := json.Unmarshal(body, &retry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retry.Attempt != 1 || retry.UsageMb == nil || *retry.UsageMb != usage || retry.QueryDate != message.QueryDate || retry.LastError == "" || delay != 30*time.Second || target != consts.QueueTrafficUsageDailyRetry || queue == target {
|
||||
t.Fatalf("invalid retry: %+v", retry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); err != nil || calls != 1 {
|
||||
t.Fatalf("calls=%d err=%v", calls, err)
|
||||
}
|
||||
message.Attempt = 9
|
||||
dlqCalls := 0
|
||||
trafficUsagePublish = func(_ context.Context, queue string, body []byte, _ amqp.Table) error {
|
||||
dlqCalls++
|
||||
if queue != consts.QueueTrafficUsageDailyRetry+".dlq" {
|
||||
t.Fatal(queue)
|
||||
}
|
||||
var retry model.TrafficUsageDailyRetryMessage
|
||||
if err := json.Unmarshal(body, &retry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retry.Attempt != 10 || retry.UsageMb == nil {
|
||||
t.Fatal("DLQ lost snapshot")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); err != nil || dlqCalls != 1 || calls != 1 {
|
||||
t.Fatalf("DLQ not published: %v", err)
|
||||
}
|
||||
expected := errors.New("broker unavailable")
|
||||
trafficUsagePublish = func(context.Context, string, []byte, amqp.Table) error { return expected }
|
||||
if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); !errors.Is(err, expected) {
|
||||
t.Fatal("failed publication must not ack original")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initWorkOrderAutoClose(ctx context.Context) error {
|
||||
if scanner, ok := c.workOrder.(interface{ CloseOverdue(context.Context) error }); ok {
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if err := scanner.CloseOverdue(ctx); err != nil {
|
||||
g.Log().Errorf(ctx, "工单到期补偿失败: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "work order auto close consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.WorkOrderAutoCloseMessage](
|
||||
ctx,
|
||||
consts.QueueWorkOrderAutoClose,
|
||||
"iotlcm-work-order-auto-close",
|
||||
c.handleWorkOrderAutoClose,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "work order auto close consumer started, queue:%s", consts.QueueWorkOrderAutoClose)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleWorkOrderAutoClose(ctx context.Context, message model.WorkOrderAutoCloseMessage) error {
|
||||
if message.WoNo == "" {
|
||||
g.Log().Errorf(ctx, "work order auto close message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
return c.workOrder.AutoCloseResolved(ctx, message)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,706 @@
|
||||
package zset
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
type redisCall struct {
|
||||
command string
|
||||
args []any
|
||||
}
|
||||
|
||||
type fakeRedis struct {
|
||||
mu sync.Mutex
|
||||
calls []redisCall
|
||||
responses []*gvar.Var
|
||||
}
|
||||
|
||||
func (f *fakeRedis) Do(_ context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, redisCall{command: command, args: args})
|
||||
if len(f.responses) == 0 {
|
||||
return gvar.New(int64(1)), nil
|
||||
}
|
||||
response := f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (f *fakeRedis) Eval(_ context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
allArgs := []any{script, numKeys}
|
||||
for _, key := range keys {
|
||||
allArgs = append(allArgs, key)
|
||||
}
|
||||
allArgs = append(allArgs, args...)
|
||||
return f.Do(context.Background(), "EVAL", allArgs...)
|
||||
}
|
||||
|
||||
type testHandler struct {
|
||||
query func(context.Context, queue.Task) Result
|
||||
apply func(context.Context, []Result) error
|
||||
}
|
||||
|
||||
func (h *testHandler) Query(ctx context.Context, task queue.Task) Result {
|
||||
if h.query != nil {
|
||||
return h.query(ctx, task)
|
||||
}
|
||||
return Result{Task: task, Success: true, UsageMB: float64(task.CardID)}
|
||||
}
|
||||
|
||||
func (h *testHandler) Apply(ctx context.Context, results []Result) error {
|
||||
if h.apply != nil {
|
||||
return h.apply(ctx, results)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func taskMember(task queue.Task) string {
|
||||
value, _ := json.Marshal(task)
|
||||
return string(value)
|
||||
}
|
||||
|
||||
func claimedTask(task queue.Task) queue.ClaimedTask {
|
||||
return queue.ClaimedTask{Task: task, Member: taskMember(task), LeaseToken: "token-" + task.TaskID}
|
||||
}
|
||||
|
||||
func newTestConsumer(t *testing.T, fake *fakeRedis, handler Handler, config Config) *Consumer {
|
||||
t.Helper()
|
||||
store, err := queue.New(fake, "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer, err := New(store, handler, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return consumer
|
||||
}
|
||||
|
||||
func TestRoundSummaryAccumulatesAndFlushes(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New([]any{"runId", "run-1"})}}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{RoundRetryEnabled: true})
|
||||
started := time.Date(2026, 9, 22, 3, 0, 0, 0, time.UTC)
|
||||
consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{RunID: "run-1", TaskID: "a"}),
|
||||
claimedTask(queue.Task{RunID: "run-1", TaskID: "b"}),
|
||||
}, []queue.TaskOutcome{
|
||||
{Success: true},
|
||||
{Retry: true},
|
||||
}, started)
|
||||
consumer.recordRoundBatch("run-1", 1, []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{RunID: "run-1", TaskID: "c"}),
|
||||
}, []queue.TaskOutcome{{Success: false}}, started.Add(2*time.Second))
|
||||
|
||||
key := "run-1\x001"
|
||||
consumer.summaryMu.Lock()
|
||||
summary := *consumer.roundSummaries[key]
|
||||
consumer.summaryMu.Unlock()
|
||||
if summary.inputCount != 3 || summary.successCount != 1 || summary.failedCount != 2 || summary.deferredCount != 1 || summary.terminalFailedCount != 1 {
|
||||
t.Fatalf("unexpected summary: %+v", summary)
|
||||
}
|
||||
if !summary.startedAt.Equal(started) {
|
||||
t.Fatalf("startedAt = %v, want %v", summary.startedAt, started)
|
||||
}
|
||||
|
||||
consumer.flushRoundSummary(context.Background(), "run-1", 1)
|
||||
consumer.summaryMu.Lock()
|
||||
_, exists := consumer.roundSummaries[key]
|
||||
consumer.summaryMu.Unlock()
|
||||
if exists {
|
||||
t.Fatal("round summary was not removed after flush")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleClaimsAppliesSuccessOnceAndAcknowledgesBatch(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
|
||||
applyCalls := 0
|
||||
var applied []Result
|
||||
handler := &testHandler{apply: func(_ context.Context, results []Result) error {
|
||||
applyCalls++
|
||||
applied = append(applied, results...)
|
||||
return nil
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if applyCalls != 1 || len(applied) != 2 {
|
||||
t.Fatalf("apply calls = %d, results = %d", applyCalls, len(applied))
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v, want one batch ack", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseFailureLeavesEntireBatchLeased(t *testing.T) {
|
||||
fake := &fakeRedis{}
|
||||
handler := &testHandler{
|
||||
query: func(_ context.Context, task queue.Task) Result {
|
||||
if task.CardID == 2 {
|
||||
return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
|
||||
}
|
||||
return Result{Task: task, Success: true, UsageMB: 1.5}
|
||||
},
|
||||
apply: func(context.Context, []Result) error { return errors.New("database unavailable") },
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{ClaimBatch: 2, MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "failed", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
|
||||
}
|
||||
err := consumer.handleClaims(context.Background(), claimed)
|
||||
if err == nil || err.Error() != "apply monthly usage result batch: database unavailable" {
|
||||
t.Fatalf("handleClaims() error = %v", err)
|
||||
}
|
||||
if len(fake.calls) != 0 {
|
||||
t.Fatalf("Redis calls = %+v, want no ACK when database update fails", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBusinessFailuresAreTerminalWithoutRetry(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
|
||||
failureCount := 0
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
return Result{Task: task, ErrorClass: "business", Code: 13000, Message: "busy"}
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{
|
||||
MobileConcurrency: 2,
|
||||
TelecomConcurrency: 1,
|
||||
OnFailure: func(context.Context, Result) {
|
||||
failureCount++
|
||||
},
|
||||
})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failureCount != 2 {
|
||||
t.Fatalf("failure count = %d, want 2", failureCount)
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v; business failures must only be acknowledged", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryableFailuresRemainLeasedWhileSuccessesAreAcknowledged(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(1))}}
|
||||
var applied []Result
|
||||
handler := &testHandler{
|
||||
query: func(_ context.Context, task queue.Task) Result {
|
||||
if task.CardID == 1 {
|
||||
return Result{Task: task, ErrorClass: "network", Retryable: true, Message: "temporary failure"}
|
||||
}
|
||||
return Result{Task: task, Success: true, UsageMB: 2}
|
||||
},
|
||||
apply: func(_ context.Context, results []Result) error {
|
||||
applied = append(applied, results...)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "retry", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "success", RunID: "run", ChanKey: "mobile-a", Operator: 1, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(applied) != 2 {
|
||||
t.Fatalf("applied results = %d, want both result rows for handler", len(applied))
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v, want one ACK for successful/terminal result", fake.calls)
|
||||
}
|
||||
if len(fake.calls[0].args) == 0 {
|
||||
t.Fatal("ACK call did not include arguments")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsAnotherActiveConsumer(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(nil)}}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
|
||||
if err := consumer.Run(context.Background()); !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("Run error = %v, want ErrAlreadyRunning", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessOnceStartsEveryRegisteredChannel(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{
|
||||
gvar.New([]any{"mobile-a", "telecom-a", "mobile-b"}),
|
||||
gvar.New([]any{}),
|
||||
gvar.New([]any{}),
|
||||
gvar.New([]any{}),
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{})
|
||||
processed, err := consumer.processOnce(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !processed {
|
||||
t.Fatal("processOnce() did not start registered channels")
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
fake.mu.Lock()
|
||||
callCount := len(fake.calls)
|
||||
fake.mu.Unlock()
|
||||
if callCount == 4 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("Redis calls = %d, want channels + 3 claims", callCount)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
consumer.channelGroup.Wait()
|
||||
if len(fake.calls) != 4 {
|
||||
t.Fatalf("Redis calls = %d, want channels + 3 claims", len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
type channelRedis struct {
|
||||
mu sync.Mutex
|
||||
claims map[string][]queue.Task
|
||||
claimCounts map[string]int
|
||||
}
|
||||
|
||||
func (r *channelRedis) Do(_ context.Context, command string, _ ...any) (*gvar.Var, error) {
|
||||
return nil, errors.New("unexpected command " + command)
|
||||
}
|
||||
|
||||
func (r *channelRedis) Eval(_ context.Context, script string, _ int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if strings.Contains(script, "ZRANGEBYSCORE") {
|
||||
chanKey := strings.TrimPrefix(keys[0], "flow:")
|
||||
r.claimCounts[chanKey]++
|
||||
tasks := r.claims[chanKey]
|
||||
if len(tasks) == 0 {
|
||||
return gvar.New([]any{}), nil
|
||||
}
|
||||
task := tasks[0]
|
||||
r.claims[chanKey] = tasks[1:]
|
||||
return gvar.New([]any{taskMember(task), "token-" + task.TaskID}), nil
|
||||
}
|
||||
if len(keys) == 2 && len(args) == 2 {
|
||||
return gvar.New(int64(1)), nil
|
||||
}
|
||||
return nil, errors.New("unexpected eval")
|
||||
}
|
||||
|
||||
func TestSlowChannelDoesNotBlockAnotherChannelFromClaimingAgain(t *testing.T) {
|
||||
redis := &channelRedis{
|
||||
claims: map[string][]queue.Task{
|
||||
"telecom-a": {{TaskID: "t1", RunID: "run", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}},
|
||||
"mobile-a": {
|
||||
{TaskID: "m1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
|
||||
{TaskID: "m2", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1},
|
||||
},
|
||||
},
|
||||
claimCounts: make(map[string]int),
|
||||
}
|
||||
telecomStarted := make(chan struct{})
|
||||
releaseTelecom := make(chan struct{})
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
if task.ChanKey == "telecom-a" {
|
||||
close(telecomStarted)
|
||||
<-releaseTelecom
|
||||
}
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
store, err := queue.New(redis, "flow:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer, err := New(store, handler, Config{ClaimBatch: 1, MobileConcurrency: 1, TelecomConcurrency: 1, LeaseTTL: time.Minute})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
consumer.startChannel(ctx, "telecom-a")
|
||||
consumer.startChannel(ctx, "mobile-a")
|
||||
select {
|
||||
case <-telecomStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("telecom channel did not start")
|
||||
}
|
||||
if consumer.startChannel(ctx, "telecom-a") {
|
||||
t.Fatal("channel executor started twice while first executor is running")
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
redis.mu.Lock()
|
||||
mobileClaims := redis.claimCounts["mobile-a"]
|
||||
redis.mu.Unlock()
|
||||
if mobileClaims >= 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("mobile claim count = %d while telecom is blocked", mobileClaims)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(releaseTelecom)
|
||||
consumer.channelGroup.Wait()
|
||||
}
|
||||
|
||||
func TestClaimedBatchRenewsLeaseUntilAcknowledged(t *testing.T) {
|
||||
fake := &fakeRedis{}
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
time.Sleep(45 * time.Millisecond)
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
if len(fake.calls) < 2 {
|
||||
t.Fatalf("Redis calls = %d, want lease renewal and ACK", len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLostBatchLeasePreventsApplyAndAck(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(0))}}
|
||||
applyCalls := 0
|
||||
handler := &testHandler{
|
||||
query: func(ctx context.Context, task queue.Task) Result {
|
||||
<-ctx.Done()
|
||||
return Result{Task: task, ErrorClass: "canceled"}
|
||||
},
|
||||
apply: func(context.Context, []Result) error {
|
||||
applyCalls++
|
||||
return nil
|
||||
},
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{LeaseTTL: 30 * time.Millisecond, TaskTimeout: 20 * time.Millisecond, MobileConcurrency: 1, TelecomConcurrency: 1})
|
||||
claimed := []queue.ClaimedTask{claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1})}
|
||||
err := consumer.handleClaims(context.Background(), claimed)
|
||||
if !errors.Is(err, ErrLeaseLost) {
|
||||
t.Fatalf("handleClaims() error = %v, want ErrLeaseLost", err)
|
||||
}
|
||||
if applyCalls != 0 {
|
||||
t.Fatalf("apply calls = %d, want 0 after lease loss", applyCalls)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
if len(fake.calls) != 1 {
|
||||
t.Fatalf("Redis calls = %d, want only failed lease renewal", len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func newTrackingHandler(release chan struct{}, current, maximum *atomic.Int32) *testHandler {
|
||||
return &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
active := current.Add(1)
|
||||
for {
|
||||
observed := maximum.Load()
|
||||
if active <= observed || maximum.CompareAndSwap(observed, active) {
|
||||
break
|
||||
}
|
||||
}
|
||||
<-release
|
||||
current.Add(-1)
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
}
|
||||
|
||||
func TestMobileChannelsHaveIndependentConcurrencyPools(t *testing.T) {
|
||||
var current atomic.Int32
|
||||
var maximum atomic.Int32
|
||||
release := make(chan struct{})
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, ¤t, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
var group sync.WaitGroup
|
||||
for index := 0; index < 3; index++ {
|
||||
group.Add(1)
|
||||
go func(index int) {
|
||||
defer group.Done()
|
||||
task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-" + string(rune('a'+index)), TempKey: mobileTempKey, CardID: int64(index + 1)}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
|
||||
}(index)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if maximum.Load() != 3 {
|
||||
t.Fatalf("maximum in-flight across channels = %d, want 3 (one pool per channel)", maximum.Load())
|
||||
}
|
||||
close(release)
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func TestChannelConcurrencyPoolCapsWithinChannel(t *testing.T) {
|
||||
var current atomic.Int32
|
||||
var maximum atomic.Int32
|
||||
release := make(chan struct{})
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, newTrackingHandler(release, ¤t, &maximum), Config{MobileConcurrency: 2, TelecomConcurrency: 1})
|
||||
var group sync.WaitGroup
|
||||
for index := 0; index < 3; index++ {
|
||||
group.Add(1)
|
||||
go func(index int) {
|
||||
defer group.Done()
|
||||
task := queue.Task{TaskID: string(rune('1' + index)), ChanKey: "mobile-a", TempKey: mobileTempKey, CardID: int64(index + 1)}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
|
||||
}(index)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if maximum.Load() != 2 {
|
||||
t.Fatalf("maximum in-flight within one channel = %d, want 2", maximum.Load())
|
||||
}
|
||||
close(release)
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func TestMobileAndTelecomUseIndependentChannelPools(t *testing.T) {
|
||||
started := make(chan carrier, 2)
|
||||
release := make(chan struct{})
|
||||
handler := &testHandler{query: func(_ context.Context, task queue.Task) Result {
|
||||
channelCarrier, _ := policyForTemplate(task.TempKey, Config{})
|
||||
started <- channelCarrier
|
||||
<-release
|
||||
return Result{Task: task, Success: true}
|
||||
}}
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, handler, Config{MobileConcurrency: 1, TelecomConcurrency: 1})
|
||||
go func() {
|
||||
task := queue.Task{TaskID: "mobile", ChanKey: "mobile-a", TempKey: mobileTempKey, Operator: 1}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierMobile, mobileQPS)
|
||||
}()
|
||||
go func() {
|
||||
task := queue.Task{TaskID: "telecom", ChanKey: "telecom-a", TempKey: telecomTempKey, Operator: 2}
|
||||
_ = consumer.queryClaim(context.Background(), claimedTask(task), carrierTelecom, telecomQPS)
|
||||
}()
|
||||
|
||||
seen := map[carrier]bool{}
|
||||
for len(seen) < 2 {
|
||||
select {
|
||||
case value := <-started:
|
||||
seen[value] = true
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("carrier pools did not progress independently: %+v", seen)
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
func TestUnknownTemplateUsesGenericStrategy(t *testing.T) {
|
||||
fake := &fakeRedis{responses: []*gvar.Var{gvar.New(int64(2))}}
|
||||
queryCalls := 0
|
||||
failures := 0
|
||||
applyCalls := 0
|
||||
handler := &testHandler{
|
||||
query: func(_ context.Context, task queue.Task) Result {
|
||||
queryCalls++
|
||||
return Result{Task: task, Success: true, UsageMB: 1}
|
||||
},
|
||||
apply: func(_ context.Context, results []Result) error {
|
||||
applyCalls++
|
||||
for _, result := range results {
|
||||
if !result.Success || result.BlockChannel {
|
||||
t.Fatalf("unexpected result = %+v", result)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
consumer := newTestConsumer(t, fake, handler, Config{OnFailure: func(context.Context, Result) { failures++ }})
|
||||
claimed := []queue.ClaimedTask{
|
||||
claimedTask(queue.Task{TaskID: "1", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 1, CardID: 1}),
|
||||
claimedTask(queue.Task{TaskID: "2", RunID: "run", ChanKey: "unknown-a", TempKey: "unknown-template", Operator: 2, CardID: 2}),
|
||||
}
|
||||
if err := consumer.handleClaims(context.Background(), claimed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queryCalls != 2 || failures != 0 || applyCalls != 1 {
|
||||
t.Fatalf("query calls = %d, failures = %d, apply calls = %d", queryCalls, failures, applyCalls)
|
||||
}
|
||||
if len(fake.calls) != 1 || fake.calls[0].command != "EVAL" {
|
||||
t.Fatalf("Redis calls = %+v, want one ack script", fake.calls)
|
||||
}
|
||||
if len(fake.responses) != 0 {
|
||||
t.Fatalf("unexpected leftover scripted responses: %+v", fake.responses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelPacerLimitsEachChannelIndependently(t *testing.T) {
|
||||
pacer := newChannelPacer()
|
||||
now := time.Unix(100, 0)
|
||||
pacedQPS := 30.0
|
||||
if delay := pacer.reserve("mobile-a", pacedQPS, now); delay != 0 {
|
||||
t.Fatalf("first reservation delay = %s", delay)
|
||||
}
|
||||
if delay := pacer.reserve("mobile-a", pacedQPS, now); delay < 33*time.Millisecond || delay > 34*time.Millisecond {
|
||||
t.Fatalf("second reservation delay = %s", delay)
|
||||
}
|
||||
if delay := pacer.reserve("mobile-b", pacedQPS, now); delay != 0 {
|
||||
t.Fatalf("independent channel delay = %s", delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelPacerSkipsWhenQPSIsDisabled(t *testing.T) {
|
||||
pacer := newChannelPacer()
|
||||
now := time.Unix(100, 0)
|
||||
if delay := pacer.reserve("mobile-a", 0, now); delay != 0 {
|
||||
t.Fatalf("disabled qps delay = %s", delay)
|
||||
}
|
||||
if len(pacer.next) != 0 {
|
||||
t.Fatalf("disabled qps should not reserve channel slots, got %+v", pacer.next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewDefaultsTaskTimeout(t *testing.T) {
|
||||
consumer := newTestConsumer(t, &fakeRedis{}, &testHandler{}, Config{})
|
||||
if consumer.config.TaskTimeout != 30*time.Second {
|
||||
t.Fatalf("TaskTimeout = %s, want 30s default", consumer.config.TaskTimeout)
|
||||
}
|
||||
if consumer.config.GenericQPS != defaultGenericQPS {
|
||||
t.Fatalf("GenericQPS = %v, want %v default", consumer.config.GenericQPS, defaultGenericQPS)
|
||||
}
|
||||
if _, err := New(consumer.store, &testHandler{}, Config{LeaseTTL: 10 * time.Second}); err == nil {
|
||||
t.Fatal("New() with LeaseTTL below default TaskTimeout should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyForTemplateUsesTemplateValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
tempKey string
|
||||
carrier carrier
|
||||
qps float64
|
||||
}{
|
||||
{name: "mobile", tempKey: mobileTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "mobile mock", tempKey: mobileMockKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Tengyu", tempKey: sharedTengyuTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Xuzhou mobile", tempKey: sharedXuzhouMobileTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Lelian", tempKey: sharedLelianTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Qianxun", tempKey: sharedQianxunTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Henan", tempKey: sharedHenanTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Daihe", tempKey: sharedDaiheTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Beixin", tempKey: sharedBeixinTempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
// lanshang_8000/8003 are OneLink shared-token templates whose names
|
||||
// omit the sim_onelink_ prefix. Before this case was added they fell
|
||||
// through to carrierGeneric and were throttled to 1.5 QPS, which made a
|
||||
// 6k-card status/traffic/IMEI run take over an hour per channel.
|
||||
{name: "shared Lanshang 8000", tempKey: sharedLanshang8000TempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Lanshang 8003", tempKey: sharedLanshang8003TempKey, carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "shared Lanshang case and padding", tempKey: " LANSHANG_8000 ", carrier: carrierMobile, qps: mobileQPS},
|
||||
{name: "telecom", tempKey: telecomTempKey, carrier: carrierTelecom, qps: telecomQPS},
|
||||
{name: "telecom v3", tempKey: " SIM_DX_V3 ", carrier: carrierTelecom, qps: telecomQPS},
|
||||
{name: "unknown telecom version", tempKey: "sim_dx_v2_future", carrier: carrierGeneric, qps: defaultGenericQPS},
|
||||
{name: "unknown template", tempKey: "unknown-template", carrier: carrierGeneric, qps: defaultGenericQPS},
|
||||
{name: "missing template", tempKey: "", carrier: carrierGeneric, qps: defaultGenericQPS},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
gotCarrier, gotQPS := policyForTemplate(test.tempKey, Config{})
|
||||
if gotCarrier != test.carrier || gotQPS != test.qps {
|
||||
t.Fatalf("policyForTemplate(%q) = %q,%v", test.tempKey, gotCarrier, gotQPS)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyForTemplateConfigurationOverridesDefaults(t *testing.T) {
|
||||
config := Config{
|
||||
GenericQPS: 2.5,
|
||||
TemplateQPS: map[string]float64{
|
||||
" SIM_YD_V5 ": 12,
|
||||
"unknown-template": 3,
|
||||
"invalid": 0,
|
||||
},
|
||||
}
|
||||
config.setDefaults()
|
||||
|
||||
if gotCarrier, gotQPS := policyForTemplate("sim_yd_v5", config); gotCarrier != carrierMobile || gotQPS != 12 {
|
||||
t.Fatalf("mobile override = %q,%v, want mobile,12", gotCarrier, gotQPS)
|
||||
}
|
||||
if gotCarrier, gotQPS := policyForTemplate("unknown-template", config); gotCarrier != carrierGeneric || gotQPS != 3 {
|
||||
t.Fatalf("generic override = %q,%v, want generic,3", gotCarrier, gotQPS)
|
||||
}
|
||||
if gotCarrier, gotQPS := policyForTemplate("other-template", config); gotCarrier != carrierGeneric || gotQPS != 2.5 {
|
||||
t.Fatalf("generic default = %q,%v, want generic,2.5", gotCarrier, gotQPS)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailurePoolAlertThresholdIsObservabilityOnly 覆盖失败池告警：
|
||||
// - 未配置阀值（0）时不发 HLEN 查询
|
||||
// - 失败数低于阀值时不告警
|
||||
// - 超过阀值时发一次 HLEN 并打 WARN，但绝不截断 Run
|
||||
//
|
||||
// 告警本身是纯观测行为，所以断言的是「有没有多打 Redis 命令」，
|
||||
// 而不是「Run 有没有被改坏」——后者必须永远为「没被改坏」。
|
||||
func TestFailurePoolAlertThresholdIsObservabilityOnly(t *testing.T) {
|
||||
newMeta := func(failed int64) queue.RunMeta {
|
||||
return queue.RunMeta{RunID: "run-1", Status: "completed_with_failures", FailedCount: failed, CurrentRound: 1}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
threshold int64
|
||||
failed int64
|
||||
wantHLenCalls int
|
||||
}{
|
||||
{"未配置阀值不查询", 0, 1_000_000, 0},
|
||||
{"低于阀值不告警", 50000, 49999, 0},
|
||||
{"等于阀值不告警", 50000, 50000, 0},
|
||||
{"超过阀值告警一次", 50000, 50001, 1},
|
||||
{"远超阀值也只告警一次", 50000, 500_000, 1},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
fake := &fakeRedis{}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{
|
||||
FailurePoolAlertThreshold: tc.threshold,
|
||||
})
|
||||
consumer.reportFailurePoolSize(context.Background(), "run-1", newMeta(tc.failed))
|
||||
|
||||
hlen := 0
|
||||
for _, call := range fake.calls {
|
||||
if call.command == "HLEN" {
|
||||
hlen++
|
||||
}
|
||||
}
|
||||
if hlen != tc.wantHLenCalls {
|
||||
t.Fatalf("%s: HLEN 调用 %d 次, want %d", tc.name, hlen, tc.wantHLenCalls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailurePoolAlertReadsCorrectKey 告警必须查该 Run 自己的失败池键。
|
||||
func TestFailurePoolAlertReadsCorrectKey(t *testing.T) {
|
||||
fake := &fakeRedis{}
|
||||
consumer := newTestConsumer(t, fake, &testHandler{}, Config{FailurePoolAlertThreshold: 10})
|
||||
consumer.reportFailurePoolSize(context.Background(), "run-abc", queue.RunMeta{
|
||||
RunID: "run-abc", Status: "completed_with_failures", FailedCount: 11, CurrentRound: 1,
|
||||
})
|
||||
|
||||
var key string
|
||||
for _, call := range fake.calls {
|
||||
if call.command == "HLEN" {
|
||||
if len(call.args) > 0 {
|
||||
key, _ = call.args[0].(string)
|
||||
}
|
||||
}
|
||||
}
|
||||
want, _ := consumer.store.FailedKey("run-abc")
|
||||
if key != want {
|
||||
t.Fatalf("HLEN key = %q, want %q", key, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
package zset
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type runHandler struct {
|
||||
query func(queue.Task) Result
|
||||
apply func([]Result) error
|
||||
}
|
||||
|
||||
func (h runHandler) Query(_ context.Context, task queue.Task) Result { return h.query(task) }
|
||||
func (h runHandler) Apply(_ context.Context, results []Result) error {
|
||||
if h.apply != nil {
|
||||
return h.apply(results)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runTasks(t *testing.T, store *queue.Store, run string, count int) []queue.Task {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if err := store.CreateRun(ctx, queue.RunMeta{RunID: run}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks := make([]queue.Task, count)
|
||||
for i := range tasks {
|
||||
tasks[i] = queue.Task{RunID: run, TaskID: fmt.Sprintf("%s:%d", run, i), CardID: int64(i + 1), ChanKey: "channel", TempKey: mobileTempKey, ICCID: fmt.Sprintf("card-%d", i)}
|
||||
}
|
||||
if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != count {
|
||||
t.Fatalf("enqueue %d: %v", n, err)
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
func runMeta(t *testing.T, store *queue.Store, run string) queue.RunMeta {
|
||||
t.Helper()
|
||||
meta, err := store.GetRunMeta(context.Background(), run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return meta
|
||||
}
|
||||
|
||||
func runConsumer(t *testing.T, store *queue.Store, handler Handler, rounds bool) *Consumer {
|
||||
t.Helper()
|
||||
c, err := New(store, handler, Config{RunQueueEnabled: true, RoundRetryEnabled: rounds, BlockedTasksAsFailures: !rounds, EventPrefix: "sim_status_queue", TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func claimRun(t *testing.T, store *queue.Store, run string, now time.Time, count int) []queue.ClaimedTask {
|
||||
t.Helper()
|
||||
claims, err := store.ClaimRun(context.Background(), run, "channel", now, now.Add(time.Minute), count)
|
||||
if err != nil || len(claims) != count {
|
||||
t.Fatalf("claim got %d want %d: %v", len(claims), count, err)
|
||||
}
|
||||
return claims
|
||||
}
|
||||
|
||||
func finishProducer(t *testing.T, store *queue.Store, run string) {
|
||||
t.Helper()
|
||||
if err := store.MarkProducerDone(context.Background(), run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkTerminal(t *testing.T, store *queue.Store, run string, success, failed int64) {
|
||||
t.Helper()
|
||||
meta := runMeta(t, store, run)
|
||||
want := "completed"
|
||||
if failed > 0 {
|
||||
want = "completed_with_failures"
|
||||
}
|
||||
if meta.Status != want || meta.CompletedAt.IsZero() || meta.SuccessCount != success || meta.FailedCount != failed || meta.ActiveCount != 0 || meta.InflightCount != 0 || meta.NextRoundCount != 0 {
|
||||
t.Fatalf("unexpected terminal metadata: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerCountsAndCompletes(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
ctx := context.Background()
|
||||
for _, failure := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(failure), func(t *testing.T) {
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
run := fmt.Sprint("counts-", failure)
|
||||
tasks := runTasks(t, store, run, 3)
|
||||
if n, err := store.EnqueueRunBatch(ctx, run, "channel", tasks, time.Now()); err != nil || n != 0 {
|
||||
t.Fatalf("duplicate enqueue: %d %v", n, err)
|
||||
}
|
||||
finishProducer(t, store, run)
|
||||
handler := runHandler{query: func(task queue.Task) Result {
|
||||
return Result{Task: task, Success: !failure || task.CardID != 2, ErrorClass: "business"}
|
||||
}, apply: func(_ []Result) error {
|
||||
if meta := runMeta(t, store, run); meta.SuccessCount != 0 || meta.FailedCount != 0 {
|
||||
return errors.New("counted before DB apply")
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
c := runConsumer(t, store, handler, false)
|
||||
claims := claimRun(t, store, run, time.Now(), 3)
|
||||
if err := c.handleRunClaims(ctx, claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var failures int64
|
||||
if failure {
|
||||
failures = 1
|
||||
}
|
||||
checkTerminal(t, store, run, 3-failures, failures)
|
||||
outcomes := []queue.TaskOutcome{{Success: true}, {Success: true}, {Success: true}}
|
||||
if n, err := store.FinalizeRunTasks(ctx, claims, outcomes); err != nil || n != 0 {
|
||||
t.Fatalf("duplicate finalize: %d %v", n, err)
|
||||
}
|
||||
checkTerminal(t, store, run, 3-failures, failures)
|
||||
key, _ := store.RunMetaKey(run)
|
||||
ttl, err := redis.Do(ctx, "TTL", key)
|
||||
if err != nil || ttl.Int64() < 29*24*3600 {
|
||||
t.Fatalf("retention=%v err=%v", ttl, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerLateProducerEmptyAndHistoricalRunCompletion(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
if err := store.CreateRun(ctx, queue.RunMeta{RunID: "historical", Status: "running", ProducerDone: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runTasks(t, store, "empty", 0)
|
||||
runTasks(t, store, "late", 1)
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}, false)
|
||||
claims := claimRun(t, store, "late", time.Now(), 1)
|
||||
if err := c.handleRunClaims(ctx, claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta := runMeta(t, store, "late"); !meta.CompletedAt.IsZero() || meta.SuccessCount != 1 {
|
||||
t.Fatalf("premature completion: %+v", meta)
|
||||
}
|
||||
finishProducer(t, store, "late")
|
||||
finishProducer(t, store, "empty")
|
||||
if _, err := c.processOnce(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.channelGroup.Wait()
|
||||
checkTerminal(t, store, "late", 1, 0)
|
||||
checkTerminal(t, store, "empty", 0, 0)
|
||||
checkTerminal(t, store, "historical", 0, 0)
|
||||
}
|
||||
|
||||
func TestRunConsumerRecoveryAndFencing(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
for _, mode := range []string{"database", "network", "stale"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
runTasks(t, store, mode, 1)
|
||||
finishProducer(t, store, mode)
|
||||
now := time.Now()
|
||||
claims := claimRun(t, store, mode, now, 1)
|
||||
good := runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }}
|
||||
handler := good
|
||||
if mode == "database" {
|
||||
handler.apply = func([]Result) error { return errors.New("database unavailable") }
|
||||
}
|
||||
if mode == "network" {
|
||||
handler.query = func(task queue.Task) Result { return Result{Task: task, Retryable: true, ErrorClass: "network"} }
|
||||
}
|
||||
c := runConsumer(t, store, handler, false)
|
||||
if mode != "stale" {
|
||||
err := c.handleRunClaims(ctx, claims)
|
||||
if (err != nil) != (mode == "database") {
|
||||
t.Fatalf("handle: %v", err)
|
||||
}
|
||||
if meta := runMeta(t, store, mode); meta.SuccessCount != 0 || meta.FailedCount != 0 || meta.ActiveCount != 1 || meta.InflightCount != 1 || meta.Status != "running" {
|
||||
t.Fatalf("lost pending task: %+v", meta)
|
||||
}
|
||||
}
|
||||
fresh := claimRun(t, store, mode, now.Add(2*time.Minute), 1)
|
||||
if n, err := store.RenewRunBatch(ctx, claims, now.Add(3*time.Minute)); err != nil || n != 0 {
|
||||
t.Fatalf("stale renewal: %d %v", n, err)
|
||||
}
|
||||
if n, err := store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil || n != 0 {
|
||||
t.Fatalf("stale ack: %d %v", n, err)
|
||||
}
|
||||
if n, err := store.BlockRunScopedAsFailed(ctx, claims[0], queue.TaskOutcome{}); err != nil || n != -1 {
|
||||
t.Fatalf("stale block: %d %v", n, err)
|
||||
}
|
||||
c.handler = good
|
||||
if err := c.handleRunClaims(ctx, fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, mode, 1, 0)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerBlockedChannelCountsOnlyUnfinishedTasks(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "blocked", 5)
|
||||
runTasks(t, store, "other", 1)
|
||||
// Production continues while the channel is blocked.
|
||||
claims := claimRun(t, store, "blocked", time.Now(), 2)
|
||||
var applied int
|
||||
successQueried := make(chan struct{})
|
||||
handler := runHandler{query: func(task queue.Task) Result {
|
||||
if task.CardID == 1 {
|
||||
close(successQueried)
|
||||
return Result{Task: task, Success: true}
|
||||
}
|
||||
<-successQueried
|
||||
return Result{Task: task, BlockChannel: true, ErrorClass: "channel_limited", Code: 797, ErrCode: "S00004", Message: "limited"}
|
||||
}, apply: func(results []Result) error { applied = countSuccessful(results); return nil }}
|
||||
c := runConsumer(t, store, handler, false)
|
||||
// Ensure the success response arrives before the channel-limit response.
|
||||
if err := c.handleRunClaims(ctx, claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := runMeta(t, store, "blocked")
|
||||
if applied != 1 || meta.SuccessCount != 1 || meta.FailedCount != 4 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
|
||||
t.Fatalf("applied=%d meta=%+v", applied, meta)
|
||||
}
|
||||
if n, err := store.BlockRunScopedAsFailed(ctx, claims[1], queue.TaskOutcome{}); err != nil || n != -1 {
|
||||
t.Fatalf("duplicate block: %d %v", n, err)
|
||||
}
|
||||
late := queue.Task{TaskID: "late", RunID: "blocked", ChanKey: "channel", CardID: 99}
|
||||
if n, err := store.EnqueueRunBatch(ctx, "blocked", "channel", []queue.Task{late}, time.Now()); err != nil || n != 0 {
|
||||
t.Fatalf("blocked enqueue: %d %v", n, err)
|
||||
}
|
||||
key, _ := store.FailedKey("blocked")
|
||||
failures, err := redis.Do(ctx, "HLEN", key)
|
||||
if err != nil || failures.Int() != 4 {
|
||||
t.Fatalf("failure records=%v err=%v", failures, err)
|
||||
}
|
||||
stored, err := redis.Do(ctx, "HGET", key, "blocked:4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var record queue.FailureRecord
|
||||
if err := json.Unmarshal([]byte(stored.String()), &record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.TaskID != "blocked:4" || record.CardID != 5 || record.ICCID != "card-4" || record.Attempts != 0 || record.FailureType != "channel_blocked" || record.ErrCode != "S00004" {
|
||||
t.Fatalf("failure identity/reason: %+v", record)
|
||||
}
|
||||
stateKey, _ := store.TaskStateKey("blocked")
|
||||
state, err := redis.Do(ctx, "HGET", stateKey, "blocked:4")
|
||||
if err != nil || state.MapStrAny()["status"] != "channel_blocked" {
|
||||
t.Fatalf("blocked state=%v err=%v", state, err)
|
||||
}
|
||||
if meta := runMeta(t, store, "other"); meta.ActiveCount != 1 || meta.FailedCount != 0 {
|
||||
t.Fatalf("other run affected: %+v", meta)
|
||||
}
|
||||
finishProducer(t, store, "blocked")
|
||||
if err := c.completeRun(ctx, "blocked"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, "blocked", 1, 4)
|
||||
}
|
||||
|
||||
func TestRunConsumerMonthlyRoundsRemainEnabled(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "flow:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "monthly", 1)
|
||||
finishProducer(t, store, "monthly")
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, true)
|
||||
for round := 1; round <= 5; round++ {
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "monthly", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if round < 5 {
|
||||
if meta := runMeta(t, store, "monthly"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
|
||||
t.Fatalf("round %d: %+v", round, meta)
|
||||
}
|
||||
if ok, err := store.AdvanceRunRound(ctx, "monthly", 5); err != nil || !ok {
|
||||
t.Fatalf("advance %v %v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkTerminal(t, store, "monthly", 0, 1)
|
||||
}
|
||||
|
||||
func TestRunConsumerStatusRetriesRetryableResultsForThreeRounds(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "status", 1)
|
||||
finishProducer(t, store, "status")
|
||||
c, err := New(store, runHandler{query: func(task queue.Task) Result {
|
||||
return Result{Task: task, ErrorClass: "carrier_retry", Retryable: true, Code: 9999, ErrCode: "S00004"}
|
||||
}}, Config{
|
||||
RunQueueEnabled: true,
|
||||
RoundRetryEnabled: true,
|
||||
MaxRounds: 3,
|
||||
EventPrefix: "sim_status_queue",
|
||||
ShouldRetryNextRound: func(result Result) bool {
|
||||
return result.Retryable
|
||||
},
|
||||
TemplateQPS: map[string]float64{mobileTempKey: 1000000},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for round := 1; round <= 3; round++ {
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "status", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if round < 3 {
|
||||
if meta := runMeta(t, store, "status"); meta.NextRoundCount != 1 || meta.FailedCount != 0 || meta.Status != "running" {
|
||||
t.Fatalf("round %d: %+v", round, meta)
|
||||
}
|
||||
if ok, err := store.AdvanceRunRound(ctx, "status", 3); err != nil || !ok {
|
||||
t.Fatalf("advance %v %v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkTerminal(t, store, "status", 0, 1)
|
||||
}
|
||||
|
||||
func TestReplayMaxRoundOverridesConsumerDefault(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
if err := store.CreateRun(ctx, queue.RunMeta{RunID: "replay-one", Operation: "status", SyncType: "replay", MaxRound: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := queue.Task{RunID: "replay-one", TaskID: "task", CardID: 1, ICCID: "card", ChanKey: "channel", TempKey: mobileTempKey}
|
||||
if _, err := store.EnqueueRunBatch(ctx, task.RunID, task.ChanKey, []queue.Task{task}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finishProducer(t, store, "replay-one")
|
||||
c, err := New(store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, Config{RunQueueEnabled: true, RoundRetryEnabled: true, MaxRounds: 3, TemplateQPS: map[string]float64{mobileTempKey: 1000000}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "replay-one", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, "replay-one", 0, 1)
|
||||
meta := runMeta(t, store, "replay-one")
|
||||
if meta.CurrentRound != 1 || meta.NextRoundCount != 0 {
|
||||
t.Fatalf("replay advanced rounds: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunQueueModeDoesNotEnableRoundRetry(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
runTasks(t, store, "single-round", 1)
|
||||
finishProducer(t, store, "single-round")
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, ErrorClass: "mobile_busy", Retryable: true} }}, false)
|
||||
if err := c.handleRunClaims(context.Background(), claimRun(t, store, "single-round", time.Now(), 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTerminal(t, store, "single-round", 0, 1)
|
||||
if meta := runMeta(t, store, "single-round"); meta.CurrentRound != 1 {
|
||||
t.Fatalf("unexpected retry: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonthlyChannelBlockKeepsExistingCountingPolicy(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "flow:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "monthly-block", 2)
|
||||
claimRun(t, store, "monthly-block", time.Now(), 1)
|
||||
for i := 0; i < 2; i++ {
|
||||
want := int64(2)
|
||||
if i == 1 {
|
||||
want = 0
|
||||
}
|
||||
if n, err := store.BlockRunScoped(ctx, "monthly-block", "channel"); err != nil || n != want {
|
||||
t.Fatalf("monthly block=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
if meta := runMeta(t, store, "monthly-block"); meta.FailedCount != 0 || meta.ActiveCount != 0 || meta.InflightCount != 0 {
|
||||
t.Fatalf("monthly counting changed: %+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConsumerConcurrentRuns(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "sim-status:")
|
||||
ctx := context.Background()
|
||||
var mu sync.Mutex
|
||||
applied := map[string]int{}
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result { return Result{Task: task, Success: true} }, apply: func(results []Result) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, result := range results {
|
||||
applied[result.Task.RunID]++
|
||||
}
|
||||
return nil
|
||||
}}, false)
|
||||
for _, run := range []string{"a", "b"} {
|
||||
runTasks(t, store, run, 3)
|
||||
finishProducer(t, store, run)
|
||||
}
|
||||
if started, err := c.processOnce(ctx); err != nil || !started {
|
||||
t.Fatalf("process: %v %v", started, err)
|
||||
}
|
||||
c.channelGroup.Wait()
|
||||
for _, run := range []string{"a", "b"} {
|
||||
checkTerminal(t, store, run, 3, 0)
|
||||
if applied[run] != 3 {
|
||||
t.Fatalf("applied %s: %d", run, applied[run])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunConsumerCoveredResultsCompleteWithoutFailures drives the full
|
||||
// consumer path for a covered result: the round must close, the run must end
|
||||
// as "completed" rather than "completed_with_failures", and the failure pool
|
||||
// must stay empty. This is the IMEI 14000 "carrier has no record" case — on
|
||||
// 2026-10-20 it accounted for every failure recorded in production while the
|
||||
// run success rate read 21%.
|
||||
func TestRunConsumerCoveredResultsCompleteWithoutFailures(t *testing.T) {
|
||||
redis := testutil.Redis(t)
|
||||
store, _ := queue.New(redis, "imei:")
|
||||
ctx := context.Background()
|
||||
runTasks(t, store, "covered", 4)
|
||||
finishProducer(t, store, "covered")
|
||||
queried := 0
|
||||
c := runConsumer(t, store, runHandler{query: func(task queue.Task) Result {
|
||||
queried++
|
||||
switch task.CardID {
|
||||
case 1:
|
||||
return Result{Task: task, Success: true, Imei: "865466070817587", ImeiSynced: true}
|
||||
case 2:
|
||||
return Result{Task: task, ErrorClass: "no_record", Code: 14000, ImeiSynced: true, Covered: true, Message: "系统无记录，请检查输入参数是否正确"}
|
||||
case 3:
|
||||
return Result{Task: task, ErrorClass: "data_invalid", ImeiSynced: true, Covered: true, Message: "carrier returned no imei value"}
|
||||
default:
|
||||
return Result{Task: task, ErrorClass: "business", Code: 11011, Message: "invalid card"}
|
||||
}
|
||||
}}, false)
|
||||
if err := c.handleRunClaims(ctx, claimRun(t, store, "covered", time.Now(), 4)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queried != 4 {
|
||||
t.Fatalf("queried = %d, want 4", queried)
|
||||
}
|
||||
meta := runMeta(t, store, "covered")
|
||||
// Three cards produced a usable or definitive answer (success + 2 covered),
|
||||
// and only the business failure is counted. Without the covered channel the
|
||||
// two 14000-style answers would have made failedCount=3.
|
||||
if meta.Status != "completed_with_failures" || meta.SuccessCount != 3 || meta.FailedCount != 1 || meta.NextRoundCount != 0 {
|
||||
t.Fatalf("unexpected run accounting: %+v", meta)
|
||||
}
|
||||
failureKey, _ := store.FailedKey("covered")
|
||||
size, err := redis.Do(ctx, "HLEN", failureKey)
|
||||
if err != nil || size.Int64() != 1 {
|
||||
t.Fatalf("failure pool entries = %d err=%v, want 1 (only the business failure)", size.Int64(), err)
|
||||
}
|
||||
for _, taskID := range []string{"covered:1", "covered:2"} {
|
||||
exists, err := redis.Do(ctx, "HEXISTS", failureKey, taskID)
|
||||
if err != nil || exists.Int() != 0 {
|
||||
t.Fatalf("covered card %s leaked into the failure pool: %v %v", taskID, exists, err)
|
||||
}
|
||||
}
|
||||
exists, err := redis.Do(ctx, "HEXISTS", failureKey, "covered:3")
|
||||
if err != nil || exists.Int() != 1 {
|
||||
t.Fatalf("the business failure is missing from the pool: %v %v", exists, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user