chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,103 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
// defaultBalanceRechargeMaxRetry 是充值超时消息的默认最大重试次数。
|
||||
// 支付宝凭据未配置等永久性失败会让 CheckRechargeTimeout 一直返回错误，
|
||||
// 必须设置上限，否则消息会无限重投并刷屏日志。
|
||||
const defaultBalanceRechargeMaxRetry = 100
|
||||
|
||||
// 包级变量便于测试注入发布行为，与 trafficUsagePublish 保持一致。
|
||||
var balanceRechargePublish = mq.Publish
|
||||
var balanceRechargePublishDelayed = mq.PublishDelayed
|
||||
|
||||
func (c *Consumers) initBalanceRechargeTimeout(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "balance recharge timeout consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if c.balance == nil {
|
||||
return gerror.New("balance recharge handler is not configured")
|
||||
}
|
||||
retryDelay, err := getBalanceRechargeRetryDelay(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maxRetry, err := getBalanceRechargeMaxRetry(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.balanceRechargeRetryDelay = retryDelay
|
||||
c.balanceRechargeMaxRetry = maxRetry
|
||||
if err := mq.DeclareQueue(ctx, consts.QueueBalanceRechargeDeadLetter); err != nil {
|
||||
return gerror.Wrap(err, "declare balance recharge dead-letter queue failed")
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.BalanceRechargeTimeoutMessage](
|
||||
ctx,
|
||||
consts.QueueBalanceRechargeTimeout,
|
||||
"iotlcm-balance-recharge-timeout",
|
||||
c.handleBalanceRechargeTimeout,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "balance recharge timeout consumer started, queue:%s, retryDelay:%s, maxRetry:%d", consts.QueueBalanceRechargeTimeout, retryDelay, maxRetry)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleBalanceRechargeTimeout(ctx context.Context, message model.BalanceRechargeTimeoutMessage) error {
|
||||
if message.OutTradeNo == "" {
|
||||
g.Log().Errorf(ctx, "balance recharge timeout message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
if c.balance == nil {
|
||||
return gerror.New("balance recharge handler is not configured")
|
||||
}
|
||||
if err := c.balance.CheckRechargeTimeout(ctx, message.OutTradeNo); err != nil {
|
||||
message.RetryCount++
|
||||
if shouldDeadLetterBalanceRecharge(message.RetryCount, c.balanceRechargeMaxRetry) {
|
||||
if publishErr := balanceRechargePublish(ctx, consts.QueueBalanceRechargeDeadLetter, message); publishErr != nil {
|
||||
return gerror.Wrap(publishErr, "publish balance recharge dead-letter message failed")
|
||||
}
|
||||
g.Log().Errorf(ctx, "balance recharge timeout retries exhausted and message sent to dead-letter queue, outTradeNo:%s, retryCount:%d, maxRetry:%d, err:%+v", message.OutTradeNo, message.RetryCount, c.balanceRechargeMaxRetry, err)
|
||||
return nil
|
||||
}
|
||||
if retryErr := balanceRechargePublishDelayed(ctx, consts.QueueBalanceRechargeDelay, consts.QueueBalanceRechargeTimeout, c.balanceRechargeRetryDelay, message); retryErr != nil {
|
||||
return gerror.Wrap(retryErr, "republish balance recharge timeout message failed")
|
||||
}
|
||||
g.Log().Warningf(ctx, "balance recharge timeout query failed and will retry, outTradeNo:%s, retryCount:%d, err:%+v", message.OutTradeNo, message.RetryCount, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shouldDeadLetterBalanceRecharge 判断本次失败后是否放弃重试。
|
||||
// retryCount 是自增后的重试次数，因此 maxRetry 表示最多重投多少次。
|
||||
func shouldDeadLetterBalanceRecharge(retryCount, maxRetry int) bool {
|
||||
return retryCount > maxRetry
|
||||
}
|
||||
|
||||
func getBalanceRechargeRetryDelay(ctx context.Context) (time.Duration, error) {
|
||||
value := g.Cfg().MustGet(ctx, "rabbitmq.balanceRecharge.retryDelay", "1m").String()
|
||||
delay, err := time.ParseDuration(value)
|
||||
if err != nil || delay <= 0 {
|
||||
return 0, gerror.Newf("balance recharge retry delay is invalid: %q", value)
|
||||
}
|
||||
return delay, nil
|
||||
}
|
||||
|
||||
func getBalanceRechargeMaxRetry(ctx context.Context) (int, error) {
|
||||
maxRetry := g.Cfg().MustGet(ctx, "rabbitmq.balanceRecharge.maxRetry", defaultBalanceRechargeMaxRetry).Int()
|
||||
if maxRetry <= 0 {
|
||||
return 0, gerror.Newf("balance recharge max retry is invalid: %d", maxRetry)
|
||||
}
|
||||
return maxRetry, nil
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func stubBalanceRechargePublish(t *testing.T) {
|
||||
t.Helper()
|
||||
originalPublish, originalDelayed := balanceRechargePublish, balanceRechargePublishDelayed
|
||||
t.Cleanup(func() {
|
||||
balanceRechargePublish, balanceRechargePublishDelayed = originalPublish, originalDelayed
|
||||
})
|
||||
balanceRechargePublish = func(context.Context, string, any) error { return nil }
|
||||
balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error { return nil }
|
||||
}
|
||||
|
||||
func setBalanceRechargeTestConfig(t *testing.T, content string) {
|
||||
t.Helper()
|
||||
oldAdapter := g.Cfg().GetAdapter()
|
||||
adapter, err := gcfg.NewAdapterContent(content)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
t.Cleanup(func() { g.Cfg().SetAdapter(oldAdapter) })
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeoutRepublishesDelayedBelowMaxRetry(t *testing.T) {
|
||||
stubBalanceRechargePublish(t)
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
|
||||
consumers.balanceRechargeRetryDelay = time.Minute
|
||||
consumers.balanceRechargeMaxRetry = 3
|
||||
|
||||
delayedCalls := 0
|
||||
balanceRechargePublishDelayed = func(_ context.Context, delayQueue, targetQueue string, delay time.Duration, data any) error {
|
||||
delayedCalls++
|
||||
if delayQueue != consts.QueueBalanceRechargeDelay || targetQueue != consts.QueueBalanceRechargeTimeout || delay != time.Minute {
|
||||
t.Fatalf("unexpected delayed publish: %s -> %s, delay:%s", delayQueue, targetQueue, delay)
|
||||
}
|
||||
message, ok := data.(model.BalanceRechargeTimeoutMessage)
|
||||
if !ok || message.OutTradeNo != "ALI-1" || message.RetryCount != 3 {
|
||||
t.Fatalf("unexpected retry payload: %+v", data)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
deadLetterCalls := 0
|
||||
balanceRechargePublish = func(_ context.Context, queue string, _ any) error {
|
||||
deadLetterCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
// 自增后达到上限仍未超过，应继续重投延迟队列。
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 2}); err != nil {
|
||||
t.Fatalf("retry below cap returned error: %v", err)
|
||||
}
|
||||
if delayedCalls != 1 || deadLetterCalls != 0 {
|
||||
t.Fatalf("delayedCalls=%d deadLetterCalls=%d, want 1 and 0", delayedCalls, deadLetterCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeoutDeadLettersAboveMaxRetry(t *testing.T) {
|
||||
stubBalanceRechargePublish(t)
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
|
||||
consumers.balanceRechargeRetryDelay = time.Minute
|
||||
consumers.balanceRechargeMaxRetry = 3
|
||||
|
||||
delayedCalls := 0
|
||||
balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error {
|
||||
delayedCalls++
|
||||
return nil
|
||||
}
|
||||
deadLetterCalls := 0
|
||||
balanceRechargePublish = func(_ context.Context, queue string, data any) error {
|
||||
deadLetterCalls++
|
||||
if queue != consts.QueueBalanceRechargeDeadLetter {
|
||||
t.Fatalf("dead letter queue = %s, want %s", queue, consts.QueueBalanceRechargeDeadLetter)
|
||||
}
|
||||
message, ok := data.(model.BalanceRechargeTimeoutMessage)
|
||||
if !ok || message.OutTradeNo != "ALI-1" || message.RetryCount != 4 {
|
||||
t.Fatalf("unexpected dead-letter payload: %+v", data)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 3}); err != nil {
|
||||
t.Fatalf("dead-letter path returned error: %v", err)
|
||||
}
|
||||
if delayedCalls != 0 || deadLetterCalls != 1 {
|
||||
t.Fatalf("delayedCalls=%d deadLetterCalls=%d, want 0 and 1", delayedCalls, deadLetterCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeoutReturnsPublishErrors(t *testing.T) {
|
||||
stubBalanceRechargePublish(t)
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{err: errors.New("Alipay credentials are not configured")})
|
||||
consumers.balanceRechargeRetryDelay = time.Minute
|
||||
consumers.balanceRechargeMaxRetry = 3
|
||||
|
||||
expected := errors.New("broker unavailable")
|
||||
balanceRechargePublish = func(context.Context, string, any) error { return expected }
|
||||
balanceRechargePublishDelayed = func(context.Context, string, string, time.Duration, any) error { return expected }
|
||||
|
||||
// 重投失败必须向上返回错误，让消费框架 Nack 重新入队，不能静默丢失消息。
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 0}); !errors.Is(err, expected) {
|
||||
t.Fatalf("delayed publish failure = %v, want %v", err, expected)
|
||||
}
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{OutTradeNo: "ALI-1", RetryCount: 3}); !errors.Is(err, expected) {
|
||||
t.Fatalf("dead-letter publish failure = %v, want %v", err, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBalanceRechargeMaxRetry(t *testing.T) {
|
||||
t.Run("defaults when unconfigured", func(t *testing.T) {
|
||||
setBalanceRechargeTestConfig(t, "rabbitmq:\n balanceRecharge:\n retryDelay: \"1m\"\n")
|
||||
maxRetry, err := getBalanceRechargeMaxRetry(context.Background())
|
||||
if err != nil || maxRetry != defaultBalanceRechargeMaxRetry {
|
||||
t.Fatalf("maxRetry = %d (err=%v), want %d", maxRetry, err, defaultBalanceRechargeMaxRetry)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reads configured value", func(t *testing.T) {
|
||||
setBalanceRechargeTestConfig(t, "rabbitmq:\n balanceRecharge:\n maxRetry: 7\n")
|
||||
maxRetry, err := getBalanceRechargeMaxRetry(context.Background())
|
||||
if err != nil || maxRetry != 7 {
|
||||
t.Fatalf("maxRetry = %d (err=%v), want 7", maxRetry, err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{name: "zero", value: "0"},
|
||||
{name: "negative", value: "-1"},
|
||||
{name: "non-numeric", value: "\"abc\""},
|
||||
} {
|
||||
t.Run("rejects "+test.name, func(t *testing.T) {
|
||||
setBalanceRechargeTestConfig(t, "rabbitmq:\n balanceRecharge:\n maxRetry: "+test.value+"\n")
|
||||
if _, err := getBalanceRechargeMaxRetry(context.Background()); err == nil {
|
||||
t.Fatalf("maxRetry=%s was accepted, want error", test.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
)
|
||||
|
||||
const batchRetryCountHeader = "x-iotlcm-retry-count"
|
||||
|
||||
func (c *Consumers) initBatchExecute(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "batch execute consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if c.batch == nil {
|
||||
return gerror.New("batch execute handler is not configured")
|
||||
}
|
||||
retryDelay, maxRetries, err := getBatchExecuteRetryConfig(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.batchExecuteRetryDelay = retryDelay
|
||||
c.batchExecuteMaxRetries = maxRetries
|
||||
if err = mq.DeclareQueue(ctx, consts.QueueBatchExecuteDeadLetter); err != nil {
|
||||
return gerror.Wrap(err, "declare batch execute dead-letter queue failed")
|
||||
}
|
||||
if err = mq.ConsumeJSON[batch.Message](
|
||||
ctx,
|
||||
consts.QueueBatchExecute,
|
||||
"iotlcm-batch-execute",
|
||||
c.handleBatchExecute,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "batch execute consumer started, queue:%s, retryDelay:%s, maxRetries:%d", consts.QueueBatchExecute, retryDelay, maxRetries)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleBatchExecute(ctx context.Context, message batch.Message) error {
|
||||
if c.batch == nil {
|
||||
return gerror.New("batch execute handler is not configured")
|
||||
}
|
||||
g.Log().Infof(ctx, "batch execute message received, batchNo:%s, batchType:%d", message.BatchNo, message.BatchType)
|
||||
if err := c.batch.Handle(ctx, message); err != nil {
|
||||
if _, delivered := mq.DeliveryMetadataFromContext(ctx); !delivered {
|
||||
return err
|
||||
}
|
||||
if batch.IsRetryable(err) {
|
||||
message.Retryable = true
|
||||
}
|
||||
var failureMarkErr *batch.FailureMarkError
|
||||
if errors.As(err, &failureMarkErr) {
|
||||
message.FailureReason = failureMarkErr.Reason
|
||||
}
|
||||
return c.retryOrDeadLetterBatchMessage(ctx, message, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) retryOrDeadLetterBatchMessage(ctx context.Context, message batch.Message, handlerErr error) error {
|
||||
metadata, _ := mq.DeliveryMetadataFromContext(ctx)
|
||||
retryCount := batchRetryCount(metadata.Headers)
|
||||
headers := amqp.Table{batchRetryCountHeader: retryCount + 1}
|
||||
if shouldDeadLetterBatch(retryCount, c.batchExecuteMaxRetries) {
|
||||
if message.Retryable {
|
||||
finalizationMessage := message
|
||||
finalizationMessage.Retryable = false
|
||||
finalizationMessage.FailureReason = handlerErr.Error()
|
||||
if err := c.batch.Handle(ctx, finalizationMessage); err != nil {
|
||||
return gerror.Wrap(err, "finalize retryable batch failure before dead-lettering")
|
||||
}
|
||||
message.FailureReason = handlerErr.Error()
|
||||
}
|
||||
deadLetterBody, marshalErr := json.Marshal(batchDeadLetterMessage{
|
||||
Message: message,
|
||||
RetryCount: retryCount,
|
||||
Error: handlerErr.Error(),
|
||||
FailedAt: time.Now().Format(time.RFC3339),
|
||||
})
|
||||
if marshalErr != nil {
|
||||
return gerror.Wrap(marshalErr, "marshal batch execute dead-letter message failed")
|
||||
}
|
||||
if err := mq.PublishBytesWithHeaders(ctx, consts.QueueBatchExecuteDeadLetter, deadLetterBody, nil); err != nil {
|
||||
return gerror.Wrap(err, "publish batch execute dead-letter message failed")
|
||||
}
|
||||
g.Log().Errorf(ctx, "batch execute message sent to dead-letter queue, batchNo:%s, batchType:%d, retryCount:%d, err:%+v", message.BatchNo, message.BatchType, retryCount, handlerErr)
|
||||
return nil
|
||||
}
|
||||
body, marshalErr := json.Marshal(message)
|
||||
if marshalErr != nil {
|
||||
return gerror.Wrap(marshalErr, "marshal batch execute retry message failed")
|
||||
}
|
||||
if err := mq.PublishDelayedBytesWithHeaders(
|
||||
ctx,
|
||||
consts.QueueBatchExecuteRetry,
|
||||
consts.QueueBatchExecute,
|
||||
c.batchExecuteRetryDelay,
|
||||
body,
|
||||
headers,
|
||||
); err != nil {
|
||||
return gerror.Wrap(err, "republish batch execute retry message failed")
|
||||
}
|
||||
g.Log().Warningf(ctx, "batch execute message scheduled for retry, batchNo:%s, batchType:%d, retryCount:%d, delay:%s, err:%+v", message.BatchNo, message.BatchType, retryCount+1, c.batchExecuteRetryDelay, handlerErr)
|
||||
return nil
|
||||
}
|
||||
|
||||
type batchDeadLetterMessage struct {
|
||||
Message batch.Message `json:"message"`
|
||||
RetryCount int `json:"retryCount"`
|
||||
Error string `json:"error"`
|
||||
FailedAt string `json:"failedAt"`
|
||||
}
|
||||
|
||||
func shouldDeadLetterBatch(retryCount, maxRetries int) bool {
|
||||
return retryCount >= maxRetries
|
||||
}
|
||||
|
||||
func batchRetryCount(headers map[string]any) int {
|
||||
if headers == nil {
|
||||
return 0
|
||||
}
|
||||
value, ok := headers[batchRetryCountHeader]
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
switch count := value.(type) {
|
||||
case int:
|
||||
return max(count, 0)
|
||||
case int8:
|
||||
return max(int(count), 0)
|
||||
case int16:
|
||||
return max(int(count), 0)
|
||||
case int32:
|
||||
return max(int(count), 0)
|
||||
case int64:
|
||||
return max(int(count), 0)
|
||||
case uint8:
|
||||
return int(count)
|
||||
case uint16:
|
||||
return int(count)
|
||||
case uint32:
|
||||
return int(count)
|
||||
case uint64:
|
||||
if count > uint64(^uint(0)>>1) {
|
||||
return int(^uint(0) >> 1)
|
||||
}
|
||||
return int(count)
|
||||
case string:
|
||||
parsed, err := strconv.Atoi(count)
|
||||
if err == nil {
|
||||
return max(parsed, 0)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func getBatchExecuteRetryConfig(ctx context.Context) (time.Duration, int, error) {
|
||||
delay := g.Cfg().MustGet(ctx, "rabbitmq.batchExecute.retryDelay", "30s").Duration()
|
||||
if delay <= 0 {
|
||||
return 0, 0, gerror.New("batch execute retry delay must be positive")
|
||||
}
|
||||
maxRetries := g.Cfg().MustGet(ctx, "rabbitmq.batchExecute.maxRetries", 3).Int()
|
||||
if maxRetries < 0 {
|
||||
return 0, 0, gerror.New("batch execute max retries cannot be negative")
|
||||
}
|
||||
return delay, maxRetries, nil
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/gogf/gf/contrib/drivers/sqlite/v2"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
batchrepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/batch"
|
||||
datarepo "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/repository/data"
|
||||
dataservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/data"
|
||||
)
|
||||
|
||||
type durableBatchRepo struct{ failed int }
|
||||
|
||||
func (*durableBatchRepo) FindMain(context.Context, string) (*entity.BatchMain, error) {
|
||||
return &entity.BatchMain{BatchNo: "APN-RECOVER", BatchType: 109, BatchAsyncStatus: batch.AsyncStatusPending}, nil
|
||||
}
|
||||
func (*durableBatchRepo) MarkProcessing(context.Context, string, int) (bool, error) { return true, nil }
|
||||
func (r *durableBatchRepo) MarkFailed(context.Context, string, int, string) (bool, error) {
|
||||
r.failed++
|
||||
return true, nil
|
||||
}
|
||||
func (*durableBatchRepo) MarkSuccess(context.Context, string, int, int, int, string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type durableBatchExecutor struct {
|
||||
calls, protections int
|
||||
retryable bool
|
||||
protectionErr error
|
||||
}
|
||||
|
||||
func (e *durableBatchExecutor) Execute(context.Context, string) error {
|
||||
e.calls++
|
||||
if e.retryable {
|
||||
return batch.Retryable(errors.New("准备中断"))
|
||||
}
|
||||
return errors.New("本地执行中断")
|
||||
}
|
||||
func (e *durableBatchExecutor) ProtectExecutionFailure(ctx context.Context, _ string, _ int, _ string) (bool, error) {
|
||||
e.protections++
|
||||
if ctx.Err() != nil {
|
||||
return true, ctx.Err()
|
||||
}
|
||||
return true, e.protectionErr
|
||||
}
|
||||
|
||||
func TestDurableAPNConsumerNeverFallsThroughToGenericFailure(t *testing.T) {
|
||||
repo := &durableBatchRepo{}
|
||||
executor := &durableBatchExecutor{}
|
||||
dispatcher := batch.NewDispatcher(nil)
|
||||
dispatcher.Register(109, executor)
|
||||
consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher)}
|
||||
if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if executor.calls != 1 || executor.protections != 1 || repo.failed != 0 {
|
||||
t.Fatalf("消费者普通错误越过durable保护: calls=%d protections=%d failed=%d", executor.calls, executor.protections, repo.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableAPNConsumerRetryExhaustionProtectsBeforePublish(t *testing.T) {
|
||||
repo := &durableBatchRepo{}
|
||||
dbDown := errors.New("模拟保护数据库不可用")
|
||||
executor := &durableBatchExecutor{retryable: true, protectionErr: dbDown}
|
||||
dispatcher := batch.NewDispatcher(nil)
|
||||
dispatcher.Register(109, executor)
|
||||
consumers := &Consumers{batch: batch.NewRuntime(repo, dispatcher), batchExecuteMaxRetries: 0}
|
||||
ctx := mq.WithDeliveryMetadata(context.Background(), mq.DeliveryMetadata{Headers: map[string]any{batchRetryCountHeader: 3}})
|
||||
err := consumers.handleBatchExecute(ctx, batch.Message{BatchNo: "APN-RECOVER", BatchType: 109})
|
||||
// 保护失败必须在消息发布前返回，测试不会连接真实RabbitMQ。
|
||||
if !errors.Is(err, dbDown) || executor.calls != 1 || executor.protections != 1 || repo.failed != 0 {
|
||||
t.Fatalf("重试耗尽错误终结或重发: err=%v calls=%d protections=%d failed=%d", err, executor.calls, executor.protections, repo.failed)
|
||||
}
|
||||
executor.protectionErr = nil
|
||||
if err := consumers.handleBatchExecute(context.Background(), batch.Message{BatchNo: "APN-RECOVER", BatchType: 109, FailureReason: "重试耗尽"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if executor.calls != 1 || executor.protections != 2 || repo.failed != 0 {
|
||||
t.Fatal("失败重投重新执行或MarkFailed")
|
||||
}
|
||||
}
|
||||
|
||||
// reentryWriteCounter 同时覆盖同步写、异步提交与结果查询，重入准备期间均不得调用。
|
||||
type reentryWriteCounter struct{ syncWrites, asyncWrites, queries int }
|
||||
|
||||
func (u *reentryWriteCounter) ChangeSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.syncWrites++
|
||||
return nil, nil
|
||||
}
|
||||
func (u *reentryWriteCounter) SubmitSimApnFunction(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.asyncWrites++
|
||||
return nil, nil
|
||||
}
|
||||
func (u *reentryWriteCounter) QuerySimApnFunctionResult(context.Context, *model.SimCardApnChangeInput) ([]model.SimBaseRes[model.SimCardApnChangeOutputItem], error) {
|
||||
u.queries++
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestAPNProcessingRabbitMQReentryRequiresDurableParent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if err := gdb.SetConfig(gdb.Config{"default": gdb.ConfigGroup{{Type: "sqlite", Link: "sqlite::@file(" + filepath.ToSlash(filepath.Join(t.TempDir(), "apn-reentry.db")) + ")"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db := g.DB()
|
||||
t.Cleanup(func() {
|
||||
if err := db.Close(ctx); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
for _, sql := range []string{
|
||||
"CREATE TABLE batch_main (id INTEGER PRIMARY KEY,batch_no TEXT UNIQUE,batch_type INTEGER,batch_async_status INTEGER,tenant_id TEXT,start_time DATETIME,batch_info TEXT,run_result INTEGER DEFAULT 0,valid_count INTEGER DEFAULT 0,success_count INTEGER DEFAULT 0,fail_count INTEGER DEFAULT 0,fail_reason TEXT,end_time DATETIME,updated_at DATETIME,created_by INTEGER,created_dept INTEGER,updated_by INTEGER)",
|
||||
"CREATE TABLE batch_detail (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,row_no INTEGER,item_id TEXT,detail_status INTEGER,error_reason TEXT,snapshot TEXT,updated_by INTEGER)",
|
||||
"CREATE TABLE t_sim_card_info (id INTEGER PRIMARY KEY AUTOINCREMENT,tenant_id TEXT,iccid TEXT,msisdn TEXT,imsi TEXT,chan_key TEXT,data_service_status INTEGER)",
|
||||
"CREATE TABLE data_operation_parent (batch_no TEXT PRIMARY KEY,ready INTEGER DEFAULT 0,finalize_token TEXT,finalize_until DATETIME,recovery_attempt_at DATETIME,last_reason TEXT)",
|
||||
"CREATE TABLE data_operation_sub_batch (id INTEGER PRIMARY KEY AUTOINCREMENT,batch_no TEXT,batch_type INTEGER,chan_key TEXT,operation_id TEXT UNIQUE,detail_ids TEXT,request_data TEXT,job_id TEXT,status TEXT,poll_count INTEGER DEFAULT 0,next_check_at DATETIME,submitted_at DATETIME,lease_token TEXT,lease_until DATETIME,created_at DATETIME DEFAULT CURRENT_TIMESTAMP,last_reason TEXT)",
|
||||
"CREATE TABLE data_operation_sub_detail (detail_id INTEGER PRIMARY KEY,operation_id TEXT,batch_no TEXT,iccid TEXT,UNIQUE(batch_no,iccid))",
|
||||
"CREATE TABLE data_operation_card_claim (tenant_id TEXT,iccid TEXT,batch_id INTEGER,operation_id TEXT,active_batch_id INTEGER DEFAULT 0,active_operation_id TEXT DEFAULT '',PRIMARY KEY(tenant_id,iccid))",
|
||||
} {
|
||||
if _, err := db.Exec(ctx, sql); err != nil {
|
||||
t.Fatalf("初始化本机 SQLite fixture: %v", err)
|
||||
}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
old, retryable bool
|
||||
}{
|
||||
{name: "legacy-ordinary", old: true},
|
||||
{name: "legacy-retryable", old: true, retryable: true},
|
||||
{name: "durable-ordinary"},
|
||||
{name: "durable-retryable", retryable: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
status := batch.AsyncStatusPending
|
||||
if tc.old {
|
||||
status = batch.AsyncStatusProcessing
|
||||
}
|
||||
// 使用 NULL 旧首发时间以确保 ReclaimProcessing 的 WHERE start_time IS NULL 可以命中。
|
||||
if _, err := db.Exec(ctx, "INSERT INTO batch_main(batch_no,batch_type,batch_async_status,tenant_id) VALUES (?,109,?,'tenant')", tc.name, status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
iccid := tc.name + "-card-" + string(rune('0'+i))
|
||||
detailStatus := batch.DetailStatusPending
|
||||
if tc.old {
|
||||
detailStatus = batch.DetailStatusRunning
|
||||
}
|
||||
if _, err := db.Exec(ctx, "INSERT INTO batch_detail(batch_no,batch_type,row_no,item_id,detail_status,snapshot) VALUES (?,109,?,?,?,?)", tc.name, i+1, iccid, detailStatus, `{"authorized":true}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(ctx, "INSERT INTO t_sim_card_info(tenant_id,iccid,msisdn,imsi,chan_key,data_service_status) VALUES ('tenant',?,?,?,'mock-apn',0)", iccid, "130000000"+string(rune('0'+i)), "460000000"+string(rune('0'+i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mainRepo := batchrepo.New()
|
||||
if tc.old && tc.retryable {
|
||||
// 先证实同一 fixture 的旧批确实可被重领，再复位空首发时间。
|
||||
claimed, err := mainRepo.ReclaimProcessing(ctx, tc.name, 109, nil)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("重领 CAS 必须可命中，避免假阳性: claimed=%t err=%v", claimed, err)
|
||||
}
|
||||
if _, err := db.Exec(ctx, "UPDATE batch_main SET start_time=NULL WHERE batch_no=?", tc.name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !tc.old {
|
||||
claimed, err := mainRepo.MarkProcessing(ctx, tc.name, 109)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("新批需同事务领取 Processing 与 ready=0 父凭据: claimed=%t err=%v", claimed, err)
|
||||
}
|
||||
}
|
||||
parent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=0", tc.name)
|
||||
if err != nil || parent.Int() != 1 && !tc.old || parent.Int() != 0 && tc.old {
|
||||
t.Fatalf("前置凭据不符: old=%t ready0=%d err=%v", tc.old, parent.Int(), err)
|
||||
}
|
||||
before, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upstream := &reentryWriteCounter{}
|
||||
dispatcher := batch.NewDispatcher(nil)
|
||||
dispatcher.Register(109, dataservice.NewExecutor(datarepo.New(), upstream))
|
||||
consumer := &Consumers{batch: batch.NewRuntime(mainRepo, dispatcher)}
|
||||
err = consumer.handleBatchExecute(ctx, batch.Message{BatchNo: tc.name, BatchType: 109, Retryable: tc.retryable})
|
||||
if err != nil {
|
||||
t.Errorf("重入应安全跳过旧批或准备新批，不能进入消息重发路径: %v", err)
|
||||
}
|
||||
if upstream.syncWrites != 0 || upstream.asyncWrites != 0 || upstream.queries != 0 {
|
||||
t.Errorf("重入意外调用上游: sync=%d submit=%d query=%d", upstream.syncWrites, upstream.asyncWrites, upstream.queries)
|
||||
}
|
||||
main, err := mainRepo.FindMain(ctx, tc.name)
|
||||
if err != nil || main == nil || main.BatchAsyncStatus != batch.AsyncStatusProcessing {
|
||||
t.Fatalf("批状态被意外终结: main=%+v err=%v", main, err)
|
||||
}
|
||||
remaining, err := db.GetValue(ctx, "SELECT COUNT(*) FROM batch_detail WHERE batch_no=? AND detail_status=1", tc.name)
|
||||
if err != nil || remaining.Int() != 10 {
|
||||
t.Fatalf("运行中明细不能被更改: count=%d err=%v", remaining.Int(), err)
|
||||
}
|
||||
intent, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_batch WHERE batch_no=? AND status='prepared'", tc.name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.old {
|
||||
if intent.Int() != 0 {
|
||||
t.Errorf("无凭据历史批不应产生新提交意图: %d", intent.Int())
|
||||
}
|
||||
assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name)
|
||||
if err != nil || assigned.Int() != 0 {
|
||||
t.Errorf("历史批不可冻结明细: count=%d err=%v", assigned.Int(), err)
|
||||
}
|
||||
claims, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_card_claim WHERE operation_id!='' AND iccid LIKE ?", tc.name+"-%")
|
||||
if err != nil || claims.Int() != 0 {
|
||||
t.Errorf("历史批不可建立卡级提交占位: count=%d err=%v", claims.Int(), err)
|
||||
}
|
||||
parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=?", tc.name)
|
||||
if err != nil || parent.Int() != 0 {
|
||||
t.Errorf("历史批不可补造父凭据: count=%d err=%v", parent.Int(), err)
|
||||
}
|
||||
after, err := db.GetValue(ctx, "SELECT start_time FROM batch_main WHERE batch_no=?", tc.name)
|
||||
if err != nil || after.String() != before.String() {
|
||||
t.Errorf("旧批重入不可重领取: before=%s after=%s err=%v", before.String(), after.String(), err)
|
||||
}
|
||||
} else {
|
||||
if intent.Int() != 1 {
|
||||
t.Errorf("新批 ready=0 未正确恢复成 10 卡一个冻结意图: %d", intent.Int())
|
||||
}
|
||||
parent, err = db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_parent WHERE batch_no=? AND ready=1", tc.name)
|
||||
if err != nil || parent.Int() != 1 {
|
||||
t.Errorf("新批冻结意图与 ready=1 未原子提交: count=%d err=%v", parent.Int(), err)
|
||||
}
|
||||
assigned, err := db.GetValue(ctx, "SELECT COUNT(*) FROM data_operation_sub_detail WHERE batch_no=?", tc.name)
|
||||
if err != nil || assigned.Int() != 10 {
|
||||
t.Errorf("新批冻结明细必须覆盖全部 10 卡: count=%d err=%v", assigned.Int(), err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package rabbitmq
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestShouldDeadLetterBatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
retryCount int
|
||||
maxRetries int
|
||||
want bool
|
||||
}{
|
||||
{name: "first attempt retries", retryCount: 0, maxRetries: 3, want: false},
|
||||
{name: "second attempt retries", retryCount: 1, maxRetries: 3, want: false},
|
||||
{name: "last configured retry retries", retryCount: 2, maxRetries: 3, want: false},
|
||||
{name: "next failure dead letters", retryCount: 3, maxRetries: 3, want: true},
|
||||
{name: "zero retries dead letters immediately", retryCount: 0, maxRetries: 0, want: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if actual := shouldDeadLetterBatch(tt.retryCount, tt.maxRetries); actual != tt.want {
|
||||
t.Fatalf("shouldDeadLetterBatch(%d, %d) = %t, want %t", tt.retryCount, tt.maxRetries, actual, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchRetryCount(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headers map[string]any
|
||||
want int
|
||||
}{
|
||||
{name: "missing", want: 0},
|
||||
{name: "int", headers: map[string]any{batchRetryCountHeader: 2}, want: 2},
|
||||
{name: "int64", headers: map[string]any{batchRetryCountHeader: int64(3)}, want: 3},
|
||||
{name: "uint", headers: map[string]any{batchRetryCountHeader: uint16(4)}, want: 4},
|
||||
{name: "string", headers: map[string]any{batchRetryCountHeader: "5"}, want: 5},
|
||||
{name: "negative", headers: map[string]any{batchRetryCountHeader: -1}, want: 0},
|
||||
{name: "invalid", headers: map[string]any{batchRetryCountHeader: "not-a-number"}, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if actual := batchRetryCount(tt.headers); actual != tt.want {
|
||||
t.Fatalf("batchRetryCount() = %d, want %d", actual, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
const channelRefreshConsumer = "iotlcm-worker-channel-refresh"
|
||||
|
||||
type channelRefreshConfig struct {
|
||||
Address string `json:"address"`
|
||||
Timeout time.Duration `json:"timeout"`
|
||||
}
|
||||
|
||||
type channelRefreshResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (c *Consumers) initChannelRefresh(ctx context.Context) error {
|
||||
if !rabbitmq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "channel refresh consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := rabbitmq.ConsumeJSON[model.ChannelRefreshMessage](
|
||||
ctx,
|
||||
consts.QueueChannelRefresh,
|
||||
channelRefreshConsumer,
|
||||
c.handleChannelRefresh,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "channel refresh consumer started, queue:%s", consts.QueueChannelRefresh)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleChannelRefresh(ctx context.Context, message model.ChannelRefreshMessage) error {
|
||||
switch strings.ToLower(strings.TrimSpace(message.Type)) {
|
||||
case "channel":
|
||||
if strings.TrimSpace(message.ChanKey) == "" {
|
||||
g.Log().Errorf(ctx, "channel refresh message dropped, chanKey is empty")
|
||||
return nil
|
||||
}
|
||||
if err := refreshChannelWithRetry(ctx, message.ChanKey); err != nil {
|
||||
g.Log().Errorf(ctx, "channel refresh failed, chanKey=%s, err=%+v", message.ChanKey, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
case "template":
|
||||
if strings.TrimSpace(message.TempKey) == "" {
|
||||
g.Log().Errorf(ctx, "channel refresh message dropped, tempKey is empty")
|
||||
return nil
|
||||
}
|
||||
return refreshTemplateChannels(ctx, message.TempKey)
|
||||
case "channel_rename":
|
||||
oldChanKey := strings.TrimSpace(message.OldChanKey)
|
||||
newChanKey := strings.TrimSpace(message.NewChanKey)
|
||||
if oldChanKey == "" || newChanKey == "" || oldChanKey == newChanKey {
|
||||
g.Log().Errorf(ctx, "channel rename message dropped, oldChanKey=%q newChanKey=%q", oldChanKey, newChanKey)
|
||||
return nil
|
||||
}
|
||||
if err := renameChannelWithRetry(ctx, oldChanKey, newChanKey); err != nil {
|
||||
// 返回 error 让消费框架 Nack 并重新入队：rename 各步骤幂等，
|
||||
// 重复投递安全；静默 Ack 会导致旧通道实例永久残留，只能重启恢复。
|
||||
g.Log().Errorf(ctx, "channel rename failed after retries, will requeue, oldChanKey=%s newChanKey=%s err=%+v", oldChanKey, newChanKey, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
g.Log().Errorf(ctx, "channel refresh message dropped, unsupported type=%s", message.Type)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func renameChannelWithRetry(ctx context.Context, oldChanKey, newChanKey string) error {
|
||||
var err error
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
err = renameChannel(ctx, oldChanKey, newChanKey)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if attempt == 3 {
|
||||
break
|
||||
}
|
||||
delay := time.Second
|
||||
if attempt == 2 {
|
||||
delay = 3 * time.Second
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
return gerror.Wrapf(err, "channel rename failed after %d attempts, oldChanKey=%s newChanKey=%s", 3, oldChanKey, newChanKey)
|
||||
}
|
||||
|
||||
// renameChannel is ordered so that the old runtime remains available until
|
||||
// both upstreams have built the new instance. Card data and ICCID routes are
|
||||
// intentionally left untouched for operations staff to migrate manually.
|
||||
func renameChannel(ctx context.Context, oldChanKey, newChanKey string) error {
|
||||
if err := refreshChannel(ctx, newChanKey); err != nil {
|
||||
return gerror.Wrap(err, "refresh new channel failed")
|
||||
}
|
||||
if err := removeChannelTarget(ctx, "upstream.gateway", oldChanKey); err != nil {
|
||||
return gerror.Wrap(err, "remove old gateway channel failed")
|
||||
}
|
||||
if err := removeChannelTarget(ctx, "upstream.proxy", oldChanKey); err != nil {
|
||||
return gerror.Wrap(err, "remove old proxy channel failed")
|
||||
}
|
||||
g.Log().Infof(ctx, "channel instance rebuilt after rename, oldChanKey=%s newChanKey=%s", oldChanKey, newChanKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
func refreshTemplateChannels(ctx context.Context, tempKey string) error {
|
||||
var rows []struct {
|
||||
ChanKey string `json:"chanKey"`
|
||||
}
|
||||
if err := g.DB().Model("t_sim_channels").Ctx(ctx).
|
||||
Fields("chan_key").
|
||||
Where("temp_key", strings.TrimSpace(tempKey)).
|
||||
Scan(&rows); err != nil {
|
||||
return gerror.Wrap(err, "query template channels failed")
|
||||
}
|
||||
|
||||
var firstErr error
|
||||
for _, row := range rows {
|
||||
if err := refreshChannelWithRetry(ctx, row.ChanKey); err != nil {
|
||||
g.Log().Errorf(ctx, "template channel refresh failed, tempKey=%s, chanKey=%s, err=%+v", tempKey, row.ChanKey, err)
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func refreshChannelWithRetry(ctx context.Context, chanKey string) error {
|
||||
var err error
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
err = refreshChannel(ctx, chanKey)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if attempt == 3 {
|
||||
break
|
||||
}
|
||||
delay := time.Second
|
||||
if attempt == 2 {
|
||||
delay = 3 * time.Second
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return gerror.Wrapf(err, "channel refresh canceled after %d attempts, chanKey=%s", attempt, chanKey)
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
return gerror.Wrapf(err, "channel refresh failed after %d attempts, chanKey=%s", 3, chanKey)
|
||||
}
|
||||
|
||||
func refreshChannel(ctx context.Context, chanKey string) error {
|
||||
chanKey = strings.TrimSpace(chanKey)
|
||||
if chanKey == "" {
|
||||
return gerror.New("chanKey is required")
|
||||
}
|
||||
if err := refreshChannelTarget(ctx, "upstream.gateway", chanKey); err != nil {
|
||||
return gerror.Wrap(err, "refresh gateway channel failed")
|
||||
}
|
||||
if err := refreshChannelTarget(ctx, "upstream.proxy", chanKey); err != nil {
|
||||
return gerror.Wrap(err, "refresh proxy channel failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func refreshChannelTarget(ctx context.Context, configKey, chanKey string) error {
|
||||
return channelControlTarget(ctx, configKey, "/internal/channel/refresh", chanKey)
|
||||
}
|
||||
|
||||
func removeChannelTarget(ctx context.Context, configKey, chanKey string) error {
|
||||
return channelControlTarget(ctx, configKey, "/internal/channel/remove", chanKey)
|
||||
}
|
||||
|
||||
func channelControlTarget(ctx context.Context, configKey, path, chanKey string) error {
|
||||
configValue, err := g.Cfg().Get(ctx, configKey)
|
||||
if err != nil {
|
||||
return gerror.Wrapf(err, "read upstream configuration failed: %s", configKey)
|
||||
}
|
||||
if configValue == nil || configValue.IsNil() {
|
||||
return gerror.Newf("upstream configuration is missing: %s", configKey)
|
||||
}
|
||||
config := channelRefreshConfig{}
|
||||
if err = configValue.Scan(&config); err != nil {
|
||||
return gerror.Wrapf(err, "parse upstream configuration failed: %s", configKey)
|
||||
}
|
||||
address := strings.TrimSpace(config.Address)
|
||||
if address == "" {
|
||||
return gerror.Newf("upstream address is empty: %s", configKey)
|
||||
}
|
||||
if config.Timeout <= 0 {
|
||||
config.Timeout = 10 * time.Second
|
||||
}
|
||||
requestCtx, cancel := context.WithTimeout(ctx, config.Timeout)
|
||||
defer cancel()
|
||||
|
||||
client := g.Client()
|
||||
client.SetTimeout(config.Timeout)
|
||||
controlURL := fmt.Sprintf("http://%s%s?chanKey=%s", address, path, url.QueryEscape(chanKey))
|
||||
response, err := client.Post(requestCtx, controlURL, nil)
|
||||
if err != nil {
|
||||
return gerror.Wrapf(err, "request upstream channel control failed: target=%s path=%s", configKey, path)
|
||||
}
|
||||
defer response.Close()
|
||||
|
||||
output := channelRefreshResponse{}
|
||||
statusCode := 0
|
||||
if response.Response != nil {
|
||||
statusCode = response.StatusCode
|
||||
}
|
||||
if err = json.Unmarshal(response.ReadAll(), &output); err != nil {
|
||||
return gerror.Wrapf(err, "parse upstream channel control response failed: target=%s path=%s status=%d", configKey, path, statusCode)
|
||||
}
|
||||
if output.Code != 0 {
|
||||
if output.Message == "" {
|
||||
output.Message = "upstream channel control failed"
|
||||
}
|
||||
return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d message=%s", configKey, path, statusCode, output.Message)
|
||||
}
|
||||
if statusCode < 200 || statusCode >= 300 {
|
||||
return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d", configKey, path, statusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func setChannelRefreshTestConfig(t *testing.T, gateway, proxy string) {
|
||||
t.Helper()
|
||||
oldAdapter := g.Cfg().GetAdapter()
|
||||
adapter, err := gcfg.NewAdapterContent(fmt.Sprintf(`
|
||||
upstream:
|
||||
gateway:
|
||||
address: %q
|
||||
timeout: 1s
|
||||
proxy:
|
||||
address: %q
|
||||
timeout: 1s
|
||||
`, gateway, proxy))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
t.Cleanup(func() { g.Cfg().SetAdapter(oldAdapter) })
|
||||
}
|
||||
|
||||
func testServerAddress(server *httptest.Server) string {
|
||||
return strings.TrimPrefix(server.URL, "http://")
|
||||
}
|
||||
|
||||
func TestRefreshChannelCallsGatewayAndProxy(t *testing.T) {
|
||||
var gatewayCalls atomic.Int32
|
||||
var proxyCalls atomic.Int32
|
||||
newServer := func(counter *atomic.Int32) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
counter.Add(1)
|
||||
if request.URL.Path != "/internal/channel/refresh" || request.URL.Query().Get("chanKey") != "channel-a" {
|
||||
t.Fatalf("unexpected request: %s", request.URL.String())
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success"}`))
|
||||
}))
|
||||
}
|
||||
gateway := newServer(&gatewayCalls)
|
||||
defer gateway.Close()
|
||||
proxy := newServer(&proxyCalls)
|
||||
defer proxy.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), testServerAddress(proxy))
|
||||
|
||||
if err := refreshChannel(context.Background(), "channel-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gatewayCalls.Load() != 1 || proxyCalls.Load() != 1 {
|
||||
t.Fatalf("gateway calls=%d proxy calls=%d", gatewayCalls.Load(), proxyCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameChannelOnlyRebuildsRuntimeInstances(t *testing.T) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
)
|
||||
newServer := func(target string) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
mu.Lock()
|
||||
events = append(events, target+":"+request.URL.Path+":"+request.URL.Query().Get("chanKey"))
|
||||
mu.Unlock()
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success"}`))
|
||||
}))
|
||||
}
|
||||
gateway := newServer("gateway")
|
||||
defer gateway.Close()
|
||||
proxy := newServer("proxy")
|
||||
defer proxy.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), testServerAddress(proxy))
|
||||
|
||||
if err := renameChannel(context.Background(), "channel-old", "channel-new"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"gateway:/internal/channel/refresh:channel-new",
|
||||
"proxy:/internal/channel/refresh:channel-new",
|
||||
"gateway:/internal/channel/remove:channel-old",
|
||||
"proxy:/internal/channel/remove:channel-old",
|
||||
}
|
||||
if strings.Join(events, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("rename events = %q, want %q", events, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelControlErrorIncludesTargetStatusAndMessage(t *testing.T) {
|
||||
gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
_, _ = w.Write([]byte(`{"code":50,"message":"carrier unavailable"}`))
|
||||
}))
|
||||
defer gateway.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), "127.0.0.1:1")
|
||||
|
||||
err := refreshChannelTarget(context.Background(), "upstream.gateway", "channel-a")
|
||||
if err == nil {
|
||||
t.Fatal("expected target error")
|
||||
}
|
||||
message := err.Error()
|
||||
for _, part := range []string{"target=upstream.gateway", "status=502", "carrier unavailable"} {
|
||||
if !strings.Contains(message, part) {
|
||||
t.Fatalf("error %q does not contain %q", message, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleChannelRefreshReturnsError(t *testing.T) {
|
||||
gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"code":50,"message":"refresh rejected"}`))
|
||||
}))
|
||||
defer gateway.Close()
|
||||
setChannelRefreshTestConfig(t, testServerAddress(gateway), "127.0.0.1:1")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
consumer := &Consumers{}
|
||||
if err := consumer.handleChannelRefresh(ctx, model.ChannelRefreshMessage{Type: "channel", ChanKey: "channel-a"}); err == nil {
|
||||
t.Fatal("refresh failure was acknowledged as success")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
type TrafficUsageHandler interface {
|
||||
RetryDaily(context.Context, model.TrafficUsageDailyRetryMessage) error
|
||||
RetryMonthly(context.Context, model.TrafficUsageMonthlyRetryMessage) error
|
||||
}
|
||||
|
||||
type WorkOrderHandler interface {
|
||||
AutoCloseResolved(context.Context, model.WorkOrderAutoCloseMessage) error
|
||||
}
|
||||
|
||||
type BalanceRechargeHandler interface {
|
||||
CheckRechargeTimeout(context.Context, string) error
|
||||
}
|
||||
|
||||
type BatchHandler interface {
|
||||
Handle(context.Context, batch.Message) error
|
||||
}
|
||||
|
||||
type PackageCustomerSyncHandler interface {
|
||||
SyncDeptPackages(context.Context, model.PackageCustomerSyncMessage) error
|
||||
}
|
||||
|
||||
type TrafficUsageMonthlyStartHandler interface {
|
||||
Produce(context.Context, model.TrafficUsageMonthlyStartMessage) error
|
||||
}
|
||||
|
||||
type TrafficUsageDailyStartHandler interface {
|
||||
Produce(context.Context, model.TrafficUsageDailyStartMessage) error
|
||||
}
|
||||
|
||||
type TrafficUsageMonthlyHistoryStartHandler interface {
|
||||
Produce(context.Context, model.TrafficUsageMonthlyHistoryStartMessage) error
|
||||
}
|
||||
|
||||
type SimStatusQueueStartHandler interface {
|
||||
Produce(context.Context, model.SimStatusQueueStartMessage) error
|
||||
}
|
||||
|
||||
type SimStatusQueueTaskHandler interface {
|
||||
EnqueueCard(context.Context, model.SimStatusQueueTaskMessage) error
|
||||
}
|
||||
|
||||
type ImeiQueueStartHandler interface {
|
||||
Produce(context.Context, model.ImeiQueueStartMessage) error
|
||||
}
|
||||
|
||||
// DataReviewQueryHandler 执行"手动同步上游结果"：只读补查既有 operationId。
|
||||
type DataReviewQueryHandler interface {
|
||||
QueryReviewBatch(ctx context.Context, batchNo string) (int, error)
|
||||
}
|
||||
|
||||
type Consumers struct {
|
||||
trafficUsage TrafficUsageHandler
|
||||
workOrder WorkOrderHandler
|
||||
balance BalanceRechargeHandler
|
||||
balanceRechargeRetryDelay time.Duration
|
||||
balanceRechargeMaxRetry int
|
||||
batch BatchHandler
|
||||
batchExecuteRetryDelay time.Duration
|
||||
batchExecuteMaxRetries int
|
||||
packageCustomerSync PackageCustomerSyncHandler
|
||||
trafficUsageMonthlyStart TrafficUsageMonthlyStartHandler
|
||||
trafficUsageDailyStart TrafficUsageDailyStartHandler
|
||||
trafficUsageMonthlyHistoryStart TrafficUsageMonthlyHistoryStartHandler
|
||||
simStatusQueueStart SimStatusQueueStartHandler
|
||||
simStatusQueueTask SimStatusQueueTaskHandler
|
||||
imeiQueueStart ImeiQueueStartHandler
|
||||
dataReview DataReviewQueryHandler
|
||||
}
|
||||
|
||||
// WithDataReviewQuery adds the manual "sync upstream result" consumer without
|
||||
// changing any existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithDataReviewQuery(handler DataReviewQueryHandler) *Consumers {
|
||||
c.dataReview = handler
|
||||
return c
|
||||
}
|
||||
|
||||
// WithDailyUsageStart adds the previous-day usage producer without changing any
|
||||
// existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithDailyUsageStart(start TrafficUsageDailyStartHandler) *Consumers {
|
||||
c.trafficUsageDailyStart = start
|
||||
return c
|
||||
}
|
||||
|
||||
// WithMonthlyHistoryStart adds the previous-month usage history producer without
|
||||
// changing any existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithMonthlyHistoryStart(start TrafficUsageMonthlyHistoryStartHandler) *Consumers {
|
||||
c.trafficUsageMonthlyHistoryStart = start
|
||||
return c
|
||||
}
|
||||
|
||||
// WithSimStatusQueue adds the independent status queue handlers without
|
||||
// changing any existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithSimStatusQueue(start SimStatusQueueStartHandler, task SimStatusQueueTaskHandler) *Consumers {
|
||||
c.simStatusQueueStart = start
|
||||
c.simStatusQueueTask = task
|
||||
return c
|
||||
}
|
||||
|
||||
// WithImeiQueue adds the independent IMEI queue handler without changing any
|
||||
// existing RabbitMQ consumer wiring.
|
||||
func (c *Consumers) WithImeiQueue(start ImeiQueueStartHandler) *Consumers {
|
||||
c.imeiQueueStart = start
|
||||
return c
|
||||
}
|
||||
|
||||
func New(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, batchHandlers ...BatchHandler) *Consumers {
|
||||
consumers := &Consumers{
|
||||
trafficUsage: trafficUsage,
|
||||
workOrder: workOrder,
|
||||
balance: balance,
|
||||
}
|
||||
if len(batchHandlers) > 0 {
|
||||
consumers.batch = batchHandlers[0]
|
||||
}
|
||||
return consumers
|
||||
}
|
||||
|
||||
func NewWithPackageCustomerSync(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, packageCustomerSync PackageCustomerSyncHandler, batchHandlers ...BatchHandler) *Consumers {
|
||||
consumers := New(trafficUsage, workOrder, balance, batchHandlers...)
|
||||
consumers.packageCustomerSync = packageCustomerSync
|
||||
return consumers
|
||||
}
|
||||
|
||||
func NewWithPackageCustomerSyncAndMonthlyUsage(trafficUsage TrafficUsageHandler, workOrder WorkOrderHandler, balance BalanceRechargeHandler, packageCustomerSync PackageCustomerSyncHandler, monthlyStart TrafficUsageMonthlyStartHandler, batchHandlers ...BatchHandler) *Consumers {
|
||||
consumers := NewWithPackageCustomerSync(trafficUsage, workOrder, balance, packageCustomerSync, batchHandlers...)
|
||||
consumers.trafficUsageMonthlyStart = monthlyStart
|
||||
return consumers
|
||||
}
|
||||
|
||||
func (c *Consumers) Start(ctx context.Context) error {
|
||||
initializers := c.initializers(g.Cfg().MustGet(ctx, "workerDiagnostic.enabled", false).Bool())
|
||||
for _, initialize := range initializers {
|
||||
if err := initialize(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) initializers(workerDiagnosticEnabled bool) []func(context.Context) error {
|
||||
initializers := []func(context.Context) error{
|
||||
c.initTrafficUsageDailyRetry,
|
||||
c.initTrafficUsageMonthlyRetry,
|
||||
c.initWorkOrderAutoClose,
|
||||
c.initBalanceRechargeTimeout,
|
||||
c.initBatchExecute,
|
||||
c.initChannelRefresh,
|
||||
c.initDataReviewQuery,
|
||||
}
|
||||
if c.packageCustomerSync != nil {
|
||||
initializers = append(initializers, c.initPackageCustomerSync)
|
||||
}
|
||||
if c.trafficUsageMonthlyStart != nil {
|
||||
initializers = append(initializers, c.initTrafficUsageMonthlyStart)
|
||||
}
|
||||
if c.trafficUsageDailyStart != nil {
|
||||
initializers = append(initializers, c.initTrafficUsageDailyStart)
|
||||
}
|
||||
if c.trafficUsageMonthlyHistoryStart != nil {
|
||||
initializers = append(initializers, c.initTrafficUsageMonthlyHistoryStart)
|
||||
}
|
||||
if c.simStatusQueueStart != nil && c.simStatusQueueTask != nil {
|
||||
initializers = append(initializers, c.initSimStatusQueueStart, c.initSimStatusQueueTask)
|
||||
}
|
||||
if c.imeiQueueStart != nil {
|
||||
initializers = append(initializers, c.initImeiQueueStart)
|
||||
}
|
||||
if workerDiagnosticEnabled {
|
||||
initializers = append(initializers, c.initWorkerDiagnostic)
|
||||
}
|
||||
return initializers
|
||||
}
|
||||
|
||||
func (c *Consumers) Stop(ctx context.Context) error {
|
||||
return mq.Close(ctx)
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
dailyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
|
||||
monthlyhistoryproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
|
||||
packagecustomerservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/packagecustomer"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
g.Log().SetWriter(io.Discard)
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
type trafficUsageHandlerStub struct {
|
||||
dailyCalls int
|
||||
monthlyCalls int
|
||||
dailyMessage model.TrafficUsageDailyRetryMessage
|
||||
monthMessage model.TrafficUsageMonthlyRetryMessage
|
||||
dailyErr error
|
||||
monthlyErr error
|
||||
}
|
||||
|
||||
func (s *trafficUsageHandlerStub) RetryDaily(_ context.Context, message model.TrafficUsageDailyRetryMessage) error {
|
||||
s.dailyCalls++
|
||||
s.dailyMessage = message
|
||||
return s.dailyErr
|
||||
}
|
||||
|
||||
func (s *trafficUsageHandlerStub) RetryMonthly(_ context.Context, message model.TrafficUsageMonthlyRetryMessage) error {
|
||||
s.monthlyCalls++
|
||||
s.monthMessage = message
|
||||
return s.monthlyErr
|
||||
}
|
||||
|
||||
type workOrderHandlerStub struct {
|
||||
calls int
|
||||
message model.WorkOrderAutoCloseMessage
|
||||
err error
|
||||
}
|
||||
|
||||
type balanceRechargeHandlerStub struct {
|
||||
calls int
|
||||
outTradeNo string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *balanceRechargeHandlerStub) CheckRechargeTimeout(_ context.Context, outTradeNo string) error {
|
||||
s.calls++
|
||||
s.outTradeNo = outTradeNo
|
||||
return s.err
|
||||
}
|
||||
|
||||
type batchHandlerStub struct {
|
||||
calls int
|
||||
message batch.Message
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *batchHandlerStub) Handle(_ context.Context, message batch.Message) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
type packageCustomerSyncHandlerStub struct {
|
||||
calls int
|
||||
message model.PackageCustomerSyncMessage
|
||||
err error
|
||||
}
|
||||
|
||||
type dailyStartHandlerStub struct {
|
||||
calls int
|
||||
message model.TrafficUsageDailyStartMessage
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *dailyStartHandlerStub) Produce(_ context.Context, message model.TrafficUsageDailyStartMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
type monthlyHistoryStartHandlerStub struct {
|
||||
calls int
|
||||
message model.TrafficUsageMonthlyHistoryStartMessage
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *monthlyHistoryStartHandlerStub) Produce(_ context.Context, message model.TrafficUsageMonthlyHistoryStartMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
func (s *packageCustomerSyncHandlerStub) SyncDeptPackages(_ context.Context, message model.PackageCustomerSyncMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
func (s *workOrderHandlerStub) AutoCloseResolved(_ context.Context, message model.WorkOrderAutoCloseMessage) error {
|
||||
s.calls++
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageDailyRetry(t *testing.T) {
|
||||
stubTrafficPublish(t)
|
||||
t.Run("invalid messages are dead lettered", func(t *testing.T) {
|
||||
messages := []model.TrafficUsageDailyRetryMessage{
|
||||
{},
|
||||
{SimKey: &model.SimKey{}, Card: model.TrafficUsageSimCard{Iccid: "card"}},
|
||||
{SimKey: &model.SimKey{ICCID: "sim"}},
|
||||
}
|
||||
for _, message := range messages {
|
||||
handler := &trafficUsageHandlerStub{}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageDailyRetry(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.dailyCalls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.dailyCalls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expectedErr := errors.New("daily failed")
|
||||
expected := model.TrafficUsageDailyRetryMessage{
|
||||
SimKey: &model.SimKey{ICCID: "sim"},
|
||||
Card: model.TrafficUsageSimCard{Iccid: "sim", PackageCode: "package"},
|
||||
QueryDate: "20260714",
|
||||
}
|
||||
handler := &trafficUsageHandlerStub{dailyErr: expectedErr}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageDailyRetry(context.Background(), expected); err != nil {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.dailyCalls != 1 || !reflect.DeepEqual(handler.dailyMessage, expected) {
|
||||
t.Fatalf("daily message was not delegated exactly once: %+v", handler.dailyMessage)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageMonthlyRetry(t *testing.T) {
|
||||
stubTrafficPublish(t)
|
||||
t.Run("invalid messages are dead lettered", func(t *testing.T) {
|
||||
messages := []model.TrafficUsageMonthlyRetryMessage{
|
||||
{},
|
||||
{SimKey: &model.SimKey{}, Card: model.TrafficUsageSimCard{Iccid: "card"}},
|
||||
{SimKey: &model.SimKey{ICCID: "sim"}},
|
||||
}
|
||||
for _, message := range messages {
|
||||
handler := &trafficUsageHandlerStub{}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageMonthlyRetry(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.monthlyCalls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.monthlyCalls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expectedErr := errors.New("monthly failed")
|
||||
expected := model.TrafficUsageMonthlyRetryMessage{
|
||||
SimKey: &model.SimKey{ICCID: "sim"},
|
||||
Card: model.TrafficUsageSimCard{Iccid: "sim", PackageCode: "package"},
|
||||
QueryDate: "202607",
|
||||
}
|
||||
handler := &trafficUsageHandlerStub{monthlyErr: expectedErr}
|
||||
consumers := New(handler, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleTrafficUsageMonthlyRetry(context.Background(), expected); err != nil {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.monthlyCalls != 1 || !reflect.DeepEqual(handler.monthMessage, expected) {
|
||||
t.Fatalf("monthly message was not delegated exactly once: %+v", handler.monthMessage)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func stubTrafficPublish(t *testing.T) {
|
||||
t.Helper()
|
||||
original, delayed := trafficUsagePublish, trafficUsagePublishDelayed
|
||||
t.Cleanup(func() { trafficUsagePublish, trafficUsagePublishDelayed = original, delayed })
|
||||
trafficUsagePublish = func(context.Context, string, []byte, amqp.Table) error { return nil }
|
||||
trafficUsagePublishDelayed = func(context.Context, string, string, time.Duration, []byte, amqp.Table) error { return nil }
|
||||
}
|
||||
|
||||
func TestHandleWorkOrderAutoClose(t *testing.T) {
|
||||
handler := &workOrderHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, handler, &balanceRechargeHandlerStub{})
|
||||
if err := consumers.handleWorkOrderAutoClose(context.Background(), model.WorkOrderAutoCloseMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.calls)
|
||||
}
|
||||
|
||||
expectedErr := errors.New("work order failed")
|
||||
expected := model.WorkOrderAutoCloseMessage{WoNo: "WO-1", ResolveTimeTs: 1}
|
||||
handler.err = expectedErr
|
||||
if err := consumers.handleWorkOrderAutoClose(context.Background(), expected); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("work order message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBalanceRechargeTimeout(t *testing.T) {
|
||||
handler := &balanceRechargeHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, handler)
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), model.BalanceRechargeTimeoutMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.calls)
|
||||
}
|
||||
|
||||
message := model.BalanceRechargeTimeoutMessage{OutTradeNo: "BAL-1"}
|
||||
if err := consumers.handleBalanceRechargeTimeout(context.Background(), message); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || handler.outTradeNo != message.OutTradeNo {
|
||||
t.Fatalf("balance recharge message was not delegated exactly once: %+v", handler)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBatchExecute(t *testing.T) {
|
||||
expectedErr := errors.New("batch failed")
|
||||
expected := batch.Message{BatchNo: "BATCH-1", BatchType: batch.Type103}
|
||||
handler := &batchHandlerStub{err: expectedErr}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
|
||||
if err := consumers.handleBatchExecute(context.Background(), expected); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("batch message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePackageCustomerSync(t *testing.T) {
|
||||
t.Run("invalid message is dropped", func(t *testing.T) {
|
||||
handler := &packageCustomerSyncHandlerStub{}
|
||||
consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
if err := consumers.handlePackageCustomerSync(context.Background(), model.PackageCustomerSyncMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called handler %d times", handler.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing dept is dropped", func(t *testing.T) {
|
||||
expected := model.PackageCustomerSyncMessage{TenantId: "000000", DeptId: 164, Trigger: "package_lifecycle", SourceBatchNo: "898604017"}
|
||||
handler := &packageCustomerSyncHandlerStub{err: packagecustomerservice.ErrDeptNotFound}
|
||||
consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
if err := consumers.handlePackageCustomerSync(context.Background(), expected); err != nil {
|
||||
t.Fatalf("missing dept returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("package customer sync message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("other errors are retried", func(t *testing.T) {
|
||||
expectedErr := errors.New("database failed")
|
||||
expected := model.PackageCustomerSyncMessage{TenantId: "000000", DeptId: 117}
|
||||
handler := &packageCustomerSyncHandlerStub{err: expectedErr}
|
||||
consumers := NewWithPackageCustomerSync(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}, handler)
|
||||
if err := consumers.handlePackageCustomerSync(context.Background(), expected); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected delegated error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestHandleWorkerDiagnostic(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
|
||||
if err := consumers.handleWorkerDiagnostic(context.Background(), model.WorkerDiagnosticMessage{}); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
|
||||
message := model.WorkerDiagnosticMessage{
|
||||
TraceID: "worker-test-1",
|
||||
SentAt: 1784030000000,
|
||||
Message: "worker ping",
|
||||
}
|
||||
if err := consumers.handleWorkerDiagnostic(context.Background(), message); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDiagnosticMessageJSONContract(t *testing.T) {
|
||||
message := model.WorkerDiagnosticMessage{
|
||||
TraceID: "worker-test-1",
|
||||
SentAt: 1784030000000,
|
||||
Message: "worker ping",
|
||||
}
|
||||
body, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal diagnostic message: %v", err)
|
||||
}
|
||||
const expected = `{"traceId":"worker-test-1","sentAt":1784030000000,"message":"worker ping"}`
|
||||
if string(body) != expected {
|
||||
t.Fatalf("unexpected diagnostic JSON: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelRefreshMessageJSONContract(t *testing.T) {
|
||||
message := model.ChannelRefreshMessage{
|
||||
Type: "channel_rename",
|
||||
OldChanKey: "old_channel",
|
||||
NewChanKey: "new_channel",
|
||||
}
|
||||
body, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal channel rename message: %v", err)
|
||||
}
|
||||
const expected = `{"type":"channel_rename","oldChanKey":"old_channel","newChanKey":"new_channel"}`
|
||||
if string(body) != expected {
|
||||
t.Fatalf("unexpected channel rename JSON: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleChannelRefreshDropsInvalidRename(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
for _, message := range []model.ChannelRefreshMessage{
|
||||
{Type: "channel_rename"},
|
||||
{Type: "channel_rename", OldChanKey: "old_channel"},
|
||||
{Type: "channel_rename", NewChanKey: "new_channel"},
|
||||
{Type: "channel_rename", OldChanKey: "same", NewChanKey: "same"},
|
||||
} {
|
||||
if err := consumers.handleChannelRefresh(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid channel rename returned error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleChannelRefreshRenameFailureReturnsError(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
message := model.ChannelRefreshMessage{
|
||||
Type: "channel_rename",
|
||||
OldChanKey: "old_channel",
|
||||
NewChanKey: "new_channel",
|
||||
}
|
||||
// ctx 已取消时 renameChannelWithRetry 会在首次重试等待处返回 ctx.Err()，
|
||||
// 消费者必须把错误向上抛出，让框架 Nack 重新入队而不是 Ack 丢弃。
|
||||
if err := consumers.handleChannelRefresh(ctx, message); err == nil {
|
||||
t.Fatalf("expected failed channel rename to return error for Nack+requeue")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageDailyStart(t *testing.T) {
|
||||
t.Run("invalid message is dropped", func(t *testing.T) {
|
||||
for _, message := range []model.TrafficUsageDailyStartMessage{
|
||||
{},
|
||||
{RunID: "run-1"},
|
||||
{QueryDate: "20261009"},
|
||||
} {
|
||||
handler := &dailyStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called producer %d times", handler.calls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expected := model.TrafficUsageDailyStartMessage{
|
||||
RunID: "daily-usage-20261009-abc",
|
||||
Operation: model.OperationPreviousDayUsageFullSync,
|
||||
QueryDate: "20261009",
|
||||
TriggeredAt: "2026-10-10T06:00:00+08:00",
|
||||
Scope: &model.SyncScope{Channels: []string{"217"}},
|
||||
}
|
||||
handler := &dailyStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), expected); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("daily start message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an already running producer is acked", func(t *testing.T) {
|
||||
handler := &dailyStartHandlerStub{err: dailyusageproducer.ErrAlreadyRunning}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); err != nil {
|
||||
t.Fatalf("ErrAlreadyRunning must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a finished run is acked instead of being resurrected", func(t *testing.T) {
|
||||
handler := &dailyStartHandlerStub{err: dailyusageproducer.ErrRunFinished}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); err != nil {
|
||||
t.Fatalf("ErrRunFinished must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("other errors are surfaced for requeue", func(t *testing.T) {
|
||||
expectedErr := errors.New("redis unavailable")
|
||||
handler := &dailyStartHandlerStub{err: expectedErr}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(handler)
|
||||
if err := consumers.handleTrafficUsageDailyStart(context.Background(), model.TrafficUsageDailyStartMessage{RunID: "r", QueryDate: "20261009"}); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected the producer error to propagate, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestDailyUsageStartConsumerIsOptIn pins the assembly contract: without an
|
||||
// explicit WithDailyUsageStart the initializer list must be unchanged, so a
|
||||
// disabled daily chain cannot alter existing deployments.
|
||||
func TestDailyUsageStartConsumerIsOptIn(t *testing.T) {
|
||||
base := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
withDaily := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithDailyUsageStart(&dailyStartHandlerStub{})
|
||||
if got, want := len(withDaily.initializers(false)), len(base.initializers(false))+1; got != want {
|
||||
t.Fatalf("initializers = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTrafficUsageMonthlyHistoryStart(t *testing.T) {
|
||||
t.Run("invalid message is dropped", func(t *testing.T) {
|
||||
for _, message := range []model.TrafficUsageMonthlyHistoryStartMessage{
|
||||
{},
|
||||
{RunID: "run-1"},
|
||||
{QueryMonth: "202609"},
|
||||
} {
|
||||
handler := &monthlyHistoryStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), message); err != nil {
|
||||
t.Fatalf("invalid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 0 {
|
||||
t.Fatalf("invalid message called producer %d times", handler.calls)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid message is delegated", func(t *testing.T) {
|
||||
expected := model.TrafficUsageMonthlyHistoryStartMessage{
|
||||
RunID: "monthly-history-202609-abc",
|
||||
Operation: model.OperationPreviousMonthUsageFullSync,
|
||||
QueryMonth: "202609",
|
||||
TriggeredAt: "2026-10-01T06:00:00+08:00",
|
||||
Scope: &model.SyncScope{Channels: []string{"217"}},
|
||||
}
|
||||
handler := &monthlyHistoryStartHandlerStub{}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), expected); err != nil {
|
||||
t.Fatalf("valid message returned error: %v", err)
|
||||
}
|
||||
if handler.calls != 1 || !reflect.DeepEqual(handler.message, expected) {
|
||||
t.Fatalf("monthly history start message was not delegated exactly once: %+v", handler.message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an already running producer is acked", func(t *testing.T) {
|
||||
handler := &monthlyHistoryStartHandlerStub{err: monthlyhistoryproducer.ErrAlreadyRunning}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); err != nil {
|
||||
t.Fatalf("ErrAlreadyRunning must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a finished run is acked instead of being resurrected", func(t *testing.T) {
|
||||
handler := &monthlyHistoryStartHandlerStub{err: monthlyhistoryproducer.ErrRunFinished}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); err != nil {
|
||||
t.Fatalf("ErrRunFinished must be acked, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("other errors are surfaced for requeue", func(t *testing.T) {
|
||||
expectedErr := errors.New("redis unavailable")
|
||||
handler := &monthlyHistoryStartHandlerStub{err: expectedErr}
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(handler)
|
||||
if err := consumers.handleTrafficUsageMonthlyHistoryStart(context.Background(), model.TrafficUsageMonthlyHistoryStartMessage{RunID: "r", QueryMonth: "202609"}); !errors.Is(err, expectedErr) {
|
||||
t.Fatalf("expected the producer error to propagate, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMonthlyHistoryStartConsumerIsOptIn mirrors the daily chain: a disabled
|
||||
// monthly-history chain must not add an initializer.
|
||||
func TestMonthlyHistoryStartConsumerIsOptIn(t *testing.T) {
|
||||
base := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
withHistory := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{}).WithMonthlyHistoryStart(&monthlyHistoryStartHandlerStub{})
|
||||
if got, want := len(withHistory.initializers(false)), len(base.initializers(false))+1; got != want {
|
||||
t.Fatalf("initializers = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsumerInitializersRespectWorkerDiagnosticSwitch(t *testing.T) {
|
||||
consumers := New(&trafficUsageHandlerStub{}, &workOrderHandlerStub{}, &balanceRechargeHandlerStub{})
|
||||
// 手动同步上游结果（initDataReviewQuery）随业务消费者常驻，无处理器时自跳过。
|
||||
if got := len(consumers.initializers(false)); got != 7 {
|
||||
t.Fatalf("diagnostic disabled: expected 7 business consumers, got %d", got)
|
||||
}
|
||||
if got := len(consumers.initializers(true)); got != 8 {
|
||||
t.Fatalf("diagnostic enabled: expected 8 consumers, got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initDataReviewQuery(ctx context.Context) error {
|
||||
if c.dataReview == nil {
|
||||
return nil
|
||||
}
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "data review query consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.DataReviewQueryMessage](
|
||||
ctx,
|
||||
consts.QueueDataReviewQuery,
|
||||
"iotlcm-data-review-query",
|
||||
c.handleDataReviewQuery,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "data review query consumer started, queue:%s", consts.QueueDataReviewQuery)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleDataReviewQuery(ctx context.Context, message model.DataReviewQueryMessage) error {
|
||||
if c.dataReview == nil || message.BatchNo == "" {
|
||||
g.Log().Errorf(ctx, "data review query message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
queried, err := c.dataReview.QueryReviewBatch(ctx, message.BatchNo)
|
||||
if err != nil {
|
||||
// 手动同步失败只记日志，不重投：重投也不会改变上游结果，且可能放大查询压力。
|
||||
g.Log().Errorf(ctx, "手动同步APN上游结果失败 batchNo=%s 已同步=%d error=%v", message.BatchNo, queried, err)
|
||||
return nil
|
||||
}
|
||||
g.Log().Infof(ctx, "手动同步APN上游结果完成 batchNo=%s 已同步=%d", message.BatchNo, queried)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initWorkerDiagnostic(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "worker diagnostic consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.WorkerDiagnosticMessage](
|
||||
ctx,
|
||||
consts.QueueWorkerDiagnostic,
|
||||
"iotlcm-worker-diagnostic",
|
||||
c.handleWorkerDiagnostic,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "worker diagnostic consumer started, queue:%s", consts.QueueWorkerDiagnostic)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleWorkerDiagnostic(ctx context.Context, message model.WorkerDiagnosticMessage) error {
|
||||
if message.TraceID == "" {
|
||||
g.Log().Errorf(ctx, "worker diagnostic message dropped, invalid traceId, message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
g.Log().Infof(
|
||||
ctx,
|
||||
"worker diagnostic message consumed, traceId:%s, sentAt:%d, message:%s",
|
||||
message.TraceID,
|
||||
message.SentAt,
|
||||
message.Message,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
imeiproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/imei"
|
||||
)
|
||||
|
||||
func (c *Consumers) initImeiQueueStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "imei queue start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.ImeiQueueStartMessage](ctx, consts.QueueImeiQueueStart, "iotlcm-imei-queue-start", c.handleImeiQueueStart); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "imei queue start consumer started, queue:%s", consts.QueueImeiQueueStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleImeiQueueStart(ctx context.Context, message model.ImeiQueueStartMessage) error {
|
||||
if message.RunID == "" {
|
||||
g.Log().Errorf(ctx, "imei queue start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.imeiQueueStart.Produce(ctx, message)
|
||||
if errors.Is(err, imeiproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "imei queue start skipped because a run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, imeiproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "imei queue start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
packagecustomerservice "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/packagecustomer"
|
||||
)
|
||||
|
||||
func (c *Consumers) initPackageCustomerSync(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "package customer sync consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if c.packageCustomerSync == nil {
|
||||
return gerror.New("package customer sync handler is not configured")
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.PackageCustomerSyncMessage](
|
||||
ctx,
|
||||
consts.QueuePackageCustomerSync,
|
||||
"iotlcm-package-customer-sync",
|
||||
c.handlePackageCustomerSync,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "package customer sync consumer started, queue:%s", consts.QueuePackageCustomerSync)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handlePackageCustomerSync(ctx context.Context, message model.PackageCustomerSyncMessage) error {
|
||||
if c.packageCustomerSync == nil {
|
||||
return gerror.New("package customer sync handler is not configured")
|
||||
}
|
||||
if message.TenantId == "" || message.DeptId <= 0 {
|
||||
g.Log().Errorf(ctx, "package customer sync message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
if err := c.packageCustomerSync.SyncDeptPackages(ctx, message); err != nil {
|
||||
if errors.Is(err, packagecustomerservice.ErrDeptNotFound) {
|
||||
g.Log().Warningf(ctx, "package customer sync message dropped, dept not found, tenantId:%s, deptId:%d, trigger:%s, sourceBatchNo:%s",
|
||||
message.TenantId, message.DeptId, message.Trigger, message.SourceBatchNo)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
simstatusproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/simstatus"
|
||||
)
|
||||
|
||||
func (c *Consumers) initSimStatusQueueStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "sim status queue start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.SimStatusQueueStartMessage](ctx, consts.QueueSimStatusQueueStart, "iotlcm-sim-status-queue-start", c.handleSimStatusQueueStart); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "sim status queue start consumer started, queue:%s", consts.QueueSimStatusQueueStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleSimStatusQueueStart(ctx context.Context, message model.SimStatusQueueStartMessage) error {
|
||||
if message.RunID == "" {
|
||||
g.Log().Errorf(ctx, "sim status queue start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.simStatusQueueStart.Produce(ctx, message)
|
||||
if errors.Is(err, simstatusproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "sim status queue start skipped because a run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, simstatusproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "sim status queue start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initSimStatusQueueTask(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "sim status queue task consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.SimStatusQueueTaskMessage](ctx, consts.QueueSimStatusQueueTask, "iotlcm-sim-status-queue-task", c.handleSimStatusQueueTask); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "sim status queue task consumer started, queue:%s", consts.QueueSimStatusQueueTask)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleSimStatusQueueTask(ctx context.Context, message model.SimStatusQueueTaskMessage) error {
|
||||
if message.CardID <= 0 || message.ICCID == "" || message.ChanKey == "" || message.TempKey == "" {
|
||||
g.Log().Errorf(ctx, "sim status queue task dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
return c.simStatusQueueTask.EnqueueCard(ctx, message)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageDailyRetry(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage daily retry consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageDailyRetryMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageDailyRetry,
|
||||
"iotlcm-traffic-usage-daily",
|
||||
c.handleTrafficUsageDailyRetry,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage daily retry consumer started, queue:%s", consts.QueueTrafficUsageDailyRetry)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageDailyRetry(ctx context.Context, message model.TrafficUsageDailyRetryMessage) error {
|
||||
err := validateTrafficUsageIdentity(message.SimKey, message.Card)
|
||||
if err == nil {
|
||||
err = c.trafficUsage.RetryDaily(ctx, message)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
message.Attempt++
|
||||
message.LastError = err.Error()
|
||||
permanent := trafficUsageFailureSnapshot(err, &message.UsageMb)
|
||||
return publishTrafficUsageFailure(ctx, consts.QueueTrafficUsageDailyRetry, message, message.Attempt, permanent)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
dailyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/dailyusage"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageDailyStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage daily start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageDailyStartMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageDailyStart,
|
||||
"iotlcm-traffic-usage-daily-start",
|
||||
c.handleTrafficUsageDailyStart,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage daily start consumer started, queue:%s", consts.QueueTrafficUsageDailyStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageDailyStart(ctx context.Context, message model.TrafficUsageDailyStartMessage) error {
|
||||
if message.RunID == "" || message.QueryDate == "" {
|
||||
g.Log().Errorf(ctx, "traffic usage daily start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.trafficUsageDailyStart.Produce(ctx, message)
|
||||
if errors.Is(err, dailyusageproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "traffic usage daily start skipped because previous run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, dailyusageproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "traffic usage daily start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageMonthlyRetry(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage monthly retry consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageMonthlyRetryMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageMonthlyRetry,
|
||||
"iotlcm-traffic-usage-monthly",
|
||||
c.handleTrafficUsageMonthlyRetry,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage monthly retry consumer started, queue:%s", consts.QueueTrafficUsageMonthlyRetry)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageMonthlyRetry(ctx context.Context, message model.TrafficUsageMonthlyRetryMessage) error {
|
||||
err := validateTrafficUsageIdentity(message.SimKey, message.Card)
|
||||
if err == nil {
|
||||
err = c.trafficUsage.RetryMonthly(ctx, message)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
message.Attempt++
|
||||
message.LastError = err.Error()
|
||||
permanent := trafficUsageFailureSnapshot(err, &message.UsageMb)
|
||||
return publishTrafficUsageFailure(ctx, consts.QueueTrafficUsageMonthlyRetry, message, message.Attempt, permanent)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
monthlyhistoryproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusagehistory"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageMonthlyHistoryStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage monthly history start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageMonthlyHistoryStartMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageMonthlyHistoryStart,
|
||||
"iotlcm-traffic-usage-monthly-history-start",
|
||||
c.handleTrafficUsageMonthlyHistoryStart,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage monthly history start consumer started, queue:%s", consts.QueueTrafficUsageMonthlyHistoryStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageMonthlyHistoryStart(ctx context.Context, message model.TrafficUsageMonthlyHistoryStartMessage) error {
|
||||
if message.RunID == "" || message.QueryMonth == "" {
|
||||
g.Log().Errorf(ctx, "traffic usage monthly history start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.trafficUsageMonthlyHistoryStart.Produce(ctx, message)
|
||||
if errors.Is(err, monthlyhistoryproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly history start skipped because previous run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, monthlyhistoryproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly history start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
monthlyusageproducer "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/producer/monthlyusage"
|
||||
)
|
||||
|
||||
func (c *Consumers) initTrafficUsageMonthlyStart(ctx context.Context) error {
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "traffic usage monthly start consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.TrafficUsageMonthlyStartMessage](
|
||||
ctx,
|
||||
consts.QueueTrafficUsageMonthlyStart,
|
||||
"iotlcm-traffic-usage-monthly-start",
|
||||
c.handleTrafficUsageMonthlyStart,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "traffic usage monthly start consumer started, queue:%s", consts.QueueTrafficUsageMonthlyStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleTrafficUsageMonthlyStart(ctx context.Context, message model.TrafficUsageMonthlyStartMessage) error {
|
||||
if message.RunID == "" || message.QueryMonth == "" {
|
||||
g.Log().Errorf(ctx, "traffic usage monthly start message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
err := c.trafficUsageMonthlyStart.Produce(ctx, message)
|
||||
if errors.Is(err, monthlyusageproducer.ErrAlreadyRunning) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly start skipped because previous run is active, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
// A run that already finished must not be resurrected by a redelivered
|
||||
// message. Ack it instead of retrying forever.
|
||||
if errors.Is(err, monthlyusageproducer.ErrRunFinished) {
|
||||
g.Log().Warningf(ctx, "traffic usage monthly start skipped because run already finished, runId:%s", message.RunID)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficusage"
|
||||
)
|
||||
|
||||
var trafficUsagePublish = mq.PublishBytesWithHeaders
|
||||
var trafficUsagePublishDelayed = mq.PublishDelayedBytesWithHeaders
|
||||
|
||||
func validateTrafficUsageIdentity(simKey *model.SimKey, card model.TrafficUsageSimCard) error {
|
||||
if simKey == nil || card.Iccid == "" || simKey.ICCID != card.Iccid {
|
||||
return &trafficusage.Failure{Cause: gerror.New("invalid traffic usage card identity"), Permanent: true}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func trafficUsageFailureSnapshot(err error, usage **float64) bool {
|
||||
var failure *trafficusage.Failure
|
||||
if errors.As(err, &failure) {
|
||||
if failure.UsageMb != nil {
|
||||
*usage = failure.UsageMb
|
||||
}
|
||||
return failure.Permanent
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func publishTrafficUsageFailure(ctx context.Context, queue string, message any, attempt int, permanent bool) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
body, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if permanent || attempt >= 10 {
|
||||
g.Log().Errorf(ctx, "traffic usage retry exhausted or invalid, queue:%s, attempt:%d, payload:%s", queue, attempt, body)
|
||||
err = trafficUsagePublish(ctx, queue+".dlq", body, nil)
|
||||
return waitTrafficUsagePublishFailure(ctx, err)
|
||||
}
|
||||
delay := 30 * time.Second
|
||||
if attempt >= 3 {
|
||||
delay = 2 * time.Minute
|
||||
}
|
||||
if attempt >= 6 {
|
||||
delay = 5 * time.Minute
|
||||
}
|
||||
err = trafficUsagePublishDelayed(ctx, queue+".delay."+delay.String(), queue, delay, body, nil)
|
||||
return waitTrafficUsagePublishFailure(ctx, err)
|
||||
}
|
||||
|
||||
func waitTrafficUsagePublishFailure(ctx context.Context, err error) error {
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/trafficusage"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTrafficUsageRetryRetainsPayload(t *testing.T) {
|
||||
stubTrafficPublish(t)
|
||||
usage := 12.5
|
||||
handler := &trafficUsageHandlerStub{dailyErr: &trafficusage.Failure{Cause: errors.New("database down"), UsageMb: &usage}}
|
||||
consumer := New(handler, nil, nil)
|
||||
message := model.TrafficUsageDailyRetryMessage{SimKey: &model.SimKey{ICCID: "card"}, Card: model.TrafficUsageSimCard{Iccid: "card"}, QueryDate: "20260831"}
|
||||
calls := 0
|
||||
trafficUsagePublishDelayed = func(_ context.Context, queue, target string, delay time.Duration, body []byte, _ amqp.Table) error {
|
||||
calls++
|
||||
var retry model.TrafficUsageDailyRetryMessage
|
||||
if err := json.Unmarshal(body, &retry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retry.Attempt != 1 || retry.UsageMb == nil || *retry.UsageMb != usage || retry.QueryDate != message.QueryDate || retry.LastError == "" || delay != 30*time.Second || target != consts.QueueTrafficUsageDailyRetry || queue == target {
|
||||
t.Fatalf("invalid retry: %+v", retry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); err != nil || calls != 1 {
|
||||
t.Fatalf("calls=%d err=%v", calls, err)
|
||||
}
|
||||
message.Attempt = 9
|
||||
dlqCalls := 0
|
||||
trafficUsagePublish = func(_ context.Context, queue string, body []byte, _ amqp.Table) error {
|
||||
dlqCalls++
|
||||
if queue != consts.QueueTrafficUsageDailyRetry+".dlq" {
|
||||
t.Fatal(queue)
|
||||
}
|
||||
var retry model.TrafficUsageDailyRetryMessage
|
||||
if err := json.Unmarshal(body, &retry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retry.Attempt != 10 || retry.UsageMb == nil {
|
||||
t.Fatal("DLQ lost snapshot")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); err != nil || dlqCalls != 1 || calls != 1 {
|
||||
t.Fatalf("DLQ not published: %v", err)
|
||||
}
|
||||
expected := errors.New("broker unavailable")
|
||||
trafficUsagePublish = func(context.Context, string, []byte, amqp.Table) error { return expected }
|
||||
if err := consumer.handleTrafficUsageDailyRetry(context.Background(), message); !errors.Is(err, expected) {
|
||||
t.Fatal("failed publication must not ack original")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
mq "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
func (c *Consumers) initWorkOrderAutoClose(ctx context.Context) error {
|
||||
if scanner, ok := c.workOrder.(interface{ CloseOverdue(context.Context) error }); ok {
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if err := scanner.CloseOverdue(ctx); err != nil {
|
||||
g.Log().Errorf(ctx, "工单到期补偿失败: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
if !mq.Enabled(ctx) {
|
||||
g.Log().Info(ctx, "work order auto close consumer skipped, RabbitMQ disabled")
|
||||
return nil
|
||||
}
|
||||
if err := mq.ConsumeJSON[model.WorkOrderAutoCloseMessage](
|
||||
ctx,
|
||||
consts.QueueWorkOrderAutoClose,
|
||||
"iotlcm-work-order-auto-close",
|
||||
c.handleWorkOrderAutoClose,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "work order auto close consumer started, queue:%s", consts.QueueWorkOrderAutoClose)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Consumers) handleWorkOrderAutoClose(ctx context.Context, message model.WorkOrderAutoCloseMessage) error {
|
||||
if message.WoNo == "" {
|
||||
g.Log().Errorf(ctx, "work order auto close message dropped, invalid message:%+v", message)
|
||||
return nil
|
||||
}
|
||||
return c.workOrder.AutoCloseResolved(ctx, message)
|
||||
}
|
||||
Reference in New Issue
Block a user