chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,806 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gcfg"
|
||||
"github.com/xuri/excelize/v2"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/batch"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
adapter, err := gcfg.NewAdapterContent("batch:\n inbound:\n upstreamBatchSize: 1\n")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
g.Cfg().SetAdapter(adapter)
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestAdminSourceTypeContract(t *testing.T) {
|
||||
if consts.BatchSourceTypeRange != 1 || consts.BatchSourceTypeBatchInput != 2 || consts.BatchSourceTypeFile != 3 {
|
||||
t.Fatalf("source_type contract mismatch: range=%d input=%d file=%d",
|
||||
consts.BatchSourceTypeRange, consts.BatchSourceTypeBatchInput, consts.BatchSourceTypeFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCascadeFailurePreservesPerCardValidation(t *testing.T) {
|
||||
task := &Task{Rows: []*Row{
|
||||
{ICCID: "invalid", Reason: "卡号不合法"},
|
||||
{ICCID: "valid"},
|
||||
}}
|
||||
applyCascadeFailure(task)
|
||||
if countFailedRows(task.Rows) != 1 || !IsCascadeFailure(task.Rows[1]) {
|
||||
t.Fatalf("unexpected validation outcomes: %+v", task.Rows)
|
||||
}
|
||||
if task.Rows[0].Reason != "卡号不合法" || inboundRowStatus(task.Rows[0]) != "失败" || inboundRowStatus(task.Rows[1]) != "未执行" {
|
||||
t.Fatalf("wrong per-card status: %+v", task.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCollectsLaterInboundFailuresBeforeBlockingBatch(t *testing.T) {
|
||||
task := &Task{Rows: []*Row{
|
||||
{ICCID: "bad", ChanKey: "channel"},
|
||||
{ICCID: "8986000000000000001", MSISDN: "13800008000", IMSI: "460001234567890", ChanKey: "channel"},
|
||||
{ICCID: "8986000000000000002", MSISDN: "invalid", IMSI: "invalid", ChanKey: "channel"},
|
||||
}}
|
||||
executor := &executor{repository: &inboundRepositoryStub{}}
|
||||
if err := executor.validate(context.Background(), task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyCascadeFailure(task)
|
||||
if !strings.Contains(task.Rows[0].Reason, "ICCID格式不正确") {
|
||||
t.Fatalf("first failure was lost: %q", task.Rows[0].Reason)
|
||||
}
|
||||
if !IsCascadeFailure(task.Rows[1]) {
|
||||
t.Fatalf("validated card was not blocked: %q", task.Rows[1].Reason)
|
||||
}
|
||||
for _, reason := range []string{"MSISDN格式不正确", "IMSI格式不正确"} {
|
||||
if !strings.Contains(task.Rows[2].Reason, reason) {
|
||||
t.Fatalf("later card was not fully validated: %q", task.Rows[2].Reason)
|
||||
}
|
||||
}
|
||||
if countFailedRows(task.Rows) != 2 || countValidRows(task.Rows) != 0 {
|
||||
t.Fatalf("batch must have two validation failures and no executable cards: %+v", task)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRowRequiresAtLeastOneCardIdentifierAndChanKey(t *testing.T) {
|
||||
row := &Row{}
|
||||
validateRow(row, "")
|
||||
|
||||
for _, want := range []string{"ICCID/MSISDN/IMSI至少填写一个", "chanKey不能为空"} {
|
||||
if !containsReason(row.Reason, want) {
|
||||
t.Fatalf("expected reason %q in %q", want, row.Reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInputTextAcceptsOneIdentifierPerLine(t *testing.T) {
|
||||
rows := parseInputText(strings.Join([]string{
|
||||
"89860401786527567308",
|
||||
"14100007308",
|
||||
"460046527567308",
|
||||
}, "\n"))
|
||||
|
||||
if len(rows) != 3 {
|
||||
t.Fatalf("row count = %d, want 3", len(rows))
|
||||
}
|
||||
if rows[0].ICCID != "89860401786527567308" || rows[1].MSISDN != "14100007308" || rows[2].IMSI != "460046527567308" {
|
||||
t.Fatalf("unexpected parsed rows: %+v", rows)
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.InputCode == "" {
|
||||
t.Fatalf("expected input code to be recorded: %+v", row)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRowRejectsTooShortIdentifiers(t *testing.T) {
|
||||
row := &Row{ICCID: "4", MSISDN: "4", IMSI: "4", ChanKey: "chan-a"}
|
||||
validateRow(row, "")
|
||||
|
||||
for _, want := range []string{"ICCID格式不正确", "MSISDN格式不正确", "IMSI格式不正确"} {
|
||||
if !containsReason(row.Reason, want) {
|
||||
t.Fatalf("expected reason %q in %q", want, row.Reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandRangeUsesSelectedMSISDN(t *testing.T) {
|
||||
rows, err := expandRange(Range{RangeType: "msisdn", StartMSISDN: "13800008000", EndMSISDN: "13800008001"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 2 || rows[0].MSISDN != "13800008000" || rows[1].MSISDN != "13800008001" {
|
||||
t.Fatalf("unexpected MSISDN rows: %+v", rows)
|
||||
}
|
||||
if rows[0].ICCID != "" || rows[0].IMSI != "" {
|
||||
t.Fatalf("unselected identifiers must remain empty: %+v", rows[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandRangeAllowsAlphaNumericICCIDRange(t *testing.T) {
|
||||
rows, err := expandRange(Range{RangeType: "iccid", StartICCID: "898604F000000000001", EndICCID: "898604F000000000003"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"898604F000000000001",
|
||||
"898604F000000000002",
|
||||
"898604F000000000003",
|
||||
}
|
||||
if len(rows) != len(want) {
|
||||
t.Fatalf("row count = %d, want %d", len(rows), len(want))
|
||||
}
|
||||
for i, value := range want {
|
||||
if rows[i].ICCID != value {
|
||||
t.Fatalf("row %d ICCID = %q, want %q", i, rows[i].ICCID, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandRangeRejectsAlphaNumericMSISDNRange(t *testing.T) {
|
||||
_, err := expandRange(Range{RangeType: "msisdn", StartMSISDN: "1380013800A", EndMSISDN: "1380013800B"})
|
||||
if err == nil || !strings.Contains(err.Error(), "range values must be numeric") {
|
||||
t.Fatalf("expected numeric range error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRowSelectedMSISDNDoesNotRequireOtherIdentifiers(t *testing.T) {
|
||||
row := &Row{MSISDN: "13800008000", ChanKey: "chan-a"}
|
||||
validateRow(row, "msisdn")
|
||||
if row.Reason != "" {
|
||||
t.Fatalf("unexpected validation reason: %q", row.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRowAllowsAlphaNumericICCID(t *testing.T) {
|
||||
row := &Row{ICCID: "898604F4152391115473", MSISDN: "13800008000", IMSI: "460001234567890", ChanKey: "chan-a"}
|
||||
validateRow(row, "")
|
||||
if row.Reason != "" {
|
||||
t.Fatalf("unexpected validation reason: %q", row.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkSameBatchDuplicatesChecksICCIDAndMSISDN(t *testing.T) {
|
||||
rows := []*Row{
|
||||
{RowNo: 1, ICCID: "8986", MSISDN: "13800008000"},
|
||||
{RowNo: 2, ICCID: "8986", MSISDN: "13800008001"},
|
||||
{RowNo: 3, ICCID: "8987", MSISDN: "13800008000"},
|
||||
}
|
||||
|
||||
markSameBatchDuplicates(rows)
|
||||
|
||||
if !containsReason(rows[1].Reason, "同批重复ICCID") {
|
||||
t.Fatalf("expected row 2 duplicate ICCID, got %q", rows[1].Reason)
|
||||
}
|
||||
if !containsReason(rows[2].Reason, "同批重复MSISDN") {
|
||||
t.Fatalf("expected row 3 duplicate MSISDN, got %q", rows[2].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkSameBatchDuplicatesChecksIMSI(t *testing.T) {
|
||||
rows := []*Row{
|
||||
{RowNo: 1, IMSI: "460001234567890"},
|
||||
{RowNo: 2, IMSI: "460001234567890"},
|
||||
}
|
||||
|
||||
markSameBatchDuplicates(rows)
|
||||
|
||||
if !containsReason(rows[1].Reason, "同批重复IMSI") {
|
||||
t.Fatalf("expected duplicate IMSI, got %q", rows[1].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoAcceptsMatchingIdentifiers(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, ICCID: "898604F4152391115473", MSISDN: "1441543615473", IMSI: "460087436015473", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{response: []model.SimBaseRes[model.SimBasicInfoItem]{
|
||||
{
|
||||
Code: 0,
|
||||
SimKey: &model.SimKey{ChanKey: "chan-a", ICCID: "898604F4152391115473"},
|
||||
ResData: model.SimBasicInfoItem{
|
||||
ICCID: "898604F4152391115473",
|
||||
MSISDN: "1441543615473",
|
||||
IMSI: "460087436015473",
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
err := (&executor{upstream: upstream}).validateUpstreamBasicInfo(context.Background(), &Task{Rows: rows})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows[0].Reason != "" {
|
||||
t.Fatalf("unexpected reason: %q", rows[0].Reason)
|
||||
}
|
||||
if upstream.calls != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1", upstream.calls)
|
||||
}
|
||||
if len(upstream.inputs) != 1 || len(upstream.inputs[0].SimKeys) != 1 {
|
||||
t.Fatalf("unexpected upstream inputs: %+v", upstream.inputs)
|
||||
}
|
||||
requestKey := upstream.inputs[0].SimKeys[0]
|
||||
if requestKey.ChanKey != "chan-a" || requestKey.ICCID != "898604F4152391115473" {
|
||||
t.Fatalf("unexpected upstream request key: %+v", requestKey)
|
||||
}
|
||||
if requestKey.MSISDN != "" || requestKey.IMSI != "" {
|
||||
t.Fatalf("upstream basic-info request must only carry chanKey and ICCID: %+v", requestKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoRequiresCompleteResData(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, ICCID: "898604F4152391115473", MSISDN: "1441543615473", IMSI: "460087436015473", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{response: []model.SimBaseRes[model.SimBasicInfoItem]{
|
||||
{
|
||||
Code: 0,
|
||||
SimKey: &model.SimKey{ChanKey: "chan-a", ICCID: "898604f4152391115473", MSISDN: "1441543615473", IMSI: "460087436015473"},
|
||||
ResData: model.SimBasicInfoItem{
|
||||
ActiveDate: "2026-08-12 00:00:00",
|
||||
OpenDate: "2026-08-12 00:00:00",
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
err := (&executor{upstream: upstream}).validateUpstreamBasicInfo(context.Background(), &Task{Rows: rows})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsReason(rows[0].Reason, "上游未返回三码信息") {
|
||||
t.Fatalf("expected missing upstream data failure, got %q", rows[0].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoPreservesNotFoundMessage(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, ICCID: "8986112223600594371", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{response: []model.SimBaseRes[model.SimBasicInfoItem]{
|
||||
{
|
||||
Code: 0,
|
||||
Message: "未查询到对应SIM卡信息",
|
||||
SimKey: &model.SimKey{ChanKey: "chan-a", ICCID: "8986112223600594371"},
|
||||
},
|
||||
}}
|
||||
|
||||
err := (&executor{upstream: upstream}).validateUpstreamBasicInfo(context.Background(), &Task{Rows: rows})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows[0].Reason != "上游三码校验失败: 未查询到对应SIM卡信息" {
|
||||
t.Fatalf("unexpected reason: %q", rows[0].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoMatchesByMSISDNAndBackfillsCodes(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, MSISDN: "14100007308", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{response: []model.SimBaseRes[model.SimBasicInfoItem]{
|
||||
{
|
||||
Code: 0,
|
||||
SimKey: &model.SimKey{ChanKey: "chan-a", MSISDN: "14100007308"},
|
||||
ResData: model.SimBasicInfoItem{
|
||||
ICCID: "89860401786527567308",
|
||||
MSISDN: "14100007308",
|
||||
IMSI: "460046527567308",
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
err := (&executor{upstream: upstream}).validateUpstreamBasicInfo(context.Background(), &Task{Rows: rows})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows[0].Reason != "" {
|
||||
t.Fatalf("unexpected reason: %q", rows[0].Reason)
|
||||
}
|
||||
if rows[0].ICCID != "89860401786527567308" || rows[0].IMSI != "460046527567308" {
|
||||
t.Fatalf("expected upstream codes to be backfilled, got %+v", rows[0])
|
||||
}
|
||||
requestKey := upstream.inputs[0].SimKeys[0]
|
||||
if requestKey.ICCID != "" || requestKey.MSISDN != "14100007308" || requestKey.IMSI != "" {
|
||||
t.Fatalf("unexpected upstream request key: %+v", requestKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBatchInputSingleMSISDNBackfillsBeforeCompletenessCheck(t *testing.T) {
|
||||
rows := parseInputText("14100007308")
|
||||
repo := &inboundRepositoryStub{}
|
||||
upstream := &basicInfoUpstreamStub{response: []model.SimBaseRes[model.SimBasicInfoItem]{
|
||||
{
|
||||
Code: 0,
|
||||
SimKey: &model.SimKey{ChanKey: "chan-a", MSISDN: "14100007308"},
|
||||
ResData: model.SimBasicInfoItem{
|
||||
ICCID: "89860401786527567308",
|
||||
MSISDN: "14100007308",
|
||||
IMSI: "460046527567308",
|
||||
},
|
||||
},
|
||||
}}
|
||||
task := &Task{
|
||||
Batch: Batch{SourceType: consts.BatchSourceTypeBatchInput},
|
||||
Options: Options{ChanKey: "chan-a", ValidateUpstreamBasicInfo: "1"},
|
||||
Rows: rows,
|
||||
}
|
||||
|
||||
err := (&executor{repository: repo, upstream: upstream}).validate(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.ValidCount != 1 {
|
||||
t.Fatalf("valid count = %d, want 1, reason=%q", task.ValidCount, rows[0].Reason)
|
||||
}
|
||||
if rows[0].ICCID != "89860401786527567308" || rows[0].IMSI != "460046527567308" {
|
||||
t.Fatalf("expected upstream codes to be backfilled, got %+v", rows[0])
|
||||
}
|
||||
if upstream.calls != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1", upstream.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSkipsUpstreamBasicInfoWhenDisabled(t *testing.T) {
|
||||
rows := []*Row{{
|
||||
RowNo: 1,
|
||||
ICCID: "89860401786527567308",
|
||||
MSISDN: "14100007308",
|
||||
IMSI: "460046527567308",
|
||||
ChanKey: "chan-a",
|
||||
}}
|
||||
upstream := &basicInfoUpstreamStub{err: errors.New("gateway unavailable")}
|
||||
task := &Task{
|
||||
Batch: Batch{SourceType: consts.BatchSourceTypeFile},
|
||||
Options: Options{ValidateUpstreamBasicInfo: "0"},
|
||||
Rows: rows,
|
||||
}
|
||||
|
||||
err := (&executor{repository: &inboundRepositoryStub{}, upstream: upstream}).validate(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 0 {
|
||||
t.Fatalf("upstream calls = %d, want 0", upstream.calls)
|
||||
}
|
||||
if task.ValidCount != 1 {
|
||||
t.Fatalf("valid count = %d, want 1, reason=%q", task.ValidCount, rows[0].Reason)
|
||||
}
|
||||
if task.Options.ValidateUpstreamBasicInfo != "0" {
|
||||
t.Fatalf("validate option = %q, want 0", task.Options.ValidateUpstreamBasicInfo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRunsUpstreamBasicInfoWhenEnabled(t *testing.T) {
|
||||
rows := []*Row{{
|
||||
RowNo: 1,
|
||||
ICCID: "89860401786527567308",
|
||||
MSISDN: "14100007308",
|
||||
IMSI: "460046527567308",
|
||||
ChanKey: "chan-a",
|
||||
}}
|
||||
upstream := &basicInfoUpstreamStub{err: errors.New("gateway unavailable")}
|
||||
task := &Task{
|
||||
Batch: Batch{SourceType: consts.BatchSourceTypeFile},
|
||||
Options: Options{ValidateUpstreamBasicInfo: "1"},
|
||||
Rows: rows,
|
||||
}
|
||||
|
||||
err := (&executor{repository: &inboundRepositoryStub{}, upstream: upstream}).validate(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1", upstream.calls)
|
||||
}
|
||||
if task.ValidCount != 0 {
|
||||
t.Fatalf("valid count = %d, want 0, reason=%q", task.ValidCount, rows[0].Reason)
|
||||
}
|
||||
if task.Options.ValidateUpstreamBasicInfo != "1" {
|
||||
t.Fatalf("validate option = %q, want 1", task.Options.ValidateUpstreamBasicInfo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPopulateExcelRowsReturnsRetryableErrorWhenSourceIsUnreadable(t *testing.T) {
|
||||
sourceErr := errors.New("OSS local file does not exist")
|
||||
task := &Task{Source: Source{ImportFileOssID: 1270}}
|
||||
repository := &inboundRepositoryStub{readOssErr: sourceErr}
|
||||
|
||||
err := (&executor{repository: repository}).populateExcelRows(context.Background(), task)
|
||||
if !errors.Is(err, sourceErr) || !batch.IsRetryable(err) {
|
||||
t.Fatalf("populateExcelRows() error = %v, want retryable wrapper of %v", err, sourceErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoWaitsForRateLimitBeforeQuery(t *testing.T) {
|
||||
var order []string
|
||||
rows := []*Row{{RowNo: 1, ICCID: "89860401786527567308", MSISDN: "14100007308", IMSI: "460046527567308", ChanKey: "chan-a"}}
|
||||
limiter := &recordingRateLimiter{
|
||||
onWait: func(context.Context, string, int) error {
|
||||
order = append(order, "limit")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
upstream := &basicInfoUpstreamStub{
|
||||
onQuery: func() {
|
||||
order = append(order, "query")
|
||||
},
|
||||
response: []model.SimBaseRes[model.SimBasicInfoItem]{
|
||||
{
|
||||
Code: 0,
|
||||
SimKey: &model.SimKey{ChanKey: "chan-a", ICCID: "89860401786527567308"},
|
||||
ResData: model.SimBasicInfoItem{
|
||||
ICCID: "89860401786527567308",
|
||||
MSISDN: "14100007308",
|
||||
IMSI: "460046527567308",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := (&executor{upstream: upstream, rateLimiter: limiter}).validateUpstreamBasicInfo(context.Background(), &Task{
|
||||
Batch: Batch{BatchNo: "B-1"},
|
||||
Rows: rows,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if limiter.calls != 1 || limiter.chanKey != "chan-a" || limiter.itemCount != 1 {
|
||||
t.Fatalf("unexpected limiter call: %+v", limiter)
|
||||
}
|
||||
if upstream.calls != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1", upstream.calls)
|
||||
}
|
||||
if len(order) != 2 || order[0] != "limit" || order[1] != "query" {
|
||||
t.Fatalf("unexpected call order: %+v", order)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoStopsWhenRateLimitFails(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, ICCID: "89860401786527567308", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{}
|
||||
limiter := &recordingRateLimiter{err: errors.New("redis down")}
|
||||
|
||||
err := (&executor{upstream: upstream, rateLimiter: limiter}).validateUpstreamBasicInfo(context.Background(), &Task{
|
||||
Batch: Batch{BatchNo: "B-1"},
|
||||
Rows: rows,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 0 {
|
||||
t.Fatalf("upstream calls = %d, want 0", upstream.calls)
|
||||
}
|
||||
if !strings.Contains(rows[0].Reason, "redis down") {
|
||||
t.Fatalf("expected rate limit failure reason, got %q", rows[0].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoRejectsMismatch(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, ICCID: "89860001", MSISDN: "13800008000", IMSI: "460001", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{response: []model.SimBaseRes[model.SimBasicInfoItem]{
|
||||
{
|
||||
Code: 0,
|
||||
SimKey: &model.SimKey{ChanKey: "chan-a", ICCID: "89860001"},
|
||||
ResData: model.SimBasicInfoItem{ICCID: "89860001", MSISDN: "13800008099", IMSI: "460001"},
|
||||
},
|
||||
}}
|
||||
|
||||
err := (&executor{upstream: upstream}).validateUpstreamBasicInfo(context.Background(), &Task{Rows: rows})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(rows[0].Reason, "MSISDN") {
|
||||
t.Fatalf("expected MSISDN mismatch reason, got %q", rows[0].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamBasicInfoRejectsUpstreamError(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, ICCID: "89860001", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{err: errors.New("gateway unavailable")}
|
||||
|
||||
err := (&executor{upstream: upstream}).validateUpstreamBasicInfo(context.Background(), &Task{Rows: rows})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(rows[0].Reason, "gateway unavailable") {
|
||||
t.Fatalf("expected upstream error reason, got %q", rows[0].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTableRowsAcceptsChineseInboundHeaders(t *testing.T) {
|
||||
rows := parseTableRows([][]string{
|
||||
{"ICCID号", "手机号", "国际移动用户识别码", "国际移动设备识别码", "通道编码", "卡类型", "上游套餐编码", "供应商编码"},
|
||||
{"898604F4152391115473", "1441543615473", "460087436015473", "865111111111111", "channel_003", "通用插拔 SIM 卡", "UP001", "SUP001"},
|
||||
})
|
||||
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d, want 1", len(rows))
|
||||
}
|
||||
row := rows[0]
|
||||
if row.ICCID != "898604F4152391115473" || row.MSISDN != "1441543615473" ||
|
||||
row.IMSI != "460087436015473" || row.IMEI != "865111111111111" || row.ChanKey != "channel_003" {
|
||||
t.Fatalf("unexpected parsed row: %+v", row)
|
||||
}
|
||||
if row.CardType != "通用插拔 SIM 卡" || row.UpPkgCode != "UP001" || row.SuppliersCode != "SUP001" {
|
||||
t.Fatalf("unexpected inbound options: %+v", row)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTableRowsAcceptsRequiredMarkerHeaders(t *testing.T) {
|
||||
rows := parseTableRows([][]string{
|
||||
{"ICCID*", "MSISDN*", "IMSI*", "通道编号*"},
|
||||
{"89860401786527567308", "14100007308", "460046527567308", "channel_003"},
|
||||
})
|
||||
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d, want 1", len(rows))
|
||||
}
|
||||
row := rows[0]
|
||||
if row.ICCID != "89860401786527567308" || row.MSISDN != "14100007308" ||
|
||||
row.IMSI != "460046527567308" || row.ChanKey != "channel_003" {
|
||||
t.Fatalf("unexpected parsed row: %+v", row)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundExcelHeaderRequiresRecognizedIdentifier(t *testing.T) {
|
||||
for _, fields := range [][]string{{"随机内容", "乱码"}, {"通道编号", "卡类型"}, {"ICCID", "通道编号"}, {"MSISDN", "通道编号"}, {"IMSI", "通道编号"}} {
|
||||
got := hasInboundIdentifierHeader(headerIndex(fields))
|
||||
want := fields[0] == "ICCID" || fields[0] == "MSISDN" || fields[0] == "IMSI"
|
||||
if got != want {
|
||||
t.Fatalf("header %v valid = %t, want %t", fields, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPopulateExcelRowsRejectsInvalidHeaderAndAcceptsMSISDNHeader(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
header []string
|
||||
valid bool
|
||||
}{
|
||||
{name: "invalid", header: []string{"乱码", "通道编号"}},
|
||||
{name: "msisdn", header: []string{"MSISDN", "通道编号"}, valid: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
workbook := excelize.NewFile()
|
||||
defer func() { _ = workbook.Close() }()
|
||||
for index, value := range test.header {
|
||||
cell, err := excelize.CoordinatesToCellName(index+1, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := workbook.SetCellValue("Sheet1", cell, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := workbook.SetCellValue("Sheet1", "A2", "14100007308"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, err := workbook.WriteToBuffer()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{Source: Source{ImportFileOssID: 1}}
|
||||
repository := &inboundRepositoryStub{readOssContent: content.Bytes(), readOssName: "cards.xlsx"}
|
||||
err = (&executor{repository: repository}).populateExcelRows(context.Background(), task)
|
||||
if !test.valid {
|
||||
if err == nil || len(task.Rows) != 0 {
|
||||
t.Fatalf("invalid header: err=%v, rows=%v", err, task.Rows)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || len(task.Rows) != 1 || task.Rows[0].MSISDN != "14100007308" {
|
||||
t.Fatalf("valid MSISDN file: err=%v, rows=%v", err, task.Rows)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateInvalidRowSkipsUpstreamForEntireBatch(t *testing.T) {
|
||||
rows := []*Row{
|
||||
{RowNo: 2, ICCID: "invalid", ChanKey: "channel_003"},
|
||||
{RowNo: 3, ICCID: "89860401786527567308", ChanKey: "channel_003"},
|
||||
}
|
||||
upstream := &basicInfoUpstreamStub{}
|
||||
task := &Task{Rows: rows, Options: Options{ChanKey: "channel_003", ValidateUpstreamBasicInfo: "1"}}
|
||||
if err := (&executor{repository: &inboundRepositoryStub{}, upstream: upstream}).validate(context.Background(), task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 0 || task.ValidCount != 0 || !hasFailedRows(rows) {
|
||||
t.Fatalf("invalid batch should skip upstream: calls=%d valid=%d", upstream.calls, task.ValidCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSkipsUpstreamWhenLocalIdentifierFormatFails(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, ICCID: "4", MSISDN: "4", IMSI: "4", ChanKey: "chan-a"}}
|
||||
upstream := &basicInfoUpstreamStub{}
|
||||
repo := &inboundRepositoryStub{}
|
||||
task := &Task{
|
||||
Rows: rows,
|
||||
Options: Options{ChanKey: "chan-a"},
|
||||
}
|
||||
|
||||
err := (&executor{repository: repo, upstream: upstream}).validate(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if upstream.calls != 0 {
|
||||
t.Fatalf("upstream calls = %d, want 0", upstream.calls)
|
||||
}
|
||||
if !containsReason(rows[0].Reason, "ICCID格式不正确") {
|
||||
t.Fatalf("expected local validation failure, got %q", rows[0].Reason)
|
||||
}
|
||||
if task.ValidCount != 0 {
|
||||
t.Fatalf("valid count = %d, want 0", task.ValidCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRequiresCompleteIdentifiersAfterUpstreamStep(t *testing.T) {
|
||||
rows := []*Row{{RowNo: 1, MSISDN: "14100007308", ChanKey: "chan-a"}}
|
||||
repo := &inboundRepositoryStub{}
|
||||
task := &Task{
|
||||
Rows: rows,
|
||||
Source: Source{Range: Range{RangeType: "msisdn"}},
|
||||
}
|
||||
|
||||
err := (&executor{repository: repo}).validate(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsReason(rows[0].Reason, "三码信息不完整") {
|
||||
t.Fatalf("expected incomplete identifiers failure, got %q", rows[0].Reason)
|
||||
}
|
||||
if task.ValidCount != 0 {
|
||||
t.Fatalf("valid count = %d, want 0", task.ValidCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNormalizesExcelInboundOptionsAndBackfillsBatchInfo(t *testing.T) {
|
||||
rows := parseTableRows([][]string{
|
||||
{"ICCID*", "MSISDN*", "IMSI*", "通道编号*", "卡类型", "上游套餐编码", "供应商编码", "是否贴片转插拔", "是否运营商达量断网"},
|
||||
{"89860401786527567308", "14100007308", "460046527567308", "channel_003", "MP1-消费级插拔物联卡", "UP001", "SUP001", "是", "否"},
|
||||
{"89860401786527567309", "14100007309", "460046527567309", "channel_003", "", "UP001", "SUP001", "", ""},
|
||||
})
|
||||
task := &Task{Rows: rows, Batch: Batch{SourceType: consts.BatchSourceTypeFile}}
|
||||
|
||||
err := (&executor{repository: &inboundRepositoryStub{}}).validate(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.ValidCount != 2 {
|
||||
t.Fatalf("valid count = %d, want 2", task.ValidCount)
|
||||
}
|
||||
if task.Options.ChanKey != "channel_003" || task.Options.CardType != "4" ||
|
||||
task.Options.UpPkgCode != "UP001" || task.Options.SuppliersCode != "SUP001" {
|
||||
t.Fatalf("unexpected task options: %+v", task.Options)
|
||||
}
|
||||
if rows[1].CardType != "4" || rows[1].InsertToPlugEnabled != "0" || rows[1].CarrierTrafficCutoffEnabled != "1" {
|
||||
t.Fatalf("expected blank row options to be backfilled from batch, got %+v", rows[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsMixedInboundOptionsInOneExcelBatch(t *testing.T) {
|
||||
rows := parseTableRows([][]string{
|
||||
{"ICCID*", "MSISDN*", "IMSI*", "通道编号*", "卡类型"},
|
||||
{"89860401786527567308", "14100007308", "460046527567308", "channel_003", "1"},
|
||||
{"89860401786527567309", "14100007309", "460046527567309", "channel_003", "2"},
|
||||
})
|
||||
task := &Task{Rows: rows, Batch: Batch{SourceType: consts.BatchSourceTypeFile}}
|
||||
|
||||
err := (&executor{repository: &inboundRepositoryStub{}}).validate(context.Background(), task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.ValidCount != 0 {
|
||||
t.Fatalf("valid count = %d, want 0", task.ValidCount)
|
||||
}
|
||||
if task.FailureReason != "同批入库配置必须一致" {
|
||||
t.Fatalf("unexpected task failure reason: %q", task.FailureReason)
|
||||
}
|
||||
if !containsReason(rows[0].Reason, "卡类型必须同批一致") || !containsReason(rows[1].Reason, "卡类型必须同批一致") {
|
||||
t.Fatalf("expected consistency errors, got row1=%q row2=%q", rows[0].Reason, rows[1].Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func containsReason(actual, want string) bool {
|
||||
return strings.Contains(actual, want)
|
||||
}
|
||||
|
||||
type basicInfoUpstreamStub struct {
|
||||
calls int
|
||||
inputs []*model.SimBasicInfoInput
|
||||
response []model.SimBaseRes[model.SimBasicInfoItem]
|
||||
err error
|
||||
onQuery func()
|
||||
}
|
||||
|
||||
func (s *basicInfoUpstreamStub) QuerySimBasicInfo(_ context.Context, input *model.SimBasicInfoInput) ([]model.SimBaseRes[model.SimBasicInfoItem], error) {
|
||||
if s.onQuery != nil {
|
||||
s.onQuery()
|
||||
}
|
||||
s.calls++
|
||||
if input == nil || len(input.SimKeys) == 0 {
|
||||
return nil, errors.New("missing sim keys")
|
||||
}
|
||||
s.inputs = append(s.inputs, cloneBasicInfoInput(input))
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
return s.response, nil
|
||||
}
|
||||
|
||||
func cloneBasicInfoInput(input *model.SimBasicInfoInput) *model.SimBasicInfoInput {
|
||||
clone := &model.SimBasicInfoInput{
|
||||
ReqData: input.ReqData,
|
||||
SimKeys: make([]*model.SimKey, 0, len(input.SimKeys)),
|
||||
}
|
||||
for _, item := range input.SimKeys {
|
||||
if item == nil {
|
||||
clone.SimKeys = append(clone.SimKeys, nil)
|
||||
continue
|
||||
}
|
||||
copied := *item
|
||||
clone.SimKeys = append(clone.SimKeys, &copied)
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
type recordingRateLimiter struct {
|
||||
calls int
|
||||
chanKey string
|
||||
itemCount int
|
||||
err error
|
||||
onWait func(context.Context, string, int) error
|
||||
}
|
||||
|
||||
func (l *recordingRateLimiter) Wait(ctx context.Context, chanKey string, itemCount int) error {
|
||||
l.calls++
|
||||
l.chanKey = chanKey
|
||||
l.itemCount = itemCount
|
||||
if l.onWait != nil {
|
||||
return l.onWait(ctx, chanKey, itemCount)
|
||||
}
|
||||
return l.err
|
||||
}
|
||||
|
||||
type inboundRepositoryStub struct {
|
||||
readOssContent []byte
|
||||
readOssName string
|
||||
readOssErr error
|
||||
}
|
||||
|
||||
func (s *inboundRepositoryStub) Load(context.Context, string) (*Task, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (s *inboundRepositoryStub) ReadOssFile(context.Context, int64) ([]byte, string, error) {
|
||||
if s.readOssErr != nil {
|
||||
return nil, "", s.readOssErr
|
||||
}
|
||||
return s.readOssContent, s.readOssName, nil
|
||||
}
|
||||
|
||||
func (s *inboundRepositoryStub) FindExistingStocks(context.Context, string, []string, []string, []string) ([]Stock, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *inboundRepositoryStub) SaveValidationFailure(context.Context, *Task) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (s *inboundRepositoryStub) SaveSuccess(context.Context, *Task) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (s *inboundRepositoryStub) SaveBusinessFailure(context.Context, *Task, string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (s *inboundRepositoryStub) SaveResultFile(context.Context, *Task, batchresult.Request) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
@@ -0,0 +1,738 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
inboundUpstreamRateLimitConfigKey = "batch.inbound.upstreamRateLimit"
|
||||
|
||||
defaultInboundUpstreamRateLimitMode = "redis"
|
||||
defaultInboundUpstreamRateLimitRedisGroup = "default"
|
||||
defaultInboundUpstreamRateLimitKeyPrefix = "iotlcm:worker:inbound:upstream_basic_info"
|
||||
defaultInboundUpstreamQPS = 20
|
||||
defaultInboundUpstreamBurst = 1
|
||||
defaultInboundUpstreamRequestCost = 1
|
||||
defaultInboundChannelTempKeyCacheTTL = time.Minute
|
||||
|
||||
inboundRateLimitFallbackLocal = "local"
|
||||
inboundRateLimitFallbackError = "error"
|
||||
|
||||
inboundCarrierTelecom = "telecom"
|
||||
inboundCarrierMobile = "mobile"
|
||||
)
|
||||
|
||||
type upstreamBasicInfoRateLimiter interface {
|
||||
Wait(ctx context.Context, chanKey string, requestCount int) error
|
||||
}
|
||||
|
||||
type redisEvaler interface {
|
||||
Eval(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error)
|
||||
}
|
||||
|
||||
type inboundUpstreamRateLimitConfig struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
Mode string `json:"mode"`
|
||||
RedisGroup string `json:"redisGroup"`
|
||||
KeyPrefix string `json:"keyPrefix"`
|
||||
RedisFailureFallback string `json:"redisFailureFallback"`
|
||||
QPS float64 `json:"qps"`
|
||||
Burst int `json:"burst"`
|
||||
RequestCost int `json:"requestCost"`
|
||||
Default inboundUpstreamRateLimitRule `json:"default"`
|
||||
Carriers map[string]inboundUpstreamRateLimitRule `json:"carriers"`
|
||||
Channels map[string]inboundUpstreamRateLimitRule `json:"channels"`
|
||||
}
|
||||
|
||||
type inboundUpstreamRateLimitRule struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
QPS float64 `json:"qps"`
|
||||
Burst int `json:"burst"`
|
||||
RequestCost int `json:"requestCost"`
|
||||
}
|
||||
|
||||
type inboundUpstreamRateLimitSetting struct {
|
||||
enabled bool
|
||||
mode string
|
||||
redisGroup string
|
||||
keyPrefix string
|
||||
redisFailureFallback string
|
||||
qps float64
|
||||
burst int
|
||||
requestCost int
|
||||
carrier string
|
||||
tempKey string
|
||||
skipReason string
|
||||
}
|
||||
|
||||
type inboundChannelMetadata struct {
|
||||
TempKey string `orm:"temp_key"`
|
||||
Operator int `orm:"operator"`
|
||||
}
|
||||
|
||||
type inboundChannelTempKeyCacheEntry struct {
|
||||
metadata inboundChannelMetadata
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type configUpstreamBasicInfoRateLimiter struct {
|
||||
redis func(string) redisEvaler
|
||||
channelMetadata func(context.Context, string) (inboundChannelMetadata, error)
|
||||
channelTempKey func(context.Context, string) (string, error)
|
||||
local *localTokenBucketLimiter
|
||||
channelCache sync.Map
|
||||
redisLogTimes sync.Map
|
||||
skipLogTimes sync.Map
|
||||
}
|
||||
|
||||
func newUpstreamBasicInfoRateLimiter() *configUpstreamBasicInfoRateLimiter {
|
||||
return &configUpstreamBasicInfoRateLimiter{
|
||||
redis: func(name string) redisEvaler {
|
||||
return g.Redis(name)
|
||||
},
|
||||
channelMetadata: queryInboundChannelMetadata,
|
||||
local: newLocalTokenBucketLimiter(),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *configUpstreamBasicInfoRateLimiter) Wait(ctx context.Context, chanKey string, requestCount int) error {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
cfg := loadInboundUpstreamRateLimitConfig(ctx)
|
||||
if !inboundUpstreamRateLimitGloballyEnabled(cfg) {
|
||||
return nil
|
||||
}
|
||||
metadata, err := l.lookupChannelMetadata(ctx, chanKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
setting := resolveInboundUpstreamRateLimitFromChannelMetadata(cfg, chanKey, metadata.Operator, metadata.TempKey)
|
||||
if !setting.enabled || setting.qps <= 0 || setting.mode == "off" {
|
||||
if setting.skipReason != "" {
|
||||
l.logSkipped(ctx, chanKey, setting)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if requestCount <= 0 {
|
||||
requestCount = 1
|
||||
}
|
||||
permits := requestCount * setting.requestCost
|
||||
key := inboundUpstreamRateLimitKey(setting.keyPrefix, chanKey, setting.carrier)
|
||||
for i := 0; i < permits; i++ {
|
||||
if err := l.waitOne(ctx, setting, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *configUpstreamBasicInfoRateLimiter) waitOne(ctx context.Context, setting inboundUpstreamRateLimitSetting, key string) error {
|
||||
if setting.mode == "local" {
|
||||
if l.local == nil {
|
||||
return fmt.Errorf("local rate limiter is nil")
|
||||
}
|
||||
return l.local.Wait(ctx, key, setting.qps, setting.burst)
|
||||
}
|
||||
if l.redis == nil {
|
||||
return l.handleRedisError(ctx, setting, key, fmt.Errorf("redis client resolver is nil"))
|
||||
}
|
||||
redis := l.redis(setting.redisGroup)
|
||||
if redis == nil {
|
||||
return l.handleRedisError(ctx, setting, key, fmt.Errorf("redis client is nil"))
|
||||
}
|
||||
if err := (&redisTokenBucketLimiter{redis: redis}).Wait(ctx, key, setting.qps, setting.burst); err != nil {
|
||||
return l.handleRedisError(ctx, setting, key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *configUpstreamBasicInfoRateLimiter) handleRedisError(ctx context.Context, setting inboundUpstreamRateLimitSetting, key string, err error) error {
|
||||
if setting.redisFailureFallback == inboundRateLimitFallbackError {
|
||||
return err
|
||||
}
|
||||
if l.local == nil {
|
||||
return err
|
||||
}
|
||||
l.logRedisFallback(ctx, key, err)
|
||||
return l.local.Wait(ctx, key, setting.qps, setting.burst)
|
||||
}
|
||||
|
||||
func (l *configUpstreamBasicInfoRateLimiter) logRedisFallback(ctx context.Context, key string, err error) {
|
||||
now := time.Now()
|
||||
if value, ok := l.redisLogTimes.Load(key); ok {
|
||||
if last, ok := value.(time.Time); ok && now.Sub(last) < 10*time.Second {
|
||||
return
|
||||
}
|
||||
}
|
||||
l.redisLogTimes.Store(key, now)
|
||||
g.Log().Errorf(ctx, "inbound upstream basic-info redis rate limiter failed, fallback local, key:%s, err:%+v", key, err)
|
||||
}
|
||||
|
||||
func (l *configUpstreamBasicInfoRateLimiter) lookupChannelMetadata(ctx context.Context, chanKey string) (inboundChannelMetadata, error) {
|
||||
chanKey = strings.TrimSpace(chanKey)
|
||||
if chanKey == "" {
|
||||
return inboundChannelMetadata{}, nil
|
||||
}
|
||||
now := time.Now()
|
||||
if value, ok := l.channelCache.Load(chanKey); ok {
|
||||
if entry, ok := value.(inboundChannelTempKeyCacheEntry); ok && now.Before(entry.expiresAt) {
|
||||
return entry.metadata, nil
|
||||
}
|
||||
}
|
||||
if l.channelMetadata != nil {
|
||||
metadata, err := l.channelMetadata(ctx, chanKey)
|
||||
if err != nil {
|
||||
return inboundChannelMetadata{}, fmt.Errorf("query channel metadata failed, chanKey:%s: %w", chanKey, err)
|
||||
}
|
||||
metadata.TempKey = strings.TrimSpace(metadata.TempKey)
|
||||
l.channelCache.Store(chanKey, inboundChannelTempKeyCacheEntry{
|
||||
metadata: metadata,
|
||||
expiresAt: now.Add(defaultInboundChannelTempKeyCacheTTL),
|
||||
})
|
||||
return metadata, nil
|
||||
}
|
||||
if l.channelTempKey == nil {
|
||||
return inboundChannelMetadata{}, fmt.Errorf("channel metadata resolver is nil")
|
||||
}
|
||||
tempKey, err := l.channelTempKey(ctx, chanKey)
|
||||
if err != nil {
|
||||
return inboundChannelMetadata{}, fmt.Errorf("query channel template failed, chanKey:%s: %w", chanKey, err)
|
||||
}
|
||||
metadata := inboundChannelMetadata{TempKey: strings.TrimSpace(tempKey)}
|
||||
l.channelCache.Store(chanKey, inboundChannelTempKeyCacheEntry{
|
||||
metadata: metadata,
|
||||
expiresAt: now.Add(defaultInboundChannelTempKeyCacheTTL),
|
||||
})
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (l *configUpstreamBasicInfoRateLimiter) logSkipped(ctx context.Context, chanKey string, setting inboundUpstreamRateLimitSetting) {
|
||||
key := strings.TrimSpace(chanKey) + ":" + setting.skipReason
|
||||
now := time.Now()
|
||||
if value, ok := l.skipLogTimes.Load(key); ok {
|
||||
if last, ok := value.(time.Time); ok && now.Sub(last) < 10*time.Second {
|
||||
return
|
||||
}
|
||||
}
|
||||
l.skipLogTimes.Store(key, now)
|
||||
g.Log().Warningf(ctx, "inbound upstream basic-info rate limit skipped, chanKey:%s, tempKey:%s, reason:%s",
|
||||
chanKey, setting.tempKey, setting.skipReason)
|
||||
}
|
||||
|
||||
func queryInboundChannelMetadata(ctx context.Context, chanKey string) (inboundChannelMetadata, error) {
|
||||
var row inboundChannelMetadata
|
||||
if err := g.DB().Model("t_sim_channels").Ctx(ctx).
|
||||
Fields("temp_key,operator").
|
||||
Where("chan_key", strings.TrimSpace(chanKey)).
|
||||
Limit(1).
|
||||
Scan(&row); err != nil {
|
||||
return inboundChannelMetadata{}, err
|
||||
}
|
||||
row.TempKey = strings.TrimSpace(row.TempKey)
|
||||
return row, nil
|
||||
}
|
||||
|
||||
func queryInboundChannelTempKey(ctx context.Context, chanKey string) (string, error) {
|
||||
metadata, err := queryInboundChannelMetadata(ctx, chanKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return metadata.TempKey, nil
|
||||
}
|
||||
|
||||
func resolveInboundUpstreamRateLimit(ctx context.Context, chanKey string) inboundUpstreamRateLimitSetting {
|
||||
return resolveInboundUpstreamRateLimitWithTempKey(ctx, chanKey, "")
|
||||
}
|
||||
|
||||
func resolveInboundUpstreamRateLimitWithTempKey(ctx context.Context, chanKey, tempKey string) inboundUpstreamRateLimitSetting {
|
||||
cfg := loadInboundUpstreamRateLimitConfig(ctx)
|
||||
return resolveInboundUpstreamRateLimitFromConfig(cfg, chanKey, tempKey)
|
||||
}
|
||||
|
||||
func resolveInboundUpstreamRateLimitWithChannelMetadata(ctx context.Context, chanKey string, operator int, tempKey string) inboundUpstreamRateLimitSetting {
|
||||
cfg := loadInboundUpstreamRateLimitConfig(ctx)
|
||||
return resolveInboundUpstreamRateLimitFromChannelMetadata(cfg, chanKey, operator, tempKey)
|
||||
}
|
||||
|
||||
func loadInboundUpstreamRateLimitConfig(ctx context.Context) inboundUpstreamRateLimitConfig {
|
||||
cfg := defaultInboundUpstreamRateLimitConfig()
|
||||
value, err := g.Cfg().Get(ctx, inboundUpstreamRateLimitConfigKey)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "load inbound upstream rate limit config failed: %+v", err)
|
||||
return normalizeInboundUpstreamRateLimitConfig(cfg)
|
||||
}
|
||||
if value != nil && !value.IsNil() {
|
||||
if err = value.Scan(&cfg); err != nil {
|
||||
g.Log().Warningf(ctx, "parse inbound upstream rate limit config failed: %+v", err)
|
||||
cfg = defaultInboundUpstreamRateLimitConfig()
|
||||
}
|
||||
}
|
||||
return normalizeInboundUpstreamRateLimitConfig(cfg)
|
||||
}
|
||||
|
||||
func defaultInboundUpstreamRateLimitConfig() inboundUpstreamRateLimitConfig {
|
||||
return inboundUpstreamRateLimitConfig{
|
||||
Mode: defaultInboundUpstreamRateLimitMode,
|
||||
RedisGroup: defaultInboundUpstreamRateLimitRedisGroup,
|
||||
KeyPrefix: defaultInboundUpstreamRateLimitKeyPrefix,
|
||||
RedisFailureFallback: inboundRateLimitFallbackError,
|
||||
Default: inboundUpstreamRateLimitRule{
|
||||
Enabled: inboundBool(false),
|
||||
},
|
||||
Carriers: map[string]inboundUpstreamRateLimitRule{
|
||||
inboundCarrierTelecom: {
|
||||
Enabled: inboundBool(true),
|
||||
QPS: defaultInboundUpstreamQPS,
|
||||
Burst: defaultInboundUpstreamBurst,
|
||||
RequestCost: defaultInboundUpstreamRequestCost,
|
||||
},
|
||||
inboundCarrierMobile: {
|
||||
Enabled: inboundBool(false),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeInboundUpstreamRateLimitConfig(cfg inboundUpstreamRateLimitConfig) inboundUpstreamRateLimitConfig {
|
||||
cfg.Mode = normalizeInboundRateLimitMode(cfg.Mode)
|
||||
cfg.RedisGroup = firstNonEmpty(strings.TrimSpace(cfg.RedisGroup), defaultInboundUpstreamRateLimitRedisGroup)
|
||||
cfg.KeyPrefix = firstNonEmpty(strings.TrimSpace(cfg.KeyPrefix), defaultInboundUpstreamRateLimitKeyPrefix)
|
||||
cfg.RedisFailureFallback = normalizeInboundRateLimitFallback(cfg.RedisFailureFallback)
|
||||
if cfg.Default.Enabled == nil {
|
||||
cfg.Default.Enabled = inboundBool(false)
|
||||
}
|
||||
if len(cfg.Carriers) == 0 {
|
||||
cfg.Carriers = make(map[string]inboundUpstreamRateLimitRule, 2)
|
||||
} else {
|
||||
normalized := make(map[string]inboundUpstreamRateLimitRule, len(cfg.Carriers))
|
||||
for key, rule := range cfg.Carriers {
|
||||
normalized[normalizeInboundCarrier(key)] = rule
|
||||
}
|
||||
cfg.Carriers = normalized
|
||||
}
|
||||
telecomRule, ok := cfg.Carriers[inboundCarrierTelecom]
|
||||
if !ok {
|
||||
telecomRule = inboundUpstreamRateLimitRule{}
|
||||
}
|
||||
if telecomRule.Enabled == nil {
|
||||
telecomRule.Enabled = inboundBool(true)
|
||||
}
|
||||
if telecomRule.QPS <= 0 {
|
||||
telecomRule.QPS = defaultInboundUpstreamQPS
|
||||
}
|
||||
if telecomRule.Burst <= 0 {
|
||||
telecomRule.Burst = defaultInboundUpstreamBurst
|
||||
}
|
||||
if telecomRule.RequestCost <= 0 {
|
||||
telecomRule.RequestCost = defaultInboundUpstreamRequestCost
|
||||
}
|
||||
cfg.Carriers[inboundCarrierTelecom] = telecomRule
|
||||
|
||||
mobileRule, ok := cfg.Carriers[inboundCarrierMobile]
|
||||
if !ok {
|
||||
mobileRule = inboundUpstreamRateLimitRule{}
|
||||
}
|
||||
if mobileRule.Enabled == nil {
|
||||
mobileRule.Enabled = inboundBool(false)
|
||||
}
|
||||
cfg.Carriers[inboundCarrierMobile] = mobileRule
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
func resolveInboundUpstreamRateLimitFromConfig(cfg inboundUpstreamRateLimitConfig, chanKey, tempKey string) inboundUpstreamRateLimitSetting {
|
||||
return resolveInboundUpstreamRateLimitFromChannelMetadata(cfg, chanKey, 0, tempKey)
|
||||
}
|
||||
|
||||
func resolveInboundUpstreamRateLimitFromChannelMetadata(cfg inboundUpstreamRateLimitConfig, chanKey string, operator int, tempKey string) inboundUpstreamRateLimitSetting {
|
||||
cfg = normalizeInboundUpstreamRateLimitConfig(cfg)
|
||||
setting := inboundUpstreamRateLimitSetting{
|
||||
enabled: true,
|
||||
mode: cfg.Mode,
|
||||
redisGroup: cfg.RedisGroup,
|
||||
keyPrefix: cfg.KeyPrefix,
|
||||
redisFailureFallback: cfg.RedisFailureFallback,
|
||||
qps: defaultInboundUpstreamQPS,
|
||||
burst: defaultInboundUpstreamBurst,
|
||||
requestCost: defaultInboundUpstreamRequestCost,
|
||||
tempKey: strings.TrimSpace(tempKey),
|
||||
}
|
||||
if cfg.Enabled != nil {
|
||||
setting.enabled = *cfg.Enabled
|
||||
}
|
||||
applyInboundRateLimitRule(&setting, inboundUpstreamRateLimitRule{
|
||||
QPS: cfg.QPS,
|
||||
Burst: cfg.Burst,
|
||||
RequestCost: cfg.RequestCost,
|
||||
})
|
||||
applyInboundRateLimitRule(&setting, cfg.Default)
|
||||
carrier, known := inboundCarrierForOperator(operator)
|
||||
if model.IsTelecomTemplate(tempKey) {
|
||||
carrier, known = inboundCarrierTelecom, true
|
||||
}
|
||||
if known {
|
||||
setting.carrier = carrier
|
||||
if rule, ok := inboundCarrierRateLimitRule(cfg.Carriers, carrier); ok {
|
||||
applyInboundRateLimitRule(&setting, rule)
|
||||
}
|
||||
} else if operator == 0 {
|
||||
setting.skipReason = "missing_channel_operator"
|
||||
} else {
|
||||
setting.skipReason = "unsupported_channel_operator"
|
||||
}
|
||||
if rule, ok := inboundChannelRateLimitRule(cfg.Channels, chanKey); ok {
|
||||
applyInboundRateLimitRule(&setting, rule)
|
||||
}
|
||||
return normalizeInboundRateLimitSetting(setting)
|
||||
}
|
||||
|
||||
func inboundUpstreamRateLimitGloballyEnabled(cfg inboundUpstreamRateLimitConfig) bool {
|
||||
cfg = normalizeInboundUpstreamRateLimitConfig(cfg)
|
||||
if cfg.Mode == "off" {
|
||||
return false
|
||||
}
|
||||
if cfg.Enabled == nil {
|
||||
return true
|
||||
}
|
||||
return *cfg.Enabled
|
||||
}
|
||||
|
||||
func applyInboundRateLimitRule(setting *inboundUpstreamRateLimitSetting, rule inboundUpstreamRateLimitRule) {
|
||||
if setting == nil {
|
||||
return
|
||||
}
|
||||
if rule.Enabled != nil {
|
||||
setting.enabled = *rule.Enabled
|
||||
}
|
||||
if rule.QPS > 0 {
|
||||
setting.qps = rule.QPS
|
||||
}
|
||||
if rule.Burst > 0 {
|
||||
setting.burst = rule.Burst
|
||||
}
|
||||
if rule.RequestCost > 0 {
|
||||
setting.requestCost = rule.RequestCost
|
||||
}
|
||||
}
|
||||
|
||||
func inboundChannelRateLimitRule(configs map[string]inboundUpstreamRateLimitRule, chanKey string) (inboundUpstreamRateLimitRule, bool) {
|
||||
chanKey = strings.TrimSpace(chanKey)
|
||||
if chanKey == "" || len(configs) == 0 {
|
||||
return inboundUpstreamRateLimitRule{}, false
|
||||
}
|
||||
if cfg, ok := configs[chanKey]; ok {
|
||||
return cfg, true
|
||||
}
|
||||
lower := strings.ToLower(chanKey)
|
||||
for key, cfg := range configs {
|
||||
if strings.ToLower(strings.TrimSpace(key)) == lower {
|
||||
return cfg, true
|
||||
}
|
||||
}
|
||||
return inboundUpstreamRateLimitRule{}, false
|
||||
}
|
||||
|
||||
func inboundCarrierRateLimitRule(configs map[string]inboundUpstreamRateLimitRule, carrier string) (inboundUpstreamRateLimitRule, bool) {
|
||||
carrier = normalizeInboundCarrier(carrier)
|
||||
if carrier == "" || len(configs) == 0 {
|
||||
return inboundUpstreamRateLimitRule{}, false
|
||||
}
|
||||
if cfg, ok := configs[carrier]; ok {
|
||||
return cfg, true
|
||||
}
|
||||
for key, cfg := range configs {
|
||||
if normalizeInboundCarrier(key) == carrier {
|
||||
return cfg, true
|
||||
}
|
||||
}
|
||||
return inboundUpstreamRateLimitRule{}, false
|
||||
}
|
||||
|
||||
func inboundCarrierForOperator(operator int) (string, bool) {
|
||||
switch operator {
|
||||
case 1:
|
||||
return inboundCarrierMobile, true
|
||||
case 2:
|
||||
return inboundCarrierTelecom, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeInboundCarrier(value string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "", "unknown":
|
||||
return ""
|
||||
case inboundCarrierTelecom, "dx", "ctcc", "china_telecom", "chinatelecom":
|
||||
return inboundCarrierTelecom
|
||||
case inboundCarrierMobile, "yd", "cmcc", "china_mobile", "chinamobile":
|
||||
return inboundCarrierMobile
|
||||
default:
|
||||
return strings.ToLower(strings.TrimSpace(value))
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeInboundRateLimitSetting(setting inboundUpstreamRateLimitSetting) inboundUpstreamRateLimitSetting {
|
||||
if setting.qps <= 0 || math.IsNaN(setting.qps) || math.IsInf(setting.qps, 0) {
|
||||
setting.qps = defaultInboundUpstreamQPS
|
||||
}
|
||||
if setting.burst <= 0 {
|
||||
setting.burst = defaultInboundUpstreamBurst
|
||||
}
|
||||
if setting.requestCost <= 0 {
|
||||
setting.requestCost = defaultInboundUpstreamRequestCost
|
||||
}
|
||||
return setting
|
||||
}
|
||||
|
||||
func normalizeInboundRateLimitMode(mode string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(mode)) {
|
||||
case "", "redis", "distributed":
|
||||
return defaultInboundUpstreamRateLimitMode
|
||||
case "local", "memory":
|
||||
return "local"
|
||||
case "off", "disable", "disabled", "none":
|
||||
return "off"
|
||||
default:
|
||||
return defaultInboundUpstreamRateLimitMode
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeInboundRateLimitFallback(value string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "", inboundRateLimitFallbackError:
|
||||
return inboundRateLimitFallbackError
|
||||
case inboundRateLimitFallbackLocal:
|
||||
return inboundRateLimitFallbackLocal
|
||||
default:
|
||||
return inboundRateLimitFallbackError
|
||||
}
|
||||
}
|
||||
|
||||
func inboundUpstreamRateLimitKey(prefix, chanKey, carrier string) string {
|
||||
prefix = strings.TrimRight(strings.TrimSpace(prefix), ":")
|
||||
if prefix == "" {
|
||||
prefix = defaultInboundUpstreamRateLimitKeyPrefix
|
||||
}
|
||||
chanKey = strings.TrimLeft(strings.TrimSpace(chanKey), ":")
|
||||
carrier = strings.TrimLeft(normalizeInboundCarrier(carrier), ":")
|
||||
if carrier != "" {
|
||||
return prefix + ":carrier:" + carrier + ":chan:" + chanKey
|
||||
}
|
||||
return prefix + ":chan:" + chanKey
|
||||
}
|
||||
|
||||
func inboundBool(value bool) *bool {
|
||||
return &value
|
||||
}
|
||||
|
||||
type localTokenBucketLimiter struct {
|
||||
mu sync.Mutex
|
||||
buckets map[string]*tokenBucket
|
||||
}
|
||||
|
||||
func newLocalTokenBucketLimiter() *localTokenBucketLimiter {
|
||||
return &localTokenBucketLimiter{buckets: make(map[string]*tokenBucket)}
|
||||
}
|
||||
|
||||
func (l *localTokenBucketLimiter) Wait(ctx context.Context, key string, qps float64, burst int) error {
|
||||
if qps <= 0 {
|
||||
return nil
|
||||
}
|
||||
bucket := l.bucket(key)
|
||||
delay := bucket.reserve(time.Now(), qps, burst)
|
||||
if delay <= 0 {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (l *localTokenBucketLimiter) bucket(key string) *tokenBucket {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
bucket := l.buckets[key]
|
||||
if bucket == nil {
|
||||
bucket = &tokenBucket{}
|
||||
l.buckets[key] = bucket
|
||||
}
|
||||
return bucket
|
||||
}
|
||||
|
||||
type tokenBucket struct {
|
||||
mu sync.Mutex
|
||||
tokens float64
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func (b *tokenBucket) reserve(now time.Time, qps float64, burst int) time.Duration {
|
||||
if qps <= 0 {
|
||||
return 0
|
||||
}
|
||||
capacity := float64(burst)
|
||||
if capacity <= 0 {
|
||||
capacity = defaultInboundUpstreamBurst
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.last.IsZero() {
|
||||
b.last = now
|
||||
b.tokens = capacity
|
||||
}
|
||||
if now.After(b.last) {
|
||||
elapsed := now.Sub(b.last).Seconds()
|
||||
b.tokens += elapsed * qps
|
||||
if b.tokens > capacity {
|
||||
b.tokens = capacity
|
||||
}
|
||||
b.last = now
|
||||
}
|
||||
if b.tokens >= 1 {
|
||||
b.tokens--
|
||||
return 0
|
||||
}
|
||||
waitSeconds := (1 - b.tokens) / qps
|
||||
wait := time.Duration(waitSeconds * float64(time.Second))
|
||||
if wait <= 0 {
|
||||
wait = time.Nanosecond
|
||||
}
|
||||
b.tokens = 0
|
||||
b.last = b.last.Add(wait)
|
||||
return b.last.Sub(now)
|
||||
}
|
||||
|
||||
type redisTokenBucketLimiter struct {
|
||||
redis redisEvaler
|
||||
}
|
||||
|
||||
const redisTokenBucketScript = `
|
||||
local now = tonumber(ARGV[1])
|
||||
local qps = tonumber(ARGV[2])
|
||||
local burst = tonumber(ARGV[3])
|
||||
local ttl = tonumber(ARGV[4])
|
||||
|
||||
if qps == nil or qps <= 0 then
|
||||
return {1, 0}
|
||||
end
|
||||
if burst == nil or burst <= 0 then
|
||||
burst = 1
|
||||
end
|
||||
if ttl == nil or ttl <= 0 then
|
||||
ttl = 60000
|
||||
end
|
||||
|
||||
local values = redis.call("HMGET", KEYS[1], "tokens", "updated_at")
|
||||
local tokens = tonumber(values[1])
|
||||
local updated_at = tonumber(values[2])
|
||||
local capacity = burst
|
||||
|
||||
if tokens == nil or updated_at == nil then
|
||||
tokens = capacity
|
||||
updated_at = now
|
||||
end
|
||||
if tokens > capacity then
|
||||
tokens = capacity
|
||||
end
|
||||
if now > updated_at then
|
||||
local elapsed = (now - updated_at) / 1000
|
||||
tokens = math.min(capacity, tokens + elapsed * qps)
|
||||
updated_at = now
|
||||
end
|
||||
|
||||
local wait = 0
|
||||
if tokens >= 1 then
|
||||
tokens = tokens - 1
|
||||
else
|
||||
wait = math.ceil((1 - tokens) / qps * 1000)
|
||||
if wait < 1 then
|
||||
wait = 1
|
||||
end
|
||||
tokens = 0
|
||||
updated_at = updated_at + wait
|
||||
wait = updated_at - now
|
||||
if wait < 0 then
|
||||
wait = 0
|
||||
end
|
||||
end
|
||||
|
||||
redis.call("HSET", KEYS[1], "tokens", tostring(tokens), "updated_at", tostring(updated_at))
|
||||
redis.call("PEXPIRE", KEYS[1], ttl)
|
||||
return {1, wait}
|
||||
`
|
||||
|
||||
func (l *redisTokenBucketLimiter) Wait(ctx context.Context, key string, qps float64, burst int) error {
|
||||
wait, err := l.reserve(ctx, key, qps, burst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if wait <= 0 {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(wait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (l *redisTokenBucketLimiter) reserve(ctx context.Context, key string, qps float64, burst int) (time.Duration, error) {
|
||||
if l == nil || l.redis == nil || qps <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if burst <= 0 {
|
||||
burst = defaultInboundUpstreamBurst
|
||||
}
|
||||
reply, err := l.redis.Eval(ctx, redisTokenBucketScript, 1, []string{key}, []any{
|
||||
time.Now().UnixMilli(),
|
||||
strconv.FormatFloat(qps, 'f', -1, 64),
|
||||
burst,
|
||||
rateLimitRedisKeyTTLMillis(qps, burst),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
values := reply.Int64s()
|
||||
if len(values) < 2 || values[0] != 1 {
|
||||
return 0, fmt.Errorf("unexpected redis rate limiter reply: %v", reply.Val())
|
||||
}
|
||||
waitMillis := values[1]
|
||||
if waitMillis <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return time.Duration(waitMillis) * time.Millisecond, nil
|
||||
}
|
||||
|
||||
func rateLimitRedisKeyTTLMillis(qps float64, burst int) int64 {
|
||||
if qps <= 0 {
|
||||
return int64(time.Minute / time.Millisecond)
|
||||
}
|
||||
if burst <= 0 {
|
||||
burst = defaultInboundUpstreamBurst
|
||||
}
|
||||
idleMillis := int64(math.Ceil(float64(burst) / qps * 1000 * 2))
|
||||
minMillis := int64(time.Minute / time.Millisecond)
|
||||
if idleMillis < minMillis {
|
||||
return minMillis
|
||||
}
|
||||
return idleMillis
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package inbound
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestInboundCarrierForOperatorUsesDictionaryValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
operator int
|
||||
carrier string
|
||||
ok bool
|
||||
}{
|
||||
{name: "mobile", operator: 1, carrier: inboundCarrierMobile, ok: true},
|
||||
{name: "telecom", operator: 2, carrier: inboundCarrierTelecom, ok: true},
|
||||
{name: "legacy telecom value is unsupported", operator: 3, carrier: "", ok: false},
|
||||
{name: "missing", operator: 0, carrier: "", ok: false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
carrier, ok := inboundCarrierForOperator(test.operator)
|
||||
if carrier != test.carrier || ok != test.ok {
|
||||
t.Fatalf("inboundCarrierForOperator(%d) = %q,%v, want %q,%v", test.operator, carrier, ok, test.carrier, test.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveInboundUpstreamRateLimitFromChannelMetadataUsesCarrierPolicy(t *testing.T) {
|
||||
cfg := defaultInboundUpstreamRateLimitConfig()
|
||||
|
||||
mobile := resolveInboundUpstreamRateLimitFromChannelMetadata(cfg, "chan-mobile", 1, "sim_yd_v5")
|
||||
if mobile.carrier != inboundCarrierMobile {
|
||||
t.Fatalf("mobile carrier = %q, want %q", mobile.carrier, inboundCarrierMobile)
|
||||
}
|
||||
if mobile.enabled {
|
||||
t.Fatalf("mobile enabled = %v, want false", mobile.enabled)
|
||||
}
|
||||
|
||||
telecom := resolveInboundUpstreamRateLimitFromChannelMetadata(cfg, "chan-telecom", 2, "sim_dx_v1")
|
||||
if telecom.carrier != inboundCarrierTelecom {
|
||||
t.Fatalf("telecom carrier = %q, want %q", telecom.carrier, inboundCarrierTelecom)
|
||||
}
|
||||
if !telecom.enabled {
|
||||
t.Fatalf("telecom enabled = %v, want true", telecom.enabled)
|
||||
}
|
||||
if telecom.qps != defaultInboundUpstreamQPS {
|
||||
t.Fatalf("telecom qps = %v, want %v", telecom.qps, defaultInboundUpstreamQPS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelecomTemplateOverridesInconsistentOperatorMetadata(t *testing.T) {
|
||||
for _, key := range []string{"sim_dx_v1", "sim_dx_v3"} {
|
||||
for _, operator := range []int{0, 1, 2, 3} {
|
||||
setting := resolveInboundUpstreamRateLimitFromChannelMetadata(defaultInboundUpstreamRateLimitConfig(), "telecom", operator, key)
|
||||
if setting.carrier != inboundCarrierTelecom || !setting.enabled || setting.skipReason != "" {
|
||||
t.Fatalf("template %q operator %d: %+v", key, operator, setting)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/encoding/gjson"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/service/batchresult"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
Load(ctx context.Context, batchNo string) (*Task, error)
|
||||
ReadOssFile(ctx context.Context, ossID int64) ([]byte, string, error)
|
||||
FindExistingStocks(ctx context.Context, tenantID string, iccids, msisdns, imsis []string) ([]Stock, error)
|
||||
SaveValidationFailure(ctx context.Context, task *Task) error
|
||||
SaveSuccess(ctx context.Context, task *Task) error
|
||||
SaveBusinessFailure(ctx context.Context, task *Task, reason string) error
|
||||
SaveResultFile(ctx context.Context, task *Task, request batchresult.Request) error
|
||||
}
|
||||
|
||||
type Upstream interface {
|
||||
QuerySimBasicInfo(ctx context.Context, input *model.SimBasicInfoInput) ([]model.SimBaseRes[model.SimBasicInfoItem], error)
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
Batch Batch
|
||||
Source Source
|
||||
Options Options
|
||||
Rows []*Row
|
||||
ValidCount int
|
||||
FailureReason string
|
||||
}
|
||||
|
||||
type Batch struct {
|
||||
ID int64
|
||||
BatchNo string
|
||||
BatchType int
|
||||
SourceType int
|
||||
TenantID string
|
||||
CreatedDept int64
|
||||
CreatedBy int64
|
||||
StartTime *gtime.Time
|
||||
BatchInfo *gjson.Json
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
ImportFileOssID int64
|
||||
InputText string
|
||||
InputFileOssID int64
|
||||
Range Range
|
||||
}
|
||||
|
||||
type Range struct {
|
||||
RangeType string
|
||||
StartICCID string
|
||||
EndICCID string
|
||||
StartMSISDN string
|
||||
EndMSISDN string
|
||||
StartIMSI string
|
||||
EndIMSI string
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
ChanKey string
|
||||
ValidateUpstreamBasicInfo string
|
||||
CardType string
|
||||
UpPkgCode string
|
||||
SuppliersCode string
|
||||
UpstreamActivationTime string
|
||||
InsertToPlugEnabled string
|
||||
CarrierTrafficCutoffEnabled string
|
||||
AgreementExpiryDate string
|
||||
HasTestPeriod string
|
||||
TestPeriodDays string
|
||||
TestPeriodLatestExpiryDate string
|
||||
HasSilentPeriod string
|
||||
SilentPeriodDays string
|
||||
SilentPeriodLatestExpiryDate string
|
||||
}
|
||||
|
||||
type Row struct {
|
||||
RowNo int
|
||||
InputCode string
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
IMEI string
|
||||
ChanKey string
|
||||
CardType string
|
||||
UpPkgCode string
|
||||
SuppliersCode string
|
||||
UpstreamActivationTime string
|
||||
InsertToPlugEnabled string
|
||||
CarrierTrafficCutoffEnabled string
|
||||
AgreementExpiryDate string
|
||||
HasTestPeriod string
|
||||
TestPeriodDays string
|
||||
TestPeriodLatestExpiryDate string
|
||||
HasSilentPeriod string
|
||||
SilentPeriodDays string
|
||||
SilentPeriodLatestExpiryDate string
|
||||
Reason string
|
||||
Raw map[string]string
|
||||
}
|
||||
|
||||
type Stock struct {
|
||||
ICCID string `orm:"iccid"`
|
||||
MSISDN string `orm:"msisdn"`
|
||||
IMSI string `orm:"imsi"`
|
||||
}
|
||||
Reference in New Issue
Block a user