chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,95 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
goredis "github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
type leaseClientFactory func() (queue.RedisClient, func() error, error)
|
||||
|
||||
// A custom client must explicitly supply the same bounded-I/O contract. There
|
||||
// is no fallback to context cancellation on an unknown Redis implementation.
|
||||
type boundedLeaseProvider interface {
|
||||
newBoundedLeaseClient(time.Duration) (queue.RedisClient, func() error, error)
|
||||
}
|
||||
|
||||
func newLeaseClientFactory(client queue.RedisClient, config *gredis.Config, budget time.Duration) (leaseClientFactory, error) {
|
||||
if budget <= 0 {
|
||||
return nil, gerror.New("IMEI lease I/O budget must be positive")
|
||||
}
|
||||
if provider, ok := client.(boundedLeaseProvider); ok {
|
||||
return func() (queue.RedisClient, func() error, error) { return provider.newBoundedLeaseClient(budget) }, nil
|
||||
}
|
||||
getter, ok := client.(interface{ GetAdapter() gredis.Adapter })
|
||||
if !ok || getter.GetAdapter() == nil {
|
||||
return nil, gerror.New("IMEI producer requires a bounded GoFrame Redis adapter")
|
||||
}
|
||||
raw, ok := getter.GetAdapter().Client().(*goredis.Client)
|
||||
if !ok {
|
||||
return nil, gerror.New("IMEI producer does not support Redis Cluster or an unknown Redis adapter")
|
||||
}
|
||||
opts := *raw.Options()
|
||||
if opts.Addr == "FailoverClient" {
|
||||
if config == nil || config.MasterName == "" || config.Cluster || config.SlaveOnly {
|
||||
return nil, gerror.New("IMEI Sentinel lease client requires its master configuration")
|
||||
}
|
||||
// Rebuild Sentinel discovery too: sharing the old dialer would retain
|
||||
// its unbounded Sentinel socket reads even if master I/O were bounded.
|
||||
cfg := *config
|
||||
return func() (queue.RedisClient, func() error, error) {
|
||||
client := goredis.NewFailoverClient(&goredis.FailoverOptions{
|
||||
MasterName: cfg.MasterName, SentinelAddrs: sentinelAddresses(cfg.Address),
|
||||
SentinelUsername: cfg.SentinelUser, SentinelPassword: cfg.SentinelPass,
|
||||
Username: cfg.User, Password: cfg.Pass, DB: cfg.Db, Protocol: cfg.Protocol,
|
||||
TLSConfig: opts.TLSConfig,
|
||||
DialTimeout: budget, ReadTimeout: budget, WriteTimeout: budget, PoolTimeout: budget,
|
||||
ContextTimeoutEnabled: true, MaxRetries: -1, DialerRetries: 1,
|
||||
PoolSize: 1, MaxActiveConns: 1,
|
||||
})
|
||||
return &leaseRedis{client}, client.Close, nil
|
||||
}, nil
|
||||
}
|
||||
// Own a separate pool so a timeout/close never closes a consumer's socket.
|
||||
opts.DialTimeout, opts.ReadTimeout, opts.WriteTimeout, opts.PoolTimeout = budget, budget, budget, budget
|
||||
opts.ContextTimeoutEnabled = true
|
||||
// Maintenance handoffs can relax socket deadlines; leases must never do so.
|
||||
opts.MaintNotificationsConfig = &maintnotifications.Config{Mode: maintnotifications.ModeDisabled}
|
||||
opts.MaxRetries, opts.DialerRetries = -1, 1
|
||||
opts.PoolSize, opts.MaxActiveConns, opts.MinIdleConns, opts.MaxIdleConns = 1, 1, 0, 1
|
||||
opts.Dialer = goredis.NewDialer(&opts)
|
||||
return func() (queue.RedisClient, func() error, error) {
|
||||
client := goredis.NewClient(&opts)
|
||||
return &leaseRedis{client}, client.Close, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sentinelAddresses(address string) []string {
|
||||
var addresses []string
|
||||
for _, addr := range strings.Split(address, ",") {
|
||||
if addr = strings.TrimSpace(addr); addr != "" {
|
||||
addresses = append(addresses, addr)
|
||||
}
|
||||
}
|
||||
return addresses
|
||||
}
|
||||
|
||||
type leaseRedis struct{ client *goredis.Client }
|
||||
|
||||
func (r *leaseRedis) Do(ctx context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
result, err := r.client.Do(ctx, append([]any{command}, args...)...).Result()
|
||||
return gvar.New(result), err
|
||||
}
|
||||
|
||||
func (r *leaseRedis) Eval(ctx context.Context, script string, _ int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
result, err := r.client.Eval(ctx, script, keys, args...).Result()
|
||||
return gvar.New(result), err
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
goredis "github.com/redis/go-redis/v9"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
// This proxy withholds a real Redis request/reply without closing its TCP socket.
|
||||
func readLeaseRESP(r *bufio.Reader) ([]byte, error) {
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch line[0] {
|
||||
case '$':
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(line[1:])))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n >= 0 {
|
||||
payload := make([]byte, n+2)
|
||||
if _, err := io.ReadFull(r, payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
line = append(line, payload...)
|
||||
}
|
||||
case '*', '%', '~', '>', '|':
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(line[1:])))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if line[0] == '%' || line[0] == '|' {
|
||||
n *= 2
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
payload, err := readLeaseRESP(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
line = append(line, payload...)
|
||||
}
|
||||
}
|
||||
return line, nil
|
||||
}
|
||||
|
||||
func TestProduceBoundsRealRedisRenewalIO(t *testing.T) {
|
||||
for _, mode := range []string{"standalone", "sentinel"} {
|
||||
t.Run(mode, func(t *testing.T) { testProduceBoundsRealRedisRenewalIO(t, mode) })
|
||||
}
|
||||
}
|
||||
|
||||
func testProduceBoundsRealRedisRenewalIO(t *testing.T, mode string) {
|
||||
for _, callerCancel := range []bool{false, true} {
|
||||
t.Run(strconv.FormatBool(callerCancel), func(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
real, held, release := leaseRedisProxy(t, backend, "redis.call('PEXPIRE'", true)
|
||||
// Release the gate even on a red-test failure so no worker leaks.
|
||||
defer release()
|
||||
opts := real.GetAdapter().Client().(*goredis.Client).Options()
|
||||
if opts.ReadTimeout != 0 || opts.WriteTimeout != 0 || opts.ContextTimeoutEnabled {
|
||||
t.Fatal("test must start with production-default unbounded socket I/O")
|
||||
}
|
||||
cause := errors.New("snapshot failed independently")
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
defer cancel(nil)
|
||||
scanner := &lockTestScanner{snapshot: func(workCtx context.Context) (int64, error) {
|
||||
select {
|
||||
case <-held:
|
||||
if callerCancel {
|
||||
cancel(cause)
|
||||
<-workCtx.Done()
|
||||
return 0, workCtx.Err()
|
||||
}
|
||||
return 0, cause
|
||||
case <-workCtx.Done():
|
||||
return 0, workCtx.Err()
|
||||
}
|
||||
}}
|
||||
store, err := queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := Config{RunLockTTL: time.Second}
|
||||
if mode == "sentinel" {
|
||||
real, config.LeaseRedisConfig = sentinelLeaseRedis(t, real)
|
||||
store, err = queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
producer, err := New(scanner, store, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "lock-test"}) }()
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("proxy did not hold renewal response")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, cause) {
|
||||
t.Fatalf("lost original cause: %v", err)
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("canceled renewal socket I/O blocked cleanup beyond its lease budget")
|
||||
}
|
||||
if producer.running.Load() != 0 {
|
||||
t.Fatal("running flag was not reset")
|
||||
}
|
||||
lock, err := backend.Do(context.Background(), "GET", producer.config.RunLockKey)
|
||||
if err != nil || !lock.IsNil() {
|
||||
t.Fatalf("lock not released: %v, %v", lock, err)
|
||||
}
|
||||
if opts.ReadTimeout != 0 || opts.WriteTimeout != 0 {
|
||||
t.Fatal("lease client mutated the shared production client")
|
||||
}
|
||||
// A fresh run must start without opening the stalled gate.
|
||||
producer.scanner = cardScanner{}
|
||||
if err := producer.Produce(context.Background(), model.ImeiQueueStartMessage{RunID: "next-run"}); err != nil {
|
||||
t.Fatalf("producer cannot restart: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceFencesDelayedRealRedisWrites(t *testing.T) {
|
||||
for _, mode := range []string{"standalone", "sentinel"} {
|
||||
t.Run(mode, func(t *testing.T) { testProduceFencesDelayedRealRedisWrites(t, mode) })
|
||||
}
|
||||
}
|
||||
|
||||
func testProduceFencesDelayedRealRedisWrites(t *testing.T, mode string) {
|
||||
for _, stage := range []string{"enqueue", "producer_done"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
match := "local taskID=ARGV[i]"
|
||||
cards := []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}}
|
||||
if stage == "producer_done" {
|
||||
match = "'producerCompletedAt'"
|
||||
cards = nil
|
||||
}
|
||||
real, held, release := leaseRedisProxy(t, backend, match, false)
|
||||
defer release()
|
||||
workContexts := make(chan context.Context, 1)
|
||||
scanner := &lockTestScanner{cardScanner: cardScanner{cards: cards}}
|
||||
scanner.snapshot = func(ctx context.Context) (int64, error) { workContexts <- ctx; return int64(len(cards)), nil }
|
||||
store, err := queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := Config{RunLockTTL: time.Second}
|
||||
if mode == "sentinel" {
|
||||
real, config.LeaseRedisConfig = sentinelLeaseRedis(t, real)
|
||||
store, err = queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
producer, err := New(scanner, store, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("proxy did not hold old-owner EVAL")
|
||||
}
|
||||
workCtx := <-workContexts
|
||||
observer, err := queue.New(backend, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ok, err := observer.AcquireLockOrTakeover(context.Background(), producer.config.RunLockKey, "new-owner", "lock-test", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("takeover: %v, %v", ok, err)
|
||||
}
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("real renewal did not detect takeover")
|
||||
}
|
||||
if !errors.Is(context.Cause(workCtx), ErrAlreadyRunning) {
|
||||
t.Fatalf("wrong loss cause: %v", context.Cause(workCtx))
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("wrong result: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not finish")
|
||||
}
|
||||
meta, err := observer.GetRunMeta(context.Background(), "lock-test")
|
||||
if err != nil || meta.ActiveCount != 0 || meta.ProducerDone {
|
||||
t.Fatalf("stale owner changed run state: %+v, %v", meta, err)
|
||||
}
|
||||
channels, err := observer.RunChannels(context.Background(), "lock-test")
|
||||
if err != nil || len(channels) != 0 {
|
||||
t.Fatalf("stale owner indexed channel: %v, %v", channels, err)
|
||||
}
|
||||
ok, err = observer.RenewLock(context.Background(), producer.config.RunLockKey, "new-owner", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("old cleanup removed new lease: %v, %v", ok, err)
|
||||
}
|
||||
added, err := observer.EnqueueRunBatch(context.Background(), "lock-test", "ch", []queue.Task{{TaskID: "next", RunID: "lock-test", ChanKey: "ch", CardID: 2}}, time.Now())
|
||||
if err != nil || added != 1 {
|
||||
t.Fatalf("new work rejected: added=%d, %v", added, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func leaseRedisProxy(t *testing.T, upstream *gredis.Redis, match string, holdReply bool) (*gredis.Redis, <-chan struct{}, func()) {
|
||||
t.Helper()
|
||||
raw := upstream.GetAdapter().Client().(*goredis.Client)
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, gate := make(chan struct{}), make(chan struct{})
|
||||
var heldOnce, releaseOnce sync.Once
|
||||
release := func() { releaseOnce.Do(func() { close(gate) }) }
|
||||
var connections sync.Map
|
||||
go func() {
|
||||
for {
|
||||
down, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
connections.Store(down, struct{}{})
|
||||
go func() {
|
||||
defer down.Close()
|
||||
defer connections.Delete(down)
|
||||
up, err := net.Dial("tcp", raw.Options().Addr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer up.Close()
|
||||
dr, ur := bufio.NewReader(down), bufio.NewReader(up)
|
||||
for {
|
||||
request, err := readLeaseRESP(dr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
holding := strings.Contains(string(request), match)
|
||||
if holding && !holdReply {
|
||||
heldOnce.Do(func() { close(held) })
|
||||
<-gate
|
||||
}
|
||||
if _, err := up.Write(request); err != nil {
|
||||
return
|
||||
}
|
||||
response, err := readLeaseRESP(ur)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if holding && holdReply {
|
||||
heldOnce.Do(func() { close(held) })
|
||||
<-gate
|
||||
}
|
||||
if _, err := down.Write(response); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
release()
|
||||
_ = listener.Close()
|
||||
connections.Range(func(key, _ any) bool { _ = key.(net.Conn).Close(); return true })
|
||||
})
|
||||
client, err := gredis.New(&gredis.Config{Address: listener.Addr().String(), DialTimeout: 100 * time.Millisecond, Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = client.Close(context.Background()) })
|
||||
if _, err := client.Do(context.Background(), "PING"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := client.GetAdapter().Client().(*goredis.Client).Options()
|
||||
t.Logf("real driver options: readTimeout=%s writeTimeout=%s contextTimeoutEnabled=%v", opts.ReadTimeout, opts.WriteTimeout, opts.ContextTimeoutEnabled)
|
||||
return client, held, release
|
||||
}
|
||||
|
||||
// A real disposable Sentinel discovers the proxy as its master; neither client
|
||||
// configuration nor server state comes from the application's Redis settings.
|
||||
func sentinelLeaseRedis(t *testing.T, master *gredis.Redis) (*gredis.Redis, *gredis.Config) {
|
||||
t.Helper()
|
||||
binary := os.Getenv("REDIS_SERVER_BINARY")
|
||||
if binary == "" {
|
||||
var err error
|
||||
binary, err = exec.LookPath("redis-server")
|
||||
if err != nil {
|
||||
t.Skip("Sentinel tests require redis-server")
|
||||
}
|
||||
}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address := listener.Addr().String()
|
||||
_, port, _ := net.SplitHostPort(address)
|
||||
_ = listener.Close()
|
||||
masterAddress := master.GetAdapter().Client().(*goredis.Client).Options().Addr
|
||||
host, masterPort, err := net.SplitHostPort(masterAddress)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "sentinel.conf")
|
||||
contents := "bind 127.0.0.1\nport " + port + "\nsentinel monitor lease-test " + host + " " + masterPort + " 1\nsentinel down-after-milliseconds lease-test 60000\n"
|
||||
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
command := exec.Command(binary, filepath.Base(path), "--sentinel")
|
||||
command.Dir = dir
|
||||
logPath := filepath.Join(dir, "sentinel.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
command.Stdout, command.Stderr = logFile, logFile
|
||||
if err := command.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = command.Process.Kill(); _ = command.Wait() })
|
||||
probe, err := gredis.New(&gredis.Config{Address: address, ReadTimeout: 100 * time.Millisecond, WriteTimeout: 100 * time.Millisecond, DialTimeout: 100 * time.Millisecond, Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = probe.Close(context.Background()) })
|
||||
for deadline := time.Now().Add(5 * time.Second); ; {
|
||||
if _, err = probe.Do(context.Background(), "SENTINEL", "get-master-addr-by-name", "lease-test"); err == nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
log, _ := os.ReadFile(logPath)
|
||||
t.Fatalf("isolated Sentinel did not start: %v; %s", err, log)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
config := &gredis.Config{Address: address, MasterName: "lease-test", DialTimeout: 100 * time.Millisecond, Protocol: 2}
|
||||
client, err := gredis.New(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = client.Close(context.Background()) })
|
||||
return client, config
|
||||
}
|
||||
|
||||
func TestLeaseClientBoundsSentinelDiscoveryIO(t *testing.T) {
|
||||
master := testutil.Redis(t)
|
||||
_, config := sentinelLeaseRedis(t, master)
|
||||
sentinel, err := gredis.New(&gredis.Config{Address: config.Address, ReadTimeout: time.Second, WriteTimeout: time.Second, Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sentinel.Close(context.Background()) })
|
||||
proxy, held, release := leaseRedisProxy(t, sentinel, "get-master-addr-by-name", true)
|
||||
defer release()
|
||||
config.Address = proxy.GetAdapter().Client().(*goredis.Client).Options().Addr
|
||||
shared, err := gredis.New(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = shared.Close(context.Background()) })
|
||||
factory, err := newLeaseClientFactory(shared, config, 100*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, closeClient, err := factory()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeClient()
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := client.Do(context.Background(), "PING"); result <- err }()
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Sentinel discovery request was not held")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if err == nil {
|
||||
t.Fatal("stalled Sentinel discovery unexpectedly succeeded")
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("Sentinel discovery has no socket I/O upper bound")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaseClientRejectsClusterAndUnknownAdapter(t *testing.T) {
|
||||
cluster, err := gredis.New(&gredis.Config{Address: "127.0.0.1:1", Cluster: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cluster.Close(context.Background())
|
||||
store, err := queue.New(cluster, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := New(cardScanner{}, store, Config{}); err == nil || !strings.Contains(err.Error(), "Cluster") {
|
||||
t.Fatalf("Cluster must be rejected at initialization before any write: %v", err)
|
||||
}
|
||||
if _, err := newLeaseClientFactory(&leaseRedis{}, nil, time.Second); err == nil {
|
||||
t.Fatal("unknown adapter must not fall back to unbounded context-only I/O")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceBoundsReleaseIO(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
real, held, release := leaseRedisProxy(t, backend, "redis.call('DEL', KEYS[1])", true)
|
||||
defer release()
|
||||
store, err := queue.New(real, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(cardScanner{}, store, Config{RunLockTTL: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("release response was not held")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("release socket I/O blocked cleanup")
|
||||
}
|
||||
if producer.running.Load() != 0 {
|
||||
t.Fatal("cleanup did not reset running")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaseClientBoundsSocketWriteIO(t *testing.T) {
|
||||
backend := testutil.Redis(t)
|
||||
upstream := backend.GetAdapter().Client().(*goredis.Client).Options().Addr
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stopReading atomic.Bool
|
||||
held, gate := make(chan struct{}), make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
release := func() { releaseOnce.Do(func() { close(gate) }) }
|
||||
defer release()
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
go func() {
|
||||
down, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer down.Close()
|
||||
up, err := net.Dial("tcp", upstream)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer up.Close()
|
||||
dr, ur := bufio.NewReader(down), bufio.NewReader(up)
|
||||
for {
|
||||
// Read only the array header once armed: the large EVAL payload
|
||||
// cannot enter the upstream or fit in TCP buffers, so Write blocks.
|
||||
header, err := dr.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if stopReading.Load() {
|
||||
close(held)
|
||||
<-gate
|
||||
return
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(header[1:])))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
part, err := readLeaseRESP(dr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
header = append(header, part...)
|
||||
}
|
||||
if _, err := up.Write(header); err != nil {
|
||||
return
|
||||
}
|
||||
response, err := readLeaseRESP(ur)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := down.Write(response); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
shared, err := gredis.New(&gredis.Config{Address: listener.Addr().String(), Protocol: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer shared.Close(context.Background())
|
||||
factory, err := newLeaseClientFactory(shared, nil, 100*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, closeClient, err := factory()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeClient()
|
||||
if _, err := client.Do(context.Background(), "PING"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := client.(*leaseRedis).client.Options()
|
||||
if opts.ReadTimeout != 100*time.Millisecond || opts.WriteTimeout != 100*time.Millisecond || !opts.ContextTimeoutEnabled || opts.MaxRetries != 0 || opts.DialerRetries != 1 {
|
||||
t.Fatalf("lease client has unsafe timeout/retry options")
|
||||
}
|
||||
stopReading.Store(true)
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.Eval(context.Background(), "return 1 --"+strings.Repeat("x", 32<<20), 0, nil, nil)
|
||||
result <- err
|
||||
}()
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("write proxy did not stop reading")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
var timeout net.Error
|
||||
if !errors.As(err, &timeout) || !timeout.Timeout() || !strings.Contains(err.Error(), "write") {
|
||||
t.Fatalf("expected real socket write timeout, got %v", err)
|
||||
}
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
release()
|
||||
<-result
|
||||
t.Fatal("socket write has no upper bound")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type lockTestRedis struct {
|
||||
queue.RedisClient
|
||||
do func(context.Context, string, []any) (*gvar.Var, error)
|
||||
eval func(context.Context, string, int64, []string, []any) (*gvar.Var, error)
|
||||
}
|
||||
|
||||
func (r *lockTestRedis) newBoundedLeaseClient(budget time.Duration) (queue.RedisClient, func() error, error) {
|
||||
factory, err := newLeaseClientFactory(r.RedisClient, nil, budget)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client, closeClient, err := factory()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return &lockTestRedis{RedisClient: client,
|
||||
do: func(ctx context.Context, command string, args []any) (*gvar.Var, error) {
|
||||
if r.do != nil {
|
||||
return r.do(ctx, command, args)
|
||||
}
|
||||
return client.Do(ctx, command, args...)
|
||||
},
|
||||
eval: func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if r.eval != nil {
|
||||
return r.eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
return client.Eval(ctx, script, numKeys, keys, args)
|
||||
},
|
||||
}, closeClient, nil
|
||||
}
|
||||
|
||||
func (r *lockTestRedis) Do(ctx context.Context, command string, args ...any) (*gvar.Var, error) {
|
||||
if r.do != nil {
|
||||
return r.do(ctx, command, args)
|
||||
}
|
||||
return r.RedisClient.Do(ctx, command, args...)
|
||||
}
|
||||
|
||||
func (r *lockTestRedis) Eval(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if r.eval != nil {
|
||||
return r.eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
return r.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
|
||||
type lockTestScanner struct {
|
||||
cardScanner
|
||||
scanCalls atomic.Int32
|
||||
snapshot func(context.Context) (int64, error)
|
||||
scan func(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
func (s *lockTestScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
if s.snapshot != nil {
|
||||
return s.snapshot(ctx)
|
||||
}
|
||||
return s.cardScanner.SnapshotMaxID(ctx)
|
||||
}
|
||||
|
||||
func (s *lockTestScanner) Scan(ctx context.Context, last, max int64, limit int) ([]Card, error) {
|
||||
s.scanCalls.Add(1)
|
||||
if s.scan != nil {
|
||||
return s.scan(ctx, last, max, limit)
|
||||
}
|
||||
return s.cardScanner.Scan(ctx, last, max, limit)
|
||||
}
|
||||
|
||||
func startLockTestProducer(t *testing.T, producer *Producer) <-chan error {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 1)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
result <- producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "lock-test"})
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("producer goroutine did not stop after cleanup cancellation")
|
||||
}
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func TestProduceStopsOnLockLossWhileWaitingForCapacity(t *testing.T) {
|
||||
backendErr := errors.New("renewal backend unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{name: "taken_over", err: ErrAlreadyRunning},
|
||||
{name: "redis_error", err: backendErr},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
client = &lockTestRedis{RedisClient: testutil.Redis(t)}
|
||||
waiting = make(chan struct{})
|
||||
failRenew = make(chan struct{})
|
||||
waitingOnce sync.Once
|
||||
capacityRead atomic.Bool
|
||||
scanner = &lockTestScanner{cardScanner: cardScanner{cards: []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
{ID: 3, ICCID: "c", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
}}}
|
||||
)
|
||||
store, err := queue.New(client, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(scanner, store, Config{
|
||||
PageSize: 2, GlobalHighWaterMark: 1,
|
||||
RunLockTTL: 300 * time.Millisecond, PollInterval: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.do = func(ctx context.Context, command string, args []any) (*gvar.Var, error) {
|
||||
result, err := client.RedisClient.Do(ctx, command, args...)
|
||||
if err == nil && command == "SMEMBERS" && args[0] == store.ActiveRunsKey() {
|
||||
capacityRead.Store(true)
|
||||
}
|
||||
if err == nil && command == "HGETALL" && capacityRead.Load() && result.MapStrStr()["activeCount"] == "2" {
|
||||
waitingOnce.Do(func() { close(waiting) })
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
client.eval = func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if strings.Contains(script, "redis.call('PEXPIRE'") {
|
||||
select {
|
||||
case <-failRenew:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if test.err == backendErr {
|
||||
return nil, backendErr
|
||||
}
|
||||
}
|
||||
return client.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
select {
|
||||
case <-waiting:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not reach backpressure")
|
||||
}
|
||||
// The long poll cannot finish naturally; renewal failure must wake it.
|
||||
select {
|
||||
case err := <-result:
|
||||
t.Fatalf("producer exited before lock loss: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if test.err == ErrAlreadyRunning {
|
||||
ok, err := store.AcquireLockOrTakeover(context.Background(), producer.config.RunLockKey, "new-owner", "lock-test", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("take over lock: acquired=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
close(failRenew)
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, test.err) {
|
||||
t.Fatalf("producer error = %v, want renewal cause %v", err, test.err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("lock loss did not interrupt backpressure before the next capacity poll")
|
||||
}
|
||||
if calls := scanner.scanCalls.Load(); calls != 1 {
|
||||
t.Fatalf("scan calls = %d, want only the first page", calls)
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "lock-test")
|
||||
if err != nil || meta.ProducerDone || meta.ActiveCount != 2 {
|
||||
t.Fatalf("run must remain incomplete without additional tasks: meta=%+v err=%v", meta, err)
|
||||
}
|
||||
if producer.running.Load() != 0 {
|
||||
t.Fatal("producer running flag was not reset")
|
||||
}
|
||||
if test.err == ErrAlreadyRunning {
|
||||
ok, err := store.RenewLock(context.Background(), producer.config.RunLockKey, "new-owner", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("old producer released the new owner's lock: renewed=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceCancelsLockedOperationsWithRenewalCause(t *testing.T) {
|
||||
for _, stage := range []string{"run_init", "snapshot", "scan", "enqueue", "producer_done"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
var (
|
||||
client = &lockTestRedis{RedisClient: testutil.Redis(t)}
|
||||
entered = make(chan context.Context, 1)
|
||||
failRenew = make(chan struct{})
|
||||
cause = errors.New("renewal failed during " + stage)
|
||||
scanner = &lockTestScanner{cardScanner: cardScanner{cards: []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
}}}
|
||||
)
|
||||
block := func(ctx context.Context) error {
|
||||
entered <- ctx
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
if stage == "snapshot" {
|
||||
scanner.snapshot = func(ctx context.Context) (int64, error) { return 0, block(ctx) }
|
||||
}
|
||||
if stage == "scan" {
|
||||
scanner.scan = func(ctx context.Context, _, _ int64, _ int) ([]Card, error) { return nil, block(ctx) }
|
||||
}
|
||||
store, err := queue.New(client, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(scanner, store, Config{RunLockTTL: 300 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.eval = func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if strings.Contains(script, "redis.call('PEXPIRE'") {
|
||||
select {
|
||||
case <-failRenew:
|
||||
return nil, cause
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if stage == "run_init" && strings.Contains(script, "'createdAt', ARGV[12]") ||
|
||||
stage == "enqueue" && strings.Contains(script, "local taskID=ARGV[i]") ||
|
||||
stage == "producer_done" && strings.Contains(script, "'producerCompletedAt'") {
|
||||
return nil, block(ctx)
|
||||
}
|
||||
return client.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
var workCtx context.Context
|
||||
select {
|
||||
case workCtx = <-entered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not enter " + stage)
|
||||
}
|
||||
close(failRenew)
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, cause) || !errors.Is(context.Cause(workCtx), cause) {
|
||||
t.Fatalf("renewal cause lost: result=%v context cause=%v", err, context.Cause(workCtx))
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("lock loss did not interrupt " + stage)
|
||||
}
|
||||
if stage == "run_init" {
|
||||
if scanner.scanCalls.Load() != 0 {
|
||||
t.Fatal("producer scanned after run initialization failed")
|
||||
}
|
||||
return
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "lock-test")
|
||||
wantActive := int64(0)
|
||||
if stage == "producer_done" {
|
||||
wantActive = 1
|
||||
}
|
||||
if err != nil || meta.ProducerDone || meta.ActiveCount != wantActive {
|
||||
t.Fatalf("unexpected work after lock loss: meta=%+v err=%v", meta, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProduceJoinsRenewalBeforeReleasingLock(t *testing.T) {
|
||||
scanErr := errors.New("snapshot failed")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{name: "success"},
|
||||
{name: "scanner_error", err: scanErr},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var (
|
||||
client = &lockTestRedis{RedisClient: testutil.Redis(t)}
|
||||
renewEntered = make(chan struct{})
|
||||
renewCanceled = make(chan struct{})
|
||||
allowRenewReturn = make(chan struct{})
|
||||
resumeSnapshot = make(chan struct{})
|
||||
released = make(chan struct{}, 1)
|
||||
allowReturnOnce sync.Once
|
||||
scanner = &lockTestScanner{}
|
||||
)
|
||||
scanner.snapshot = func(ctx context.Context) (int64, error) {
|
||||
select {
|
||||
case <-resumeSnapshot:
|
||||
return 0, test.err
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
}
|
||||
}
|
||||
store, err := queue.New(client, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
producer, err := New(scanner, store, Config{RunLockTTL: 300 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.eval = func(ctx context.Context, script string, numKeys int64, keys []string, args []any) (*gvar.Var, error) {
|
||||
if strings.Contains(script, "redis.call('PEXPIRE'") {
|
||||
close(renewEntered)
|
||||
<-ctx.Done()
|
||||
close(renewCanceled)
|
||||
<-allowRenewReturn
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if len(keys) == 1 && keys[0] == producer.config.RunLockKey && len(args) == 1 {
|
||||
released <- struct{}{}
|
||||
}
|
||||
return client.RedisClient.Eval(ctx, script, numKeys, keys, args)
|
||||
}
|
||||
result := startLockTestProducer(t, producer)
|
||||
t.Cleanup(func() { allowReturnOnce.Do(func() { close(allowRenewReturn) }) })
|
||||
select {
|
||||
case <-renewEntered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("renewal did not start")
|
||||
}
|
||||
close(resumeSnapshot)
|
||||
select {
|
||||
case <-renewCanceled:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not cancel renewal on exit")
|
||||
}
|
||||
// Keep the in-flight renewal alive to verify cleanup actually joins it.
|
||||
select {
|
||||
case err := <-result:
|
||||
t.Fatalf("producer returned before renewal exited: %v", err)
|
||||
case <-released:
|
||||
t.Fatal("producer released the lock before renewal exited")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
allowReturnOnce.Do(func() { close(allowRenewReturn) })
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, test.err) {
|
||||
t.Fatalf("cleanup changed the production error: got %v, want %v", err, test.err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("producer did not finish after renewal exited")
|
||||
}
|
||||
select {
|
||||
case <-released:
|
||||
default:
|
||||
t.Fatal("producer did not release the lock after joining renewal")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducePreservesCallerCancellationCause(t *testing.T) {
|
||||
producer, store := newProducer(t, nil)
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
defer cancel(nil)
|
||||
cause := errors.New("caller stopped the run")
|
||||
producer.scanner = &lockTestScanner{snapshot: func(ctx context.Context) (int64, error) {
|
||||
cancel(cause)
|
||||
<-ctx.Done()
|
||||
return 0, ctx.Err()
|
||||
}}
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "caller-canceled"}); !errors.Is(err, cause) {
|
||||
t.Fatalf("caller cancellation cause lost: %v", err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(context.Background(), "caller-canceled")
|
||||
if err != nil || meta.ProducerDone || producer.running.Load() != 0 {
|
||||
t.Fatalf("canceled producer did not clean up: meta=%+v err=%v running=%d", meta, err, producer.running.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForCapacityHonorsCancellationWithoutWatermark(t *testing.T) {
|
||||
producer := &Producer{}
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
cause := errors.New("lock lost")
|
||||
cancel(cause)
|
||||
if err := producer.waitForCapacity(ctx); !errors.Is(err, cause) {
|
||||
t.Fatalf("disabled watermark ignored cancellation: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model/entity"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// FullSyncOperation is recorded on the run and used by SyncKindForOperation to
|
||||
// classify the run for the admin monitor.
|
||||
const FullSyncOperation = "imei_full_sync"
|
||||
|
||||
// scopeSampleLimit bounds how many channel names are copied into run metadata.
|
||||
const scopeSampleLimit = 10
|
||||
|
||||
// productionStats mirrors the status producer so every full-sync path reports a
|
||||
// single comparable summary line.
|
||||
type productionStats struct {
|
||||
pages int64
|
||||
scanned int64
|
||||
enqueued int64
|
||||
skippedEmptyChan int64
|
||||
skippedNoTemplate int64
|
||||
skippedSyncDisabled int64
|
||||
skippedOutOfScope int64
|
||||
skippedNoImei int64
|
||||
scopeChannels int
|
||||
channels map[string]struct{}
|
||||
}
|
||||
|
||||
func newProductionStats(scopeChannels int) productionStats {
|
||||
return productionStats{channels: make(map[string]struct{}), scopeChannels: scopeChannels}
|
||||
}
|
||||
|
||||
func (s *productionStats) log(ctx context.Context, status, runID, stage, reason string, startedAt time.Time, maxID int64, err error) {
|
||||
message := fmt.Sprintf("event=imei_queue_producer status=%s runId=%s snapshotMaxId=%d pages=%d scanned=%d enqueued=%d skippedEmptyChan=%d skippedNoTemplate=%d skippedSyncDisabled=%d skippedOutOfScope=%d skippedNoImei=%d channelCount=%d scopeChannels=%d durationMs=%d", status, runID, maxID, s.pages, s.scanned, s.enqueued, s.skippedEmptyChan, s.skippedNoTemplate, s.skippedSyncDisabled, s.skippedOutOfScope, s.skippedNoImei, len(s.channels), s.scopeChannels, time.Since(startedAt).Milliseconds())
|
||||
if s.scanned > 0 {
|
||||
message += fmt.Sprintf(" cardsPerSec=%.2f", float64(s.scanned)/time.Since(startedAt).Seconds())
|
||||
}
|
||||
if stage != "" {
|
||||
message += " stage=" + stage
|
||||
}
|
||||
if reason != "" {
|
||||
message += " reason=" + reason
|
||||
}
|
||||
if err != nil {
|
||||
message += fmt.Sprintf(" error=%q", err.Error())
|
||||
}
|
||||
switch status {
|
||||
case "failed":
|
||||
g.Log().Error(ctx, message)
|
||||
case "skipped":
|
||||
g.Log().Warning(ctx, message)
|
||||
default:
|
||||
g.Log().Info(ctx, message)
|
||||
}
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
PageSize int
|
||||
RunLockKey string
|
||||
RunLockTTL time.Duration
|
||||
PollInterval time.Duration
|
||||
GlobalHighWaterMark int64
|
||||
// LeaseRedisConfig is required only for Sentinel discovery. Standalone
|
||||
// settings come from the actual adapter; Cluster is rejected in New.
|
||||
LeaseRedisConfig *gredis.Config
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.PageSize <= 0 {
|
||||
c.PageSize = 1000
|
||||
}
|
||||
if c.RunLockKey == "" {
|
||||
c.RunLockKey = "imei:producer:active"
|
||||
}
|
||||
if c.RunLockTTL <= 0 {
|
||||
c.RunLockTTL = 5 * time.Minute
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = time.Second
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type Card struct {
|
||||
ID int64
|
||||
ICCID string
|
||||
MSISDN string
|
||||
IMSI string
|
||||
ChanKey string
|
||||
TempKey string
|
||||
Operator int
|
||||
SyncEnabled bool
|
||||
Imei string
|
||||
}
|
||||
|
||||
type Scanner interface {
|
||||
SnapshotMaxID(context.Context) (int64, error)
|
||||
Scan(context.Context, int64, int64, int) ([]Card, error)
|
||||
}
|
||||
|
||||
type channelMetadata struct {
|
||||
ChanKey string `orm:"chan_key"`
|
||||
TempKey string `orm:"temp_key"`
|
||||
Operator int `orm:"operator"`
|
||||
SyncEnabled bool `orm:"sync_enabled"`
|
||||
}
|
||||
|
||||
// DatabaseScanner uses keyset pagination so a large IMEI run does not hold the
|
||||
// entire card table in memory and does not degrade through OFFSET scans.
|
||||
type DatabaseScanner struct {
|
||||
metadata map[string]channelMetadata
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) SnapshotMaxID(ctx context.Context) (int64, error) {
|
||||
var card entity.TSimCardInfo
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id).
|
||||
OrderDesc(columns.Id).
|
||||
Limit(1).
|
||||
Scan(&card); err != nil {
|
||||
return 0, fmt.Errorf("read imei snapshot max id: %w", err)
|
||||
}
|
||||
return card.Id, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseScanner) Scan(ctx context.Context, lastID, maxID int64, pageSize int) ([]Card, error) {
|
||||
if maxID <= lastID {
|
||||
return nil, nil
|
||||
}
|
||||
if s.metadata == nil {
|
||||
var rows []channelMetadata
|
||||
if err := g.DB().Model("t_sim_channels").Ctx(ctx).Fields("chan_key,temp_key,sync_enabled,operator").Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan imei channel metadata: %w", err)
|
||||
}
|
||||
s.metadata = make(map[string]channelMetadata, len(rows))
|
||||
for _, row := range rows {
|
||||
s.metadata[row.ChanKey] = row
|
||||
}
|
||||
}
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
var rows []entity.TSimCardInfo
|
||||
if err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(columns.Id, columns.Iccid, columns.Msisdn, columns.Imsi, columns.ChanKey, columns.Imei).
|
||||
WhereGT(columns.Id, lastID).
|
||||
WhereLTE(columns.Id, maxID).
|
||||
OrderAsc(columns.Id).
|
||||
Limit(pageSize).
|
||||
Scan(&rows); err != nil {
|
||||
return nil, fmt.Errorf("scan imei cards: %w", err)
|
||||
}
|
||||
cards := make([]Card, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.Id <= 0 || row.Iccid == "" || row.ChanKey == "" {
|
||||
// Keep the page cursor moving even when a row cannot become a
|
||||
// runnable task. The producer advances by the highest returned ID.
|
||||
if row.Id > 0 {
|
||||
cards = append(cards, Card{ID: row.Id})
|
||||
}
|
||||
continue
|
||||
}
|
||||
metadata := s.metadata[row.ChanKey]
|
||||
cards = append(cards, Card{ID: row.Id, ICCID: row.Iccid, MSISDN: row.Msisdn, IMSI: row.Imsi, ChanKey: row.ChanKey, TempKey: metadata.TempKey, Operator: metadata.Operator, SyncEnabled: metadata.SyncEnabled, Imei: row.Imei})
|
||||
}
|
||||
return cards, nil
|
||||
}
|
||||
|
||||
type Producer struct {
|
||||
scanner Scanner
|
||||
store *queue.Store
|
||||
config Config
|
||||
running atomic.Int32
|
||||
owner string
|
||||
leaseFactory leaseClientFactory
|
||||
}
|
||||
|
||||
var ErrAlreadyRunning = fmt.Errorf("imei producer already running")
|
||||
|
||||
// ErrRunFinished is returned when a redelivered start message names a run that
|
||||
// already reached a terminal state. The run must not be resurrected.
|
||||
var ErrRunFinished = fmt.Errorf("imei run already finished")
|
||||
|
||||
// runAlreadyFinished reports whether runID exists and has already completed.
|
||||
// A missing run is not an error: the message may be the first delivery.
|
||||
func (p *Producer) runAlreadyFinished(ctx context.Context, runID string) (bool, error) {
|
||||
meta, err := p.store.GetRunMeta(ctx, runID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return meta.Status == "completed" || meta.Status == "completed_with_failures", nil
|
||||
}
|
||||
|
||||
func New(scanner Scanner, store *queue.Store, config Config) (*Producer, error) {
|
||||
if scanner == nil || store == nil {
|
||||
return nil, fmt.Errorf("imei producer dependencies are incomplete")
|
||||
}
|
||||
config = config.withDefaults()
|
||||
factory, err := newLeaseClientFactory(store.RedisClient(), config.LeaseRedisConfig, leaseIOBudget(config.RunLockTTL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Producer{scanner: scanner, store: store, config: config, owner: newOwner(), leaseFactory: factory}, nil
|
||||
}
|
||||
|
||||
func leaseIOBudget(ttl time.Duration) time.Duration {
|
||||
return min(time.Second, ttl/6)
|
||||
}
|
||||
|
||||
func (p *Producer) Produce(ctx context.Context, message model.ImeiQueueStartMessage) (err error) {
|
||||
startedAt := time.Now()
|
||||
if message.RunID == "" {
|
||||
return fmt.Errorf("imei sync runId is empty")
|
||||
}
|
||||
operation := message.Operation
|
||||
if operation == "" {
|
||||
operation = FullSyncOperation
|
||||
}
|
||||
scope, err := model.NormalizeScope(message.Scope)
|
||||
if err != nil {
|
||||
failedStats := newProductionStats(0)
|
||||
failedStats.log(ctx, "failed", message.RunID, "validate", "invalid_scope", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
stats := newProductionStats(scope.ChannelCount())
|
||||
if !p.running.CompareAndSwap(0, 1) {
|
||||
stats.log(ctx, "skipped", message.RunID, "precheck", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer p.running.Store(0)
|
||||
leaseClient, closeLeaseClient, err := p.leaseFactory()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeLeaseClient()
|
||||
leaseStore, err := queue.New(leaseClient, p.store.Prefix())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A redelivered start message must not resurrect a run that already
|
||||
// finished; that would create a second producer pass over the same RunID.
|
||||
if finished, checkErr := p.runAlreadyFinished(ctx, message.RunID); checkErr != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "run_state", "", startedAt, 0, checkErr)
|
||||
return checkErr
|
||||
} else if finished {
|
||||
stats.log(ctx, "skipped", message.RunID, "run_state", "run_finished", startedAt, 0, nil)
|
||||
return ErrRunFinished
|
||||
}
|
||||
// Tokens are per lease, not per Producer: a delayed command from an earlier
|
||||
// attempt must never match a later lease on this same Producer instance.
|
||||
p.owner = newOwner()
|
||||
acquireCtx, cancelAcquire := context.WithTimeout(ctx, leaseIOBudget(p.config.RunLockTTL))
|
||||
acquired, err := leaseStore.AcquireLockOrTakeover(acquireCtx, p.config.RunLockKey, p.owner, message.RunID, p.config.RunLockTTL)
|
||||
cancelAcquire()
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, "lock", "", startedAt, 0, err)
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
stats.log(ctx, "skipped", message.RunID, "lock", "already_running", startedAt, 0, nil)
|
||||
return ErrAlreadyRunning
|
||||
}
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), leaseIOBudget(p.config.RunLockTTL))
|
||||
defer cancel()
|
||||
_, _ = leaseStore.ReleaseLock(releaseCtx, p.config.RunLockKey, p.owner)
|
||||
}()
|
||||
workStore, err := p.store.WithIMEIProducerLease(p.config.RunLockKey, p.owner, message.RunID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockCtx, cancelLock := context.WithCancelCause(ctx)
|
||||
// Stop renewal separately so normal cleanup does not become a lock-loss cause.
|
||||
renewCtx, stopRenew := context.WithCancel(lockCtx)
|
||||
renewDone := make(chan struct{})
|
||||
var (
|
||||
maxID int64
|
||||
stage = "run_init"
|
||||
)
|
||||
go func() {
|
||||
defer close(renewDone)
|
||||
p.renewLock(renewCtx, cancelLock, leaseStore)
|
||||
}()
|
||||
defer func() {
|
||||
stopRenew()
|
||||
<-renewDone
|
||||
// Join before reading the cause and releasing the lease: a failed renewal
|
||||
// must not be masked by a scanner/Redis context-cancellation error.
|
||||
if cause := context.Cause(lockCtx); cause != nil {
|
||||
err = cause
|
||||
}
|
||||
if errors.Is(err, queue.ErrProducerLeaseLost) {
|
||||
err = ErrAlreadyRunning
|
||||
}
|
||||
cancelLock(nil)
|
||||
if err != nil {
|
||||
stats.log(ctx, "failed", message.RunID, stage, "", startedAt, maxID, err)
|
||||
return
|
||||
}
|
||||
stats.log(ctx, "completed", message.RunID, "", "", startedAt, maxID, nil)
|
||||
}()
|
||||
if err = workStore.CreateRun(lockCtx, queue.RunMeta{
|
||||
RunID: message.RunID,
|
||||
Operation: operation,
|
||||
Status: "producing",
|
||||
CurrentRound: 1,
|
||||
ScopeChannels: scope.ChannelCount(),
|
||||
ScopeDigest: scopeDigest(scope),
|
||||
ScopeChannelsSample: strings.Join(scope.ChannelSample(scopeSampleLimit), ","),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "snapshot"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
maxID, err = p.scanner.SnapshotMaxID(lockCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.Log().Infof(ctx, "event=imei_queue_producer status=started runId=%s snapshotMaxId=%d scopeChannels=%d", message.RunID, maxID, scope.ChannelCount())
|
||||
lastID := int64(0)
|
||||
for lastID < maxID {
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stage = "scan"
|
||||
cards, scanErr := p.scanner.Scan(lockCtx, lastID, maxID, p.config.PageSize)
|
||||
if scanErr != nil {
|
||||
return scanErr
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
stats.pages++
|
||||
stats.scanned += int64(len(cards))
|
||||
groups := make(map[string][]queue.Task)
|
||||
for _, card := range cards {
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if card.ID > lastID {
|
||||
lastID = card.ID
|
||||
}
|
||||
if card.ChanKey == "" {
|
||||
stats.skippedEmptyChan++
|
||||
continue
|
||||
}
|
||||
// Scope is applied before the template and sync_enabled checks so a
|
||||
// partial run only reports on the channels it was asked for.
|
||||
if !scope.Allows(card.ChanKey) {
|
||||
stats.skippedOutOfScope++
|
||||
continue
|
||||
}
|
||||
if card.TempKey == "" {
|
||||
stats.skippedNoTemplate++
|
||||
continue
|
||||
}
|
||||
if card.SyncEnabled {
|
||||
// sync_enabled=1(否/不参与) → skip
|
||||
stats.skippedSyncDisabled++
|
||||
continue
|
||||
}
|
||||
// skippedNoImei counts cards that already have a value. They are
|
||||
// still enqueued (this is a full re-poll, so a device swap is
|
||||
// detected), but the count makes the "how much of this pass is
|
||||
// actually filling gaps" question answerable from one log line.
|
||||
if strings.TrimSpace(card.Imei) != "" {
|
||||
stats.skippedNoImei++
|
||||
}
|
||||
stats.channels[card.ChanKey] = struct{}{}
|
||||
task := queue.Task{TaskID: queue.StableTaskID(message.RunID, operation, card.ID), RunID: message.RunID, Operation: operation, ChanKey: card.ChanKey, TempKey: card.TempKey, Operator: card.Operator, CardID: card.ID, ICCID: card.ICCID, MSISDN: card.MSISDN, IMSI: card.IMSI}
|
||||
groups[card.ChanKey] = append(groups[card.ChanKey], task)
|
||||
}
|
||||
for chanKey, tasks := range groups {
|
||||
stage = "enqueue"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
added, enqueueErr := workStore.EnqueueRunBatch(lockCtx, message.RunID, chanKey, tasks, time.Now())
|
||||
if enqueueErr != nil {
|
||||
return fmt.Errorf("enqueue imei channel %s: %w", chanKey, enqueueErr)
|
||||
}
|
||||
stats.enqueued += int64(added)
|
||||
}
|
||||
stage = "capacity"
|
||||
if err = p.waitForCapacity(lockCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
stage = "producer_done"
|
||||
if err = lockCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return workStore.MarkProducerDone(lockCtx, message.RunID)
|
||||
}
|
||||
|
||||
// scopeDigest is a stable short fingerprint of the selected channels.
|
||||
func scopeDigest(scope *model.SyncScope) string {
|
||||
if scope == nil || len(scope.Channels) == 0 {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(scope.Channels, "\n")))
|
||||
return hex.EncodeToString(digest[:8])
|
||||
}
|
||||
|
||||
func (p *Producer) renewLock(ctx context.Context, cancel context.CancelCauseFunc, leaseStore *queue.Store) {
|
||||
interval := p.config.RunLockTTL / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
renewCtx, stop := context.WithTimeout(ctx, leaseIOBudget(p.config.RunLockTTL))
|
||||
ok, err := leaseStore.RenewLock(renewCtx, p.config.RunLockKey, p.owner, p.config.RunLockTTL)
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err == nil && !ok {
|
||||
err = ErrAlreadyRunning
|
||||
}
|
||||
if err != nil {
|
||||
cancel(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) waitForCapacity(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
if p.config.GlobalHighWaterMark <= 0 {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
runs, err := p.store.ActiveRuns(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var total int64
|
||||
for _, runID := range runs {
|
||||
meta, sizeErr := p.store.GetRunMeta(ctx, runID)
|
||||
if sizeErr != nil {
|
||||
return sizeErr
|
||||
}
|
||||
// ActiveCount includes leased tasks; do not count InflightCount twice.
|
||||
total += meta.ActiveCount + meta.NextRoundCount
|
||||
}
|
||||
if total <= p.config.GlobalHighWaterMark {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return context.Cause(ctx)
|
||||
case <-time.After(p.config.PollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newOwner() string {
|
||||
var b [12]byte
|
||||
if _, err := rand.Read(b[:]); err == nil {
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
return fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package imei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
||||
queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
|
||||
)
|
||||
|
||||
type cardScanner struct{ cards []Card }
|
||||
|
||||
func (s cardScanner) SnapshotMaxID(context.Context) (int64, error) { return int64(len(s.cards)), nil }
|
||||
func (s cardScanner) Scan(_ context.Context, last, max int64, limit int) ([]Card, error) {
|
||||
end := last + int64(limit)
|
||||
if end > max {
|
||||
end = max
|
||||
}
|
||||
return s.cards[last:end], nil
|
||||
}
|
||||
|
||||
func newProducer(t *testing.T, cards []Card, config ...Config) (*Producer, *queue.Store) {
|
||||
t.Helper()
|
||||
redis := testutil.Redis(t)
|
||||
store, err := queue.New(redis, "imei:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := Config{PageSize: 1}
|
||||
if len(config) > 0 {
|
||||
cfg = config[0]
|
||||
}
|
||||
producer, err := New(cardScanner{cards: cards}, store, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return producer, store
|
||||
}
|
||||
|
||||
// TestProduceEnqueuesEligibleCards 覆盖 §7 T1：全量入队，
|
||||
// 跳过无通道、无模板、不参与轮询的卡。
|
||||
func TestProduceEnqueuesEligibleCards(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}, // 合格
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5", SyncEnabled: true}, // sync_enabled=1 跳过
|
||||
{ID: 3, ICCID: "c", ChanKey: "ch"}, // 无模板
|
||||
{ID: 4, ICCID: "d", ChanKey: "ch", TempKey: "sim_yd_v5"}, // 合格
|
||||
})
|
||||
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "full"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "full")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.ProducerDone {
|
||||
t.Fatal("producerDone 应为 true")
|
||||
}
|
||||
// 用 ActiveCount 而非 EnqueuedCount：后者是 RunMeta 上的既有死字段
|
||||
// （只有 RoundStats.EnqueuedCount 会被真实写入），状态链路同样如此。
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("activeCount = %d, want 2", meta.ActiveCount)
|
||||
}
|
||||
claims, err := store.ClaimRun(ctx, "full", "ch", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(claims) != 2 {
|
||||
t.Fatalf("claims = %d, want 2", len(claims))
|
||||
}
|
||||
for _, claim := range claims {
|
||||
if claim.Task.CardID != 1 && claim.Task.CardID != 4 {
|
||||
t.Fatalf("不应入队 card %d", claim.Task.CardID)
|
||||
}
|
||||
if claim.Task.Operation != FullSyncOperation {
|
||||
t.Fatalf("operation = %q, want %q", claim.Task.Operation, FullSyncOperation)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceRePollIncludesCardsThatAlreadyHaveImei 是用户决策的直接回归测试：
|
||||
// 全量重查（而非只查空值卡），已有 IMEI 的卡也必须入队，否则无法发现换机。
|
||||
func TestProduceRePollIncludesCardsThatAlreadyHaveImei(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5", Imei: "865466070817587"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5", Imei: ""},
|
||||
})
|
||||
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "repoll"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "repoll")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("全量重查应入队全部合格卡（含已有值），activeCount = %d, want 2", meta.ActiveCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceAppliesChannelScope 覆盖 §7 T2：按通道裁剪。
|
||||
func TestProduceAppliesChannelScope(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "173", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "189", TempKey: "sim_yd_v5"},
|
||||
{ID: 3, ICCID: "c", ChanKey: "173", TempKey: "sim_yd_v5"},
|
||||
})
|
||||
|
||||
scope := &model.SyncScope{Channels: []string{"173"}, TriggerSource: model.TriggerSourceManual}
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "scoped", Scope: scope}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "scoped")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("activeCount = %d, want 2（只入 173 通道）", meta.ActiveCount)
|
||||
}
|
||||
if meta.ScopeChannels != 1 {
|
||||
t.Fatalf("scopeChannels = %d, want 1", meta.ScopeChannels)
|
||||
}
|
||||
if meta.ScopeDigest == "" {
|
||||
t.Fatal("scopeDigest 不应为空")
|
||||
}
|
||||
|
||||
claims, err := store.ClaimRun(ctx, "scoped", "173", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(claims) != 2 {
|
||||
t.Fatalf("173 通道应有 2 张卡，实际 %d", len(claims))
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceRejectsInvalidScope 非法 scope 应在建 Run 前失败。
|
||||
func TestProduceRejectsInvalidScope(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}})
|
||||
|
||||
// 超过 200 个通道
|
||||
tooMany := make([]string, 0, model.MaxScopeChannels+1)
|
||||
for i := 0; i <= model.MaxScopeChannels; i++ {
|
||||
tooMany = append(tooMany, "ch"+string(rune('a'+i%26)))
|
||||
}
|
||||
// 通道名重复会被去重，故用不重复的
|
||||
tooMany = tooMany[:0]
|
||||
for i := 0; i <= model.MaxScopeChannels; i++ {
|
||||
tooMany = append(tooMany, "ch"+padInt(i))
|
||||
}
|
||||
err := producer.Produce(ctx, model.ImeiQueueStartMessage{
|
||||
RunID: "bad-scope",
|
||||
Scope: &model.SyncScope{Channels: tooMany},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("超限 scope 应被拒绝")
|
||||
}
|
||||
if _, runErr := store.GetRunMeta(ctx, "bad-scope"); runErr == nil {
|
||||
t.Fatal("非法 scope 不应创建 Run")
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceIsIdempotentForSameRun 同一 RunID 重复投递不重复入队。
|
||||
func TestProduceIsIdempotentForSameRun(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{
|
||||
{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
{ID: 2, ICCID: "b", ChanKey: "ch", TempKey: "sim_yd_v5"},
|
||||
})
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "dup"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
meta, err := store.GetRunMeta(ctx, "dup")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.ActiveCount != 2 {
|
||||
t.Fatalf("activeCount = %d, want 2（重复投递不应翻倍）", meta.ActiveCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceSkipsFinishedRun 终态 Run 不被重投消息复活。
|
||||
func TestProduceSkipsFinishedRun(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, store := newProducer(t, []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}})
|
||||
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "done"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 排空队列让 Run 进入终态：先领取并终结任务，再收尾。
|
||||
claims, err := store.ClaimRun(ctx, "done", "ch", time.Now(), time.Now().Add(time.Minute), 10)
|
||||
if err != nil || len(claims) != 1 {
|
||||
t.Fatalf("claims=%v err=%v", claims, err)
|
||||
}
|
||||
if _, err = store.FinalizeRunTasks(ctx, claims, []queue.TaskOutcome{{Success: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if completed, err := store.CompleteRunIfIdle(ctx, "done"); err != nil || !completed {
|
||||
t.Fatalf("complete run: completed=%v err=%v", completed, err)
|
||||
}
|
||||
|
||||
err = producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "done"})
|
||||
if !errors.Is(err, ErrRunFinished) {
|
||||
t.Fatalf("终态 Run 重投应返回 ErrRunFinished，实际 %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProduceRejectsEmptyRunID 空 RunID 直接失败。
|
||||
func TestProduceRejectsEmptyRunID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
producer, _ := newProducer(t, []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}})
|
||||
if err := producer.Produce(ctx, model.ImeiQueueStartMessage{}); err == nil {
|
||||
t.Fatal("空 RunID 应被拒绝")
|
||||
}
|
||||
}
|
||||
|
||||
// TestScopeDigestIsStableAcrossOrdering 摘要对通道顺序不敏感，
|
||||
// 使「同一批通道以不同顺序请求」得到同一个 digest。
|
||||
//
|
||||
// 排序由 NormalizeScope 完成，所以必须走归一化后再算 digest
|
||||
// （直接对未排序输入调 scopeDigest 得到不同结果属预期行为）。
|
||||
func TestScopeDigestIsStableAcrossOrdering(t *testing.T) {
|
||||
a, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"173", "189", "216"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := model.NormalizeScope(&model.SyncScope{Channels: []string{"216", "173", "189"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scopeDigest(a) != scopeDigest(b) {
|
||||
t.Fatalf("digest 应对顺序不敏感: %q vs %q", scopeDigest(a), scopeDigest(b))
|
||||
}
|
||||
if scopeDigest(a) == "" {
|
||||
t.Fatal("非空 scope 的 digest 不应为空")
|
||||
}
|
||||
if scopeDigest(nil) != "" {
|
||||
t.Fatal("全量同步的 digest 应为空")
|
||||
}
|
||||
}
|
||||
|
||||
func padInt(value int) string {
|
||||
if value == 0 {
|
||||
return "0"
|
||||
}
|
||||
digits := make([]byte, 0, 8)
|
||||
for value > 0 {
|
||||
digits = append([]byte{byte('0' + value%10)}, digits...)
|
||||
value /= 10
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
Reference in New Issue
Block a user