chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）

2026-11-02 13:28:03 +08:00
parent 6de2e503cc 96f0f4eacd
commit 2923c78755
422 changed files with 80993 additions and 0 deletions
@@ -0,0 +1,576 @@
+package imei
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/gogf/gf/v2/database/gredis"
+	goredis "github.com/redis/go-redis/v9"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
+	queue "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+// This proxy withholds a real Redis request/reply without closing its TCP socket.
+func readLeaseRESP(r *bufio.Reader) ([]byte, error) {
+	line, err := r.ReadBytes('\n')
+	if err != nil {
+		return nil, err
+	}
+	switch line[0] {
+	case '$':
+		n, err := strconv.Atoi(strings.TrimSpace(string(line[1:])))
+		if err != nil {
+			return nil, err
+		}
+		if n >= 0 {
+			payload := make([]byte, n+2)
+			if _, err := io.ReadFull(r, payload); err != nil {
+				return nil, err
+			}
+			line = append(line, payload...)
+		}
+	case '*', '%', '~', '>', '|':
+		n, err := strconv.Atoi(strings.TrimSpace(string(line[1:])))
+		if err != nil {
+			return nil, err
+		}
+		if line[0] == '%' || line[0] == '|' {
+			n *= 2
+		}
+		for i := 0; i < n; i++ {
+			payload, err := readLeaseRESP(r)
+			if err != nil {
+				return nil, err
+			}
+			line = append(line, payload...)
+		}
+	}
+	return line, nil
+}
+
+func TestProduceBoundsRealRedisRenewalIO(t *testing.T) {
+	for _, mode := range []string{"standalone", "sentinel"} {
+		t.Run(mode, func(t *testing.T) { testProduceBoundsRealRedisRenewalIO(t, mode) })
+	}
+}
+
+func testProduceBoundsRealRedisRenewalIO(t *testing.T, mode string) {
+	for _, callerCancel := range []bool{false, true} {
+		t.Run(strconv.FormatBool(callerCancel), func(t *testing.T) {
+			backend := testutil.Redis(t)
+			real, held, release := leaseRedisProxy(t, backend, "redis.call('PEXPIRE'", true)
+			// Release the gate even on a red-test failure so no worker leaks.
+			defer release()
+			opts := real.GetAdapter().Client().(*goredis.Client).Options()
+			if opts.ReadTimeout != 0 || opts.WriteTimeout != 0 || opts.ContextTimeoutEnabled {
+				t.Fatal("test must start with production-default unbounded socket I/O")
+			}
+			cause := errors.New("snapshot failed independently")
+			ctx, cancel := context.WithCancelCause(context.Background())
+			defer cancel(nil)
+			scanner := &lockTestScanner{snapshot: func(workCtx context.Context) (int64, error) {
+				select {
+				case <-held:
+					if callerCancel {
+						cancel(cause)
+						<-workCtx.Done()
+						return 0, workCtx.Err()
+					}
+					return 0, cause
+				case <-workCtx.Done():
+					return 0, workCtx.Err()
+				}
+			}}
+			store, err := queue.New(real, "imei:")
+			if err != nil {
+				t.Fatal(err)
+			}
+			config := Config{RunLockTTL: time.Second}
+			if mode == "sentinel" {
+				real, config.LeaseRedisConfig = sentinelLeaseRedis(t, real)
+				store, err = queue.New(real, "imei:")
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			producer, err := New(scanner, store, config)
+			if err != nil {
+				t.Fatal(err)
+			}
+			result := make(chan error, 1)
+			go func() { result <- producer.Produce(ctx, model.ImeiQueueStartMessage{RunID: "lock-test"}) }()
+			select {
+			case <-held:
+			case <-time.After(2 * time.Second):
+				t.Fatal("proxy did not hold renewal response")
+			}
+			select {
+			case err := <-result:
+				if !errors.Is(err, cause) {
+					t.Fatalf("lost original cause: %v", err)
+				}
+			case <-time.After(600 * time.Millisecond):
+				release()
+				<-result
+				t.Fatal("canceled renewal socket I/O blocked cleanup beyond its lease budget")
+			}
+			if producer.running.Load() != 0 {
+				t.Fatal("running flag was not reset")
+			}
+			lock, err := backend.Do(context.Background(), "GET", producer.config.RunLockKey)
+			if err != nil || !lock.IsNil() {
+				t.Fatalf("lock not released: %v, %v", lock, err)
+			}
+			if opts.ReadTimeout != 0 || opts.WriteTimeout != 0 {
+				t.Fatal("lease client mutated the shared production client")
+			}
+			// A fresh run must start without opening the stalled gate.
+			producer.scanner = cardScanner{}
+			if err := producer.Produce(context.Background(), model.ImeiQueueStartMessage{RunID: "next-run"}); err != nil {
+				t.Fatalf("producer cannot restart: %v", err)
+			}
+		})
+	}
+}
+
+func TestProduceFencesDelayedRealRedisWrites(t *testing.T) {
+	for _, mode := range []string{"standalone", "sentinel"} {
+		t.Run(mode, func(t *testing.T) { testProduceFencesDelayedRealRedisWrites(t, mode) })
+	}
+}
+
+func testProduceFencesDelayedRealRedisWrites(t *testing.T, mode string) {
+	for _, stage := range []string{"enqueue", "producer_done"} {
+		t.Run(stage, func(t *testing.T) {
+			backend := testutil.Redis(t)
+			match := "local taskID=ARGV[i]"
+			cards := []Card{{ID: 1, ICCID: "a", ChanKey: "ch", TempKey: "sim_yd_v5"}}
+			if stage == "producer_done" {
+				match = "'producerCompletedAt'"
+				cards = nil
+			}
+			real, held, release := leaseRedisProxy(t, backend, match, false)
+			defer release()
+			workContexts := make(chan context.Context, 1)
+			scanner := &lockTestScanner{cardScanner: cardScanner{cards: cards}}
+			scanner.snapshot = func(ctx context.Context) (int64, error) { workContexts <- ctx; return int64(len(cards)), nil }
+			store, err := queue.New(real, "imei:")
+			if err != nil {
+				t.Fatal(err)
+			}
+			config := Config{RunLockTTL: time.Second}
+			if mode == "sentinel" {
+				real, config.LeaseRedisConfig = sentinelLeaseRedis(t, real)
+				store, err = queue.New(real, "imei:")
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			producer, err := New(scanner, store, config)
+			if err != nil {
+				t.Fatal(err)
+			}
+			result := startLockTestProducer(t, producer)
+			select {
+			case <-held:
+			case <-time.After(2 * time.Second):
+				t.Fatal("proxy did not hold old-owner EVAL")
+			}
+			workCtx := <-workContexts
+			observer, err := queue.New(backend, "imei:")
+			if err != nil {
+				t.Fatal(err)
+			}
+			ok, err := observer.AcquireLockOrTakeover(context.Background(), producer.config.RunLockKey, "new-owner", "lock-test", time.Minute)
+			if err != nil || !ok {
+				t.Fatalf("takeover: %v, %v", ok, err)
+			}
+			select {
+			case <-workCtx.Done():
+			case <-time.After(2 * time.Second):
+				t.Fatal("real renewal did not detect takeover")
+			}
+			if !errors.Is(context.Cause(workCtx), ErrAlreadyRunning) {
+				t.Fatalf("wrong loss cause: %v", context.Cause(workCtx))
+			}
+			release()
+			select {
+			case err := <-result:
+				if !errors.Is(err, ErrAlreadyRunning) {
+					t.Fatalf("wrong result: %v", err)
+				}
+			case <-time.After(2 * time.Second):
+				t.Fatal("producer did not finish")
+			}
+			meta, err := observer.GetRunMeta(context.Background(), "lock-test")
+			if err != nil || meta.ActiveCount != 0 || meta.ProducerDone {
+				t.Fatalf("stale owner changed run state: %+v, %v", meta, err)
+			}
+			channels, err := observer.RunChannels(context.Background(), "lock-test")
+			if err != nil || len(channels) != 0 {
+				t.Fatalf("stale owner indexed channel: %v, %v", channels, err)
+			}
+			ok, err = observer.RenewLock(context.Background(), producer.config.RunLockKey, "new-owner", time.Minute)
+			if err != nil || !ok {
+				t.Fatalf("old cleanup removed new lease: %v, %v", ok, err)
+			}
+			added, err := observer.EnqueueRunBatch(context.Background(), "lock-test", "ch", []queue.Task{{TaskID: "next", RunID: "lock-test", ChanKey: "ch", CardID: 2}}, time.Now())
+			if err != nil || added != 1 {
+				t.Fatalf("new work rejected: added=%d, %v", added, err)
+			}
+		})
+	}
+}
+
+func leaseRedisProxy(t *testing.T, upstream *gredis.Redis, match string, holdReply bool) (*gredis.Redis, <-chan struct{}, func()) {
+	t.Helper()
+	raw := upstream.GetAdapter().Client().(*goredis.Client)
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	held, gate := make(chan struct{}), make(chan struct{})
+	var heldOnce, releaseOnce sync.Once
+	release := func() { releaseOnce.Do(func() { close(gate) }) }
+	var connections sync.Map
+	go func() {
+		for {
+			down, err := listener.Accept()
+			if err != nil {
+				return
+			}
+			connections.Store(down, struct{}{})
+			go func() {
+				defer down.Close()
+				defer connections.Delete(down)
+				up, err := net.Dial("tcp", raw.Options().Addr)
+				if err != nil {
+					return
+				}
+				defer up.Close()
+				dr, ur := bufio.NewReader(down), bufio.NewReader(up)
+				for {
+					request, err := readLeaseRESP(dr)
+					if err != nil {
+						return
+					}
+					holding := strings.Contains(string(request), match)
+					if holding && !holdReply {
+						heldOnce.Do(func() { close(held) })
+						<-gate
+					}
+					if _, err := up.Write(request); err != nil {
+						return
+					}
+					response, err := readLeaseRESP(ur)
+					if err != nil {
+						return
+					}
+					if holding && holdReply {
+						heldOnce.Do(func() { close(held) })
+						<-gate
+					}
+					if _, err := down.Write(response); err != nil {
+						return
+					}
+				}
+			}()
+		}
+	}()
+	t.Cleanup(func() {
+		release()
+		_ = listener.Close()
+		connections.Range(func(key, _ any) bool { _ = key.(net.Conn).Close(); return true })
+	})
+	client, err := gredis.New(&gredis.Config{Address: listener.Addr().String(), DialTimeout: 100 * time.Millisecond, Protocol: 2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = client.Close(context.Background()) })
+	if _, err := client.Do(context.Background(), "PING"); err != nil {
+		t.Fatal(err)
+	}
+	opts := client.GetAdapter().Client().(*goredis.Client).Options()
+	t.Logf("real driver options: readTimeout=%s writeTimeout=%s contextTimeoutEnabled=%v", opts.ReadTimeout, opts.WriteTimeout, opts.ContextTimeoutEnabled)
+	return client, held, release
+}
+
+// A real disposable Sentinel discovers the proxy as its master; neither client
+// configuration nor server state comes from the application's Redis settings.
+func sentinelLeaseRedis(t *testing.T, master *gredis.Redis) (*gredis.Redis, *gredis.Config) {
+	t.Helper()
+	binary := os.Getenv("REDIS_SERVER_BINARY")
+	if binary == "" {
+		var err error
+		binary, err = exec.LookPath("redis-server")
+		if err != nil {
+			t.Skip("Sentinel tests require redis-server")
+		}
+	}
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	address := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(address)
+	_ = listener.Close()
+	masterAddress := master.GetAdapter().Client().(*goredis.Client).Options().Addr
+	host, masterPort, err := net.SplitHostPort(masterAddress)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sentinel.conf")
+	contents := "bind 127.0.0.1\nport " + port + "\nsentinel monitor lease-test " + host + " " + masterPort + " 1\nsentinel down-after-milliseconds lease-test 60000\n"
+	if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
+		t.Fatal(err)
+	}
+	command := exec.Command(binary, filepath.Base(path), "--sentinel")
+	command.Dir = dir
+	logPath := filepath.Join(dir, "sentinel.log")
+	logFile, err := os.Create(logPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer logFile.Close()
+	command.Stdout, command.Stderr = logFile, logFile
+	if err := command.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = command.Process.Kill(); _ = command.Wait() })
+	probe, err := gredis.New(&gredis.Config{Address: address, ReadTimeout: 100 * time.Millisecond, WriteTimeout: 100 * time.Millisecond, DialTimeout: 100 * time.Millisecond, Protocol: 2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = probe.Close(context.Background()) })
+	for deadline := time.Now().Add(5 * time.Second); ; {
+		if _, err = probe.Do(context.Background(), "SENTINEL", "get-master-addr-by-name", "lease-test"); err == nil {
+			break
+		}
+		if time.Now().After(deadline) {
+			log, _ := os.ReadFile(logPath)
+			t.Fatalf("isolated Sentinel did not start: %v; %s", err, log)
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	config := &gredis.Config{Address: address, MasterName: "lease-test", DialTimeout: 100 * time.Millisecond, Protocol: 2}
+	client, err := gredis.New(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = client.Close(context.Background()) })
+	return client, config
+}
+
+func TestLeaseClientBoundsSentinelDiscoveryIO(t *testing.T) {
+	master := testutil.Redis(t)
+	_, config := sentinelLeaseRedis(t, master)
+	sentinel, err := gredis.New(&gredis.Config{Address: config.Address, ReadTimeout: time.Second, WriteTimeout: time.Second, Protocol: 2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = sentinel.Close(context.Background()) })
+	proxy, held, release := leaseRedisProxy(t, sentinel, "get-master-addr-by-name", true)
+	defer release()
+	config.Address = proxy.GetAdapter().Client().(*goredis.Client).Options().Addr
+	shared, err := gredis.New(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = shared.Close(context.Background()) })
+	factory, err := newLeaseClientFactory(shared, config, 100*time.Millisecond)
+	if err != nil {
+		t.Fatal(err)
+	}
+	client, closeClient, err := factory()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeClient()
+	result := make(chan error, 1)
+	go func() { _, err := client.Do(context.Background(), "PING"); result <- err }()
+	select {
+	case <-held:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Sentinel discovery request was not held")
+	}
+	select {
+	case err := <-result:
+		if err == nil {
+			t.Fatal("stalled Sentinel discovery unexpectedly succeeded")
+		}
+	case <-time.After(600 * time.Millisecond):
+		release()
+		<-result
+		t.Fatal("Sentinel discovery has no socket I/O upper bound")
+	}
+}
+
+func TestLeaseClientRejectsClusterAndUnknownAdapter(t *testing.T) {
+	cluster, err := gredis.New(&gredis.Config{Address: "127.0.0.1:1", Cluster: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cluster.Close(context.Background())
+	store, err := queue.New(cluster, "imei:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := New(cardScanner{}, store, Config{}); err == nil || !strings.Contains(err.Error(), "Cluster") {
+		t.Fatalf("Cluster must be rejected at initialization before any write: %v", err)
+	}
+	if _, err := newLeaseClientFactory(&leaseRedis{}, nil, time.Second); err == nil {
+		t.Fatal("unknown adapter must not fall back to unbounded context-only I/O")
+	}
+}
+
+func TestProduceBoundsReleaseIO(t *testing.T) {
+	backend := testutil.Redis(t)
+	real, held, release := leaseRedisProxy(t, backend, "redis.call('DEL', KEYS[1])", true)
+	defer release()
+	store, err := queue.New(real, "imei:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	producer, err := New(cardScanner{}, store, Config{RunLockTTL: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	result := startLockTestProducer(t, producer)
+	select {
+	case <-held:
+	case <-time.After(2 * time.Second):
+		t.Fatal("release response was not held")
+	}
+	select {
+	case err := <-result:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-time.After(600 * time.Millisecond):
+		release()
+		<-result
+		t.Fatal("release socket I/O blocked cleanup")
+	}
+	if producer.running.Load() != 0 {
+		t.Fatal("cleanup did not reset running")
+	}
+}
+
+func TestLeaseClientBoundsSocketWriteIO(t *testing.T) {
+	backend := testutil.Redis(t)
+	upstream := backend.GetAdapter().Client().(*goredis.Client).Options().Addr
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var stopReading atomic.Bool
+	held, gate := make(chan struct{}), make(chan struct{})
+	var releaseOnce sync.Once
+	release := func() { releaseOnce.Do(func() { close(gate) }) }
+	defer release()
+	t.Cleanup(func() { _ = listener.Close() })
+	go func() {
+		down, err := listener.Accept()
+		if err != nil {
+			return
+		}
+		defer down.Close()
+		up, err := net.Dial("tcp", upstream)
+		if err != nil {
+			return
+		}
+		defer up.Close()
+		dr, ur := bufio.NewReader(down), bufio.NewReader(up)
+		for {
+			// Read only the array header once armed: the large EVAL payload
+			// cannot enter the upstream or fit in TCP buffers, so Write blocks.
+			header, err := dr.ReadBytes('\n')
+			if err != nil {
+				return
+			}
+			if stopReading.Load() {
+				close(held)
+				<-gate
+				return
+			}
+			n, err := strconv.Atoi(strings.TrimSpace(string(header[1:])))
+			if err != nil {
+				return
+			}
+			for i := 0; i < n; i++ {
+				part, err := readLeaseRESP(dr)
+				if err != nil {
+					return
+				}
+				header = append(header, part...)
+			}
+			if _, err := up.Write(header); err != nil {
+				return
+			}
+			response, err := readLeaseRESP(ur)
+			if err != nil {
+				return
+			}
+			if _, err := down.Write(response); err != nil {
+				return
+			}
+		}
+	}()
+	shared, err := gredis.New(&gredis.Config{Address: listener.Addr().String(), Protocol: 2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer shared.Close(context.Background())
+	factory, err := newLeaseClientFactory(shared, nil, 100*time.Millisecond)
+	if err != nil {
+		t.Fatal(err)
+	}
+	client, closeClient, err := factory()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeClient()
+	if _, err := client.Do(context.Background(), "PING"); err != nil {
+		t.Fatal(err)
+	}
+	opts := client.(*leaseRedis).client.Options()
+	if opts.ReadTimeout != 100*time.Millisecond || opts.WriteTimeout != 100*time.Millisecond || !opts.ContextTimeoutEnabled || opts.MaxRetries != 0 || opts.DialerRetries != 1 {
+		t.Fatalf("lease client has unsafe timeout/retry options")
+	}
+	stopReading.Store(true)
+	result := make(chan error, 1)
+	go func() {
+		_, err := client.Eval(context.Background(), "return 1 --"+strings.Repeat("x", 32<<20), 0, nil, nil)
+		result <- err
+	}()
+	select {
+	case <-held:
+	case <-time.After(2 * time.Second):
+		t.Fatal("write proxy did not stop reading")
+	}
+	select {
+	case err := <-result:
+		var timeout net.Error
+		if !errors.As(err, &timeout) || !timeout.Timeout() || !strings.Contains(err.Error(), "write") {
+			t.Fatalf("expected real socket write timeout, got %v", err)
+		}
+	case <-time.After(600 * time.Millisecond):
+		release()
+		<-result
+		t.Fatal("socket write has no upper bound")
+	}
+}