chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）

worker-svr/internal/producer/monthlyusage/zombie_test.go

@@ -0,0 +1,174 @@
+package monthlyusage
+
+import (
+	"context"
+	"testing"
+
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
+	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
+)
+
+func newZombieTestProducer(t *testing.T) (*Producer, *zset.Store) {
+	t.Helper()
+	store, err := zset.New(testutil.Redis(t), "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	producer, err := New(&pageScanner{cards: nil}, store, Config{PageSize: 100, RoundRetryEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return producer, store
+}
+
+// TestHasPendingTasksReclaimsZombieRun guards the "zombie run" regression.
+//
+// A producer that dies before MarkProducerDone (for example the container is
+// replaced mid-run) leaves its run in status=producing with producerDone=0.
+// completeRunIfIdle and advanceRunRound both require producerDone=1, so such a
+// run can never be closed, never leaves the active index, and never gets the
+// retention TTL. Treating it as backpressure rejected every later request
+// forever; merely warning about it leaked keys and an index entry indefinitely.
+func TestHasPendingTasksReclaimsZombieRun(t *testing.T) {
+	producer, store := newZombieTestProducer(t)
+	ctx := context.Background()
+
+	// A genuinely in-flight run (producerDone set, consumer working) must keep
+	// providing backpressure.
+	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "running-run", Status: "running", ProducerDone: true, CurrentRound: 1}); err != nil {
+		t.Fatal(err)
+	}
+	pending, err := producer.hasPendingTasks(ctx, "new-run")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !pending {
+		t.Fatal("a running run must still provide backpressure")
+	}
+	if _, err := store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), "running-run"); err != nil {
+		t.Fatal(err)
+	}
+
+	// A zombie run must not block a new request, and must be promoted so the
+	// normal lifecycle can drain and close it.
+	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
+		t.Fatal(err)
+	}
+	pending, err = producer.hasPendingTasks(ctx, "new-run-2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pending {
+		t.Fatal("a producing run with producerDone=0 must not block new requests (zombie run)")
+	}
+
+	meta, err := store.GetRunMeta(ctx, "zombie-run")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !meta.ProducerDone {
+		t.Fatal("the zombie run must be promoted (producerDone=1) so it can be closed")
+	}
+	if meta.Status != "running" {
+		t.Fatalf("promoted run status = %q, want running", meta.Status)
+	}
+	// Promotion makes the run closable, which is what applies the retention TTL
+	// and releases the active index entry.
+	if completed, err := store.CompleteRunIfIdle(ctx, "zombie-run"); err != nil || !completed {
+		t.Fatalf("promoted run must be closable: completed=%v err=%v", completed, err)
+	}
+}
+
+// TestHasPendingTasksNeverReclaimsReplayRun guards the boundary: a replay target
+// is also created as "producing", but it is driven by the replay engine under
+// its own lock, so its producerDone is not the producer's to set.
+func TestHasPendingTasksNeverReclaimsReplayRun(t *testing.T) {
+	producer, store := newZombieTestProducer(t)
+	ctx := context.Background()
+
+	if err := store.CreateRun(ctx, zset.RunMeta{
+		RunID: "replay-run", Status: "producing", ProducerDone: false, CurrentRound: 1,
+		SyncType: "replay", SourceRunID: "source-run", ReplayCycle: 1,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := producer.hasPendingTasks(ctx, "new-run"); err != nil {
+		t.Fatal(err)
+	}
+	meta, err := store.GetRunMeta(ctx, "replay-run")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if meta.ProducerDone {
+		t.Fatal("a replay run must not be promoted by the full-sync producer")
+	}
+	if meta.Status != "producing" {
+		t.Fatalf("replay run status = %q, want producing", meta.Status)
+	}
+}
+
+// TestHasPendingTasksDropsStaleIndexEntry covers a second unrecoverable path:
+// the active index can list a run whose hash is already gone. Returning that as
+// an error made the request fail permanently (the consumer redelivers it
+// forever), which is the same "only fixable by hand-cleaning Redis" failure.
+func TestHasPendingTasksDropsStaleIndexEntry(t *testing.T) {
+	producer, store := newZombieTestProducer(t)
+	ctx := context.Background()
+
+	// Index a run that has no metadata hash.
+	if _, err := store.RedisClient().Do(ctx, "SADD", store.ActiveRunsKey(), "ghost-run"); err != nil {
+		t.Fatal(err)
+	}
+	pending, err := producer.hasPendingTasks(ctx, "new-run")
+	if err != nil {
+		t.Fatalf("a stale index entry must not fail the request: %v", err)
+	}
+	if pending {
+		t.Fatal("a stale index entry must not provide backpressure")
+	}
+	// The stale entry is removed so it does not have to be handled again.
+	members, err := store.ActiveRuns(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, member := range members {
+		if member == "ghost-run" {
+			t.Fatal("the stale index entry must be removed")
+		}
+	}
+}
+
+// TestProduceCreatesRunAfterZombie covers the end-to-end path through Produce:
+// a zombie run must not stop a new run from being created.
+func TestProduceCreatesRunAfterZombie(t *testing.T) {
+	store, err := zset.New(testutil.Redis(t), "flow:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	producer, err := New(&pageScanner{cards: []Card{
+		{ID: 1, ChanKey: "channel-a", TempKey: "sim_yd_v5", Operator: 1},
+	}}, store, Config{PageSize: 100, RoundRetryEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A zombie run from a crashed producer.
+	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
+		t.Fatal(err)
+	}
+
+	// A new request must still produce its own run.
+	if err := producer.Produce(ctx, scopedMessage("fresh-run", "channel-a")); err != nil {
+		t.Fatalf("a zombie run must not stop a new run: %v", err)
+	}
+	meta, err := store.GetRunMeta(ctx, "fresh-run")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The producer enqueues into the active ZSet; ActiveCount is the count it
+	// maintains. EnqueuedCount is an accounting field the producer does not set.
+	if !meta.ProducerDone || meta.ActiveCount != 1 {
+		t.Fatalf("fresh run metadata = %+v, want producerDone with 1 active task", meta)
+	}
+}