package dailyusage import ( "context" "testing" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil" ) func newZombieTestProducer(t *testing.T) (*Producer, *zset.Store) { t.Helper() store, err := zset.New(testutil.Redis(t), "flow:") if err != nil { t.Fatal(err) } producer, err := New(&pageScanner{cards: nil}, store, Config{PageSize: 100, RoundRetryEnabled: true}) if err != nil { t.Fatal(err) } ctx := context.Background() if acquired, err := store.AcquireLockOrTakeover(ctx, producer.config.RunLockKey, producer.owner, "pending-check", producer.config.RunLockTTL); err != nil || !acquired { t.Fatalf("acquire producer lock=%t err=%v", acquired, err) } t.Cleanup(func() { _, _ = store.ReleaseLock(ctx, producer.config.RunLockKey, producer.owner) }) return producer, store } // TestHasPendingTasksReclaimsZombieRun guards the "zombie run" regression. // // A producer that dies before MarkProducerDone (for example the container is // replaced mid-run) leaves its run in status=producing with producerDone=0. // completeRunIfIdle and advanceRunRound both require producerDone=1, so such a // run can never be closed, never leaves the active index, and never gets the // retention TTL. Treating it as backpressure rejected every later request // forever; merely warning about it leaked keys and an index entry indefinitely. func TestHasPendingTasksReclaimsZombieRun(t *testing.T) { producer, store := newZombieTestProducer(t) ctx := context.Background() // A genuinely in-flight run (producerDone set, consumer working) must keep // providing backpressure. if err := store.CreateRun(ctx, zset.RunMeta{RunID: "running-run", Status: "running", ProducerDone: true, CurrentRound: 1}); err != nil { t.Fatal(err) } pending, err := producer.hasPendingTasks(ctx, "new-run") if err != nil { t.Fatal(err) } if !pending { t.Fatal("a running run must still provide backpressure") } if _, err := store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), "running-run"); err != nil { t.Fatal(err) } // A zombie run must not block a new request, and must be promoted so the // normal lifecycle can drain and close it. if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil { t.Fatal(err) } pending, err = producer.hasPendingTasks(ctx, "new-run-2") if err != nil { t.Fatal(err) } if pending { t.Fatal("a producing run with producerDone=0 must not block new requests (zombie run)") } meta, err := store.GetRunMeta(ctx, "zombie-run") if err != nil { t.Fatal(err) } if !meta.ProducerDone { t.Fatal("the zombie run must be promoted (producerDone=1) so it can be closed") } if meta.Status != "running" { t.Fatalf("promoted run status = %q, want running", meta.Status) } // Promotion makes the run closable, which is what applies the retention TTL // and releases the active index entry. if completed, err := store.CompleteRunIfIdle(ctx, "zombie-run"); err != nil || !completed { t.Fatalf("promoted run must be closable: completed=%v err=%v", completed, err) } } // TestHasPendingTasksNeverReclaimsReplayRun guards the boundary: a replay target // is also created as "producing", but it is driven by the replay engine under // its own lock, so its producerDone is not the producer's to set. func TestHasPendingTasksNeverReclaimsReplayRun(t *testing.T) { producer, store := newZombieTestProducer(t) ctx := context.Background() if err := store.CreateRun(ctx, zset.RunMeta{ RunID: "replay-run", Status: "producing", ProducerDone: false, CurrentRound: 1, SyncType: "replay", SourceRunID: "source-run", ReplayCycle: 1, }); err != nil { t.Fatal(err) } if pending, err := producer.hasPendingTasks(ctx, "new-run"); err != nil || !pending { t.Fatalf("replay must retain backpressure: pending=%t err=%v", pending, err) } meta, err := store.GetRunMeta(ctx, "replay-run") if err != nil { t.Fatal(err) } if meta.ProducerDone { t.Fatal("a replay run must not be promoted by the full-sync producer") } if meta.Status != "producing" { t.Fatalf("replay run status = %q, want producing", meta.Status) } } // TestHasPendingTasksDropsStaleIndexEntry covers a second unrecoverable path: // the active index can list a run whose hash is already gone. Returning that as // an error made the request fail permanently (the consumer redelivers it // forever), which is the same "only fixable by hand-cleaning Redis" failure. func TestHasPendingTasksDropsStaleIndexEntry(t *testing.T) { producer, store := newZombieTestProducer(t) ctx := context.Background() // Index a run that has no metadata hash. if _, err := store.RedisClient().Do(ctx, "SADD", store.ActiveRunsKey(), "ghost-run"); err != nil { t.Fatal(err) } pending, err := producer.hasPendingTasks(ctx, "new-run") if err != nil { t.Fatalf("a stale index entry must not fail the request: %v", err) } if pending { t.Fatal("a stale index entry must not provide backpressure") } // The stale entry is removed so it does not have to be handled again. members, err := store.ActiveRuns(ctx) if err != nil { t.Fatal(err) } for _, member := range members { if member == "ghost-run" { t.Fatal("the stale index entry must be removed") } } } // TestProduceCreatesRunAfterZombie covers the end-to-end path through Produce: // a zombie run must not stop a new run from being created. func TestProduceCreatesRunAfterZombie(t *testing.T) { store, err := zset.New(testutil.Redis(t), "flow:") if err != nil { t.Fatal(err) } ctx := context.Background() producer, err := New(&pageScanner{cards: []Card{ {ID: 1, ChanKey: "channel-a", TempKey: "sim_yd_v5", Operator: 1}, }}, store, Config{PageSize: 100, RoundRetryEnabled: true}) if err != nil { t.Fatal(err) } // A zombie run from a crashed producer. if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil { t.Fatal(err) } // A new request must still produce its own run. if err := producer.Produce(ctx, scopedMessage("fresh-run", "channel-a")); err != nil { t.Fatalf("a zombie run must not stop a new run: %v", err) } meta, err := store.GetRunMeta(ctx, "fresh-run") if err != nil { t.Fatal(err) } // The producer enqueues into the active ZSet; ActiveCount is the count it // maintains. EnqueuedCount is an accounting field the producer does not set. if !meta.ProducerDone || meta.ActiveCount != 1 { t.Fatalf("fresh run metadata = %+v, want producerDone with 1 active task", meta) } }