iotlcm/internal/producer/monthlyusagehistory/zombie_test.go

package monthlyusagehistory

import (
	"context"
	"testing"

	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
)

func newZombieTestProducer(t *testing.T) (*Producer, *zset.Store) {
	t.Helper()
	store, err := zset.New(testutil.Redis(t), "flow:")
	if err != nil {
		t.Fatal(err)
	}
	producer, err := New(&pageScanner{cards: nil}, store, Config{PageSize: 100, RoundRetryEnabled: true})
	if err != nil {
		t.Fatal(err)
	}
	ctx := context.Background()
	if acquired, err := store.AcquireLockOrTakeover(ctx, producer.config.RunLockKey, producer.owner, "pending-check", producer.config.RunLockTTL); err != nil || !acquired {
		t.Fatalf("acquire producer lock=%t err=%v", acquired, err)
	}
	t.Cleanup(func() { _, _ = store.ReleaseLock(ctx, producer.config.RunLockKey, producer.owner) })
	return producer, store
}

// TestHasPendingTasksReclaimsZombieRun guards the "zombie run" regression.
//
// A producer that dies before MarkProducerDone (for example the container is
// replaced mid-run) leaves its run in status=producing with producerDone=0.
// completeRunIfIdle and advanceRunRound both require producerDone=1, so such a
// run can never be closed, never leaves the active index, and never gets the
// retention TTL. Treating it as backpressure rejected every later request
// forever; merely warning about it leaked keys and an index entry indefinitely.
func TestHasPendingTasksReclaimsZombieRun(t *testing.T) {
	producer, store := newZombieTestProducer(t)
	ctx := context.Background()

	// A genuinely in-flight run (producerDone set, consumer working) must keep
	// providing backpressure.
	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "running-run", Status: "running", ProducerDone: true, CurrentRound: 1}); err != nil {
		t.Fatal(err)
	}
	pending, err := producer.hasPendingTasks(ctx, "new-run")
	if err != nil {
		t.Fatal(err)
	}
	if !pending {
		t.Fatal("a running run must still provide backpressure")
	}
	if _, err := store.RedisClient().Do(ctx, "SREM", store.ActiveRunsKey(), "running-run"); err != nil {
		t.Fatal(err)
	}

	// A zombie run must not block a new request, and must be promoted so the
	// normal lifecycle can drain and close it.
	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
		t.Fatal(err)
	}
	pending, err = producer.hasPendingTasks(ctx, "new-run-2")
	if err != nil {
		t.Fatal(err)
	}
	if pending {
		t.Fatal("a producing run with producerDone=0 must not block new requests (zombie run)")
	}

	meta, err := store.GetRunMeta(ctx, "zombie-run")
	if err != nil {
		t.Fatal(err)
	}
	if !meta.ProducerDone {
		t.Fatal("the zombie run must be promoted (producerDone=1) so it can be closed")
	}
	if meta.Status != "running" {
		t.Fatalf("promoted run status = %q, want running", meta.Status)
	}
	// Promotion makes the run closable, which is what applies the retention TTL
	// and releases the active index entry.
	if completed, err := store.CompleteRunIfIdle(ctx, "zombie-run"); err != nil || !completed {
		t.Fatalf("promoted run must be closable: completed=%v err=%v", completed, err)
	}
}

// TestHasPendingTasksNeverReclaimsReplayRun guards the boundary: a replay target
// is also created as "producing", but it is driven by the replay engine under
// its own lock, so its producerDone is not the producer's to set.
func TestHasPendingTasksNeverReclaimsReplayRun(t *testing.T) {
	producer, store := newZombieTestProducer(t)
	ctx := context.Background()

	if err := store.CreateRun(ctx, zset.RunMeta{
		RunID: "replay-run", Status: "producing", ProducerDone: false, CurrentRound: 1,
		SyncType: "replay", SourceRunID: "source-run", ReplayCycle: 1,
	}); err != nil {
		t.Fatal(err)
	}
	if pending, err := producer.hasPendingTasks(ctx, "new-run"); err != nil || !pending {
		t.Fatalf("replay must retain backpressure: pending=%t err=%v", pending, err)
	}
	meta, err := store.GetRunMeta(ctx, "replay-run")
	if err != nil {
		t.Fatal(err)
	}
	if meta.ProducerDone {
		t.Fatal("a replay run must not be promoted by the full-sync producer")
	}
	if meta.Status != "producing" {
		t.Fatalf("replay run status = %q, want producing", meta.Status)
	}
}

// TestHasPendingTasksDropsStaleIndexEntry covers a second unrecoverable path:
// the active index can list a run whose hash is already gone. Returning that as
// an error made the request fail permanently (the consumer redelivers it
// forever), which is the same "only fixable by hand-cleaning Redis" failure.
func TestHasPendingTasksDropsStaleIndexEntry(t *testing.T) {
	producer, store := newZombieTestProducer(t)
	ctx := context.Background()

	// Index a run that has no metadata hash.
	if _, err := store.RedisClient().Do(ctx, "SADD", store.ActiveRunsKey(), "ghost-run"); err != nil {
		t.Fatal(err)
	}
	pending, err := producer.hasPendingTasks(ctx, "new-run")
	if err != nil {
		t.Fatalf("a stale index entry must not fail the request: %v", err)
	}
	if pending {
		t.Fatal("a stale index entry must not provide backpressure")
	}
	// The stale entry is removed so it does not have to be handled again.
	members, err := store.ActiveRuns(ctx)
	if err != nil {
		t.Fatal(err)
	}
	for _, member := range members {
		if member == "ghost-run" {
			t.Fatal("the stale index entry must be removed")
		}
	}
}

// TestProduceCreatesRunAfterZombie covers the end-to-end path through Produce:
// a zombie run must not stop a new run from being created.
func TestProduceCreatesRunAfterZombie(t *testing.T) {
	store, err := zset.New(testutil.Redis(t), "flow:")
	if err != nil {
		t.Fatal(err)
	}
	ctx := context.Background()
	producer, err := New(&pageScanner{cards: []Card{
		{ID: 1, ChanKey: "channel-a", TempKey: "sim_yd_v5", Operator: 1},
	}}, store, Config{PageSize: 100, RoundRetryEnabled: true})
	if err != nil {
		t.Fatal(err)
	}

	// A zombie run from a crashed producer.
	if err := store.CreateRun(ctx, zset.RunMeta{RunID: "zombie-run", Status: "producing", ProducerDone: false, CurrentRound: 1}); err != nil {
		t.Fatal(err)
	}

	// A new request must still produce its own run.
	if err := producer.Produce(ctx, scopedMessage("fresh-run", "channel-a")); err != nil {
		t.Fatalf("a zombie run must not stop a new run: %v", err)
	}
	meta, err := store.GetRunMeta(ctx, "fresh-run")
	if err != nil {
		t.Fatal(err)
	}
	// The producer enqueues into the active ZSet; ActiveCount is the count it
	// maintains. EnqueuedCount is an accounting field the producer does not set.
	if !meta.ProducerDone || meta.ActiveCount != 1 {
		t.Fatalf("fresh run metadata = %+v, want producerDone with 1 active task", meta)
	}
}