iotlcm/worker-svr/internal/queue/zset/lock_test.go

package zset

import (
	"context"
	"testing"
	"time"

	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil"
)

func newLockStore(t *testing.T) *Store {
	t.Helper()
	store, err := New(testutil.Redis(t), "flow:")
	if err != nil {
		t.Fatal(err)
	}
	return store
}

func TestAcquireLockOrTakeoverBlocksOtherRuns(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "run-1", time.Minute)
	if err != nil || !acquired {
		t.Fatalf("first acquire: acquired=%v err=%v", acquired, err)
	}
	// A different run must not steal the lock.
	acquired, err = store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-b", "run-2", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if acquired {
		t.Fatal("a different run took over the lock")
	}
}

func TestAcquireLockOrTakeoverResumesSameRunAfterCrash(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	// Simulate a producer that died without releasing its lock.
	if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "dead-owner", "run-1", time.Minute); err != nil || !acquired {
		t.Fatalf("first acquire: acquired=%v err=%v", acquired, err)
	}
	// The redelivered message arrives with a fresh owner token but the same run.
	acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if !acquired {
		t.Fatal("the same run could not resume after a producer crash")
	}
	// The new owner must now hold the lock for renew and release.
	renewed, err := store.RenewLock(ctx, "flow:producer:active", "new-owner", time.Minute)
	if err != nil || !renewed {
		t.Fatalf("new owner cannot renew: renewed=%v err=%v", renewed, err)
	}
	released, err := store.ReleaseLock(ctx, "flow:producer:active", "new-owner")
	if err != nil || !released {
		t.Fatalf("new owner cannot release: released=%v err=%v", released, err)
	}
}

func TestAcquireLockOrTakeoverEmptyRunIDDegradesToAcquireLock(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "", time.Minute); err != nil || !acquired {
		t.Fatalf("first acquire: acquired=%v err=%v", acquired, err)
	}
	acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-b", "", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if acquired {
		t.Fatal("an empty runID must not permit takeover")
	}
}

// TestAcquireLockOrTakeoverRefusesLiveLegacyLock covers the rolling-deploy case:
// an older build still holds a bare token and keeps renewing it. Stealing that
// lock would let two producers run at once, so it must be refused.
func TestAcquireLockOrTakeoverRefusesLiveLegacyLock(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	// A live legacy holder renews every TTL/3, so its remaining TTL stays above
	// TTL/2. AcquireLock writes the bare token with the full TTL, which is
	// indistinguishable from a just-renewed lock.
	if acquired, err := store.AcquireLock(ctx, "flow:producer:active", "legacy-owner", time.Minute); err != nil || !acquired {
		t.Fatalf("legacy acquire: acquired=%v err=%v", acquired, err)
	}

	acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if acquired {
		t.Fatal("a still-live legacy lock must not be taken over: that would let two producers run at once")
	}
	// The legacy owner keeps the lock.
	if renewed, err := store.RenewLock(ctx, "flow:producer:active", "legacy-owner", time.Minute); err != nil || !renewed {
		t.Fatalf("legacy owner lost the lock: renewed=%v err=%v", renewed, err)
	}
}

// TestAcquireLockOrTakeoverTakesOverStaleLegacyLock covers the case the takeover
// exists for: the legacy writer died, stopped renewing, and its TTL has fallen
// past the point a live holder could ever reach.
func TestAcquireLockOrTakeoverTakesOverStaleLegacyLock(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	// A short TTL lets the test wait for the stale window without sleeping a
	// full production TTL. After ~60% of the TTL has elapsed the remaining TTL is
	// below TTL/2, which a renewing holder can never reach.
	if acquired, err := store.AcquireLock(ctx, "flow:producer:active", "legacy-owner", 2*time.Second); err != nil || !acquired {
		t.Fatalf("legacy acquire: acquired=%v err=%v", acquired, err)
	}
	time.Sleep(1300 * time.Millisecond)

	acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if !acquired {
		t.Fatal("a stale legacy lock must be taken over; refusing strands the key and drops the request")
	}
	// The new owner must now hold the lock, and the old token must not.
	if renewed, err := store.RenewLock(ctx, "flow:producer:active", "new-owner", time.Minute); err != nil || !renewed {
		t.Fatalf("new owner cannot renew: renewed=%v err=%v", renewed, err)
	}
	if released, err := store.ReleaseLock(ctx, "flow:producer:active", "legacy-owner"); err != nil || released {
		t.Fatalf("legacy owner must no longer hold the lock: released=%v err=%v", released, err)
	}
}

// TestAcquireLockOrTakeoverTakesOverLegacyLockWithoutExpiry treats a legacy lock
// that has no expiry as stale: every writer in this codebase sets one.
func TestAcquireLockOrTakeoverTakesOverLegacyLockWithoutExpiry(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	// Simulate a legacy lock written without a TTL.
	if _, err := store.RedisClient().Do(ctx, "SET", "flow:producer:active", "legacy-owner"); err != nil {
		t.Fatal(err)
	}
	acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if !acquired {
		t.Fatal("a legacy lock with no expiry must be treated as stale and taken over")
	}
}

func TestAcquireLockOrTakeoverBlocksDifferentStructuredRun(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	// A structured lock for a different run means another run is genuinely live.
	if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "run-1", time.Minute); err != nil || !acquired {
		t.Fatalf("first acquire: acquired=%v err=%v", acquired, err)
	}
	acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-b", "run-2", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if acquired {
		t.Fatal("a structured lock held by a different run must not be taken over")
	}
}

func TestAcquireLockOrTakeoverRejectsNonOwnerRelease(t *testing.T) {
	store := newLockStore(t)
	ctx := context.Background()

	if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "run-1", time.Minute); err != nil || !acquired {
		t.Fatalf("acquire: acquired=%v err=%v", acquired, err)
	}
	released, err := store.ReleaseLock(ctx, "flow:producer:active", "owner-b")
	if err != nil {
		t.Fatal(err)
	}
	if released {
		t.Fatal("a non-owner released the lock")
	}
}