package zset import ( "context" "testing" "time" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil" ) func newLockStore(t *testing.T) *Store { t.Helper() store, err := New(testutil.Redis(t), "flow:") if err != nil { t.Fatal(err) } return store } func TestAcquireLockOrTakeoverBlocksOtherRuns(t *testing.T) { store := newLockStore(t) ctx := context.Background() acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "run-1", time.Minute) if err != nil || !acquired { t.Fatalf("first acquire: acquired=%v err=%v", acquired, err) } // A different run must not steal the lock. acquired, err = store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-b", "run-2", time.Minute) if err != nil { t.Fatal(err) } if acquired { t.Fatal("a different run took over the lock") } } func TestAcquireLockOrTakeoverResumesSameRunAfterCrash(t *testing.T) { store := newLockStore(t) ctx := context.Background() // Simulate a producer that died without releasing its lock. if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "dead-owner", "run-1", time.Minute); err != nil || !acquired { t.Fatalf("first acquire: acquired=%v err=%v", acquired, err) } // The redelivered message arrives with a fresh owner token but the same run. acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute) if err != nil { t.Fatal(err) } if !acquired { t.Fatal("the same run could not resume after a producer crash") } // The new owner must now hold the lock for renew and release. renewed, err := store.RenewLock(ctx, "flow:producer:active", "new-owner", time.Minute) if err != nil || !renewed { t.Fatalf("new owner cannot renew: renewed=%v err=%v", renewed, err) } released, err := store.ReleaseLock(ctx, "flow:producer:active", "new-owner") if err != nil || !released { t.Fatalf("new owner cannot release: released=%v err=%v", released, err) } } func TestAcquireLockOrTakeoverEmptyRunIDDegradesToAcquireLock(t *testing.T) { store := newLockStore(t) ctx := context.Background() if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "", time.Minute); err != nil || !acquired { t.Fatalf("first acquire: acquired=%v err=%v", acquired, err) } acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-b", "", time.Minute) if err != nil { t.Fatal(err) } if acquired { t.Fatal("an empty runID must not permit takeover") } } // TestAcquireLockOrTakeoverRefusesLiveLegacyLock covers the rolling-deploy case: // an older build still holds a bare token and keeps renewing it. Stealing that // lock would let two producers run at once, so it must be refused. func TestAcquireLockOrTakeoverRefusesLiveLegacyLock(t *testing.T) { store := newLockStore(t) ctx := context.Background() // A live legacy holder renews every TTL/3, so its remaining TTL stays above // TTL/2. AcquireLock writes the bare token with the full TTL, which is // indistinguishable from a just-renewed lock. if acquired, err := store.AcquireLock(ctx, "flow:producer:active", "legacy-owner", time.Minute); err != nil || !acquired { t.Fatalf("legacy acquire: acquired=%v err=%v", acquired, err) } acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute) if err != nil { t.Fatal(err) } if acquired { t.Fatal("a still-live legacy lock must not be taken over: that would let two producers run at once") } // The legacy owner keeps the lock. if renewed, err := store.RenewLock(ctx, "flow:producer:active", "legacy-owner", time.Minute); err != nil || !renewed { t.Fatalf("legacy owner lost the lock: renewed=%v err=%v", renewed, err) } } // TestAcquireLockOrTakeoverTakesOverStaleLegacyLock covers the case the takeover // exists for: the legacy writer died, stopped renewing, and its TTL has fallen // past the point a live holder could ever reach. func TestAcquireLockOrTakeoverTakesOverStaleLegacyLock(t *testing.T) { store := newLockStore(t) ctx := context.Background() // A short TTL lets the test wait for the stale window without sleeping a // full production TTL. After ~60% of the TTL has elapsed the remaining TTL is // below TTL/2, which a renewing holder can never reach. if acquired, err := store.AcquireLock(ctx, "flow:producer:active", "legacy-owner", 2*time.Second); err != nil || !acquired { t.Fatalf("legacy acquire: acquired=%v err=%v", acquired, err) } time.Sleep(1300 * time.Millisecond) acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute) if err != nil { t.Fatal(err) } if !acquired { t.Fatal("a stale legacy lock must be taken over; refusing strands the key and drops the request") } // The new owner must now hold the lock, and the old token must not. if renewed, err := store.RenewLock(ctx, "flow:producer:active", "new-owner", time.Minute); err != nil || !renewed { t.Fatalf("new owner cannot renew: renewed=%v err=%v", renewed, err) } if released, err := store.ReleaseLock(ctx, "flow:producer:active", "legacy-owner"); err != nil || released { t.Fatalf("legacy owner must no longer hold the lock: released=%v err=%v", released, err) } } // TestAcquireLockOrTakeoverTakesOverLegacyLockWithoutExpiry treats a legacy lock // that has no expiry as stale: every writer in this codebase sets one. func TestAcquireLockOrTakeoverTakesOverLegacyLockWithoutExpiry(t *testing.T) { store := newLockStore(t) ctx := context.Background() // Simulate a legacy lock written without a TTL. if _, err := store.RedisClient().Do(ctx, "SET", "flow:producer:active", "legacy-owner"); err != nil { t.Fatal(err) } acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "new-owner", "run-1", time.Minute) if err != nil { t.Fatal(err) } if !acquired { t.Fatal("a legacy lock with no expiry must be treated as stale and taken over") } } func TestAcquireLockOrTakeoverBlocksDifferentStructuredRun(t *testing.T) { store := newLockStore(t) ctx := context.Background() // A structured lock for a different run means another run is genuinely live. if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "run-1", time.Minute); err != nil || !acquired { t.Fatalf("first acquire: acquired=%v err=%v", acquired, err) } acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-b", "run-2", time.Minute) if err != nil { t.Fatal(err) } if acquired { t.Fatal("a structured lock held by a different run must not be taken over") } } func TestAcquireLockOrTakeoverRejectsNonOwnerRelease(t *testing.T) { store := newLockStore(t) ctx := context.Background() if acquired, err := store.AcquireLockOrTakeover(ctx, "flow:producer:active", "owner-a", "run-1", time.Minute); err != nil || !acquired { t.Fatalf("acquire: acquired=%v err=%v", acquired, err) } released, err := store.ReleaseLock(ctx, "flow:producer:active", "owner-b") if err != nil { t.Fatal(err) } if released { t.Fatal("a non-owner released the lock") } }