chore: 并入 iotlcm-worker-svr 历史到 worker-svr/（原提交 96f0f4e）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,206 @@
|
||||
package dailyusage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
|
||||
)
|
||||
|
||||
// Pause the real producer after its first enqueue, before MarkProducerDone.
|
||||
type interruptedScanner struct {
|
||||
*datasetScanner
|
||||
paused chan struct{}
|
||||
}
|
||||
|
||||
func TestInterruptedRunReclamationGuards(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
for _, status := range []string{"producing", "running"} {
|
||||
for _, guard := range []string{"no_lock", "foreign_lock", "no_current_run", "same_run", "producer_done", "replay"} {
|
||||
t.Run(mode+"/"+status+"/"+guard, func(t *testing.T) {
|
||||
ctx := contextForTest(t)
|
||||
store := testStore(t)
|
||||
p := testProducer(t, scannerFor(nil, 0), store, Config{ScanMode: mode})
|
||||
meta := zset.RunMeta{RunID: "guarded", Status: status, ProducerDone: guard == "producer_done"}
|
||||
if guard == "replay" {
|
||||
meta.SyncType = "replay"
|
||||
}
|
||||
if err := store.CreateRun(ctx, meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := "fresh"
|
||||
owner := p.owner
|
||||
if guard == "same_run" {
|
||||
current = meta.RunID
|
||||
}
|
||||
if guard == "no_current_run" {
|
||||
current = ""
|
||||
}
|
||||
if guard == "foreign_lock" {
|
||||
owner = "other-owner"
|
||||
}
|
||||
if guard != "no_lock" {
|
||||
if ok, err := store.AcquireLockOrTakeover(ctx, p.config.RunLockKey, owner, current, p.config.RunLockTTL); err != nil || !ok {
|
||||
t.Fatalf("acquire=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
key, _ := store.RunMetaKey(meta.RunID)
|
||||
before, err := store.RedisClient().Do(ctx, "HGETALL", key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := p.hasPendingTasks(ctx, current)
|
||||
if err != nil || pending != (guard != "same_run") {
|
||||
t.Fatalf("guard=%s pending=%t err=%v", guard, pending, err)
|
||||
}
|
||||
after, err := store.RedisClient().Do(ctx, "HGETALL", key)
|
||||
if err != nil || !reflect.DeepEqual(before.MapStrStr(), after.MapStrStr()) {
|
||||
t.Fatalf("guarded metadata changed: before=%v after=%v err=%v", before, after, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *interruptedScanner) Scan(ctx context.Context, last, upper int64, size int) ([]Card, error) {
|
||||
if last > 0 {
|
||||
close(s.paused)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return s.datasetScanner.Scan(ctx, last, upper, size)
|
||||
}
|
||||
|
||||
func (s *interruptedScanner) ScanChannel(ctx context.Context, key string, last int64, size int) ([]Card, error) {
|
||||
if last > 0 {
|
||||
close(s.paused)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return s.datasetScanner.ScanChannel(ctx, key, last, size)
|
||||
}
|
||||
|
||||
func TestBlockedInterruptedProducerRecovery(t *testing.T) {
|
||||
for _, mode := range []string{"serial", "channel"} {
|
||||
for _, backlog := range []string{"none", "queued", "leased", "waiting"} {
|
||||
t.Run(mode+"/"+backlog, func(t *testing.T) {
|
||||
ctx := contextForTest(t)
|
||||
store := testStore(t)
|
||||
scanner := &interruptedScanner{datasetScanner: scannerFor([]Card{{ID: 1, ChanKey: "blocked"}, {ID: 2, ChanKey: "blocked"}}, 2), paused: make(chan struct{})}
|
||||
old := testProducer(t, scanner, store, Config{ScanMode: mode, PageSize: 1})
|
||||
oldCtx, interrupt := context.WithCancel(ctx)
|
||||
defer interrupt()
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- old.Produce(oldCtx, scopedMessage("interrupted")) }()
|
||||
select {
|
||||
case <-scanner.paused:
|
||||
case err := <-result:
|
||||
t.Fatalf("producer exited before pause: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
claim := func(channel string) zset.ClaimedTask {
|
||||
t.Helper()
|
||||
tasks, err := store.ClaimRun(ctx, "interrupted", channel, time.Now().Add(time.Second), time.Now().Add(time.Minute), 1)
|
||||
if err != nil || len(tasks) != 1 {
|
||||
t.Fatalf("claim %s=%v err=%v", channel, tasks, err)
|
||||
}
|
||||
return tasks[0]
|
||||
}
|
||||
if n, err := store.BlockRunScopedAsFailed(ctx, claim("blocked"), zset.TaskOutcome{Code: 797, ErrCode: "S00004"}); err != nil || n != 1 {
|
||||
t.Fatalf("blocked=%d err=%v", n, err)
|
||||
}
|
||||
var remaining zset.ClaimedTask
|
||||
if backlog != "none" {
|
||||
tasks := []zset.Task{{TaskID: "remaining", RunID: "interrupted", ChanKey: "other"}}
|
||||
if n, err := store.EnqueueRunBatch(ctx, "interrupted", "other", tasks, time.Now()); err != nil || n != 1 {
|
||||
t.Fatalf("enqueue=%d err=%v", n, err)
|
||||
}
|
||||
if backlog != "queued" {
|
||||
remaining = claim("other")
|
||||
}
|
||||
if backlog == "waiting" {
|
||||
if ok, err := store.FinalizeRunTask(ctx, remaining, zset.TaskOutcome{Retry: true}); err != nil || !ok {
|
||||
t.Fatalf("defer=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
before, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || before.Status != "running" || before.ProducerDone || before.FailedCount != 1 {
|
||||
t.Fatalf("blocked metadata=%+v err=%v", before, err)
|
||||
}
|
||||
fresh := testProducer(t, scannerFor([]Card{{ID: 3, ChanKey: "fresh"}}, 3), store, Config{ScanMode: mode})
|
||||
if err := fresh.Produce(ctx, scopedMessage("fresh")); !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("live producer lock must prevent reclamation: %v", err)
|
||||
}
|
||||
requireMeta(t, store, "interrupted", false, before.ActiveCount)
|
||||
interrupt()
|
||||
if err := <-result; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("interruption=%v", err)
|
||||
}
|
||||
lock, err := store.RedisClient().Do(ctx, "EXISTS", old.config.RunLockKey)
|
||||
if err != nil || lock.Int() != 0 {
|
||||
t.Fatalf("old lock=%v err=%v", lock, err)
|
||||
}
|
||||
err = fresh.Produce(ctx, scopedMessage("fresh"))
|
||||
if backlog == "none" && err != nil || backlog != "none" && !errors.Is(err, ErrAlreadyRunning) {
|
||||
t.Fatalf("fresh start with %s backlog: %v", backlog, err)
|
||||
}
|
||||
after, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || !after.ProducerDone || after.Status != "running" || after.ActiveCount != before.ActiveCount || after.InflightCount != before.InflightCount || after.NextRoundCount != before.NextRoundCount || after.FailedCount != before.FailedCount {
|
||||
t.Fatalf("reclaimed metadata=%+v before=%+v err=%v", after, before, err)
|
||||
}
|
||||
if backlog != "none" {
|
||||
if _, err := store.GetRunMeta(ctx, "fresh"); !errors.Is(err, zset.ErrRunNotFound) {
|
||||
t.Fatalf("backlog must prevent creating fresh run: %v", err)
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "interrupted"); err != nil || complete {
|
||||
t.Fatalf("premature completion=%t err=%v", complete, err)
|
||||
}
|
||||
if backlog == "waiting" {
|
||||
if advanced, err := store.AdvanceRunRound(ctx, "interrupted"); err != nil || !advanced {
|
||||
t.Fatalf("advance=%t err=%v", advanced, err)
|
||||
}
|
||||
}
|
||||
if backlog != "leased" {
|
||||
remaining = claim("other")
|
||||
}
|
||||
if ok, err := store.FinalizeRunTask(ctx, remaining, zset.TaskOutcome{Success: true}); err != nil || !ok {
|
||||
t.Fatalf("drain=%t err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
if complete, err := store.CompleteRunIfIdle(ctx, "interrupted", time.Hour); err != nil || !complete {
|
||||
t.Fatalf("normal completion=%t err=%v", complete, err)
|
||||
}
|
||||
closed, err := store.GetRunMeta(ctx, "interrupted")
|
||||
if err != nil || closed.Status != "completed_with_failures" || closed.FailedCount != 1 || closed.ActiveCount != 0 || closed.InflightCount != 0 || closed.NextRoundCount != 0 {
|
||||
t.Fatalf("closed metadata=%+v err=%v", closed, err)
|
||||
}
|
||||
metaKey, _ := store.RunMetaKey("interrupted")
|
||||
failedKey, _ := store.FailedKey("interrupted")
|
||||
stateKey, _ := store.TaskStateKey("interrupted")
|
||||
channelsKey, _ := store.RunChannelsKey("interrupted")
|
||||
for _, key := range []string{metaKey, failedKey, stateKey, channelsKey} {
|
||||
ttl, err := store.RedisClient().Do(ctx, "PTTL", key)
|
||||
if err != nil || ttl.Int64() <= 0 || ttl.Int64() > time.Hour.Milliseconds() {
|
||||
t.Fatalf("retention %s=%v err=%v", key, ttl, err)
|
||||
}
|
||||
}
|
||||
if backlog != "none" {
|
||||
if err := fresh.Produce(ctx, scopedMessage("fresh")); err != nil {
|
||||
t.Fatalf("fresh start after draining: %v", err)
|
||||
}
|
||||
}
|
||||
requireMeta(t, store, "fresh", true, 1)
|
||||
active, err := store.ActiveRuns(ctx)
|
||||
if err != nil || len(active) != 1 || active[0] != "fresh" {
|
||||
t.Fatalf("only closed run must leave active index: %v err=%v", active, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user