package runtime import ( "context" "encoding/json" "fmt" "testing" "time" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/testutil" ) func TestSimpleReplayBothNamespacesAndRecovery(t *testing.T) { redis := testutil.Redis(t) ctx := context.Background() traffic, _ := zset.New(redis, "flow:") status, _ := zset.New(redis, "sim-status:") for _, store := range []*zset.Store{traffic, status} { name := store.Prefix() + "source" key, _ := store.RunMetaKey(name) failed, _ := store.FailedKey(name) _, err := redis.Do(ctx, "HSET", key, "runId", name, "operation", "operation", "status", "completed_with_failures", "producerDone", "1", "failedCount", "3", "currentRound", "1") if err != nil { t.Fatal(err) } for i := 1; i <= 3; i++ { rec := zset.FailureRecord{TaskID: fmt.Sprintf("task-%d", i), CardID: int64(i), ICCID: fmt.Sprintf("iccid-%d", i), ChanKey: "ch", TempKey: "template", Operation: "operation", FailureType: "business"} payload, _ := json.Marshal(rec) if _, err := redis.Do(ctx, "HSET", failed, rec.TaskID, string(payload)); err != nil { t.Fatal(err) } } } svc, _ := NewSimpleReplay(traffic, status) for _, prefix := range []string{"flow:", "sim-status:"} { source := prefix + "source" receipt, err := svc.Request(ctx, source, "42", 0) if err != nil { t.Fatal(err) } repeat, err := svc.Request(ctx, source, "42", 0) if err != nil || repeat.ReplayRunID != receipt.ReplayRunID { t.Fatalf("repeat = %+v, %v", repeat, err) } // Recovery competes with the initial goroutine but must keep its ID. svc.reconcile(ctx) deadline := time.Now().Add(4 * time.Second) for time.Now().Before(deadline) { state, err := svc.Status(ctx, receipt.ReplayRunID) if err == nil && state.Status == "completed" { if state.Remaining != 0 || state.Enqueued != 3 { t.Fatalf("state = %+v", state) } store := traffic if prefix == "sim-status:" { store = status } meta, err := store.GetRunMeta(ctx, receipt.ReplayRunID) if err != nil || meta.MaxRound != DefaultReplayMaxRounds || meta.SourceRunID != source || !meta.ProducerDone || meta.ActiveCount != 3 { t.Fatalf("target meta = %+v, %v", meta, err) } sourceMeta, err := store.GetRunMeta(ctx, source) if err != nil || sourceMeta.FailedCount != 3 { t.Fatalf("source meta = %+v, %v", sourceMeta, err) } break } time.Sleep(20 * time.Millisecond) } state, _ := svc.Status(ctx, receipt.ReplayRunID) if state.Status != "completed" { t.Fatalf("handoff timed out: %+v", state) } } } func TestSimpleReplayRejectsRunningAndEmpty(t *testing.T) { redis := testutil.Redis(t) store, _ := zset.New(redis, "flow:") svc, _ := NewSimpleReplay(store) ctx := context.Background() if _, err := svc.Request(ctx, "missing", "1", 0); err != contract.ErrNotFound { t.Fatal(err) } key, _ := store.RunMetaKey("source") _, _ = redis.Do(ctx, "HSET", key, "status", "running") if _, err := svc.Request(ctx, "source", "1", 0); err != contract.ErrConflict { t.Fatal(err) } _, _ = redis.Do(ctx, "HSET", key, "status", "completed_with_failures") if _, err := svc.Request(ctx, "source", "1", 0); err != ErrNoFailures { t.Fatal(err) } }