// Package runtime provides the small concrete storage adapters used by the // full-sync HTTP and archive services. It intentionally keeps queue protocol // transitions in queue/zset and only reads or records durable metadata here. package runtime import ( "context" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "strconv" "strings" "time" "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/frame/g" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract" "git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset" ) type Repository struct { store *zset.Store redis zset.RedisClient keys contract.Keys root string } func (r *Repository) GetRoundStats(ctx context.Context, runID string) ([]contract.RoundStats, error) { stats, err := r.store.GetRoundStats(ctx, runID) if err != nil { return nil, wrapNotFound(err) } out := make([]contract.RoundStats, 0, len(stats)) for _, stat := range stats { out = append(out, contract.RoundStats{ RunID: stat.RunID, Round: stat.Round, EnqueuedCount: stat.EnqueuedCount, ActiveCount: stat.ActiveCount, NextRoundCount: stat.NextRoundCount, SuccessCount: stat.SuccessCount, FailedCount: stat.FailedCount, UpstreamAttemptCount: stat.UpstreamAttemptCount, BatchCount: stat.BatchCount, StartedAt: stat.StartedAt, CompletedAt: stat.CompletedAt, }) } return out, nil } func (r *Repository) RequestReplay(ctx context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) { if err := req.Validate(); err != nil { return contract.ReplayAccepted{}, err } fp, err := req.Fingerprint() if err != nil { return contract.ReplayAccepted{}, err } key, err := r.keys.ReplayIdempotencyKey(req.SourceRunID, req.IdempotencyKey) if err != nil { return contract.ReplayAccepted{}, err } actionID := "replay-" + fp[:24] actionKey, _ := r.keys.ReplayActionKey(actionID) targetID := "replay-run-" + fp[:24] now := time.Now().UTC() action := contract.ReplayAction{ActionID: actionID, SourceRunID: req.SourceRunID, TargetRunID: targetID, RequestedBy: req.RequestedBy, IdempotencyKey: req.IdempotencyKey, Fingerprint: fp, Status: contract.ActionAccepted, ReplayCycle: 1, Progress: contract.ReplayProgress{}, CreatedAt: now, UpdatedAt: now} b, _ := json.Marshal(action) // Reserve idempotency first. A repeat reads the original action; a changed // request under the same key is rejected by fingerprint comparison. set, e := r.redis.Do(ctx, "SET", key, actionID, "NX", "EX", 604800) if e != nil { return contract.ReplayAccepted{}, e } if set.String() == "" || strings.EqualFold(set.String(), "nil") { reserved, e := r.redis.Do(ctx, "GET", key) if e != nil { return contract.ReplayAccepted{}, e } reservedID := reserved.String() old, e := r.GetReplayAction(ctx, reservedID) if e != nil { return contract.ReplayAccepted{}, e } if old.Fingerprint != fp { return contract.ReplayAccepted{}, contract.ErrConflict } u, _ := contract.ActionStatusURL(old.ActionID) return contract.ReplayAccepted{ActionID: old.ActionID, TargetRunID: old.TargetRunID, Status: old.Status, StatusURL: u}, nil } if _, e = r.redis.Do(ctx, "SET", actionKey, string(b), "EX", 604800); e != nil { return contract.ReplayAccepted{}, e } _, _ = r.redis.Do(ctx, "SADD", r.keys.PendingReplayActionsKey(), actionID) go r.executeReplay(actionID) u, _ := contract.ActionStatusURL(actionID) return contract.ReplayAccepted{ActionID: actionID, TargetRunID: targetID, Status: action.Status, StatusURL: u}, nil } // executeReplay is deliberately small: it snapshots the source failure hash, // enqueues equivalent tasks into one target run, then removes only entries // that were accepted. Redis action JSON is the durable retry checkpoint. func (r *Repository) executeReplay(actionID string) { ctx := context.Background() action, err := r.GetReplayAction(ctx, actionID) if err != nil { return } action.Status = contract.ActionProcessing action.UpdatedAt = time.Now().UTC() _ = r.saveAction(ctx, action) source, err := r.GetRunMeta(ctx, action.SourceRunID) if err != nil { action.Error = err.Error() action.Status = contract.ActionNeedsAttention action.UpdatedAt = time.Now().UTC() _ = r.saveAction(ctx, action) return } target := zset.RunMeta{RunID: action.TargetRunID, Operation: source.Operation, QueryMonth: source.QueryMonth, Status: "producing", CurrentRound: 1, CreatedAt: time.Now().UTC(), SyncType: string(contract.SyncReplay), SourceRunID: action.SourceRunID, ReplayCycle: action.ReplayCycle, TriggerSource: "replay"} if err := r.store.CreateRun(ctx, target); err != nil { if _, getErr := r.store.GetRunMeta(ctx, target.RunID); getErr != nil { action.Error = err.Error() action.Status = contract.ActionNeedsAttention action.UpdatedAt = time.Now().UTC() _ = r.saveAction(ctx, action) return } } fkey, _ := r.store.FailedKey(action.SourceRunID) var records []struct { field string rec zset.FailureRecord } var cursor uint64 for { v, e := r.redis.Do(ctx, "HSCAN", fkey, cursor, "COUNT", 500) if e != nil { action.Error = e.Error() action.Status = contract.ActionNeedsAttention action.UpdatedAt = time.Now().UTC() _ = r.saveAction(ctx, action) return } cursor, pairs, e := parseScanResult(v) if e != nil { action.Error = e.Error() action.Status = contract.ActionNeedsAttention action.UpdatedAt = time.Now().UTC() _ = r.saveAction(ctx, action) return } for i := 0; i < len(pairs); i += 2 { var rec zset.FailureRecord if json.Unmarshal([]byte(pairs[i+1].String()), &rec) == nil { records = append(records, struct { field string rec zset.FailureRecord }{pairs[i].String(), rec}) } } if cursor == 0 { break } } groups := map[string][]zset.Task{} for _, item := range records { rec := item.rec chanKey := rec.ChanKey if chanKey == "" { chanKey = "default" } groups[chanKey] = append(groups[chanKey], zset.Task{TaskID: action.TargetRunID + ":" + rec.TaskID, RunID: action.TargetRunID, Operation: rec.Operation, QueryMonth: rec.QueryMonth, ChanKey: chanKey, TempKey: rec.TempKey, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI}) } for chanKey, tasks := range groups { if _, err := r.store.EnqueueRunBatch(ctx, action.TargetRunID, chanKey, tasks, time.Now()); err != nil { action.Error = err.Error() action.Status = contract.ActionNeedsAttention action.UpdatedAt = time.Now().UTC() _ = r.saveAction(ctx, action) return } } for _, item := range records { _, _ = r.redis.Do(ctx, "HDEL", fkey, item.field) } _ = r.store.MarkProducerDone(ctx, action.TargetRunID) action.Progress = contract.ReplayProgress{DiscoveredCount: int64(len(records)), EnqueuedCount: int64(len(records)), CleanedCount: int64(len(records)), FailureCount: int64(len(records))} action.Status = contract.ActionCompleted action.UpdatedAt = time.Now().UTC() completed := action.UpdatedAt action.CompletedAt = &completed digest := sha256.Sum256([]byte(action.ActionID + action.TargetRunID)) action.Receipt = &contract.ArchiveReceipt{Version: 1, Path: "replay://" + action.ActionID, Checksum: hex.EncodeToString(digest[:]), ArchivedAt: completed} _ = r.saveAction(ctx, action) } func (r *Repository) saveAction(ctx context.Context, action contract.ReplayAction) error { b, err := json.Marshal(action) if err != nil { return err } key, err := r.keys.ReplayActionKey(action.ActionID) if err != nil { return err } _, err = r.redis.Do(ctx, "SET", key, string(b), "EX", 604800) return err } func (r *Repository) GetReplayAction(ctx context.Context, actionID string) (contract.ReplayAction, error) { if err := contract.ValidateID(actionID); err != nil { return contract.ReplayAction{}, err } key, _ := r.keys.ReplayActionKey(actionID) v, err := r.redis.Do(ctx, "GET", key) if err != nil { return contract.ReplayAction{}, err } if v == nil || v.IsNil() || strings.TrimSpace(v.String()) == "" { return contract.ReplayAction{}, contract.ErrNotFound } var action contract.ReplayAction if err := json.Unmarshal([]byte(v.String()), &action); err != nil { return contract.ReplayAction{}, err } return action, nil } func New(store *zset.Store, archiveRoot string) (*Repository, error) { if store == nil { return nil, errors.New("full-sync runtime store is nil") } keys, err := contract.NewKeys(store.Prefix()) if err != nil { return nil, err } return &Repository{store: store, redis: store.RedisClient(), keys: keys, root: archiveRoot}, nil } func (r *Repository) GetRunMeta(ctx context.Context, runID string) (contract.RunMeta, error) { m, err := r.store.GetRunMeta(ctx, runID) if err != nil { return contract.RunMeta{}, wrapNotFound(err) } return toContractMeta(m), nil } func (r *Repository) GetArchivedRun(ctx context.Context, runID string) (contract.RunArchive, error) { if err := contract.ValidateID(runID); err != nil { return contract.RunArchive{}, err } if strings.TrimSpace(r.root) == "" { return contract.RunArchive{}, contract.ErrNotFound } b, err := os.ReadFile(filepath.Join(r.root, "run-"+runID+".json")) if err != nil { if errors.Is(err, os.ErrNotExist) { return contract.RunArchive{}, contract.ErrNotFound } return contract.RunArchive{}, err } var archive contract.RunArchive if err := json.Unmarshal(b, &archive); err != nil { return contract.RunArchive{}, fmt.Errorf("decode run archive: %w", err) } if archive.Run.RunID != runID { return contract.RunArchive{}, fmt.Errorf("archive run ID does not match filename") } return archive, nil } func (r *Repository) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) { p, err := page.Normalized() if err != nil { return contract.FailurePage{}, err } key, err := r.store.FailedKey(runID) if err != nil { return contract.FailurePage{}, err } start := uint64(0) if p.Cursor != "" { start, err = decodeCursor(p.Cursor, runID) if err != nil { return contract.FailurePage{}, err } } var items []contract.Failure var next uint64 for { v, e := r.redis.Do(ctx, "HSCAN", key, start, "COUNT", p.Limit) if e != nil { return contract.FailurePage{}, e } next, pairs, e := parseScanResult(v) if e != nil { return contract.FailurePage{}, e } for i := 1; i < len(pairs); i += 2 { var rec zset.FailureRecord if json.Unmarshal([]byte(pairs[i].String()), &rec) == nil { items = append(items, toFailure(rec)) if len(items) >= p.Limit { break } } } if len(items) >= p.Limit || next == 0 { break } start = next } result := contract.FailurePage{Items: items} if next != 0 { result.NextCursor = encodeCursor(runID, next) } if result.Items == nil { result.Items = []contract.Failure{} } return result, nil } func (r *Repository) ListArchiveWork(ctx context.Context, page contract.PageRequest) (contract.ArchiveWorkPage, error) { p, err := page.Normalized() if err != nil { return contract.ArchiveWorkPage{}, err } start := uint64(0) if p.Cursor != "" { start, err = decodeCursor(p.Cursor, "archive") if err != nil { return contract.ArchiveWorkPage{}, err } } v, err := r.redis.Do(ctx, "ZSCAN", r.keys.PendingArchivesKey(), start, "COUNT", p.Limit) if err != nil { return contract.ArchiveWorkPage{}, err } out := contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}} next, pairs, err := parseScanResult(v) if err != nil { return contract.ArchiveWorkPage{}, err } for i := 0; i < len(pairs); i += 2 { out.Items = append(out.Items, contract.ArchiveWork{RunID: pairs[i].String(), Version: pairs[i+1].String()}) } if next != 0 { out.NextCursor = encodeCursor("archive", next) } return out, nil } func parseScanResult(value *gvar.Var) (uint64, []*gvar.Var, error) { if value == nil { return 0, nil, errors.New("redis scan returned nil response") } values := value.Vars() if len(values) != 2 { return 0, nil, fmt.Errorf("unexpected Redis scan response length: %d", len(values)) } pairs := values[1].Vars() if len(pairs)%2 != 0 { return 0, nil, fmt.Errorf("redis scan returned odd number of entry values: %d", len(pairs)) } return uint64(values[0].Int64()), pairs, nil } func (r *Repository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) { if archived, err := r.GetArchivedRun(ctx, work.RunID); err == nil { return archived, nil } meta, err := r.GetRunMeta(ctx, work.RunID) if err != nil { return contract.RunArchive{}, err } rounds, err := r.GetRoundStats(ctx, work.RunID) if err != nil { g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=read error=%q", work.RunID, err.Error()) return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil } if err := validateRoundAccounting(meta, rounds); err != nil { g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=validate error=%q", work.RunID, err.Error()) return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil } return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: rounds}, nil } func validateRoundAccounting(meta contract.RunMeta, rounds []contract.RoundStats) error { if len(rounds) == 0 { return nil } var success, failed int64 seen := make(map[int]struct{}, len(rounds)) for _, round := range rounds { if round.RunID != meta.RunID { return fmt.Errorf("round %d belongs to run %q", round.Round, round.RunID) } if _, ok := seen[round.Round]; ok { return fmt.Errorf("duplicate round %d", round.Round) } seen[round.Round] = struct{}{} if round.EnqueuedCount != round.SuccessCount+round.NextRoundCount+round.FailedCount { return fmt.Errorf("round %d counters do not close", round.Round) } success += round.SuccessCount failed += round.FailedCount } if success != meta.SuccessCount || failed != meta.FailedCount { return fmt.Errorf("run counters do not match round totals: success=%d/%d failed=%d/%d", success, meta.SuccessCount, failed, meta.FailedCount) } return nil } func (r *Repository) RegisterArchiveWork(ctx context.Context, work contract.ArchiveWork) error { if err := contract.ValidateID(work.RunID); err != nil { return err } if strings.TrimSpace(work.Version) == "" { return contract.ErrInvalidArgument } _, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), work.Version, work.RunID) return err } func (r *Repository) RegisterArchive(ctx context.Context, work contract.ArchiveWork, receipt contract.ArchiveReceipt) error { if err := receipt.Validate(); err != nil { return err } key, err := r.keys.ArchiveStateKey(work.RunID) if err != nil { return err } _, err = r.redis.Do(ctx, "HSET", key, "version", receipt.Version, "path", receipt.Path, "checksum", receipt.Checksum, "archivedAt", receipt.ArchivedAt.UTC().Format(time.RFC3339Nano)) if err != nil { return err } _, err = r.redis.Do(ctx, "ZREM", r.keys.PendingArchivesKey(), work.RunID) return err } func (r *Repository) RegisterArchiveFailure(ctx context.Context, work contract.ArchiveWork, message string) error { _, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), time.Now().UnixMilli(), work.RunID) return err } func toContractMeta(m zset.RunMeta) contract.RunMeta { syncType := contract.SyncType(m.SyncType) if syncType == "" { syncType = contract.SyncNormal } return contract.RunMeta{SupplyObservation: m.SupplyObservation, RunID: m.RunID, Operation: m.Operation, QueryMonth: m.QueryMonth, Status: contract.RunStatus(m.Status), ProducerDone: m.ProducerDone, CurrentRound: m.CurrentRound, ActiveCount: m.ActiveCount, InflightCount: m.InflightCount, NextRoundCount: m.NextRoundCount, SuccessCount: m.SuccessCount, FailedCount: m.FailedCount, CreatedAt: m.CreatedAt, ProducerCompletedAt: m.ProducerCompletedAt, CompletedAt: m.CompletedAt, SyncType: syncType, SourceRunID: m.SourceRunID, ReplayCycle: m.ReplayCycle, TriggerSource: m.TriggerSource, ScannedCount: m.ScannedCount, EnqueuedCount: m.EnqueuedCount, UpstreamAttemptCount: m.UpstreamAttemptCount, BatchCount: m.BatchCount, MaxRound: m.MaxRound, PendingFailureCount: m.PendingFailureCount, ArchiveStatus: contract.ArchiveStatus(m.ArchiveStatus), ArchiveVersion: m.ArchiveVersion, ArchivePath: m.ArchivePath, ArchiveChecksum: m.ArchiveChecksum, ReplayDisposition: contract.ReplayDisposition(m.ReplayDisposition), ScopeChannels: m.ScopeChannels, ScopeDigest: m.ScopeDigest, ScopeChannelsSample: m.ScopeChannelsSample, StartedAt: m.StartedAt, ScanCompletedAt: m.ScanCompletedAt, ArchivedAt: m.ArchivedAt} } func toFailure(f zset.FailureRecord) contract.Failure { return contract.Failure{TaskID: f.TaskID, CardID: f.CardID, ErrorCategory: contract.NormalizeErrorCategory(f.FailureType), ErrorMessage: contract.BoundedErrorMessage(f.Message), FailedAt: f.LastFailedAt, Round: f.Round} } func wrapNotFound(err error) error { if strings.Contains(strings.ToLower(err.Error()), "not found") { return contract.ErrNotFound } return err } func encodeCursor(scope string, n uint64) contract.Cursor { return contract.Cursor(base64.RawURLEncoding.EncodeToString([]byte(scope + ":" + strconv.FormatUint(n, 10)))) } func decodeCursor(c contract.Cursor, scope string) (uint64, error) { b, e := base64.RawURLEncoding.DecodeString(string(c)) if e != nil { return 0, contract.ErrInvalidArgument } p := strings.Split(string(b), ":") if len(p) != 2 || p[0] != scope { return 0, contract.ErrInvalidArgument } n, e := strconv.ParseUint(p[1], 10, 64) if e != nil { return 0, contract.ErrInvalidArgument } return n, nil } var _ contract.LiveRunReader = (*Repository)(nil) var _ contract.ArchivedRunReader = (*Repository)(nil) var _ contract.FailurePager = (*Repository)(nil) var _ contract.ArchiveWorkDiscovery = (*Repository)(nil) var _ contract.ArchiveWorkLoader = (*Repository)(nil) var _ contract.ArchiveWorkRegistrar = (*Repository)(nil) var _ contract.ArchiveRegistrar = (*Repository)(nil) var _ contract.ReplayRequester = (*Repository)(nil) var _ contract.ActionReader = (*Repository)(nil)