iotlcm/worker-svr/internal/fullsync/runtime/runtime.go

// Package runtime provides the small concrete storage adapters used by the
// full-sync HTTP and archive services. It intentionally keeps queue protocol
// transitions in queue/zset and only reads or records durable metadata here.
package runtime

import (
	"context"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/gogf/gf/v2/container/gvar"
	"github.com/gogf/gf/v2/frame/g"

	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
	"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/queue/zset"
)

type Repository struct {
	store *zset.Store
	redis zset.RedisClient
	keys  contract.Keys
	root  string
}

func (r *Repository) GetRoundStats(ctx context.Context, runID string) ([]contract.RoundStats, error) {
	stats, err := r.store.GetRoundStats(ctx, runID)
	if err != nil {
		return nil, wrapNotFound(err)
	}
	out := make([]contract.RoundStats, 0, len(stats))
	for _, stat := range stats {
		out = append(out, contract.RoundStats{
			RunID:                stat.RunID,
			Round:                stat.Round,
			EnqueuedCount:        stat.EnqueuedCount,
			ActiveCount:          stat.ActiveCount,
			NextRoundCount:       stat.NextRoundCount,
			SuccessCount:         stat.SuccessCount,
			FailedCount:          stat.FailedCount,
			UpstreamAttemptCount: stat.UpstreamAttemptCount,
			BatchCount:           stat.BatchCount,
			StartedAt:            stat.StartedAt,
			CompletedAt:          stat.CompletedAt,
		})
	}
	return out, nil
}

func (r *Repository) RequestReplay(ctx context.Context, req contract.ReplayRequest) (contract.ReplayAccepted, error) {
	if err := req.Validate(); err != nil {
		return contract.ReplayAccepted{}, err
	}
	fp, err := req.Fingerprint()
	if err != nil {
		return contract.ReplayAccepted{}, err
	}
	key, err := r.keys.ReplayIdempotencyKey(req.SourceRunID, req.IdempotencyKey)
	if err != nil {
		return contract.ReplayAccepted{}, err
	}
	actionID := "replay-" + fp[:24]
	actionKey, _ := r.keys.ReplayActionKey(actionID)
	targetID := "replay-run-" + fp[:24]
	now := time.Now().UTC()
	action := contract.ReplayAction{ActionID: actionID, SourceRunID: req.SourceRunID, TargetRunID: targetID, RequestedBy: req.RequestedBy, IdempotencyKey: req.IdempotencyKey, Fingerprint: fp, Status: contract.ActionAccepted, ReplayCycle: 1, Progress: contract.ReplayProgress{}, CreatedAt: now, UpdatedAt: now}
	b, _ := json.Marshal(action)
	// Reserve idempotency first. A repeat reads the original action; a changed
	// request under the same key is rejected by fingerprint comparison.
	set, e := r.redis.Do(ctx, "SET", key, actionID, "NX", "EX", 604800)
	if e != nil {
		return contract.ReplayAccepted{}, e
	}
	if set.String() == "" || strings.EqualFold(set.String(), "nil") {
		reserved, e := r.redis.Do(ctx, "GET", key)
		if e != nil {
			return contract.ReplayAccepted{}, e
		}
		reservedID := reserved.String()
		old, e := r.GetReplayAction(ctx, reservedID)
		if e != nil {
			return contract.ReplayAccepted{}, e
		}
		if old.Fingerprint != fp {
			return contract.ReplayAccepted{}, contract.ErrConflict
		}
		u, _ := contract.ActionStatusURL(old.ActionID)
		return contract.ReplayAccepted{ActionID: old.ActionID, TargetRunID: old.TargetRunID, Status: old.Status, StatusURL: u}, nil
	}
	if _, e = r.redis.Do(ctx, "SET", actionKey, string(b), "EX", 604800); e != nil {
		return contract.ReplayAccepted{}, e
	}
	_, _ = r.redis.Do(ctx, "SADD", r.keys.PendingReplayActionsKey(), actionID)
	go r.executeReplay(actionID)
	u, _ := contract.ActionStatusURL(actionID)
	return contract.ReplayAccepted{ActionID: actionID, TargetRunID: targetID, Status: action.Status, StatusURL: u}, nil
}

// executeReplay is deliberately small: it snapshots the source failure hash,
// enqueues equivalent tasks into one target run, then removes only entries
// that were accepted. Redis action JSON is the durable retry checkpoint.
func (r *Repository) executeReplay(actionID string) {
	ctx := context.Background()
	action, err := r.GetReplayAction(ctx, actionID)
	if err != nil {
		return
	}
	action.Status = contract.ActionProcessing
	action.UpdatedAt = time.Now().UTC()
	_ = r.saveAction(ctx, action)
	source, err := r.GetRunMeta(ctx, action.SourceRunID)
	if err != nil {
		action.Error = err.Error()
		action.Status = contract.ActionNeedsAttention
		action.UpdatedAt = time.Now().UTC()
		_ = r.saveAction(ctx, action)
		return
	}
	target := zset.RunMeta{RunID: action.TargetRunID, Operation: source.Operation, QueryMonth: source.QueryMonth, Status: "producing", CurrentRound: 1, CreatedAt: time.Now().UTC(), SyncType: string(contract.SyncReplay), SourceRunID: action.SourceRunID, ReplayCycle: action.ReplayCycle, TriggerSource: "replay"}
	if err := r.store.CreateRun(ctx, target); err != nil {
		if _, getErr := r.store.GetRunMeta(ctx, target.RunID); getErr != nil {
			action.Error = err.Error()
			action.Status = contract.ActionNeedsAttention
			action.UpdatedAt = time.Now().UTC()
			_ = r.saveAction(ctx, action)
			return
		}
	}
	fkey, _ := r.store.FailedKey(action.SourceRunID)
	var records []struct {
		field string
		rec   zset.FailureRecord
	}
	var cursor uint64
	for {
		v, e := r.redis.Do(ctx, "HSCAN", fkey, cursor, "COUNT", 500)
		if e != nil {
			action.Error = e.Error()
			action.Status = contract.ActionNeedsAttention
			action.UpdatedAt = time.Now().UTC()
			_ = r.saveAction(ctx, action)
			return
		}
		cursor, pairs, e := parseScanResult(v)
		if e != nil {
			action.Error = e.Error()
			action.Status = contract.ActionNeedsAttention
			action.UpdatedAt = time.Now().UTC()
			_ = r.saveAction(ctx, action)
			return
		}
		for i := 0; i < len(pairs); i += 2 {
			var rec zset.FailureRecord
			if json.Unmarshal([]byte(pairs[i+1].String()), &rec) == nil {
				records = append(records, struct {
					field string
					rec   zset.FailureRecord
				}{pairs[i].String(), rec})
			}
		}
		if cursor == 0 {
			break
		}
	}
	groups := map[string][]zset.Task{}
	for _, item := range records {
		rec := item.rec
		chanKey := rec.ChanKey
		if chanKey == "" {
			chanKey = "default"
		}
		groups[chanKey] = append(groups[chanKey], zset.Task{TaskID: action.TargetRunID + ":" + rec.TaskID, RunID: action.TargetRunID, Operation: rec.Operation, QueryMonth: rec.QueryMonth, ChanKey: chanKey, TempKey: rec.TempKey, CardID: rec.CardID, ICCID: rec.ICCID, MSISDN: rec.MSISDN, IMSI: rec.IMSI})
	}
	for chanKey, tasks := range groups {
		if _, err := r.store.EnqueueRunBatch(ctx, action.TargetRunID, chanKey, tasks, time.Now()); err != nil {
			action.Error = err.Error()
			action.Status = contract.ActionNeedsAttention
			action.UpdatedAt = time.Now().UTC()
			_ = r.saveAction(ctx, action)
			return
		}
	}
	for _, item := range records {
		_, _ = r.redis.Do(ctx, "HDEL", fkey, item.field)
	}
	_ = r.store.MarkProducerDone(ctx, action.TargetRunID)
	action.Progress = contract.ReplayProgress{DiscoveredCount: int64(len(records)), EnqueuedCount: int64(len(records)), CleanedCount: int64(len(records)), FailureCount: int64(len(records))}
	action.Status = contract.ActionCompleted
	action.UpdatedAt = time.Now().UTC()
	completed := action.UpdatedAt
	action.CompletedAt = &completed
	digest := sha256.Sum256([]byte(action.ActionID + action.TargetRunID))
	action.Receipt = &contract.ArchiveReceipt{Version: 1, Path: "replay://" + action.ActionID, Checksum: hex.EncodeToString(digest[:]), ArchivedAt: completed}
	_ = r.saveAction(ctx, action)
}

func (r *Repository) saveAction(ctx context.Context, action contract.ReplayAction) error {
	b, err := json.Marshal(action)
	if err != nil {
		return err
	}
	key, err := r.keys.ReplayActionKey(action.ActionID)
	if err != nil {
		return err
	}
	_, err = r.redis.Do(ctx, "SET", key, string(b), "EX", 604800)
	return err
}

func (r *Repository) GetReplayAction(ctx context.Context, actionID string) (contract.ReplayAction, error) {
	if err := contract.ValidateID(actionID); err != nil {
		return contract.ReplayAction{}, err
	}
	key, _ := r.keys.ReplayActionKey(actionID)
	v, err := r.redis.Do(ctx, "GET", key)
	if err != nil {
		return contract.ReplayAction{}, err
	}
	if v == nil || v.IsNil() || strings.TrimSpace(v.String()) == "" {
		return contract.ReplayAction{}, contract.ErrNotFound
	}
	var action contract.ReplayAction
	if err := json.Unmarshal([]byte(v.String()), &action); err != nil {
		return contract.ReplayAction{}, err
	}
	return action, nil
}

func New(store *zset.Store, archiveRoot string) (*Repository, error) {
	if store == nil {
		return nil, errors.New("full-sync runtime store is nil")
	}
	keys, err := contract.NewKeys(store.Prefix())
	if err != nil {
		return nil, err
	}
	return &Repository{store: store, redis: store.RedisClient(), keys: keys, root: archiveRoot}, nil
}

func (r *Repository) GetRunMeta(ctx context.Context, runID string) (contract.RunMeta, error) {
	m, err := r.store.GetRunMeta(ctx, runID)
	if err != nil {
		return contract.RunMeta{}, wrapNotFound(err)
	}
	return toContractMeta(m), nil
}

func (r *Repository) GetArchivedRun(ctx context.Context, runID string) (contract.RunArchive, error) {
	if err := contract.ValidateID(runID); err != nil {
		return contract.RunArchive{}, err
	}
	if strings.TrimSpace(r.root) == "" {
		return contract.RunArchive{}, contract.ErrNotFound
	}
	b, err := os.ReadFile(filepath.Join(r.root, "run-"+runID+".json"))
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return contract.RunArchive{}, contract.ErrNotFound
		}
		return contract.RunArchive{}, err
	}
	var archive contract.RunArchive
	if err := json.Unmarshal(b, &archive); err != nil {
		return contract.RunArchive{}, fmt.Errorf("decode run archive: %w", err)
	}
	if archive.Run.RunID != runID {
		return contract.RunArchive{}, fmt.Errorf("archive run ID does not match filename")
	}
	return archive, nil
}

func (r *Repository) ListFailures(ctx context.Context, runID string, page contract.PageRequest) (contract.FailurePage, error) {
	p, err := page.Normalized()
	if err != nil {
		return contract.FailurePage{}, err
	}
	key, err := r.store.FailedKey(runID)
	if err != nil {
		return contract.FailurePage{}, err
	}
	start := uint64(0)
	if p.Cursor != "" {
		start, err = decodeCursor(p.Cursor, runID)
		if err != nil {
			return contract.FailurePage{}, err
		}
	}
	var items []contract.Failure
	var next uint64
	for {
		v, e := r.redis.Do(ctx, "HSCAN", key, start, "COUNT", p.Limit)
		if e != nil {
			return contract.FailurePage{}, e
		}
		next, pairs, e := parseScanResult(v)
		if e != nil {
			return contract.FailurePage{}, e
		}
		for i := 1; i < len(pairs); i += 2 {
			var rec zset.FailureRecord
			if json.Unmarshal([]byte(pairs[i].String()), &rec) == nil {
				items = append(items, toFailure(rec))
				if len(items) >= p.Limit {
					break
				}
			}
		}
		if len(items) >= p.Limit || next == 0 {
			break
		}
		start = next
	}
	result := contract.FailurePage{Items: items}
	if next != 0 {
		result.NextCursor = encodeCursor(runID, next)
	}
	if result.Items == nil {
		result.Items = []contract.Failure{}
	}
	return result, nil
}

func (r *Repository) ListArchiveWork(ctx context.Context, page contract.PageRequest) (contract.ArchiveWorkPage, error) {
	p, err := page.Normalized()
	if err != nil {
		return contract.ArchiveWorkPage{}, err
	}
	start := uint64(0)
	if p.Cursor != "" {
		start, err = decodeCursor(p.Cursor, "archive")
		if err != nil {
			return contract.ArchiveWorkPage{}, err
		}
	}
	v, err := r.redis.Do(ctx, "ZSCAN", r.keys.PendingArchivesKey(), start, "COUNT", p.Limit)
	if err != nil {
		return contract.ArchiveWorkPage{}, err
	}
	out := contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}
	next, pairs, err := parseScanResult(v)
	if err != nil {
		return contract.ArchiveWorkPage{}, err
	}
	for i := 0; i < len(pairs); i += 2 {
		out.Items = append(out.Items, contract.ArchiveWork{RunID: pairs[i].String(), Version: pairs[i+1].String()})
	}
	if next != 0 {
		out.NextCursor = encodeCursor("archive", next)
	}
	return out, nil
}

func parseScanResult(value *gvar.Var) (uint64, []*gvar.Var, error) {
	if value == nil {
		return 0, nil, errors.New("redis scan returned nil response")
	}
	values := value.Vars()
	if len(values) != 2 {
		return 0, nil, fmt.Errorf("unexpected Redis scan response length: %d", len(values))
	}
	pairs := values[1].Vars()
	if len(pairs)%2 != 0 {
		return 0, nil, fmt.Errorf("redis scan returned odd number of entry values: %d", len(pairs))
	}
	return uint64(values[0].Int64()), pairs, nil
}

func (r *Repository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
	if archived, err := r.GetArchivedRun(ctx, work.RunID); err == nil {
		return archived, nil
	}
	meta, err := r.GetRunMeta(ctx, work.RunID)
	if err != nil {
		return contract.RunArchive{}, err
	}
	rounds, err := r.GetRoundStats(ctx, work.RunID)
	if err != nil {
		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=read error=%q", work.RunID, err.Error())
		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
	}
	if err := validateRoundAccounting(meta, rounds); err != nil {
		g.Log().Warningf(ctx, "event=full_sync_archive_rounds status=failed runId=%s stage=validate error=%q", work.RunID, err.Error())
		return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: []contract.RoundStats{}}, nil
	}
	return contract.RunArchive{Version: contract.ArchiveSchemaVersion, Run: meta, Rounds: rounds}, nil
}

func validateRoundAccounting(meta contract.RunMeta, rounds []contract.RoundStats) error {
	if len(rounds) == 0 {
		return nil
	}
	var success, failed int64
	seen := make(map[int]struct{}, len(rounds))
	for _, round := range rounds {
		if round.RunID != meta.RunID {
			return fmt.Errorf("round %d belongs to run %q", round.Round, round.RunID)
		}
		if _, ok := seen[round.Round]; ok {
			return fmt.Errorf("duplicate round %d", round.Round)
		}
		seen[round.Round] = struct{}{}
		if round.EnqueuedCount != round.SuccessCount+round.NextRoundCount+round.FailedCount {
			return fmt.Errorf("round %d counters do not close", round.Round)
		}
		success += round.SuccessCount
		failed += round.FailedCount
	}
	if success != meta.SuccessCount || failed != meta.FailedCount {
		return fmt.Errorf("run counters do not match round totals: success=%d/%d failed=%d/%d", success, meta.SuccessCount, failed, meta.FailedCount)
	}
	return nil
}

func (r *Repository) RegisterArchiveWork(ctx context.Context, work contract.ArchiveWork) error {
	if err := contract.ValidateID(work.RunID); err != nil {
		return err
	}
	if strings.TrimSpace(work.Version) == "" {
		return contract.ErrInvalidArgument
	}
	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), work.Version, work.RunID)
	return err
}

func (r *Repository) RegisterArchive(ctx context.Context, work contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
	if err := receipt.Validate(); err != nil {
		return err
	}
	key, err := r.keys.ArchiveStateKey(work.RunID)
	if err != nil {
		return err
	}
	_, err = r.redis.Do(ctx, "HSET", key, "version", receipt.Version, "path", receipt.Path, "checksum", receipt.Checksum, "archivedAt", receipt.ArchivedAt.UTC().Format(time.RFC3339Nano))
	if err != nil {
		return err
	}
	_, err = r.redis.Do(ctx, "ZREM", r.keys.PendingArchivesKey(), work.RunID)
	return err
}

func (r *Repository) RegisterArchiveFailure(ctx context.Context, work contract.ArchiveWork, message string) error {
	_, err := r.redis.Do(ctx, "ZADD", r.keys.PendingArchivesKey(), time.Now().UnixMilli(), work.RunID)
	return err
}

func toContractMeta(m zset.RunMeta) contract.RunMeta {
	syncType := contract.SyncType(m.SyncType)
	if syncType == "" {
		syncType = contract.SyncNormal
	}
	return contract.RunMeta{SupplyObservation: m.SupplyObservation, RunID: m.RunID, Operation: m.Operation, QueryMonth: m.QueryMonth, Status: contract.RunStatus(m.Status), ProducerDone: m.ProducerDone, CurrentRound: m.CurrentRound, ActiveCount: m.ActiveCount, InflightCount: m.InflightCount, NextRoundCount: m.NextRoundCount, SuccessCount: m.SuccessCount, FailedCount: m.FailedCount, CreatedAt: m.CreatedAt, ProducerCompletedAt: m.ProducerCompletedAt, CompletedAt: m.CompletedAt, SyncType: syncType, SourceRunID: m.SourceRunID, ReplayCycle: m.ReplayCycle, TriggerSource: m.TriggerSource, ScannedCount: m.ScannedCount, EnqueuedCount: m.EnqueuedCount, UpstreamAttemptCount: m.UpstreamAttemptCount, BatchCount: m.BatchCount, MaxRound: m.MaxRound, PendingFailureCount: m.PendingFailureCount, ArchiveStatus: contract.ArchiveStatus(m.ArchiveStatus), ArchiveVersion: m.ArchiveVersion, ArchivePath: m.ArchivePath, ArchiveChecksum: m.ArchiveChecksum, ReplayDisposition: contract.ReplayDisposition(m.ReplayDisposition), ScopeChannels: m.ScopeChannels, ScopeDigest: m.ScopeDigest, ScopeChannelsSample: m.ScopeChannelsSample, StartedAt: m.StartedAt, ScanCompletedAt: m.ScanCompletedAt, ArchivedAt: m.ArchivedAt}
}
func toFailure(f zset.FailureRecord) contract.Failure {
	return contract.Failure{TaskID: f.TaskID, CardID: f.CardID, ErrorCategory: contract.NormalizeErrorCategory(f.FailureType), ErrorMessage: contract.BoundedErrorMessage(f.Message), FailedAt: f.LastFailedAt, Round: f.Round}
}
func wrapNotFound(err error) error {
	if strings.Contains(strings.ToLower(err.Error()), "not found") {
		return contract.ErrNotFound
	}
	return err
}
func encodeCursor(scope string, n uint64) contract.Cursor {
	return contract.Cursor(base64.RawURLEncoding.EncodeToString([]byte(scope + ":" + strconv.FormatUint(n, 10))))
}
func decodeCursor(c contract.Cursor, scope string) (uint64, error) {
	b, e := base64.RawURLEncoding.DecodeString(string(c))
	if e != nil {
		return 0, contract.ErrInvalidArgument
	}
	p := strings.Split(string(b), ":")
	if len(p) != 2 || p[0] != scope {
		return 0, contract.ErrInvalidArgument
	}
	n, e := strconv.ParseUint(p[1], 10, 64)
	if e != nil {
		return 0, contract.ErrInvalidArgument
	}
	return n, nil
}

var _ contract.LiveRunReader = (*Repository)(nil)
var _ contract.ArchivedRunReader = (*Repository)(nil)
var _ contract.FailurePager = (*Repository)(nil)
var _ contract.ArchiveWorkDiscovery = (*Repository)(nil)
var _ contract.ArchiveWorkLoader = (*Repository)(nil)
var _ contract.ArchiveWorkRegistrar = (*Repository)(nil)
var _ contract.ArchiveRegistrar = (*Repository)(nil)
var _ contract.ReplayRequester = (*Repository)(nil)
var _ contract.ActionReader = (*Repository)(nil)