chore: 初始开源发布
iotlcm-worker-svr：异步执行中枢（RabbitMQ 消费者 + 全量同步编排）。 - Go module：iotlcm-worker-svr → git.5iot.com/iotlcm-public/iotlcm-worker-svr（214 文件） - 凭据与内部标识脱敏：MySQL/Redis 凭据、私网与生产 IP、内网域名、 库名 xiujie_admin → iotlcm、哨兵名占位 - 159 处真实手机号 → 合法 11 位编造号（保留号段与后 4 位，维持互异， 不破坏「校验 MSISDN 格式」等测试断言） - 移除内部资产：CI 触发器、Jenkins 触发测试文档、预演门禁与镜像 digest 记录 - Dockerfile 改用公共基础镜像（golang:1.25-alpine → alpine:3.23）， 移除写死的 git 凭据 - README 重写（架构/全量同步模型/配置/已知失败），补 Apache-2.0 LICENSE，LF 行尾 + .gitattributes 已知：3 个测试包在源仓同一提交上同样失败（client/upstream、 service/allocate、service/inbound），属既有缺陷，本仓未引入亦未修复。
This commit is contained in:
MiFaZhan
@@ -0,0 +1,802 @@
|
||||
// Package archive durably stores summary-only receipts for completed full-sync
|
||||
// runs and replay actions. It depends only on the shared full-sync contracts;
|
||||
// repository implementations and service wiring live elsewhere.
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultInterval = time.Minute
|
||||
defaultBackoff = time.Second
|
||||
defaultMaxBackoff = time.Minute
|
||||
defaultMaxAttempts = 4
|
||||
defaultReplayLeaseTTL = time.Minute
|
||||
defaultReplayLeaseName = "fullsync-archive"
|
||||
defaultReleaseTimeout = 5 * time.Second
|
||||
maxRounds = 10000
|
||||
)
|
||||
|
||||
var errChecksumMismatch = gerror.New("archive checksum verification failed")
|
||||
|
||||
// Config controls the recovery scanner. Root is the archive directory. Zero
|
||||
// timing values select conservative defaults.
|
||||
type Config struct {
|
||||
Root string
|
||||
Interval time.Duration
|
||||
InitialBackoff time.Duration
|
||||
MaxBackoff time.Duration
|
||||
MaxAttempts int
|
||||
ReplayLeaseOwner string
|
||||
ReplayLeaseTTL time.Duration
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// ReplayRepositories are the contract boundaries used to recover replay
|
||||
// actions left in receipt_pending after a process restart.
|
||||
type ReplayRepositories struct {
|
||||
Discovery contract.ReplayWorkDiscovery
|
||||
Leases contract.ReplayLeaseStore
|
||||
Progress contract.ReplayProgressStore
|
||||
}
|
||||
|
||||
// Service publishes archive files and scans durable repository indexes at
|
||||
// startup and periodically. A Service may be configured for runs, actions, or
|
||||
// both.
|
||||
type Service struct {
|
||||
root string
|
||||
runDiscovery contract.ArchiveWorkDiscovery
|
||||
runLoader contract.ArchiveWorkLoader
|
||||
runRegistrar contract.ArchiveRegistrar
|
||||
replay ReplayRepositories
|
||||
interval time.Duration
|
||||
initialBackoff time.Duration
|
||||
maxBackoff time.Duration
|
||||
maxAttempts int
|
||||
leaseOwner string
|
||||
leaseTTL time.Duration
|
||||
now func() time.Time
|
||||
|
||||
// Publications are serialized so repeat writes in this process observe the
|
||||
// same final file timestamp and therefore produce the same receipt.
|
||||
publishMu sync.Mutex
|
||||
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
|
||||
// These hooks keep failure-path tests deterministic without weakening the
|
||||
// production filesystem path.
|
||||
replaceFile func(string, string) error
|
||||
afterReplace func(string) error
|
||||
}
|
||||
|
||||
// New constructs a run-archive service with default scanner settings.
|
||||
func New(root string, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
|
||||
return NewService(Config{Root: root}, discovery, loader, registrar)
|
||||
}
|
||||
|
||||
// NewService constructs a run-archive service. Use NewServiceWithReplay when
|
||||
// replay receipt recovery is also required.
|
||||
func NewService(cfg Config, discovery contract.ArchiveWorkDiscovery, loader contract.ArchiveWorkLoader, registrar contract.ArchiveRegistrar) *Service {
|
||||
return NewServiceWithReplay(cfg, discovery, loader, registrar, ReplayRepositories{})
|
||||
}
|
||||
|
||||
// NewServiceWithReplay constructs a lifecycle-enabled run and replay archive
|
||||
// service using only shared contract interfaces.
|
||||
func NewServiceWithReplay(
|
||||
cfg Config,
|
||||
discovery contract.ArchiveWorkDiscovery,
|
||||
loader contract.ArchiveWorkLoader,
|
||||
registrar contract.ArchiveRegistrar,
|
||||
replay ReplayRepositories,
|
||||
) *Service {
|
||||
if cfg.Interval <= 0 {
|
||||
cfg.Interval = defaultInterval
|
||||
}
|
||||
if cfg.InitialBackoff <= 0 {
|
||||
cfg.InitialBackoff = defaultBackoff
|
||||
}
|
||||
if cfg.MaxBackoff <= 0 {
|
||||
cfg.MaxBackoff = defaultMaxBackoff
|
||||
}
|
||||
if cfg.MaxBackoff < cfg.InitialBackoff {
|
||||
cfg.MaxBackoff = cfg.InitialBackoff
|
||||
}
|
||||
if cfg.MaxAttempts <= 0 {
|
||||
cfg.MaxAttempts = defaultMaxAttempts
|
||||
}
|
||||
if strings.TrimSpace(cfg.ReplayLeaseOwner) == "" {
|
||||
cfg.ReplayLeaseOwner = defaultReplayLeaseName
|
||||
}
|
||||
if cfg.ReplayLeaseTTL <= 0 {
|
||||
cfg.ReplayLeaseTTL = defaultReplayLeaseTTL
|
||||
}
|
||||
if cfg.Now == nil {
|
||||
cfg.Now = time.Now
|
||||
}
|
||||
|
||||
return &Service{
|
||||
root: cfg.Root,
|
||||
runDiscovery: discovery,
|
||||
runLoader: loader,
|
||||
runRegistrar: registrar,
|
||||
replay: replay,
|
||||
interval: cfg.Interval,
|
||||
initialBackoff: cfg.InitialBackoff,
|
||||
maxBackoff: cfg.MaxBackoff,
|
||||
maxAttempts: cfg.MaxAttempts,
|
||||
leaseOwner: cfg.ReplayLeaseOwner,
|
||||
leaseTTL: cfg.ReplayLeaseTTL,
|
||||
now: cfg.Now,
|
||||
replaceFile: atomicReplace,
|
||||
afterReplace: func(string) error { return nil },
|
||||
}
|
||||
}
|
||||
|
||||
// Start launches an immediate recovery scan followed by periodic scans. It is
|
||||
// idempotent while the service is running.
|
||||
func (s *Service) Start(parent context.Context) error {
|
||||
if parent == nil {
|
||||
return gerror.New("archive parent context is nil")
|
||||
}
|
||||
if err := s.validateConfiguration(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if s.running {
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
done := make(chan struct{})
|
||||
s.cancel, s.done, s.running = cancel, done, true
|
||||
s.mu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
if s.done == done {
|
||||
s.running = false
|
||||
s.cancel = nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
close(done)
|
||||
}()
|
||||
s.scanLoop(ctx)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop cancels the scanner and waits for it to finish. It is safe to call more
|
||||
// than once.
|
||||
func (s *Service) Stop() error {
|
||||
s.mu.Lock()
|
||||
if !s.running {
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
cancel, done := s.cancel, s.done
|
||||
s.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
<-done
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close implements the common service lifecycle convention.
|
||||
func (s *Service) Close() error { return s.Stop() }
|
||||
|
||||
// Done is closed when the current scanner invocation exits. It returns nil
|
||||
// before the service has been started.
|
||||
func (s *Service) Done() <-chan struct{} {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.done
|
||||
}
|
||||
|
||||
// RunOnce performs one recovery pass. It is useful for explicit lifecycle
|
||||
// wiring and deterministic integration tests.
|
||||
func (s *Service) RunOnce(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return gerror.New("archive scan context is nil")
|
||||
}
|
||||
if err := s.validateConfiguration(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var scanErrors []error
|
||||
if s.runDiscovery != nil {
|
||||
if err := s.scanRuns(ctx); err != nil {
|
||||
scanErrors = append(scanErrors, gerror.Wrap(err, "scan run archives"))
|
||||
}
|
||||
}
|
||||
if s.replay.Discovery != nil && ctx.Err() == nil {
|
||||
if err := s.scanActions(ctx); err != nil {
|
||||
scanErrors = append(scanErrors, gerror.Wrap(err, "scan replay action archives"))
|
||||
}
|
||||
}
|
||||
return errors.Join(scanErrors...)
|
||||
}
|
||||
|
||||
// PublishRun atomically writes a run summary and then conditionally registers
|
||||
// the durable receipt against the work version.
|
||||
func (s *Service) PublishRun(ctx context.Context, work contract.ArchiveWork, snapshot contract.RunArchive) (contract.ArchiveReceipt, error) {
|
||||
if err := validateWork(work); err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
normalized, err := normalizeRunArchive(work, snapshot)
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
|
||||
pathTime := normalized.Run.CompletedAt
|
||||
if pathTime.IsZero() {
|
||||
pathTime = normalized.Run.CreatedAt
|
||||
}
|
||||
receipt, err := s.publishJSON(ctx, "runs", work.RunID, pathTime, normalized)
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
if s.runRegistrar != nil {
|
||||
if err := s.runRegistrar.RegisterArchive(ctx, work, receipt); err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "register run archive")
|
||||
}
|
||||
}
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
// PublishAction atomically writes a receipt_pending replay action summary. The
|
||||
// repository update is performed by ArchiveAction because it requires a fenced
|
||||
// replay lease.
|
||||
func (s *Service) PublishAction(ctx context.Context, action contract.ReplayAction) (contract.ArchiveReceipt, error) {
|
||||
normalized, err := normalizeAction(action)
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
return s.publishJSON(ctx, "actions", normalized.ActionID, normalized.CreatedAt, normalized)
|
||||
}
|
||||
|
||||
// ArchiveRun loads, publishes, and registers one durable run work item.
|
||||
func (s *Service) ArchiveRun(ctx context.Context, work contract.ArchiveWork) error {
|
||||
if s.runLoader == nil {
|
||||
return gerror.New("archive run loader is nil")
|
||||
}
|
||||
if s.runRegistrar == nil {
|
||||
return gerror.New("archive run registrar is nil")
|
||||
}
|
||||
|
||||
snapshot, err := s.runLoader.LoadArchiveWork(ctx, work)
|
||||
if err != nil {
|
||||
s.recordRunFailure(ctx, work, err)
|
||||
return gerror.Wrap(err, "load run archive work")
|
||||
}
|
||||
if _, err = s.PublishRun(ctx, work, snapshot); err != nil {
|
||||
s.recordRunFailure(ctx, work, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ArchiveAction acquires a replay lease, atomically publishes the action
|
||||
// summary, and fences the completed receipt update through ReplayProgressStore.
|
||||
func (s *Service) ArchiveAction(ctx context.Context, action contract.ReplayAction) error {
|
||||
if s.replay.Leases == nil || s.replay.Progress == nil {
|
||||
return gerror.New("replay archive repositories are incomplete")
|
||||
}
|
||||
if action.Status != contract.ActionReceiptPending {
|
||||
return gerror.Wrap(contract.ErrInvalidArgument, "replay action is not receipt_pending")
|
||||
}
|
||||
if _, err := normalizeAction(action); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lease, err := s.replay.Leases.AcquireReplayLease(ctx, action.ActionID, s.leaseOwner, s.leaseTTL)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "acquire replay archive lease")
|
||||
}
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultReleaseTimeout)
|
||||
defer cancel()
|
||||
_ = s.replay.Leases.ReleaseReplayLease(releaseCtx, lease)
|
||||
}()
|
||||
|
||||
receipt, err := s.PublishAction(ctx, action)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lease, err = s.replay.Leases.RenewReplayLease(ctx, lease, s.leaseTTL)
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "renew replay archive lease")
|
||||
}
|
||||
completedAt := receipt.ArchivedAt
|
||||
_, err = s.replay.Progress.UpdateReplayAction(ctx, contract.ReplayActionUpdate{
|
||||
Lease: lease,
|
||||
ExpectedRevision: action.Revision,
|
||||
Status: contract.ActionCompleted,
|
||||
Progress: action.Progress,
|
||||
Receipt: &receipt,
|
||||
CompletedAt: &completedAt,
|
||||
})
|
||||
if err != nil {
|
||||
return gerror.Wrap(err, "register replay action archive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) scanLoop(ctx context.Context) {
|
||||
_ = s.RunOnce(ctx)
|
||||
ticker := time.NewTicker(s.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
_ = s.RunOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) scanRuns(ctx context.Context) error {
|
||||
page := contract.PageRequest{Limit: contract.DefaultPageLimit}
|
||||
seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
|
||||
var itemErrors []error
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := s.runDiscovery.ListArchiveWork(ctx, page)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, work := range result.Items {
|
||||
err := s.retry(ctx, func() error { return s.ArchiveRun(ctx, work) })
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
itemErrors = append(itemErrors, gerror.Wrapf(err, "archive run %q", work.RunID))
|
||||
}
|
||||
}
|
||||
if result.NextCursor == "" {
|
||||
return errors.Join(itemErrors...)
|
||||
}
|
||||
if _, repeated := seenCursors[result.NextCursor]; repeated {
|
||||
return errors.Join(append(itemErrors, gerror.New("run archive pagination cursor did not advance"))...)
|
||||
}
|
||||
seenCursors[result.NextCursor] = struct{}{}
|
||||
page.Cursor = result.NextCursor
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) scanActions(ctx context.Context) error {
|
||||
page := contract.PageRequest{Limit: contract.DefaultPageLimit}
|
||||
seenCursors := map[contract.Cursor]struct{}{page.Cursor: {}}
|
||||
var itemErrors []error
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := s.replay.Discovery.ListReplayWork(ctx, page)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, action := range result.Items {
|
||||
if action.Status != contract.ActionReceiptPending {
|
||||
continue
|
||||
}
|
||||
err := s.retry(ctx, func() error { return s.ArchiveAction(ctx, action) })
|
||||
if errors.Is(err, contract.ErrBusy) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
itemErrors = append(itemErrors, gerror.Wrapf(err, "archive replay action %q", action.ActionID))
|
||||
}
|
||||
}
|
||||
if result.NextCursor == "" {
|
||||
return errors.Join(itemErrors...)
|
||||
}
|
||||
if _, repeated := seenCursors[result.NextCursor]; repeated {
|
||||
return errors.Join(append(itemErrors, gerror.New("replay archive pagination cursor did not advance"))...)
|
||||
}
|
||||
seenCursors[result.NextCursor] = struct{}{}
|
||||
page.Cursor = result.NextCursor
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) retry(ctx context.Context, fn func() error) error {
|
||||
delay := s.initialBackoff
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < s.maxAttempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
lastErr = fn()
|
||||
if lastErr == nil || errors.Is(lastErr, contract.ErrBusy) {
|
||||
return lastErr
|
||||
}
|
||||
if attempt+1 == s.maxAttempts {
|
||||
break
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
if delay <= s.maxBackoff/2 {
|
||||
delay *= 2
|
||||
} else {
|
||||
delay = s.maxBackoff
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (s *Service) recordRunFailure(ctx context.Context, work contract.ArchiveWork, archiveErr error) {
|
||||
if s.runRegistrar == nil || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
_ = s.runRegistrar.RegisterArchiveFailure(ctx, work, contract.BoundedErrorMessage(archiveErr.Error()))
|
||||
}
|
||||
|
||||
func (s *Service) publishJSON(ctx context.Context, kind, id string, pathTime time.Time, value any) (contract.ArchiveReceipt, error) {
|
||||
if ctx == nil {
|
||||
return contract.ArchiveReceipt{}, gerror.New("archive publication context is nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
dir, final, relative, err := s.storagePath(kind, id, pathTime)
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
|
||||
s.publishMu.Lock()
|
||||
defer s.publishMu.Unlock()
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive directory")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
|
||||
temp, err := os.CreateTemp(dir, "."+id+"-*.tmp")
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "create archive temporary file")
|
||||
}
|
||||
tempName := temp.Name()
|
||||
defer func() { _ = os.Remove(tempName) }()
|
||||
|
||||
hasher := sha256.New()
|
||||
buffer := bufio.NewWriter(io.MultiWriter(contextWriter{ctx: ctx, writer: temp}, hasher))
|
||||
encoder := json.NewEncoder(buffer)
|
||||
if err = encoder.Encode(value); err == nil {
|
||||
err = buffer.Flush()
|
||||
}
|
||||
if err == nil {
|
||||
err = temp.Chmod(0o644)
|
||||
}
|
||||
if err == nil {
|
||||
err = temp.Sync()
|
||||
}
|
||||
closeErr := temp.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
_ = temp.Close()
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "encode archive temporary file")
|
||||
}
|
||||
expectedChecksum := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
if receipt, ok, existingErr := s.existingReceipt(final, relative, kind, id, expectedChecksum); existingErr != nil {
|
||||
return contract.ArchiveReceipt{}, existingErr
|
||||
} else if ok {
|
||||
if err := syncDirectory(dir); err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
|
||||
}
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
if err := s.replaceFile(tempName, final); err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "publish archive file")
|
||||
}
|
||||
if err := syncDirectory(dir); err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "sync archive directory")
|
||||
}
|
||||
if err := s.afterReplace(final); err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "archive post-publish verification hook")
|
||||
}
|
||||
|
||||
stored, err := os.ReadFile(final)
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "read published archive file")
|
||||
}
|
||||
storedSum := sha256.Sum256(stored)
|
||||
if hex.EncodeToString(storedSum[:]) != expectedChecksum {
|
||||
return contract.ArchiveReceipt{}, errChecksumMismatch
|
||||
}
|
||||
info, err := os.Stat(final)
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, gerror.Wrap(err, "stat published archive file")
|
||||
}
|
||||
receipt := contract.ArchiveReceipt{
|
||||
Version: contract.ArchiveSchemaVersion,
|
||||
Path: relative,
|
||||
Checksum: expectedChecksum,
|
||||
ArchivedAt: archiveModTime(info, s.now),
|
||||
}
|
||||
if err := receipt.Validate(); err != nil {
|
||||
return contract.ArchiveReceipt{}, err
|
||||
}
|
||||
return receipt, nil
|
||||
}
|
||||
|
||||
func (s *Service) existingReceipt(final, relative, kind, id, expectedChecksum string) (contract.ArchiveReceipt, bool, error) {
|
||||
info, err := os.Lstat(final)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return contract.ArchiveReceipt{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "inspect existing archive file")
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return contract.ArchiveReceipt{}, false, nil
|
||||
}
|
||||
stored, err := os.ReadFile(final)
|
||||
if err != nil {
|
||||
return contract.ArchiveReceipt{}, false, gerror.Wrap(err, "read existing archive file")
|
||||
}
|
||||
storedSum := sha256.Sum256(stored)
|
||||
if hex.EncodeToString(storedSum[:]) != expectedChecksum {
|
||||
if validExistingArchive(kind, id, stored) {
|
||||
return contract.ArchiveReceipt{}, false, gerror.Wrap(contract.ErrConflict, "archive path already contains different valid content")
|
||||
}
|
||||
return contract.ArchiveReceipt{}, false, nil
|
||||
}
|
||||
receipt := contract.ArchiveReceipt{
|
||||
Version: contract.ArchiveSchemaVersion,
|
||||
Path: relative,
|
||||
Checksum: expectedChecksum,
|
||||
ArchivedAt: archiveModTime(info, s.now),
|
||||
}
|
||||
if err := receipt.Validate(); err != nil {
|
||||
return contract.ArchiveReceipt{}, false, err
|
||||
}
|
||||
return receipt, true, nil
|
||||
}
|
||||
|
||||
func validExistingArchive(kind, id string, data []byte) bool {
|
||||
switch kind {
|
||||
case "runs":
|
||||
var snapshot contract.RunArchive
|
||||
if json.Unmarshal(data, &snapshot) != nil {
|
||||
return false
|
||||
}
|
||||
_, err := normalizeRunArchive(contract.ArchiveWork{RunID: id, Version: "existing"}, snapshot)
|
||||
return err == nil
|
||||
case "actions":
|
||||
var action contract.ReplayAction
|
||||
if json.Unmarshal(data, &action) != nil || action.ActionID != id {
|
||||
return false
|
||||
}
|
||||
_, err := normalizeAction(action)
|
||||
return err == nil
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func archiveModTime(info os.FileInfo, now func() time.Time) time.Time {
|
||||
archivedAt := info.ModTime().UTC()
|
||||
if archivedAt.IsZero() {
|
||||
archivedAt = now().UTC()
|
||||
}
|
||||
return archivedAt
|
||||
}
|
||||
|
||||
func (s *Service) storagePath(kind, id string, pathTime time.Time) (string, string, string, error) {
|
||||
if err := validateStorageID(id); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
if kind != "runs" && kind != "actions" {
|
||||
return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "unknown archive kind")
|
||||
}
|
||||
if strings.TrimSpace(s.root) == "" {
|
||||
return "", "", "", gerror.New("archive root is empty")
|
||||
}
|
||||
if pathTime.IsZero() {
|
||||
pathTime = s.now().UTC()
|
||||
} else {
|
||||
pathTime = pathTime.UTC()
|
||||
}
|
||||
if pathTime.Year() < 1 || pathTime.Year() > 9999 {
|
||||
return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive timestamp is outside the supported year range")
|
||||
}
|
||||
|
||||
root := filepath.Clean(s.root)
|
||||
prefix := "run-"
|
||||
if kind == "actions" {
|
||||
prefix = "action-"
|
||||
}
|
||||
relativeOS := prefix + id + ".json"
|
||||
final := filepath.Join(root, relativeOS)
|
||||
contained, err := filepath.Rel(root, final)
|
||||
if err != nil || filepath.IsAbs(contained) || contained == ".." || strings.HasPrefix(contained, ".."+string(filepath.Separator)) {
|
||||
return "", "", "", gerror.Wrap(contract.ErrInvalidArgument, "archive path escapes root")
|
||||
}
|
||||
return filepath.Dir(final), final, filepath.ToSlash(relativeOS), nil
|
||||
}
|
||||
|
||||
func (s *Service) validateConfiguration() error {
|
||||
if strings.TrimSpace(s.root) == "" {
|
||||
return gerror.New("archive root is empty")
|
||||
}
|
||||
if s.runDiscovery != nil && (s.runLoader == nil || s.runRegistrar == nil) {
|
||||
return gerror.New("run archive repositories are incomplete")
|
||||
}
|
||||
if s.replay.Discovery != nil && (s.replay.Leases == nil || s.replay.Progress == nil) {
|
||||
return gerror.New("replay archive repositories are incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateWork(work contract.ArchiveWork) error {
|
||||
if err := validateStorageID(work.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(work.Version) == "" {
|
||||
return gerror.Wrap(contract.ErrInvalidArgument, "archive work version is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateStorageID(id string) error {
|
||||
if err := contract.ValidateID(id); err != nil {
|
||||
return err
|
||||
}
|
||||
if id == "." || id == ".." || filepath.VolumeName(id) != "" || strings.ContainsRune(id, ':') || filepath.Base(id) != id {
|
||||
return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is not safe for archive storage")
|
||||
}
|
||||
name := strings.ToUpper(strings.TrimSuffix(id, filepath.Ext(id)))
|
||||
switch name {
|
||||
case "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9":
|
||||
return gerror.Wrap(contract.ErrInvalidArgument, "resource identifier is reserved by the filesystem")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeRunArchive(work contract.ArchiveWork, snapshot contract.RunArchive) (contract.RunArchive, error) {
|
||||
if snapshot.Version == 0 {
|
||||
snapshot.Version = contract.ArchiveSchemaVersion
|
||||
}
|
||||
if snapshot.Version != contract.ArchiveSchemaVersion {
|
||||
return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "invalid archive schema version")
|
||||
}
|
||||
if snapshot.Run.RunID != work.RunID {
|
||||
return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archive run ID does not match work")
|
||||
}
|
||||
if !snapshot.Run.Status.Terminal() {
|
||||
return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "only terminal runs can be archived")
|
||||
}
|
||||
if err := snapshot.Run.Validate(); err != nil {
|
||||
return contract.RunArchive{}, err
|
||||
}
|
||||
if len(snapshot.Rounds) > maxRounds {
|
||||
return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "too many archived rounds")
|
||||
}
|
||||
|
||||
snapshot.Run = normalizeRunMeta(snapshot.Run)
|
||||
snapshot.Rounds = append([]contract.RoundStats(nil), snapshot.Rounds...)
|
||||
if snapshot.Rounds == nil {
|
||||
snapshot.Rounds = make([]contract.RoundStats, 0)
|
||||
}
|
||||
seen := make(map[int]struct{}, len(snapshot.Rounds))
|
||||
for i := range snapshot.Rounds {
|
||||
round := &snapshot.Rounds[i]
|
||||
if round.RunID != work.RunID {
|
||||
return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "archived round belongs to another run")
|
||||
}
|
||||
if _, duplicate := seen[round.Round]; duplicate {
|
||||
return contract.RunArchive{}, gerror.Wrap(contract.ErrInvalidArgument, "duplicate archived round")
|
||||
}
|
||||
seen[round.Round] = struct{}{}
|
||||
if err := round.Validate(); err != nil {
|
||||
return contract.RunArchive{}, err
|
||||
}
|
||||
normalizedCounts, err := round.ErrorCounts.Normalized()
|
||||
if err != nil {
|
||||
return contract.RunArchive{}, err
|
||||
}
|
||||
round.ErrorCounts = normalizedCounts
|
||||
round.StartedAt = utcTimePtr(round.StartedAt)
|
||||
round.CompletedAt = utcTimePtr(round.CompletedAt)
|
||||
}
|
||||
sort.Slice(snapshot.Rounds, func(i, j int) bool { return snapshot.Rounds[i].Round < snapshot.Rounds[j].Round })
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func normalizeRunMeta(meta contract.RunMeta) contract.RunMeta {
|
||||
meta.CreatedAt = meta.CreatedAt.UTC()
|
||||
meta.ProducerCompletedAt = meta.ProducerCompletedAt.UTC()
|
||||
meta.CompletedAt = meta.CompletedAt.UTC()
|
||||
meta.StartedAt = utcTimePtr(meta.StartedAt)
|
||||
meta.ScanCompletedAt = utcTimePtr(meta.ScanCompletedAt)
|
||||
// Archive state describes the repository registration and is intentionally
|
||||
// excluded from the file whose receipt populates that state.
|
||||
meta.ArchiveStatus = ""
|
||||
meta.ArchiveVersion = 0
|
||||
meta.ArchivePath = ""
|
||||
meta.ArchiveChecksum = ""
|
||||
meta.ArchivedAt = nil
|
||||
return meta
|
||||
}
|
||||
|
||||
func normalizeAction(action contract.ReplayAction) (contract.ReplayAction, error) {
|
||||
if err := validateStorageID(action.ActionID); err != nil {
|
||||
return contract.ReplayAction{}, err
|
||||
}
|
||||
if action.Status != contract.ActionReceiptPending {
|
||||
return contract.ReplayAction{}, gerror.Wrap(contract.ErrInvalidArgument, "only receipt_pending replay actions can be archived")
|
||||
}
|
||||
action.Error = contract.BoundedErrorMessage(action.Error)
|
||||
action.CreatedAt = action.CreatedAt.UTC()
|
||||
action.UpdatedAt = action.UpdatedAt.UTC()
|
||||
action.CompletedAt = nil
|
||||
action.Receipt = nil
|
||||
if err := action.Validate(); err != nil {
|
||||
return contract.ReplayAction{}, err
|
||||
}
|
||||
return action, nil
|
||||
}
|
||||
|
||||
func utcTimePtr(value *time.Time) *time.Time {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
utc := value.UTC()
|
||||
return &utc
|
||||
}
|
||||
|
||||
type contextWriter struct {
|
||||
ctx context.Context
|
||||
writer io.Writer
|
||||
}
|
||||
|
||||
func (w contextWriter) Write(p []byte) (int, error) {
|
||||
if err := w.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return w.writer.Write(p)
|
||||
}
|
||||
@@ -0,0 +1,588 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
|
||||
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/fullsync/contract"
|
||||
)
|
||||
|
||||
func TestPublishRunIsDurableDeterministicAndSummaryOnly(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-1"))
|
||||
service := New(root, nil, nil, repository)
|
||||
work := contract.ArchiveWork{RunID: "run-1", Version: "version-1"}
|
||||
|
||||
staleDir := root
|
||||
if err := os.MkdirAll(staleDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stalePath := filepath.Join(staleDir, ".run-1-stale.tmp")
|
||||
if err := os.WriteFile(stalePath, []byte("incomplete"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("PublishRun() error = %v", err)
|
||||
}
|
||||
if receipt.Path != "run-run-1.json" {
|
||||
t.Fatalf("receipt path = %q", receipt.Path)
|
||||
}
|
||||
if err := receipt.Validate(); err != nil {
|
||||
t.Fatalf("receipt validation failed: %v", err)
|
||||
}
|
||||
|
||||
archivePath := filepath.Join(root, filepath.FromSlash(receipt.Path))
|
||||
info, err := os.Stat(archivePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
if got := info.Mode().Perm(); got != 0o644 {
|
||||
t.Fatalf("archive mode = %o, want 644", got)
|
||||
}
|
||||
}
|
||||
data, err := os.ReadFile(archivePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
if got := hex.EncodeToString(sum[:]); got != receipt.Checksum {
|
||||
t.Fatalf("checksum = %q, want %q", receipt.Checksum, got)
|
||||
}
|
||||
for _, forbidden := range [][]byte{[]byte(`"archiveStatus"`), []byte(`"archivePath"`), []byte(`"taskId"`), []byte(`"card"`), []byte(`"message"`)} {
|
||||
if bytes.Contains(data, forbidden) {
|
||||
t.Fatalf("archive contains forbidden detail %s: %s", forbidden, data)
|
||||
}
|
||||
}
|
||||
var decoded contract.RunArchive
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(decoded.Rounds) != 2 || decoded.Rounds[0].Round != 1 || decoded.Rounds[1].Round != 2 {
|
||||
t.Fatalf("rounds were not deterministically sorted: %+v", decoded.Rounds)
|
||||
}
|
||||
if decoded.Rounds[1].ErrorCounts[contract.ErrorRateLimited] != 1 {
|
||||
t.Fatalf("error categories were not normalized: %+v", decoded.Rounds[1].ErrorCounts)
|
||||
}
|
||||
|
||||
repeated, err := service.PublishRun(context.Background(), work, repository.snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("repeated PublishRun() error = %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(repeated, receipt) {
|
||||
t.Fatalf("repeated receipt changed:\nfirst: %+v\nsecond: %+v", receipt, repeated)
|
||||
}
|
||||
if _, err := os.Stat(stalePath); err != nil {
|
||||
t.Fatalf("stale temp residue should not block publication: %v", err)
|
||||
}
|
||||
repository.mu.Lock()
|
||||
defer repository.mu.Unlock()
|
||||
if len(repository.receipts) != 2 || !reflect.DeepEqual(repository.receipts[0], repository.receipts[1]) {
|
||||
t.Fatalf("registered receipts = %+v", repository.receipts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveActionPublishesAndRegistersFencedReceipt(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
action := testReplayAction("action-1")
|
||||
action.Error = strings.Repeat("x", contract.MaxErrorMessageBytes+50)
|
||||
repository := newFakeReplayRepository(action)
|
||||
service := NewServiceWithReplay(
|
||||
Config{Root: root},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
|
||||
)
|
||||
|
||||
if err := service.ArchiveAction(context.Background(), action); err != nil {
|
||||
t.Fatalf("ArchiveAction() error = %v", err)
|
||||
}
|
||||
repository.mu.Lock()
|
||||
update := repository.update
|
||||
released := repository.released
|
||||
repository.mu.Unlock()
|
||||
if update == nil || update.Status != contract.ActionCompleted || update.Receipt == nil || update.CompletedAt == nil {
|
||||
t.Fatalf("registered update = %+v", update)
|
||||
}
|
||||
if update.ExpectedRevision != action.Revision || update.Receipt.ArchivedAt != *update.CompletedAt {
|
||||
t.Fatalf("receipt registration lost fencing/time: %+v", update)
|
||||
}
|
||||
if update.Lease.Token != "lease-token-2" {
|
||||
t.Fatalf("receipt registration used the unrenewed lease: %+v", update.Lease)
|
||||
}
|
||||
if released.Token != "lease-token-2" {
|
||||
t.Fatalf("archive released the stale lease: %+v", released)
|
||||
}
|
||||
if update.Receipt.Path != "action-action-1.json" {
|
||||
t.Fatalf("action receipt path = %q", update.Receipt.Path)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(update.Receipt.Path)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range [][]byte{[]byte("idempotencyKey"), []byte("fingerprint"), []byte("leaseOwner"), []byte("leaseToken"), []byte("revision"), []byte(`"receipt"`), []byte("completedAt")} {
|
||||
if bytes.Contains(data, forbidden) {
|
||||
t.Fatalf("action archive contains internal field %q: %s", forbidden, data)
|
||||
}
|
||||
}
|
||||
var decoded contract.ReplayAction
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded.Status != contract.ActionReceiptPending || len(decoded.Error) != contract.MaxErrorMessageBytes {
|
||||
t.Fatalf("archived action was not bounded receipt_pending summary: %+v", decoded)
|
||||
}
|
||||
repeated, err := service.PublishAction(context.Background(), action)
|
||||
if err != nil {
|
||||
t.Fatalf("repeated PublishAction() error = %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(repeated, *update.Receipt) {
|
||||
t.Fatalf("repeated action receipt changed: %+v != %+v", repeated, *update.Receipt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRunCleansTempAfterRenameFailure(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-rename"))
|
||||
service := New(root, nil, nil, repository)
|
||||
renameErr := gerror.New("injected rename failure")
|
||||
service.replaceFile = func(string, string) error { return renameErr }
|
||||
|
||||
_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: "run-rename", Version: "v1"}, repository.snapshot)
|
||||
if !errors.Is(err, renameErr) {
|
||||
t.Fatalf("PublishRun() error = %v, want rename failure", err)
|
||||
}
|
||||
if matches, globErr := filepath.Glob(filepath.Join(root, ".run-rename-*.tmp")); globErr != nil || len(matches) != 0 {
|
||||
t.Fatalf("temporary files after failure = %v, error = %v", matches, globErr)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(root, "run-run-rename.json")); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("final file exists after failed rename: %v", statErr)
|
||||
}
|
||||
if len(repository.receipts) != 0 {
|
||||
t.Fatalf("registered receipt after failed rename: %+v", repository.receipts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChecksumFailureIsRecoveredByExactPathReplacement(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-checksum"))
|
||||
service := New(root, nil, nil, repository)
|
||||
service.afterReplace = func(path string) error {
|
||||
return os.WriteFile(path, []byte("corrupt"), 0o600)
|
||||
}
|
||||
work := contract.ArchiveWork{RunID: "run-checksum", Version: "v1"}
|
||||
|
||||
if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, errChecksumMismatch) {
|
||||
t.Fatalf("PublishRun() error = %v, want checksum mismatch", err)
|
||||
}
|
||||
if len(repository.receipts) != 0 {
|
||||
t.Fatalf("registered receipt after checksum failure: %+v", repository.receipts)
|
||||
}
|
||||
service.afterReplace = func(string) error { return nil }
|
||||
receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("recovery PublishRun() error = %v", err)
|
||||
}
|
||||
if receipt.Path != "run-run-checksum.json" {
|
||||
t.Fatalf("replacement used different path: %q", receipt.Path)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(receipt.Path)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) == "corrupt" {
|
||||
t.Fatal("corrupt final file was not replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrationRetryReusesIdenticalReceipt(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-register"))
|
||||
repository.registerErr = gerror.New("registration result was uncertain")
|
||||
service := New(root, nil, nil, repository)
|
||||
work := contract.ArchiveWork{RunID: "run-register", Version: "v1"}
|
||||
|
||||
if _, err := service.PublishRun(context.Background(), work, repository.snapshot); !errors.Is(err, repository.registerErr) {
|
||||
t.Fatalf("first PublishRun() error = %v", err)
|
||||
}
|
||||
repository.mu.Lock()
|
||||
firstReceipt := repository.receipts[0]
|
||||
repository.registerErr = nil
|
||||
repository.mu.Unlock()
|
||||
|
||||
receipt, err := service.PublishRun(context.Background(), work, repository.snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("retry PublishRun() error = %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(receipt, firstReceipt) {
|
||||
t.Fatalf("retry changed uncertain receipt:\nfirst: %+v\nretry: %+v", firstReceipt, receipt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRunRejectsDifferentValidContentAtSamePath(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-conflict"))
|
||||
service := New(root, nil, nil, repository)
|
||||
work := contract.ArchiveWork{RunID: "run-conflict", Version: "v1"}
|
||||
|
||||
first, err := service.PublishRun(context.Background(), work, repository.snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed := repository.snapshot
|
||||
changed.Run.TriggerSource = "different-valid-summary"
|
||||
if _, err = service.PublishRun(context.Background(), work, changed); !errors.Is(err, contract.ErrConflict) {
|
||||
t.Fatalf("PublishRun() error = %v, want conflict", err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(first.Path)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
if got := hex.EncodeToString(sum[:]); got != first.Checksum {
|
||||
t.Fatalf("conflicting publication replaced the original file: %q != %q", got, first.Checksum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerRecoversFailedRunAfterRestart(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-restart"))
|
||||
repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
|
||||
return contract.RunArchive{}, gerror.New("source temporarily unavailable")
|
||||
}
|
||||
first := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
|
||||
if err := first.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitSignal(t, repository.failed, "initial archive failure")
|
||||
if err := first.Stop(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
repository.mu.Lock()
|
||||
repository.load = nil
|
||||
repository.mu.Unlock()
|
||||
second := NewService(Config{Root: root, Interval: time.Hour, MaxAttempts: 1}, repository, repository, repository)
|
||||
if err := second.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitSignal(t, repository.succeeded, "restart archive success")
|
||||
if err := second.Stop(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "run-run-restart.json")); err != nil {
|
||||
t.Fatalf("recovered archive file missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeriodicScannerRetriesPendingRun(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-periodic"))
|
||||
var calls atomic.Int32
|
||||
repository.load = func(context.Context, contract.ArchiveWork) (contract.RunArchive, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
return contract.RunArchive{}, gerror.New("first scan fails")
|
||||
}
|
||||
return repository.snapshot, nil
|
||||
}
|
||||
service := NewService(Config{Root: root, Interval: 10 * time.Millisecond, MaxAttempts: 1}, repository, repository, repository)
|
||||
if err := service.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitSignal(t, repository.succeeded, "periodic archive success")
|
||||
if err := service.Stop(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() < 2 {
|
||||
t.Fatalf("loader calls = %d, want at least 2", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartupScannerArchivesReceiptPendingAction(t *testing.T) {
|
||||
action := testReplayAction("action-startup")
|
||||
repository := newFakeReplayRepository(action)
|
||||
service := NewServiceWithReplay(
|
||||
Config{Root: t.TempDir(), Interval: time.Hour, MaxAttempts: 1},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
|
||||
)
|
||||
if err := service.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitSignal(t, repository.completed, "startup replay archive success")
|
||||
if err := service.Stop(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellationStopsBlockedScannerAndPublication(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repository := newFakeRunRepository(testRunArchive("run-cancel"))
|
||||
started := make(chan struct{})
|
||||
repository.load = func(ctx context.Context, _ contract.ArchiveWork) (contract.RunArchive, error) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
return contract.RunArchive{}, ctx.Err()
|
||||
}
|
||||
service := NewService(Config{Root: root, Interval: time.Hour, InitialBackoff: time.Hour}, repository, repository, repository)
|
||||
if err := service.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitSignal(t, started, "blocked loader start")
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
_ = service.Stop()
|
||||
close(stopped)
|
||||
}()
|
||||
waitSignal(t, stopped, "scanner cancellation")
|
||||
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := service.PublishRun(canceled, contract.ArchiveWork{RunID: "run-cancel", Version: "v1"}, repository.snapshot)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled PublishRun() error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "run-run-cancel.json")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("canceled publication wrote final file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellationAfterRenameStillVerifiesPublicationAndReleasesLease(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
action := testReplayAction("action-cancel-after-rename")
|
||||
repository := newFakeReplayRepository(action)
|
||||
service := NewServiceWithReplay(
|
||||
Config{Root: t.TempDir()},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
ReplayRepositories{Discovery: repository, Leases: repository, Progress: repository},
|
||||
)
|
||||
service.afterReplace = func(string) error {
|
||||
cancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := service.ArchiveAction(ctx, action); err != nil {
|
||||
t.Fatalf("ArchiveAction() error = %v", err)
|
||||
}
|
||||
repository.mu.Lock()
|
||||
releaseCtxErr := repository.releaseCtxErr
|
||||
repository.mu.Unlock()
|
||||
if releaseCtxErr != nil {
|
||||
t.Fatalf("lease release inherited canceled context: %v", releaseCtxErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveRejectsTraversalAndUnsafeFilesystemIDs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
service := New(root, nil, nil, newFakeRunRepository(contract.RunArchive{}))
|
||||
for _, id := range []string{"../escape", `..\escape`, "/absolute", `C:\escape`, "drive:name", ".", "..", "{hash}", "CON"} {
|
||||
t.Run(strings.ReplaceAll(id, "/", "_"), func(t *testing.T) {
|
||||
snapshot := testRunArchive(id)
|
||||
_, err := service.PublishRun(context.Background(), contract.ArchiveWork{RunID: id, Version: "v1"}, snapshot)
|
||||
if !errors.Is(err, contract.ErrInvalidArgument) {
|
||||
t.Fatalf("PublishRun(%q) error = %v, want invalid argument", id, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("unsafe IDs created archive entries: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func testRunArchive(runID string) contract.RunArchive {
|
||||
created := time.Date(2026, time.August, 31, 23, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||
completed := time.Date(2026, time.September, 20, 16, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||
archived := completed.Add(time.Minute)
|
||||
return contract.RunArchive{
|
||||
Version: contract.ArchiveSchemaVersion,
|
||||
Run: contract.RunMeta{
|
||||
RunID: runID, Status: contract.RunCompletedWithFailures, ProducerDone: true,
|
||||
CurrentRound: 2, MaxRound: 2, EnqueuedCount: 3, SuccessCount: 2, FailedCount: 1, PendingFailureCount: 1,
|
||||
CreatedAt: created, ProducerCompletedAt: completed, CompletedAt: completed,
|
||||
ArchiveStatus: contract.ArchiveFailed, ArchiveVersion: 99, ArchivePath: "old.json",
|
||||
ArchiveChecksum: strings.Repeat("a", 64), ArchivedAt: &archived,
|
||||
},
|
||||
Rounds: []contract.RoundStats{
|
||||
{RunID: runID, Round: 2, EnqueuedCount: 1, FailedCount: 1, ErrorCounts: contract.ErrorCounts{"mobile_busy": 1}},
|
||||
{RunID: runID, Round: 1, EnqueuedCount: 2, SuccessCount: 2, ErrorCounts: nil},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func testReplayAction(actionID string) contract.ReplayAction {
|
||||
created := time.Date(2026, time.September, 20, 12, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||
return contract.ReplayAction{
|
||||
ActionID: actionID, SourceRunID: "source-1", TargetRunID: "target-1", RequestedBy: "operator",
|
||||
IdempotencyKey: "secret-key", Fingerprint: "secret-fingerprint", Status: contract.ActionReceiptPending,
|
||||
ReplayCycle: 1, Progress: contract.ReplayProgress{DiscoveredCount: 2, EnqueuedCount: 2, CleanedCount: 2},
|
||||
LeaseOwner: "engine", LeaseToken: "secret-token", LeaseUntil: created.Add(time.Hour), Revision: 7,
|
||||
CreatedAt: created, UpdatedAt: created.Add(time.Minute),
|
||||
}
|
||||
}
|
||||
|
||||
type fakeRunRepository struct {
|
||||
mu sync.Mutex
|
||||
work contract.ArchiveWork
|
||||
snapshot contract.RunArchive
|
||||
load func(context.Context, contract.ArchiveWork) (contract.RunArchive, error)
|
||||
registered bool
|
||||
receipts []contract.ArchiveReceipt
|
||||
failures []string
|
||||
registerErr error
|
||||
succeeded chan struct{}
|
||||
failed chan struct{}
|
||||
}
|
||||
|
||||
func newFakeRunRepository(snapshot contract.RunArchive) *fakeRunRepository {
|
||||
return &fakeRunRepository{
|
||||
work: contract.ArchiveWork{RunID: snapshot.Run.RunID, Version: "repository-version"},
|
||||
snapshot: snapshot,
|
||||
succeeded: make(chan struct{}, 8),
|
||||
failed: make(chan struct{}, 8),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *fakeRunRepository) ListArchiveWork(ctx context.Context, _ contract.PageRequest) (contract.ArchiveWorkPage, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contract.ArchiveWorkPage{}, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.registered {
|
||||
return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{}}, nil
|
||||
}
|
||||
return contract.ArchiveWorkPage{Items: []contract.ArchiveWork{r.work}}, nil
|
||||
}
|
||||
|
||||
func (r *fakeRunRepository) LoadArchiveWork(ctx context.Context, work contract.ArchiveWork) (contract.RunArchive, error) {
|
||||
r.mu.Lock()
|
||||
load := r.load
|
||||
snapshot := r.snapshot
|
||||
r.mu.Unlock()
|
||||
if load != nil {
|
||||
return load(ctx, work)
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func (r *fakeRunRepository) RegisterArchive(_ context.Context, _ contract.ArchiveWork, receipt contract.ArchiveReceipt) error {
|
||||
r.mu.Lock()
|
||||
r.receipts = append(r.receipts, receipt)
|
||||
err := r.registerErr
|
||||
if err == nil {
|
||||
r.registered = true
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
notify(r.succeeded)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeRunRepository) RegisterArchiveFailure(_ context.Context, _ contract.ArchiveWork, message string) error {
|
||||
r.mu.Lock()
|
||||
r.failures = append(r.failures, message)
|
||||
r.mu.Unlock()
|
||||
notify(r.failed)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeReplayRepository struct {
|
||||
mu sync.Mutex
|
||||
action contract.ReplayAction
|
||||
update *contract.ReplayActionUpdate
|
||||
released contract.ReplayLease
|
||||
releaseCtxErr error
|
||||
completed chan struct{}
|
||||
}
|
||||
|
||||
func newFakeReplayRepository(action contract.ReplayAction) *fakeReplayRepository {
|
||||
return &fakeReplayRepository{action: action, completed: make(chan struct{}, 4)}
|
||||
}
|
||||
|
||||
func (r *fakeReplayRepository) ListReplayWork(ctx context.Context, _ contract.PageRequest) (contract.ReplayActionPage, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contract.ReplayActionPage{}, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.update != nil {
|
||||
return contract.ReplayActionPage{Items: []contract.ReplayAction{}}, nil
|
||||
}
|
||||
return contract.ReplayActionPage{Items: []contract.ReplayAction{r.action}}, nil
|
||||
}
|
||||
|
||||
func (r *fakeReplayRepository) AcquireReplayLease(_ context.Context, actionID, owner string, ttl time.Duration) (contract.ReplayLease, error) {
|
||||
return contract.ReplayLease{ActionID: actionID, Owner: owner, Token: "lease-token", Until: time.Now().Add(ttl)}, nil
|
||||
}
|
||||
|
||||
func (r *fakeReplayRepository) RenewReplayLease(_ context.Context, lease contract.ReplayLease, ttl time.Duration) (contract.ReplayLease, error) {
|
||||
lease.Token = "lease-token-2"
|
||||
lease.Until = time.Now().Add(ttl)
|
||||
return lease, nil
|
||||
}
|
||||
|
||||
func (r *fakeReplayRepository) ReleaseReplayLease(ctx context.Context, lease contract.ReplayLease) error {
|
||||
r.mu.Lock()
|
||||
r.released = lease
|
||||
r.releaseCtxErr = ctx.Err()
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeReplayRepository) UpdateReplayAction(_ context.Context, update contract.ReplayActionUpdate) (contract.ReplayAction, error) {
|
||||
r.mu.Lock()
|
||||
copyOfUpdate := update
|
||||
r.update = ©OfUpdate
|
||||
action := r.action
|
||||
action.Status = update.Status
|
||||
action.Receipt = update.Receipt
|
||||
action.CompletedAt = update.CompletedAt
|
||||
r.mu.Unlock()
|
||||
notify(r.completed)
|
||||
return action, nil
|
||||
}
|
||||
|
||||
func notify(ch chan struct{}) {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func waitSignal(t *testing.T, ch <-chan struct{}, operation string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("timed out waiting for %s", operation)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//go:build !windows
|
||||
|
||||
package archive
|
||||
|
||||
import "os"
|
||||
|
||||
func atomicReplace(from, to string) error {
|
||||
return os.Rename(from, to)
|
||||
}
|
||||
|
||||
func syncDirectory(path string) error {
|
||||
dir, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := dir.Sync(); err != nil {
|
||||
_ = dir.Close()
|
||||
return err
|
||||
}
|
||||
return dir.Close()
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//go:build windows
|
||||
|
||||
package archive
|
||||
|
||||
import "golang.org/x/sys/windows"
|
||||
|
||||
func atomicReplace(from, to string) error {
|
||||
fromUTF16, err := windows.UTF16PtrFromString(from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
toUTF16, err := windows.UTF16PtrFromString(to)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return windows.MoveFileEx(fromUTF16, toUTF16, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH)
|
||||
}
|
||||
|
||||
// Windows has no portable directory fsync operation. MoveFileEx with
|
||||
// MOVEFILE_WRITE_THROUGH provides the strongest supported replacement flush.
|
||||
func syncDirectory(string) error { return nil }
|
||||
Reference in New Issue
Block a user