chore: 并入 iotlcm-worker-svr 历史到 worker-svr/(原提交 96f0f4e)

This commit is contained in:
MiFaZhan
2026-11-02 13:28:03 +08:00
422 changed files with 80993 additions and 0 deletions
@@ -0,0 +1,287 @@
// Package supply holds bounded, per-channel telemetry independent of card volume.
package supply
import (
"sort"
"sync"
"time"
)
const SchemaVersion = 1
type Wait struct {
Count int64 `json:"count"`
TotalMS int64 `json:"totalMs"`
LongestMS int64 `json:"longestMs"`
}
type Channel struct {
Key string `json:"chanKey"`
FirstQueryMS *int64 `json:"firstQueryMs"`
FirstEnqueueMS *int64 `json:"firstEnqueueMs"`
Startup Wait `json:"startupWait"`
Confirmed Wait `json:"confirmedWait"`
Unconfirmed Wait `json:"unconfirmedIdle"`
Consumer Wait `json:"consumerStartupWait"`
UnknownMS int64 `json:"unknownMs"`
EmptyClaims int64 `json:"emptyClaims"`
ValidSamples int64 `json:"validSamples"`
MissingSamples int64 `json:"missingSamples"`
Stages map[string]int64 `json:"producerStageMs"`
Completeness string `json:"completeness"`
Termination string `json:"termination,omitempty"`
CoverageStart time.Time `json:"coverageStart"`
CoverageEnd time.Time `json:"coverageEnd"`
}
type Summary struct {
Targets int `json:"targetChannels"`
Valid int `json:"validChannels"`
Confirmed int `json:"confirmedChannels"`
Unknown int `json:"unknownChannels"`
UnknownMS int64 `json:"unknownChannelMs"`
UnconfirmedMS int64 `json:"unconfirmedChannelMs"`
ConsumerMS int64 `json:"consumerStartupChannelMs"`
TotalMS int64 `json:"confirmedChannelMs"`
LongestMS int64 `json:"longestMs"`
Worst string `json:"worstChannel,omitempty"`
}
type Snapshot struct {
SchemaVersion int `json:"schemaVersion"`
Attempt string `json:"attempt"`
ResolutionMS int `json:"resolutionMs"`
Partial bool `json:"partial"`
Truncated bool `json:"truncated"`
Summary Summary `json:"summary"`
Channels []Channel `json:"channels"`
}
type View struct {
Version uint64
Pending int
Processing int
Stage string
Ended bool
}
type Sample struct {
Ready int64
Leases int64
Future int64
Eligible bool
Known bool
}
type state struct {
Channel
View
last time.Time
kind string
continuous int64
open bool
}
type Observer struct {
mu sync.Mutex
started time.Time
attempt string
partial bool
retained Summary
truncated bool
channels map[string]*state
}
func New(attempt string, started time.Time, keys []string) *Observer {
o := &Observer{attempt: attempt, started: started, channels: make(map[string]*state, len(keys))}
for _, k := range keys {
o.channels[k] = &state{Channel: Channel{Key: k, Stages: map[string]int64{}, CoverageStart: started}, View: View{Stage: "initializing"}}
}
return o
}
func (o *Observer) Event(key, stage string, pending int, added bool, ended bool, now time.Time) {
o.mu.Lock()
defer o.mu.Unlock()
s := o.channels[key]
if s == nil {
return
}
s.Version++
s.Stage = stage
s.Pending = pending
if stage == "sql_query" && s.FirstQueryMS == nil {
v := now.Sub(o.started).Milliseconds()
s.FirstQueryMS = &v
}
if added && s.FirstEnqueueMS == nil {
v := now.Sub(o.started).Milliseconds()
s.FirstEnqueueMS = &v
s.close()
}
if ended {
s.Ended = true
s.Termination = stage
s.CoverageEnd = now
s.close()
}
}
func (o *Observer) Empty(key string) {
o.mu.Lock()
defer o.mu.Unlock()
if s := o.channels[key]; s != nil && !s.Ended {
s.EmptyClaims++
}
}
func (o *Observer) Processing(key string, delta int) {
o.mu.Lock()
defer o.mu.Unlock()
if s := o.channels[key]; s != nil {
s.Version++
s.Processing = max(0, s.Processing+delta)
if delta > 0 {
s.close()
}
}
}
func (o *Observer) Views() map[string]View {
o.mu.Lock()
defer o.mu.Unlock()
out := make(map[string]View, len(o.channels))
for k, s := range o.channels {
out[k] = s.View
}
return out
}
func (s *state) close() { s.kind = ""; s.continuous = 0; s.open = false; s.last = time.Time{} }
func (o *Observer) Partial() { o.mu.Lock(); o.partial = true; o.mu.Unlock() }
func (o *Observer) Apply(key string, before View, sample Sample, now time.Time) {
o.mu.Lock()
defer o.mu.Unlock()
s := o.channels[key]
if s == nil || s.Ended {
return
}
kind := "unknown"
if sample.Known && before.Version == s.Version {
s.ValidSamples++
switch {
case !sample.Eligible:
s.close()
s.CoverageEnd = now
return
case sample.Ready > 0:
if s.Processing == 0 && sample.Leases == 0 {
kind = "consumer"
} else {
kind = "stock"
}
case s.Processing > 0 || sample.Leases > 0:
kind = "processing"
case sample.Future > 0:
kind = "scheduled"
case s.Pending > 0:
if s.FirstEnqueueMS == nil {
kind = "startup"
} else {
kind = "confirmed"
}
default:
kind = "unconfirmed"
}
} else {
s.MissingSamples++
o.partial = true
}
elapsed := now.Sub(s.last)
if !s.last.IsZero() && elapsed > 2*time.Second {
s.MissingSamples += int64(elapsed/time.Second) - 1
o.partial = true
s.close()
}
if s.kind == kind && !s.last.IsZero() {
ms := elapsed.Milliseconds()
if ms < 0 {
ms = 0
}
s.continuous += ms
var w *Wait
switch kind {
case "startup":
w = &s.Startup
case "confirmed":
w = &s.Confirmed
case "unconfirmed":
w = &s.Unconfirmed
case "consumer":
w = &s.Consumer
case "unknown":
s.UnknownMS += ms
}
if w != nil {
if !s.open {
w.Count++
s.open = true
}
w.TotalMS += ms
if s.continuous > w.LongestMS {
w.LongestMS = s.continuous
}
if kind == "startup" || kind == "confirmed" {
s.Stages[s.Stage] += ms
}
}
} else {
s.continuous = 0
s.open = false
}
s.kind = kind
s.last = now
s.CoverageEnd = now
}
func (o *Observer) Snapshot() Snapshot {
o.mu.Lock()
defer o.mu.Unlock()
out := Snapshot{SchemaVersion: 1, Attempt: o.attempt, ResolutionMS: 1000, Partial: o.partial, Truncated: o.truncated, Channels: make([]Channel, 0, len(o.channels))}
out.Summary.Targets = len(o.channels)
for _, s := range o.channels {
c := s.Channel
c.Completeness = "observing"
if s.Ended {
c.Completeness = "complete"
}
if c.MissingSamples > 0 || c.UnknownMS > 0 || c.ValidSamples == 0 || c.Termination == "cancelled" {
c.Completeness = "partial"
}
c.Stages = make(map[string]int64, len(s.Stages))
for k, v := range s.Stages {
c.Stages[k] = v
}
out.Channels = append(out.Channels, c)
if c.ValidSamples == 0 {
out.Partial = true
}
out.Summary.UnknownMS += c.UnknownMS
out.Summary.UnconfirmedMS += c.Unconfirmed.TotalMS
out.Summary.ConsumerMS += c.Consumer.TotalMS
if c.ValidSamples > 0 {
out.Summary.Valid++
}
total := c.Startup.TotalMS + c.Confirmed.TotalMS
longest := max(c.Startup.LongestMS, c.Confirmed.LongestMS)
if total > 0 {
out.Summary.Confirmed++
}
if c.MissingSamples > 0 || c.UnknownMS > 0 || c.ValidSamples == 0 {
out.Summary.Unknown++
}
out.Summary.TotalMS += total
if longest > out.Summary.LongestMS {
out.Summary.LongestMS = longest
out.Summary.Worst = c.Key
}
}
out.Summary.Confirmed = min(out.Summary.Targets, out.Summary.Confirmed+o.retained.Confirmed)
out.Summary.TotalMS += o.retained.TotalMS
out.Summary.UnknownMS += o.retained.UnknownMS
out.Summary.UnconfirmedMS += o.retained.UnconfirmedMS
out.Summary.ConsumerMS += o.retained.ConsumerMS
if o.retained.LongestMS > out.Summary.LongestMS {
out.Summary.LongestMS = o.retained.LongestMS
out.Summary.Worst = o.retained.Worst
}
sort.Slice(out.Channels, func(i, j int) bool { return out.Channels[i].Key < out.Channels[j].Key })
return out
}
@@ -0,0 +1,134 @@
package supply
import (
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
)
func TestSnapshotLimitPreservesSummaryAndRanking(t *testing.T) {
keys := make([]string, 700)
for i := range keys {
keys[i] = fmt.Sprintf("%04d-%s", i, strings.Repeat("x", 200))
}
start := time.Unix(100, 0)
o := New("attempt", start, keys)
for i, k := range keys {
o.Event(k, "global_capacity", 1, false, false, start)
o.Apply(k, o.Views()[k], Sample{Known: true, Eligible: true}, start.Add(time.Second))
o.Apply(k, o.Views()[k], Sample{Known: true, Eligible: true}, start.Add(time.Duration(1000+i)*time.Millisecond+time.Second))
}
body, err := o.Encode()
if err != nil {
t.Fatal(err)
}
var s Snapshot
if err = json.Unmarshal(body, &s); err != nil {
t.Fatal(err)
}
if len(body) > 256*1024 || !s.Truncated || len(s.Channels) != 100 || s.Summary.Targets != 700 || s.Channels[0].Key != keys[699] {
t.Fatalf("invalid bounded snapshot bytes=%d %+v", len(body), s.Summary)
}
restored := New("new", start.Add(time.Hour), keys)
restored.Restore(s)
r := restored.Snapshot()
if !r.Partial || !r.Truncated || r.Summary.TotalMS != s.Summary.TotalMS {
t.Fatalf("lost historical totals %+v %+v", s.Summary, r.Summary)
}
}
func TestConfirmedIntervalsAndStages(t *testing.T) {
start := time.Unix(100, 0)
o := New("attempt", start, []string{"a", "empty"})
o.Event("a", "global_capacity", 2, false, false, start)
tick := func(n int) {
v := o.Views()["a"]
o.Apply("a", v, Sample{Known: true, Eligible: true}, start.Add(time.Duration(n)*time.Second))
}
tick(1)
tick(2)
o.Event("a", "redis_write", 2, false, false, start.Add(2*time.Second))
tick(3)
s := o.Snapshot()
c := s.Channels[0]
if c.Startup.Count != 1 || c.Startup.TotalMS != 2000 || c.Stages["global_capacity"] != 1000 || c.Stages["redis_write"] != 1000 {
t.Fatalf("unexpected %+v", c)
}
o.Processing("a", 1)
tick(4)
tick(5)
if o.Snapshot().Channels[0].Startup.TotalMS != 2000 {
t.Fatal("processing counted as idle")
}
}
func TestUncertainAndGap(t *testing.T) {
start := time.Unix(100, 0)
o := New("a", start, []string{"a"})
v := o.Views()["a"]
o.Event("a", "sql_query", 0, false, false, start)
o.Apply("a", v, Sample{Known: true, Eligible: true}, start.Add(time.Second))
o.Apply("a", o.Views()["a"], Sample{Known: false}, start.Add(2*time.Second))
o.Apply("a", o.Views()["a"], Sample{Known: true, Eligible: true}, start.Add(6*time.Second))
s := o.Snapshot()
if !s.Partial || s.Channels[0].UnknownMS != 1000 || s.Channels[0].MissingSamples != 5 || s.Summary.Confirmed != 0 {
t.Fatalf("unexpected %+v", s)
}
}
func TestSupplyExclusions(t *testing.T) {
for _, tc := range []struct {
name string
sample Sample
processing bool
}{{"stock", Sample{Known: true, Eligible: true, Ready: 1}, false}, {"lease", Sample{Known: true, Eligible: true, Leases: 1}, false}, {"ended_round_or_blocked", Sample{Known: true, Eligible: false}, false}, {"consumer", Sample{Known: true, Eligible: true}, true}, {"unknown", Sample{Eligible: true}, false}} {
t.Run(tc.name, func(t *testing.T) {
start := time.Unix(100, 0)
o := New("a", start, []string{"a"})
o.Event("a", "global_capacity", 100, false, false, start)
if tc.processing {
o.Processing("a", 1)
}
for i := 1; i <= 3; i++ {
o.Apply("a", o.Views()["a"], tc.sample, start.Add(time.Duration(i)*time.Second))
}
if o.Snapshot().Summary.Confirmed != 0 {
t.Fatal("excluded sample counted as supply idle")
}
})
}
}
func TestConcurrentEvents(t *testing.T) {
start := time.Unix(100, 0)
o := New("a", start, []string{"a"})
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for n := 0; n < 100; n++ {
o.Event("a", "sql_query", 0, false, false, start)
o.Processing("a", 1)
o.Empty("a")
o.Apply("a", o.Views()["a"], Sample{Known: true, Eligible: true}, start)
o.Snapshot()
o.Processing("a", -1)
}
}()
}
wg.Wait()
}
func TestEmptyNeverConfirms(t *testing.T) {
start := time.Unix(100, 0)
o := New("a", start, []string{"a"})
for i := 1; i <= 3; i++ {
o.Apply("a", o.Views()["a"], Sample{Known: true, Eligible: true}, start.Add(time.Duration(i)*time.Second))
}
o.Event("a", "empty", 0, false, true, start.Add(3*time.Second))
s := o.Snapshot()
if s.Summary.Confirmed != 0 || s.Channels[0].Unconfirmed.TotalMS != 2000 {
t.Fatalf("unexpected %+v", s)
}
}
@@ -0,0 +1,90 @@
package supply
import (
"encoding/json"
"fmt"
"sort"
"sync"
"sync/atomic"
)
// Tracker routes optional consumer events to the local producer attempt.
type Tracker struct {
mu sync.RWMutex
runs map[string]*Observer
LocalConsumer bool
ConsumerActive atomic.Bool
}
func NewTracker(localConsumer bool) *Tracker {
return &Tracker{runs: map[string]*Observer{}, LocalConsumer: localConsumer}
}
func (t *Tracker) Put(run string, o *Observer) { t.mu.Lock(); t.runs[run] = o; t.mu.Unlock() }
func (t *Tracker) Remove(run string) { t.mu.Lock(); delete(t.runs, run); t.mu.Unlock() }
func (t *Tracker) Empty(run, key string) {
t.mu.RLock()
defer t.mu.RUnlock()
if o := t.runs[run]; o != nil {
o.Empty(key)
}
}
func (t *Tracker) Processing(run, key string, delta int) {
t.mu.RLock()
defer t.mu.RUnlock()
if o := t.runs[run]; o != nil {
o.Processing(key, delta)
}
}
func (o *Observer) Restore(previous Snapshot) {
o.mu.Lock()
defer o.mu.Unlock()
o.partial = true
o.retained = previous.Summary
o.truncated = previous.Truncated
for _, c := range previous.Channels {
if s := o.channels[c.Key]; s != nil {
if c.Startup.TotalMS+c.Confirmed.TotalMS > 0 {
o.retained.Confirmed--
}
o.retained.TotalMS -= c.Startup.TotalMS + c.Confirmed.TotalMS
o.retained.UnknownMS -= c.UnknownMS
o.retained.UnconfirmedMS -= c.Unconfirmed.TotalMS
o.retained.ConsumerMS -= c.Consumer.TotalMS
s.Channel = c
if s.Stages == nil {
s.Stages = map[string]int64{}
}
s.Termination = ""
s.MissingSamples++
s.close()
}
}
}
func (o *Observer) Encode() ([]byte, error) {
snapshot := o.Snapshot()
encoded, err := json.Marshal(snapshot)
if err != nil || len(encoded) <= 256*1024 {
return encoded, err
}
snapshot.Truncated = true
snapshot.Partial = true
sort.Slice(snapshot.Channels, func(i, j int) bool {
a, b := snapshot.Channels[i], snapshot.Channels[j]
av, bv := max(a.Startup.LongestMS, a.Confirmed.LongestMS), max(b.Startup.LongestMS, b.Confirmed.LongestMS)
if av == bv {
return a.Key < b.Key
}
return av > bv
})
snapshot.Channels = snapshot.Channels[:min(100, len(snapshot.Channels))]
for {
encoded, err = json.Marshal(snapshot)
if err != nil || len(encoded) <= 256*1024 {
return encoded, err
}
if len(snapshot.Channels) == 0 {
return nil, fmt.Errorf("supply summary exceeds snapshot limit")
}
snapshot.Channels = snapshot.Channels[:len(snapshot.Channels)-1]
}
}
@@ -0,0 +1,27 @@
package supply
// Valid checks only telemetry. Callers must degrade invalid observations without
// rejecting the business run or its archive.
func (s Snapshot) Valid() bool {
if s.SchemaVersion != 1 || s.ResolutionMS != 1000 || s.Summary.Targets < 0 || s.Summary.Valid < 0 || s.Summary.Valid > s.Summary.Targets || s.Summary.Confirmed < 0 || s.Summary.Confirmed > s.Summary.Targets || s.Summary.Unknown < 0 || s.Summary.Unknown > s.Summary.Targets || s.Summary.TotalMS < 0 || s.Summary.LongestMS < 0 || s.Summary.UnknownMS < 0 || s.Summary.UnconfirmedMS < 0 || s.Summary.ConsumerMS < 0 {
return false
}
seen := make(map[string]bool, len(s.Channels))
for _, c := range s.Channels {
if c.Key == "" || seen[c.Key] || c.ValidSamples < 0 || c.MissingSamples < 0 || c.UnknownMS < 0 {
return false
}
seen[c.Key] = true
for _, w := range []Wait{c.Startup, c.Confirmed, c.Unconfirmed, c.Consumer} {
if w.Count < 0 || w.TotalMS < 0 || w.LongestMS < 0 || w.LongestMS > w.TotalMS {
return false
}
}
for _, v := range c.Stages {
if v < 0 {
return false
}
}
}
return len(s.Channels) <= s.Summary.Targets
}