Files
iotlcm/bot/internal/notify/health_test.go
T

199 lines
6.8 KiB
Go

package notify
import (
"context"
"errors"
"log/slog"
"os"
"path/filepath"
"testing"
"time"
"git.5iot.com/iotlcm-public/iotlcm-bot/internal/state"
)
func writeArchive(t *testing.T, dir, name, body string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
func discardLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError + 1}))
}
// countingLogger 统计 ERROR 级日志条数,用于验证去重告警。
func countingLogger(count *int) *slog.Logger {
return slog.New(slog.NewTextHandler(countingWriter{count}, &slog.HandlerOptions{Level: slog.LevelError}))
}
type countingWriter struct{ count *int }
func (w countingWriter) Write(p []byte) (int, error) {
*w.count++
return len(p), nil
}
const validArchiveBody = `{"version":1,"run":{"runId":"z-good","operation":"op","status":"completed","successCount":1,"failedCount":0,"createdAt":"2026-10-12T00:00:00Z","completedAt":"2026-10-12T00:01:00Z"},"rounds":[{"round":1,"enqueuedCount":1,"activeCount":0,"nextRoundCount":0,"successCount":1,"failedCount":0,"startedAt":"2026-10-12T00:00:00Z","completedAt":"2026-10-12T00:01:00Z"}]}`
// 一个坏归档不能挡住后面的归档,但也不能每轮扫描都刷一条日志。
func TestScanSkipsBrokenArchiveAndLogsOnlyOnce(t *testing.T) {
root := t.TempDir()
// 字典序在前:旧格式(rounds 为空)会校验失败。
writeArchive(t, root, "run-broken.json", `{"version":1,"run":{"runId":"broken","operation":"op","status":"completed"},"rounds":[]}`)
writeArchive(t, root, "run-z-good.json", validArchiveBody)
fake := &fakeFeishu{}
var logged int
svc := Service{archiveDir: root, environment: "preview", store: state.New(t.TempDir()), client: fake, logger: countingLogger(&logged), warned: map[string]struct{}{}}
for i := 0; i < 3; i++ {
if err := svc.Scan(context.Background()); err != nil {
t.Fatalf("Scan() returned error for a broken archive: %v", err)
}
}
if fake.sent != 1 {
t.Fatalf("expected the valid archive to be sent once, sent=%d", fake.sent)
}
if logged != 1 {
t.Fatalf("broken archive should be reported once across 3 scans, logged=%d", logged)
}
}
// 文件被重写(内容变了)后应该重新告警。
func TestScanWarnsAgainAfterArchiveIsRewritten(t *testing.T) {
root := t.TempDir()
writeArchive(t, root, "run-broken.json", `{"version":1,"run":{"runId":"broken","operation":"op","status":"completed"},"rounds":[]}`)
var logged int
svc := Service{archiveDir: root, environment: "preview", store: state.New(t.TempDir()), client: &fakeFeishu{}, logger: countingLogger(&logged), warned: map[string]struct{}{}}
if err := svc.Scan(context.Background()); err != nil {
t.Fatal(err)
}
// 重写为另一个仍然无效的内容,mtime/大小变化
time.Sleep(10 * time.Millisecond)
writeArchive(t, root, "run-broken.json", `{"version":1,"run":{"runId":"broken","operation":"op","status":"completed","queryMonth":"202610"},"rounds":[]}`)
if err := svc.Scan(context.Background()); err != nil {
t.Fatal(err)
}
if logged != 2 {
t.Fatalf("rewritten broken archive should be reported again, logged=%d", logged)
}
}
// 发送失败(飞书临时故障)必须每轮都报,不能被去重掉:
// 否则一次网络抖动就会让通知静默丢失。
func TestScanSendFailureIsNotDeduped(t *testing.T) {
root := t.TempDir()
writeArchive(t, root, "run-z-good.json", validArchiveBody)
fail := &fakeFeishu{err: errors.New("feishu timeout")}
var logged int
svc := Service{archiveDir: root, environment: "preview", store: state.New(t.TempDir()), client: fail, logger: countingLogger(&logged), warned: map[string]struct{}{}}
for i := 0; i < 3; i++ {
if err := svc.Scan(context.Background()); err != nil {
t.Fatal(err)
}
}
if logged != 3 {
t.Fatalf("transient send failures must be reported every scan, logged=%d want=3", logged)
}
}
func TestHealthReportsEmptyDirectory(t *testing.T) {
svc := Service{archiveDir: t.TempDir()}
h := svc.Health()
if !h.Mounted || h.Degraded {
t.Fatalf("empty but mounted directory should not be degraded: %+v", h)
}
if h.Entries != 0 || h.Reason == "" {
t.Fatalf("unexpected health: %+v", h)
}
}
func TestHealthReportsMissingDirectory(t *testing.T) {
svc := Service{archiveDir: filepath.Join(t.TempDir(), "gone")}
h := svc.Health()
if h.Mounted || !h.Degraded {
t.Fatalf("missing directory must be degraded: %+v", h)
}
}
// 挂载失效时必须退出进程,交给 restart policy 重新拉起以重新解析 bind mount。
// 这是「不再需要人工同批重建 bot」的关键:重启后挂载指向宿主当前目录。
func TestWatchExitsWhenArchiveMountIsDegraded(t *testing.T) {
shortenExitDelay(t)
if !ExitOnDegraded {
t.Skip("ExitOnDegraded disabled")
}
stateDir := t.TempDir()
svc := Service{
archiveDir: filepath.Join(t.TempDir(), "gone"),
environment: "test",
stateDir: stateDir,
pollInterval: 10 * time.Millisecond,
store: state.New(stateDir),
logger: discardLogger(),
}
done := make(chan error, 1)
go func() { done <- svc.Watch(context.Background()) }()
select {
case err := <-done:
if err == nil {
t.Fatal("Watch must return an error so the process exits non-zero and restarts")
}
case <-time.After(3 * time.Second):
t.Fatal("Watch did not exit after detecting a degraded archive mount")
}
}
// 目录长期为空时才缩短轮询;有内容或空窗很短时保持配置周期。
func TestIntervalShortensOnlyAfterLongEmptyPeriod(t *testing.T) {
root := t.TempDir()
svc := Service{archiveDir: root, pollInterval: time.Minute}
if got := svc.Interval(time.Time{}); got != time.Minute {
t.Fatalf("fresh empty dir should keep poll interval, got %s", got)
}
if got := svc.Interval(time.Now().Add(-StaleAfter - time.Second)); got != 10*time.Second {
t.Fatalf("long-empty dir should poll every 10s, got %s", got)
}
writeArchive(t, root, "run-anything.json", "{}")
if got := svc.Interval(time.Now().Add(-time.Hour)); got != time.Minute {
t.Fatalf("non-empty dir should keep poll interval, got %s", got)
}
}
type fakeFeishu struct {
sent int
err error
textCount int
texts []string
}
func (f *fakeFeishu) Send(ctx context.Context, card []byte) error {
if f.err != nil {
return f.err
}
f.sent++
return nil
}
func (f *fakeFeishu) SendText(ctx context.Context, text string) error {
f.textCount++
f.texts = append(f.texts, text)
return nil
}
// newTestStore 构造一个 state.Store 并返回其 Dir,供 alertstate 测试使用。
func newTestStore(dir string) state.Store { return state.New(dir) }
// shortenExitDelay 把退出等待改短,避免测试白等 30 秒(仅测试内使用)。
func shortenExitDelay(t *testing.T) {
t.Helper()
original := DegradedExitDelay
DegradedExitDelay = 10 * time.Millisecond
t.Cleanup(func() { DegradedExitDelay = original })
}