chore: 并入 iotlcm-admin-svr 历史到 admin-svr/(原提交 49818c6)

This commit is contained in:
MiFaZhan
2026-11-02 13:27:59 +08:00
1250 changed files with 154111 additions and 0 deletions
@@ -0,0 +1,305 @@
package fullsyncretry
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/gogf/gf/v2/errors/gcode"
"github.com/gogf/gf/v2/errors/gerror"
)
const maxResponse = 512 << 10
var ErrConfig = errors.New("full-sync worker is not configured")
type Client struct {
BaseURL string
HTTP *http.Client
}
type Receipt struct {
ReplayRunID string `json:"replayRunId"`
Status string `json:"status"`
}
type Status struct {
ReplayRunID string `json:"replayRunId"`
SourceRunID string `json:"sourceRunId"`
Status string `json:"status"`
Remaining int64 `json:"remaining"`
Enqueued int64 `json:"enqueued"`
RunStatus string `json:"runStatus,omitempty"`
Error string `json:"error,omitempty"`
}
type RunMeta struct {
RunID string `json:"runId"`
Operation string `json:"operation"`
QueryMonth string `json:"queryMonth"`
Status string `json:"status"`
ProducerDone bool `json:"producerDone"`
CurrentRound int `json:"currentRound"`
ActiveCount int64 `json:"activeCount"`
InflightCount int64 `json:"inflightCount"`
NextRoundCount int64 `json:"nextRoundCount"`
SuccessCount int64 `json:"successCount"`
FailedCount int64 `json:"failedCount"`
CreatedAt time.Time `json:"createdAt"`
ProducerCompletedAt time.Time `json:"producerCompletedAt"`
CompletedAt time.Time `json:"completedAt"`
SyncType string `json:"syncType,omitempty"`
SourceRunID string `json:"sourceRunId,omitempty"`
ReplayCycle int `json:"replayCycle,omitempty"`
TriggerSource string `json:"triggerSource,omitempty"`
ScannedCount int64 `json:"scannedCount,omitempty"`
EnqueuedCount int64 `json:"enqueuedCount,omitempty"`
UpstreamAttemptCount int64 `json:"upstreamAttemptCount,omitempty"`
BatchCount int64 `json:"batchCount,omitempty"`
MaxRound int `json:"maxRound,omitempty"`
PendingFailureCount int64 `json:"pendingFailureCount,omitempty"`
StartedAt *time.Time `json:"startedAt,omitempty"`
ScanCompletedAt *time.Time `json:"scanCompletedAt,omitempty"`
ArchiveStatus string `json:"archiveStatus,omitempty"`
ArchivedAt *time.Time `json:"archivedAt,omitempty"`
}
type RoundStats struct {
RunID string `json:"runId"`
Round int `json:"round"`
EnqueuedCount int64 `json:"enqueuedCount"`
ActiveCount int64 `json:"activeCount"`
NextRoundCount int64 `json:"nextRoundCount"`
SuccessCount int64 `json:"successCount"`
FailedCount int64 `json:"failedCount"`
UpstreamAttemptCount int64 `json:"upstreamAttemptCount"`
BatchCount int64 `json:"batchCount"`
StartedAt *time.Time `json:"startedAt,omitempty"`
CompletedAt *time.Time `json:"completedAt,omitempty"`
}
type RunView struct {
Source string `json:"source"`
Run RunMeta `json:"run"`
Rounds []RoundStats `json:"rounds,omitempty"`
}
type RunList struct {
Items []RunView `json:"items"`
Total int `json:"total"`
}
type Failure struct {
TaskID string `json:"taskId"`
CardID int64 `json:"cardId"`
ErrorCategory string `json:"errorCategory"`
ErrorMessage string `json:"errorMessage"`
FailedAt time.Time `json:"failedAt"`
Round int `json:"round"`
}
type FailurePage struct {
Items []Failure `json:"items"`
NextCursor string `json:"nextCursor"`
}
type StartReceipt struct {
RunID string `json:"runId"`
SyncKind string `json:"syncKind"`
Status string `json:"status"`
}
func FromEnvironment() Client {
return Client{BaseURL: "http://iotlcm-worker-svr:18081"}
}
func ValidateID(id string) bool {
if len(id) == 0 || len(id) > 128 {
return false
}
for _, c := range id {
if c < '0' || c > '9' {
if c < 'a' || c > 'z' {
if c < 'A' || c > 'Z' {
if c != '-' && c != '_' && c != ':' && c != '.' {
return false
}
}
}
}
}
return true
}
// MaxReplayRounds mirrors the Worker's server-side ceiling so an oversized
// request is rejected at the edge instead of surfacing as an opaque 400.
const MaxReplayRounds = 100
// Replay requests a re-run of the source run's failures. maxRounds == 0 asks
// the Worker to apply its configured default; any other value is passed
// through and must not exceed MaxReplayRounds.
func (c Client) Replay(ctx context.Context, id, actor string, maxRounds int) (Receipt, error) {
if !ValidateID(id) || actor == "" {
return Receipt{}, gerror.NewCode(gcode.CodeInvalidParameter, "Invalid replay request")
}
if maxRounds < 0 || maxRounds > MaxReplayRounds {
return Receipt{}, gerror.NewCode(gcode.CodeInvalidParameter, "Invalid replay maxRounds")
}
body, _ := json.Marshal(struct {
RequestedBy string `json:"requestedBy"`
MaxRounds int `json:"maxRounds,omitempty"`
}{actor, maxRounds})
var receipt Receipt
if err := c.do(ctx, http.MethodPost, "/internal/v1/full-sync-runs/"+url.PathEscape(id)+"/replay", body, http.StatusAccepted, &receipt); err != nil {
return Receipt{}, err
}
if !ValidateID(receipt.ReplayRunID) {
return Receipt{}, gerror.NewCode(gcode.CodeInternalError, "Invalid Worker response")
}
return receipt, nil
}
// ValidSyncKind matches the Worker's full-sync kinds for both monitoring and starts.
func ValidSyncKind(kind string) bool {
switch kind {
case "traffic_usage", "sim_status", "imei", "traffic_usage_daily", "traffic_usage_monthly":
return true
default:
return false
}
}
func (c Client) ListRuns(ctx context.Context, kind string, limit int) (RunList, error) {
if (kind != "" && !ValidSyncKind(kind)) || limit < 0 || limit > 100 {
return RunList{}, gerror.NewCode(gcode.CodeInvalidParameter, "Invalid full-sync filter")
}
query := url.Values{}
if kind != "" {
query.Set("syncKind", kind)
}
if limit > 0 {
query.Set("limit", fmt.Sprint(limit))
}
path := "/internal/v1/full-sync-runs"
if len(query) > 0 {
path += "?" + query.Encode()
}
var result RunList
if err := c.do(ctx, http.MethodGet, path, nil, http.StatusOK, &result); err != nil {
return RunList{}, err
}
if result.Items == nil {
result.Items = []RunView{}
}
return result, nil
}
func (c Client) Run(ctx context.Context, id string) (RunView, error) {
if !ValidateID(id) {
return RunView{}, gerror.NewCode(gcode.CodeInvalidParameter, "Invalid run ID")
}
var result RunView
if err := c.do(ctx, http.MethodGet, "/internal/v1/full-sync-runs/"+url.PathEscape(id), nil, http.StatusOK, &result); err != nil {
return RunView{}, err
}
if result.Run.RunID != id {
return RunView{}, gerror.NewCode(gcode.CodeInternalError, "Invalid Worker response")
}
return result, nil
}
func (c Client) Failures(ctx context.Context, id string, limit int, cursor string) (FailurePage, error) {
if !ValidateID(id) || limit < 0 || limit > 500 || len(cursor) > 1024 {
return FailurePage{}, gerror.NewCode(gcode.CodeInvalidParameter, "Invalid failure query")
}
query := url.Values{}
if limit > 0 {
query.Set("limit", fmt.Sprint(limit))
}
if cursor != "" {
query.Set("cursor", cursor)
}
path := "/internal/v1/full-sync-runs/" + url.PathEscape(id) + "/failures"
if len(query) > 0 {
path += "?" + query.Encode()
}
var result FailurePage
if err := c.do(ctx, http.MethodGet, path, nil, http.StatusOK, &result); err != nil {
return FailurePage{}, err
}
if result.Items == nil {
result.Items = []Failure{}
}
return result, nil
}
func (c Client) Get(ctx context.Context, id string) (Status, error) {
if !ValidateID(id) {
return Status{}, gerror.NewCode(gcode.CodeInvalidParameter, "Invalid replay run ID")
}
var status Status
if err := c.do(ctx, http.MethodGet, "/internal/v1/full-sync-replays/"+url.PathEscape(id), nil, http.StatusOK, &status); err != nil {
return Status{}, err
}
if status.ReplayRunID != id {
return Status{}, gerror.NewCode(gcode.CodeInternalError, "Invalid Worker response")
}
return status, nil
}
func (c Client) do(ctx context.Context, method, path string, body []byte, expected int, result any) error {
base, err := url.Parse(c.BaseURL)
if err != nil || base == nil || base.Scheme != "http" || base.Host == "" || base.User != nil || base.RawQuery != "" || base.Fragment != "" || base.Path != "" {
return gerror.NewCode(gcode.CodeInternalError, ErrConfig.Error())
}
client := c.HTTP
if client == nil {
client = &http.Client{Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
}
request, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(base.String(), "/")+path, bytes.NewReader(body))
if err != nil {
return gerror.NewCode(gcode.CodeInternalError, "Invalid Worker request")
}
if body != nil {
request.Header.Set("Content-Type", "application/json")
}
response, err := client.Do(request)
if err != nil {
return gerror.NewCode(gcode.CodeInternalError, "Worker unavailable")
}
defer response.Body.Close()
if response.StatusCode != expected {
return workerError(response.StatusCode)
}
decoder := json.NewDecoder(io.LimitReader(response.Body, maxResponse))
if err := decoder.Decode(result); err != nil {
return gerror.NewCode(gcode.CodeInternalError, "Invalid Worker response")
}
return nil
}
func workerError(status int) error {
switch status {
case http.StatusBadRequest:
return gerror.NewCode(gcode.CodeInvalidParameter, "Invalid replay request")
case http.StatusNotFound:
return gerror.NewCode(gcode.CodeNotFound, "Run not found")
case http.StatusGone:
return gerror.NewCode(gcode.CodeNotFound, "Failure details are no longer available")
case http.StatusServiceUnavailable:
return gerror.NewCode(gcode.CodeInternalError, "Worker is busy")
case http.StatusConflict:
return gerror.NewCode(gcode.CodeInvalidParameter, "Run is not ready for replay")
case http.StatusUnprocessableEntity:
return gerror.NewCode(gcode.CodeInvalidParameter, "Run has no failures")
default:
return gerror.NewCode(gcode.CodeInternalError, fmt.Sprintf("Worker failed (HTTP %d)", status))
}
}
@@ -0,0 +1,159 @@
package fullsyncretry
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gogf/gf/v2/errors/gcode"
"github.com/gogf/gf/v2/errors/gerror"
)
func TestReplayAndStatusProxy(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
if r.Header.Get("Authorization") != "" {
t.Error("Admin JWT must not be forwarded to Worker")
}
if r.Method == "POST" {
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
if len(body) != 1 || body["requestedBy"] != "42" {
t.Errorf("unexpected Worker request: %+v", body)
}
w.WriteHeader(http.StatusAccepted)
_, _ = w.Write([]byte(`{"replayRunId":"replay-1","status":"accepted"}`))
return
}
_, _ = w.Write([]byte(`{"replayRunId":"replay-1","sourceRunId":"source","status":"completed","remaining":0,"enqueued":3}`))
}))
defer server.Close()
client := Client{BaseURL: server.URL}
r, err := client.Replay(context.Background(), "source", "42", 0)
if err != nil || r.ReplayRunID != "replay-1" {
t.Fatalf("replay: %+v %v", r, err)
}
s, err := client.Get(context.Background(), "replay-1")
if err != nil || s.Enqueued != 3 || calls != 2 {
t.Fatalf("status: %+v %v, calls=%d", s, err, calls)
}
}
func TestRunQueriesAndFailures(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
switch {
case r.Method == http.MethodGet && r.URL.Path == "/internal/v1/full-sync-runs":
if r.URL.Query().Get("syncKind") != "traffic_usage" || r.URL.Query().Get("limit") != "7" {
t.Fatalf("query = %s", r.URL.RawQuery)
}
_, _ = w.Write([]byte(`{"items":[{"source":"live","run":{"runId":"run-1","operation":"current_month_usage_full_sync","status":"running"}}],"total":1}`))
case r.Method == http.MethodGet && r.URL.Path == "/internal/v1/full-sync-runs/run-1":
_, _ = w.Write([]byte(`{"source":"archive","run":{"runId":"run-1","operation":"current_month_usage_full_sync","status":"completed"},"rounds":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/internal/v1/full-sync-runs/run-1/failures":
_, _ = w.Write([]byte(`{"items":[{"taskId":"task-1","cardId":1,"errorCategory":"business","errorMessage":"failed","round":2}],"nextCursor":"next"}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client := Client{BaseURL: server.URL}
list, err := client.ListRuns(context.Background(), "traffic_usage", 7)
if err != nil || list.Total != 1 || len(list.Items) != 1 {
t.Fatalf("list = %+v, err = %v", list, err)
}
view, err := client.Run(context.Background(), "run-1")
if err != nil || view.Source != "archive" {
t.Fatalf("run = %+v, err = %v", view, err)
}
page, err := client.Failures(context.Background(), "run-1", 100, "cursor")
if err != nil || len(page.Items) != 1 || page.NextCursor != "next" || calls != 3 {
t.Fatalf("failures = %+v, err = %v, calls = %d", page, err, calls)
}
}
func TestReplayWorkerTimeout(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(50 * time.Millisecond) }))
defer server.Close()
client := Client{BaseURL: server.URL, HTTP: &http.Client{Timeout: time.Millisecond}}
_, err := client.Replay(context.Background(), "source", "42", 0)
if err == nil || !strings.Contains(err.Error(), "Worker unavailable") {
t.Fatalf("timeout: %v", err)
}
}
func TestReplayValidationAndWorkerErrors(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
w.WriteHeader(http.StatusConflict)
_, _ = w.Write([]byte(`{"error":{"message":"ICCID secret"}}`))
}))
defer server.Close()
client := Client{BaseURL: server.URL}
for _, id := range []string{"", "a/b", "{bad}", strings.Repeat("a", 129)} {
if _, err := client.Replay(context.Background(), id, "42", 0); gerror.Code(err) != gcode.CodeInvalidParameter {
t.Fatalf("id=%q: %v", id, err)
}
}
if calls != 0 {
t.Fatal("invalid source triggered Worker")
}
// The edge must reject an oversized round budget before calling Worker.
for _, rounds := range []int{-1, MaxReplayRounds + 1} {
if _, err := client.Replay(context.Background(), "source", "42", rounds); gerror.Code(err) != gcode.CodeInvalidParameter {
t.Fatalf("maxRounds=%d: %v", rounds, err)
}
}
if calls != 0 {
t.Fatal("invalid maxRounds triggered Worker")
}
_, err := client.Replay(context.Background(), "source", "42", 0)
if gerror.Code(err) != gcode.CodeInvalidParameter || strings.Contains(err.Error(), "ICCID") {
t.Fatalf("error leaked Worker body: %v", err)
}
for _, status := range []int{400, 401, 404, 409, 422, 500} {
if workerError(status) == nil {
t.Fatalf("missing error mapping for %d", status)
}
}
}
// The round budget must reach Worker only when the caller asks for one, so an
// omitted value keeps the Worker-side configured default authoritative.
func TestReplayForwardsMaxRoundsOnlyWhenSet(t *testing.T) {
var bodies []map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body map[string]any
_ = json.NewDecoder(r.Body).Decode(&body)
bodies = append(bodies, body)
w.WriteHeader(http.StatusAccepted)
_, _ = w.Write([]byte(`{"replayRunId":"replay-1","status":"accepted"}`))
}))
defer server.Close()
client := Client{BaseURL: server.URL}
if _, err := client.Replay(context.Background(), "source", "42", 0); err != nil {
t.Fatal(err)
}
if _, err := client.Replay(context.Background(), "source", "42", 7); err != nil {
t.Fatal(err)
}
if len(bodies) != 2 {
t.Fatalf("expected 2 Worker calls, got %d", len(bodies))
}
if _, present := bodies[0]["maxRounds"]; present {
t.Fatalf("omitted budget must not be sent: %+v", bodies[0])
}
if got := bodies[1]["maxRounds"]; got != float64(7) {
t.Fatalf("explicit budget must be forwarded, got %+v", bodies[1])
}
}
@@ -0,0 +1,39 @@
package fullsyncretry
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/gogf/gf/v2/errors/gcode"
"github.com/gogf/gf/v2/errors/gerror"
)
func TestListRunsAcceptsFiveKindsAndRejectsUnknownFilters(t *testing.T) {
var filters []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
filters = append(filters, r.URL.Query().Get("syncKind"))
_, _ = w.Write([]byte(`{"items":null,"total":0}`))
}))
defer server.Close()
client := Client{BaseURL: server.URL}
kinds := []string{"", "traffic_usage", "sim_status", "imei", "traffic_usage_daily", "traffic_usage_monthly"}
for i, kind := range kinds {
result, err := client.ListRuns(context.Background(), kind, 100)
if err != nil || result.Items == nil || len(filters) != i+1 || filters[i] != kind {
t.Fatalf("kind=%q result=%+v filters=%v error=%v", kind, result, filters, err)
}
}
for _, query := range []struct {
kind string
limit int
}{{"unknown", 10}, {"IMEI", 10}, {"", -1}, {"imei", 101}} {
if _, err := client.ListRuns(context.Background(), query.kind, query.limit); gerror.Code(err) != gcode.CodeInvalidParameter {
t.Fatalf("invalid filter %+v: %v", query, err)
}
}
if len(filters) != len(kinds) {
t.Fatal("invalid filter reached Worker")
}
}