chore: 并入 iotlcm-admin-svr 历史到 admin-svr/（原提交 49818c6）
This commit is contained in:
MiFaZhan
@@ -0,0 +1,491 @@
|
||||
package simwarning
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/logic/warningnotify"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
||||
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type sSimWarning struct{}
|
||||
|
||||
func NewSimWarning() *sSimWarning {
|
||||
return &sSimWarning{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
service.RegisterSimWarning(NewSimWarning())
|
||||
}
|
||||
|
||||
func (s *sSimWarning) Scan(ctx context.Context) {
|
||||
s.scan(ctx, scanScopeAll)
|
||||
}
|
||||
|
||||
func (s *sSimWarning) ScanTraffic(ctx context.Context) {
|
||||
s.scan(ctx, scanScopeTraffic)
|
||||
}
|
||||
|
||||
func (s *sSimWarning) ScanFlowPool(ctx context.Context) {
|
||||
s.scan(ctx, scanScopeFlowPool)
|
||||
}
|
||||
|
||||
func (s *sSimWarning) ScanExpire(ctx context.Context) {
|
||||
s.ScanExpireCandidates(ctx)
|
||||
}
|
||||
|
||||
func (s *sSimWarning) scan(ctx context.Context, scope scanScope) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
scope = normalizeScanScope(scope)
|
||||
runTime := time.Now()
|
||||
stats := scanStats{}
|
||||
collector := warningnotify.NewCollector()
|
||||
g.Log().Infof(ctx, "SimWarning.Scan begin, runTime:%s, scope:%s, mode:warn_log_only", runTime.Format("2006-01-02 15:04:05"), scope)
|
||||
|
||||
var lastID int64
|
||||
for page := 1; ; page++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Warningf(ctx, "SimWarning.Scan canceled, page:%d, scope:%s, err:%+v", page, scope, err)
|
||||
break
|
||||
}
|
||||
configs, err := s.queryConfigs(ctx, lastID, scope)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan query configs failed, page:%d, err:%+v", page, err)
|
||||
break
|
||||
}
|
||||
if len(configs) == 0 {
|
||||
break
|
||||
}
|
||||
lastID = configs[len(configs)-1].Id
|
||||
stats.ConfigCount += len(configs)
|
||||
|
||||
cardMap, err := s.queryCards(ctx, configs)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan query cards failed, page:%d, err:%+v", page, err)
|
||||
break
|
||||
}
|
||||
poolMap, err := s.queryFlowPools(ctx, configs)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan query flow pools failed, page:%d, err:%+v", page, err)
|
||||
break
|
||||
}
|
||||
pendingWarnings := make([]pendingWarning, 0, len(configs))
|
||||
for _, cfg := range configs {
|
||||
if pending := s.scanConfig(ctx, cfg, cardMap[itemKey(cfg.TenantId, cfg.ItemId)], poolMap[itemKey(cfg.TenantId, cfg.ItemId)], runTime, &stats); pending != nil {
|
||||
pendingWarnings = append(pendingWarnings, *pending)
|
||||
}
|
||||
}
|
||||
stored := s.storePendingWarningsWithoutNotify(ctx, pendingWarnings, runTime, &stats)
|
||||
events := make([]warningnotify.Event, 0, len(stored))
|
||||
for _, item := range stored {
|
||||
events = append(events, warningNotifyEvent(ctx, item, runTime))
|
||||
s.scanAutoAction(ctx, item.Hit, item.Record.WarnLogID, runTime, &stats)
|
||||
}
|
||||
if err := collector.Add(ctx, events); err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "warning recipient aggregation failed: %+v", err)
|
||||
}
|
||||
if len(configs) < scanPageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := collector.Flush(ctx); err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "warning notification failed: %+v", err)
|
||||
}
|
||||
|
||||
g.Log().Infof(
|
||||
ctx,
|
||||
"SimWarning.Scan end, configs:%d, hits:%d, duplicated:%d, skipped:%d, errors:%d, autoActions:%d, autoActionErrors:%d, scope:%s, mode:warn_log_only",
|
||||
stats.ConfigCount,
|
||||
stats.HitCount,
|
||||
stats.DupeCount,
|
||||
stats.SkipCount,
|
||||
stats.ErrorCount,
|
||||
stats.AutoActionCount,
|
||||
stats.AutoActionErrors,
|
||||
scope,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *sSimWarning) queryConfigs(ctx context.Context, lastID int64, scope scanScope) ([]warnConfig, error) {
|
||||
scope = normalizeScanScope(scope)
|
||||
columns := dao.TWarnConfig.Columns()
|
||||
mod := dao.TWarnConfig.Ctx(ctx).
|
||||
Fields(
|
||||
columns.Id,
|
||||
columns.TenantId,
|
||||
columns.ItemType,
|
||||
columns.ItemId,
|
||||
columns.ItemCode,
|
||||
columns.WarnUnit,
|
||||
columns.FirstWarn,
|
||||
columns.SecondWarn,
|
||||
columns.ThirdWarn,
|
||||
columns.WarnMethods,
|
||||
columns.WarnPhones,
|
||||
columns.WarnEmails,
|
||||
columns.AutoAction,
|
||||
columns.AppNotifyEnabled,
|
||||
columns.IsEnabled,
|
||||
columns.WxTemplateId,
|
||||
columns.CreatedDept,
|
||||
columns.CreatedBy,
|
||||
).
|
||||
Where(columns.IsEnabled, 1)
|
||||
|
||||
switch scope {
|
||||
case scanScopeTraffic:
|
||||
mod = mod.Where(
|
||||
columns.ItemType+" = ? AND "+columns.WarnUnit+" IN (?, ?)",
|
||||
warnItemTypeCard,
|
||||
warnUnitMb,
|
||||
warnUnitPercent,
|
||||
)
|
||||
case scanScopeFlowPool:
|
||||
mod = mod.Where(
|
||||
columns.ItemType+" = ? AND "+columns.WarnUnit+" = ?",
|
||||
warnItemTypeFlowPool,
|
||||
warnUnitPercent,
|
||||
)
|
||||
case scanScopeExpire:
|
||||
mod = mod.Where("1 = 0")
|
||||
default:
|
||||
mod = mod.Where(
|
||||
"("+columns.ItemType+" = ? AND "+columns.WarnUnit+" IN (?, ?)) OR ("+
|
||||
columns.ItemType+" = ? AND "+columns.WarnUnit+" = ?)",
|
||||
warnItemTypeCard,
|
||||
warnUnitMb,
|
||||
warnUnitPercent,
|
||||
warnItemTypeFlowPool,
|
||||
warnUnitPercent,
|
||||
)
|
||||
}
|
||||
|
||||
entities := make([]entity.TWarnConfig, 0)
|
||||
err := mod.WhereGT(columns.Id, lastID).OrderAsc(columns.Id).
|
||||
Limit(scanPageSize).
|
||||
Scan(&entities)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
configs := make([]warnConfig, 0, len(entities))
|
||||
for i := range entities {
|
||||
configs = append(configs, warnConfig{
|
||||
TWarnConfig: entities[i],
|
||||
WarnType: deriveWarnType(entities[i].ItemType, entities[i].WarnUnit),
|
||||
})
|
||||
}
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
func normalizeScanScope(scope scanScope) scanScope {
|
||||
switch scope {
|
||||
case scanScopeTraffic, scanScopeFlowPool, scanScopeExpire:
|
||||
return scope
|
||||
default:
|
||||
return scanScopeAll
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sSimWarning) scanConfig(ctx context.Context, cfg warnConfig, card *entity.TSimCardInfo, pool *entity.TSimFlowPool, runTime time.Time, stats *scanStats) *pendingWarning {
|
||||
if cfg.ItemType == warnItemTypeFlowPool {
|
||||
return s.scanFlowPoolConfig(ctx, cfg, pool, runTime, stats)
|
||||
}
|
||||
return s.scanCardConfig(ctx, cfg, card, runTime, stats)
|
||||
}
|
||||
|
||||
func (s *sSimWarning) scanCardConfig(ctx context.Context, cfg warnConfig, card *entity.TSimCardInfo, runTime time.Time, stats *scanStats) *pendingWarning {
|
||||
if card == nil {
|
||||
stats.SkipCount++
|
||||
g.Log().Warningf(ctx, "SimWarning.Scan skipped, card not found, configId:%d, tenantId:%s, itemId:%d", cfg.Id, cfg.TenantId, cfg.ItemId)
|
||||
return nil
|
||||
}
|
||||
|
||||
hit, reason := evaluateConfig(cfg, *card, runTime)
|
||||
if hit == nil {
|
||||
stats.SkipCount++
|
||||
g.Log().Debugf(ctx, "SimWarning.Scan skipped, configId:%d, cardId:%d, reason:%s", cfg.Id, card.Id, reason)
|
||||
return nil
|
||||
}
|
||||
|
||||
exists, err := s.existsInPeriod(ctx, *hit)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan idempotent check failed, configId:%d, cardId:%d, err:%+v", cfg.Id, card.Id, err)
|
||||
return nil
|
||||
}
|
||||
if exists {
|
||||
stats.DupeCount++
|
||||
g.Log().Infof(ctx, "SimWarning.Scan duplicated, configId:%d, cardId:%d, stage:%d", cfg.Id, card.Id, hit.WarnStage)
|
||||
s.scanAutoAction(ctx, *hit, 0, runTime, stats)
|
||||
return nil
|
||||
}
|
||||
|
||||
acquired, dedupeKey, err := acquireWarningDedupe(ctx, *hit, runTime)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan acquire dedupe failed, configId:%d, cardId:%d, stage:%d, err:%+v", cfg.Id, card.Id, hit.WarnStage, err)
|
||||
return nil
|
||||
}
|
||||
if !acquired {
|
||||
stats.DupeCount++
|
||||
g.Log().Infof(ctx, "SimWarning.Scan duplicated by redis, configId:%d, cardId:%d, stage:%d", cfg.Id, card.Id, hit.WarnStage)
|
||||
return nil
|
||||
}
|
||||
|
||||
return &pendingWarning{Hit: *hit, DedupeKey: dedupeKey}
|
||||
}
|
||||
|
||||
func (s *sSimWarning) scanFlowPoolConfig(ctx context.Context, cfg warnConfig, pool *entity.TSimFlowPool, runTime time.Time, stats *scanStats) *pendingWarning {
|
||||
if pool == nil {
|
||||
stats.SkipCount++
|
||||
g.Log().Warningf(ctx, "SimWarning.Scan skipped, flow pool not found, configId:%d, tenantId:%s, itemId:%d", cfg.Id, cfg.TenantId, cfg.ItemId)
|
||||
return nil
|
||||
}
|
||||
|
||||
hit, reason := evaluatePoolTrafficPercent(cfg, *pool, runTime)
|
||||
if hit == nil {
|
||||
stats.SkipCount++
|
||||
g.Log().Debugf(ctx, "SimWarning.Scan skipped, configId:%d, flowPoolId:%d, reason:%s", cfg.Id, pool.Id, reason)
|
||||
return nil
|
||||
}
|
||||
|
||||
exists, err := s.existsInPeriod(ctx, *hit)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan idempotent check failed, configId:%d, flowPoolId:%d, err:%+v", cfg.Id, pool.Id, err)
|
||||
return nil
|
||||
}
|
||||
if exists {
|
||||
stats.DupeCount++
|
||||
g.Log().Infof(ctx, "SimWarning.Scan duplicated, configId:%d, flowPoolId:%d, stage:%d", cfg.Id, pool.Id, hit.WarnStage)
|
||||
return nil
|
||||
}
|
||||
|
||||
acquired, dedupeKey, err := acquireWarningDedupe(ctx, *hit, runTime)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan acquire dedupe failed, configId:%d, flowPoolId:%d, stage:%d, err:%+v", cfg.Id, pool.Id, hit.WarnStage, err)
|
||||
return nil
|
||||
}
|
||||
if !acquired {
|
||||
stats.DupeCount++
|
||||
g.Log().Infof(ctx, "SimWarning.Scan duplicated by redis, configId:%d, flowPoolId:%d, stage:%d", cfg.Id, pool.Id, hit.WarnStage)
|
||||
return nil
|
||||
}
|
||||
|
||||
return &pendingWarning{Hit: *hit, DedupeKey: dedupeKey}
|
||||
}
|
||||
|
||||
func (s *sSimWarning) storePendingWarnings(ctx context.Context, pendingWarnings []pendingWarning, runTime time.Time, stats *scanStats) []storedWarning {
|
||||
stored := s.storePendingWarningsWithoutNotify(ctx, pendingWarnings, runTime, stats)
|
||||
notifyStoredWarnings(ctx, stored, runTime)
|
||||
return stored
|
||||
}
|
||||
|
||||
func (s *sSimWarning) storePendingWarningsWithoutNotify(ctx context.Context, pendingWarnings []pendingWarning, runTime time.Time, stats *scanStats) []storedWarning {
|
||||
if len(pendingWarnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
storedWarnings, err := writeWarningBatch(ctx, pendingWarnings, runTime)
|
||||
if err != nil {
|
||||
for _, pending := range pendingWarnings {
|
||||
releaseWarningDedupe(ctx, pending.DedupeKey)
|
||||
}
|
||||
stats.ErrorCount++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan write warning batch failed, warnings:%d, err:%+v", len(pendingWarnings), err)
|
||||
return nil
|
||||
}
|
||||
|
||||
stats.HitCount += len(storedWarnings)
|
||||
logWarningBatch(ctx, storedWarnings)
|
||||
for _, item := range storedWarnings {
|
||||
logWarningHit(ctx, item.Hit, item.Record)
|
||||
}
|
||||
return storedWarnings
|
||||
}
|
||||
|
||||
func (s *sSimWarning) scanAutoAction(ctx context.Context, hit warningHit, warnLogID int64, runTime time.Time, stats *scanStats) {
|
||||
handled, err := s.handleAutoAction(ctx, hit, warnLogID, runTime)
|
||||
if err != nil {
|
||||
stats.ErrorCount++
|
||||
stats.AutoActionErrors++
|
||||
g.Log().Errorf(ctx, "SimWarning.Scan auto action failed, configId:%d, cardId:%d, stage:%d, warnLogId:%d, err:%+v", hit.Config.Id, hit.Card.Id, hit.WarnStage, warnLogID, err)
|
||||
return
|
||||
}
|
||||
if handled {
|
||||
stats.AutoActionCount++
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sSimWarning) queryCards(ctx context.Context, configs []warnConfig) (map[string]*entity.TSimCardInfo, error) {
|
||||
result := make(map[string]*entity.TSimCardInfo)
|
||||
if len(configs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
itemIds, tenantIds := configKeys(configs, warnItemTypeCard)
|
||||
if len(itemIds) == 0 || len(tenantIds) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
cards := make([]entity.TSimCardInfo, 0, len(itemIds))
|
||||
columns := dao.TSimCardInfo.Columns()
|
||||
err := dao.TSimCardInfo.Ctx(ctx).
|
||||
Fields(
|
||||
columns.Id,
|
||||
columns.Iccid,
|
||||
columns.Msisdn,
|
||||
columns.Imsi,
|
||||
columns.CurrentPackageCode,
|
||||
columns.PackageStatus,
|
||||
columns.BossCardStatus,
|
||||
columns.IsFlowPool,
|
||||
columns.DataServiceStatus,
|
||||
columns.FlowPkgBeginAt,
|
||||
columns.FlowPkgEndAt,
|
||||
columns.PackageTotalFlowMb,
|
||||
columns.UsedFlowMb,
|
||||
columns.ChanKey,
|
||||
columns.TenantId,
|
||||
columns.CreatedDept,
|
||||
columns.CreatedBy,
|
||||
).
|
||||
WhereIn(columns.Id, itemIds).
|
||||
WhereIn(columns.TenantId, tenantIds).
|
||||
Scan(&cards)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range cards {
|
||||
card := cards[i]
|
||||
result[itemKey(card.TenantId, card.Id)] = &cards[i]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *sSimWarning) queryFlowPools(ctx context.Context, configs []warnConfig) (map[string]*entity.TSimFlowPool, error) {
|
||||
result := make(map[string]*entity.TSimFlowPool)
|
||||
if len(configs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
itemIds, tenantIds := configKeys(configs, warnItemTypeFlowPool)
|
||||
if len(itemIds) == 0 || len(tenantIds) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
pools := make([]entity.TSimFlowPool, 0, len(itemIds))
|
||||
columns := dao.TSimFlowPool.Columns()
|
||||
err := dao.TSimFlowPool.Ctx(ctx).
|
||||
Fields(
|
||||
columns.Id,
|
||||
columns.FlowPoolCode,
|
||||
columns.TotalFlowMb,
|
||||
columns.UsedFlowMb,
|
||||
columns.RemainFlowMb,
|
||||
columns.PoolStatus,
|
||||
columns.TenantId,
|
||||
columns.CreatedDept,
|
||||
columns.CreatedBy,
|
||||
).
|
||||
WhereIn(columns.Id, itemIds).
|
||||
WhereIn(columns.TenantId, tenantIds).
|
||||
Scan(&pools)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range pools {
|
||||
pool := pools[i]
|
||||
result[itemKey(pool.TenantId, pool.Id)] = &pools[i]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *sSimWarning) existsInPeriod(ctx context.Context, hit warningHit) (bool, error) {
|
||||
windowBegin, windowEnd := warningWindow(hit)
|
||||
if windowBegin.IsZero() || windowEnd.IsZero() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
columns := dao.TWarnLog.Columns()
|
||||
count, err := dao.TWarnLog.Ctx(ctx).
|
||||
Where(columns.TenantId, hit.Config.TenantId).
|
||||
Where(columns.ConfigId, hit.Config.Id).
|
||||
Where(columns.ItemType, hit.Config.ItemType).
|
||||
Where(columns.ItemId, hit.Config.ItemId).
|
||||
Where(columns.WarnUnit, hit.Config.WarnUnit).
|
||||
Where(columns.WarnStage, hit.WarnStage).
|
||||
Where(columns.WarnTime+" >= ?", windowBegin).
|
||||
Where(columns.WarnTime+" < ?", windowEnd).
|
||||
Count()
|
||||
if err != nil {
|
||||
if isTableNotExist(err) {
|
||||
g.Log().Warningf(ctx, "SimWarning.Scan skip idempotent check, warn log table not found: %s", dao.TWarnLog.Table())
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func isTableNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
for err != nil {
|
||||
text := strings.ToLower(err.Error())
|
||||
if strings.Contains(text, "doesn't exist") ||
|
||||
strings.Contains(text, "does not exist") ||
|
||||
strings.Contains(text, "not exist") ||
|
||||
strings.Contains(text, "no such table") ||
|
||||
strings.Contains(text, "unknown table") {
|
||||
return true
|
||||
}
|
||||
err = errors.Unwrap(err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func configKeys(configs []warnConfig, itemType int) ([]int64, []string) {
|
||||
itemIdSet := make(map[int64]struct{})
|
||||
tenantIdSet := make(map[string]struct{})
|
||||
itemIds := make([]int64, 0, len(configs))
|
||||
tenantIds := make([]string, 0, len(configs))
|
||||
for _, cfg := range configs {
|
||||
if itemType > 0 && cfg.ItemType != itemType {
|
||||
continue
|
||||
}
|
||||
if cfg.ItemId > 0 {
|
||||
if _, ok := itemIdSet[cfg.ItemId]; !ok {
|
||||
itemIdSet[cfg.ItemId] = struct{}{}
|
||||
itemIds = append(itemIds, cfg.ItemId)
|
||||
}
|
||||
}
|
||||
if cfg.TenantId != "" {
|
||||
if _, ok := tenantIdSet[cfg.TenantId]; !ok {
|
||||
tenantIdSet[cfg.TenantId] = struct{}{}
|
||||
tenantIds = append(tenantIds, cfg.TenantId)
|
||||
}
|
||||
}
|
||||
}
|
||||
return itemIds, tenantIds
|
||||
}
|
||||
|
||||
func itemKey(tenantId string, itemId int64) string {
|
||||
return tenantId + ":" + strconv.FormatInt(itemId, 10)
|
||||
}
|
||||
Reference in New Issue
Block a user