package simwarning import ( "context" "errors" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service" "strconv" "strings" "time" "github.com/gogf/gf/v2/frame/g" ) type sSimWarning struct{} func NewSimWarning() *sSimWarning { return &sSimWarning{} } func init() { service.RegisterSimWarning(NewSimWarning()) } func (s *sSimWarning) Scan(ctx context.Context) { s.scan(ctx, scanScopeAll) } func (s *sSimWarning) ScanTraffic(ctx context.Context) { s.scan(ctx, scanScopeTraffic) } func (s *sSimWarning) ScanFlowPool(ctx context.Context) { s.scan(ctx, scanScopeFlowPool) } func (s *sSimWarning) ScanExpire(ctx context.Context) { s.ScanExpireCandidates(ctx) } func (s *sSimWarning) scan(ctx context.Context, scope scanScope) { if ctx == nil { ctx = context.Background() } scope = normalizeScanScope(scope) runTime := time.Now() stats := scanStats{} pendingWarnings := make([]pendingWarning, 0) g.Log().Infof(ctx, "SimWarning.Scan begin, runTime:%s, scope:%s, mode:warn_log_only", runTime.Format("2006-01-02 15:04:05"), scope) for page := 1; ; page++ { if err := ctx.Err(); err != nil { stats.ErrorCount++ g.Log().Warningf(ctx, "SimWarning.Scan canceled, page:%d, scope:%s, err:%+v", page, scope, err) break } configs, err := s.queryConfigs(ctx, page, scope) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan query configs failed, page:%d, err:%+v", page, err) break } if len(configs) == 0 { break } stats.ConfigCount += len(configs) cardMap, err := s.queryCards(ctx, configs) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan query cards failed, page:%d, err:%+v", page, err) break } poolMap, err := s.queryFlowPools(ctx, configs) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan query flow pools failed, page:%d, err:%+v", page, err) break } for _, cfg := range configs { if pending := s.scanConfig(ctx, cfg, cardMap[itemKey(cfg.TenantId, cfg.ItemId)], poolMap[itemKey(cfg.TenantId, cfg.ItemId)], runTime, &stats); pending != nil { pendingWarnings = append(pendingWarnings, *pending) } } if len(configs) < scanPageSize { break } } storedWarnings := s.storePendingWarnings(ctx, pendingWarnings, runTime, &stats) for _, stored := range storedWarnings { s.scanAutoAction(ctx, stored.Hit, stored.Record.WarnLogID, runTime, &stats) } g.Log().Infof( ctx, "SimWarning.Scan end, configs:%d, hits:%d, duplicated:%d, skipped:%d, errors:%d, autoActions:%d, autoActionErrors:%d, scope:%s, mode:warn_log_only", stats.ConfigCount, stats.HitCount, stats.DupeCount, stats.SkipCount, stats.ErrorCount, stats.AutoActionCount, stats.AutoActionErrors, scope, ) } func (s *sSimWarning) queryConfigs(ctx context.Context, page int, scope scanScope) ([]warnConfig, error) { scope = normalizeScanScope(scope) columns := dao.TWarnConfig.Columns() mod := dao.TWarnConfig.Ctx(ctx). Fields( columns.Id, columns.TenantId, columns.ItemType, columns.ItemId, columns.ItemCode, columns.WarnUnit, columns.FirstWarn, columns.SecondWarn, columns.ThirdWarn, columns.WarnMethods, columns.WarnPhones, columns.WarnEmails, columns.AutoAction, columns.AppNotifyEnabled, columns.IsEnabled, columns.WxTemplateId, columns.CreatedDept, columns.CreatedBy, ). Where(columns.IsEnabled, 1) switch scope { case scanScopeTraffic: mod = mod.Where( columns.ItemType+" = ? AND "+columns.WarnUnit+" IN (?, ?)", warnItemTypeCard, warnUnitMb, warnUnitPercent, ) case scanScopeFlowPool: mod = mod.Where( columns.ItemType+" = ? AND "+columns.WarnUnit+" = ?", warnItemTypeFlowPool, warnUnitPercent, ) case scanScopeExpire: mod = mod.Where("1 = 0") default: mod = mod.Where( "("+columns.ItemType+" = ? AND "+columns.WarnUnit+" IN (?, ?)) OR ("+ columns.ItemType+" = ? AND "+columns.WarnUnit+" = ?)", warnItemTypeCard, warnUnitMb, warnUnitPercent, warnItemTypeFlowPool, warnUnitPercent, ) } entities := make([]entity.TWarnConfig, 0) err := mod.OrderAsc(columns.Id). Page(page, scanPageSize). Scan(&entities) if err != nil { return nil, err } configs := make([]warnConfig, 0, len(entities)) for i := range entities { configs = append(configs, warnConfig{ TWarnConfig: entities[i], WarnType: deriveWarnType(entities[i].ItemType, entities[i].WarnUnit), }) } return configs, nil } func normalizeScanScope(scope scanScope) scanScope { switch scope { case scanScopeTraffic, scanScopeFlowPool, scanScopeExpire: return scope default: return scanScopeAll } } func (s *sSimWarning) scanConfig(ctx context.Context, cfg warnConfig, card *entity.TSimCardInfo, pool *entity.TSimFlowPool, runTime time.Time, stats *scanStats) *pendingWarning { if cfg.ItemType == warnItemTypeFlowPool { return s.scanFlowPoolConfig(ctx, cfg, pool, runTime, stats) } return s.scanCardConfig(ctx, cfg, card, runTime, stats) } func (s *sSimWarning) scanCardConfig(ctx context.Context, cfg warnConfig, card *entity.TSimCardInfo, runTime time.Time, stats *scanStats) *pendingWarning { if card == nil { stats.SkipCount++ g.Log().Warningf(ctx, "SimWarning.Scan skipped, card not found, configId:%d, tenantId:%s, itemId:%d", cfg.Id, cfg.TenantId, cfg.ItemId) return nil } hit, reason := evaluateConfig(cfg, *card, runTime) if hit == nil { stats.SkipCount++ g.Log().Debugf(ctx, "SimWarning.Scan skipped, configId:%d, cardId:%d, reason:%s", cfg.Id, card.Id, reason) return nil } exists, err := s.existsInPeriod(ctx, *hit) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan idempotent check failed, configId:%d, cardId:%d, err:%+v", cfg.Id, card.Id, err) return nil } if exists { stats.DupeCount++ g.Log().Infof(ctx, "SimWarning.Scan duplicated, configId:%d, cardId:%d, stage:%d", cfg.Id, card.Id, hit.WarnStage) s.scanAutoAction(ctx, *hit, 0, runTime, stats) return nil } acquired, dedupeKey, err := acquireWarningDedupe(ctx, *hit, runTime) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan acquire dedupe failed, configId:%d, cardId:%d, stage:%d, err:%+v", cfg.Id, card.Id, hit.WarnStage, err) return nil } if !acquired { stats.DupeCount++ g.Log().Infof(ctx, "SimWarning.Scan duplicated by redis, configId:%d, cardId:%d, stage:%d", cfg.Id, card.Id, hit.WarnStage) return nil } return &pendingWarning{Hit: *hit, DedupeKey: dedupeKey} } func (s *sSimWarning) scanFlowPoolConfig(ctx context.Context, cfg warnConfig, pool *entity.TSimFlowPool, runTime time.Time, stats *scanStats) *pendingWarning { if pool == nil { stats.SkipCount++ g.Log().Warningf(ctx, "SimWarning.Scan skipped, flow pool not found, configId:%d, tenantId:%s, itemId:%d", cfg.Id, cfg.TenantId, cfg.ItemId) return nil } hit, reason := evaluatePoolTrafficPercent(cfg, *pool, runTime) if hit == nil { stats.SkipCount++ g.Log().Debugf(ctx, "SimWarning.Scan skipped, configId:%d, flowPoolId:%d, reason:%s", cfg.Id, pool.Id, reason) return nil } exists, err := s.existsInPeriod(ctx, *hit) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan idempotent check failed, configId:%d, flowPoolId:%d, err:%+v", cfg.Id, pool.Id, err) return nil } if exists { stats.DupeCount++ g.Log().Infof(ctx, "SimWarning.Scan duplicated, configId:%d, flowPoolId:%d, stage:%d", cfg.Id, pool.Id, hit.WarnStage) return nil } acquired, dedupeKey, err := acquireWarningDedupe(ctx, *hit, runTime) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan acquire dedupe failed, configId:%d, flowPoolId:%d, stage:%d, err:%+v", cfg.Id, pool.Id, hit.WarnStage, err) return nil } if !acquired { stats.DupeCount++ g.Log().Infof(ctx, "SimWarning.Scan duplicated by redis, configId:%d, flowPoolId:%d, stage:%d", cfg.Id, pool.Id, hit.WarnStage) return nil } return &pendingWarning{Hit: *hit, DedupeKey: dedupeKey} } func (s *sSimWarning) storePendingWarnings(ctx context.Context, pendingWarnings []pendingWarning, runTime time.Time, stats *scanStats) []storedWarning { if len(pendingWarnings) == 0 { return nil } storedWarnings, err := writeWarningBatch(ctx, pendingWarnings, runTime) if err != nil { for _, pending := range pendingWarnings { releaseWarningDedupe(ctx, pending.DedupeKey) } stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.Scan write warning batch failed, warnings:%d, err:%+v", len(pendingWarnings), err) return nil } stats.HitCount += len(storedWarnings) logWarningBatch(ctx, storedWarnings) for _, item := range storedWarnings { logWarningHit(ctx, item.Hit, item.Record) } notifyStoredWarnings(ctx, storedWarnings, runTime) return storedWarnings } func (s *sSimWarning) scanAutoAction(ctx context.Context, hit warningHit, warnLogID int64, runTime time.Time, stats *scanStats) { handled, err := s.handleAutoAction(ctx, hit, warnLogID, runTime) if err != nil { stats.ErrorCount++ stats.AutoActionErrors++ g.Log().Errorf(ctx, "SimWarning.Scan auto action failed, configId:%d, cardId:%d, stage:%d, warnLogId:%d, err:%+v", hit.Config.Id, hit.Card.Id, hit.WarnStage, warnLogID, err) return } if handled { stats.AutoActionCount++ } } func (s *sSimWarning) queryCards(ctx context.Context, configs []warnConfig) (map[string]*entity.TSimCardInfo, error) { result := make(map[string]*entity.TSimCardInfo) if len(configs) == 0 { return result, nil } itemIds, tenantIds := configKeys(configs, warnItemTypeCard) if len(itemIds) == 0 || len(tenantIds) == 0 { return result, nil } cards := make([]entity.TSimCardInfo, 0, len(itemIds)) columns := dao.TSimCardInfo.Columns() err := dao.TSimCardInfo.Ctx(ctx). Fields( columns.Id, columns.Iccid, columns.Msisdn, columns.Imsi, columns.CurrentPackageCode, columns.PackageStatus, columns.BossCardStatus, columns.IsFlowPool, columns.DataServiceStatus, columns.FlowPkgBeginAt, columns.FlowPkgEndAt, columns.PackageTotalFlowMb, columns.UsedFlowMb, columns.ChanKey, columns.TenantId, columns.CreatedDept, columns.CreatedBy, ). WhereIn(columns.Id, itemIds). WhereIn(columns.TenantId, tenantIds). Scan(&cards) if err != nil { return nil, err } for i := range cards { card := cards[i] result[itemKey(card.TenantId, card.Id)] = &cards[i] } return result, nil } func (s *sSimWarning) queryFlowPools(ctx context.Context, configs []warnConfig) (map[string]*entity.TSimFlowPool, error) { result := make(map[string]*entity.TSimFlowPool) if len(configs) == 0 { return result, nil } itemIds, tenantIds := configKeys(configs, warnItemTypeFlowPool) if len(itemIds) == 0 || len(tenantIds) == 0 { return result, nil } pools := make([]entity.TSimFlowPool, 0, len(itemIds)) columns := dao.TSimFlowPool.Columns() err := dao.TSimFlowPool.Ctx(ctx). Fields( columns.Id, columns.FlowPoolCode, columns.TotalFlowMb, columns.UsedFlowMb, columns.RemainFlowMb, columns.PoolStatus, columns.TenantId, columns.CreatedDept, columns.CreatedBy, ). WhereIn(columns.Id, itemIds). WhereIn(columns.TenantId, tenantIds). Scan(&pools) if err != nil { return nil, err } for i := range pools { pool := pools[i] result[itemKey(pool.TenantId, pool.Id)] = &pools[i] } return result, nil } func (s *sSimWarning) existsInPeriod(ctx context.Context, hit warningHit) (bool, error) { windowBegin, windowEnd := warningWindow(hit) if windowBegin.IsZero() || windowEnd.IsZero() { return false, nil } columns := dao.TWarnLog.Columns() count, err := dao.TWarnLog.Ctx(ctx). Where(columns.TenantId, hit.Config.TenantId). Where(columns.ConfigId, hit.Config.Id). Where(columns.ItemType, hit.Config.ItemType). Where(columns.ItemId, hit.Config.ItemId). Where(columns.WarnUnit, hit.Config.WarnUnit). Where(columns.WarnStage, hit.WarnStage). Where(columns.WarnTime+" >= ?", windowBegin). Where(columns.WarnTime+" < ?", windowEnd). Count() if err != nil { if isTableNotExist(err) { g.Log().Warningf(ctx, "SimWarning.Scan skip idempotent check, warn log table not found: %s", dao.TWarnLog.Table()) return false, nil } return false, err } return count > 0, nil } func isTableNotExist(err error) bool { if err == nil { return false } for err != nil { text := strings.ToLower(err.Error()) if strings.Contains(text, "doesn't exist") || strings.Contains(text, "does not exist") || strings.Contains(text, "not exist") || strings.Contains(text, "no such table") || strings.Contains(text, "unknown table") { return true } err = errors.Unwrap(err) } return false } func configKeys(configs []warnConfig, itemType int) ([]int64, []string) { itemIdSet := make(map[int64]struct{}) tenantIdSet := make(map[string]struct{}) itemIds := make([]int64, 0, len(configs)) tenantIds := make([]string, 0, len(configs)) for _, cfg := range configs { if itemType > 0 && cfg.ItemType != itemType { continue } if cfg.ItemId > 0 { if _, ok := itemIdSet[cfg.ItemId]; !ok { itemIdSet[cfg.ItemId] = struct{}{} itemIds = append(itemIds, cfg.ItemId) } } if cfg.TenantId != "" { if _, ok := tenantIdSet[cfg.TenantId]; !ok { tenantIdSet[cfg.TenantId] = struct{}{} tenantIds = append(tenantIds, cfg.TenantId) } } } return itemIds, tenantIds } func itemKey(tenantId string, itemId int64) string { return tenantId + ":" + strconv.FormatInt(itemId, 10) }