package simwarning import ( "context" "encoding/json" "fmt" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity" "net/mail" "sort" "strconv" "strings" "time" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" ) const ( expireCandidateKeyPrefix = warnDedupeKeyPrefix + ":sim_card:expire:candidate" expireCandidateSetPrefix = warnDedupeKeyPrefix + ":sim_card:expire:candidates" expireCandidateTTLBuffer = 7 * 24 * time.Hour simTerminalTable = "t_sim_terminal" terminalTable = "t_terminal" ) type expireCandidate struct { TenantID string `json:"tenantId"` CardID int64 `json:"cardId"` ICCID string `json:"iccid"` MSISDN string `json:"msisdn"` PackageCode string `json:"packageCode"` ExpireUnix int64 `json:"expireUnix"` ExpireAt string `json:"expireAt"` TargetMonth string `json:"targetMonth"` CreatedDept int64 `json:"createdDept"` CreatedBy int64 `json:"createdBy"` LoadedAt string `json:"loadedAt"` } type expireCandidateStats struct { CandidateCount int CachedCount int DeletedCount int scanStats } type expireWarningContactRow struct { UserID int64 `orm:"user_id"` Email string `orm:"email"` Phonenumber string `orm:"phonenumber"` } func (s *sSimWarning) PreloadExpireCandidates(ctx context.Context) { if ctx == nil { ctx = context.Background() } runTime := time.Now() targetBegin, targetEnd := expirePreloadTargetMonth(runTime) targetMonth := targetBegin.Format("200601") stats := expireCandidateStats{} lastID := int64(0) g.Log().Infof(ctx, "SimWarning.ExpirePreload begin, runTime:%s, targetMonth:%s", runTime.Format("2006-01-02 15:04:05"), targetMonth) for { if err := ctx.Err(); err != nil { stats.ErrorCount++ g.Log().Warningf(ctx, "SimWarning.ExpirePreload canceled, targetMonth:%s, lastID:%d, err:%+v", targetMonth, lastID, err) break } cards, err := s.queryExpireCandidateCards(ctx, lastID, targetBegin, targetEnd) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.ExpirePreload query candidates failed, targetMonth:%s, lastID:%d, err:%+v", targetMonth, lastID, err) break } if len(cards) == 0 { break } stats.CandidateCount += len(cards) for _, card := range cards { if card.Id > lastID { lastID = card.Id } if reason := validExpireCandidateCard(card, runTime); reason != "" { stats.SkipCount++ g.Log().Debugf(ctx, "SimWarning.ExpirePreload skipped, targetMonth:%s, cardId:%d, iccid:%s, reason:%s", targetMonth, card.Id, card.Iccid, reason) continue } if err := storeExpireCandidate(ctx, card, targetMonth, targetEnd.Add(expireCandidateTTLBuffer), runTime); err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.ExpirePreload store candidate failed, targetMonth:%s, cardId:%d, iccid:%s, err:%+v", targetMonth, card.Id, card.Iccid, err) continue } stats.CachedCount++ } if len(cards) < scanPageSize { break } } g.Log().Infof(ctx, "SimWarning.ExpirePreload end, targetMonth:%s, candidates:%d, cached:%d, skipped:%d, errors:%d", targetMonth, stats.CandidateCount, stats.CachedCount, stats.SkipCount, stats.ErrorCount) } func (s *sSimWarning) ScanExpireCandidates(ctx context.Context) { if ctx == nil { ctx = context.Background() } runTime := time.Now() stats := expireCandidateStats{} pendingWarnings := make([]pendingWarning, 0) g.Log().Infof(ctx, "SimWarning.ExpireCandidateScan begin, runTime:%s, mode:warn_log_only", runTime.Format("2006-01-02 15:04:05")) for _, targetMonth := range expireCandidateScanMonths(runTime) { setKey := expireCandidateSetKey(targetMonth) members, err := g.Redis().SMembers(ctx, setKey) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.ExpireCandidateScan read redis set failed, setKey:%s, err:%+v", setKey, err) continue } keys := members.Strings() stats.CandidateCount += len(keys) for _, key := range keys { if err := ctx.Err(); err != nil { stats.ErrorCount++ g.Log().Warningf(ctx, "SimWarning.ExpireCandidateScan canceled, setKey:%s, err:%+v", setKey, err) break } pending := s.scanExpireCandidateKey(ctx, setKey, key, runTime, &stats) if pending != nil { pendingWarnings = append(pendingWarnings, *pending) } } } storedWarnings := s.storePendingWarnings(ctx, pendingWarnings, runTime, &stats.scanStats) g.Log().Infof( ctx, "SimWarning.ExpireCandidateScan end, candidates:%d, hits:%d, duplicated:%d, deleted:%d, skipped:%d, errors:%d, mode:warn_log_only", stats.CandidateCount, len(storedWarnings), stats.DupeCount, stats.DeletedCount, stats.SkipCount, stats.ErrorCount, ) } func (s *sSimWarning) scanExpireCandidateKey(ctx context.Context, setKey string, key string, runTime time.Time, stats *expireCandidateStats) *pendingWarning { key = strings.TrimSpace(key) if key == "" { stats.SkipCount++ return nil } candidate, ok, err := readExpireCandidate(ctx, key) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.ExpireCandidateScan read candidate failed, key:%s, err:%+v", key, err) return nil } if !ok { stats.DeletedCount++ removeExpireCandidate(ctx, setKey, key) return nil } card, err := s.queryExpireCandidateCard(ctx, candidate.TenantID, candidate.CardID) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.ExpireCandidateScan query card failed, key:%s, tenantId:%s, cardId:%d, err:%+v", key, candidate.TenantID, candidate.CardID, err) return nil } if reason := removeExpireCandidateReason(candidate, card, runTime); reason != "" { stats.DeletedCount++ removeExpireCandidate(ctx, setKey, key) g.Log().Infof(ctx, "SimWarning.ExpireCandidateScan candidate removed, key:%s, reason:%s", key, reason) return nil } warnPhones, warnEmails, err := s.resolveExpireWarningContacts(ctx, *card) if err != nil { stats.ErrorCount++ g.Log().Warningf(ctx, "SimWarning.ExpireCandidateScan resolve contacts failed, cardId:%d, iccid:%s, err:%+v", card.Id, card.Iccid, err) } cfg := systemExpireWarnConfig(ctx, *card, warnPhones, warnEmails) hit, reason := evaluateSystemExpireDay(cfg, *card, runTime) if hit == nil { stats.SkipCount++ g.Log().Debugf(ctx, "SimWarning.ExpireCandidateScan skipped, key:%s, cardId:%d, reason:%s", key, card.Id, reason) return nil } exists, err := s.existsInPeriod(ctx, *hit) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.ExpireCandidateScan idempotent check failed, cardId:%d, stage:%d, err:%+v", card.Id, hit.WarnStage, err) return nil } if exists { stats.DupeCount++ g.Log().Infof(ctx, "SimWarning.ExpireCandidateScan duplicated by warn log, cardId:%d, stage:%d", card.Id, hit.WarnStage) return nil } acquired, dedupeKey, err := acquireWarningDedupe(ctx, *hit, runTime) if err != nil { stats.ErrorCount++ g.Log().Errorf(ctx, "SimWarning.ExpireCandidateScan acquire dedupe failed, cardId:%d, stage:%d, err:%+v", card.Id, hit.WarnStage, err) return nil } if !acquired { stats.DupeCount++ g.Log().Infof(ctx, "SimWarning.ExpireCandidateScan duplicated by redis, cardId:%d, stage:%d", card.Id, hit.WarnStage) return nil } return &pendingWarning{Hit: *hit, DedupeKey: dedupeKey} } func (s *sSimWarning) queryExpireCandidateCards(ctx context.Context, lastID int64, targetBegin time.Time, targetEnd time.Time) ([]entity.TSimCardInfo, error) { columns := dao.TSimCardInfo.Columns() cards := make([]entity.TSimCardInfo, 0, scanPageSize) err := dao.TSimCardInfo.Ctx(ctx). Fields( columns.Id, columns.Iccid, columns.Msisdn, columns.CurrentPackageCode, columns.PackageStatus, columns.IsFlowPool, columns.FlowPkgBeginAt, columns.FlowPkgEndAt, columns.TenantId, columns.DeptId, columns.CreatedDept, columns.CreatedBy, ). WhereGT(columns.Id, lastID). Where(columns.PackageStatus, consts.PackageStatusEffective). Where(columns.IsFlowPool, 0). Where(columns.CurrentPackageCode+" <> ?", ""). Where(columns.FlowPkgEndAt+" >= ?", targetBegin). Where(columns.FlowPkgEndAt+" < ?", targetEnd). OrderAsc(columns.Id). Limit(scanPageSize). Scan(&cards) return cards, err } func (s *sSimWarning) queryExpireCandidateCard(ctx context.Context, tenantID string, cardID int64) (*entity.TSimCardInfo, error) { if strings.TrimSpace(tenantID) == "" || cardID <= 0 { return nil, nil } columns := dao.TSimCardInfo.Columns() var card *entity.TSimCardInfo err := dao.TSimCardInfo.Ctx(ctx). Fields( columns.Id, columns.Iccid, columns.Msisdn, columns.CurrentPackageCode, columns.PackageStatus, columns.IsFlowPool, columns.FlowPkgBeginAt, columns.FlowPkgEndAt, columns.TenantId, columns.DeptId, columns.CreatedDept, columns.CreatedBy, ). Where(columns.Id, cardID). Where(columns.TenantId, tenantID). Scan(&card) return card, err } func validExpireCandidateCard(card entity.TSimCardInfo, runTime time.Time) string { if reason := validCommonCard(card); reason != "" { return reason } if card.FlowPkgEndAt == nil || card.FlowPkgEndAt.IsZero() { return "package end time is empty" } if !runTime.Before(card.FlowPkgEndAt.Time) { return "package expired" } return "" } func removeExpireCandidateReason(candidate expireCandidate, card *entity.TSimCardInfo, runTime time.Time) string { if strings.TrimSpace(candidate.TenantID) == "" || candidate.CardID <= 0 || candidate.ExpireUnix <= 0 { return "candidate data invalid" } if card == nil || card.Id == 0 { return "card not found" } if reason := validExpireCandidateCard(*card, runTime); reason != "" { return reason } if card.FlowPkgEndAt.Time.Unix() != candidate.ExpireUnix { return "package renewed" } return "" } func storeExpireCandidate(ctx context.Context, card entity.TSimCardInfo, targetMonth string, expireAt time.Time, runTime time.Time) error { if card.FlowPkgEndAt == nil || card.FlowPkgEndAt.IsZero() { return gerror.New("package end time is empty") } candidate := expireCandidate{ TenantID: strings.TrimSpace(card.TenantId), CardID: card.Id, ICCID: strings.TrimSpace(card.Iccid), MSISDN: strings.TrimSpace(card.Msisdn), PackageCode: strings.TrimSpace(card.CurrentPackageCode), ExpireUnix: card.FlowPkgEndAt.Time.Unix(), ExpireAt: card.FlowPkgEndAt.Time.Format("2006-01-02 15:04:05"), TargetMonth: targetMonth, CreatedDept: expireWarningDeptID(card), CreatedBy: card.CreatedBy, LoadedAt: runTime.Format("2006-01-02 15:04:05"), } key := expireCandidateKey(candidate.TenantID, candidate.CardID, candidate.ExpireUnix) if key == "" { return gerror.New("expire candidate key is empty") } payload, err := json.Marshal(candidate) if err != nil { return err } ttl := expireCandidateTTL(expireAt, runTime) if _, err = g.Redis().Do(ctx, "SET", key, string(payload), "EX", int(ttl.Seconds())); err != nil { return err } setKey := expireCandidateSetKey(targetMonth) if _, err = g.Redis().Do(ctx, "SADD", setKey, key); err != nil { return err } _, err = g.Redis().Do(ctx, "EXPIRE", setKey, int(ttl.Seconds())) return err } func readExpireCandidate(ctx context.Context, key string) (expireCandidate, bool, error) { res, err := g.Redis().Do(ctx, "GET", key) if err != nil { return expireCandidate{}, false, err } raw := strings.TrimSpace(res.String()) if raw == "" { return expireCandidate{}, false, nil } var candidate expireCandidate if err = json.Unmarshal([]byte(raw), &candidate); err != nil { return expireCandidate{}, false, err } return candidate, true, nil } func removeExpireCandidate(ctx context.Context, setKey string, key string) { key = strings.TrimSpace(key) if key == "" { return } if strings.TrimSpace(setKey) != "" { _, _ = g.Redis().Do(ctx, "SREM", setKey, key) } _, _ = g.Redis().Do(ctx, "DEL", key) } func systemExpireWarnConfig(ctx context.Context, card entity.TSimCardInfo, phones string, emails string) warnConfig { thresholds := systemExpireWarnThresholds(ctx) return warnConfig{ TWarnConfig: entity.TWarnConfig{ Id: systemExpireConfigID, TenantId: strings.TrimSpace(card.TenantId), ItemType: warnItemTypeCard, ItemId: card.Id, ItemCode: strings.TrimSpace(card.Iccid), WarnUnit: warnUnitDay, FirstWarn: thresholds.First, SecondWarn: thresholds.Second, ThirdWarn: thresholds.Third, WarnMethods: systemExpireWarnMethods(phones, emails), WarnPhones: phones, WarnEmails: emails, AppNotifyEnabled: 1, WxTemplateId: consts.WarningWechatTemplateIDDefault, IsEnabled: 1, CreatedDept: expireWarningDeptID(card), CreatedBy: card.CreatedBy, }, WarnType: warnTypeExpireDay, } } func systemExpireWarnMethods(phones string, emails string) string { methods := []string{warnMethodPlatform} if strings.TrimSpace(emails) != "" { methods = append(methods, warnMethodEmail) } if strings.TrimSpace(phones) != "" { methods = append(methods, warnMethodSMS) } return strings.Join(methods, ",") } func (s *sSimWarning) resolveExpireWarningContacts(ctx context.Context, card entity.TSimCardInfo) (string, string, error) { rows := make([]expireWarningContactRow, 0) agentRows, err := queryExpireAgentContactRows(ctx, card) if err != nil { return "", "", err } rows = append(rows, agentRows...) terminalRows, err := queryExpireTerminalContactRows(ctx, card) if err != nil { return "", "", err } rows = append(rows, terminalRows...) return normalizeExpireContactRows(rows) } func queryExpireAgentContactRows(ctx context.Context, card entity.TSimCardInfo) ([]expireWarningContactRow, error) { tenantID := strings.TrimSpace(card.TenantId) deptID := expireWarningDeptID(card) if tenantID == "" || (deptID <= 0 && card.CreatedBy <= 0) { return nil, nil } columns := dao.SysUser.Columns() mod := dao.SysUser.Ctx(ctx). Fields(columns.UserId, columns.Email, columns.Phonenumber). Where(columns.TenantId, tenantID). Where(columns.UserType, consts.SysUserTypeSys). Where(columns.Status, consts.SysDeptStatusNormal) switch { case deptID > 0 && card.CreatedBy > 0: mod = mod.Where("("+columns.DeptId+" = ? OR "+columns.UserId+" = ?)", deptID, card.CreatedBy) case deptID > 0: mod = mod.Where(columns.DeptId, deptID) default: mod = mod.Where(columns.UserId, card.CreatedBy) } var rows []expireWarningContactRow err := mod.Scan(&rows) return rows, err } func queryExpireTerminalContactRows(ctx context.Context, card entity.TSimCardInfo) ([]expireWarningContactRow, error) { rows := make([]expireWarningContactRow, 0) tenantID := strings.TrimSpace(card.TenantId) if tenantID == "" { return rows, nil } createdRows, err := queryExpireCreatedTerminalContactRows(ctx, tenantID, card.CreatedBy) if err != nil { return nil, err } rows = append(rows, createdRows...) iccid := strings.TrimSpace(card.Iccid) msisdn := strings.TrimSpace(card.Msisdn) if iccid == "" && msisdn == "" { return rows, nil } mod := g.DB().Model(simTerminalTable+" st").Ctx(ctx).Safe(). LeftJoin(terminalTable+" t", "t.terminal_id = st.terminal_id AND t.tenant_id = st.tenant_id"). LeftJoin(dao.SysUser.Table()+" u", "(u.user_id = t.leader OR u.user_id = st.terminal_id) AND u.tenant_id = st.tenant_id"). Fields("DISTINCT u.user_id, u.email, u.phonenumber"). Where("st.tenant_id", tenantID). Where("u.user_type", consts.SysUserTypeAPP). Where("u.status", consts.SysDeptStatusNormal) if iccid != "" && msisdn != "" { mod = mod.Where("(st.iccid = ? OR st.msisdn = ?)", iccid, msisdn) } else if iccid != "" { mod = mod.Where("st.iccid", iccid) } else { mod = mod.Where("st.msisdn", msisdn) } var relatedRows []expireWarningContactRow if err = mod.Scan(&relatedRows); err != nil { return nil, err } rows = append(rows, relatedRows...) return rows, nil } func queryExpireCreatedTerminalContactRows(ctx context.Context, tenantID string, createdBy int64) ([]expireWarningContactRow, error) { if strings.TrimSpace(tenantID) == "" || createdBy <= 0 { return nil, nil } columns := dao.SysUser.Columns() var rows []expireWarningContactRow err := dao.SysUser.Ctx(ctx). Fields(columns.UserId, columns.Email, columns.Phonenumber). Where(columns.TenantId, tenantID). Where(columns.UserId, createdBy). Where(columns.UserType, consts.SysUserTypeAPP). Where(columns.Status, consts.SysDeptStatusNormal). Scan(&rows) return rows, err } func normalizeExpireContactRows(rows []expireWarningContactRow) (string, string, error) { phoneSeen := make(map[string]struct{}) emailSeen := make(map[string]struct{}) phones := make([]string, 0) emails := make([]string, 0) for _, row := range rows { phone := strings.TrimSpace(row.Phonenumber) if phone != "" && isValidConfigWarnPhone(phone) { phones = appendUniqueString(phones, phoneSeen, phone) } email := strings.TrimSpace(row.Email) if email != "" { address, err := mail.ParseAddress(email) if err == nil && address != nil && strings.TrimSpace(address.Address) != "" { emails = appendUniqueString(emails, emailSeen, strings.ToLower(strings.TrimSpace(address.Address))) } } } sort.Strings(phones) sort.Strings(emails) return strings.Join(phones, ","), strings.Join(emails, ","), nil } func appendUniqueString(values []string, seen map[string]struct{}, value string) []string { value = strings.TrimSpace(value) if value == "" { return values } key := strings.ToLower(value) if _, ok := seen[key]; ok { return values } seen[key] = struct{}{} return append(values, value) } func expireWarningDeptID(card entity.TSimCardInfo) int64 { if card.CreatedDept > 0 { return card.CreatedDept } return card.DeptId } func expirePreloadTargetMonth(runTime time.Time) (time.Time, time.Time) { begin := calendarMonthBegin(runTime) targetBegin := begin.AddDate(0, 2, 0) return targetBegin, targetBegin.AddDate(0, 1, 0) } func expireCandidateScanMonths(runTime time.Time) []string { begin := calendarMonthBegin(runTime) seen := make(map[string]struct{}, 4) months := make([]string, 0, 4) for _, offset := range []int{-1, 0, 1, 2} { month := begin.AddDate(0, offset, 0).Format("200601") if _, ok := seen[month]; ok { continue } seen[month] = struct{}{} months = append(months, month) } return months } func calendarMonthBegin(runTime time.Time) time.Time { if runTime.IsZero() { runTime = time.Now() } location := runTime.Location() local := runTime.In(location) return time.Date(local.Year(), local.Month(), 1, 0, 0, 0, 0, location) } func expireCandidateTTL(expireAt time.Time, runTime time.Time) time.Duration { ttl := expireAt.Sub(runTime) if ttl < warnDedupeMinTTL { return warnDedupeMinTTL } return ttl } func expireCandidateKey(tenantID string, cardID int64, expireUnix int64) string { tenantID = strings.TrimSpace(tenantID) if tenantID == "" || cardID <= 0 || expireUnix <= 0 { return "" } return fmt.Sprintf("%s:%s:%d:%d", expireCandidateKeyPrefix, tenantID, cardID, expireUnix) } func expireCandidateSetKey(targetMonth string) string { targetMonth = strings.TrimSpace(targetMonth) if targetMonth == "" { targetMonth = "unknown" } return expireCandidateSetPrefix + ":" + targetMonth } func parseExpireCandidateKey(key string) (tenantID string, cardID int64, expireUnix int64, ok bool) { key = strings.TrimSpace(key) prefix := expireCandidateKeyPrefix + ":" if !strings.HasPrefix(key, prefix) { return "", 0, 0, false } parts := strings.Split(strings.TrimPrefix(key, prefix), ":") if len(parts) != 3 { return "", 0, 0, false } cardID, err := strconv.ParseInt(parts[1], 10, 64) if err != nil || cardID <= 0 { return "", 0, 0, false } expireUnix, err = strconv.ParseInt(parts[2], 10, 64) if err != nil || expireUnix <= 0 { return "", 0, 0, false } return parts[0], cardID, expireUnix, true }