package exportcenter import ( "context" "os" "github.com/gogf/gf/v2/errors/gerror" "time" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gctx" "github.com/gogf/gf/v2/os/gtime" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity" ) const ( cleanupBatchSize = 200 cleanupInterval = 5 * time.Minute ) // RemoveTaskFile 清理文件与OSS记录,保留任务历史。 // 文件删除不可事务回滚;数据库提交失败时保留索引,下次按文件不存在幂等重试。 func RemoveTaskFile(ctx context.Context, taskID int64, reason string, validate func(*entity.TExportTask) error) (affected int64, err error) { err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { columns := dao.TExportTask.Columns() task := &entity.TExportTask{} if err := dao.TExportTask.Ctx(ctx).TX(tx).Where(columns.Id, taskID).LockUpdate().Scan(task); err != nil { return err } if task.Id == 0 { return gerror.New("导出任务不存在") } if err := validate(task); err != nil { return err } if task.DeletedTime != nil { return nil } if task.OssId > 0 { oss := &entity.SysOss{} if err := dao.SysOss.Ctx(ctx).TX(tx).Where(dao.SysOss.Columns().OssId, task.OssId).LockUpdate().Scan(oss); err != nil { return err } if oss.OssId > 0 { if info, statErr := os.Stat(oss.Path); statErr == nil { if !info.Mode().IsRegular() { return gerror.New("导出文件路径不是普通文件") } if err := os.Remove(oss.Path); err != nil && !os.IsNotExist(err) { return err } } else if !os.IsNotExist(statErr) { return statErr } if _, err := dao.SysOss.Ctx(ctx).TX(tx).Where(dao.SysOss.Columns().OssId, oss.OssId).Delete(); err != nil { return err } } } var updateErr error affected, updateErr = dao.TExportTask.Ctx(ctx).TX(tx).Where(columns.Id, task.Id).WhereNull(columns.DeletedTime). Data(g.Map{columns.DeletedTime: gtime.Now(), columns.DeleteReason: reason, columns.OssId: 0}).UpdateAndGetAffected() if updateErr != nil { return updateErr } if affected != 1 { return gerror.New("任务状态已变更,请刷新后重试") } return nil }) return affected, err } // StartCleanup 每五分钟清理过期任务和失败任务残留文件。 func StartCleanup(ctx context.Context) { go func() { ticker := time.NewTicker(cleanupInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: cleanupOnce(gctx.New()) } } }() } func cleanupOnce(ctx context.Context) { var lastID int64 for { tasks := make([]*entity.TExportTask, 0, cleanupBatchSize) if err := dao.TExportTask.Ctx(ctx). WhereNull(dao.TExportTask.Columns().DeletedTime). WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusGenerated, consts.DownloadStatusFailed}). WhereLT(dao.TExportTask.Columns().ExpireTime, gtime.Now()). WhereGT(dao.TExportTask.Columns().Id, lastID). OrderAsc(dao.TExportTask.Columns().Id). Limit(cleanupBatchSize).Scan(&tasks); err != nil { g.Log().Errorf(ctx, "[exportcenter] 查询过期导出任务失败: %v", err) return } progress := false for _, task := range tasks { affected, err := RemoveTaskFile(ctx, task.Id, "expired", func(locked *entity.TExportTask) error { if (locked.Status != consts.DownloadStatusGenerated && locked.Status != consts.DownloadStatusFailed) || locked.ExpireTime == nil || !locked.ExpireTime.Before(gtime.Now()) { return gerror.New("任务尚未过期或状态已变更") } return nil }) if err != nil { g.Log().Warningf(ctx, "[exportcenter] 清理过期任务失败 taskNo=%s err=%v", task.TaskNo, err) } else if affected > 0 { progress = true } } if len(tasks) < cleanupBatchSize { return } nextID := tasks[len(tasks)-1].Id if !progress && nextID <= lastID { return } lastID = nextID } }