sync: 同步 v1.0.4 并脱敏配置与镜像

server/internal/library/exportcenter/cleanup.go

@@ -0,0 +1,130 @@
+package exportcenter
+
+import (
+	"context"
+	"os"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"time"
+
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gctx"
+	"github.com/gogf/gf/v2/os/gtime"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
+)
+
+const (
+	cleanupBatchSize = 200
+	cleanupInterval  = 5 * time.Minute
+)
+
+// RemoveTaskFile 清理文件与OSS记录，保留任务历史。
+// 文件删除不可事务回滚；数据库提交失败时保留索引，下次按文件不存在幂等重试。
+func RemoveTaskFile(ctx context.Context, taskID int64, reason string, validate func(*entity.TExportTask) error) (affected int64, err error) {
+	err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
+		columns := dao.TExportTask.Columns()
+		task := &entity.TExportTask{}
+		if err := dao.TExportTask.Ctx(ctx).TX(tx).Where(columns.Id, taskID).LockUpdate().Scan(task); err != nil {
+			return err
+		}
+		if task.Id == 0 {
+			return gerror.New("导出任务不存在")
+		}
+		if err := validate(task); err != nil {
+			return err
+		}
+		if task.DeletedTime != nil {
+			return nil
+		}
+		if task.OssId > 0 {
+			oss := &entity.SysOss{}
+			if err := dao.SysOss.Ctx(ctx).TX(tx).Where(dao.SysOss.Columns().OssId, task.OssId).LockUpdate().Scan(oss); err != nil {
+				return err
+			}
+			if oss.OssId > 0 {
+				if info, statErr := os.Stat(oss.Path); statErr == nil {
+					if !info.Mode().IsRegular() {
+						return gerror.New("导出文件路径不是普通文件")
+					}
+					if err := os.Remove(oss.Path); err != nil && !os.IsNotExist(err) {
+						return err
+					}
+				} else if !os.IsNotExist(statErr) {
+					return statErr
+				}
+				if _, err := dao.SysOss.Ctx(ctx).TX(tx).Where(dao.SysOss.Columns().OssId, oss.OssId).Delete(); err != nil {
+					return err
+				}
+			}
+		}
+		var updateErr error
+		affected, updateErr = dao.TExportTask.Ctx(ctx).TX(tx).Where(columns.Id, task.Id).WhereNull(columns.DeletedTime).
+			Data(g.Map{columns.DeletedTime: gtime.Now(), columns.DeleteReason: reason, columns.OssId: 0}).UpdateAndGetAffected()
+		if updateErr != nil {
+			return updateErr
+		}
+		if affected != 1 {
+			return gerror.New("任务状态已变更，请刷新后重试")
+		}
+		return nil
+	})
+	return affected, err
+}
+
+// StartCleanup 每五分钟清理过期任务和失败任务残留文件。
+func StartCleanup(ctx context.Context) {
+	go func() {
+		ticker := time.NewTicker(cleanupInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				cleanupOnce(gctx.New())
+			}
+		}
+	}()
+}
+
+func cleanupOnce(ctx context.Context) {
+	var lastID int64
+	for {
+		tasks := make([]*entity.TExportTask, 0, cleanupBatchSize)
+		if err := dao.TExportTask.Ctx(ctx).
+			WhereNull(dao.TExportTask.Columns().DeletedTime).
+			WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusGenerated, consts.DownloadStatusFailed}).
+			WhereLT(dao.TExportTask.Columns().ExpireTime, gtime.Now()).
+			WhereGT(dao.TExportTask.Columns().Id, lastID).
+			OrderAsc(dao.TExportTask.Columns().Id).
+			Limit(cleanupBatchSize).Scan(&tasks); err != nil {
+			g.Log().Errorf(ctx, "[exportcenter] 查询过期导出任务失败: %v", err)
+			return
+		}
+		progress := false
+		for _, task := range tasks {
+			affected, err := RemoveTaskFile(ctx, task.Id, "expired", func(locked *entity.TExportTask) error {
+				if (locked.Status != consts.DownloadStatusGenerated && locked.Status != consts.DownloadStatusFailed) || locked.ExpireTime == nil || !locked.ExpireTime.Before(gtime.Now()) {
+					return gerror.New("任务尚未过期或状态已变更")
+				}
+				return nil
+			})
+			if err != nil {
+				g.Log().Warningf(ctx, "[exportcenter] 清理过期任务失败 taskNo=%s err=%v", task.TaskNo, err)
+			} else if affected > 0 {
+				progress = true
+			}
+		}
+		if len(tasks) < cleanupBatchSize {
+			return
+		}
+		nextID := tasks[len(tasks)-1].Id
+		if !progress && nextID <= lastID {
+			return
+		}
+		lastID = nextID
+	}
+}