chore: 并入 iotlcm-admin-svr 历史到 admin-svr/（原提交 49818c6）

2026-11-02 13:27:59 +08:00
parent 976980a3f5 49818c6dbe
commit 193f7393c0
1250 changed files with 154111 additions and 0 deletions
@@ -0,0 +1,498 @@
+package exportcenter
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime/debug"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/crypto/gmd5"
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gctx"
+	"github.com/gogf/gf/v2/os/gtime"
+	"github.com/xuri/excelize/v2"
+
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/contexts"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
+	"git.5iot.com/iotlcm-public/iotlcm-admin-svr/utility/hash"
+)
+
+// slots 在提交时预留容量，避免无限积压等待执行的 goroutine。
+var slots = make(chan struct{}, MaxConcurrent+MaxQueued)
+
+// sem 限制同时执行的导出任务数，避免把 DB 连接与磁盘打满。
+var sem = make(chan struct{}, MaxConcurrent)
+
+func run(taskNo string, req *Request) {
+	defer func() { <-slots }()
+	baseCtx := gctx.New()
+	var task *entity.TExportTask
+	var generatedPath string
+	defer func() {
+		if p := recover(); p != nil {
+			g.Log().Errorf(baseCtx, "[exportcenter] 导出协程异常 taskNo=%s panic=%v stack=%s", taskNo, p, debug.Stack())
+			if task == nil || task.Id == 0 {
+				task = &entity.TExportTask{}
+				if err := dao.TExportTask.Ctx(baseCtx).Where(dao.TExportTask.Columns().TaskNo, taskNo).Scan(task); err != nil {
+					g.Log().Errorf(baseCtx, "[exportcenter] panic 后加载任务失败 taskNo=%s err=%v", taskNo, err)
+				}
+			}
+			if task != nil && task.Id > 0 {
+				markFailed(baseCtx, task, gerror.Newf("导出任务执行异常: %v", p))
+			}
+		}
+	}()
+	sem <- struct{}{}
+	defer func() { <-sem }()
+
+	task = &entity.TExportTask{}
+	if err := dao.TExportTask.Ctx(baseCtx).
+		Where(dao.TExportTask.Columns().TaskNo, taskNo).
+		Scan(task); err != nil || task.Id <= 0 {
+		g.Log().Errorf(baseCtx, "[exportcenter] 加载任务失败 taskNo=%s err=%v", taskNo, err)
+		return
+	}
+
+	// CAS 抢占：只有把 0 改成 2 的协程才能执行
+	now := gtime.Now()
+	affected, err := dao.TExportTask.Ctx(baseCtx).
+		Data(g.Map{
+			dao.TExportTask.Columns().Status:    consts.DownloadStatusRunning,
+			dao.TExportTask.Columns().StartTime: now,
+		}).
+		Where(dao.TExportTask.Columns().Id, task.Id).
+		WhereNull(dao.TExportTask.Columns().DeletedTime).
+		Where(dao.TExportTask.Columns().Status, consts.DownloadStatusPending).
+		UpdateAndGetAffected()
+	if err != nil {
+		g.Log().Errorf(baseCtx, "[exportcenter] 抢占任务失败 taskNo=%s err=%v", taskNo, err)
+		return
+	}
+	if affected == 0 {
+		return
+	}
+
+	execCtx, cancel := context.WithTimeout(restoreContext(baseCtx, task), TaskTimeout)
+	defer cancel()
+
+	markBatchRunning(execCtx, taskNo)
+
+	if err = generate(execCtx, task, req, &generatedPath); err != nil {
+		g.Log().Errorf(execCtx, "[exportcenter] 导出失败 taskNo=%s err=%v", taskNo, err)
+		markFailed(baseCtx, task, err)
+	}
+}
+
+// restoreContext 用提交时的身份快照还原 ctx，保证 FilterAuth 仍按提交人范围过滤。
+func restoreContext(base context.Context, task *entity.TExportTask) context.Context {
+	snap := &model.ExportIdentitySnapshot{}
+	if task.IdentitySnapshot != nil {
+		if err := task.IdentitySnapshot.Scan(snap); err != nil {
+			g.Log().Warningf(base, "[exportcenter] 身份快照解析失败 taskNo=%s err=%v", task.TaskNo, err)
+		}
+	}
+	if snap.UserId <= 0 {
+		snap.UserId = task.OwnerUserId
+		snap.DeptId = task.DeptId
+		snap.TenantId = task.TenantId
+	}
+
+	identity := &model.Identity{}
+	identity.ID = snap.UserId
+	identity.DeptId = snap.DeptId
+	identity.TenantId = snap.TenantId
+	identity.Roles = snap.Roles
+	identity.AccessCodeList = snap.AccessCodeList
+
+	data := g.Map{}
+	if snap.CurrentRoleId != "" {
+		data["currentRoleId"] = snap.CurrentRoleId
+	}
+	if snap.CurrentRoleDataScope != "" {
+		data["currentRoleDataScope"] = snap.CurrentRoleDataScope
+	}
+
+	return contexts.Set(base, &model.Context{User: identity, Data: data})
+}
+
+func generate(ctx context.Context, task *entity.TExportTask, req *Request, generatedPath *string) (err error) {
+	header, fields, err := resolveColumns(req)
+	if err != nil {
+		return gerror.Wrap(err, "生成表头失败")
+	}
+
+	dicts, err := loadColumnDicts(ctx, req.Columns, func(ctx context.Context, types []string) ([]model.SysDictDataListModel, error) {
+		return service.SysDictData().GetDictListByTypes(ctx, types)
+	})
+	if err != nil {
+		return gerror.Wrap(err, "加载导出字典失败")
+	}
+
+	sheet := req.SheetName
+	if sheet == "" {
+		sheet = req.BizName
+	}
+	sheet = truncateSheetName(sheet)
+
+	f := excelize.NewFile()
+	defer f.Close()
+	if err = f.SetSheetName(f.GetSheetName(0), sheet); err != nil {
+		return gerror.Wrap(err, "设置工作表名称失败")
+	}
+	stream, err := f.NewStreamWriter(sheet)
+	if err != nil {
+		return gerror.Wrap(err, "创建工作表流失败")
+	}
+
+	if err = stream.SetRow("A1", header); err != nil {
+		return gerror.Wrap(err, "写入表头失败")
+	}
+
+	rowNum := 2
+	written := 0
+	for page := 1; ; page++ {
+		rows, total, fetchErr := req.Fetch(ctx, page, DefaultPageSize)
+		if fetchErr != nil {
+			// 页码只留给日志排查；对用户展示业务侧给出的可读原因，不暴露分页与驱动细节。
+			g.Log().Errorf(ctx, "[exportcenter] 第 %d 页取数失败 taskNo=%s err=%v", page, task.TaskNo, fetchErr)
+			return fetchErr
+		}
+		if len(rows) == 0 {
+			break
+		}
+		for _, v := range rows {
+			cell, _ := excelize.CoordinatesToCellName(1, rowNum)
+			values := rowValues(v, fields, req)
+			translateRow(values, req.Columns, dicts)
+			if err = stream.SetRow(cell, values); err != nil {
+				return gerror.Wrapf(err, "写入第 %d 行失败", rowNum)
+			}
+			rowNum++
+			written++
+			if written > MaxRows {
+				return gerror.Newf("导出行数超过上限 %d 行", MaxRows)
+			}
+		}
+		// 已取满或不足一页，结束
+		if len(rows) < DefaultPageSize || (total > 0 && written >= total) {
+			break
+		}
+	}
+	if err = stream.Flush(); err != nil {
+		return gerror.Wrap(err, "刷新工作表失败")
+	}
+
+	content, err := toBytes(f)
+	if err != nil {
+		return err
+	}
+	if int64(len(content)) > MaxFileSize {
+		return gerror.Newf("导出文件超过 %d MB 上限，请缩小筛选范围", MaxFileSize>>20)
+	}
+
+	oss, err := savePrivateExport(ctx, task, content)
+	if err != nil {
+		return gerror.Wrap(err, "保存导出文件失败")
+	}
+	*generatedPath = oss.Path
+	task.OssId = oss.OssId
+	if err = markSuccess(ctx, task, oss, written, req); err != nil {
+		return err
+	}
+	*generatedPath = ""
+	return nil
+}
+
+func savePrivateExport(ctx context.Context, task *entity.TExportTask, content []byte) (*model.SysOssSaveContentModel, error) {
+	uploadRoot := strings.TrimSpace(g.Cfg().MustGet(ctx, "system.upload.local.path").String())
+	if uploadRoot == "" {
+		return nil, gerror.New("上传路径未配置")
+	}
+	root, err := filepath.Abs(uploadRoot)
+	if err != nil {
+		return nil, err
+	}
+	// 私有目录必须与上传静态根目录隔离，不能生成可访问的 OSS URL。
+	path := filepath.Join(filepath.Dir(root), "private-export", task.TenantId, task.TaskNo+".xlsx")
+	if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return nil, err
+	}
+	registered := false
+	defer func() {
+		if !registered {
+			if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) {
+				g.Log().Errorf(gctx.New(), "[exportcenter] 清理未登记文件失败 path=%s err=%v", path, removeErr)
+			}
+		}
+	}()
+	if err = os.WriteFile(path, content, 0600); err != nil {
+		return nil, err
+	}
+	md5, err := gmd5.EncryptFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var ossId int64
+	// 这里仅登记元数据；通用 OSS API 拒绝导出文件，实际下载走下载中心。
+	err = g.DB().Transaction(gctx.New(), func(ctx context.Context, tx gdb.TX) error {
+		var insertErr error
+		ossId, insertErr = dao.SysOss.Ctx(ctx).TX(tx).Data(g.Map{
+			dao.SysOss.Columns().TenantId:     task.TenantId,
+			dao.SysOss.Columns().FileName:     task.FileName,
+			dao.SysOss.Columns().OriginalName: task.FileName,
+			dao.SysOss.Columns().FileSuffix:   "xlsx",
+			dao.SysOss.Columns().Path:         path,
+			dao.SysOss.Columns().Url:          "",
+			dao.SysOss.Columns().CreatedDept:  task.DeptId,
+			dao.SysOss.Columns().CreatedBy:    task.OwnerUserId,
+			dao.SysOss.Columns().CreatedAt:    gtime.Now(),
+			dao.SysOss.Columns().Service:      "local",
+			dao.SysOss.Columns().Md5:          md5,
+			dao.SysOss.Columns().FileSize:     len(content),
+			dao.SysOss.Columns().FileCrc16:    hash.Crc16ChecksumFile(path),
+			dao.SysOss.Columns().FileSum:      hash.Sum16ChecksumFile(path),
+		}).InsertAndGetId()
+		if insertErr != nil {
+			return insertErr
+		}
+		affected, updateErr := dao.TExportTask.Ctx(ctx).TX(tx).
+			Data(g.Map{dao.TExportTask.Columns().OssId: ossId}).
+			Where(dao.TExportTask.Columns().Id, task.Id).
+			Where(dao.TExportTask.Columns().Status, consts.DownloadStatusRunning).UpdateAndGetAffected()
+		if updateErr != nil {
+			return updateErr
+		}
+		if affected != 1 {
+			return gerror.New("登记导出文件失败：任务状态已变更")
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	registered = true
+	task.OssId = ossId
+	return &model.SysOssSaveContentModel{SysOss: &entity.SysOss{OssId: ossId, Path: path, FileSize: len(content), Md5: md5}}, nil
+}
+
+func structValues(v interface{}) []interface{} {
+	rv := reflect.ValueOf(v)
+	for rv.Kind() == reflect.Ptr {
+		if rv.IsNil() {
+			return nil
+		}
+		rv = rv.Elem()
+	}
+	if rv.Kind() != reflect.Struct {
+		return []interface{}{v}
+	}
+	fields := rv.NumField()
+	out := make([]interface{}, fields)
+	for i := 0; i < fields; i++ {
+		out[i] = rv.Field(i).Interface()
+	}
+	return out
+}
+
+func toBytes(f *excelize.File) ([]byte, error) {
+	tmp, err := os.CreateTemp("", "iotlcm-export-*.xlsx")
+	if err != nil {
+		return nil, gerror.Wrap(err, "创建临时文件失败")
+	}
+	name := tmp.Name()
+	defer os.Remove(name)
+	defer tmp.Close()
+
+	if err = f.Write(tmp); err != nil {
+		return nil, gerror.Wrap(err, "写入临时文件失败")
+	}
+	if err = tmp.Sync(); err != nil {
+		return nil, gerror.Wrap(err, "刷新临时文件失败")
+	}
+	content, err := os.ReadFile(name)
+	if err != nil {
+		return nil, gerror.Wrap(err, "读取临时文件失败")
+	}
+	return content, nil
+}
+
+func truncateSheetName(name string) string {
+	if name == "" {
+		return "Sheet1"
+	}
+	rs := []rune(name)
+	if len(rs) > 31 {
+		rs = rs[:31]
+	}
+	return string(rs)
+}
+
+func markBatchRunning(ctx context.Context, taskNo string) {
+	_, err := dao.BatchMain.Ctx(ctx).
+		Data(g.Map{
+			dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
+			dao.BatchMain.Columns().RunResult:        consts.BatchRunResultRunning,
+			dao.BatchMain.Columns().StartTime:        gtime.Now(),
+		}).
+		Where(dao.BatchMain.Columns().BatchNo, taskNo).
+		Update()
+	if err != nil {
+		g.Log().Warningf(ctx, "[exportcenter] 更新执行日志开始状态失败 taskNo=%s err=%v", taskNo, err)
+	}
+}
+
+func markSuccess(ctx context.Context, task *entity.TExportTask, oss *model.SysOssSaveContentModel, written int, req *Request) error {
+	now := gtime.Now()
+	// 默认按全部成功回填；查询类导出可通过 Request.Result 改判为部分成功/失败并写汇总说明。
+	// 注意：即使判定为失败，文件本身仍已生成并可下载（文件里写明每条失败原因），
+	// 因此这里只改执行日志的 async_status/run_result，不改 t_export_task 的下载状态。
+	runResult := consts.BatchRunResultSuccess
+	asyncStatus := consts.BatchAsyncStatusSuccess
+	summary := ""
+	if req != nil && req.Result != nil {
+		if result, text := req.Result(written); result > 0 {
+			runResult = result
+			summary = text
+			switch result {
+			case consts.BatchRunResultPartialSuccess:
+				asyncStatus = consts.BatchAsyncStatusPartialSuccess
+				summary = text
+			case consts.BatchRunResultFailed:
+				asyncStatus = consts.BatchAsyncStatusFailed
+				summary = text
+			default:
+				asyncStatus = consts.BatchAsyncStatusSuccess
+			}
+		}
+	}
+	// 文件已写磁盘，两张表必须同一事务提交；失败保留 OSS 索引供过期清理重试。
+	return g.DB().Transaction(gctx.New(), func(ctx context.Context, tx gdb.TX) error {
+		affected, err := dao.TExportTask.Ctx(ctx).TX(tx).
+			Data(g.Map{
+				dao.TExportTask.Columns().Status:     consts.DownloadStatusGenerated,
+				dao.TExportTask.Columns().OssId:      oss.OssId,
+				dao.TExportTask.Columns().FileSize:   oss.FileSize,
+				dao.TExportTask.Columns().FileMd5:    oss.Md5,
+				dao.TExportTask.Columns().TotalCount: written,
+				dao.TExportTask.Columns().FinishTime: now,
+				dao.TExportTask.Columns().ExpireTime: now.AddDate(0, 0, ExpireDays),
+			}).
+			Where(dao.TExportTask.Columns().Id, task.Id).
+			Where(dao.TExportTask.Columns().Status, consts.DownloadStatusRunning).
+			UpdateAndGetAffected()
+		if err != nil {
+			return gerror.Wrap(err, "回填导出任务失败")
+		}
+		if affected != 1 {
+			return gerror.New("回填导出任务失败：任务状态已变更")
+		}
+		affected, err = dao.BatchMain.Ctx(ctx).TX(tx).
+			Data(g.Map{
+				dao.BatchMain.Columns().BatchAsyncStatus: asyncStatus,
+				dao.BatchMain.Columns().RunResult:        runResult,
+				dao.BatchMain.Columns().FailReason:       summary,
+				dao.BatchMain.Columns().ResultOssId:      oss.OssId,
+				dao.BatchMain.Columns().TotalCount:       written,
+				dao.BatchMain.Columns().EndTime:          now,
+			}).
+			Where(dao.BatchMain.Columns().BatchNo, task.TaskNo).
+			UpdateAndGetAffected()
+		if err != nil {
+			return gerror.Wrap(err, "回填执行日志失败")
+		}
+		if affected != 1 {
+			return gerror.New("回填执行日志失败：记录不存在")
+		}
+		return nil
+	})
+}
+
+func markFailed(ctx context.Context, task *entity.TExportTask, cause error) {
+	// 失败也要落库，用独立 ctx 避免任务 ctx 已超时
+	ctx = gctx.New()
+	msg := cause.Error()
+	if len([]rune(msg)) > 1900 {
+		msg = string([]rune(msg)[:1900])
+	}
+	now := gtime.Now()
+
+	affected, err := dao.TExportTask.Ctx(ctx).
+		Data(g.Map{
+			dao.TExportTask.Columns().Status:       consts.DownloadStatusFailed,
+			dao.TExportTask.Columns().ErrorMessage: msg,
+			dao.TExportTask.Columns().FinishTime:   now,
+		}).
+		Where(dao.TExportTask.Columns().Id, task.Id).
+		WhereNull(dao.TExportTask.Columns().DeletedTime).
+		WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusPending, consts.DownloadStatusRunning}).
+		UpdateAndGetAffected()
+	if err != nil {
+		g.Log().Errorf(ctx, "[exportcenter] 回填失败状态失败 taskNo=%s err=%v", task.TaskNo, err)
+		return
+	}
+	if affected == 0 {
+		current := &entity.TExportTask{}
+		if err := dao.TExportTask.Ctx(ctx).Where(dao.TExportTask.Columns().Id, task.Id).Scan(current); err != nil {
+			g.Log().Errorf(ctx, "[exportcenter] 核对失败状态失败 taskNo=%s err=%v", task.TaskNo, err)
+			return
+		}
+		if current.Id == 0 || current.Status != consts.DownloadStatusFailed {
+			return
+		}
+	}
+
+	_, err = dao.BatchMain.Ctx(ctx).
+		Data(g.Map{
+			dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusFailed,
+			dao.BatchMain.Columns().RunResult:        consts.BatchRunResultFailed,
+			dao.BatchMain.Columns().FailReason:       msg,
+			dao.BatchMain.Columns().EndTime:          now,
+		}).
+		Where(dao.BatchMain.Columns().BatchNo, task.TaskNo).
+		Update()
+	if err != nil {
+		g.Log().Errorf(ctx, "[exportcenter] 回填执行日志失败状态失败 taskNo=%s err=%v", task.TaskNo, err)
+	}
+}
+
+// RecoverOnStart 进程启动时把残留的「排队中/生成中」任务标记为失败。
+// 进程内任务不跨重启恢复，失败后用户可在下载中心点重试重新提交。
+func RecoverOnStart(ctx context.Context) {
+	now := time.Now()
+	_, err := dao.TExportTask.Ctx(ctx).
+		Data(g.Map{
+			dao.TExportTask.Columns().Status:       consts.DownloadStatusFailed,
+			dao.TExportTask.Columns().ErrorMessage: "服务重启，任务已中断，请重新导出",
+			dao.TExportTask.Columns().FinishTime:   now,
+		}).
+		WhereNull(dao.TExportTask.Columns().DeletedTime).
+		WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusPending, consts.DownloadStatusRunning}).
+		Update()
+	if err != nil {
+		g.Log().Errorf(ctx, "[exportcenter] 启动补偿失败: %v", err)
+		return
+	}
+
+	_, err = dao.BatchMain.Ctx(ctx).
+		Data(g.Map{
+			dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusFailed,
+			dao.BatchMain.Columns().RunResult:        consts.BatchRunResultFailed,
+			dao.BatchMain.Columns().FailReason:       "服务重启，任务已中断",
+			dao.BatchMain.Columns().EndTime:          now,
+		}).
+		Where(dao.BatchMain.Columns().BatchType, consts.BatchTypeExport).
+		WhereIn(dao.BatchMain.Columns().BatchAsyncStatus, []int{consts.BatchAsyncStatusCreated, consts.BatchAsyncStatusProcessing}).
+		Update()
+	if err != nil {
+		g.Log().Errorf(ctx, "[exportcenter] 启动补偿执行日志失败: %v", err)
+	}
+}