MiFaZhan
499 lines
17 KiB
Go
499 lines
17 KiB
Go
package exportcenter
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"runtime/debug"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gogf/gf/v2/crypto/gmd5"
|
|
"github.com/gogf/gf/v2/database/gdb"
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
"github.com/gogf/gf/v2/os/gctx"
|
|
"github.com/gogf/gf/v2/os/gtime"
|
|
"github.com/xuri/excelize/v2"
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/contexts"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service"
|
|
"git.5iot.com/iotlcm-public/iotlcm-admin-svr/utility/hash"
|
|
)
|
|
|
|
// slots 在提交时预留容量，避免无限积压等待执行的 goroutine。
|
|
var slots = make(chan struct{}, MaxConcurrent+MaxQueued)
|
|
|
|
// sem 限制同时执行的导出任务数，避免把 DB 连接与磁盘打满。
|
|
var sem = make(chan struct{}, MaxConcurrent)
|
|
|
|
func run(taskNo string, req *Request) {
|
|
defer func() { <-slots }()
|
|
baseCtx := gctx.New()
|
|
var task *entity.TExportTask
|
|
var generatedPath string
|
|
defer func() {
|
|
if p := recover(); p != nil {
|
|
g.Log().Errorf(baseCtx, "[exportcenter] 导出协程异常 taskNo=%s panic=%v stack=%s", taskNo, p, debug.Stack())
|
|
if task == nil || task.Id == 0 {
|
|
task = &entity.TExportTask{}
|
|
if err := dao.TExportTask.Ctx(baseCtx).Where(dao.TExportTask.Columns().TaskNo, taskNo).Scan(task); err != nil {
|
|
g.Log().Errorf(baseCtx, "[exportcenter] panic 后加载任务失败 taskNo=%s err=%v", taskNo, err)
|
|
}
|
|
}
|
|
if task != nil && task.Id > 0 {
|
|
markFailed(baseCtx, task, gerror.Newf("导出任务执行异常: %v", p))
|
|
}
|
|
}
|
|
}()
|
|
sem <- struct{}{}
|
|
defer func() { <-sem }()
|
|
|
|
task = &entity.TExportTask{}
|
|
if err := dao.TExportTask.Ctx(baseCtx).
|
|
Where(dao.TExportTask.Columns().TaskNo, taskNo).
|
|
Scan(task); err != nil || task.Id <= 0 {
|
|
g.Log().Errorf(baseCtx, "[exportcenter] 加载任务失败 taskNo=%s err=%v", taskNo, err)
|
|
return
|
|
}
|
|
|
|
// CAS 抢占：只有把 0 改成 2 的协程才能执行
|
|
now := gtime.Now()
|
|
affected, err := dao.TExportTask.Ctx(baseCtx).
|
|
Data(g.Map{
|
|
dao.TExportTask.Columns().Status: consts.DownloadStatusRunning,
|
|
dao.TExportTask.Columns().StartTime: now,
|
|
}).
|
|
Where(dao.TExportTask.Columns().Id, task.Id).
|
|
WhereNull(dao.TExportTask.Columns().DeletedTime).
|
|
Where(dao.TExportTask.Columns().Status, consts.DownloadStatusPending).
|
|
UpdateAndGetAffected()
|
|
if err != nil {
|
|
g.Log().Errorf(baseCtx, "[exportcenter] 抢占任务失败 taskNo=%s err=%v", taskNo, err)
|
|
return
|
|
}
|
|
if affected == 0 {
|
|
return
|
|
}
|
|
|
|
execCtx, cancel := context.WithTimeout(restoreContext(baseCtx, task), TaskTimeout)
|
|
defer cancel()
|
|
|
|
markBatchRunning(execCtx, taskNo)
|
|
|
|
if err = generate(execCtx, task, req, &generatedPath); err != nil {
|
|
g.Log().Errorf(execCtx, "[exportcenter] 导出失败 taskNo=%s err=%v", taskNo, err)
|
|
markFailed(baseCtx, task, err)
|
|
}
|
|
}
|
|
|
|
// restoreContext 用提交时的身份快照还原 ctx，保证 FilterAuth 仍按提交人范围过滤。
|
|
func restoreContext(base context.Context, task *entity.TExportTask) context.Context {
|
|
snap := &model.ExportIdentitySnapshot{}
|
|
if task.IdentitySnapshot != nil {
|
|
if err := task.IdentitySnapshot.Scan(snap); err != nil {
|
|
g.Log().Warningf(base, "[exportcenter] 身份快照解析失败 taskNo=%s err=%v", task.TaskNo, err)
|
|
}
|
|
}
|
|
if snap.UserId <= 0 {
|
|
snap.UserId = task.OwnerUserId
|
|
snap.DeptId = task.DeptId
|
|
snap.TenantId = task.TenantId
|
|
}
|
|
|
|
identity := &model.Identity{}
|
|
identity.ID = snap.UserId
|
|
identity.DeptId = snap.DeptId
|
|
identity.TenantId = snap.TenantId
|
|
identity.Roles = snap.Roles
|
|
identity.AccessCodeList = snap.AccessCodeList
|
|
|
|
data := g.Map{}
|
|
if snap.CurrentRoleId != "" {
|
|
data["currentRoleId"] = snap.CurrentRoleId
|
|
}
|
|
if snap.CurrentRoleDataScope != "" {
|
|
data["currentRoleDataScope"] = snap.CurrentRoleDataScope
|
|
}
|
|
|
|
return contexts.Set(base, &model.Context{User: identity, Data: data})
|
|
}
|
|
|
|
func generate(ctx context.Context, task *entity.TExportTask, req *Request, generatedPath *string) (err error) {
|
|
header, fields, err := resolveColumns(req)
|
|
if err != nil {
|
|
return gerror.Wrap(err, "生成表头失败")
|
|
}
|
|
|
|
dicts, err := loadColumnDicts(ctx, req.Columns, func(ctx context.Context, types []string) ([]model.SysDictDataListModel, error) {
|
|
return service.SysDictData().GetDictListByTypes(ctx, types)
|
|
})
|
|
if err != nil {
|
|
return gerror.Wrap(err, "加载导出字典失败")
|
|
}
|
|
|
|
sheet := req.SheetName
|
|
if sheet == "" {
|
|
sheet = req.BizName
|
|
}
|
|
sheet = truncateSheetName(sheet)
|
|
|
|
f := excelize.NewFile()
|
|
defer f.Close()
|
|
if err = f.SetSheetName(f.GetSheetName(0), sheet); err != nil {
|
|
return gerror.Wrap(err, "设置工作表名称失败")
|
|
}
|
|
stream, err := f.NewStreamWriter(sheet)
|
|
if err != nil {
|
|
return gerror.Wrap(err, "创建工作表流失败")
|
|
}
|
|
|
|
if err = stream.SetRow("A1", header); err != nil {
|
|
return gerror.Wrap(err, "写入表头失败")
|
|
}
|
|
|
|
rowNum := 2
|
|
written := 0
|
|
for page := 1; ; page++ {
|
|
rows, total, fetchErr := req.Fetch(ctx, page, DefaultPageSize)
|
|
if fetchErr != nil {
|
|
// 页码只留给日志排查；对用户展示业务侧给出的可读原因，不暴露分页与驱动细节。
|
|
g.Log().Errorf(ctx, "[exportcenter] 第 %d 页取数失败 taskNo=%s err=%v", page, task.TaskNo, fetchErr)
|
|
return fetchErr
|
|
}
|
|
if len(rows) == 0 {
|
|
break
|
|
}
|
|
for _, v := range rows {
|
|
cell, _ := excelize.CoordinatesToCellName(1, rowNum)
|
|
values := rowValues(v, fields, req)
|
|
translateRow(values, req.Columns, dicts)
|
|
if err = stream.SetRow(cell, values); err != nil {
|
|
return gerror.Wrapf(err, "写入第 %d 行失败", rowNum)
|
|
}
|
|
rowNum++
|
|
written++
|
|
if written > MaxRows {
|
|
return gerror.Newf("导出行数超过上限 %d 行", MaxRows)
|
|
}
|
|
}
|
|
// 已取满或不足一页，结束
|
|
if len(rows) < DefaultPageSize || (total > 0 && written >= total) {
|
|
break
|
|
}
|
|
}
|
|
if err = stream.Flush(); err != nil {
|
|
return gerror.Wrap(err, "刷新工作表失败")
|
|
}
|
|
|
|
content, err := toBytes(f)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if int64(len(content)) > MaxFileSize {
|
|
return gerror.Newf("导出文件超过 %d MB 上限，请缩小筛选范围", MaxFileSize>>20)
|
|
}
|
|
|
|
oss, err := savePrivateExport(ctx, task, content)
|
|
if err != nil {
|
|
return gerror.Wrap(err, "保存导出文件失败")
|
|
}
|
|
*generatedPath = oss.Path
|
|
task.OssId = oss.OssId
|
|
if err = markSuccess(ctx, task, oss, written, req); err != nil {
|
|
return err
|
|
}
|
|
*generatedPath = ""
|
|
return nil
|
|
}
|
|
|
|
func savePrivateExport(ctx context.Context, task *entity.TExportTask, content []byte) (*model.SysOssSaveContentModel, error) {
|
|
uploadRoot := strings.TrimSpace(g.Cfg().MustGet(ctx, "system.upload.local.path").String())
|
|
if uploadRoot == "" {
|
|
return nil, gerror.New("上传路径未配置")
|
|
}
|
|
root, err := filepath.Abs(uploadRoot)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// 私有目录必须与上传静态根目录隔离，不能生成可访问的 OSS URL。
|
|
path := filepath.Join(filepath.Dir(root), "private-export", task.TenantId, task.TaskNo+".xlsx")
|
|
if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
|
return nil, err
|
|
}
|
|
registered := false
|
|
defer func() {
|
|
if !registered {
|
|
if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) {
|
|
g.Log().Errorf(gctx.New(), "[exportcenter] 清理未登记文件失败 path=%s err=%v", path, removeErr)
|
|
}
|
|
}
|
|
}()
|
|
if err = os.WriteFile(path, content, 0600); err != nil {
|
|
return nil, err
|
|
}
|
|
md5, err := gmd5.EncryptFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var ossId int64
|
|
// 这里仅登记元数据；通用 OSS API 拒绝导出文件，实际下载走下载中心。
|
|
err = g.DB().Transaction(gctx.New(), func(ctx context.Context, tx gdb.TX) error {
|
|
var insertErr error
|
|
ossId, insertErr = dao.SysOss.Ctx(ctx).TX(tx).Data(g.Map{
|
|
dao.SysOss.Columns().TenantId: task.TenantId,
|
|
dao.SysOss.Columns().FileName: task.FileName,
|
|
dao.SysOss.Columns().OriginalName: task.FileName,
|
|
dao.SysOss.Columns().FileSuffix: "xlsx",
|
|
dao.SysOss.Columns().Path: path,
|
|
dao.SysOss.Columns().Url: "",
|
|
dao.SysOss.Columns().CreatedDept: task.DeptId,
|
|
dao.SysOss.Columns().CreatedBy: task.OwnerUserId,
|
|
dao.SysOss.Columns().CreatedAt: gtime.Now(),
|
|
dao.SysOss.Columns().Service: "local",
|
|
dao.SysOss.Columns().Md5: md5,
|
|
dao.SysOss.Columns().FileSize: len(content),
|
|
dao.SysOss.Columns().FileCrc16: hash.Crc16ChecksumFile(path),
|
|
dao.SysOss.Columns().FileSum: hash.Sum16ChecksumFile(path),
|
|
}).InsertAndGetId()
|
|
if insertErr != nil {
|
|
return insertErr
|
|
}
|
|
affected, updateErr := dao.TExportTask.Ctx(ctx).TX(tx).
|
|
Data(g.Map{dao.TExportTask.Columns().OssId: ossId}).
|
|
Where(dao.TExportTask.Columns().Id, task.Id).
|
|
Where(dao.TExportTask.Columns().Status, consts.DownloadStatusRunning).UpdateAndGetAffected()
|
|
if updateErr != nil {
|
|
return updateErr
|
|
}
|
|
if affected != 1 {
|
|
return gerror.New("登记导出文件失败：任务状态已变更")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
registered = true
|
|
task.OssId = ossId
|
|
return &model.SysOssSaveContentModel{SysOss: &entity.SysOss{OssId: ossId, Path: path, FileSize: len(content), Md5: md5}}, nil
|
|
}
|
|
|
|
func structValues(v interface{}) []interface{} {
|
|
rv := reflect.ValueOf(v)
|
|
for rv.Kind() == reflect.Ptr {
|
|
if rv.IsNil() {
|
|
return nil
|
|
}
|
|
rv = rv.Elem()
|
|
}
|
|
if rv.Kind() != reflect.Struct {
|
|
return []interface{}{v}
|
|
}
|
|
fields := rv.NumField()
|
|
out := make([]interface{}, fields)
|
|
for i := 0; i < fields; i++ {
|
|
out[i] = rv.Field(i).Interface()
|
|
}
|
|
return out
|
|
}
|
|
|
|
func toBytes(f *excelize.File) ([]byte, error) {
|
|
tmp, err := os.CreateTemp("", "iotlcm-export-*.xlsx")
|
|
if err != nil {
|
|
return nil, gerror.Wrap(err, "创建临时文件失败")
|
|
}
|
|
name := tmp.Name()
|
|
defer os.Remove(name)
|
|
defer tmp.Close()
|
|
|
|
if err = f.Write(tmp); err != nil {
|
|
return nil, gerror.Wrap(err, "写入临时文件失败")
|
|
}
|
|
if err = tmp.Sync(); err != nil {
|
|
return nil, gerror.Wrap(err, "刷新临时文件失败")
|
|
}
|
|
content, err := os.ReadFile(name)
|
|
if err != nil {
|
|
return nil, gerror.Wrap(err, "读取临时文件失败")
|
|
}
|
|
return content, nil
|
|
}
|
|
|
|
func truncateSheetName(name string) string {
|
|
if name == "" {
|
|
return "Sheet1"
|
|
}
|
|
rs := []rune(name)
|
|
if len(rs) > 31 {
|
|
rs = rs[:31]
|
|
}
|
|
return string(rs)
|
|
}
|
|
|
|
func markBatchRunning(ctx context.Context, taskNo string) {
|
|
_, err := dao.BatchMain.Ctx(ctx).
|
|
Data(g.Map{
|
|
dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusProcessing,
|
|
dao.BatchMain.Columns().RunResult: consts.BatchRunResultRunning,
|
|
dao.BatchMain.Columns().StartTime: gtime.Now(),
|
|
}).
|
|
Where(dao.BatchMain.Columns().BatchNo, taskNo).
|
|
Update()
|
|
if err != nil {
|
|
g.Log().Warningf(ctx, "[exportcenter] 更新执行日志开始状态失败 taskNo=%s err=%v", taskNo, err)
|
|
}
|
|
}
|
|
|
|
func markSuccess(ctx context.Context, task *entity.TExportTask, oss *model.SysOssSaveContentModel, written int, req *Request) error {
|
|
now := gtime.Now()
|
|
// 默认按全部成功回填；查询类导出可通过 Request.Result 改判为部分成功/失败并写汇总说明。
|
|
// 注意：即使判定为失败，文件本身仍已生成并可下载（文件里写明每条失败原因），
|
|
// 因此这里只改执行日志的 async_status/run_result，不改 t_export_task 的下载状态。
|
|
runResult := consts.BatchRunResultSuccess
|
|
asyncStatus := consts.BatchAsyncStatusSuccess
|
|
summary := ""
|
|
if req != nil && req.Result != nil {
|
|
if result, text := req.Result(written); result > 0 {
|
|
runResult = result
|
|
summary = text
|
|
switch result {
|
|
case consts.BatchRunResultPartialSuccess:
|
|
asyncStatus = consts.BatchAsyncStatusPartialSuccess
|
|
summary = text
|
|
case consts.BatchRunResultFailed:
|
|
asyncStatus = consts.BatchAsyncStatusFailed
|
|
summary = text
|
|
default:
|
|
asyncStatus = consts.BatchAsyncStatusSuccess
|
|
}
|
|
}
|
|
}
|
|
// 文件已写磁盘，两张表必须同一事务提交；失败保留 OSS 索引供过期清理重试。
|
|
return g.DB().Transaction(gctx.New(), func(ctx context.Context, tx gdb.TX) error {
|
|
affected, err := dao.TExportTask.Ctx(ctx).TX(tx).
|
|
Data(g.Map{
|
|
dao.TExportTask.Columns().Status: consts.DownloadStatusGenerated,
|
|
dao.TExportTask.Columns().OssId: oss.OssId,
|
|
dao.TExportTask.Columns().FileSize: oss.FileSize,
|
|
dao.TExportTask.Columns().FileMd5: oss.Md5,
|
|
dao.TExportTask.Columns().TotalCount: written,
|
|
dao.TExportTask.Columns().FinishTime: now,
|
|
dao.TExportTask.Columns().ExpireTime: now.AddDate(0, 0, ExpireDays),
|
|
}).
|
|
Where(dao.TExportTask.Columns().Id, task.Id).
|
|
Where(dao.TExportTask.Columns().Status, consts.DownloadStatusRunning).
|
|
UpdateAndGetAffected()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "回填导出任务失败")
|
|
}
|
|
if affected != 1 {
|
|
return gerror.New("回填导出任务失败：任务状态已变更")
|
|
}
|
|
affected, err = dao.BatchMain.Ctx(ctx).TX(tx).
|
|
Data(g.Map{
|
|
dao.BatchMain.Columns().BatchAsyncStatus: asyncStatus,
|
|
dao.BatchMain.Columns().RunResult: runResult,
|
|
dao.BatchMain.Columns().FailReason: summary,
|
|
dao.BatchMain.Columns().ResultOssId: oss.OssId,
|
|
dao.BatchMain.Columns().TotalCount: written,
|
|
dao.BatchMain.Columns().EndTime: now,
|
|
}).
|
|
Where(dao.BatchMain.Columns().BatchNo, task.TaskNo).
|
|
UpdateAndGetAffected()
|
|
if err != nil {
|
|
return gerror.Wrap(err, "回填执行日志失败")
|
|
}
|
|
if affected != 1 {
|
|
return gerror.New("回填执行日志失败：记录不存在")
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func markFailed(ctx context.Context, task *entity.TExportTask, cause error) {
|
|
// 失败也要落库，用独立 ctx 避免任务 ctx 已超时
|
|
ctx = gctx.New()
|
|
msg := cause.Error()
|
|
if len([]rune(msg)) > 1900 {
|
|
msg = string([]rune(msg)[:1900])
|
|
}
|
|
now := gtime.Now()
|
|
|
|
affected, err := dao.TExportTask.Ctx(ctx).
|
|
Data(g.Map{
|
|
dao.TExportTask.Columns().Status: consts.DownloadStatusFailed,
|
|
dao.TExportTask.Columns().ErrorMessage: msg,
|
|
dao.TExportTask.Columns().FinishTime: now,
|
|
}).
|
|
Where(dao.TExportTask.Columns().Id, task.Id).
|
|
WhereNull(dao.TExportTask.Columns().DeletedTime).
|
|
WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusPending, consts.DownloadStatusRunning}).
|
|
UpdateAndGetAffected()
|
|
if err != nil {
|
|
g.Log().Errorf(ctx, "[exportcenter] 回填失败状态失败 taskNo=%s err=%v", task.TaskNo, err)
|
|
return
|
|
}
|
|
if affected == 0 {
|
|
current := &entity.TExportTask{}
|
|
if err := dao.TExportTask.Ctx(ctx).Where(dao.TExportTask.Columns().Id, task.Id).Scan(current); err != nil {
|
|
g.Log().Errorf(ctx, "[exportcenter] 核对失败状态失败 taskNo=%s err=%v", task.TaskNo, err)
|
|
return
|
|
}
|
|
if current.Id == 0 || current.Status != consts.DownloadStatusFailed {
|
|
return
|
|
}
|
|
}
|
|
|
|
_, err = dao.BatchMain.Ctx(ctx).
|
|
Data(g.Map{
|
|
dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
|
dao.BatchMain.Columns().RunResult: consts.BatchRunResultFailed,
|
|
dao.BatchMain.Columns().FailReason: msg,
|
|
dao.BatchMain.Columns().EndTime: now,
|
|
}).
|
|
Where(dao.BatchMain.Columns().BatchNo, task.TaskNo).
|
|
Update()
|
|
if err != nil {
|
|
g.Log().Errorf(ctx, "[exportcenter] 回填执行日志失败状态失败 taskNo=%s err=%v", task.TaskNo, err)
|
|
}
|
|
}
|
|
|
|
// RecoverOnStart 进程启动时把残留的「排队中/生成中」任务标记为失败。
|
|
// 进程内任务不跨重启恢复，失败后用户可在下载中心点重试重新提交。
|
|
func RecoverOnStart(ctx context.Context) {
|
|
now := time.Now()
|
|
_, err := dao.TExportTask.Ctx(ctx).
|
|
Data(g.Map{
|
|
dao.TExportTask.Columns().Status: consts.DownloadStatusFailed,
|
|
dao.TExportTask.Columns().ErrorMessage: "服务重启，任务已中断，请重新导出",
|
|
dao.TExportTask.Columns().FinishTime: now,
|
|
}).
|
|
WhereNull(dao.TExportTask.Columns().DeletedTime).
|
|
WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusPending, consts.DownloadStatusRunning}).
|
|
Update()
|
|
if err != nil {
|
|
g.Log().Errorf(ctx, "[exportcenter] 启动补偿失败: %v", err)
|
|
return
|
|
}
|
|
|
|
_, err = dao.BatchMain.Ctx(ctx).
|
|
Data(g.Map{
|
|
dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusFailed,
|
|
dao.BatchMain.Columns().RunResult: consts.BatchRunResultFailed,
|
|
dao.BatchMain.Columns().FailReason: "服务重启，任务已中断",
|
|
dao.BatchMain.Columns().EndTime: now,
|
|
}).
|
|
Where(dao.BatchMain.Columns().BatchType, consts.BatchTypeExport).
|
|
WhereIn(dao.BatchMain.Columns().BatchAsyncStatus, []int{consts.BatchAsyncStatusCreated, consts.BatchAsyncStatusProcessing}).
|
|
Update()
|
|
if err != nil {
|
|
g.Log().Errorf(ctx, "[exportcenter] 启动补偿执行日志失败: %v", err)
|
|
}
|
|
}
|