package exportcenter import ( "context" "os" "path/filepath" "reflect" "runtime/debug" "strings" "time" "github.com/gogf/gf/v2/crypto/gmd5" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gctx" "github.com/gogf/gf/v2/os/gtime" "github.com/xuri/excelize/v2" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/consts" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/dao" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/library/contexts" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/model/entity" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/internal/service" "git.5iot.com/iotlcm-public/iotlcm-admin-svr/utility/hash" ) // slots 在提交时预留容量,避免无限积压等待执行的 goroutine。 var slots = make(chan struct{}, MaxConcurrent+MaxQueued) // sem 限制同时执行的导出任务数,避免把 DB 连接与磁盘打满。 var sem = make(chan struct{}, MaxConcurrent) func run(taskNo string, req *Request) { defer func() { <-slots }() baseCtx := gctx.New() var task *entity.TExportTask var generatedPath string defer func() { if p := recover(); p != nil { g.Log().Errorf(baseCtx, "[exportcenter] 导出协程异常 taskNo=%s panic=%v stack=%s", taskNo, p, debug.Stack()) if task == nil || task.Id == 0 { task = &entity.TExportTask{} if err := dao.TExportTask.Ctx(baseCtx).Where(dao.TExportTask.Columns().TaskNo, taskNo).Scan(task); err != nil { g.Log().Errorf(baseCtx, "[exportcenter] panic 后加载任务失败 taskNo=%s err=%v", taskNo, err) } } if task != nil && task.Id > 0 { markFailed(baseCtx, task, gerror.Newf("导出任务执行异常: %v", p)) } } }() sem <- struct{}{} defer func() { <-sem }() task = &entity.TExportTask{} if err := dao.TExportTask.Ctx(baseCtx). Where(dao.TExportTask.Columns().TaskNo, taskNo). Scan(task); err != nil || task.Id <= 0 { g.Log().Errorf(baseCtx, "[exportcenter] 加载任务失败 taskNo=%s err=%v", taskNo, err) return } // CAS 抢占:只有把 0 改成 2 的协程才能执行 now := gtime.Now() affected, err := dao.TExportTask.Ctx(baseCtx). Data(g.Map{ dao.TExportTask.Columns().Status: consts.DownloadStatusRunning, dao.TExportTask.Columns().StartTime: now, }). Where(dao.TExportTask.Columns().Id, task.Id). WhereNull(dao.TExportTask.Columns().DeletedTime). Where(dao.TExportTask.Columns().Status, consts.DownloadStatusPending). UpdateAndGetAffected() if err != nil { g.Log().Errorf(baseCtx, "[exportcenter] 抢占任务失败 taskNo=%s err=%v", taskNo, err) return } if affected == 0 { return } execCtx, cancel := context.WithTimeout(restoreContext(baseCtx, task), TaskTimeout) defer cancel() markBatchRunning(execCtx, taskNo) if err = generate(execCtx, task, req, &generatedPath); err != nil { g.Log().Errorf(execCtx, "[exportcenter] 导出失败 taskNo=%s err=%v", taskNo, err) markFailed(baseCtx, task, err) } } // restoreContext 用提交时的身份快照还原 ctx,保证 FilterAuth 仍按提交人范围过滤。 func restoreContext(base context.Context, task *entity.TExportTask) context.Context { snap := &model.ExportIdentitySnapshot{} if task.IdentitySnapshot != nil { if err := task.IdentitySnapshot.Scan(snap); err != nil { g.Log().Warningf(base, "[exportcenter] 身份快照解析失败 taskNo=%s err=%v", task.TaskNo, err) } } if snap.UserId <= 0 { snap.UserId = task.OwnerUserId snap.DeptId = task.DeptId snap.TenantId = task.TenantId } identity := &model.Identity{} identity.ID = snap.UserId identity.DeptId = snap.DeptId identity.TenantId = snap.TenantId identity.Roles = snap.Roles identity.AccessCodeList = snap.AccessCodeList data := g.Map{} if snap.CurrentRoleId != "" { data["currentRoleId"] = snap.CurrentRoleId } if snap.CurrentRoleDataScope != "" { data["currentRoleDataScope"] = snap.CurrentRoleDataScope } return contexts.Set(base, &model.Context{User: identity, Data: data}) } func generate(ctx context.Context, task *entity.TExportTask, req *Request, generatedPath *string) (err error) { header, fields, err := resolveColumns(req) if err != nil { return gerror.Wrap(err, "生成表头失败") } dicts, err := loadColumnDicts(ctx, req.Columns, func(ctx context.Context, types []string) ([]model.SysDictDataListModel, error) { return service.SysDictData().GetDictListByTypes(ctx, types) }) if err != nil { return gerror.Wrap(err, "加载导出字典失败") } sheet := req.SheetName if sheet == "" { sheet = req.BizName } sheet = truncateSheetName(sheet) f := excelize.NewFile() defer f.Close() if err = f.SetSheetName(f.GetSheetName(0), sheet); err != nil { return gerror.Wrap(err, "设置工作表名称失败") } stream, err := f.NewStreamWriter(sheet) if err != nil { return gerror.Wrap(err, "创建工作表流失败") } if err = stream.SetRow("A1", header); err != nil { return gerror.Wrap(err, "写入表头失败") } rowNum := 2 written := 0 for page := 1; ; page++ { rows, total, fetchErr := req.Fetch(ctx, page, DefaultPageSize) if fetchErr != nil { // 页码只留给日志排查;对用户展示业务侧给出的可读原因,不暴露分页与驱动细节。 g.Log().Errorf(ctx, "[exportcenter] 第 %d 页取数失败 taskNo=%s err=%v", page, task.TaskNo, fetchErr) return fetchErr } if len(rows) == 0 { break } for _, v := range rows { cell, _ := excelize.CoordinatesToCellName(1, rowNum) values := rowValues(v, fields, req) translateRow(values, req.Columns, dicts) if err = stream.SetRow(cell, values); err != nil { return gerror.Wrapf(err, "写入第 %d 行失败", rowNum) } rowNum++ written++ if written > MaxRows { return gerror.Newf("导出行数超过上限 %d 行", MaxRows) } } // 已取满或不足一页,结束 if len(rows) < DefaultPageSize || (total > 0 && written >= total) { break } } if err = stream.Flush(); err != nil { return gerror.Wrap(err, "刷新工作表失败") } content, err := toBytes(f) if err != nil { return err } if int64(len(content)) > MaxFileSize { return gerror.Newf("导出文件超过 %d MB 上限,请缩小筛选范围", MaxFileSize>>20) } oss, err := savePrivateExport(ctx, task, content) if err != nil { return gerror.Wrap(err, "保存导出文件失败") } *generatedPath = oss.Path task.OssId = oss.OssId if err = markSuccess(ctx, task, oss, written, req); err != nil { return err } *generatedPath = "" return nil } func savePrivateExport(ctx context.Context, task *entity.TExportTask, content []byte) (*model.SysOssSaveContentModel, error) { uploadRoot := strings.TrimSpace(g.Cfg().MustGet(ctx, "system.upload.local.path").String()) if uploadRoot == "" { return nil, gerror.New("上传路径未配置") } root, err := filepath.Abs(uploadRoot) if err != nil { return nil, err } // 私有目录必须与上传静态根目录隔离,不能生成可访问的 OSS URL。 path := filepath.Join(filepath.Dir(root), "private-export", task.TenantId, task.TaskNo+".xlsx") if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { return nil, err } registered := false defer func() { if !registered { if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) { g.Log().Errorf(gctx.New(), "[exportcenter] 清理未登记文件失败 path=%s err=%v", path, removeErr) } } }() if err = os.WriteFile(path, content, 0600); err != nil { return nil, err } md5, err := gmd5.EncryptFile(path) if err != nil { return nil, err } var ossId int64 // 这里仅登记元数据;通用 OSS API 拒绝导出文件,实际下载走下载中心。 err = g.DB().Transaction(gctx.New(), func(ctx context.Context, tx gdb.TX) error { var insertErr error ossId, insertErr = dao.SysOss.Ctx(ctx).TX(tx).Data(g.Map{ dao.SysOss.Columns().TenantId: task.TenantId, dao.SysOss.Columns().FileName: task.FileName, dao.SysOss.Columns().OriginalName: task.FileName, dao.SysOss.Columns().FileSuffix: "xlsx", dao.SysOss.Columns().Path: path, dao.SysOss.Columns().Url: "", dao.SysOss.Columns().CreatedDept: task.DeptId, dao.SysOss.Columns().CreatedBy: task.OwnerUserId, dao.SysOss.Columns().CreatedAt: gtime.Now(), dao.SysOss.Columns().Service: "local", dao.SysOss.Columns().Md5: md5, dao.SysOss.Columns().FileSize: len(content), dao.SysOss.Columns().FileCrc16: hash.Crc16ChecksumFile(path), dao.SysOss.Columns().FileSum: hash.Sum16ChecksumFile(path), }).InsertAndGetId() if insertErr != nil { return insertErr } affected, updateErr := dao.TExportTask.Ctx(ctx).TX(tx). Data(g.Map{dao.TExportTask.Columns().OssId: ossId}). Where(dao.TExportTask.Columns().Id, task.Id). Where(dao.TExportTask.Columns().Status, consts.DownloadStatusRunning).UpdateAndGetAffected() if updateErr != nil { return updateErr } if affected != 1 { return gerror.New("登记导出文件失败:任务状态已变更") } return nil }) if err != nil { return nil, err } registered = true task.OssId = ossId return &model.SysOssSaveContentModel{SysOss: &entity.SysOss{OssId: ossId, Path: path, FileSize: len(content), Md5: md5}}, nil } func structValues(v interface{}) []interface{} { rv := reflect.ValueOf(v) for rv.Kind() == reflect.Ptr { if rv.IsNil() { return nil } rv = rv.Elem() } if rv.Kind() != reflect.Struct { return []interface{}{v} } fields := rv.NumField() out := make([]interface{}, fields) for i := 0; i < fields; i++ { out[i] = rv.Field(i).Interface() } return out } func toBytes(f *excelize.File) ([]byte, error) { tmp, err := os.CreateTemp("", "iotlcm-export-*.xlsx") if err != nil { return nil, gerror.Wrap(err, "创建临时文件失败") } name := tmp.Name() defer os.Remove(name) defer tmp.Close() if err = f.Write(tmp); err != nil { return nil, gerror.Wrap(err, "写入临时文件失败") } if err = tmp.Sync(); err != nil { return nil, gerror.Wrap(err, "刷新临时文件失败") } content, err := os.ReadFile(name) if err != nil { return nil, gerror.Wrap(err, "读取临时文件失败") } return content, nil } func truncateSheetName(name string) string { if name == "" { return "Sheet1" } rs := []rune(name) if len(rs) > 31 { rs = rs[:31] } return string(rs) } func markBatchRunning(ctx context.Context, taskNo string) { _, err := dao.BatchMain.Ctx(ctx). Data(g.Map{ dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusProcessing, dao.BatchMain.Columns().RunResult: consts.BatchRunResultRunning, dao.BatchMain.Columns().StartTime: gtime.Now(), }). Where(dao.BatchMain.Columns().BatchNo, taskNo). Update() if err != nil { g.Log().Warningf(ctx, "[exportcenter] 更新执行日志开始状态失败 taskNo=%s err=%v", taskNo, err) } } func markSuccess(ctx context.Context, task *entity.TExportTask, oss *model.SysOssSaveContentModel, written int, req *Request) error { now := gtime.Now() // 默认按全部成功回填;查询类导出可通过 Request.Result 改判为部分成功/失败并写汇总说明。 // 注意:即使判定为失败,文件本身仍已生成并可下载(文件里写明每条失败原因), // 因此这里只改执行日志的 async_status/run_result,不改 t_export_task 的下载状态。 runResult := consts.BatchRunResultSuccess asyncStatus := consts.BatchAsyncStatusSuccess summary := "" if req != nil && req.Result != nil { if result, text := req.Result(written); result > 0 { runResult = result summary = text switch result { case consts.BatchRunResultPartialSuccess: asyncStatus = consts.BatchAsyncStatusPartialSuccess summary = text case consts.BatchRunResultFailed: asyncStatus = consts.BatchAsyncStatusFailed summary = text default: asyncStatus = consts.BatchAsyncStatusSuccess } } } // 文件已写磁盘,两张表必须同一事务提交;失败保留 OSS 索引供过期清理重试。 return g.DB().Transaction(gctx.New(), func(ctx context.Context, tx gdb.TX) error { affected, err := dao.TExportTask.Ctx(ctx).TX(tx). Data(g.Map{ dao.TExportTask.Columns().Status: consts.DownloadStatusGenerated, dao.TExportTask.Columns().OssId: oss.OssId, dao.TExportTask.Columns().FileSize: oss.FileSize, dao.TExportTask.Columns().FileMd5: oss.Md5, dao.TExportTask.Columns().TotalCount: written, dao.TExportTask.Columns().FinishTime: now, dao.TExportTask.Columns().ExpireTime: now.AddDate(0, 0, ExpireDays), }). Where(dao.TExportTask.Columns().Id, task.Id). Where(dao.TExportTask.Columns().Status, consts.DownloadStatusRunning). UpdateAndGetAffected() if err != nil { return gerror.Wrap(err, "回填导出任务失败") } if affected != 1 { return gerror.New("回填导出任务失败:任务状态已变更") } affected, err = dao.BatchMain.Ctx(ctx).TX(tx). Data(g.Map{ dao.BatchMain.Columns().BatchAsyncStatus: asyncStatus, dao.BatchMain.Columns().RunResult: runResult, dao.BatchMain.Columns().FailReason: summary, dao.BatchMain.Columns().ResultOssId: oss.OssId, dao.BatchMain.Columns().TotalCount: written, dao.BatchMain.Columns().EndTime: now, }). Where(dao.BatchMain.Columns().BatchNo, task.TaskNo). UpdateAndGetAffected() if err != nil { return gerror.Wrap(err, "回填执行日志失败") } if affected != 1 { return gerror.New("回填执行日志失败:记录不存在") } return nil }) } func markFailed(ctx context.Context, task *entity.TExportTask, cause error) { // 失败也要落库,用独立 ctx 避免任务 ctx 已超时 ctx = gctx.New() msg := cause.Error() if len([]rune(msg)) > 1900 { msg = string([]rune(msg)[:1900]) } now := gtime.Now() affected, err := dao.TExportTask.Ctx(ctx). Data(g.Map{ dao.TExportTask.Columns().Status: consts.DownloadStatusFailed, dao.TExportTask.Columns().ErrorMessage: msg, dao.TExportTask.Columns().FinishTime: now, }). Where(dao.TExportTask.Columns().Id, task.Id). WhereNull(dao.TExportTask.Columns().DeletedTime). WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusPending, consts.DownloadStatusRunning}). UpdateAndGetAffected() if err != nil { g.Log().Errorf(ctx, "[exportcenter] 回填失败状态失败 taskNo=%s err=%v", task.TaskNo, err) return } if affected == 0 { current := &entity.TExportTask{} if err := dao.TExportTask.Ctx(ctx).Where(dao.TExportTask.Columns().Id, task.Id).Scan(current); err != nil { g.Log().Errorf(ctx, "[exportcenter] 核对失败状态失败 taskNo=%s err=%v", task.TaskNo, err) return } if current.Id == 0 || current.Status != consts.DownloadStatusFailed { return } } _, err = dao.BatchMain.Ctx(ctx). Data(g.Map{ dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusFailed, dao.BatchMain.Columns().RunResult: consts.BatchRunResultFailed, dao.BatchMain.Columns().FailReason: msg, dao.BatchMain.Columns().EndTime: now, }). Where(dao.BatchMain.Columns().BatchNo, task.TaskNo). Update() if err != nil { g.Log().Errorf(ctx, "[exportcenter] 回填执行日志失败状态失败 taskNo=%s err=%v", task.TaskNo, err) } } // RecoverOnStart 进程启动时把残留的「排队中/生成中」任务标记为失败。 // 进程内任务不跨重启恢复,失败后用户可在下载中心点重试重新提交。 func RecoverOnStart(ctx context.Context) { now := time.Now() _, err := dao.TExportTask.Ctx(ctx). Data(g.Map{ dao.TExportTask.Columns().Status: consts.DownloadStatusFailed, dao.TExportTask.Columns().ErrorMessage: "服务重启,任务已中断,请重新导出", dao.TExportTask.Columns().FinishTime: now, }). WhereNull(dao.TExportTask.Columns().DeletedTime). WhereIn(dao.TExportTask.Columns().Status, []int{consts.DownloadStatusPending, consts.DownloadStatusRunning}). Update() if err != nil { g.Log().Errorf(ctx, "[exportcenter] 启动补偿失败: %v", err) return } _, err = dao.BatchMain.Ctx(ctx). Data(g.Map{ dao.BatchMain.Columns().BatchAsyncStatus: consts.BatchAsyncStatusFailed, dao.BatchMain.Columns().RunResult: consts.BatchRunResultFailed, dao.BatchMain.Columns().FailReason: "服务重启,任务已中断", dao.BatchMain.Columns().EndTime: now, }). Where(dao.BatchMain.Columns().BatchType, consts.BatchTypeExport). WhereIn(dao.BatchMain.Columns().BatchAsyncStatus, []int{consts.BatchAsyncStatusCreated, consts.BatchAsyncStatusProcessing}). Update() if err != nil { g.Log().Errorf(ctx, "[exportcenter] 启动补偿执行日志失败: %v", err) } }