iotlcm-public/iotlcm
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: 初始开源发布

Browse Source 
iotlcm-app：小程序端 + 用户端服务。

- app/：uni-app 小程序前端（基于 unibest，MIT）
  - 微信 AppID → YOUR_WX_APPID，uni-app AppID → __UNI__YOURAPPID
  - 内网后端地址 → admin.iotlcm.example.com
  - 应用标题与 unibest 模板 README 重写（保留上游致谢与 MIT 声明）
- server/：GoFrame 用户端服务（基于 XiuAdmin，Apache-2.0）
  - module xiuadmin → git.5iot.com/iotlcm-public/iotlcm-app（360 文件）
  - 支付回调地址 → app.iotlcm.example.com，商户名与应用名泛化
  - 真实手机号 → 合法 11 位编造号（保留号段前 3 位 + 后 4 位）
  - 依赖号码尾部的断言同步更新
  - 补 server/LICENSE（Apache-2.0）
- 移除内部资产：CI 触发器
- 补根 README，LF 行尾 + .gitattributes

已知：TestSysDept_RefreshDeptAncestors 在源仓同一提交上同样失败，属既有缺陷。
This commit is contained in:
MiFaZhan
2026-10-26 14:01:25 +08:00
commit 2ed5dfa6a1
1013 changed files with 138466 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/library/batchimport/async.go
+168
View File
@@ -0,0 +1,168 @@
package batchimport
import (
"context"
"strings"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/service"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
type PendingOptions struct {
PrepareRows func(ctx context.Context, rows []Row) error
}
func CreatePendingUpload(ctx context.Context, req SubmitRequest, p Processor, options PendingOptions) (*Result, error) {
cfg := p.Config()
if req.File == nil {
return nil, gerror.New("请选择上传文件")
}
if err := service.SysOss().CheckType(ctx, consts.SysUploadFileTypeFile, req.File); err != nil {
return nil, err
}
if req.File.Size > cfg.MaxFileSize {
return nil, gerror.Newf("上传文件大小超过限制，最大为%dM", cfg.MaxFileSize>>20)
}
batchNo, err := NextBatchNoContext(ctx, cfg.BatchNoPrefix)
if err != nil {
return nil, err
}
req.BatchNo = batchNo
content, err := readUploadContent(req.File)
if err != nil {
return nil, err
}
importOss, err := service.SysOss().SaveContent(ctx, &model.SysOssSaveContentParam{
Content: content,
FileName: batchNo + "-import.xlsx",
NewFileType: consts.SysUploadNewFileTypeUserFile,
SubDirName: cfg.ImportDir,
})
if err != nil {
return nil, gerror.Wrap(err, "保存原始Excel失败")
}
rows, err := p.ParseRows(ctx, importOss.Path)
if err != nil {
return nil, err
}
return createPending(ctx, &req, p, rows, importOss, options)
}
func CreatePendingRows(ctx context.Context, req SubmitRequest, p Processor, rows []Row, options PendingOptions) (*Result, error) {
batchNo, err := NextBatchNoContext(ctx, p.Config().BatchNoPrefix)
if err != nil {
return nil, err
}
req.BatchNo = batchNo
return createPending(ctx, &req, p, rows, nil, options)
}
func createPending(ctx context.Context, req *SubmitRequest, p Processor, rows []Row, importOss *model.SysOssSaveContentModel, options PendingOptions) (*Result, error) {
if len(rows) == 0 {
return nil, gerror.New("导入数据为空")
}
if err := p.ValidateRows(ctx, rows); err != nil {
return nil, err
}
if options.PrepareRows != nil {
if err := options.PrepareRows(ctx, rows); err != nil {
return nil, err
}
}
result := &Result{
ImportStatus: consts.ImportStatusSuccess,
BatchNo: req.BatchNo,
BatchType: p.Config().BatchType,
TotalRows: len(rows),
}
if importOss != nil {
result.ImportFileOssId = importOss.OssId
result.ImportFileUrl = importOss.Url
}
if err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
sourceBatchNo := strings.TrimSpace(req.SourceBatchNo)
if sourceBatchNo == "" {
sourceBatchNo = req.BatchNo
}
if _, err := dao.BatchMain.Ctx(ctx).TX(tx).Data(do.BatchMain{
BatchNo: req.BatchNo,
OrderSn: "",
BatchType: p.Config().BatchType,
BatchStatus: consts.BatchAsyncStatusPending,
BatchAsyncStatus: consts.BatchAsyncStatusPending,
SourceType: req.SourceType,
SourceBatchNo: sourceBatchNo,
IsCrossDept: 0,
TargetDeptId: req.TargetDeptId,
DeptId: req.Operator.DeptId,
TotalCount: len(rows),
ValidCount: 0,
SuccessCount: 0,
FailCount: 0,
RunResult: consts.BatchRunResultPending,
FailReason: "",
BatchInfo: p.BuildBatchInfo(ctx, *req, result, importOss, nil, 0),
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
UpdatedBy: req.Operator.UserId,
}).Insert(); err != nil {
return gerror.Wrap(err, "写入待处理批次失败")
}
details := make([]do.BatchDetail, 0, len(rows))
for _, row := range rows {
details = append(details, do.BatchDetail{
BatchNo: req.BatchNo,
BatchType: p.Config().BatchType,
DetailStatus: consts.BatchDetailStatusPending,
BlankOutStatus: consts.BlankOutStatusNo,
RowNo: row.GetRowNo(),
ItemId: row.GetItemID(),
ErrorCode: "",
ErrorReason: strings.Join(row.GetErrorReasons(), "；"),
Snapshot: row.Snapshot(),
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
UpdatedBy: req.Operator.UserId,
})
}
_, err := dao.BatchDetail.Ctx(ctx).TX(tx).Data(details).Insert()
return gerror.Wrap(err, "写入待处理批次明细失败")
}); err != nil {
return nil, err
}
return result, nil
}
func SaveAsyncResultFiles(
ctx context.Context,
req SubmitRequest,
p Processor,
result *Result,
rows []Row,
importOss *model.SysOssSaveContentModel,
startTime, endTime *gtime.Time,
) (*model.SysOssSaveContentModel, int64, error) {
resultOss, downloadId, err := saveResultExcelAndDownload(ctx, req, p, result, rows, importOss, startTime, endTime)
if err != nil {
return nil, 0, err
}
result.ResultDownloadId = downloadId
result.ResultFileOssId = resultOss.OssId
result.ResultFileUrl = resultOss.Url
return resultOss, downloadId, nil
}
server/internal/library/batchimport/batch_no.go
+117
View File
@@ -0,0 +1,117 @@
package batchimport
import (
"context"
"crypto/rand"
"fmt"
"strings"
"time"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
)
const (
batchNoPrefixLength = 3
batchNoSequenceLength = 3
batchNoSequenceMax = 999
batchNoSequenceTTL = 48 * time.Hour
batchNoSequenceRedisPrefix = "iotlcm:batch:no:seq:"
randomNoSuffixLength = 8
randomNoAlphabet = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"
)
func NextBatchNo(prefix string) (string, error) {
return NextBatchNoContext(context.Background(), prefix)
}
func NextBatchNoContext(ctx context.Context, prefix string) (string, error) {
prefix, err := normalizeBatchNoPrefix(prefix)
if err != nil {
return "", err
}
if ctx == nil {
ctx = context.Background()
}
now := time.Now()
seq, err := nextDailyBatchSequence(ctx, now)
if err != nil {
return "", err
}
return formatBatchNo(prefix, now, seq)
}
func NextRandomNo(prefix string) (string, error) {
prefix = strings.TrimSpace(strings.ToUpper(prefix))
if prefix == "" {
return "", gerror.New("编号前缀不能为空")
}
random, err := randomNoSuffix(randomNoSuffixLength)
if err != nil {
return "", err
}
return fmt.Sprintf("%s%s%s", prefix, time.Now().Format("20060102150405"), random), nil
}
func normalizeBatchNoPrefix(prefix string) (string, error) {
prefix = strings.TrimSpace(strings.ToUpper(prefix))
if prefix == "" {
return "", gerror.New("批次号前缀不能为空")
}
if len(prefix) != batchNoPrefixLength {
return "", gerror.Newf("批次号前缀必须为%d位", batchNoPrefixLength)
}
return prefix, nil
}
func nextDailyBatchSequence(ctx context.Context, now time.Time) (int64, error) {
key := batchNoSequenceKey(now)
script := `
local v = redis.call("INCR", KEYS[1])
if v == 1 then
redis.call("EXPIRE", KEYS[1], ARGV[1])
end
return v`
res, err := g.Redis().Do(ctx, "EVAL", script, 1, key, int(batchNoSequenceTTL.Seconds()))
if err != nil {
return 0, gerror.Wrap(err, "生成批次号序号失败")
}
seq := res.Int64()
if seq <= 0 {
return 0, gerror.New("生成批次号序号失败")
}
if seq > batchNoSequenceMax {
return 0, gerror.Newf("当日批次号序号超过%03d", batchNoSequenceMax)
}
return seq, nil
}
func batchNoSequenceKey(now time.Time) string {
return batchNoSequenceRedisPrefix + now.Format("20060102")
}
func formatBatchNo(prefix string, now time.Time, seq int64) (string, error) {
if seq <= 0 || seq > batchNoSequenceMax {
return "", gerror.Newf("批次号序号必须在1-%03d之间", batchNoSequenceMax)
}
return fmt.Sprintf("%s%s%0*d", prefix, now.Format("20060102150405"), batchNoSequenceLength, seq), nil
}
func randomNoSuffix(length int) (string, error) {
if length <= 0 {
return "", gerror.New("随机码长度必须大于0")
}
buf := make([]byte, length)
randomBytes := make([]byte, length)
if _, err := rand.Read(randomBytes); err != nil {
return "", gerror.Wrap(err, "生成随机码失败")
}
alphabetLen := byte(len(randomNoAlphabet))
for i, b := range randomBytes {
buf[i] = randomNoAlphabet[b%alphabetLen]
}
return string(buf), nil
}
server/internal/library/batchimport/batch_no_test.go
+45
View File
@@ -0,0 +1,45 @@
package batchimport
import (
"strings"
"testing"
"time"
)
func TestNormalizeBatchNoPrefix(t *testing.T) {
prefix, err := normalizeBatchNoPrefix(" inb ")
if err != nil {
t.Fatalf("normalize prefix: %v", err)
}
if prefix != "INB" {
t.Fatalf("unexpected prefix: %s", prefix)
}
if _, err = normalizeBatchNoPrefix("IN"); err == nil {
t.Fatal("expected error for non-three-character prefix")
}
}
func TestFormatBatchNo(t *testing.T) {
now := time.Date(2026, 8, 10, 12, 34, 56, 789000000, time.Local)
batchNo, err := formatBatchNo("REN", now, 7)
if err != nil {
t.Fatalf("format batch no: %v", err)
}
if batchNo != "REN20260810123456007" {
t.Fatalf("unexpected batch no: %s", batchNo)
}
if len(batchNo) != 20 {
t.Fatalf("unexpected batch no length: %d", len(batchNo))
}
if strings.Contains(batchNo, "789") {
t.Fatalf("batch no should not include milliseconds: %s", batchNo)
}
}
func TestFormatBatchNoRejectsOverflowSequence(t *testing.T) {
_, err := formatBatchNo("REN", time.Now(), 1000)
if err == nil {
t.Fatal("expected overflow sequence error")
}
}
server/internal/library/batchimport/runner.go
+596
View File
@@ -0,0 +1,596 @@
package batchimport
import (
"context"
"fmt"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/consts"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/dao"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/model"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/model/do"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/model/entity"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/service"
"git.5iot.com/iotlcm-public/iotlcm-app/utility/hash"
"git.5iot.com/iotlcm-public/iotlcm-app/utility/iot_excelize"
"io"
"os"
"strings"
"github.com/gogf/gf/v2/crypto/gmd5"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gtime"
"github.com/xuri/excelize/v2"
)
func SubmitAndRun(ctx context.Context, req SubmitRequest, p Processor) (*Result, error) {
cfg := p.Config()
if req.File == nil {
return nil, gerror.New("请选择上传文件")
}
if err := service.SysOss().CheckType(ctx, consts.SysUploadFileTypeFile, req.File); err != nil {
return nil, err
}
if req.File.Size > cfg.MaxFileSize {
return nil, gerror.Newf("上传文件大小超过限制，最大为%dM", cfg.MaxFileSize>>20)
}
batchNo, err := NextBatchNoContext(ctx, cfg.BatchNoPrefix)
if err != nil {
return nil, err
}
req.BatchNo = batchNo
result := &Result{
ImportStatus: consts.ImportStatusFailed,
BatchNo: batchNo,
}
content, err := readUploadContent(req.File)
if err != nil {
return nil, err
}
importOss, err := service.SysOss().SaveContent(ctx, &model.SysOssSaveContentParam{
Content: content,
FileName: batchNo + "-import.xlsx",
NewFileType: consts.SysUploadNewFileTypeUserFile,
SubDirName: cfg.ImportDir,
NotAddDate: 0,
})
if err != nil {
return nil, gerror.Wrap(err, "保存原始Excel失败")
}
result.ImportFileOssId = importOss.OssId
result.ImportFileUrl = importOss.Url
rows, err := p.ParseRows(ctx, importOss.Path)
if err != nil {
result.FailReason = userFacingFailReason(err)
if err = archiveFailedBatchAfterImport(ctx, req, p, result, nil, importOss); err != nil {
return nil, err
}
return result, nil
}
result.TotalRows = len(rows)
if err = p.ValidateRows(ctx, rows); err != nil {
for _, row := range rows {
row.AddErrorReason(userFacingFailReason(err))
}
}
fillResultCounts(result, rows)
startTime := gtime.Now()
endTime := gtime.Now()
if err = saveBatchAndExecute(ctx, &req, p, result, rows, importOss, startTime, endTime); err != nil {
markRowsFailed(result, rows, userFacingFailReason(err))
if err = saveBatchAndExecute(ctx, &req, p, result, rows, importOss, startTime, endTime); err != nil {
return nil, err
}
}
result.ExecutionResult = req.ExecutionResult
resultOss, downloadId, err := saveResultExcelAndDownload(ctx, req, p, result, rows, importOss, startTime, endTime)
if err != nil {
return nil, err
}
result.ResultDownloadId = downloadId
result.ResultFileOssId = resultOss.OssId
result.ResultFileUrl = resultOss.Url
if err = updateBatchResultFileInfo(ctx, req, p, result, importOss, resultOss, downloadId); err != nil {
return nil, err
}
return result, nil
}
func RunRows(ctx context.Context, req SubmitRequest, p Processor, rows []Row, sourceName string) (*Result, error) {
cfg := p.Config()
batchNo, err := NextBatchNoContext(ctx, cfg.BatchNoPrefix)
if err != nil {
return nil, err
}
req.BatchNo = batchNo
result := &Result{
ImportStatus: consts.ImportStatusFailed,
BatchNo: batchNo,
TotalRows: len(rows),
}
if len(rows) == 0 {
result.FailReason = "导入数据为空"
} else if err = p.ValidateRows(ctx, rows); err != nil {
for _, row := range rows {
row.AddErrorReason(userFacingFailReason(err))
}
}
fillResultCounts(result, rows)
if result.FailReason != "" {
result.ImportStatus = consts.ImportStatusFailed
result.InsertedRows = 0
}
startTime := gtime.Now()
endTime := gtime.Now()
importOss := &model.SysOssSaveContentModel{SysOss: &entity.SysOss{}}
if sourceName != "" {
importOss.OriginalName = sourceName
}
if err = saveBatchAndExecute(ctx, &req, p, result, rows, importOss, startTime, endTime); err != nil {
markRowsFailed(result, rows, userFacingFailReason(err))
if err = saveBatchAndExecute(ctx, &req, p, result, rows, importOss, startTime, endTime); err != nil {
return nil, err
}
}
result.ExecutionResult = req.ExecutionResult
resultOss, downloadId, err := saveResultExcelAndDownload(ctx, req, p, result, rows, importOss, startTime, endTime)
if err != nil {
return nil, err
}
result.ResultDownloadId = downloadId
result.ResultFileOssId = resultOss.OssId
result.ResultFileUrl = resultOss.Url
if err = updateBatchResultFileInfo(ctx, req, p, result, importOss, resultOss, downloadId); err != nil {
return nil, err
}
return result, nil
}
func archiveFailedBatchAfterImport(ctx context.Context, req SubmitRequest, p Processor, result *Result, rows []Row, importOss *model.SysOssSaveContentModel) error {
result.ImportStatus = consts.ImportStatusFailed
result.InsertedRows = 0
if result.FailReason == "" {
result.FailReason = "导入失败"
}
startTime := gtime.Now()
endTime := gtime.Now()
if err := saveBatchAndExecute(ctx, &req, p, result, rows, importOss, startTime, endTime); err != nil {
return err
}
result.ExecutionResult = req.ExecutionResult
resultOss, downloadId, err := saveResultExcelAndDownload(ctx, req, p, result, rows, importOss, startTime, endTime)
if err != nil {
return err
}
result.ResultDownloadId = downloadId
result.ResultFileOssId = resultOss.OssId
result.ResultFileUrl = resultOss.Url
return updateBatchResultFileInfo(ctx, req, p, result, importOss, resultOss, downloadId)
}
func readUploadContent(file *ghttp.UploadFile) ([]byte, error) {
reader, err := file.Open()
if err != nil {
return nil, gerror.Wrap(err, "打开上传文件失败")
}
defer reader.Close()
content, err := io.ReadAll(reader)
if err != nil {
return nil, gerror.Wrap(err, "读取上传文件失败")
}
return content, nil
}
func fillResultCounts(result *Result, rows []Row) {
failCount := 0
for _, row := range rows {
if len(row.GetErrorReasons()) > 0 {
failCount++
}
}
result.ErrorRows = failCount
result.ValidRows = result.TotalRows - failCount
if result.ValidRows < 0 {
result.ValidRows = 0
}
if failCount == 0 {
result.ImportStatus = consts.ImportStatusSuccess
result.InsertedRows = len(rows)
return
}
result.ImportStatus = consts.ImportStatusFailed
result.InsertedRows = 0
}
func markRowsFailed(result *Result, rows []Row, reason string) {
if result == nil {
return
}
if reason == "" {
reason = "业务处理失败"
}
for _, row := range rows {
row.AddErrorReason(reason)
}
result.ImportStatus = consts.ImportStatusFailed
result.FailReason = reason
result.TotalRows = len(rows)
result.ValidRows = 0
result.ErrorRows = len(rows)
result.InsertedRows = 0
}
// saveBatchAndExecute writes stock data only when all rows pass validation.
// Failed validation keeps the batch and result file, but skips business inserts.
func saveBatchAndExecute(ctx context.Context, req *SubmitRequest, p Processor, result *Result, rows []Row, importOss *model.SysOssSaveContentModel, startTime, endTime *gtime.Time) error {
return g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
now := gtime.Now()
failReason := result.FailReason
if failReason == "" && result.ErrorRows > 0 {
failReason = "存在校验失败数据，本次未写入"
}
batchType := batchTypeByImportStatus(req, p, result.ImportStatus)
result.BatchType = batchType
if result.ImportStatus == consts.ImportStatusSuccess && len(rows) > 0 && req.BeforeSave != nil {
if err := req.BeforeSave(ctx, tx, req, rows); err != nil {
return err
}
}
sourceBatchNo := req.SourceBatchNo
if sourceBatchNo == "" {
sourceBatchNo = result.BatchNo
}
if _, err := dao.BatchMain.Ctx(ctx).TX(tx).Data(do.BatchMain{
BatchNo: result.BatchNo,
OrderSn: "",
BatchType: batchType,
BatchStatus: batchStatusByImportStatus(result.ImportStatus),
BatchAsyncStatus: batchStatusByImportStatus(result.ImportStatus),
SourceType: req.SourceType,
SourceBatchNo: sourceBatchNo,
IsCrossDept: 0,
TargetDeptId: req.TargetDeptId,
DeptId: req.Operator.DeptId,
TotalCount: result.TotalRows,
ValidCount: result.ValidRows,
SuccessCount: result.InsertedRows,
FailCount: result.ErrorRows,
RunResult: runResultByImportStatus(result.ImportStatus),
FailReason: failReason,
BatchInfo: p.BuildBatchInfo(ctx, *req, result, importOss, nil, 0),
StartTime: startTime,
EndTime: endTime,
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
CreatedAt: now,
UpdatedBy: req.Operator.UserId,
UpdatedAt: now,
}).Insert(); err != nil {
return gerror.Wrap(err, "写入批次主表失败")
}
details := make([]do.BatchDetail, 0, len(rows))
for _, row := range rows {
details = append(details, do.BatchDetail{
BatchNo: result.BatchNo,
BatchType: batchType,
DetailStatus: detailStatus(row),
BlankOutStatus: consts.BlankOutStatusNo,
RowNo: row.GetRowNo(),
ItemId: row.GetItemID(),
ErrorCode: "",
ErrorReason: strings.Join(row.GetErrorReasons(), "；"),
Snapshot: row.Snapshot(),
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
CreatedAt: now,
UpdatedBy: req.Operator.UserId,
UpdatedAt: now,
})
}
if len(details) > 0 {
if _, err := dao.BatchDetail.Ctx(ctx).TX(tx).Data(details).Insert(); err != nil {
return gerror.Wrap(err, "写入批次明细失败")
}
}
if result.ImportStatus == consts.ImportStatusSuccess && len(rows) > 0 && !req.SkipExecute {
if err := p.Execute(ctx, tx, req, rows); err != nil {
return err
}
}
return nil
})
}
func saveResultExcelAndDownload(ctx context.Context, req SubmitRequest, p Processor, result *Result, rows []Row, importOss *model.SysOssSaveContentModel, startTime, endTime *gtime.Time) (*model.SysOssSaveContentModel, int64, error) {
cfg := p.Config()
resultRows := make([]iot_excelize.BatchResultRow, 0, len(rows))
for _, row := range rows {
resultRows = append(resultRows, iot_excelize.BatchResultRow{
RowNo: row.GetRowNo(),
ItemID: row.GetItemID(),
Status: displayRowStatus(row),
ErrorReason: strings.Join(row.GetErrorReasons(), "；"),
})
}
importName := ""
importOssId := ""
if importOss != nil {
importName = importOss.OriginalName
if importOss.OssId > 0 {
importOssId = fmt.Sprintf("%d", importOss.OssId)
}
}
infos := []iot_excelize.BatchInfoItem{
{Field: "批次号", Value: result.BatchNo},
{Field: "批次类型", Value: taskTypeName(p)},
{Field: "处理结果", Value: displayImportStatus(result.ImportStatus)},
{Field: "总行数", Value: fmt.Sprintf("%d", result.TotalRows)},
{Field: "校验通过行数", Value: fmt.Sprintf("%d", result.ValidRows)},
{Field: "失败行数", Value: fmt.Sprintf("%d", result.ErrorRows)},
{Field: "实际写入行数", Value: fmt.Sprintf("%d", result.InsertedRows)},
{Field: "原始文件名", Value: importName},
{Field: "原始文件OSS ID", Value: importOssId},
{Field: "结果文件OSS ID", Value: ""},
{Field: "操作人", Value: fmt.Sprintf("%d", req.Operator.UserId)},
{Field: "开始时间", Value: formatGTime(startTime)},
{Field: "结束时间", Value: formatGTime(endTime)},
{Field: "备注", Value: resultRemark(result)},
}
f, err := iot_excelize.BuildBatchResultExcel(resultRows, infos)
if err != nil {
return nil, 0, err
}
defer f.Close()
reader, err := iot_excelize.ToReadSeeker(f)
if err != nil {
return nil, 0, err
}
content, err := io.ReadAll(reader)
if err != nil {
return nil, 0, err
}
resultOss, err := service.SysOss().SaveContent(ctx, &model.SysOssSaveContentParam{
Content: content,
FileName: result.BatchNo + "-result.xlsx",
NewFileType: consts.SysUploadNewFileTypeUserFile,
SubDirName: cfg.ResultDir,
NotAddDate: 0,
})
if err != nil {
return nil, 0, gerror.Wrap(err, "保存结果Excel失败")
}
if err = fillResultOssId(ctx, req, resultOss); err != nil {
return nil, 0, err
}
now := gtime.Now()
downloadId, err := dao.TResourceBatchDownload.Ctx(ctx).Data(do.TResourceBatchDownload{
BatchNo: result.BatchNo,
TaskName: cfg.TaskName,
ResourceType: consts.ResourceTypeExcel,
ExportScene: cfg.ExportScene,
TotalCount: result.TotalRows,
SuccessCount: result.InsertedRows,
FailCount: result.ErrorRows,
DownloadStatus: consts.DownloadStatusGenerated,
OssId: resultOss.OssId,
FileName: resultOss.FileName,
FileExt: resultOss.FileSuffix,
FileUrl: resultOss.Url,
FilePath: resultOss.Path,
FileSize: resultOss.FileSize,
FileMd5: resultOss.Md5,
DownloadCount: 0,
MaxDownloadCount: 0,
DeptId: req.Operator.DeptId,
StartTime: startTime,
FinishTime: now,
TenantId: req.Operator.TenantId,
CreatedDept: req.Operator.DeptId,
CreatedBy: req.Operator.UserId,
CreatedAt: now,
UpdatedBy: req.Operator.UserId,
UpdatedAt: now,
}).InsertAndGetId()
if err != nil {
return nil, 0, gerror.Wrap(err, "写入结果下载任务失败")
}
return resultOss, downloadId, nil
}
func fillResultOssId(ctx context.Context, req SubmitRequest, resultOss *model.SysOssSaveContentModel) error {
f, err := excelize.OpenFile(resultOss.Path)
if err != nil {
return gerror.Wrap(err, "打开结果Excel失败")
}
defer f.Close()
const batchInfoSheet = "批次信息"
rows, err := f.GetRows(batchInfoSheet)
if err != nil {
return gerror.Wrap(err, "读取结果Excel批次信息失败")
}
found := false
for i, row := range rows {
if len(row) == 0 || strings.TrimSpace(row[0]) != "结果文件OSS ID" {
continue
}
cell, err := excelize.CoordinatesToCellName(2, i+1)
if err != nil {
return err
}
if err = f.SetCellStr(batchInfoSheet, cell, fmt.Sprintf("%d", resultOss.OssId)); err != nil {
return err
}
found = true
break
}
if !found {
return gerror.New("结果Excel缺少结果文件OSS ID字段")
}
if err = f.SaveAs(resultOss.Path); err != nil {
return gerror.Wrap(err, "回写结果Excel OSS ID失败")
}
fileInfo, err := os.Stat(resultOss.Path)
if err != nil {
return gerror.Wrap(err, "读取结果Excel文件信息失败")
}
md5, _ := gmd5.EncryptFile(resultOss.Path)
crc16 := hash.Crc16ChecksumFile(resultOss.Path)
sum16 := hash.Sum16ChecksumFile(resultOss.Path)
resultOss.FileSize = int(fileInfo.Size())
resultOss.Md5 = md5
resultOss.FileCrc16 = int(crc16)
resultOss.FileSum = int(sum16)
_, err = dao.SysOss.Ctx(ctx).
Where(dao.SysOss.Columns().OssId, resultOss.OssId).
Data(do.SysOss{
Md5: resultOss.Md5,
FileSize: resultOss.FileSize,
FileCrc16: resultOss.FileCrc16,
FileSum: resultOss.FileSum,
UpdatedAt: gtime.Now(),
UpdatedBy: req.Operator.UserId,
}).
Update()
if err != nil {
return gerror.Wrap(err, "更新结果Excel文件信息失败")
}
return nil
}
func updateBatchResultFileInfo(ctx context.Context, req SubmitRequest, p Processor, result *Result, importOss *model.SysOssSaveContentModel, resultOss *model.SysOssSaveContentModel, downloadId int64) error {
_, err := dao.BatchMain.Ctx(ctx).
Where(dao.BatchMain.Columns().BatchNo, result.BatchNo).
Data(do.BatchMain{
BatchInfo: p.BuildBatchInfo(ctx, req, result, importOss, resultOss, downloadId),
UpdatedBy: req.Operator.UserId,
UpdatedAt: gtime.Now(),
}).
Update()
if err != nil {
return gerror.Wrap(err, "更新批次结果文件信息失败")
}
return nil
}
func batchStatusByImportStatus(importStatus string) int {
if importStatus == consts.ImportStatusSuccess {
return consts.BatchAsyncStatusSuccess
}
return consts.BatchAsyncStatusFailed
}
func runResultByImportStatus(importStatus string) int {
if importStatus == consts.ImportStatusSuccess {
return consts.BatchRunResultSuccess
}
return consts.BatchRunResultFailed
}
func batchTypeByImportStatus(req *SubmitRequest, p Processor, importStatus string) int {
if req == nil {
return p.Config().BatchType
}
if importStatus != consts.ImportStatusSuccess && req.FailureBatchType > 0 {
return req.FailureBatchType
}
if req.BatchType > 0 {
return req.BatchType
}
return p.Config().BatchType
}
func detailStatus(row Row) int {
if len(row.GetErrorReasons()) > 0 {
return consts.BatchDetailStatusFailed
}
return consts.BatchDetailStatusSuccess
}
func displayRowStatus(row Row) string {
if len(row.GetErrorReasons()) > 0 {
return "失败"
}
return "成功"
}
func displayImportStatus(status string) string {
if status == consts.ImportStatusSuccess {
return "成功"
}
return "失败"
}
func resultRemark(result *Result) string {
if result != nil && result.ImportStatus == consts.ImportStatusSuccess {
return "导入成功"
}
if result != nil && result.FailReason != "" {
return result.FailReason
}
return "存在校验失败数据，本次未写入"
}
func userFacingFailReason(err error) string {
if err == nil {
return ""
}
reason := strings.TrimSpace(err.Error())
if reason == "" {
return "导入失败"
}
const maxLen = 500
if len([]rune(reason)) <= maxLen {
return reason
}
runes := []rune(reason)
return string(runes[:maxLen])
}
func taskTypeName(p Processor) string {
taskName := p.Config().TaskName
return strings.TrimSuffix(taskName, "结果")
}
func formatGTime(t *gtime.Time) string {
if t == nil {
return ""
}
return t.Format("Y-m-d H:i:s")
}
server/internal/library/batchimport/types.go
+82
View File
@@ -0,0 +1,82 @@
package batchimport
import (
"context"
"git.5iot.com/iotlcm-public/iotlcm-app/internal/model"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/encoding/gjson"
"github.com/gogf/gf/v2/net/ghttp"
)
type SubmitRequest struct {
File *ghttp.UploadFile
BizParams any
SourceType int
SourceBatchNo string
TargetDeptId int64
Operator OperatorContext
BatchNo string
BatchType int
FailureBatchType int
SkipExecute bool
BeforeSave func(ctx context.Context, tx gdb.TX, req *SubmitRequest, rows []Row) error
ExecutionResult any
}
type OperatorContext struct {
TenantId string
DeptId int64
UserId int64
}
type Result struct {
ImportStatus string
BatchNo string
BatchType int
FailReason string
ImportFileOssId int64
ImportFileUrl string
TotalRows int
ValidRows int
ErrorRows int
InsertedRows int
ResultDownloadId int64
ResultFileOssId int64
ResultFileUrl string
ExecutionResult any
}
type Row interface {
GetRowNo() int
GetItemID() string
GetErrorReasons() []string
AddErrorReason(reason string)
Snapshot() *gjson.Json
}
type ProcessorConfig struct {
Code string
BatchType int
BatchNoPrefix string
TaskName string
ExportScene int
ImportDir string
ResultDir string
MaxFileSize int64
TemplateHeaders []string
RequiredHeaders []string
}
type Processor interface {
Config() ProcessorConfig
ParseRows(ctx context.Context, filePath string) ([]Row, error)
ValidateRows(ctx context.Context, rows []Row) error
Execute(ctx context.Context, tx gdb.TX, req *SubmitRequest, rows []Row) error
BuildBatchInfo(ctx context.Context, req SubmitRequest, result *Result, importOss *model.SysOssSaveContentModel, resultOss *model.SysOssSaveContentModel, downloadId int64) *gjson.Json
}
type Message struct {
BatchNo string `json:"batchNo"`
BatchType int `json:"batchType"`
}