chore: 并入 iotlcm-admin-svr 历史到 admin-svr/（原提交 49818c6）

2026-11-02 13:27:59 +08:00
parent 976980a3f5 49818c6dbe
commit 193f7393c0
1250 changed files with 154111 additions and 0 deletions
@@ -0,0 +1,248 @@
+package excel
+
+import (
+	"context"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/xuri/excelize/v2"
+)
+
+type ParseOptions struct {
+	RequiredHeaders []string
+	SingleSheet     bool
+	// MaxDataRows 限制单次导入的有效数据行数（不含表头、不含空行）。
+	// 0 表示不限制（向后兼容已有批量功能）。超过该值立即停止读取并返回错误，
+	// 避免超大 Excel 在解析阶段占用过多内存或解析完成后才报错。
+	MaxDataRows int
+}
+
+func ParseFile(ctx context.Context, filePath string, options ParseOptions) ([]ParsedRow, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer file.Close()
+	return ParseReader(ctx, file, options)
+}
+
+// StreamFile parses the first data sheet row by row and keeps only the current
+// row in memory.
+func StreamFile(ctx context.Context, filePath string, options ParseOptions, handler func(ParsedRow) error) (int, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return 0, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer file.Close()
+	return StreamReader(ctx, file, options, handler)
+}
+
+func StreamReader(ctx context.Context, reader io.Reader, options ParseOptions, handler func(ParsedRow) error) (int, error) {
+	if handler == nil {
+		return 0, gerror.New("Excel row handler cannot be nil")
+	}
+	f, err := excelize.OpenReader(reader)
+	if err != nil {
+		return 0, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer func() { _ = f.Close() }()
+
+	sheetList := f.GetSheetList()
+	if len(sheetList) == 0 {
+		return 0, gerror.New("Excel file has no worksheet")
+	}
+	if options.SingleSheet {
+		if err = ensureSingleDataSheet(f, sheetList); err != nil {
+			return 0, err
+		}
+	}
+
+	rows, err := f.Rows(sheetList[0])
+	if err != nil {
+		return 0, gerror.Wrap(err, "read Excel rows failed")
+	}
+	defer rows.Close()
+
+	if !rows.Next() {
+		if err = rows.Error(); err != nil {
+			return 0, gerror.Wrap(err, "read Excel header failed")
+		}
+		return 0, gerror.New("Excel file has no header")
+	}
+	headers, err := rows.Columns()
+	if err != nil {
+		return 0, gerror.Wrap(err, "read Excel header failed")
+	}
+	colIndex, err := buildColumnIndex(headers)
+	if err != nil {
+		return 0, err
+	}
+	for _, col := range options.RequiredHeaders {
+		if _, ok := colIndex[normalizeHeader(col)]; !ok {
+			return 0, gerror.Newf("Excel file is missing required column: %s", col)
+		}
+	}
+
+	rowNo := 1
+	parsedCount := 0
+	for rows.Next() {
+		rowNo++
+		if err = ctx.Err(); err != nil {
+			return parsedCount, err
+		}
+		values, columnsErr := rows.Columns()
+		if columnsErr != nil {
+			return parsedCount, gerror.Wrapf(columnsErr, "read Excel row %d failed", rowNo)
+		}
+		if isEmptyRow(values) {
+			continue
+		}
+		fields := make(map[string]string, len(colIndex))
+		for colName, idx := range colIndex {
+			fields[colName] = cellValue(values, idx)
+		}
+		if err = handler(ParsedRow{RowNo: rowNo, Fields: fields}); err != nil {
+			return parsedCount, err
+		}
+		parsedCount++
+	}
+	if err = rows.Error(); err != nil {
+		return parsedCount, gerror.Wrap(err, "read Excel rows failed")
+	}
+	if parsedCount == 0 {
+		return 0, gerror.New("Excel file has no data rows")
+	}
+	return parsedCount, nil
+}
+
+func ParseReader(ctx context.Context, reader io.Reader, options ParseOptions) ([]ParsedRow, error) {
+	f, err := excelize.OpenReader(reader)
+	if err != nil {
+		return nil, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer func() { _ = f.Close() }()
+
+	sheetList := f.GetSheetList()
+	if len(sheetList) == 0 {
+		return nil, gerror.New("Excel file has no worksheet")
+	}
+	if options.SingleSheet {
+		if err = ensureSingleDataSheet(f, sheetList); err != nil {
+			return nil, err
+		}
+	}
+
+	// 使用流式迭代器逐行读取，避免在超过 MaxDataRows 时仍把整张表读入内存。
+	rowsIter, err := f.Rows(sheetList[0])
+	if err != nil {
+		return nil, gerror.Wrap(err, "read Excel rows failed")
+	}
+	defer func() { _ = rowsIter.Close() }()
+
+	if !rowsIter.Next() {
+		return nil, gerror.New("Excel文件缺少表头，请检查文件内容")
+	}
+	header, err := rowsIter.Columns()
+	if err != nil {
+		return nil, gerror.Wrap(err, "读取Excel表头失败，请检查文件内容")
+	}
+	colIndex, err := buildColumnIndex(header)
+	if err != nil {
+		return nil, err
+	}
+	for _, col := range options.RequiredHeaders {
+		if _, ok := colIndex[normalizeHeader(col)]; !ok {
+			return nil, gerror.Newf("Excel file is missing required column: %s", col)
+		}
+	}
+
+	initCap := options.MaxDataRows
+	if initCap <= 0 || initCap > 100000 {
+		initCap = 1024
+	}
+	parsedRows := make([]ParsedRow, 0, initCap)
+	lineNo := 2
+	dataCount := 0
+	for rowsIter.Next() {
+		if err = ctx.Err(); err != nil {
+			return nil, err
+		}
+		row, err := rowsIter.Columns()
+		if err != nil {
+			return nil, gerror.Wrap(err, "读取Excel数据失败，请检查文件内容")
+		}
+		if isEmptyRow(row) {
+			lineNo++
+			continue
+		}
+		dataCount++
+		if options.MaxDataRows > 0 && dataCount > options.MaxDataRows {
+			return nil, gerror.Newf("单次导入最多支持%d张卡，请拆分后重试", options.MaxDataRows)
+		}
+		fields := make(map[string]string, len(colIndex))
+		for colName, idx := range colIndex {
+			fields[colName] = cellValue(row, idx)
+		}
+		parsedRows = append(parsedRows, ParsedRow{
+			RowNo:  lineNo,
+			Fields: fields,
+		})
+		lineNo++
+	}
+	if len(parsedRows) == 0 {
+		return nil, gerror.New("Excel file has no data rows")
+	}
+	return parsedRows, nil
+}
+
+func buildColumnIndex(headers []string) (map[string]int, error) {
+	colIndex := make(map[string]int)
+	for i, col := range headers {
+		header := NormalizeHeader(col)
+		if header == "" {
+			continue
+		}
+		if _, exists := colIndex[header]; exists {
+			return nil, gerror.Newf("Excel file has duplicate column: %s", strings.TrimSpace(col))
+		}
+		colIndex[header] = i
+	}
+	return colIndex, nil
+}
+
+func ensureSingleDataSheet(f *excelize.File, sheetList []string) error {
+	for _, sheetName := range sheetList[1:] {
+		rows, err := f.GetRows(sheetName)
+		if err != nil {
+			return gerror.Wrap(err, "read Excel worksheet failed")
+		}
+		for _, row := range rows {
+			if !isEmptyRow(row) {
+				return gerror.New("only one non-empty worksheet is supported")
+			}
+		}
+	}
+	return nil
+}
+
+func cellValue(row []string, index int) string {
+	if index < 0 || index >= len(row) {
+		return ""
+	}
+	return strings.TrimSpace(row[index])
+}
+
+func isEmptyRow(row []string) bool {
+	for _, col := range row {
+		if strings.TrimSpace(col) != "" {
+			return false
+		}
+	}
+	return true
+}
+
+func normalizeHeader(header string) string {
+	return NormalizeHeader(header)
+}