iotlcm/server/internal/library/batchinput/excel/parser.go

package excel

import (
	"context"
	"io"
	"os"
	"strings"

	"github.com/gogf/gf/v2/errors/gerror"
	"github.com/xuri/excelize/v2"
)

type ParseOptions struct {
	RequiredHeaders []string
	SingleSheet     bool
	// MaxDataRows 限制单次导入的有效数据行数（不含表头、不含空行）。
	// 0 表示不限制（向后兼容已有批量功能）。超过该值立即停止读取并返回错误，
	// 避免超大 Excel 在解析阶段占用过多内存或解析完成后才报错。
	MaxDataRows int
}

func ParseFile(ctx context.Context, filePath string, options ParseOptions) ([]ParsedRow, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, gerror.Wrap(err, "open Excel file failed")
	}
	defer file.Close()
	return ParseReader(ctx, file, options)
}

// StreamFile parses the first data sheet row by row and keeps only the current
// row in memory.
func StreamFile(ctx context.Context, filePath string, options ParseOptions, handler func(ParsedRow) error) (int, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return 0, gerror.Wrap(err, "open Excel file failed")
	}
	defer file.Close()
	return StreamReader(ctx, file, options, handler)
}

func StreamReader(ctx context.Context, reader io.Reader, options ParseOptions, handler func(ParsedRow) error) (int, error) {
	if handler == nil {
		return 0, gerror.New("Excel row handler cannot be nil")
	}
	f, err := excelize.OpenReader(reader)
	if err != nil {
		return 0, gerror.Wrap(err, "open Excel file failed")
	}
	defer func() { _ = f.Close() }()

	sheetList := f.GetSheetList()
	if len(sheetList) == 0 {
		return 0, gerror.New("Excel file has no worksheet")
	}
	if options.SingleSheet {
		if err = ensureSingleDataSheet(f, sheetList); err != nil {
			return 0, err
		}
	}

	rows, err := f.Rows(sheetList[0])
	if err != nil {
		return 0, gerror.Wrap(err, "read Excel rows failed")
	}
	defer rows.Close()

	if !rows.Next() {
		if err = rows.Error(); err != nil {
			return 0, gerror.Wrap(err, "read Excel header failed")
		}
		return 0, gerror.New("Excel file has no header")
	}
	headers, err := rows.Columns()
	if err != nil {
		return 0, gerror.Wrap(err, "read Excel header failed")
	}
	colIndex, err := buildColumnIndex(headers)
	if err != nil {
		return 0, err
	}
	for _, col := range options.RequiredHeaders {
		if _, ok := colIndex[normalizeHeader(col)]; !ok {
			return 0, gerror.Newf("Excel file is missing required column: %s", col)
		}
	}

	rowNo := 1
	parsedCount := 0
	for rows.Next() {
		rowNo++
		if err = ctx.Err(); err != nil {
			return parsedCount, err
		}
		values, columnsErr := rows.Columns()
		if columnsErr != nil {
			return parsedCount, gerror.Wrapf(columnsErr, "read Excel row %d failed", rowNo)
		}
		if isEmptyRow(values) {
			continue
		}
		fields := make(map[string]string, len(colIndex))
		for colName, idx := range colIndex {
			fields[colName] = cellValue(values, idx)
		}
		if err = handler(ParsedRow{RowNo: rowNo, Fields: fields}); err != nil {
			return parsedCount, err
		}
		parsedCount++
	}
	if err = rows.Error(); err != nil {
		return parsedCount, gerror.Wrap(err, "read Excel rows failed")
	}
	if parsedCount == 0 {
		return 0, gerror.New("Excel file has no data rows")
	}
	return parsedCount, nil
}

func ParseReader(ctx context.Context, reader io.Reader, options ParseOptions) ([]ParsedRow, error) {
	f, err := excelize.OpenReader(reader)
	if err != nil {
		return nil, gerror.Wrap(err, "open Excel file failed")
	}
	defer func() { _ = f.Close() }()

	sheetList := f.GetSheetList()
	if len(sheetList) == 0 {
		return nil, gerror.New("Excel file has no worksheet")
	}
	if options.SingleSheet {
		if err = ensureSingleDataSheet(f, sheetList); err != nil {
			return nil, err
		}
	}

	// 使用流式迭代器逐行读取，避免在超过 MaxDataRows 时仍把整张表读入内存。
	rowsIter, err := f.Rows(sheetList[0])
	if err != nil {
		return nil, gerror.Wrap(err, "read Excel rows failed")
	}
	defer func() { _ = rowsIter.Close() }()

	if !rowsIter.Next() {
		return nil, gerror.New("Excel文件缺少表头，请检查文件内容")
	}
	header, err := rowsIter.Columns()
	if err != nil {
		return nil, gerror.Wrap(err, "读取Excel表头失败，请检查文件内容")
	}
	colIndex, err := buildColumnIndex(header)
	if err != nil {
		return nil, err
	}
	for _, col := range options.RequiredHeaders {
		if _, ok := colIndex[normalizeHeader(col)]; !ok {
			return nil, gerror.Newf("Excel file is missing required column: %s", col)
		}
	}

	initCap := options.MaxDataRows
	if initCap <= 0 || initCap > 100000 {
		initCap = 1024
	}
	parsedRows := make([]ParsedRow, 0, initCap)
	lineNo := 2
	dataCount := 0
	for rowsIter.Next() {
		if err = ctx.Err(); err != nil {
			return nil, err
		}
		row, err := rowsIter.Columns()
		if err != nil {
			return nil, gerror.Wrap(err, "读取Excel数据失败，请检查文件内容")
		}
		if isEmptyRow(row) {
			lineNo++
			continue
		}
		dataCount++
		if options.MaxDataRows > 0 && dataCount > options.MaxDataRows {
			return nil, gerror.Newf("单次导入最多支持%d张卡，请拆分后重试", options.MaxDataRows)
		}
		fields := make(map[string]string, len(colIndex))
		for colName, idx := range colIndex {
			fields[colName] = cellValue(row, idx)
		}
		parsedRows = append(parsedRows, ParsedRow{
			RowNo:  lineNo,
			Fields: fields,
		})
		lineNo++
	}
	if len(parsedRows) == 0 {
		return nil, gerror.New("Excel file has no data rows")
	}
	return parsedRows, nil
}

func buildColumnIndex(headers []string) (map[string]int, error) {
	colIndex := make(map[string]int)
	for i, col := range headers {
		header := NormalizeHeader(col)
		if header == "" {
			continue
		}
		if _, exists := colIndex[header]; exists {
			return nil, gerror.Newf("Excel file has duplicate column: %s", strings.TrimSpace(col))
		}
		colIndex[header] = i
	}
	return colIndex, nil
}

func ensureSingleDataSheet(f *excelize.File, sheetList []string) error {
	for _, sheetName := range sheetList[1:] {
		rows, err := f.GetRows(sheetName)
		if err != nil {
			return gerror.Wrap(err, "read Excel worksheet failed")
		}
		for _, row := range rows {
			if !isEmptyRow(row) {
				return gerror.New("only one non-empty worksheet is supported")
			}
		}
	}
	return nil
}

func cellValue(row []string, index int) string {
	if index < 0 || index >= len(row) {
		return ""
	}
	return strings.TrimSpace(row[index])
}

func isEmptyRow(row []string) bool {
	for _, col := range row {
		if strings.TrimSpace(col) != "" {
			return false
		}
	}
	return true
}

func normalizeHeader(header string) string {
	return NormalizeHeader(header)
}