chore: 并入 iotlcm-admin-svr 历史到 admin-svr/（原提交 49818c6）

admin-svr/server/internal/library/batchinput/excel/csv.go

@@ -0,0 +1,78 @@
+package excel
+
+import (
+	"context"
+	"encoding/csv"
+	"io"
+	"os"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+func StreamCSVFile(ctx context.Context, filePath string, options ParseOptions, handler func(ParsedRow) error) (int, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return 0, gerror.Wrap(err, "open CSV file failed")
+	}
+	defer file.Close()
+	return StreamCSVReader(ctx, file, options, handler)
+}
+
+func StreamCSVReader(ctx context.Context, reader io.Reader, options ParseOptions, handler func(ParsedRow) error) (int, error) {
+	if handler == nil {
+		return 0, gerror.New("CSV row handler cannot be nil")
+	}
+
+	csvReader := csv.NewReader(reader)
+	csvReader.FieldsPerRecord = -1
+	csvReader.ReuseRecord = true
+
+	headers, err := csvReader.Read()
+	if err != nil {
+		if err == io.EOF {
+			return 0, gerror.New("CSV file has no header")
+		}
+		return 0, gerror.Wrap(err, "read CSV header failed")
+	}
+	columnIndex, err := buildColumnIndex(headers)
+	if err != nil {
+		return 0, err
+	}
+	for _, requiredHeader := range options.RequiredHeaders {
+		if _, ok := columnIndex[normalizeHeader(requiredHeader)]; !ok {
+			return 0, gerror.Newf("CSV file is missing required column: %s", requiredHeader)
+		}
+	}
+
+	rowNo := 1
+	parsedCount := 0
+	for {
+		if err = ctx.Err(); err != nil {
+			return parsedCount, err
+		}
+		values, readErr := csvReader.Read()
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			return parsedCount, gerror.Wrapf(readErr, "read CSV row %d failed", rowNo+1)
+		}
+		rowNo++
+		if isEmptyRow(values) {
+			continue
+		}
+
+		fields := make(map[string]string, len(columnIndex))
+		for columnName, index := range columnIndex {
+			fields[columnName] = cellValue(values, index)
+		}
+		if err = handler(ParsedRow{RowNo: rowNo, Fields: fields}); err != nil {
+			return parsedCount, err
+		}
+		parsedCount++
+	}
+	if parsedCount == 0 {
+		return 0, gerror.New("CSV file has no data rows")
+	}
+	return parsedCount, nil
+}

admin-svr/server/internal/library/batchinput/excel/parser.go

@@ -0,0 +1,248 @@
+package excel
+
+import (
+	"context"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/xuri/excelize/v2"
+)
+
+type ParseOptions struct {
+	RequiredHeaders []string
+	SingleSheet     bool
+	// MaxDataRows 限制单次导入的有效数据行数（不含表头、不含空行）。
+	// 0 表示不限制（向后兼容已有批量功能）。超过该值立即停止读取并返回错误，
+	// 避免超大 Excel 在解析阶段占用过多内存或解析完成后才报错。
+	MaxDataRows int
+}
+
+func ParseFile(ctx context.Context, filePath string, options ParseOptions) ([]ParsedRow, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer file.Close()
+	return ParseReader(ctx, file, options)
+}
+
+// StreamFile parses the first data sheet row by row and keeps only the current
+// row in memory.
+func StreamFile(ctx context.Context, filePath string, options ParseOptions, handler func(ParsedRow) error) (int, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return 0, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer file.Close()
+	return StreamReader(ctx, file, options, handler)
+}
+
+func StreamReader(ctx context.Context, reader io.Reader, options ParseOptions, handler func(ParsedRow) error) (int, error) {
+	if handler == nil {
+		return 0, gerror.New("Excel row handler cannot be nil")
+	}
+	f, err := excelize.OpenReader(reader)
+	if err != nil {
+		return 0, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer func() { _ = f.Close() }()
+
+	sheetList := f.GetSheetList()
+	if len(sheetList) == 0 {
+		return 0, gerror.New("Excel file has no worksheet")
+	}
+	if options.SingleSheet {
+		if err = ensureSingleDataSheet(f, sheetList); err != nil {
+			return 0, err
+		}
+	}
+
+	rows, err := f.Rows(sheetList[0])
+	if err != nil {
+		return 0, gerror.Wrap(err, "read Excel rows failed")
+	}
+	defer rows.Close()
+
+	if !rows.Next() {
+		if err = rows.Error(); err != nil {
+			return 0, gerror.Wrap(err, "read Excel header failed")
+		}
+		return 0, gerror.New("Excel file has no header")
+	}
+	headers, err := rows.Columns()
+	if err != nil {
+		return 0, gerror.Wrap(err, "read Excel header failed")
+	}
+	colIndex, err := buildColumnIndex(headers)
+	if err != nil {
+		return 0, err
+	}
+	for _, col := range options.RequiredHeaders {
+		if _, ok := colIndex[normalizeHeader(col)]; !ok {
+			return 0, gerror.Newf("Excel file is missing required column: %s", col)
+		}
+	}
+
+	rowNo := 1
+	parsedCount := 0
+	for rows.Next() {
+		rowNo++
+		if err = ctx.Err(); err != nil {
+			return parsedCount, err
+		}
+		values, columnsErr := rows.Columns()
+		if columnsErr != nil {
+			return parsedCount, gerror.Wrapf(columnsErr, "read Excel row %d failed", rowNo)
+		}
+		if isEmptyRow(values) {
+			continue
+		}
+		fields := make(map[string]string, len(colIndex))
+		for colName, idx := range colIndex {
+			fields[colName] = cellValue(values, idx)
+		}
+		if err = handler(ParsedRow{RowNo: rowNo, Fields: fields}); err != nil {
+			return parsedCount, err
+		}
+		parsedCount++
+	}
+	if err = rows.Error(); err != nil {
+		return parsedCount, gerror.Wrap(err, "read Excel rows failed")
+	}
+	if parsedCount == 0 {
+		return 0, gerror.New("Excel file has no data rows")
+	}
+	return parsedCount, nil
+}
+
+func ParseReader(ctx context.Context, reader io.Reader, options ParseOptions) ([]ParsedRow, error) {
+	f, err := excelize.OpenReader(reader)
+	if err != nil {
+		return nil, gerror.Wrap(err, "open Excel file failed")
+	}
+	defer func() { _ = f.Close() }()
+
+	sheetList := f.GetSheetList()
+	if len(sheetList) == 0 {
+		return nil, gerror.New("Excel file has no worksheet")
+	}
+	if options.SingleSheet {
+		if err = ensureSingleDataSheet(f, sheetList); err != nil {
+			return nil, err
+		}
+	}
+
+	// 使用流式迭代器逐行读取，避免在超过 MaxDataRows 时仍把整张表读入内存。
+	rowsIter, err := f.Rows(sheetList[0])
+	if err != nil {
+		return nil, gerror.Wrap(err, "read Excel rows failed")
+	}
+	defer func() { _ = rowsIter.Close() }()
+
+	if !rowsIter.Next() {
+		return nil, gerror.New("Excel文件缺少表头，请检查文件内容")
+	}
+	header, err := rowsIter.Columns()
+	if err != nil {
+		return nil, gerror.Wrap(err, "读取Excel表头失败，请检查文件内容")
+	}
+	colIndex, err := buildColumnIndex(header)
+	if err != nil {
+		return nil, err
+	}
+	for _, col := range options.RequiredHeaders {
+		if _, ok := colIndex[normalizeHeader(col)]; !ok {
+			return nil, gerror.Newf("Excel file is missing required column: %s", col)
+		}
+	}
+
+	initCap := options.MaxDataRows
+	if initCap <= 0 || initCap > 100000 {
+		initCap = 1024
+	}
+	parsedRows := make([]ParsedRow, 0, initCap)
+	lineNo := 2
+	dataCount := 0
+	for rowsIter.Next() {
+		if err = ctx.Err(); err != nil {
+			return nil, err
+		}
+		row, err := rowsIter.Columns()
+		if err != nil {
+			return nil, gerror.Wrap(err, "读取Excel数据失败，请检查文件内容")
+		}
+		if isEmptyRow(row) {
+			lineNo++
+			continue
+		}
+		dataCount++
+		if options.MaxDataRows > 0 && dataCount > options.MaxDataRows {
+			return nil, gerror.Newf("单次导入最多支持%d张卡，请拆分后重试", options.MaxDataRows)
+		}
+		fields := make(map[string]string, len(colIndex))
+		for colName, idx := range colIndex {
+			fields[colName] = cellValue(row, idx)
+		}
+		parsedRows = append(parsedRows, ParsedRow{
+			RowNo:  lineNo,
+			Fields: fields,
+		})
+		lineNo++
+	}
+	if len(parsedRows) == 0 {
+		return nil, gerror.New("Excel file has no data rows")
+	}
+	return parsedRows, nil
+}
+
+func buildColumnIndex(headers []string) (map[string]int, error) {
+	colIndex := make(map[string]int)
+	for i, col := range headers {
+		header := NormalizeHeader(col)
+		if header == "" {
+			continue
+		}
+		if _, exists := colIndex[header]; exists {
+			return nil, gerror.Newf("Excel file has duplicate column: %s", strings.TrimSpace(col))
+		}
+		colIndex[header] = i
+	}
+	return colIndex, nil
+}
+
+func ensureSingleDataSheet(f *excelize.File, sheetList []string) error {
+	for _, sheetName := range sheetList[1:] {
+		rows, err := f.GetRows(sheetName)
+		if err != nil {
+			return gerror.Wrap(err, "read Excel worksheet failed")
+		}
+		for _, row := range rows {
+			if !isEmptyRow(row) {
+				return gerror.New("only one non-empty worksheet is supported")
+			}
+		}
+	}
+	return nil
+}
+
+func cellValue(row []string, index int) string {
+	if index < 0 || index >= len(row) {
+		return ""
+	}
+	return strings.TrimSpace(row[index])
+}
+
+func isEmptyRow(row []string) bool {
+	for _, col := range row {
+		if strings.TrimSpace(col) != "" {
+			return false
+		}
+	}
+	return true
+}
+
+func normalizeHeader(header string) string {
+	return NormalizeHeader(header)
+}

admin-svr/server/internal/library/batchinput/excel/row.go

@@ -0,0 +1,15 @@
+package excel
+
+import "strings"
+
+type ParsedRow struct {
+	RowNo  int
+	Fields map[string]string
+}
+
+func Field(row ParsedRow, name string) string {
+	if row.Fields == nil {
+		return ""
+	}
+	return strings.TrimSpace(row.Fields[NormalizeHeader(name)])
+}

admin-svr/server/internal/library/batchinput/excel/schema.go

@@ -0,0 +1,33 @@
+package excel
+
+type ColumnType string
+
+const (
+	ColumnTypeString         ColumnType = "string"
+	ColumnTypeInt            ColumnType = "int"
+	ColumnTypePositiveInt    ColumnType = "positive_int"
+	ColumnTypeNonNegativeInt ColumnType = "non_negative_int"
+	ColumnTypeBool           ColumnType = "bool"
+	ColumnTypeDate           ColumnType = "date"
+	ColumnTypeDateTime       ColumnType = "datetime"
+	ColumnTypeAmount         ColumnType = "amount"
+)
+
+type ColumnSpec struct {
+	Name     string
+	Required bool
+	Type     ColumnType
+}
+
+type ConsistencyRule struct {
+	Name       string
+	Type       ColumnType
+	AllowEmpty bool
+}
+
+type RowIssue struct {
+	RowNo   int
+	Column  string
+	Value   string
+	Message string
+}

admin-svr/server/internal/library/batchinput/excel/validator.go

@@ -0,0 +1,99 @@
+package excel
+
+import (
+	"fmt"
+	"strings"
+)
+
+func ValidateRequiredFields(rows []ParsedRow, specs []ColumnSpec) []RowIssue {
+	issues := make([]RowIssue, 0)
+	for _, row := range rows {
+		for _, spec := range specs {
+			if !spec.Required {
+				continue
+			}
+			value := Field(row, spec.Name)
+			if value != "" {
+				continue
+			}
+			issues = append(issues, RowIssue{
+				RowNo:   row.RowNo,
+				Column:  spec.Name,
+				Value:   value,
+				Message: fmt.Sprintf("%s is required", spec.Name),
+			})
+		}
+	}
+	return issues
+}
+
+func ValidateRowTypes(rows []ParsedRow, specs []ColumnSpec) []RowIssue {
+	issues := make([]RowIssue, 0)
+	for _, row := range rows {
+		for _, spec := range specs {
+			value := Field(row, spec.Name)
+			if value == "" {
+				continue
+			}
+			if _, err := normalizedValue(value, spec.Type); err != nil {
+				issues = append(issues, RowIssue{
+					RowNo:   row.RowNo,
+					Column:  spec.Name,
+					Value:   value,
+					Message: fmt.Sprintf("%s format is invalid", spec.Name),
+				})
+			}
+		}
+	}
+	return issues
+}
+
+func ValidateFileConsistency(rows []ParsedRow, rules []ConsistencyRule) []RowIssue {
+	issues := make([]RowIssue, 0)
+	for _, rule := range rules {
+		baseValue := ""
+		baseSet := false
+		for _, row := range rows {
+			raw := Field(row, rule.Name)
+			normalized := ""
+			if strings.TrimSpace(raw) == "" {
+				if !rule.AllowEmpty {
+					issues = append(issues, RowIssue{
+						RowNo:   row.RowNo,
+						Column:  rule.Name,
+						Value:   raw,
+						Message: fmt.Sprintf("%s must not be empty", rule.Name),
+					})
+					continue
+				}
+			} else {
+				value, err := normalizedValue(raw, rule.Type)
+				if err != nil {
+					issues = append(issues, RowIssue{
+						RowNo:   row.RowNo,
+						Column:  rule.Name,
+						Value:   raw,
+						Message: fmt.Sprintf("%s format is invalid", rule.Name),
+					})
+					continue
+				}
+				normalized = value
+			}
+
+			if !baseSet {
+				baseValue = normalized
+				baseSet = true
+				continue
+			}
+			if normalized != baseValue {
+				issues = append(issues, RowIssue{
+					RowNo:   row.RowNo,
+					Column:  rule.Name,
+					Value:   raw,
+					Message: fmt.Sprintf("%s must be consistent", rule.Name),
+				})
+			}
+		}
+	}
+	return issues
+}

admin-svr/server/internal/library/batchinput/excel/value.go

@@ -0,0 +1,220 @@
+package excel
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gogf/gf/v2/errors/gerror"
+	"github.com/gogf/gf/v2/os/gtime"
+)
+
+func NormalizeHeader(header string) string {
+	header = strings.ReplaceAll(header, "\uFEFF", "")
+	header = strings.TrimSpace(header)
+	header = strings.TrimRight(header, "*＊")
+	return strings.ToUpper(strings.TrimSpace(header))
+}
+
+func ParseString(value string) string {
+	return strings.TrimSpace(value)
+}
+
+func ParseInt(value string) (int, error) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return 0, strconv.ErrSyntax
+	}
+	return strconv.Atoi(value)
+}
+
+func ParsePositiveInt(value string) (int, error) {
+	parsed, err := ParseInt(value)
+	if err != nil || parsed <= 0 {
+		return 0, strconv.ErrSyntax
+	}
+	return parsed, nil
+}
+
+func ParseNonNegativeInt(value string) (int, error) {
+	parsed, err := ParseInt(value)
+	if err != nil || parsed < 0 {
+		return 0, strconv.ErrSyntax
+	}
+	return parsed, nil
+}
+
+func ParseBool(value string) (bool, error) {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "1", "true", "yes", "y", "\u662f", "\u5df2\u5b8c\u6210", "\u5b8c\u6210":
+		return true, nil
+	case "0", "false", "no", "n", "\u5426", "\u672a\u5b8c\u6210":
+		return false, nil
+	default:
+		return false, strconv.ErrSyntax
+	}
+}
+
+func ParseDate(value string) (*gtime.Time, error) {
+	parsed, err := parseTimeValue(value)
+	if err != nil {
+		return nil, err
+	}
+	return gtime.NewFromTime(time.Date(
+		parsed.Year(),
+		time.Month(parsed.Month()),
+		parsed.Day(),
+		0,
+		0,
+		0,
+		0,
+		parsed.Location(),
+	)), nil
+}
+
+func ParseDateTime(value string) (*gtime.Time, error) {
+	return parseTimeValue(value)
+}
+
+func ParseAmount(value string) (float64, error) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return 0, strconv.ErrSyntax
+	}
+	value = strings.ReplaceAll(value, ",", "")
+	parsed, err := strconv.ParseFloat(value, 64)
+	if err != nil || parsed < 0 {
+		return 0, strconv.ErrSyntax
+	}
+	return math.Round(parsed*100) / 100, nil
+}
+
+func normalizedValue(value string, typ ColumnType) (string, error) {
+	switch typ {
+	case "", ColumnTypeString:
+		return ParseString(value), nil
+	case ColumnTypeInt:
+		parsed, err := ParseInt(value)
+		if err != nil {
+			return "", err
+		}
+		return strconv.Itoa(parsed), nil
+	case ColumnTypePositiveInt:
+		parsed, err := ParsePositiveInt(value)
+		if err != nil {
+			return "", err
+		}
+		return strconv.Itoa(parsed), nil
+	case ColumnTypeNonNegativeInt:
+		parsed, err := ParseNonNegativeInt(value)
+		if err != nil {
+			return "", err
+		}
+		return strconv.Itoa(parsed), nil
+	case ColumnTypeBool:
+		parsed, err := ParseBool(value)
+		if err != nil {
+			return "", err
+		}
+		return strconv.FormatBool(parsed), nil
+	case ColumnTypeDate:
+		parsed, err := ParseDate(value)
+		if err != nil {
+			return "", err
+		}
+		return parsed.Format("Y-m-d"), nil
+	case ColumnTypeDateTime:
+		parsed, err := ParseDateTime(value)
+		if err != nil {
+			return "", err
+		}
+		return parsed.Format("Y-m-d H:i:s"), nil
+	case ColumnTypeAmount:
+		parsed, err := ParseAmount(value)
+		if err != nil {
+			return "", err
+		}
+		return fmt.Sprintf("%.2f", parsed), nil
+	default:
+		return "", gerror.Newf("unsupported column type: %s", typ)
+	}
+}
+
+func parseTimeValue(value string) (*gtime.Time, error) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return nil, strconv.ErrSyntax
+	}
+	if parsed, ok := parseCompactTime(value); ok {
+		return parsed, nil
+	}
+	if parsed, ok := parseLayoutTime(value); ok {
+		return parsed, nil
+	}
+	parsed, err := gtime.StrToTime(value)
+	if err != nil || parsed == nil || parsed.IsZero() || parsed.Year() <= 0 {
+		if err != nil {
+			return nil, err
+		}
+		return nil, strconv.ErrSyntax
+	}
+	return parsed, nil
+}
+
+func parseCompactTime(value string) (*gtime.Time, bool) {
+	if !isDigits(value) {
+		return nil, false
+	}
+	layout := ""
+	switch len(value) {
+	case 8:
+		layout = "20060102"
+	case 12:
+		layout = "200601021504"
+	case 14:
+		layout = "20060102150405"
+	default:
+		return nil, false
+	}
+	parsed, err := time.ParseInLocation(layout, value, time.Local)
+	if err != nil {
+		return nil, false
+	}
+	return gtime.NewFromTime(parsed), true
+}
+
+func parseLayoutTime(value string) (*gtime.Time, bool) {
+	layouts := []string{
+		time.RFC3339,
+		time.RFC3339Nano,
+		"2006-1-2 15:04:05",
+		"2006-1-2 15:04",
+		"2006/1/2 15:04:05",
+		"2006/1/2 15:04",
+		"2006.1.2 15:04:05",
+		"2006.1.2 15:04",
+		"2006-1-2T15:04:05",
+		"2006-1-2T15:04",
+		"2006-1-2",
+		"2006/1/2",
+		"2006.1.2",
+	}
+	for _, layout := range layouts {
+		parsed, err := time.ParseInLocation(layout, value, time.Local)
+		if err == nil {
+			return gtime.NewFromTime(parsed), true
+		}
+	}
+	return nil, false
+}
+
+func isDigits(value string) bool {
+	for _, r := range value {
+		if r < '0' || r > '9' {
+			return false
+		}
+	}
+	return value != ""
+}