MiFaZhan
79 lines
1.9 KiB
Go
79 lines
1.9 KiB
Go
package excel
|
|
|
|
import (
|
|
"context"
|
|
"encoding/csv"
|
|
"io"
|
|
"os"
|
|
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
)
|
|
|
|
func StreamCSVFile(ctx context.Context, filePath string, options ParseOptions, handler func(ParsedRow) error) (int, error) {
|
|
file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return 0, gerror.Wrap(err, "open CSV file failed")
|
|
}
|
|
defer file.Close()
|
|
return StreamCSVReader(ctx, file, options, handler)
|
|
}
|
|
|
|
func StreamCSVReader(ctx context.Context, reader io.Reader, options ParseOptions, handler func(ParsedRow) error) (int, error) {
|
|
if handler == nil {
|
|
return 0, gerror.New("CSV row handler cannot be nil")
|
|
}
|
|
|
|
csvReader := csv.NewReader(reader)
|
|
csvReader.FieldsPerRecord = -1
|
|
csvReader.ReuseRecord = true
|
|
|
|
headers, err := csvReader.Read()
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return 0, gerror.New("CSV file has no header")
|
|
}
|
|
return 0, gerror.Wrap(err, "read CSV header failed")
|
|
}
|
|
columnIndex, err := buildColumnIndex(headers)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
for _, requiredHeader := range options.RequiredHeaders {
|
|
if _, ok := columnIndex[normalizeHeader(requiredHeader)]; !ok {
|
|
return 0, gerror.Newf("CSV file is missing required column: %s", requiredHeader)
|
|
}
|
|
}
|
|
|
|
rowNo := 1
|
|
parsedCount := 0
|
|
for {
|
|
if err = ctx.Err(); err != nil {
|
|
return parsedCount, err
|
|
}
|
|
values, readErr := csvReader.Read()
|
|
if readErr == io.EOF {
|
|
break
|
|
}
|
|
if readErr != nil {
|
|
return parsedCount, gerror.Wrapf(readErr, "read CSV row %d failed", rowNo+1)
|
|
}
|
|
rowNo++
|
|
if isEmptyRow(values) {
|
|
continue
|
|
}
|
|
|
|
fields := make(map[string]string, len(columnIndex))
|
|
for columnName, index := range columnIndex {
|
|
fields[columnName] = cellValue(values, index)
|
|
}
|
|
if err = handler(ParsedRow{RowNo: rowNo, Fields: fields}); err != nil {
|
|
return parsedCount, err
|
|
}
|
|
parsedCount++
|
|
}
|
|
if parsedCount == 0 {
|
|
return 0, gerror.New("CSV file has no data rows")
|
|
}
|
|
return parsedCount, nil
|
|
}
|