Skip to content
Snippets Groups Projects
Commit a2217c7f authored by Alexander Rose's avatar Alexander Rose
Browse files

fix csv parser chunking

parent 3eec30aa
No related branches found
No related tags found
No related merge requests found
...@@ -209,22 +209,26 @@ function moveNext(state: State) { ...@@ -209,22 +209,26 @@ function moveNext(state: State) {
function readRecordsChunk(chunkSize: number, state: State) { function readRecordsChunk(chunkSize: number, state: State) {
if (state.tokenType === CsvTokenType.End) return 0 if (state.tokenType === CsvTokenType.End) return 0
let newRecord = moveNext(state); let counter = 0;
if (newRecord) ++state.recordCount let newRecord: boolean | undefined
const { tokens, tokenizer } = state; const { tokens, tokenizer } = state;
let counter = 0;
while (state.tokenType === CsvTokenType.Value && counter < chunkSize) { while (state.tokenType === CsvTokenType.Value && counter < chunkSize) {
TokenBuilder.add(tokens[state.fieldCount % state.columnCount], tokenizer.tokenStart, tokenizer.tokenEnd); TokenBuilder.add(tokens[state.fieldCount % state.columnCount], tokenizer.tokenStart, tokenizer.tokenEnd);
++state.fieldCount ++state.fieldCount
newRecord = moveNext(state); newRecord = moveNext(state);
if (newRecord) ++state.recordCount if (newRecord) {
++state.recordCount
++counter; ++counter;
} }
}
return counter; return counter;
} }
function readRecordsChunks(state: State) { function readRecordsChunks(state: State) {
let newRecord = moveNext(state);
if (newRecord) ++state.recordCount
return chunkedSubtask(state.runtimeCtx, 100000, state, readRecordsChunk, return chunkedSubtask(state.runtimeCtx, 100000, state, readRecordsChunk,
(ctx, state) => ctx.update({ message: 'Parsing...', current: state.tokenizer.position, max: state.data.length })); (ctx, state) => ctx.update({ message: 'Parsing...', current: state.tokenizer.position, max: state.data.length }));
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment