diff --git a/src/mol-io/reader/common/text/tokenizer.ts b/src/mol-io/reader/common/text/tokenizer.ts index 60f14c1b135bfca7d215f3a3d8939997b4ec2ae5..523bb90617941b88d113a4919c8064dec4bbd472 100644 --- a/src/mol-io/reader/common/text/tokenizer.ts +++ b/src/mol-io/reader/common/text/tokenizer.ts @@ -52,7 +52,7 @@ export namespace Tokenizer { /** * Eat everything until a newline occurs. */ - export function eatLine(state: Tokenizer) { + export function eatLine(state: Tokenizer): boolean { const { data } = state; while (state.position < state.length) { switch (data.charCodeAt(state.position)) { @@ -60,7 +60,7 @@ export namespace Tokenizer { state.tokenEnd = state.position; ++state.position; ++state.lineNumber; - return; + return true; case 13: // \r state.tokenEnd = state.position; ++state.position; @@ -68,13 +68,14 @@ export namespace Tokenizer { if (data.charCodeAt(state.position) === 10) { ++state.position; } - return; + return true; default: ++state.position; break; } } state.tokenEnd = state.position; + return state.tokenStart !== state.tokenEnd; } /** Sets the current token start to the current position */ @@ -85,7 +86,7 @@ export namespace Tokenizer { /** Sets the current token start to current position and moves to the next line. */ export function markLine(state: Tokenizer) { state.tokenStart = state.position; - eatLine(state); + return eatLine(state); } /** Advance the state by the given number of lines and return line starts/ends as tokens. */ @@ -95,10 +96,13 @@ export namespace Tokenizer { } function readLinesChunk(state: Tokenizer, count: number, tokens: Tokens) { + let read = 0; for (let i = 0; i < count; i++) { - markLine(state); + if (!markLine(state)) return read; TokenBuilder.addUnchecked(tokens, state.tokenStart, state.tokenEnd); + read++; } + return read; } /** Advance the state by the given number of lines and return line starts/ends as tokens. */ @@ -124,6 +128,27 @@ export namespace Tokenizer { return lineTokens; } + export function readAllLines(data: string) { + const state = Tokenizer(data); + const tokens = TokenBuilder.create(state, Math.max(data.length / 160, 2)) + while (markLine(state)) { + TokenBuilder.add(tokens, state.tokenStart, state.tokenEnd); + } + return tokens; + } + + export async function readAllLinesAsync(data: string, ctx: RuntimeContext, chunkSize = 100000) { + const state = Tokenizer(data); + const tokens = TokenBuilder.create(state, Math.max(data.length / 160, 2)); + + await chunkedSubtask(ctx, chunkSize, state, (chunkSize, state) => { + readLinesChunk(state, chunkSize, tokens); + return state.position < state.length ? chunkSize : 0; + }, (ctx, state) => ctx.update({ message: 'Parsing...', current: state.position, max: length })); + + return tokens; + } + /** * Eat everything until a whitespace/newline occurs. */ diff --git a/src/mol-plugin/state/actions/basic.ts b/src/mol-plugin/state/actions/basic.ts index d54a87b9aef0e984040fca191052c81dd30c1270..68bef84e34dfa245565b30aa2d25e2c4ff484a9f 100644 --- a/src/mol-plugin/state/actions/basic.ts +++ b/src/mol-plugin/state/actions/basic.ts @@ -14,7 +14,7 @@ import { ParamDefinition as PD } from 'mol-util/param-definition'; import { PluginStateObject } from '../objects'; import { StateTransforms } from '../transforms'; import { Download } from '../transforms/data'; -import { StructureRepresentation3DHelpers, VolumeRepresentation3DHelpers } from '../transforms/representation'; +import { StructureRepresentation3DHelpers } from '../transforms/representation'; import { getFileInfo, FileInput } from 'mol-util/file-info'; import { Task } from 'mol-task';