From 93b97ac91e4acecdbd51545fc41c58a82bf8e1e4 Mon Sep 17 00:00:00 2001
From: David Sehnal <david.sehnal@gmail.com>
Date: Wed, 13 Feb 2019 16:50:44 +0100
Subject: [PATCH] mol-io: readAllLines
---
src/mol-io/reader/common/text/tokenizer.ts | 35 ++++++++++++++++++----
src/mol-plugin/state/actions/basic.ts | 2 +-
2 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/src/mol-io/reader/common/text/tokenizer.ts b/src/mol-io/reader/common/text/tokenizer.ts
index 60f14c1b1..523bb9061 100644
--- a/src/mol-io/reader/common/text/tokenizer.ts
+++ b/src/mol-io/reader/common/text/tokenizer.ts
@@ -52,7 +52,7 @@ export namespace Tokenizer {
/**
* Eat everything until a newline occurs.
*/
- export function eatLine(state: Tokenizer) {
+ export function eatLine(state: Tokenizer): boolean {
const { data } = state;
while (state.position < state.length) {
switch (data.charCodeAt(state.position)) {
@@ -60,7 +60,7 @@ export namespace Tokenizer {
state.tokenEnd = state.position;
++state.position;
++state.lineNumber;
- return;
+ return true;
case 13: // \r
state.tokenEnd = state.position;
++state.position;
@@ -68,13 +68,14 @@ export namespace Tokenizer {
if (data.charCodeAt(state.position) === 10) {
++state.position;
}
- return;
+ return true;
default:
++state.position;
break;
}
}
state.tokenEnd = state.position;
+ return state.tokenStart !== state.tokenEnd;
}
/** Sets the current token start to the current position */
@@ -85,7 +86,7 @@ export namespace Tokenizer {
/** Sets the current token start to current position and moves to the next line. */
export function markLine(state: Tokenizer) {
state.tokenStart = state.position;
- eatLine(state);
+ return eatLine(state);
}
/** Advance the state by the given number of lines and return line starts/ends as tokens. */
@@ -95,10 +96,13 @@ export namespace Tokenizer {
}
function readLinesChunk(state: Tokenizer, count: number, tokens: Tokens) {
+ let read = 0;
for (let i = 0; i < count; i++) {
- markLine(state);
+ if (!markLine(state)) return read;
TokenBuilder.addUnchecked(tokens, state.tokenStart, state.tokenEnd);
+ read++;
}
+ return read;
}
/** Advance the state by the given number of lines and return line starts/ends as tokens. */
@@ -124,6 +128,27 @@ export namespace Tokenizer {
return lineTokens;
}
+ export function readAllLines(data: string) {
+ const state = Tokenizer(data);
+ const tokens = TokenBuilder.create(state, Math.max(data.length / 160, 2))
+ while (markLine(state)) {
+ TokenBuilder.add(tokens, state.tokenStart, state.tokenEnd);
+ }
+ return tokens;
+ }
+
+ export async function readAllLinesAsync(data: string, ctx: RuntimeContext, chunkSize = 100000) {
+ const state = Tokenizer(data);
+ const tokens = TokenBuilder.create(state, Math.max(data.length / 160, 2));
+
+ await chunkedSubtask(ctx, chunkSize, state, (chunkSize, state) => {
+ readLinesChunk(state, chunkSize, tokens);
+ return state.position < state.length ? chunkSize : 0;
+ }, (ctx, state) => ctx.update({ message: 'Parsing...', current: state.position, max: length }));
+
+ return tokens;
+ }
+
/**
* Eat everything until a whitespace/newline occurs.
*/
diff --git a/src/mol-plugin/state/actions/basic.ts b/src/mol-plugin/state/actions/basic.ts
index d54a87b9a..68bef84e3 100644
--- a/src/mol-plugin/state/actions/basic.ts
+++ b/src/mol-plugin/state/actions/basic.ts
@@ -14,7 +14,7 @@ import { ParamDefinition as PD } from 'mol-util/param-definition';
import { PluginStateObject } from '../objects';
import { StateTransforms } from '../transforms';
import { Download } from '../transforms/data';
-import { StructureRepresentation3DHelpers, VolumeRepresentation3DHelpers } from '../transforms/representation';
+import { StructureRepresentation3DHelpers } from '../transforms/representation';
import { getFileInfo, FileInput } from 'mol-util/file-info';
import { Task } from 'mol-task';
--
GitLab