Skip to content
Snippets Groups Projects
Commit 9951dce6 authored by David Sehnal's avatar David Sehnal
Browse files

fixed readLinesAsync

parent d46610f4
Branches
Tags
No related merge requests found
...@@ -397,7 +397,7 @@ function createTokenizer(data: string, ctx: Computation.Context): TokenizerState ...@@ -397,7 +397,7 @@ function createTokenizer(data: string, ctx: Computation.Context): TokenizerState
currentTokenType: CifTokenType.End, currentTokenType: CifTokenType.End,
currentLineNumber: 1, currentLineNumber: 1,
isEscaped: false, isEscaped: false,
computation: new Computation.Chunked(ctx, 1000000) computation: Computation.chunked(ctx, 1000000)
}; };
} }
......
...@@ -35,7 +35,7 @@ export function Tokenizer(data: string, ctx: Computation.Context): Tokenizer { ...@@ -35,7 +35,7 @@ export function Tokenizer(data: string, ctx: Computation.Context): Tokenizer {
currentLineNumber: 1, currentLineNumber: 1,
currentTokenStart: 0, currentTokenStart: 0,
currentTokenEnd: 0, currentTokenEnd: 0,
computation: new Computation.Chunked(ctx, 1000000) computation: Computation.chunked(ctx, 1000000)
}; };
} }
...@@ -90,21 +90,37 @@ export namespace Tokenizer { ...@@ -90,21 +90,37 @@ export namespace Tokenizer {
return getTokenString(state); return getTokenString(state);
} }
function readLinesChunk(state: Tokenizer, count: number, tokens: Tokens) {
for (let i = 0; i < count; i++) {
markLine(state);
TokenBuilder.addUnchecked(tokens, state.currentTokenStart, state.currentTokenEnd);
}
}
/** Advance the state by the given number of lines and return line starts/ends as tokens. */ /** Advance the state by the given number of lines and return line starts/ends as tokens. */
export async function readLines(state: Tokenizer, count: number): Promise<Tokens> { export function readLines(state: Tokenizer, count: number): Tokens {
const { computation, position, length } = state
const lineTokens = TokenBuilder.create(state, count * 2); const lineTokens = TokenBuilder.create(state, count * 2);
readLinesChunk(state, count, lineTokens);
return lineTokens;
}
for (let i = 0; i < count; i++) { /** Advance the state by the given number of lines and return line starts/ends as tokens. */
markLine(state); export async function readLinesAsync(state: Tokenizer, count: number): Promise<Tokens> {
TokenBuilder.addUnchecked(lineTokens, state.currentTokenStart, state.currentTokenEnd); const { computation, length } = state
const lineTokens = TokenBuilder.create(state, count * 2);
computation.chunkSize = 100000;
let linesAlreadyRead = 0;
while (linesAlreadyRead < count) {
const linesToRead = Math.min(count - linesAlreadyRead, computation.chunkSize);
readLinesChunk(state, linesToRead, lineTokens);
linesAlreadyRead += linesToRead;
if (computation.requiresUpdate) { if (computation.requiresUpdate) {
await computation.updateProgress('Parsing...', void 0, position, length); await computation.updateProgress('Parsing...', void 0, state.position, length);
} }
} }
return { data: state.data, count, indices: lineTokens.indices }; return lineTokens;
} }
/** /**
......
...@@ -88,7 +88,7 @@ function handleNumberOfAtoms(state: State) { ...@@ -88,7 +88,7 @@ function handleNumberOfAtoms(state: State) {
*/ */
async function handleAtoms(state: State): Promise<Schema.Atoms> { async function handleAtoms(state: State): Promise<Schema.Atoms> {
const { tokenizer, numberOfAtoms } = state; const { tokenizer, numberOfAtoms } = state;
const lines = await Tokenizer.readLines(tokenizer, numberOfAtoms); const lines = await Tokenizer.readLinesAsync(tokenizer, numberOfAtoms);
const positionSample = tokenizer.data.substring(lines.indices[0], lines.indices[1]).substring(20); const positionSample = tokenizer.data.substring(lines.indices[0], lines.indices[1]).substring(20);
const precisions = positionSample.match(/\.\d+/g)!; const precisions = positionSample.match(/\.\d+/g)!;
......
...@@ -19,7 +19,7 @@ const file = 'md_1u19_trj.gro' ...@@ -19,7 +19,7 @@ const file = 'md_1u19_trj.gro'
async function runGro(input: string) { async function runGro(input: string) {
console.time('parseGro'); console.time('parseGro');
const comp = Gro(input); const comp = Gro(input);
const running = comp.runWithContext(new Computation.ObservableContext({ updateRateMs: 250 })); const running = comp.runObservable(Computation.observableContext({ updateRateMs: 250 }));
running.subscribe(p => console.log(`[Gro] ${(p.current / p.max * 100).toFixed(2)} (${p.elapsedMs | 0}ms)`)); running.subscribe(p => console.log(`[Gro] ${(p.current / p.max * 100).toFixed(2)} (${p.elapsedMs | 0}ms)`));
const parsed = await running.result; const parsed = await running.result;
console.timeEnd('parseGro'); console.timeEnd('parseGro');
...@@ -70,7 +70,7 @@ async function runGro(input: string) { ...@@ -70,7 +70,7 @@ async function runGro(input: string) {
console.log(residueNumber.length, residueNumber[0], residueNumber[residueNumber.length - 1]) console.log(residueNumber.length, residueNumber[0], residueNumber[residueNumber.length - 1])
} }
async function _gro() { function _gro() {
fs.readFile(`./examples/${file}`, 'utf8', function (err, input) { fs.readFile(`./examples/${file}`, 'utf8', function (err, input) {
if (err) { if (err) {
return console.log(err); return console.log(err);
...@@ -85,7 +85,7 @@ async function runCIF(input: string | Uint8Array) { ...@@ -85,7 +85,7 @@ async function runCIF(input: string | Uint8Array) {
console.time('parseCIF'); console.time('parseCIF');
const comp = typeof input === 'string' ? CIF.parseText(input) : CIF.parseBinary(input); const comp = typeof input === 'string' ? CIF.parseText(input) : CIF.parseBinary(input);
const running = comp.runWithContext(new Computation.ObservableContext({ updateRateMs: 250 })); const running = comp.runObservable(Computation.observableContext({ updateRateMs: 250 }));
running.subscribe(p => console.log(`[CIF] ${(p.current / p.max * 100).toFixed(2)} (${p.elapsedMs | 0}ms)`)); running.subscribe(p => console.log(`[CIF] ${(p.current / p.max * 100).toFixed(2)} (${p.elapsedMs | 0}ms)`));
const parsed = await running.result; const parsed = await running.result;
console.timeEnd('parseCIF'); console.timeEnd('parseCIF');
...@@ -131,7 +131,7 @@ export function _cif() { ...@@ -131,7 +131,7 @@ export function _cif() {
}); });
} }
_cif(); //_cif();
import Computation from './utils/computation' import Computation from './utils/computation'
const comp = new Computation(async ctx => { const comp = new Computation(async ctx => {
...@@ -142,7 +142,7 @@ const comp = new Computation(async ctx => { ...@@ -142,7 +142,7 @@ const comp = new Computation(async ctx => {
return 42; return 42;
}); });
async function testComp() { async function testComp() {
const running = comp.runWithContext(); const running = comp.runObservable();
running.subscribe(p => console.log(JSON.stringify(p))); running.subscribe(p => console.log(JSON.stringify(p)));
const ret = await running.result; const ret = await running.result;
console.log('computation returned', ret); console.log('computation returned', ret);
......
...@@ -9,14 +9,14 @@ import Scheduler from './scheduler' ...@@ -9,14 +9,14 @@ import Scheduler from './scheduler'
class Computation<A> { class Computation<A> {
run(ctx?: Computation.Context) { run(ctx?: Computation.Context) {
return this.runWithContext(ctx).result; return this.runObservable(ctx).result;
} }
runWithContext(ctx?: Computation.Context): Computation.Running<A> { runObservable(ctx?: Computation.Context): Computation.Running<A> {
const context = ctx ? ctx as Computation.ObservableContext : new Computation.ObservableContext(); const context = ctx ? ctx as ObservableContext : new ObservableContext();
return { return {
subscribe: (context as Computation.ObservableContext).subscribe || NoOpSubscribe, subscribe: (context as ObservableContext).subscribe || NoOpSubscribe,
result: new Promise<A>(async (resolve, reject) => { result: new Promise<A>(async (resolve, reject) => {
try { try {
if (context.started) context.started(); if (context.started) context.started();
...@@ -37,19 +37,20 @@ class Computation<A> { ...@@ -37,19 +37,20 @@ class Computation<A> {
} }
} }
const NoOpSubscribe = () => {}
namespace Computation { namespace Computation {
const DefaulUpdateRateMs = 100;
export let PRINT_CONSOLE_ERROR = false; export let PRINT_CONSOLE_ERROR = false;
export function create<A>(computation: (ctx: Context) => Promise<A>) {
return new Computation(computation);
}
export function resolve<A>(a: A) { export function resolve<A>(a: A) {
return new Computation<A>(() => Promise.resolve(a)); return new Computation<A>(_ => Promise.resolve(a));
} }
export function reject<A>(reason: any) { export function reject<A>(reason: any) {
return new Computation<A>(() => Promise.reject(reason)); return new Computation<A>(_ => Promise.reject(reason));
} }
export interface Params { export interface Params {
...@@ -80,38 +81,80 @@ namespace Computation { ...@@ -80,38 +81,80 @@ namespace Computation {
updateProgress(msg: string, abort?: boolean | (() => void), current?: number, max?: number): Promise<void> | void updateProgress(msg: string, abort?: boolean | (() => void), current?: number, max?: number): Promise<void> | void
} }
type ProgressObserver = (progress: Readonly<Progress>) => void; export type ProgressObserver = (progress: Readonly<Progress>) => void;
export interface Running<A> { export interface Running<A> {
subscribe(onProgress: ProgressObserver): void; subscribe(onProgress: ProgressObserver): void;
result: Promise<A> result: Promise<A>
} }
export const ContextWithoutUpdates: Context = { export const contextWithoutUpdates: Context = {
requiresUpdate: false, requiresUpdate: false,
requestAbort() { }, requestAbort() { },
updateProgress(msg, abort, current, max) { } updateProgress(msg, abort, current, max) { }
} }
export class ObservableContext implements Context { export function observableContext(params?: Partial<Params>) {
return new ObservableContext(params);
}
declare var process: any;
declare var window: any;
export const now: () => number = (function () {
if (typeof window !== 'undefined' && window.performance) {
const perf = window.performance;
return function () { return perf.now(); }
} else if (typeof process !== 'undefined' && process.hrtime !== 'undefined') {
return function () {
let t = process.hrtime();
return t[0] * 1000 + t[1] / 1000000;
};
} else {
return function () { return +new Date(); }
}
})();
export interface Chunked {
/**
* Get automatically computed chunk size
* Or set it a default value.
*/
chunkSize: number,
readonly requiresUpdate: boolean,
updateProgress: Context['updateProgress'],
context: Context
}
export function chunked(ctx: Context, defaultChunkSize: number): Chunked {
return new ChunkedImpl(ctx, defaultChunkSize);
}
}
const DefaulUpdateRateMs = 150;
const NoOpSubscribe = () => { }
class ObservableContext implements Computation.Context {
readonly updateRate: number; readonly updateRate: number;
private isSynchronous: boolean; private isSynchronous: boolean;
private level = 0; private level = 0;
private startedTime = 0; private startedTime = 0;
private abortRequested = false; private abortRequested = false;
private lastUpdated = 0; private lastUpdated = 0;
private observers: ProgressObserver[] | undefined = void 0; private observers: Computation.ProgressObserver[] | undefined = void 0;
private progress: Progress = { message: 'Working...', current: 0, max: 0, elapsedMs: 0, isIndeterminate: true, requestAbort: void 0 }; private progress: Computation.Progress = { message: 'Working...', current: 0, max: 0, elapsedMs: 0, isIndeterminate: true, requestAbort: void 0 };
lastDelta = 0; lastDelta = 0;
private checkAborted() { private checkAborted() {
if (this.abortRequested) throw Aborted; if (this.abortRequested) throw Computation.Aborted;
} }
private abortRequester = () => { this.abortRequested = true }; private abortRequester = () => { this.abortRequested = true };
subscribe = (obs: ProgressObserver) => { subscribe = (obs: Computation.ProgressObserver) => {
if (!this.observers) this.observers = []; if (!this.observers) this.observers = [];
this.observers.push(obs); this.observers.push(obs);
} }
...@@ -127,7 +170,7 @@ namespace Computation { ...@@ -127,7 +170,7 @@ namespace Computation {
updateProgress(msg: string, abort?: boolean | (() => void), current?: number, max?: number): Promise<void> | void { updateProgress(msg: string, abort?: boolean | (() => void), current?: number, max?: number): Promise<void> | void {
this.checkAborted(); this.checkAborted();
const time = now(); const time = Computation.now();
if (typeof abort === 'boolean') { if (typeof abort === 'boolean') {
this.progress.requestAbort = abort ? this.abortRequester : void 0; this.progress.requestAbort = abort ? this.abortRequester : void 0;
...@@ -160,11 +203,11 @@ namespace Computation { ...@@ -160,11 +203,11 @@ namespace Computation {
get requiresUpdate() { get requiresUpdate() {
this.checkAborted(); this.checkAborted();
if (this.isSynchronous) return false; if (this.isSynchronous) return false;
return now() - this.lastUpdated > this.updateRate / 2; return Computation.now() - this.lastUpdated > this.updateRate / 2;
} }
started() { started() {
if (!this.level) this.startedTime = now(); if (!this.level) this.startedTime = Computation.now();
this.level++; this.level++;
} }
...@@ -176,13 +219,14 @@ namespace Computation { ...@@ -176,13 +219,14 @@ namespace Computation {
if (!this.level) this.observers = void 0; if (!this.level) this.observers = void 0;
} }
constructor(params?: Partial<Params>) { constructor(params?: Partial<Computation.Params>) {
this.updateRate = (params && params.updateRateMs) || DefaulUpdateRateMs; this.updateRate = (params && params.updateRateMs) || DefaulUpdateRateMs;
this.isSynchronous = !!(params && params.isSynchronous); this.isSynchronous = !!(params && params.isSynchronous);
} }
} }
export class Chunked {
class ChunkedImpl implements Computation.Chunked {
private currentChunkSize: number; private currentChunkSize: number;
private computeChunkSize() { private computeChunkSize() {
...@@ -212,27 +256,9 @@ namespace Computation { ...@@ -212,27 +256,9 @@ namespace Computation {
this.defaultChunkSize = this.computeChunkSize(); this.defaultChunkSize = this.computeChunkSize();
} }
constructor(public context: Context, private defaultChunkSize: number) { constructor(public context: Computation.Context, private defaultChunkSize: number) {
this.currentChunkSize = defaultChunkSize; this.currentChunkSize = defaultChunkSize;
} }
} }
declare var process: any;
declare var window: any;
export const now: () => number = (function () {
if (typeof window !== 'undefined' && window.performance) {
const perf = window.performance;
return function () { return perf.now(); }
} else if (typeof process !== 'undefined' && process.hrtime !== 'undefined') {
return function () {
let t = process.hrtime();
return t[0] * 1000 + t[1] / 1000000;
};
} else {
return function () { return +new Date(); }
}
})();
}
export default Computation; export default Computation;
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment