From efc8144acc1cb59e45d7be57063c275572385df3 Mon Sep 17 00:00:00 2001
From: David Sehnal <david.sehnal@gmail.com>
Date: Wed, 21 Nov 2018 12:00:59 +0100
Subject: [PATCH] ajax download with progress tracking
---
src/mol-plugin/context.ts | 8 +-
src/mol-plugin/state/transforms/data.ts | 3 +-
src/mol-util/data-source.ts | 187 ++++++++++++++++++++++++
3 files changed, 193 insertions(+), 5 deletions(-)
create mode 100644 src/mol-util/data-source.ts
diff --git a/src/mol-plugin/context.ts b/src/mol-plugin/context.ts
index 38bf3dcb9..af037145a 100644
--- a/src/mol-plugin/context.ts
+++ b/src/mol-plugin/context.ts
@@ -23,6 +23,7 @@ import { PluginState } from './state';
import { TaskManager } from './util/task-manager';
import { Color } from 'mol-util/color';
import { LociLabelEntry, LociLabelManager } from './util/loci-label-manager';
+import { ajaxGet } from 'mol-util/data-source';
export class PluginContext {
private disposed = false;
@@ -96,9 +97,10 @@ export class PluginContext {
* This should be used in all transform related request so that it could be "spoofed" to allow
* "static" access to resources.
*/
- async fetch(url: string, type: 'string' | 'binary' = 'string'): Promise<string | Uint8Array> {
- const req = await fetch(url, { referrerPolicy: 'origin-when-cross-origin' });
- return type === 'string' ? await req.text() : new Uint8Array(await req.arrayBuffer());
+ fetch(url: string, type: 'string' | 'binary' = 'string'): Task<string | Uint8Array> {
+ return ajaxGet({ url, type });
+ //const req = await fetch(url, { referrerPolicy: 'origin-when-cross-origin' });
+ // return type === 'string' ? await req.text() : new Uint8Array(await req.arrayBuffer());
}
runTask<T>(task: Task<T>) {
diff --git a/src/mol-plugin/state/transforms/data.ts b/src/mol-plugin/state/transforms/data.ts
index d2f76a919..d663997c5 100644
--- a/src/mol-plugin/state/transforms/data.ts
+++ b/src/mol-plugin/state/transforms/data.ts
@@ -29,8 +29,7 @@ const Download = PluginStateTransform.Create<SO.Root, SO.Data.String | SO.Data.B
}),
apply({ params: p }, globalCtx: PluginContext) {
return Task.create('Download', async ctx => {
- // TODO: track progress
- const data = await globalCtx.fetch(p.url, p.isBinary ? 'binary' : 'string');
+ const data = await globalCtx.fetch(p.url, p.isBinary ? 'binary' : 'string').runInContext(ctx);
return p.isBinary
? new SO.Data.Binary(data as Uint8Array, { label: p.label ? p.label : p.url })
: new SO.Data.String(data as string, { label: p.label ? p.label : p.url });
diff --git a/src/mol-util/data-source.ts b/src/mol-util/data-source.ts
new file mode 100644
index 000000000..8545b9397
--- /dev/null
+++ b/src/mol-util/data-source.ts
@@ -0,0 +1,187 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ *
+ * Adapted from LiteMol
+ */
+
+import { Task, RuntimeContext } from 'mol-task';
+import { utf8Read } from 'mol-io/common/utf8';
+
+export enum DataCompressionMethod {
+ None,
+ Gzip
+}
+
+export interface AjaxGetParams {
+ url: string,
+ type: 'string' | 'binary',
+ title?: string,
+ compression?: DataCompressionMethod
+}
+
+export function readStringFromFile(file: File) {
+ return <Task<string>>readFromFileInternal(file, false);
+}
+
+export function readUint8ArrayFromFile(file: File) {
+ return <Task<Uint8Array>>readFromFileInternal(file, true);
+}
+
+export function readFromFile(file: File, type: 'string' | 'binary') {
+ return <Task<Uint8Array | string>>readFromFileInternal(file, type === 'binary');
+}
+
+export function ajaxGetString(url: string, title?: string) {
+ return <Task<string>>ajaxGetInternal(title, url, false, false);
+}
+
+export function ajaxGetUint8Array(url: string, title?: string) {
+ return <Task<Uint8Array>>ajaxGetInternal(title, url, true, false);
+}
+
+export function ajaxGet(params: AjaxGetParams) {
+ return <Task<string | Uint8Array>>ajaxGetInternal(params.title, params.url, params.type === 'binary', params.compression === DataCompressionMethod.Gzip);
+}
+
+function decompress(buffer: Uint8Array): Uint8Array {
+ // TODO
+ throw 'nyi';
+ // const gzip = new LiteMolZlib.Gunzip(new Uint8Array(buffer));
+ // return gzip.decompress();
+}
+
+async function processFile(ctx: RuntimeContext, asUint8Array: boolean, compressed: boolean, e: any) {
+ const data = (e.target as FileReader).result;
+
+ if (compressed) {
+ await ctx.update('Decompressing...');
+
+ const decompressed = decompress(new Uint8Array(data as ArrayBuffer));
+ if (asUint8Array) {
+ return decompressed;
+ } else {
+ return utf8Read(decompressed, 0, decompressed.length);
+ }
+ } else {
+ return asUint8Array ? new Uint8Array(data as ArrayBuffer) : data as string;
+ }
+}
+
+function readData(ctx: RuntimeContext, action: string, data: XMLHttpRequest | FileReader, asUint8Array: boolean): Promise<any> {
+ return new Promise<any>((resolve, reject) => {
+ data.onerror = (e: any) => {
+ const error = (<FileReader>e.target).error;
+ reject(error ? error : 'Failed.');
+ };
+
+ data.onabort = () => reject(Task.Aborted(''));
+
+ data.onprogress = (e: ProgressEvent) => {
+ if (e.lengthComputable) {
+ ctx.update({ message: action, isIndeterminate: false, current: e.loaded, max: e.total });
+ } else {
+ ctx.update({ message: `${action} ${(e.loaded / 1024 / 1024).toFixed(2)} MB`, isIndeterminate: true });
+ }
+ }
+ data.onload = (e: any) => resolve(e);
+ });
+}
+
+function readFromFileInternal(file: File, asUint8Array: boolean): Task<string | Uint8Array> {
+ let reader: FileReader | undefined = void 0;
+ return Task.create('Read File', async ctx => {
+ try {
+ reader = new FileReader();
+ const isCompressed = /\.gz$/i.test(file.name);
+
+ if (isCompressed || asUint8Array) reader.readAsArrayBuffer(file);
+ else reader.readAsBinaryString(file);
+
+ ctx.update({ message: 'Opening file...', canAbort: true });
+ const e = await readData(ctx, 'Reading...', reader, asUint8Array);
+ const result = processFile(ctx, asUint8Array, isCompressed, e);
+ return result;
+ } finally {
+ reader = void 0;
+ }
+ }, () => {
+ if (reader) reader.abort();
+ });
+}
+
+class RequestPool {
+ private static pool: XMLHttpRequest[] = [];
+ private static poolSize = 15;
+
+ static get() {
+ if (this.pool.length) {
+ return this.pool.pop()!;
+ }
+ return new XMLHttpRequest();
+ }
+
+ static emptyFunc() { }
+
+ static deposit(req: XMLHttpRequest) {
+ if (this.pool.length < this.poolSize) {
+ req.onabort = RequestPool.emptyFunc;
+ req.onerror = RequestPool.emptyFunc;
+ req.onload = RequestPool.emptyFunc;
+ req.onprogress = RequestPool.emptyFunc;
+ this.pool.push(req);
+ }
+ }
+}
+
+async function processAjax(ctx: RuntimeContext, asUint8Array: boolean, decompressGzip: boolean, e: any) {
+ const req = (e.target as XMLHttpRequest);
+ if (req.status >= 200 && req.status < 400) {
+ if (asUint8Array) {
+ const buff = new Uint8Array(e.target.response);
+ RequestPool.deposit(e.target);
+
+ if (decompressGzip) {
+ return decompress(buff);
+ } else {
+ return buff;
+ }
+ }
+ else {
+ const text = e.target.responseText;
+ RequestPool.deposit(e.target);
+ return text;
+ }
+ } else {
+ const status = req.statusText;
+ RequestPool.deposit(e.target);
+ throw status;
+ }
+}
+
+function ajaxGetInternal(title: string | undefined, url: string, asUint8Array: boolean, decompressGzip: boolean): Task<string | Uint8Array> {
+ let xhttp: XMLHttpRequest | undefined = void 0;
+ return Task.create(title ? title : 'Download', async ctx => {
+ try {
+ if (!asUint8Array && decompressGzip) {
+ throw 'Decompress is only available when downloading binary data.';
+ }
+
+ xhttp = RequestPool.get();
+
+ xhttp.open('get', url, true);
+ xhttp.responseType = asUint8Array ? 'arraybuffer' : 'text';
+ xhttp.send();
+
+ ctx.update({ message: 'Waiting for server...', canAbort: true });
+ const e = await readData(ctx, 'Downloading...', xhttp, asUint8Array);
+ const result = await processAjax(ctx, asUint8Array, decompressGzip, e)
+ return result;
+ } finally {
+ xhttp = void 0;
+ }
+ }, () => {
+ if (xhttp) xhttp.abort();
+ });
+}
\ No newline at end of file
--
GitLab