From ae306d17617a7f644fd85613d0cf93a6218d11f0 Mon Sep 17 00:00:00 2001
From: David Sehnal <david.sehnal@gmail.com>
Date: Fri, 17 Apr 2020 00:06:41 +0200
Subject: [PATCH] data asset handling improvements
---
src/apps/state-docs/pd-to-md.ts | 1 +
src/apps/viewer/index.ts | 3 +-
src/examples/basic-wrapper/index.ts | 3 +-
src/examples/basic-wrapper/superposition.ts | 3 +-
src/examples/lighting/index.ts | 3 +-
src/examples/proteopedia-wrapper/index.ts | 3 +-
src/extensions/cellpack/model.ts | 6 +-
src/mol-plugin-state/actions/structure.ts | 5 +-
src/mol-plugin-state/actions/volume.ts | 21 ++---
src/mol-plugin-state/transforms/data.ts | 62 ++++++++-------
src/mol-plugin-ui/controls/parameters.tsx | 28 +++++++
src/mol-plugin/behavior/static/state.ts | 26 +++----
src/mol-util/assets.ts | 86 +++++++++++----------
src/mol-util/data-source.ts | 23 +++---
src/mol-util/param-definition.ts | 11 ++-
15 files changed, 165 insertions(+), 119 deletions(-)
diff --git a/src/apps/state-docs/pd-to-md.ts b/src/apps/state-docs/pd-to-md.ts
index 8e618fecb..615dcf0ac 100644
--- a/src/apps/state-docs/pd-to-md.ts
+++ b/src/apps/state-docs/pd-to-md.ts
@@ -22,6 +22,7 @@ function paramInfo(param: PD.Any, offset: number): string {
case 'color-list': return `A list of colors as 0xrrggbb`;
case 'vec3': return `3D vector [x, y, z]`;
case 'mat4': return `4x4 transformation matrix`;
+ case 'url': return `URL couple with unique identifier`;
case 'file': return `JavaScript File Handle`;
case 'file-list': return `JavaScript FileList Handle`;
case 'select': return `One of ${oToS(param.options)}`;
diff --git a/src/apps/viewer/index.ts b/src/apps/viewer/index.ts
index f27157cd1..719b18d1b 100644
--- a/src/apps/viewer/index.ts
+++ b/src/apps/viewer/index.ts
@@ -17,6 +17,7 @@ import { PluginConfig } from '../../mol-plugin/config';
import { CellPack } from '../../extensions/cellpack';
import { RCSBAssemblySymmetry, RCSBValidationReport } from '../../extensions/rcsb';
import { PDBeStructureQualityReport } from '../../extensions/pdbe';
+import { Asset } from '../../mol-util/assets';
require('mol-plugin-ui/skin/light.scss');
function getParam(name: string, regex: string): string {
@@ -88,7 +89,7 @@ async function tryLoadFromUrl(ctx: PluginContext) {
source: {
name: 'url',
params: {
- url,
+ url: Asset.Url(url),
format: format as any,
isBinary,
options: params.source.params.options,
diff --git a/src/examples/basic-wrapper/index.ts b/src/examples/basic-wrapper/index.ts
index 4e776090a..ec0f3909e 100644
--- a/src/examples/basic-wrapper/index.ts
+++ b/src/examples/basic-wrapper/index.ts
@@ -19,6 +19,7 @@ import { CustomToastMessage } from './controls';
import './index.html';
import { buildStaticSuperposition, dynamicSuperpositionTest, StaticSuperpositionTestData } from './superposition';
import { PDBeStructureQualityReport } from '../../extensions/pdbe';
+import { Asset } from '../../mol-util/assets';
require('mol-plugin-ui/skin/light.scss');
type LoadParams = { url: string, format?: BuiltInTrajectoryFormat, isBinary?: boolean, assemblyId?: string }
@@ -51,7 +52,7 @@ class BasicWrapper {
async load({ url, format = 'mmcif', isBinary = false, assemblyId = '' }: LoadParams) {
await this.plugin.clear();
- const data = await this.plugin.builders.data.download({ url, isBinary }, { state: { isGhost: true } });
+ const data = await this.plugin.builders.data.download({ url: Asset.Url(url), isBinary }, { state: { isGhost: true } });
const trajectory = await this.plugin.builders.structure.parseTrajectory(data, format);
await this.plugin.builders.structure.hierarchy.applyPreset(trajectory, 'default', {
diff --git a/src/examples/basic-wrapper/superposition.ts b/src/examples/basic-wrapper/superposition.ts
index 1145124b8..342b6782f 100644
--- a/src/examples/basic-wrapper/superposition.ts
+++ b/src/examples/basic-wrapper/superposition.ts
@@ -15,6 +15,7 @@ import { compile } from '../../mol-script/runtime/query/compiler';
import { StateObjectRef } from '../../mol-state';
import { BuiltInTrajectoryFormat } from '../../mol-plugin-state/formats/trajectory';
import { StateTransforms } from '../../mol-plugin-state/transforms';
+import { Asset } from '../../mol-util/assets';
export type SuperpositionTestInput = {
pdbId: string,
@@ -97,7 +98,7 @@ async function siteVisual(plugin: PluginContext, s: StateObjectRef<PSO.Molecule.
}
async function loadStructure(plugin: PluginContext, url: string, format: BuiltInTrajectoryFormat, assemblyId?: string) {
- const data = await plugin.builders.data.download({ url });
+ const data = await plugin.builders.data.download({ url: Asset.Url(url) });
const trajectory = await plugin.builders.structure.parseTrajectory(data, format);
const model = await plugin.builders.structure.createModel(trajectory);
const structure = await plugin.builders.structure.createStructure(model, assemblyId ? { name: 'assembly', params: { id: assemblyId } } : void 0);
diff --git a/src/examples/lighting/index.ts b/src/examples/lighting/index.ts
index 941c843a0..daf051025 100644
--- a/src/examples/lighting/index.ts
+++ b/src/examples/lighting/index.ts
@@ -10,6 +10,7 @@ import { BuiltInTrajectoryFormat } from '../../mol-plugin-state/formats/trajecto
import { PluginCommands } from '../../mol-plugin/commands';
import { PluginContext } from '../../mol-plugin/context';
import './index.html';
+import { Asset } from '../../mol-util/assets';
require('mol-plugin-ui/skin/light.scss');
type LoadParams = { url: string, format?: BuiltInTrajectoryFormat, isBinary?: boolean, assemblyId?: string }
@@ -101,7 +102,7 @@ class LightingDemo {
async load({ url, format = 'mmcif', isBinary = false, assemblyId = '' }: LoadParams) {
await this.plugin.clear();
- const data = await this.plugin.builders.data.download({ url, isBinary }, { state: { isGhost: true } });
+ const data = await this.plugin.builders.data.download({ url: Asset.Url(url), isBinary }, { state: { isGhost: true } });
const trajectory = await this.plugin.builders.structure.parseTrajectory(data, format);
const model = await this.plugin.builders.structure.createModel(trajectory);
const structure = await this.plugin.builders.structure.createStructure(model, assemblyId ? { name: 'assembly', params: { id: assemblyId } } : { name: 'deposited', params: { } });
diff --git a/src/examples/proteopedia-wrapper/index.ts b/src/examples/proteopedia-wrapper/index.ts
index 66d298d24..d09e0e3ca 100644
--- a/src/examples/proteopedia-wrapper/index.ts
+++ b/src/examples/proteopedia-wrapper/index.ts
@@ -28,6 +28,7 @@ import { DefaultCanvas3DParams, Canvas3DProps } from '../../mol-canvas3d/canvas3
import { createStructureRepresentationParams } from '../../mol-plugin-state/helpers/structure-representation-params';
import { download } from '../../mol-util/download';
import { getFormattedTime } from '../../mol-util/date';
+import { Asset } from '../../mol-util/assets';
require('../../mol-plugin-ui/skin/light.scss');
class MolStarProteopediaWrapper {
@@ -74,7 +75,7 @@ class MolStarProteopediaWrapper {
}
private download(b: StateBuilder.To<PSO.Root>, url: string) {
- return b.apply(StateTransforms.Data.Download, { url, isBinary: false });
+ return b.apply(StateTransforms.Data.Download, { url: Asset.Url(url), isBinary: false });
}
private model(b: StateBuilder.To<PSO.Data.Binary | PSO.Data.String>, format: SupportedFormats) {
diff --git a/src/extensions/cellpack/model.ts b/src/extensions/cellpack/model.ts
index 4bf40d9f3..06e0a9718 100644
--- a/src/extensions/cellpack/model.ts
+++ b/src/extensions/cellpack/model.ts
@@ -395,7 +395,7 @@ async function loadMembrane(name: string, plugin: PluginContext, runtime: Runtim
const file = ingredientFiles[fname];
b = b.apply(StateTransforms.Data.ReadFile, { file: Asset.File(file), isBinary: true, label: file.name }, { state: { isGhost: true } });
} else {
- const url = `${params.baseUrl}/membranes/${name}.bcif`;
+ const url = Asset.Url(`${params.baseUrl}/membranes/${name}.bcif`);
b = b.apply(StateTransforms.Data.Download, { url, isBinary: true, label: name }, { state: { isGhost: true } });
}
@@ -412,7 +412,7 @@ async function loadMembrane(name: string, plugin: PluginContext, runtime: Runtim
}
async function loadHivMembrane(plugin: PluginContext, runtime: RuntimeContext, state: State, params: LoadCellPackModelParams) {
- const url = `${params.baseUrl}/membranes/hiv_lipids.bcif`;
+ const url = Asset.Url(`${params.baseUrl}/membranes/hiv_lipids.bcif`);
const membrane = await state.build().toRoot()
.apply(StateTransforms.Data.Download, { label: 'hiv_lipids', url, isBinary: true }, { state: { isGhost: true } })
.apply(StateTransforms.Data.ParseCif, undefined, { state: { isGhost: true } })
@@ -431,7 +431,7 @@ async function loadHivMembrane(plugin: PluginContext, runtime: RuntimeContext, s
async function loadPackings(plugin: PluginContext, runtime: RuntimeContext, state: State, params: LoadCellPackModelParams) {
let cellPackJson: StateBuilder.To<PSO.Format.Json, StateTransformer<PSO.Data.String, PSO.Format.Json>>;
if (params.source.name === 'id') {
- const url = getCellPackModelUrl(params.source.params, params.baseUrl);
+ const url = Asset.Url(getCellPackModelUrl(params.source.params, params.baseUrl));
cellPackJson = state.build().toRoot()
.apply(StateTransforms.Data.Download, { url, isBinary: false, label: params.source.params }, { state: { isGhost: true } });
} else {
diff --git a/src/mol-plugin-state/actions/structure.ts b/src/mol-plugin-state/actions/structure.ts
index 4f5a8c475..31a560860 100644
--- a/src/mol-plugin-state/actions/structure.ts
+++ b/src/mol-plugin-state/actions/structure.ts
@@ -16,6 +16,7 @@ import { PluginStateObject } from '../objects';
import { StateTransforms } from '../transforms';
import { Download } from '../transforms/data';
import { CustomModelProperties, CustomStructureProperties, TrajectoryFromModelAndCoordinates } from '../transforms/model';
+import { Asset } from '../../mol-util/assets';
const DownloadModelRepresentationOptions = (plugin: PluginContext) => PD.Group({
type: RootStructureDefinition.getParams(void 0, 'auto').type,
@@ -65,7 +66,7 @@ const DownloadStructure = StateAction.build({
options
}, { isFlat: true, label: 'PubChem', description: 'Loads 3D conformer from PubChem.' }),
'url': PD.Group({
- url: PD.Text(''),
+ url: PD.Url(''),
format: PD.Select<BuiltInTrajectoryFormat>('mmcif', PD.arrayToOptions(BuiltInTrajectoryFormats.map(f => f[0]), f => f)),
isBinary: PD.Boolean(false),
options
@@ -160,7 +161,7 @@ function getDownloadParams(src: string, url: (id: string) => string, label: (id:
const ids = src.split(',').map(id => id.trim()).filter(id => !!id && (id.length >= 4 || /^[1-9][0-9]*$/.test(id)));
const ret: StateTransformer.Params<Download>[] = [];
for (const id of ids) {
- ret.push({ url: url(id), isBinary, label: label(id) });
+ ret.push({ url: Asset.Url(url(id)), isBinary, label: label(id) });
}
return ret;
}
diff --git a/src/mol-plugin-state/actions/volume.ts b/src/mol-plugin-state/actions/volume.ts
index 178fce410..8827bb879 100644
--- a/src/mol-plugin-state/actions/volume.ts
+++ b/src/mol-plugin-state/actions/volume.ts
@@ -13,6 +13,7 @@ import { ParamDefinition as PD } from '../../mol-util/param-definition';
import { PluginStateObject } from '../objects';
import { Download } from '../transforms/data';
import { DataFormatProvider } from '../formats/provider';
+import { Asset } from '../../mol-util/assets';
export { DownloadDensity };
type DownloadDensity = typeof DownloadDensity
@@ -45,7 +46,7 @@ const DownloadDensity = StateAction.build({
detail: PD.Numeric(3, { min: 0, max: 10, step: 1 }, { label: 'Detail' }),
}, { isFlat: true }),
'url': PD.Group({
- url: PD.Text(''),
+ url: PD.Url(''),
isBinary: PD.Boolean(false),
format: PD.Select('auto', options),
}, { isFlat: true })
@@ -70,37 +71,37 @@ const DownloadDensity = StateAction.build({
break;
case 'pdb-xray':
downloadParams = src.params.provider.server === 'pdbe' ? {
- url: src.params.type === '2fofc'
+ url: Asset.Url(src.params.type === '2fofc'
? `http://www.ebi.ac.uk/pdbe/coordinates/files/${src.params.provider.id.toLowerCase()}.ccp4`
- : `http://www.ebi.ac.uk/pdbe/coordinates/files/${src.params.provider.id.toLowerCase()}_diff.ccp4`,
+ : `http://www.ebi.ac.uk/pdbe/coordinates/files/${src.params.provider.id.toLowerCase()}_diff.ccp4`),
isBinary: true,
label: `PDBe X-ray map: ${src.params.provider.id}`
} : {
- url: src.params.type === '2fofc'
+ url: Asset.Url(src.params.type === '2fofc'
? `https://edmaps.rcsb.org/maps/${src.params.provider.id.toLowerCase()}_2fofc.dsn6`
- : `https://edmaps.rcsb.org/maps/${src.params.provider.id.toLowerCase()}_fofc.dsn6`,
+ : `https://edmaps.rcsb.org/maps/${src.params.provider.id.toLowerCase()}_fofc.dsn6`),
isBinary: true,
label: `RCSB X-ray map: ${src.params.provider.id}`
};
break;
case 'pdb-emd-ds':
downloadParams = src.params.provider.server === 'pdbe' ? {
- url: `https://www.ebi.ac.uk/pdbe/densities/emd/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+ url: Asset.Url(`https://www.ebi.ac.uk/pdbe/densities/emd/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
isBinary: true,
label: `PDBe EMD Density Server: ${src.params.provider.id}`
} : {
- url: `https://maps.rcsb.org/em/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+ url: Asset.Url(`https://maps.rcsb.org/em/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
isBinary: true,
label: `RCSB PDB EMD Density Server: ${src.params.provider.id}`
};
break;
case 'pdb-xray-ds':
downloadParams = src.params.provider.server === 'pdbe' ? {
- url: `https://www.ebi.ac.uk/pdbe/densities/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+ url: Asset.Url(`https://www.ebi.ac.uk/pdbe/densities/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
isBinary: true,
label: `PDBe X-ray Density Server: ${src.params.provider.id}`
} : {
- url: `https://maps.rcsb.org/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+ url: Asset.Url(`https://maps.rcsb.org/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
isBinary: true,
label: `RCSB PDB X-ray Density Server: ${src.params.provider.id}`
};
@@ -113,7 +114,7 @@ const DownloadDensity = StateAction.build({
switch (src.name) {
case 'url':
downloadParams = src.params;
- provider = src.params.format === 'auto' ? plugin.dataFormats.auto(getFileInfo(downloadParams.url), data.cell?.obj!) : plugin.dataFormats.get(src.params.format);
+ provider = src.params.format === 'auto' ? plugin.dataFormats.auto(getFileInfo(downloadParams.url.url), data.cell?.obj!) : plugin.dataFormats.get(src.params.format);
break;
case 'pdb-xray':
provider = src.params.provider.server === 'pdbe'
diff --git a/src/mol-plugin-state/transforms/data.ts b/src/mol-plugin-state/transforms/data.ts
index 9c856181d..9beac1756 100644
--- a/src/mol-plugin-state/transforms/data.ts
+++ b/src/mol-plugin-state/transforms/data.ts
@@ -32,6 +32,7 @@ export { ParseDsn6 };
export { ImportString };
export { ImportJson };
export { ParseJson };
+
type Download = typeof Download
const Download = PluginStateTransform.BuiltIn({
name: 'download',
@@ -39,29 +40,27 @@ const Download = PluginStateTransform.BuiltIn({
from: [SO.Root],
to: [SO.Data.String, SO.Data.Binary],
params: {
- url: PD.Text('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
+ url: PD.Url('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
label: PD.Optional(PD.Text('')),
- isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, download data as binary (string otherwise)' })),
- body: PD.Optional(PD.Text(''))
+ isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, download data as binary (string otherwise)' }))
}
})({
- apply({ params: p }, plugin: PluginContext) {
+ apply({ params: p, cache }, plugin: PluginContext) {
return Task.create('Download', async ctx => {
- const data = await plugin.managers.asset.resolve(Asset.Url(p.url, { body: p.body }), p.isBinary ? 'binary' : 'string').runInContext(ctx);
+ const asset = await plugin.managers.asset.resolve(p.url, p.isBinary ? 'binary' : 'string').runInContext(ctx);
+ (cache as any).asset = asset;
return p.isBinary
- ? new SO.Data.Binary(data as Uint8Array, { label: p.label ? p.label : p.url })
- : new SO.Data.String(data as string, { label: p.label ? p.label : p.url });
+ ? new SO.Data.Binary(asset.data as Uint8Array, { label: p.label ? p.label : p.url.url })
+ : new SO.Data.String(asset.data as string, { label: p.label ? p.label : p.url.url });
});
},
- dispose({ params: p }, plugin: PluginContext) {
- if (p) {
- plugin.managers.asset.release(Asset.Url(p.url, { body: p.body }));
- }
+ dispose({ cache }) {
+ ((cache as any)?.asset as Asset.Wrapper | undefined)?.dispose();
},
update({ oldParams, newParams, b }) {
if (oldParams.url !== newParams.url || oldParams.isBinary !== newParams.isBinary) return StateTransformer.UpdateResult.Recreate;
if (oldParams.label !== newParams.label) {
- b.label = newParams.label || newParams.url;
+ b.label = newParams.label || newParams.url.url;
return StateTransformer.UpdateResult.Updated;
}
return StateTransformer.UpdateResult.Unchanged;
@@ -77,35 +76,39 @@ const DownloadBlob = PluginStateTransform.BuiltIn({
params: {
sources: PD.ObjectList({
id: PD.Text('', { label: 'Unique ID' }),
- url: PD.Text('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
+ url: PD.Url('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, download data as binary (string otherwise)' })),
- body: PD.Optional(PD.Text('')),
canFail: PD.Optional(PD.Boolean(false, { description: 'Indicate whether the download can fail and not be included in the blob as a result.' }))
}, e => `${e.id}: ${e.url}`),
maxConcurrency: PD.Optional(PD.Numeric(4, { min: 1, max: 12, step: 1 }, { description: 'The maximum number of concurrent downloads.' }))
}
})({
- apply({ params }, plugin: PluginContext) {
+ apply({ params, cache }, plugin: PluginContext) {
return Task.create('Download Blob', async ctx => {
const entries: SO.Data.BlobEntry[] = [];
- const data = await ajaxGetMany(ctx, params.sources, params.maxConcurrency || 4, plugin.managers.asset);
+ const data = await ajaxGetMany(ctx, plugin.managers.asset, params.sources, params.maxConcurrency || 4);
+
+ const assets: Asset.Wrapper[] = [];
for (let i = 0; i < data.length; i++) {
const r = data[i], src = params.sources[i];
if (r.kind === 'error') plugin.log.warn(`Download ${r.id} (${src.url}) failed: ${r.error}`);
else {
+ assets.push(r.result);
entries.push(src.isBinary
- ? { id: r.id, kind: 'binary', data: r.result as Uint8Array }
- : { id: r.id, kind: 'string', data: r.result as string });
+ ? { id: r.id, kind: 'binary', data: r.result.data as Uint8Array }
+ : { id: r.id, kind: 'string', data: r.result.data as string });
}
}
+ (cache as any).assets = assets;
return new SO.Data.Blob(entries, { label: 'Data Blob', description: `${entries.length} ${entries.length === 1 ? 'entry' : 'entries'}` });
});
},
- dispose({ params }, plugin: PluginContext) {
- if (!params) return;
- for (const s of params.sources) {
- plugin.managers.asset.release({ url: s.url, body: s.body });
+ dispose({ cache }, plugin: PluginContext) {
+ const assets: Asset.Wrapper[] | undefined = (cache as any)?.assets;
+ if (!assets) return;
+ for (const a of assets) {
+ a.dispose();
}
}
// TODO: ??
@@ -163,25 +166,24 @@ const ReadFile = PluginStateTransform.BuiltIn({
isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, open file as as binary (string otherwise)' }))
}
})({
- apply({ params: p }, plugin: PluginContext) {
+ apply({ params: p, cache }, plugin: PluginContext) {
return Task.create('Open File', async ctx => {
if (p.file === null) {
plugin.log.error('No file(s) selected');
return StateObject.Null;
}
- const data = await plugin.managers.asset.resolve(p.file, p.isBinary ? 'binary' : 'string').runInContext(ctx);
+ const asset = await plugin.managers.asset.resolve(p.file, p.isBinary ? 'binary' : 'string').runInContext(ctx);
+ (cache as any).asset = asset;
const o = p.isBinary
- ? new SO.Data.Binary(data as Uint8Array, { label: p.label ? p.label : p.file.name })
- : new SO.Data.String(data as string, { label: p.label ? p.label : p.file.name });
+ ? new SO.Data.Binary(asset.data as Uint8Array, { label: p.label ? p.label : p.file.name })
+ : new SO.Data.String(asset.data as string, { label: p.label ? p.label : p.file.name });
return o;
});
},
- dispose({ params }, plugin: PluginContext) {
- if (params?.file) {
- plugin.managers.asset.release(params.file);
- }
+ dispose({ cache }) {
+ ((cache as any)?.asset as Asset.Wrapper | undefined)?.dispose();
},
update({ oldParams, newParams, b }) {
if (oldParams.label !== newParams.label) {
diff --git a/src/mol-plugin-ui/controls/parameters.tsx b/src/mol-plugin-ui/controls/parameters.tsx
index 45225b335..c8c0a879e 100644
--- a/src/mol-plugin-ui/controls/parameters.tsx
+++ b/src/mol-plugin-ui/controls/parameters.tsx
@@ -181,6 +181,7 @@ function controlFor(param: PD.Any): ParamControl | undefined {
case 'color-list': return ColorListControl;
case 'vec3': return Vec3Control;
case 'mat4': return Mat4Control;
+ case 'url': return UrlControl;
case 'file': return FileControl;
case 'file-list': return FileListControl;
case 'select': return SelectControl;
@@ -786,6 +787,33 @@ export class Mat4Control extends React.PureComponent<ParamProps<PD.Mat4>, { isEx
}
}
+export class UrlControl extends SimpleParam<PD.UrlParam> {
+ onChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+ const value = e.target.value;
+ if (value !== this.props.value.url) {
+ this.update(Asset.Url(value));
+ }
+ }
+
+ onKeyPress = (e: React.KeyboardEvent<HTMLInputElement>) => {
+ if ((e.keyCode === 13 || e.charCode === 13)) {
+ if (this.props.onEnter) this.props.onEnter();
+ }
+ e.stopPropagation();
+ }
+
+ renderControl() {
+ const placeholder = this.props.param.label || camelCaseToWords(this.props.name);
+ return <input type='text'
+ value={this.props.value.url || ''}
+ placeholder={placeholder}
+ onChange={this.onChange}
+ onKeyPress={this.props.onEnter ? this.onKeyPress : void 0}
+ disabled={this.props.isDisabled}
+ />;
+ }
+}
+
export class FileControl extends React.PureComponent<ParamProps<PD.FileParam>> {
change(value: File) {
this.props.onChange({ name: this.props.name, param: this.props.param, value: Asset.File(value) });
diff --git a/src/mol-plugin/behavior/static/state.ts b/src/mol-plugin/behavior/static/state.ts
index fcfce4555..20f859de7 100644
--- a/src/mol-plugin/behavior/static/state.ts
+++ b/src/mol-plugin/behavior/static/state.ts
@@ -5,22 +5,20 @@
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
-import { PluginCommands } from '../../commands';
-import { PluginContext } from '../../context';
-import { StateTree, StateTransform, State } from '../../../mol-state';
-import { PluginStateSnapshotManager } from '../../../mol-plugin-state/snapshots';
+import { utf8ByteCount, utf8Write } from '../../../mol-io/common/utf8';
+import { Structure } from '../../../mol-model/structure';
import { PluginStateObject as SO } from '../../../mol-plugin-state/objects';
-import { getFormattedTime } from '../../../mol-util/date';
+import { PluginStateSnapshotManager } from '../../../mol-plugin-state/snapshots';
+import { State, StateTransform, StateTree } from '../../../mol-state';
import { readFromFile } from '../../../mol-util/data-source';
+import { getFormattedTime } from '../../../mol-util/date';
import { download } from '../../../mol-util/download';
-import { Structure } from '../../../mol-model/structure';
+import { objectForEach } from '../../../mol-util/object';
import { urlCombine } from '../../../mol-util/url';
-import { PluginConfig } from '../../config';
import { zip } from '../../../mol-util/zip/zip';
-import { utf8Write, utf8ByteCount } from '../../../mol-io/common/utf8';
-import { objectForEach } from '../../../mol-util/object';
-import { UUID } from '../../../mol-util';
-import { Asset } from '../../../mol-util/assets';
+import { PluginCommands } from '../../commands';
+import { PluginConfig } from '../../config';
+import { PluginContext } from '../../context';
export function registerDefault(ctx: PluginContext) {
SyncBehaviors(ctx);
@@ -201,10 +199,10 @@ export function Snapshots(ctx: PluginContext) {
const assets: any[] = [];
+ // TODO: there can be duplicate entries: check for this?
for (const { asset, file } of ctx.managers.asset.assets) {
- const id = Asset.isFile(asset) ? asset.id : UUID.create22();
- assets.push([id, asset]);
- zipDataObj[`assets/${id}`] = new Uint8Array(await file.arrayBuffer());
+ assets.push([asset.id, asset]);
+ zipDataObj[`assets/${asset.id}`] = new Uint8Array(await file.arrayBuffer());
}
if (assets.length > 0) {
diff --git a/src/mol-util/assets.ts b/src/mol-util/assets.ts
index bc89f3264..ba06cfec4 100644
--- a/src/mol-util/assets.ts
+++ b/src/mol-util/assets.ts
@@ -16,27 +16,40 @@ type _File = File;
type Asset = Asset.Url | Asset.File
namespace Asset {
- export type Url = { url: string, title?: string, body?: string }
- export type File = { id: UUID, name: string, file?: _File }
+ export type Url = { kind: 'url', id: UUID, url: string, title?: string, body?: string }
+ export type File = { kind: 'file', id: UUID, name: string, file?: _File }
export function Url(url: string, options?: { body?: string, title?: string }): Url {
- return { url, ...options };
+ return { kind: 'url', id: UUID.create22(), url, ...options };
}
export function File(file: _File): File {
- return { id: UUID.create22(), name: file.name, file };
+ return { kind: 'file', id: UUID.create22(), name: file.name, file };
}
- export function isUrl(x: Asset): x is Url {
- return !!x && !!(x as any).url;
+ export function isUrl(x?: Asset): x is Url {
+ return x?.kind === 'url';
}
- export function isFile(x: Asset): x is File {
- return !!x && !!(x as any).id;
+ export function isFile(x?: Asset): x is File {
+ return x?.kind === 'file';
+ }
+
+ export class Wrapper<T extends DataType = DataType> {
+ dispose() {
+ this.manager.release(this.asset);
+ }
+
+ constructor(public readonly data: DataResponse<T>, private asset: Asset, private manager: AssetManager) {
+
+ }
}
}
class AssetManager {
+ // TODO: add URL based ref-counted cache?
+ // TODO: when serializing, check for duplicates?
+
private _assets = new Map<string, { asset: Asset, file: File }>();
get assets() {
@@ -44,51 +57,42 @@ class AssetManager {
}
set(asset: Asset, file: File) {
- if (Asset.isUrl(asset)) {
- this._assets.set(getUrlKey(asset), { asset, file });
- } else {
- this._assets.set(asset.id, { asset, file });
- }
+ this._assets.set(asset.id, { asset, file });
}
- resolve<T extends DataType>(asset: Asset, type: T, store = true): Task<DataResponse<T>> {
+ resolve<T extends DataType>(asset: Asset, type: T, store = true): Task<Asset.Wrapper<T>> {
if (Asset.isUrl(asset)) {
- const key = getUrlKey(asset);
- if (this._assets.has(key)) {
- return readFromFile(this._assets.get(key)!.file, type);
- }
+ return Task.create(`Download ${asset.title || asset.url}`, async ctx => {
+ if (this._assets.has(asset.id)) {
+ return new Asset.Wrapper(await readFromFile(this._assets.get(asset.id)!.file, type).runInContext(ctx), asset, this);
+ }
- if (!store) {
- return ajaxGet({ ...asset, type });
- }
+ if (!store) {
+ return new Asset.Wrapper(await ajaxGet({ ...asset, type }).runInContext(ctx), asset, this);
+ }
- return Task.create(`Download ${asset.title || asset.url}`, async ctx => {
const data = await ajaxGet({ ...asset, type: 'binary' }).runInContext(ctx);
const file = new File([data], 'raw-data');
- this._assets.set(key, { asset, file });
- return await readFromFile(file, type).runInContext(ctx);
+ this._assets.set(asset.id, { asset, file });
+ return new Asset.Wrapper(await readFromFile(file, type).runInContext(ctx), asset, this);
});
} else {
- if (this._assets.has(asset.id)) return readFromFile(this._assets.get(asset.id)!.file, type);
- if (!(asset.file instanceof File)) {
- return Task.fail('Resolve asset', `Cannot resolve file asset '${asset.name}' (${asset.id})`);
- }
- if (store) {
- this._assets.set(asset.id, { asset, file: asset.file });
- }
- return readFromFile(asset.file, type);
+ return Task.create(`Read ${asset.name}`, async ctx => {
+ if (this._assets.has(asset.id)) {
+ return new Asset.Wrapper(await readFromFile(this._assets.get(asset.id)!.file, type).runInContext(ctx), asset, this);
+ }
+ if (!(asset.file instanceof File)) {
+ throw new Error(`Cannot resolve file asset '${asset.name}' (${asset.id})`);
+ }
+ if (store) {
+ this._assets.set(asset.id, { asset, file: asset.file });
+ }
+ return new Asset.Wrapper(await readFromFile(asset.file, type).runInContext(ctx), asset, this);
+ });
}
}
release(asset: Asset) {
- if (Asset.isFile(asset)) {
- this._assets.delete(asset.id);
- } else {
- this._assets.delete(getUrlKey(asset));
- }
+ this._assets.delete(asset.id);
}
-}
-
-function getUrlKey(asset: Asset.Url) {
- return asset.body ? `${asset.url}_${asset.body || ''}` : asset.url;
}
\ No newline at end of file
diff --git a/src/mol-util/data-source.ts b/src/mol-util/data-source.ts
index 30b568eb0..49d44d27d 100644
--- a/src/mol-util/data-source.ts
+++ b/src/mol-util/data-source.ts
@@ -10,7 +10,7 @@
import { Task, RuntimeContext } from '../mol-task';
import { unzip, ungzip } from './zip/zip';
import { utf8Read } from '../mol-io/common/utf8';
-import { AssetManager } from './assets';
+import { AssetManager, Asset } from './assets';
// polyfill XMLHttpRequest in node.js
const XHR = typeof document === 'undefined' ? require('xhr2') as {
@@ -281,23 +281,19 @@ function ajaxGetInternal<T extends DataType>(title: string | undefined, url: str
});
}
-export type AjaxGetManyEntry<T> = { kind: 'ok', id: string, result: T } | { kind: 'error', id: string, error: any }
-export async function ajaxGetMany(ctx: RuntimeContext, sources: { id: string, url: string, isBinary?: boolean, body?: string, canFail?: boolean }[], maxConcurrency: number, assetManager?: AssetManager) {
+export type AjaxGetManyEntry = { kind: 'ok', id: string, result: Asset.Wrapper<'string' | 'binary'> } | { kind: 'error', id: string, error: any }
+export async function ajaxGetMany(ctx: RuntimeContext, assetManager: AssetManager, sources: { id: string, url: Asset.Url, isBinary?: boolean, canFail?: boolean }[], maxConcurrency: number) {
const len = sources.length;
- const slots: AjaxGetManyEntry<string | Uint8Array>[] = new Array(sources.length);
+ const slots: AjaxGetManyEntry[] = new Array(sources.length);
await ctx.update({ message: 'Downloading...', current: 0, max: len });
- let promises: Promise<AjaxGetManyEntry<any> & { index: number }>[] = [], promiseKeys: number[] = [];
+ let promises: Promise<AjaxGetManyEntry & { index: number }>[] = [], promiseKeys: number[] = [];
let currentSrc = 0;
for (let _i = Math.min(len, maxConcurrency); currentSrc < _i; currentSrc++) {
const current = sources[currentSrc];
- if (assetManager) {
- promises.push(wrapPromise(currentSrc, current.id,
- assetManager.resolve({ url: current.url, body: current.body }, current.isBinary ? 'binary' : 'string').runAsChild(ctx)));
- } else {
- promises.push(wrapPromise(currentSrc, current.id, ajaxGet({ url: current.url, type: current.isBinary ? 'binary' : 'string' }).runAsChild(ctx)));
- }
+ promises.push(wrapPromise(currentSrc, current.id,
+ assetManager.resolve(current.url, current.isBinary ? 'binary' : 'string').runAsChild(ctx)));
promiseKeys.push(currentSrc);
}
@@ -319,7 +315,8 @@ export async function ajaxGetMany(ctx: RuntimeContext, sources: { id: string, ur
promiseKeys = promiseKeys.filter(_filterRemoveIndex, idx);
if (currentSrc < len) {
const current = sources[currentSrc];
- promises.push(wrapPromise(currentSrc, current.id, ajaxGet({ url: current.url, type: current.isBinary ? 'binary' : 'string', body: current.body }).runAsChild(ctx)));
+ const asset = assetManager.resolve(current.url, current.isBinary ? 'binary' : 'string').runAsChild(ctx);
+ promises.push(wrapPromise(currentSrc, current.id, asset));
promiseKeys.push(currentSrc);
currentSrc++;
}
@@ -332,7 +329,7 @@ function _filterRemoveIndex(this: number, _: any, i: number) {
return this !== i;
}
-async function wrapPromise<T>(index: number, id: string, p: Promise<T>): Promise<AjaxGetManyEntry<T> & { index: number }> {
+async function wrapPromise(index: number, id: string, p: Promise<Asset.Wrapper<'string' | 'binary'>>): Promise<AjaxGetManyEntry & { index: number }> {
try {
const result = await p;
return { kind: 'ok', result, index, id };
diff --git a/src/mol-util/param-definition.ts b/src/mol-util/param-definition.ts
index 809bf38c6..b6c53baa0 100644
--- a/src/mol-util/param-definition.ts
+++ b/src/mol-util/param-definition.ts
@@ -148,6 +148,15 @@ export namespace ParamDefinition {
return setInfo<Mat4>({ type: 'mat4', defaultValue }, info);
}
+ export interface UrlParam extends Base<Asset.Url> {
+ type: 'url'
+ }
+ export function Url(url: string | { url: string, body?: string }, info?: Info): UrlParam {
+ const defaultValue = typeof url === 'string' ? Asset.Url(url) : Asset.Url(url.url, { body: url.body });
+ const ret = setInfo<UrlParam>({ type: 'url', defaultValue }, info);
+ return ret;
+ }
+
export interface FileParam extends Base<Asset.File | null> {
type: 'file'
accept?: string
@@ -298,7 +307,7 @@ export namespace ParamDefinition {
}
export type Any =
- | Value<any> | Select<any> | MultiSelect<any> | BooleanParam | Text | Color | Vec3 | Mat4 | Numeric | FileParam | FileListParam | Interval | LineGraph
+ | Value<any> | Select<any> | MultiSelect<any> | BooleanParam | Text | Color | Vec3 | Mat4 | Numeric | FileParam | UrlParam | FileListParam | Interval | LineGraph
| ColorList | Group<any> | Mapped<any> | Converted<any, any> | Conditioned<any, any, any> | Script | ObjectList
export type Params = { [k: string]: Any }
--
GitLab