From 65d3355b182823e3cdb82c425435714ae0d2e0be Mon Sep 17 00:00:00 2001
From: David Sehnal <david.sehnal@gmail.com>
Date: Mon, 23 Dec 2019 17:19:38 +0100
Subject: [PATCH] mol-plugin: DataManager [wip]
---
package.json | 4 +-
src/mol-plugin/context.ts | 5 ++
src/mol-plugin/state/manager/base.ts | 18 ++++---
src/mol-plugin/state/manager/data.ts | 53 ++++++++++++++++++-
src/mol-plugin/state/manager/data/actions.ts | 17 ++++++
src/mol-plugin/state/manager/data/formats.ts | 50 +++++++++++++++++
src/mol-plugin/state/manager/data/parsers.ts | 8 ---
src/mol-plugin/state/manager/data/provider.ts | 50 +++++++++++++++++
src/mol-plugin/state/manager/data/sources.ts | 21 +++++++-
src/mol-plugin/util/task-manager.ts | 21 ++++++--
src/mol-state/object.ts | 11 ++++
src/mol-task/execution/observable.ts | 5 ++
12 files changed, 241 insertions(+), 22 deletions(-)
create mode 100644 src/mol-plugin/state/manager/data/actions.ts
create mode 100644 src/mol-plugin/state/manager/data/formats.ts
delete mode 100644 src/mol-plugin/state/manager/data/parsers.ts
create mode 100644 src/mol-plugin/state/manager/data/provider.ts
diff --git a/package.json b/package.json
index 35d015648..0a76d11ed 100644
--- a/package.json
+++ b/package.json
@@ -14,11 +14,11 @@
"lint": "tslint src/**/*.ts",
"test": "npm run lint && jest",
"build": "npm run build-tsc && npm run build-extra && npm run build-webpack",
- "build-tsc": "tsc",
+ "build-tsc": "tsc --incremental",
"build-extra": "cpx \"src/**/*.{scss,woff,woff2,ttf,otf,eot,svg,html,ico}\" lib/",
"build-webpack": "webpack --mode production",
"watch": "concurrently --kill-others \"npm:watch-tsc\" \"npm:watch-extra\" \"npm:watch-webpack\"",
- "watch-tsc": "tsc -watch",
+ "watch-tsc": "tsc --watch --incremental",
"watch-extra": "cpx \"src/**/*.{scss,woff,woff2,ttf,otf,eot,svg,html,ico}\" lib/ --watch",
"watch-webpack": "webpack -w --mode development --display minimal",
"serve": "http-server -p 1338",
diff --git a/src/mol-plugin/context.ts b/src/mol-plugin/context.ts
index c31f53999..165c21963 100644
--- a/src/mol-plugin/context.ts
+++ b/src/mol-plugin/context.ts
@@ -44,6 +44,7 @@ import { PluginToastManager } from './state/toast';
import { StructureMeasurementManager } from './util/structure-measurement';
import { ViewportScreenshotHelper } from './util/viewport-screenshot';
import { StructureRepresentationManager } from './state/representation/structure';
+import { DataManager } from './state/manager/data';
interface Log {
entries: List<LogEntry>
@@ -132,6 +133,10 @@ export class PluginContext {
readonly customStructureProperties = new CustomStructureProperty.Registry();
readonly customParamEditors = new Map<string, StateTransformParameters.Class>();
+ readonly managers = {
+ data: new DataManager(this),
+ } as const;
+
readonly helpers = {
measurement: new StructureMeasurementManager(this),
structureSelectionManager: new StructureElementSelectionManager(this),
diff --git a/src/mol-plugin/state/manager/base.ts b/src/mol-plugin/state/manager/base.ts
index d612b0c21..c3433c3b0 100644
--- a/src/mol-plugin/state/manager/base.ts
+++ b/src/mol-plugin/state/manager/base.ts
@@ -6,12 +6,16 @@
// TODO: primites
-// import { StateObject, State, StateObjectCell, StateBuilder, StateTransformer } from '../../../mol-state';
-// import { RuntimeContext } from '../../../mol-task';
-// import { PluginContext } from '../../context';
+import { StateObject, State, StateObjectCell, StateBuilder, StateTransformer, StateTransform } from '../../../mol-state';
+import { RuntimeContext } from '../../../mol-task';
+import { PluginContext } from '../../context';
-// export type StateAction<P = any, O extends StateObject = StateObject, R = {}> =
-// (ctx: RuntimeContext, state: State, cell: StateObjectCell<O>, params: P, plugin: PluginContext) => Promise<R> | R;
+export { StateAction, BuilderAction }
-// export type BuilderAction<P = any, O extends StateObject = StateObject, T extends StateTransformer = StateTransformer, R = {}> =
-// (builder: StateBuilder.To<O, T>, params: P, plugin: PluginContext) => R;
\ No newline at end of file
+type StateAction<P = any, O extends StateObject = StateObject, R = {}> =
+ (cell: StateObjectCell<O>, params: P, ctx: { ctx: RuntimeContext, state: State, plugin: PluginContext }) => Promise<R> | R;
+function StateAction<P = any, O extends StateObject = StateObject, R = {}>(action: StateAction<P, O, R>) { return action; }
+
+type BuilderAction<P = any, O extends StateObject = StateObject, T extends StateTransformer = StateTransformer, R = {}> =
+ (builder: StateBuilder.To<O, T>, params: P, ctx: { options?: Partial<StateTransform.Options>, plugin: PluginContext }) => R;
+function BuilderAction<P = any, O extends StateObject = StateObject, T extends StateTransformer = StateTransformer, R = {}>(action: BuilderAction<P, O, T, R>) { return action; }
\ No newline at end of file
diff --git a/src/mol-plugin/state/manager/data.ts b/src/mol-plugin/state/manager/data.ts
index 866772880..4241ec1dc 100644
--- a/src/mol-plugin/state/manager/data.ts
+++ b/src/mol-plugin/state/manager/data.ts
@@ -4,4 +4,55 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
-// TODO: data manager (download / open file / parse)
\ No newline at end of file
+import { StateTransformer, StateTransform, StateObjectSelector, StateObjectCell } from '../../../mol-state';
+import { PluginContext } from '../../context';
+import { Download, ReadFile } from '../transforms/data';
+import { getFileInfo } from '../../../mol-util/file-info';
+import { DataFormatProvider } from './data/provider';
+import { BuiltInDataFormats } from './data/formats';
+import { objectForEach } from '../../../mol-util/object';
+
+export class DataManager {
+ readonly formats: DataFormatProvider[] = [];
+
+ addFormat(p: DataFormatProvider) {
+ this.formats.push(p);
+ }
+
+ get dataState() {
+ return this.plugin.state.dataState;
+ }
+
+ async download(params: StateTransformer.Params<Download>, options?: Partial<StateTransform.Options>) {
+ const data = this.dataState.build().toRoot().apply(Download, params, options);
+ await this.plugin.runTask(this.dataState.updateTree(data));
+ return { data: data.selector };
+ }
+
+ async readFile(params: StateTransformer.Params<ReadFile>, options?: Partial<StateTransform.Options>) {
+ const data = this.dataState.build().toRoot().apply(ReadFile, params, options);
+ const fileInfo = getFileInfo(params.file);
+ await this.plugin.runTask(this.dataState.updateTree(data));
+ return { data: data.selector, fileInfo };
+ }
+
+ async parse<K extends keyof BuiltInDataFormats, P extends BuiltInDataFormats[K]>(provider: K, data: StateObjectSelector<DataFormatProvider.Data<P>> | StateObjectCell | string, params?: DataFormatProvider.Params<P>): Promise<DataFormatProvider.Ret<P>>
+ async parse<P extends DataFormatProvider>(provider: P, data: StateObjectSelector<DataFormatProvider.Data<P>> | StateObjectCell | string, params?: DataFormatProvider.Params<P>): Promise<DataFormatProvider.Ret<P>>
+ async parse<P extends DataFormatProvider>(providerOrBuildIn: P | string, data: StateObjectSelector<DataFormatProvider.Data<P>> | StateObjectCell | StateTransform.Ref, params?: DataFormatProvider.Params<P>) {
+ const provider: P = typeof providerOrBuildIn === 'string' ? BuiltInDataFormats[providerOrBuildIn as keyof BuiltInDataFormats] as unknown as P : providerOrBuildIn as P;
+ const cell = StateObjectCell.resolve(this.dataState, data);
+ if (!cell) {
+ throw new Error('Could not resolve data cell.');
+ }
+ return provider.apply({ state: this.dataState, plugin: this.plugin }, cell, params);
+ }
+
+ // async test() {
+ // const { data } = await this.download({ url: '' });
+ // const cif = await this.parse('mmcif', data);
+ // }
+
+ constructor(public plugin: PluginContext) {
+ objectForEach(BuiltInDataFormats, f => this.formats.push(f));
+ }
+}
\ No newline at end of file
diff --git a/src/mol-plugin/state/manager/data/actions.ts b/src/mol-plugin/state/manager/data/actions.ts
new file mode 100644
index 000000000..0c036e4a6
--- /dev/null
+++ b/src/mol-plugin/state/manager/data/actions.ts
@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { BuilderAction } from '../base';
+import { StateTransformer } from '../../../../mol-state';
+import { Download as DownloadData, ReadFile } from '../../transforms/data';
+
+export const Download = BuilderAction((builder, params: StateTransformer.Params<DownloadData>, { options }) => {
+ return builder.apply(DownloadData, params, options);
+});
+
+export const OpenFile = BuilderAction((builder, params: StateTransformer.Params<ReadFile>, { options }) => {
+ return builder.apply(ReadFile, params, options);
+});
\ No newline at end of file
diff --git a/src/mol-plugin/state/manager/data/formats.ts b/src/mol-plugin/state/manager/data/formats.ts
new file mode 100644
index 000000000..22f3adf7b
--- /dev/null
+++ b/src/mol-plugin/state/manager/data/formats.ts
@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { DataFormatProvider } from './provider';
+import { PluginContext } from '../../../context';
+import { FileInfo } from '../../../../mol-util/file-info';
+import { StateTransforms } from '../../transforms';
+
+export const MmcifFormatProvider = DataFormatProvider({
+ id: 'mmcif',
+ display: { name: 'mmCIF', group: 'Molecule' },
+ extensions: { text: ['cif', 'mmcif', 'mcif'], binary: ['bcif'] }
+})({
+ isApplicable(plugin: PluginContext, data: string | Uint8Array, info?: FileInfo): boolean {
+ // TODO: check CIF variants
+ return true;
+ },
+ async apply({ plugin, state }, data) {
+ const dictionary = state.build().to(data).apply(StateTransforms.Data.ParseCif, void 0, { state: { isGhost: true } });
+ const trajectory = dictionary.apply(StateTransforms.Model.TrajectoryFromMmCif);
+ await plugin.runTask(state.updateTree(trajectory));
+ return { dictionary: dictionary.selector, trajectory: trajectory.selector };
+ }
+});
+
+export const PdbFormatProvider = DataFormatProvider({
+ id: 'pdb',
+ display: { name: 'PDB', group: 'Molecule' },
+ extensions: { text: ['pdb', 'ent'] }
+})({
+ async apply({ plugin, state }, data) {
+ const trajectory = state.build().to(data).apply(StateTransforms.Model.TrajectoryFromPDB);
+ await plugin.runTask(state.updateTree(trajectory));
+ return { trajectory: trajectory.selector };
+ }
+});
+
+export const BuiltInDataFormats = {
+ 'mmcif': MmcifFormatProvider,
+ 'pdb': PdbFormatProvider
+}
+export type BuiltInDataFormats = typeof BuiltInDataFormats
+
+// export const TrajectoryFormatProviders = {
+// 'mmcif': MmcifFormatProvider,
+// 'pdb': PdbFormatProvider
+// }
\ No newline at end of file
diff --git a/src/mol-plugin/state/manager/data/parsers.ts b/src/mol-plugin/state/manager/data/parsers.ts
deleted file mode 100644
index a1f17581d..000000000
--- a/src/mol-plugin/state/manager/data/parsers.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-/**
- * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
- *
- * @author David Sehnal <david.sehnal@gmail.com>
- */
-
-// TODO: build in format parsers, allow to define custom formats
-// basically state actions that take data and return parsed format in appropriate built in representation (e.g. string|binary => )
\ No newline at end of file
diff --git a/src/mol-plugin/state/manager/data/provider.ts b/src/mol-plugin/state/manager/data/provider.ts
new file mode 100644
index 000000000..788583f36
--- /dev/null
+++ b/src/mol-plugin/state/manager/data/provider.ts
@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { PluginContext } from '../../../context';
+import { State, StateObjectCell } from '../../../../mol-state';
+import { ParamDefinition as PD } from '../../../../mol-util/param-definition';
+import { PluginStateObject } from '../../objects';
+import { FileInfo } from '../../../../mol-util/file-info';
+
+export { DataFormatProvider }
+
+type DataFormatProvider<Id extends string = string, D extends PluginStateObject.Data.Binary | PluginStateObject.Data.String = PluginStateObject.Data.Binary | PluginStateObject.Data.String, P = any, S = {}> =
+ DataFormatProvider.Base<Id, P> & DataFormatProvider.Definition<D, P, S>
+
+function DataFormatProvider<Id extends string, P = {}>(provider: DataFormatProvider.Base<Id, P>) {
+ return function<D extends PluginStateObject.Data.Binary | PluginStateObject.Data.String, S>(p: DataFormatProvider.Definition<D, P, S>): DataFormatProvider<Id, D, P, S> {
+ return { ...provider, ...p };
+ };
+}
+
+namespace DataFormatProvider {
+ export type Ret<P extends DataFormatProvider> = P extends DataFormatProvider<any, any, any, infer T> ? T : never
+ export type Data<P extends DataFormatProvider> = P extends DataFormatProvider<any, infer T, any, any> ? T : never
+ export type Params<P extends DataFormatProvider> = P extends DataFormatProvider<any, any, infer T, any> ? T : never
+
+ export interface Base<Id extends string = string, P = any> {
+ id: Id,
+ display: { name: string, group?: string, description?: string },
+ extensions: { text?: string[], binary?: string[] },
+ params?(plugin: PluginContext, data: string | Uint8Array, info?: FileInfo): PD.Def<P>
+ // TODO: default representation
+ // defaultRepresentation?: RepresenatationProvider
+ }
+ export interface Definition<D extends PluginStateObject.Data.Binary | PluginStateObject.Data.String = PluginStateObject.Data.Binary | PluginStateObject.Data.String, P = any, S = any> {
+ isApplicable?(plugin: PluginContext, data: string | Uint8Array, info?: FileInfo): boolean,
+ apply(ctx: { state: State, plugin: PluginContext }, data: StateObjectCell<D>, params: P): Promise<S>
+ }
+}
+
+// interface DataSourceProvider<Id extends string = string, Format extends DataFormatProvider = DataFormatProvider, P = any, S = {}> {
+// id: Id,
+// display: { name: string, group?: string, description?: string },
+// format: Format,
+// apply(ctx: { ctx: RuntimeContext, state: State, plugin: PluginContext }, params: P): Promise<S>,
+// params(plugin: PluginContext): PD.Def<P>,
+// }
+// function DataSourceProvider<Id extends string, Format extends DataFormatProvider, P, S>(provider: DataSourceProvider<Id, Format, P, S>) { return provider; }
diff --git a/src/mol-plugin/state/manager/data/sources.ts b/src/mol-plugin/state/manager/data/sources.ts
index dcb98a7d0..e9ed684bc 100644
--- a/src/mol-plugin/state/manager/data/sources.ts
+++ b/src/mol-plugin/state/manager/data/sources.ts
@@ -4,5 +4,24 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
+// import { DataSourceProvider } from './provider';
+// import { MmcifFormatProvider } from './formats';
+// import { ParamDefinition as PD } from '../../../../mol-util/param-definition';
+// import { Download } from './actions';
+
// TODO: basic types string, binary (both download and from file)
-// TODO: decompress functionality (string|binary --> string|binary)
\ No newline at end of file
+// TODO: decompress functionality (string|binary --> string|binary)
+
+// export const PDBeUpdatedMmcifDataSource = DataSourceProvider({
+// id: 'pdbe-updated-mmcif',
+// display: { name: 'PDBe Updated mmCIF', group: 'Molecule' },
+// format: MmcifFormatProvider,
+// params() {
+// return { id: PD.Text('1cbs', { label: 'PDB Id(s)', description: 'One or more comma separated PDB ids.' }) }
+// },
+// async apply(ctx, params) {
+// const download = Download(ctx.state.build().toRoot(), { url: `https://www.ebi.ac.uk/pdbe/static/entry/${params.id.toLowerCase()}_updated.cif` }, ctx);
+// await ctx.state.updateTree(download).runInContext(ctx.ctx);
+// return 0;
+// }
+// })
\ No newline at end of file
diff --git a/src/mol-plugin/util/task-manager.ts b/src/mol-plugin/util/task-manager.ts
index dd8ce979e..812f35d92 100644
--- a/src/mol-plugin/util/task-manager.ts
+++ b/src/mol-plugin/util/task-manager.ts
@@ -4,9 +4,11 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
-import { Task, Progress } from '../../mol-task';
+import { Task, Progress, RuntimeContext } from '../../mol-task';
import { RxEventHelper } from '../../mol-util/rx-event-helper';
import { now } from '../../mol-util/now';
+import { CreateObservableCtx, ExecuteInContext } from '../../mol-task/execution/observable';
+import { arrayRemoveInPlace } from '../../mol-util/array';
export { TaskManager }
@@ -14,6 +16,7 @@ class TaskManager {
private ev = RxEventHelper.create();
private id = 0;
private abortRequests = new Map<number, string | undefined>();
+ private currentContext: { ctx: RuntimeContext, refCount: number }[] = [];
readonly events = {
progress: this.ev<TaskManager.ProgressEvent>(),
@@ -34,14 +37,26 @@ class TaskManager {
};
}
- async run<T>(task: Task<T>): Promise<T> {
+ async run<T>(task: Task<T>, createNewContext = false): Promise<T> {
const id = this.id++;
+
+ let ctx: TaskManager['currentContext'][0];
+
+ if (createNewContext || this.currentContext.length === 0) {
+ ctx = { ctx: CreateObservableCtx(task, this.track(id, task.id), 100), refCount: 1 };
+ } else {
+ ctx = this.currentContext[this.currentContext.length - 1];
+ ctx.refCount++;
+ }
+
try {
- const ret = await task.run(this.track(id, task.id), 100);
+ const ret = await ExecuteInContext(ctx.ctx, task);
return ret;
} finally {
this.events.finished.next({ id });
this.abortRequests.delete(task.id);
+ ctx.refCount--;
+ if (ctx.refCount === 0) arrayRemoveInPlace(this.currentContext, ctx);
}
}
diff --git a/src/mol-state/object.ts b/src/mol-state/object.ts
index 5f6333e58..d88a7fd37 100644
--- a/src/mol-state/object.ts
+++ b/src/mol-state/object.ts
@@ -93,6 +93,17 @@ namespace StateObjectCell {
const c: StateObjectCell = o;
return !!c && !!c.transform && !!c.parent && !!c.status;
}
+
+ export type Ref = StateTransform.Ref | StateObjectCell | StateObjectSelector
+
+ export function resolve(state: State, refOrCellOrSelector: StateTransform.Ref | StateObjectCell | StateObjectSelector) {
+ const ref = typeof refOrCellOrSelector === 'string'
+ ? refOrCellOrSelector
+ : StateObjectCell.is(refOrCellOrSelector)
+ ? refOrCellOrSelector.transform.ref
+ : refOrCellOrSelector.ref;
+ return state.cells.get(ref);
+ }
}
// TODO: improve the API?
diff --git a/src/mol-task/execution/observable.ts b/src/mol-task/execution/observable.ts
index c4bed0acf..99b48a399 100644
--- a/src/mol-task/execution/observable.ts
+++ b/src/mol-task/execution/observable.ts
@@ -23,6 +23,11 @@ export function ExecuteObservable<T>(task: Task<T>, observer: Progress.Observer,
return execute(task as ExposedTask<T>, ctx);
}
+export function CreateObservableCtx<T>(task: Task<T>, observer: Progress.Observer, updateRateMs = 250) {
+ const info = ProgressInfo(task, observer, updateRateMs);
+ return new ObservableRuntimeContext(info, info.root);
+}
+
export function ExecuteInContext<T>(ctx: RuntimeContext, task: Task<T>) {
return execute(task as ExposedTask<T>, ctx as ObservableRuntimeContext);
}
--
GitLab