From c8ac64a5716d73f701cf337571e781f4f8e5856b Mon Sep 17 00:00:00 2001
From: Alexander Rose <alex.rose@rcsb.org>
Date: Mon, 10 Jun 2019 16:15:14 -0700
Subject: [PATCH] wip, server/model/preprocess
---
package.json | 2 +-
src/mol-model-props/rcsb/assembly-symmetry.ts | 12 ++---
src/servers/common/util.ts | 21 +++++++++
src/servers/model/config.ts | 13 +++++-
src/servers/model/preprocess/master.ts | 20 ++++++++-
src/servers/model/preprocess/preprocess.ts | 1 -
.../model/properties/providers/pdbe.ts | 15 +------
.../model/properties/providers/rcsb.ts | 17 +++++--
.../model/properties/providers/wwpdb.ts | 44 +++++++++++++++++++
src/servers/model/properties/wwpdb.ts | 16 +++++++
src/servers/model/property-provider.ts | 17 ++++++-
src/servers/volume/local.ts | 4 +-
webpack.config.js | 6 +++
13 files changed, 157 insertions(+), 31 deletions(-)
create mode 100644 src/servers/common/util.ts
create mode 100644 src/servers/model/properties/providers/wwpdb.ts
create mode 100644 src/servers/model/properties/wwpdb.ts
diff --git a/package.json b/package.json
index 44be598d2..9d02b0b61 100644
--- a/package.json
+++ b/package.json
@@ -85,7 +85,6 @@
"tslint": "^5.17.0",
"typescript": "^3.5.1",
"uglify-js": "^3.6.0",
- "util.promisify": "^1.0.0",
"webpack": "^4.32.2",
"webpack-cli": "^3.3.2"
},
@@ -111,6 +110,7 @@
"react-dom": "^16.8.6",
"rxjs": "^6.5.2",
"swagger-ui-dist": "^3.22.2",
+ "util.promisify": "^1.0.0",
"xhr2": "^0.2.0"
}
}
diff --git a/src/mol-model-props/rcsb/assembly-symmetry.ts b/src/mol-model-props/rcsb/assembly-symmetry.ts
index 150a93ef4..bdf36072c 100644
--- a/src/mol-model-props/rcsb/assembly-symmetry.ts
+++ b/src/mol-model-props/rcsb/assembly-symmetry.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
@@ -18,7 +18,6 @@ import { CifCategory } from '../../mol-io/reader/cif';
import { PropertyWrapper } from '../../mol-model-props/common/wrapper';
import { Task, RuntimeContext } from '../../mol-task';
import { GraphQLClient } from '../../mol-util/graphql-client';
-import { ajaxGet } from '../../mol-util/data-source';
const { str, int, float, Aliased, Vector, List } = Column.Schema;
@@ -183,13 +182,12 @@ export function AssemblySymmetry(db: AssemblySymmetry.Database): AssemblySymmetr
type SymmetryKind = 'GLOBAL' | 'LOCAL' | 'PSEUDO'
type SymmetryType = 'ASYMMETRIC' | 'CYCLIC' | 'DIHEDRAL' | 'HELICAL' | 'ICOSAHEDRAL' | 'OCTAHEDRAL' | 'TETRAHEDRAL'
-const Client = new GraphQLClient(AssemblySymmetry.GraphQLEndpointURL, ajaxGet)
-
export namespace AssemblySymmetry {
export function is(x: any): x is AssemblySymmetry {
return x['@type'] === 'rcsb_assembly_symmetry'
}
- export const GraphQLEndpointURL = '//rest-dev.rcsb.org/graphql'
+ export const GraphQLEndpointURL = '//rest-staging.rcsb.org/graphql'
+
export const Schema = {
rcsb_assembly_symmetry_info: {
updated_datetime_utc: Column.Schema.str
@@ -257,7 +255,7 @@ export namespace AssemblySymmetry {
export const Descriptor = _Descriptor;
- export async function attachFromCifOrAPI(model: Model, client: GraphQLClient = Client, ctx?: RuntimeContext) {
+ export async function attachFromCifOrAPI(model: Model, client: GraphQLClient, ctx?: RuntimeContext) {
if (model.customProperties.has(Descriptor)) return true;
let db: Database
@@ -266,8 +264,10 @@ export namespace AssemblySymmetry {
db = createDatabaseFromCif(model)
} else {
let result: AssemblySymmetryGraphQL.Query
+ console.log('model.label.toLowerCase()', model.label.toLowerCase())
const variables: AssemblySymmetryGraphQL.Variables = { pdbId: model.label.toLowerCase() };
try {
+ console.log('foo', client)
result = await client.request<AssemblySymmetryGraphQL.Query>(ctx || RuntimeContext.Synchronous, query, variables);
} catch (e) {
console.error(e)
diff --git a/src/servers/common/util.ts b/src/servers/common/util.ts
new file mode 100644
index 000000000..3086ffb1f
--- /dev/null
+++ b/src/servers/common/util.ts
@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { ConsoleLogger } from '../../mol-util/console-logger';
+
+export function getParam<T>(params: any, ...path: string[]): T | undefined {
+ try {
+ let current = params;
+ for (const p of path) {
+ if (typeof current === 'undefined') return;
+ current = current[p];
+ }
+ return current;
+ } catch (e) {
+ ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
+ }
+}
\ No newline at end of file
diff --git a/src/servers/model/config.ts b/src/servers/model/config.ts
index ac963eadf..860f6a0a4 100644
--- a/src/servers/model/config.ts
+++ b/src/servers/model/config.ts
@@ -54,8 +54,9 @@ const config = {
*/
customProperties: <ModelPropertyProviderConfig | string>{
sources: [
- './properties/pdbe',
- './properties/rcsb'
+ 'pdbe',
+ 'rcsb',
+ 'wwpdb'
],
params: {
PDBe: {
@@ -68,6 +69,14 @@ const config = {
File: {
residuewise_outlier_summary: 'e:/test/mol-star/model/props/'
}
+ },
+ RCSB: {
+ API: {
+ assembly_symmetry: 'https://rest-staging.rcsb.org/graphql'
+ }
+ },
+ wwPDB: {
+ chemCompBondTablePath: ''
}
}
},
diff --git a/src/servers/model/preprocess/master.ts b/src/servers/model/preprocess/master.ts
index 9ff8b8e1f..4c7068208 100644
--- a/src/servers/model/preprocess/master.ts
+++ b/src/servers/model/preprocess/master.ts
@@ -10,9 +10,27 @@ import * as argparse from 'argparse'
import { runMaster, PreprocessEntry } from './parallel';
import { ModelPropertyProviderConfig } from '../property-provider';
+function description() {
+ const exampleCfg = {
+ numProcesses: 1,
+ customProperties: {
+ sources: [
+ 'wwpdb'
+ ],
+ params: {
+ wwPDB: {
+ chemCompBondTablePath: './build/data/ccb.bcif'
+ }
+ }
+ }
+ }
+
+ return `Preprocess CIF files to include custom properties and convert them to BinaryCIF format.\n\nExample cfg.json: ${JSON.stringify(exampleCfg, null, 2)}`
+}
+
const cmdParser = new argparse.ArgumentParser({
addHelp: true,
- description: 'Preprocess CIF files to include custom properties and convert them to BinaryCIF format.'
+ description: description()
});
cmdParser.addArgument(['--input', '-i'], { help: 'Input filename', required: false });
cmdParser.addArgument(['--outCIF', '-oc'], { help: 'Output CIF filename', required: false });
diff --git a/src/servers/model/preprocess/preprocess.ts b/src/servers/model/preprocess/preprocess.ts
index 727da1d02..397674826 100644
--- a/src/servers/model/preprocess/preprocess.ts
+++ b/src/servers/model/preprocess/preprocess.ts
@@ -21,7 +21,6 @@ export function preprocessFile(filename: string, propertyProvider?: ModelPropert
: convert(filename, outputCif, outputBcif);
}
-
async function preprocess(filename: string, propertyProvider?: ModelPropertiesProvider, outputCif?: string, outputBcif?: string) {
const input = await readStructureWrapper('entry', '_local_', filename, propertyProvider);
const categories = await classifyCif(input.cifFrame);
diff --git a/src/servers/model/properties/providers/pdbe.ts b/src/servers/model/properties/providers/pdbe.ts
index f0b254dd0..7935d4a49 100644
--- a/src/servers/model/properties/providers/pdbe.ts
+++ b/src/servers/model/properties/providers/pdbe.ts
@@ -14,6 +14,7 @@ import { PDBePreferredAssembly } from '../../../../mol-model-props/pdbe/preferre
import { PDBeStructRefDomain } from '../../../../mol-model-props/pdbe/struct-ref-domain';
import { AttachModelProperty } from '../../property-provider';
import { ConsoleLogger } from '../../../../mol-util/console-logger';
+import { getParam } from '../../../common/util';
export const PDBe_structureQualityReport: AttachModelProperty = ({ model, params, cache }) => {
const PDBe_apiSourceJson = useFileSource(params)
@@ -68,20 +69,6 @@ function useFileSource(params: any) {
return !!getParam<boolean>(params, 'PDBe', 'UseFileSource')
}
-function getParam<T>(params: any, ...path: string[]): T | undefined {
- try {
- let current = params;
- for (const p of path) {
- if (typeof current === 'undefined') return;
- current = current[p];
- }
- return current;
- } catch (e) {
- ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
- }
-}
-
-
function apiQueryProvider(urlPrefix: string, cache: any) {
const cacheKey = UUID.create22();
return async (model: Model) => {
diff --git a/src/servers/model/properties/providers/rcsb.ts b/src/servers/model/properties/providers/rcsb.ts
index e2ae21e66..011ad4b3c 100644
--- a/src/servers/model/properties/providers/rcsb.ts
+++ b/src/servers/model/properties/providers/rcsb.ts
@@ -1,12 +1,23 @@
/**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
import { AssemblySymmetry } from '../../../../mol-model-props/rcsb/assembly-symmetry';
import { AttachModelProperty } from '../../property-provider';
+import { ajaxGet } from '../../../../mol-util/data-source';
+import { GraphQLClient } from '../../../../mol-util/graphql-client';
+import { getParam } from '../../../common/util';
-export const RCSB_assemblySymmetry: AttachModelProperty = ({ model }) => {
- return AssemblySymmetry.attachFromCifOrAPI(model)
+export const RCSB_assemblySymmetry: AttachModelProperty = ({ model, params }) => {
+ const url = getApiUrl(params, 'assembly_symmetry', `https:${AssemblySymmetry.GraphQLEndpointURL}`)
+ const client = new GraphQLClient(url, ajaxGet)
+ return AssemblySymmetry.attachFromCifOrAPI(model, client)
+}
+
+function getApiUrl(params: any, name: string, fallback: string) {
+ const path = getParam<string>(params, 'RCSB', 'API', name);
+ if (!path) return fallback;
+ return path;
}
\ No newline at end of file
diff --git a/src/servers/model/properties/providers/wwpdb.ts b/src/servers/model/properties/providers/wwpdb.ts
new file mode 100644
index 000000000..d5ea7136e
--- /dev/null
+++ b/src/servers/model/properties/providers/wwpdb.ts
@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import * as fs from 'fs'
+import * as util from 'util'
+import { AttachModelProperty } from '../../property-provider';
+import { ChemCompBond } from '../../../../mol-model-props/wwpdb/chem-comp-bond';
+import { Table } from '../../../../mol-data/db';
+import { CIF } from '../../../../mol-io/reader/cif';
+import { getParam } from '../../../common/util';
+
+require('util.promisify').shim()
+const readFile = util.promisify(fs.readFile)
+
+export const wwPDB_chemCompBond: AttachModelProperty = ({ model, params }) => {
+ const wwPDB_apiSourceTable = getChemCompBondTableProvider(getTablePath(params))
+ return ChemCompBond.attachFromCifOrTable(model, { wwPDB_apiSourceTable });
+}
+
+async function read(path: string) {
+ return path.endsWith('.bcif') ? new Uint8Array(await readFile(path)) : readFile(path, 'utf8');
+}
+
+function getChemCompBondTableProvider(path: string): () => Promise<Table<ChemCompBond.Schema['chem_comp_bond']>> {
+ let chemCompBondTable: Table<ChemCompBond.Schema['chem_comp_bond']>
+ return async function() {
+ if (chemCompBondTable === undefined) {
+ const parsed = await CIF.parse(await read(path)).run()
+ if (parsed.isError) throw new Error(parsed.toString())
+ const table = CIF.toDatabase(ChemCompBond.Schema, parsed.result.blocks[0])
+ chemCompBondTable = table.chem_comp_bond
+ }
+ return chemCompBondTable
+ }
+}
+
+function getTablePath(params: any) {
+ const path = getParam<string>(params, 'wwPDB', 'chemCompBondTablePath');
+ if (!path) throw new Error(`wwPDB 'chemCompBondTablePath' not set!`);
+ return path;
+}
\ No newline at end of file
diff --git a/src/servers/model/properties/wwpdb.ts b/src/servers/model/properties/wwpdb.ts
new file mode 100644
index 000000000..cb4040db1
--- /dev/null
+++ b/src/servers/model/properties/wwpdb.ts
@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { AttachModelProperties } from '../property-provider';
+import { wwPDB_chemCompBond } from './providers/wwpdb';
+
+export const attachModelProperties: AttachModelProperties = (args) => {
+ // return a list of promises that start attaching the props in parallel
+ // (if there are downloads etc.)
+ return [
+ wwPDB_chemCompBond(args)
+ ];
+}
\ No newline at end of file
diff --git a/src/servers/model/property-provider.ts b/src/servers/model/property-provider.ts
index d3b4ad00a..d929c30c7 100644
--- a/src/servers/model/property-provider.ts
+++ b/src/servers/model/property-provider.ts
@@ -9,6 +9,17 @@ import { Model } from '../../mol-model/structure';
import Config from './config';
import { ConsoleLogger } from '../../mol-util/console-logger';
+// TODO enable dynamic imports again
+import * as pdbeProps from './properties/pdbe'
+import * as rcsbProps from './properties/rcsb'
+import * as wwpdbProps from './properties/wwpdb'
+
+const attachModelProperties: { [k: string]: AttachModelProperties } = {
+ pdbe: pdbeProps.attachModelProperties,
+ rcsb: rcsbProps.attachModelProperties,
+ wwpdb: wwpdbProps.attachModelProperties
+}
+
export interface ModelPropertyProviderConfig {
sources: string[],
params?: { [name: string]: any }
@@ -39,7 +50,11 @@ export function createModelPropertiesProvider(configOrPath: ModelPropertyProvide
const ps: AttachModelProperties[] = [];
for (const p of config.sources) {
- ps.push(require(p).attachModelProperties);
+ if (p in attachModelProperties) {
+ ps.push(attachModelProperties[p]);
+ } else {
+ ConsoleLogger.error('Config', `Could not find property provider '${p}', ignoring.`);
+ }
}
return (model, cache) => {
diff --git a/src/servers/volume/local.ts b/src/servers/volume/local.ts
index 99993cfbc..f2145c2ba 100644
--- a/src/servers/volume/local.ts
+++ b/src/servers/volume/local.ts
@@ -16,7 +16,7 @@ import { LimitsConfig, addLimitsArgs, setLimitsConfig } from './config';
console.log(`VolumeServer Local ${VERSION}, (c) 2018-2019, Mol* contributors`);
console.log();
-function help() {
+function description() {
const exampleJobs: LocalApi.JobEntry[] = [{
source: {
filename: `g:/test/mdb/xray-1tqn.mdb`,
@@ -55,7 +55,7 @@ function help() {
const parser = new argparse.ArgumentParser({
addHelp: true,
- description: help()
+ description: description()
});
addLimitsArgs(parser)
parser.addArgument(['jobs'], {
diff --git a/webpack.config.js b/webpack.config.js
index 3de9e1565..44062a2ae 100644
--- a/webpack.config.js
+++ b/webpack.config.js
@@ -73,6 +73,12 @@ function createNodeEntryPoint(name, dir, out) {
target: 'node',
entry: path.resolve(__dirname, `lib/${dir}/${name}.js`),
output: { filename: `${name}.js`, path: path.resolve(__dirname, `build/${out}`) },
+ externals: {
+ argparse: 'require("argparse")',
+ 'node-fetch': 'require("node-fetch")',
+ 'util.promisify': 'require("util.promisify")',
+ xhr2: 'require("xhr2")',
+ },
...sharedConfig
}
}
--
GitLab