Skip to content
Snippets Groups Projects
Commit c8ac64a5 authored by Alexander Rose's avatar Alexander Rose
Browse files

wip, server/model/preprocess

parent bf81b902
No related branches found
No related tags found
No related merge requests found
......@@ -85,7 +85,6 @@
"tslint": "^5.17.0",
"typescript": "^3.5.1",
"uglify-js": "^3.6.0",
"util.promisify": "^1.0.0",
"webpack": "^4.32.2",
"webpack-cli": "^3.3.2"
},
......@@ -111,6 +110,7 @@
"react-dom": "^16.8.6",
"rxjs": "^6.5.2",
"swagger-ui-dist": "^3.22.2",
"util.promisify": "^1.0.0",
"xhr2": "^0.2.0"
}
}
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
......@@ -18,7 +18,6 @@ import { CifCategory } from '../../mol-io/reader/cif';
import { PropertyWrapper } from '../../mol-model-props/common/wrapper';
import { Task, RuntimeContext } from '../../mol-task';
import { GraphQLClient } from '../../mol-util/graphql-client';
import { ajaxGet } from '../../mol-util/data-source';
const { str, int, float, Aliased, Vector, List } = Column.Schema;
......@@ -183,13 +182,12 @@ export function AssemblySymmetry(db: AssemblySymmetry.Database): AssemblySymmetr
type SymmetryKind = 'GLOBAL' | 'LOCAL' | 'PSEUDO'
type SymmetryType = 'ASYMMETRIC' | 'CYCLIC' | 'DIHEDRAL' | 'HELICAL' | 'ICOSAHEDRAL' | 'OCTAHEDRAL' | 'TETRAHEDRAL'
const Client = new GraphQLClient(AssemblySymmetry.GraphQLEndpointURL, ajaxGet)
export namespace AssemblySymmetry {
export function is(x: any): x is AssemblySymmetry {
return x['@type'] === 'rcsb_assembly_symmetry'
}
export const GraphQLEndpointURL = '//rest-dev.rcsb.org/graphql'
export const GraphQLEndpointURL = '//rest-staging.rcsb.org/graphql'
export const Schema = {
rcsb_assembly_symmetry_info: {
updated_datetime_utc: Column.Schema.str
......@@ -257,7 +255,7 @@ export namespace AssemblySymmetry {
export const Descriptor = _Descriptor;
export async function attachFromCifOrAPI(model: Model, client: GraphQLClient = Client, ctx?: RuntimeContext) {
export async function attachFromCifOrAPI(model: Model, client: GraphQLClient, ctx?: RuntimeContext) {
if (model.customProperties.has(Descriptor)) return true;
let db: Database
......@@ -266,8 +264,10 @@ export namespace AssemblySymmetry {
db = createDatabaseFromCif(model)
} else {
let result: AssemblySymmetryGraphQL.Query
console.log('model.label.toLowerCase()', model.label.toLowerCase())
const variables: AssemblySymmetryGraphQL.Variables = { pdbId: model.label.toLowerCase() };
try {
console.log('foo', client)
result = await client.request<AssemblySymmetryGraphQL.Query>(ctx || RuntimeContext.Synchronous, query, variables);
} catch (e) {
console.error(e)
......
/**
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author David Sehnal <david.sehnal@gmail.com>
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
import { ConsoleLogger } from '../../mol-util/console-logger';
export function getParam<T>(params: any, ...path: string[]): T | undefined {
try {
let current = params;
for (const p of path) {
if (typeof current === 'undefined') return;
current = current[p];
}
return current;
} catch (e) {
ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
}
}
\ No newline at end of file
......@@ -54,8 +54,9 @@ const config = {
*/
customProperties: <ModelPropertyProviderConfig | string>{
sources: [
'./properties/pdbe',
'./properties/rcsb'
'pdbe',
'rcsb',
'wwpdb'
],
params: {
PDBe: {
......@@ -68,6 +69,14 @@ const config = {
File: {
residuewise_outlier_summary: 'e:/test/mol-star/model/props/'
}
},
RCSB: {
API: {
assembly_symmetry: 'https://rest-staging.rcsb.org/graphql'
}
},
wwPDB: {
chemCompBondTablePath: ''
}
}
},
......
......@@ -10,9 +10,27 @@ import * as argparse from 'argparse'
import { runMaster, PreprocessEntry } from './parallel';
import { ModelPropertyProviderConfig } from '../property-provider';
function description() {
const exampleCfg = {
numProcesses: 1,
customProperties: {
sources: [
'wwpdb'
],
params: {
wwPDB: {
chemCompBondTablePath: './build/data/ccb.bcif'
}
}
}
}
return `Preprocess CIF files to include custom properties and convert them to BinaryCIF format.\n\nExample cfg.json: ${JSON.stringify(exampleCfg, null, 2)}`
}
const cmdParser = new argparse.ArgumentParser({
addHelp: true,
description: 'Preprocess CIF files to include custom properties and convert them to BinaryCIF format.'
description: description()
});
cmdParser.addArgument(['--input', '-i'], { help: 'Input filename', required: false });
cmdParser.addArgument(['--outCIF', '-oc'], { help: 'Output CIF filename', required: false });
......
......@@ -21,7 +21,6 @@ export function preprocessFile(filename: string, propertyProvider?: ModelPropert
: convert(filename, outputCif, outputBcif);
}
async function preprocess(filename: string, propertyProvider?: ModelPropertiesProvider, outputCif?: string, outputBcif?: string) {
const input = await readStructureWrapper('entry', '_local_', filename, propertyProvider);
const categories = await classifyCif(input.cifFrame);
......
......@@ -14,6 +14,7 @@ import { PDBePreferredAssembly } from '../../../../mol-model-props/pdbe/preferre
import { PDBeStructRefDomain } from '../../../../mol-model-props/pdbe/struct-ref-domain';
import { AttachModelProperty } from '../../property-provider';
import { ConsoleLogger } from '../../../../mol-util/console-logger';
import { getParam } from '../../../common/util';
export const PDBe_structureQualityReport: AttachModelProperty = ({ model, params, cache }) => {
const PDBe_apiSourceJson = useFileSource(params)
......@@ -68,20 +69,6 @@ function useFileSource(params: any) {
return !!getParam<boolean>(params, 'PDBe', 'UseFileSource')
}
function getParam<T>(params: any, ...path: string[]): T | undefined {
try {
let current = params;
for (const p of path) {
if (typeof current === 'undefined') return;
current = current[p];
}
return current;
} catch (e) {
ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
}
}
function apiQueryProvider(urlPrefix: string, cache: any) {
const cacheKey = UUID.create22();
return async (model: Model) => {
......
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
import { AssemblySymmetry } from '../../../../mol-model-props/rcsb/assembly-symmetry';
import { AttachModelProperty } from '../../property-provider';
import { ajaxGet } from '../../../../mol-util/data-source';
import { GraphQLClient } from '../../../../mol-util/graphql-client';
import { getParam } from '../../../common/util';
export const RCSB_assemblySymmetry: AttachModelProperty = ({ model }) => {
return AssemblySymmetry.attachFromCifOrAPI(model)
export const RCSB_assemblySymmetry: AttachModelProperty = ({ model, params }) => {
const url = getApiUrl(params, 'assembly_symmetry', `https:${AssemblySymmetry.GraphQLEndpointURL}`)
const client = new GraphQLClient(url, ajaxGet)
return AssemblySymmetry.attachFromCifOrAPI(model, client)
}
function getApiUrl(params: any, name: string, fallback: string) {
const path = getParam<string>(params, 'RCSB', 'API', name);
if (!path) return fallback;
return path;
}
\ No newline at end of file
/**
* Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
import * as fs from 'fs'
import * as util from 'util'
import { AttachModelProperty } from '../../property-provider';
import { ChemCompBond } from '../../../../mol-model-props/wwpdb/chem-comp-bond';
import { Table } from '../../../../mol-data/db';
import { CIF } from '../../../../mol-io/reader/cif';
import { getParam } from '../../../common/util';
require('util.promisify').shim()
const readFile = util.promisify(fs.readFile)
export const wwPDB_chemCompBond: AttachModelProperty = ({ model, params }) => {
const wwPDB_apiSourceTable = getChemCompBondTableProvider(getTablePath(params))
return ChemCompBond.attachFromCifOrTable(model, { wwPDB_apiSourceTable });
}
async function read(path: string) {
return path.endsWith('.bcif') ? new Uint8Array(await readFile(path)) : readFile(path, 'utf8');
}
function getChemCompBondTableProvider(path: string): () => Promise<Table<ChemCompBond.Schema['chem_comp_bond']>> {
let chemCompBondTable: Table<ChemCompBond.Schema['chem_comp_bond']>
return async function() {
if (chemCompBondTable === undefined) {
const parsed = await CIF.parse(await read(path)).run()
if (parsed.isError) throw new Error(parsed.toString())
const table = CIF.toDatabase(ChemCompBond.Schema, parsed.result.blocks[0])
chemCompBondTable = table.chem_comp_bond
}
return chemCompBondTable
}
}
function getTablePath(params: any) {
const path = getParam<string>(params, 'wwPDB', 'chemCompBondTablePath');
if (!path) throw new Error(`wwPDB 'chemCompBondTablePath' not set!`);
return path;
}
\ No newline at end of file
/**
* Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
import { AttachModelProperties } from '../property-provider';
import { wwPDB_chemCompBond } from './providers/wwpdb';
export const attachModelProperties: AttachModelProperties = (args) => {
// return a list of promises that start attaching the props in parallel
// (if there are downloads etc.)
return [
wwPDB_chemCompBond(args)
];
}
\ No newline at end of file
......@@ -9,6 +9,17 @@ import { Model } from '../../mol-model/structure';
import Config from './config';
import { ConsoleLogger } from '../../mol-util/console-logger';
// TODO enable dynamic imports again
import * as pdbeProps from './properties/pdbe'
import * as rcsbProps from './properties/rcsb'
import * as wwpdbProps from './properties/wwpdb'
const attachModelProperties: { [k: string]: AttachModelProperties } = {
pdbe: pdbeProps.attachModelProperties,
rcsb: rcsbProps.attachModelProperties,
wwpdb: wwpdbProps.attachModelProperties
}
export interface ModelPropertyProviderConfig {
sources: string[],
params?: { [name: string]: any }
......@@ -39,7 +50,11 @@ export function createModelPropertiesProvider(configOrPath: ModelPropertyProvide
const ps: AttachModelProperties[] = [];
for (const p of config.sources) {
ps.push(require(p).attachModelProperties);
if (p in attachModelProperties) {
ps.push(attachModelProperties[p]);
} else {
ConsoleLogger.error('Config', `Could not find property provider '${p}', ignoring.`);
}
}
return (model, cache) => {
......
......@@ -16,7 +16,7 @@ import { LimitsConfig, addLimitsArgs, setLimitsConfig } from './config';
console.log(`VolumeServer Local ${VERSION}, (c) 2018-2019, Mol* contributors`);
console.log();
function help() {
function description() {
const exampleJobs: LocalApi.JobEntry[] = [{
source: {
filename: `g:/test/mdb/xray-1tqn.mdb`,
......@@ -55,7 +55,7 @@ function help() {
const parser = new argparse.ArgumentParser({
addHelp: true,
description: help()
description: description()
});
addLimitsArgs(parser)
parser.addArgument(['jobs'], {
......
......@@ -73,6 +73,12 @@ function createNodeEntryPoint(name, dir, out) {
target: 'node',
entry: path.resolve(__dirname, `lib/${dir}/${name}.js`),
output: { filename: `${name}.js`, path: path.resolve(__dirname, `build/${out}`) },
externals: {
argparse: 'require("argparse")',
'node-fetch': 'require("node-fetch")',
'util.promisify': 'require("util.promisify")',
xhr2: 'require("xhr2")',
},
...sharedConfig
}
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment