From f21f5f5de0afd587209d636663076b7c67546425 Mon Sep 17 00:00:00 2001 From: David Sehnal <david.sehnal@gmail.com> Date: Wed, 3 Oct 2018 11:46:34 +0200 Subject: [PATCH] model-server: Parametize custom property definitions --- src/mol-util/console-logger.ts | 3 + src/servers/model/config.ts | 25 ++++- src/servers/model/preprocess/master.ts | 5 +- src/servers/model/preprocess/parallel.ts | 6 +- src/servers/model/properties/pdbe.ts | 10 +- .../model/properties/providers/pdbe.ts | 103 +++++++++++------- src/servers/model/property-provider.ts | 35 ++++-- src/servers/model/utils/fetch-retry.ts | 5 +- 8 files changed, 129 insertions(+), 63 deletions(-) diff --git a/src/mol-util/console-logger.ts b/src/mol-util/console-logger.ts index 5159b4040..c09f08b9a 100644 --- a/src/mol-util/console-logger.ts +++ b/src/mol-util/console-logger.ts @@ -34,6 +34,9 @@ export namespace ConsoleLogger { if (e.stack) console.error(e.stack); } + export function warn(ctx: string, e: any) { + console.error(`[Warn] (${ctx}) ${e}`); + } export function errorId(guid: string | String, e: any) { console.error(`[${guid}][Error] ${e}`); diff --git a/src/servers/model/config.ts b/src/servers/model/config.ts index ae25a2e55..301160a10 100644 --- a/src/servers/model/config.ts +++ b/src/servers/model/config.ts @@ -48,12 +48,27 @@ const config = { maxQueueLength: 30, /** - * Paths (relative to the root directory of the model server) to JavaScript files that specify custom properties + * Provide a property config or a path a JSON file with the config. */ - customPropertyProviders: [ - './properties/pdbe', - // './properties/rcsb' - ], + customProperties: <import('./property-provider').ModelPropertyProviderConfig | string>{ + sources: [ + './properties/pdbe', + // './properties/rcsb' + ], + params: { + PDBe: { + UseFileSource: false, + API: { + residuewise_outlier_summary: 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry', + preferred_assembly: 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary', + struct_ref_domain: 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains' + }, + File: { + residuewise_outlier_summary: 'e:/test/mol-star/model/props/' + } + } + } + }, /** * Maps a request identifier to a filename. diff --git a/src/servers/model/preprocess/master.ts b/src/servers/model/preprocess/master.ts index 8a4f70ef3..7dbb2ba39 100644 --- a/src/servers/model/preprocess/master.ts +++ b/src/servers/model/preprocess/master.ts @@ -8,6 +8,7 @@ import * as fs from 'fs' import * as path from 'path' import * as argparse from 'argparse' import { runMaster, PreprocessEntry } from './parallel'; +import { ModelPropertyProviderConfig } from '../property-provider'; const cmdParser = new argparse.ArgumentParser({ addHelp: true, @@ -37,13 +38,13 @@ interface CmdArgs { export interface PreprocessConfig { numProcesses?: number, - customPropertyProviders?: string[] + customProperties?: ModelPropertyProviderConfig | string } const cmdArgs = cmdParser.parseArgs() as CmdArgs; let entries: PreprocessEntry[] = [] -let config: PreprocessConfig = { numProcesses: 1, customPropertyProviders: [] } +let config: PreprocessConfig = { numProcesses: 1, customProperties: void 0 } if (cmdArgs.input) entries.push({ source: cmdArgs.input, cif: cmdArgs.outCIF, bcif: cmdArgs.outBCIF }); // else if (cmdArgs.bulk) runBulk(cmdArgs.bulk); diff --git a/src/servers/model/preprocess/parallel.ts b/src/servers/model/preprocess/parallel.ts index ff2b37db7..2e24f3e0c 100644 --- a/src/servers/model/preprocess/parallel.ts +++ b/src/servers/model/preprocess/parallel.ts @@ -9,7 +9,7 @@ import * as cluster from 'cluster' import { now } from 'mol-task'; import { PerformanceMonitor } from 'mol-util/performance-monitor'; import { preprocessFile } from './preprocess'; -import { createModelPropertiesProviderFromSources } from '../property-provider'; +import { createModelPropertiesProvider } from '../property-provider'; type PreprocessConfig = import('./master').PreprocessConfig @@ -50,7 +50,7 @@ export function runMaster(config: PreprocessConfig, entries: PreprocessEntry[]) export function runChild() { process.on('message', async ({ entries, config }: { entries: PreprocessEntry[], config: PreprocessConfig }) => { - const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []); + const props = createModelPropertiesProvider(config.customProperties); for (const entry of entries) { try { await preprocessFile(entry.source, props, entry.cif, entry.bcif); @@ -64,7 +64,7 @@ export function runChild() { } async function runSingle(entry: PreprocessEntry, config: PreprocessConfig, onMessage: (msg: any) => void) { - const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []); + const props = createModelPropertiesProvider(config.customProperties); try { await preprocessFile(entry.source, props, entry.cif, entry.bcif); } catch (e) { diff --git a/src/servers/model/properties/pdbe.ts b/src/servers/model/properties/pdbe.ts index 7f678a19b..9512ae5fb 100644 --- a/src/servers/model/properties/pdbe.ts +++ b/src/servers/model/properties/pdbe.ts @@ -5,15 +5,15 @@ * @author Alexander Rose <alexander.rose@weirdbyte.de> */ -import { Model } from 'mol-model/structure'; import { PDBe_structureQualityReport, PDBe_preferredAssembly, PDBe_structRefDomain } from './providers/pdbe'; +import { AttachModelProperties } from '../property-provider'; -export function attachModelProperties(model: Model, cache: object): Promise<any>[] { +export const attachModelProperties: AttachModelProperties = (args) => { // return a list of promises that start attaching the props in parallel // (if there are downloads etc.) return [ - PDBe_structureQualityReport(model, cache), - PDBe_preferredAssembly(model, cache), - PDBe_structRefDomain(model, cache) + PDBe_structureQualityReport(args), + PDBe_preferredAssembly(args), + PDBe_structRefDomain(args) ]; } \ No newline at end of file diff --git a/src/servers/model/properties/providers/pdbe.ts b/src/servers/model/properties/providers/pdbe.ts index 31a04a731..df2b852b4 100644 --- a/src/servers/model/properties/providers/pdbe.ts +++ b/src/servers/model/properties/providers/pdbe.ts @@ -5,73 +5,98 @@ */ import * as fs from 'fs' +import * as path from 'path' import { Model } from 'mol-model/structure'; import { StructureQualityReport } from 'mol-model-props/pdbe/structure-quality-report'; import { fetchRetry } from '../../utils/fetch-retry'; import { UUID } from 'mol-util'; import { PDBePreferredAssembly } from 'mol-model-props/pdbe/preferred-assembly'; import { PDBeStructRefDomain } from 'mol-model-props/pdbe/struct-ref-domain'; +import { AttachModelProperty } from '../../property-provider'; +import { ConsoleLogger } from 'mol-util/console-logger'; -const USE_FILE_SOURCE = false; +export const PDBe_structureQualityReport: AttachModelProperty = ({ model, params, cache }) => { + const PDBe_apiSourceJson = useFileSource(params) + ? residuewise_outlier_summary.getDataFromAggregateFile(getFilePrefix(params, 'residuewise_outlier_summary')) + : apiQueryProvider(getApiUrl(params, 'residuewise_outlier_summary', 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry'), cache); -export function PDBe_structureQualityReport(model: Model, cache: any) { - return StructureQualityReport.attachFromCifOrApi(model, { - PDBe_apiSourceJson: USE_FILE_SOURCE - ? residuewise_outlier_summary.getDataFromAggregateFile - : residuewise_outlier_summary.getDataFromApiProvider(cache) - }); + return StructureQualityReport.attachFromCifOrApi(model, { PDBe_apiSourceJson }); } -export function PDBe_preferredAssembly(model: Model, cache: any) { - return PDBePreferredAssembly.attachFromCifOrApi(model, { - PDBe_apiSourceJson: USE_FILE_SOURCE - ? void 0 - : preferred_assembly.getDataFromApiProvider(cache) - }); +export const PDBe_preferredAssembly: AttachModelProperty = ({ model, params, cache }) => { + const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'preferred_assembly', 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary'), cache); + return PDBePreferredAssembly.attachFromCifOrApi(model, { PDBe_apiSourceJson }); } -export function PDBe_structRefDomain(model: Model, cache: any) { - return PDBeStructRefDomain.attachFromCifOrApi(model, { - PDBe_apiSourceJson: USE_FILE_SOURCE - ? void 0 - : struct_ref_domain.getDataFromApiProvider(cache) - }); +export const PDBe_structRefDomain: AttachModelProperty = ({ model, params, cache }) => { + const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'struct_ref_domain', 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'), cache); + return PDBeStructRefDomain.attachFromCifOrApi(model, { PDBe_apiSourceJson }); } -namespace preferred_assembly { - export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary'); +namespace residuewise_outlier_summary { + const json = new Map<string, any>(); + export function getDataFromAggregateFile(pathPrefix: string) { + // This is for "testing" purposes and should probably only read + // a single file with the appropriate prop in the "production" version. + return async (model: Model) => { + const key = `${model.label[1]}${model.label[2]}`; + if (!json.has(key)) { + const fn = path.join(pathPrefix, `${key}.json`); + if (!fs.existsSync(fn)) json.set(key, { }); + // TODO: use async readFile? + else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8'))); + } + return json.get(key)![model.label.toLowerCase()] || { }; + } + } } -namespace struct_ref_domain { - export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'); +function getApiUrl(params: any, name: string, fallback: string) { + const url = getParam<string>(params, 'PDBe', 'API', name); + if (!url) return fallback; + if (url[url.length - 1] === '/') return url.substring(0, url.length - 1); + return url; } -namespace residuewise_outlier_summary { - const json = new Map<string, any>(); +function getFilePrefix(params: any, name: string) { + const ret = getParam<string>(params, 'PDBe', 'File', name); + if (!ret) throw new Error(`PDBe file prefix '${name}' not set!`); + return ret; +} + +function useFileSource(params: any) { + return !!getParam<boolean>(params, 'PDBe', 'UseFileSource') +} - export async function getDataFromAggregateFile(model: Model) { - const key = `${model.label[1]}${model.label[2]}`; - if (!json.has(key)) { - const fn = `e:/test/mol-star/model/props/${key}.json`; - if (!fs.existsSync(fn)) json.set(key, { }); - // TODO: use async readFile? - else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8'))); +function getParam<T>(params: any, ...path: string[]): T | undefined { + try { + let current = params; + for (const p of path) { + if (typeof current === 'undefined') return; + current = current[p]; } - return json.get(key)![model.label.toLowerCase()] || { }; + return current; + } catch (e) { + ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`); } - - export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry'); } -function apiQueryProvider(urlPrefix: string) { - return (cache: any) => { - const cacheKey = UUID.create(); - return async (model: Model) => { + +function apiQueryProvider(urlPrefix: string, cache: any) { + const cacheKey = UUID.create(); + return async (model: Model) => { + try { if (cache[cacheKey]) return cache[cacheKey]; const rawData = await fetchRetry(`${urlPrefix}/${model.label.toLowerCase()}`, 1500, 5); + // TODO: is this ok? + if (rawData.status !== 200) return { }; const json = (await rawData.json())[model.label.toLowerCase()] || { }; cache[cacheKey] = json; return json; + } catch (e) { + // TODO: handle better + ConsoleLogger.warn('Props', `Count not retrieve prop @${`${urlPrefix}/${model.label.toLowerCase()}`}`); + return { }; } } } \ No newline at end of file diff --git a/src/servers/model/property-provider.ts b/src/servers/model/property-provider.ts index 11673cf4e..d64e5b4cf 100644 --- a/src/servers/model/property-provider.ts +++ b/src/servers/model/property-provider.ts @@ -4,27 +4,48 @@ * @author David Sehnal <david.sehnal@gmail.com> */ +import * as fs from 'fs' import { Model } from 'mol-model/structure'; import Config from './config'; +import { ConsoleLogger } from 'mol-util/console-logger'; -export type ModelPropertiesProvider = (model: Model, cache: object) => Promise<any>[] +export interface ModelPropertyProviderConfig { + sources: string[], + params?: { [name: string]: any } +} + +export type AttachModelProperty = (args: { model: Model, params: any, cache: any }) => Promise<any> +export type AttachModelProperties = (args: { model: Model, params: any, cache: any }) => Promise<any>[] +export type ModelPropertiesProvider = (model: Model, cache: any) => Promise<any>[] export function createModelPropertiesProviderFromConfig(): ModelPropertiesProvider { - return createModelPropertiesProviderFromSources(Config.customPropertyProviders); + return createModelPropertiesProvider(Config.customProperties); } -export function createModelPropertiesProviderFromSources(sources: string[]): ModelPropertiesProvider { - if (!sources || sources.length === 0) return () => []; +export function createModelPropertiesProvider(configOrPath: ModelPropertyProviderConfig | string | undefined): ModelPropertiesProvider { + let config: ModelPropertyProviderConfig; + if (typeof configOrPath === 'string') { + try { + config = JSON.parse(fs.readFileSync(configOrPath, 'utf8')); + } catch { + ConsoleLogger.error('Config', `Could not read property provider config file '${configOrPath}', ignoring.`); + return () => []; + } + } else { + config = configOrPath!; + } + + if (!config || !config.sources || config.sources.length === 0) return () => []; - const ps: ModelPropertiesProvider[] = []; - for (const p of sources) { + const ps: AttachModelProperties[] = []; + for (const p of config.sources) { ps.push(require(p).attachModelProperties); } return (model, cache) => { const ret: Promise<any>[] = []; for (const p of ps) { - for (const e of p(model, cache)) ret.push(e); + for (const e of p({ model, cache, params: config.params })) ret.push(e); } return ret; } diff --git a/src/servers/model/utils/fetch-retry.ts b/src/servers/model/utils/fetch-retry.ts index 777221274..b29b7ec35 100644 --- a/src/servers/model/utils/fetch-retry.ts +++ b/src/servers/model/utils/fetch-retry.ts @@ -24,6 +24,7 @@ export async function fetchRetry(url: string, timeout: number, retryCount: numbe retryCount }); - if (result.status >= 200 && result.status < 300) return result; - throw new Error(result.statusText); + return result; + // if (result.status >= 200 && result.status < 300) return result; + // throw new Error(result.statusText); } -- GitLab