Skip to content
Snippets Groups Projects
Commit f21f5f5d authored by David Sehnal's avatar David Sehnal
Browse files

model-server: Parametize custom property definitions

parent d792c34e
No related branches found
No related tags found
No related merge requests found
...@@ -34,6 +34,9 @@ export namespace ConsoleLogger { ...@@ -34,6 +34,9 @@ export namespace ConsoleLogger {
if (e.stack) console.error(e.stack); if (e.stack) console.error(e.stack);
} }
export function warn(ctx: string, e: any) {
console.error(`[Warn] (${ctx}) ${e}`);
}
export function errorId(guid: string | String, e: any) { export function errorId(guid: string | String, e: any) {
console.error(`[${guid}][Error] ${e}`); console.error(`[${guid}][Error] ${e}`);
......
...@@ -48,12 +48,27 @@ const config = { ...@@ -48,12 +48,27 @@ const config = {
maxQueueLength: 30, maxQueueLength: 30,
/** /**
* Paths (relative to the root directory of the model server) to JavaScript files that specify custom properties * Provide a property config or a path a JSON file with the config.
*/ */
customPropertyProviders: [ customProperties: <import('./property-provider').ModelPropertyProviderConfig | string>{
'./properties/pdbe', sources: [
// './properties/rcsb' './properties/pdbe',
], // './properties/rcsb'
],
params: {
PDBe: {
UseFileSource: false,
API: {
residuewise_outlier_summary: 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry',
preferred_assembly: 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary',
struct_ref_domain: 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'
},
File: {
residuewise_outlier_summary: 'e:/test/mol-star/model/props/'
}
}
}
},
/** /**
* Maps a request identifier to a filename. * Maps a request identifier to a filename.
......
...@@ -8,6 +8,7 @@ import * as fs from 'fs' ...@@ -8,6 +8,7 @@ import * as fs from 'fs'
import * as path from 'path' import * as path from 'path'
import * as argparse from 'argparse' import * as argparse from 'argparse'
import { runMaster, PreprocessEntry } from './parallel'; import { runMaster, PreprocessEntry } from './parallel';
import { ModelPropertyProviderConfig } from '../property-provider';
const cmdParser = new argparse.ArgumentParser({ const cmdParser = new argparse.ArgumentParser({
addHelp: true, addHelp: true,
...@@ -37,13 +38,13 @@ interface CmdArgs { ...@@ -37,13 +38,13 @@ interface CmdArgs {
export interface PreprocessConfig { export interface PreprocessConfig {
numProcesses?: number, numProcesses?: number,
customPropertyProviders?: string[] customProperties?: ModelPropertyProviderConfig | string
} }
const cmdArgs = cmdParser.parseArgs() as CmdArgs; const cmdArgs = cmdParser.parseArgs() as CmdArgs;
let entries: PreprocessEntry[] = [] let entries: PreprocessEntry[] = []
let config: PreprocessConfig = { numProcesses: 1, customPropertyProviders: [] } let config: PreprocessConfig = { numProcesses: 1, customProperties: void 0 }
if (cmdArgs.input) entries.push({ source: cmdArgs.input, cif: cmdArgs.outCIF, bcif: cmdArgs.outBCIF }); if (cmdArgs.input) entries.push({ source: cmdArgs.input, cif: cmdArgs.outCIF, bcif: cmdArgs.outBCIF });
// else if (cmdArgs.bulk) runBulk(cmdArgs.bulk); // else if (cmdArgs.bulk) runBulk(cmdArgs.bulk);
......
...@@ -9,7 +9,7 @@ import * as cluster from 'cluster' ...@@ -9,7 +9,7 @@ import * as cluster from 'cluster'
import { now } from 'mol-task'; import { now } from 'mol-task';
import { PerformanceMonitor } from 'mol-util/performance-monitor'; import { PerformanceMonitor } from 'mol-util/performance-monitor';
import { preprocessFile } from './preprocess'; import { preprocessFile } from './preprocess';
import { createModelPropertiesProviderFromSources } from '../property-provider'; import { createModelPropertiesProvider } from '../property-provider';
type PreprocessConfig = import('./master').PreprocessConfig type PreprocessConfig = import('./master').PreprocessConfig
...@@ -50,7 +50,7 @@ export function runMaster(config: PreprocessConfig, entries: PreprocessEntry[]) ...@@ -50,7 +50,7 @@ export function runMaster(config: PreprocessConfig, entries: PreprocessEntry[])
export function runChild() { export function runChild() {
process.on('message', async ({ entries, config }: { entries: PreprocessEntry[], config: PreprocessConfig }) => { process.on('message', async ({ entries, config }: { entries: PreprocessEntry[], config: PreprocessConfig }) => {
const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []); const props = createModelPropertiesProvider(config.customProperties);
for (const entry of entries) { for (const entry of entries) {
try { try {
await preprocessFile(entry.source, props, entry.cif, entry.bcif); await preprocessFile(entry.source, props, entry.cif, entry.bcif);
...@@ -64,7 +64,7 @@ export function runChild() { ...@@ -64,7 +64,7 @@ export function runChild() {
} }
async function runSingle(entry: PreprocessEntry, config: PreprocessConfig, onMessage: (msg: any) => void) { async function runSingle(entry: PreprocessEntry, config: PreprocessConfig, onMessage: (msg: any) => void) {
const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []); const props = createModelPropertiesProvider(config.customProperties);
try { try {
await preprocessFile(entry.source, props, entry.cif, entry.bcif); await preprocessFile(entry.source, props, entry.cif, entry.bcif);
} catch (e) { } catch (e) {
......
...@@ -5,15 +5,15 @@ ...@@ -5,15 +5,15 @@
* @author Alexander Rose <alexander.rose@weirdbyte.de> * @author Alexander Rose <alexander.rose@weirdbyte.de>
*/ */
import { Model } from 'mol-model/structure';
import { PDBe_structureQualityReport, PDBe_preferredAssembly, PDBe_structRefDomain } from './providers/pdbe'; import { PDBe_structureQualityReport, PDBe_preferredAssembly, PDBe_structRefDomain } from './providers/pdbe';
import { AttachModelProperties } from '../property-provider';
export function attachModelProperties(model: Model, cache: object): Promise<any>[] { export const attachModelProperties: AttachModelProperties = (args) => {
// return a list of promises that start attaching the props in parallel // return a list of promises that start attaching the props in parallel
// (if there are downloads etc.) // (if there are downloads etc.)
return [ return [
PDBe_structureQualityReport(model, cache), PDBe_structureQualityReport(args),
PDBe_preferredAssembly(model, cache), PDBe_preferredAssembly(args),
PDBe_structRefDomain(model, cache) PDBe_structRefDomain(args)
]; ];
} }
\ No newline at end of file
...@@ -5,73 +5,98 @@ ...@@ -5,73 +5,98 @@
*/ */
import * as fs from 'fs' import * as fs from 'fs'
import * as path from 'path'
import { Model } from 'mol-model/structure'; import { Model } from 'mol-model/structure';
import { StructureQualityReport } from 'mol-model-props/pdbe/structure-quality-report'; import { StructureQualityReport } from 'mol-model-props/pdbe/structure-quality-report';
import { fetchRetry } from '../../utils/fetch-retry'; import { fetchRetry } from '../../utils/fetch-retry';
import { UUID } from 'mol-util'; import { UUID } from 'mol-util';
import { PDBePreferredAssembly } from 'mol-model-props/pdbe/preferred-assembly'; import { PDBePreferredAssembly } from 'mol-model-props/pdbe/preferred-assembly';
import { PDBeStructRefDomain } from 'mol-model-props/pdbe/struct-ref-domain'; import { PDBeStructRefDomain } from 'mol-model-props/pdbe/struct-ref-domain';
import { AttachModelProperty } from '../../property-provider';
import { ConsoleLogger } from 'mol-util/console-logger';
const USE_FILE_SOURCE = false; export const PDBe_structureQualityReport: AttachModelProperty = ({ model, params, cache }) => {
const PDBe_apiSourceJson = useFileSource(params)
? residuewise_outlier_summary.getDataFromAggregateFile(getFilePrefix(params, 'residuewise_outlier_summary'))
: apiQueryProvider(getApiUrl(params, 'residuewise_outlier_summary', 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry'), cache);
export function PDBe_structureQualityReport(model: Model, cache: any) { return StructureQualityReport.attachFromCifOrApi(model, { PDBe_apiSourceJson });
return StructureQualityReport.attachFromCifOrApi(model, {
PDBe_apiSourceJson: USE_FILE_SOURCE
? residuewise_outlier_summary.getDataFromAggregateFile
: residuewise_outlier_summary.getDataFromApiProvider(cache)
});
} }
export function PDBe_preferredAssembly(model: Model, cache: any) { export const PDBe_preferredAssembly: AttachModelProperty = ({ model, params, cache }) => {
return PDBePreferredAssembly.attachFromCifOrApi(model, { const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'preferred_assembly', 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary'), cache);
PDBe_apiSourceJson: USE_FILE_SOURCE return PDBePreferredAssembly.attachFromCifOrApi(model, { PDBe_apiSourceJson });
? void 0
: preferred_assembly.getDataFromApiProvider(cache)
});
} }
export function PDBe_structRefDomain(model: Model, cache: any) { export const PDBe_structRefDomain: AttachModelProperty = ({ model, params, cache }) => {
return PDBeStructRefDomain.attachFromCifOrApi(model, { const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'struct_ref_domain', 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'), cache);
PDBe_apiSourceJson: USE_FILE_SOURCE return PDBeStructRefDomain.attachFromCifOrApi(model, { PDBe_apiSourceJson });
? void 0
: struct_ref_domain.getDataFromApiProvider(cache)
});
} }
namespace preferred_assembly { namespace residuewise_outlier_summary {
export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary'); const json = new Map<string, any>();
export function getDataFromAggregateFile(pathPrefix: string) {
// This is for "testing" purposes and should probably only read
// a single file with the appropriate prop in the "production" version.
return async (model: Model) => {
const key = `${model.label[1]}${model.label[2]}`;
if (!json.has(key)) {
const fn = path.join(pathPrefix, `${key}.json`);
if (!fs.existsSync(fn)) json.set(key, { });
// TODO: use async readFile?
else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8')));
}
return json.get(key)![model.label.toLowerCase()] || { };
}
}
} }
namespace struct_ref_domain { function getApiUrl(params: any, name: string, fallback: string) {
export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'); const url = getParam<string>(params, 'PDBe', 'API', name);
if (!url) return fallback;
if (url[url.length - 1] === '/') return url.substring(0, url.length - 1);
return url;
} }
namespace residuewise_outlier_summary { function getFilePrefix(params: any, name: string) {
const json = new Map<string, any>(); const ret = getParam<string>(params, 'PDBe', 'File', name);
if (!ret) throw new Error(`PDBe file prefix '${name}' not set!`);
return ret;
}
function useFileSource(params: any) {
return !!getParam<boolean>(params, 'PDBe', 'UseFileSource')
}
export async function getDataFromAggregateFile(model: Model) { function getParam<T>(params: any, ...path: string[]): T | undefined {
const key = `${model.label[1]}${model.label[2]}`; try {
if (!json.has(key)) { let current = params;
const fn = `e:/test/mol-star/model/props/${key}.json`; for (const p of path) {
if (!fs.existsSync(fn)) json.set(key, { }); if (typeof current === 'undefined') return;
// TODO: use async readFile? current = current[p];
else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8')));
} }
return json.get(key)![model.label.toLowerCase()] || { }; return current;
} catch (e) {
ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
} }
export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry');
} }
function apiQueryProvider(urlPrefix: string) {
return (cache: any) => { function apiQueryProvider(urlPrefix: string, cache: any) {
const cacheKey = UUID.create(); const cacheKey = UUID.create();
return async (model: Model) => { return async (model: Model) => {
try {
if (cache[cacheKey]) return cache[cacheKey]; if (cache[cacheKey]) return cache[cacheKey];
const rawData = await fetchRetry(`${urlPrefix}/${model.label.toLowerCase()}`, 1500, 5); const rawData = await fetchRetry(`${urlPrefix}/${model.label.toLowerCase()}`, 1500, 5);
// TODO: is this ok?
if (rawData.status !== 200) return { };
const json = (await rawData.json())[model.label.toLowerCase()] || { }; const json = (await rawData.json())[model.label.toLowerCase()] || { };
cache[cacheKey] = json; cache[cacheKey] = json;
return json; return json;
} catch (e) {
// TODO: handle better
ConsoleLogger.warn('Props', `Count not retrieve prop @${`${urlPrefix}/${model.label.toLowerCase()}`}`);
return { };
} }
} }
} }
\ No newline at end of file
...@@ -4,27 +4,48 @@ ...@@ -4,27 +4,48 @@
* @author David Sehnal <david.sehnal@gmail.com> * @author David Sehnal <david.sehnal@gmail.com>
*/ */
import * as fs from 'fs'
import { Model } from 'mol-model/structure'; import { Model } from 'mol-model/structure';
import Config from './config'; import Config from './config';
import { ConsoleLogger } from 'mol-util/console-logger';
export type ModelPropertiesProvider = (model: Model, cache: object) => Promise<any>[] export interface ModelPropertyProviderConfig {
sources: string[],
params?: { [name: string]: any }
}
export type AttachModelProperty = (args: { model: Model, params: any, cache: any }) => Promise<any>
export type AttachModelProperties = (args: { model: Model, params: any, cache: any }) => Promise<any>[]
export type ModelPropertiesProvider = (model: Model, cache: any) => Promise<any>[]
export function createModelPropertiesProviderFromConfig(): ModelPropertiesProvider { export function createModelPropertiesProviderFromConfig(): ModelPropertiesProvider {
return createModelPropertiesProviderFromSources(Config.customPropertyProviders); return createModelPropertiesProvider(Config.customProperties);
} }
export function createModelPropertiesProviderFromSources(sources: string[]): ModelPropertiesProvider { export function createModelPropertiesProvider(configOrPath: ModelPropertyProviderConfig | string | undefined): ModelPropertiesProvider {
if (!sources || sources.length === 0) return () => []; let config: ModelPropertyProviderConfig;
if (typeof configOrPath === 'string') {
try {
config = JSON.parse(fs.readFileSync(configOrPath, 'utf8'));
} catch {
ConsoleLogger.error('Config', `Could not read property provider config file '${configOrPath}', ignoring.`);
return () => [];
}
} else {
config = configOrPath!;
}
if (!config || !config.sources || config.sources.length === 0) return () => [];
const ps: ModelPropertiesProvider[] = []; const ps: AttachModelProperties[] = [];
for (const p of sources) { for (const p of config.sources) {
ps.push(require(p).attachModelProperties); ps.push(require(p).attachModelProperties);
} }
return (model, cache) => { return (model, cache) => {
const ret: Promise<any>[] = []; const ret: Promise<any>[] = [];
for (const p of ps) { for (const p of ps) {
for (const e of p(model, cache)) ret.push(e); for (const e of p({ model, cache, params: config.params })) ret.push(e);
} }
return ret; return ret;
} }
......
...@@ -24,6 +24,7 @@ export async function fetchRetry(url: string, timeout: number, retryCount: numbe ...@@ -24,6 +24,7 @@ export async function fetchRetry(url: string, timeout: number, retryCount: numbe
retryCount retryCount
}); });
if (result.status >= 200 && result.status < 300) return result; return result;
throw new Error(result.statusText); // if (result.status >= 200 && result.status < 300) return result;
// throw new Error(result.statusText);
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment