From 76503b52f586bb6235070d26a0dad126fe1635ad Mon Sep 17 00:00:00 2001 From: David Sehnal <david.sehnal@gmail.com> Date: Wed, 14 Aug 2019 14:21:13 +0200 Subject: [PATCH] wip model-server --- src/apps/model-server-query/index.tsx | 2 +- src/servers/model/config.ts | 18 ++- src/servers/model/query/atoms.ts | 7 +- src/servers/model/query/schemas.ts | 51 ++++++++- src/servers/model/server/api-web.ts | 75 +++++++------ src/servers/model/server/api.ts | 153 ++++++++++++++++++-------- src/servers/model/server/jobs.ts | 6 +- src/servers/model/server/query.ts | 20 ++-- src/servers/model/version.ts | 4 +- 9 files changed, 225 insertions(+), 111 deletions(-) diff --git a/src/apps/model-server-query/index.tsx b/src/apps/model-server-query/index.tsx index 0c143ceba..0ed4cc3bb 100644 --- a/src/apps/model-server-query/index.tsx +++ b/src/apps/model-server-query/index.tsx @@ -112,7 +112,7 @@ const state: State = { function formatParams(def: QueryDefinition) { const prms = Object.create(null); - for (const p of def.params) { + for (const p of def.jsonParams) { prms[p.name] = p.exampleValues ? p.exampleValues[0] : void 0; } return JSON.stringify(prms, void 0, 2); diff --git a/src/servers/model/config.ts b/src/servers/model/config.ts index 860f6a0a4..7a8c495b5 100644 --- a/src/servers/model/config.ts +++ b/src/servers/model/config.ts @@ -4,8 +4,6 @@ * @author David Sehnal <david.sehnal@gmail.com> */ -import { ModelPropertyProviderConfig } from './property-provider'; - const config = { /** * Determine if and how long to cache entries after a request. @@ -52,11 +50,11 @@ const config = { /** * Provide a property config or a path a JSON file with the config. */ - customProperties: <ModelPropertyProviderConfig | string>{ + customProperties: <import('./property-provider').ModelPropertyProviderConfig | string>{ sources: [ - 'pdbe', - 'rcsb', - 'wwpdb' + // 'pdbe', + // 'rcsb', + // 'wwpdb' ], params: { PDBe: { @@ -91,10 +89,10 @@ const config = { */ mapFile(source: string, id: string) { switch (source.toLowerCase()) { - // case 'pdb': return `e:/test/quick/${id}_updated.cif`; - case 'pdb': return `e:/test/mol-star/model/out/${id}_updated.bcif`; - case 'pdb-bcif': return `c:/test/mol-star/model/out/${id}_updated.bcif`; - case 'pdb-cif': return `c:/test/mol-star/model/out/${id}_updated.cif`; + case 'pdb': return `e:/test/quick/${id}_updated.cif`; + // case 'pdb': return `e:/test/mol-star/model/out/${id}_updated.bcif`; + // case 'pdb-bcif': return `c:/test/mol-star/model/out/${id}_updated.bcif`; + // case 'pdb-cif': return `c:/test/mol-star/model/out/${id}_updated.cif`; default: return void 0; } } diff --git a/src/servers/model/query/atoms.ts b/src/servers/model/query/atoms.ts index 1742592ca..495efca81 100644 --- a/src/servers/model/query/atoms.ts +++ b/src/servers/model/query/atoms.ts @@ -7,6 +7,7 @@ import { QueryPredicate, StructureElement, StructureProperties as Props } from '../../../mol-model/structure'; import { AtomsQueryParams } from '../../../mol-model/structure/query/queries/generators'; import { AtomSiteSchema, AtomSiteSchemaElement } from '../server/api'; +import { ElementSymbol } from '../../../mol-model/structure/model/types'; export function getAtomsTests(params: AtomSiteSchema): Partial<AtomsQueryParams>[] { if (!params) return [{ }]; @@ -86,17 +87,17 @@ function atomTest(params: AtomSiteSchemaElement): QueryPredicate | undefined { if (typeof params.label_atom_id !== 'undefined') { props.push(Props.atom.label_atom_id); - values.push(+params.label_atom_id); + values.push(params.label_atom_id); } if (typeof params.auth_atom_id !== 'undefined') { props.push(Props.atom.auth_atom_id); - values.push(+params.auth_atom_id); + values.push(params.auth_atom_id); } if (typeof params.type_symbol !== 'undefined') { props.push(Props.atom.type_symbol); - values.push(+params.type_symbol); + values.push(ElementSymbol(params.type_symbol)); } return andEqual(props, values); diff --git a/src/servers/model/query/schemas.ts b/src/servers/model/query/schemas.ts index 0ffdd02fc..7f852bcae 100644 --- a/src/servers/model/query/schemas.ts +++ b/src/servers/model/query/schemas.ts @@ -1 +1,50 @@ -// TODO \ No newline at end of file +/** + * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info. + * + * @author David Sehnal <david.sehnal@gmail.com> + */ + +import { CifWriter } from '../../../mol-io/writer/cif'; + +const InteractionCategories = new Set([ + 'entry', + 'entity', + 'exptl', + 'cell', + 'symmetry', + 'struct_conf', + 'struct_sheet_range', + 'entity_poly', + 'struct_asym', + 'struct_conn', + 'struct_conn_type', + 'pdbx_struct_mod_residue', + 'chem_comp_bond', + 'atom_sites' +]); + +const AssemblyCategories = new Set([ + 'entry', + 'entity', + 'exptl', + 'cell', + 'symmetry', + 'struct_conf', + 'struct_sheet_range', + 'entity_poly', + 'entity_poly_seq', + 'pdbx_nonpoly_scheme', + 'struct_asym', + 'struct_conn', + 'struct_conn_type', + 'pdbx_struct_mod_residue', + 'chem_comp_bond', + 'atom_sites' +]); + +export const QuerySchemas = { + interaction: <CifWriter.Category.Filter>{ + includeCategory(name) { return InteractionCategories.has(name); }, + includeField(cat, field) { return true; } + } +} \ No newline at end of file diff --git a/src/servers/model/server/api-web.ts b/src/servers/model/server/api-web.ts index d1a1dbfb9..45cab6709 100644 --- a/src/servers/model/server/api-web.ts +++ b/src/servers/model/server/api-web.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info. + * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info. * * @author David Sehnal <david.sehnal@gmail.com> */ @@ -13,6 +13,7 @@ import { resolveJob } from './query'; import { JobManager } from './jobs'; import { UUID } from '../../../mol-util'; import { LandingPage } from './landing'; +import { QueryDefinition, normalizeRestQueryParams, normalizeRestCommonParams, QueryList } from './api'; function makePath(p: string) { return Config.appPrefix + '/' + p; @@ -83,21 +84,23 @@ async function processNextJob() { } } -// function mapQuery(app: express.Express, queryName: string, queryDefinition: QueryDefinition) { -// app.get(makePath(':entryId/' + queryName), (req, res) => { -// ConsoleLogger.log('Server', `Query '${req.params.entryId}/${queryName}'...`); - -// if (JobManager.size >= Config.maxQueueLength) { -// res.status(503).send('Too many queries, please try again later.'); -// res.end(); -// return; -// } - -// const jobId = JobManager.add('pdb', req.params.entryId, queryName, req.query); -// responseMap.set(jobId, res); -// if (JobManager.size === 1) processNextJob(); -// }); -// } +function mapQuery(app: express.Express, queryName: string, queryDefinition: QueryDefinition) { + app.get(makePath('api/v1/:id/' + queryName), (req, res) => { + console.log({ queryName, params: req.params, query: req.query }); + const entryId = req.params.id; + const queryParams = normalizeRestQueryParams(queryDefinition, req.query); + const commonParams = normalizeRestCommonParams(req.query); + const jobId = JobManager.add({ + sourceId: commonParams.data_source || 'pdb', + entryId, + queryName: queryName as any, + queryParams, + options: { modelNums: commonParams.model_nums, binary: commonParams.encoding === 'bcif' } + }); + responseMap.set(jobId, res); + if (JobManager.size === 1) processNextJob(); + }); +} export function initWebApi(app: express.Express) { app.get(makePath('static/:format/:id'), async (req, res) => { @@ -128,28 +131,28 @@ export function initWebApi(app: express.Express) { }); }) - app.get(makePath('api/v1'), (req, res) => { - const query = /\?(.*)$/.exec(req.url)![1]; - const args = JSON.parse(decodeURIComponent(query)); - const name = args.name; - const entryId = args.id; - const queryParams = args.params || { }; - const jobId = JobManager.add({ - sourceId: 'pdb', - entryId, - queryName: name, - queryParams, - options: { modelNums: args.modelNums, binary: args.binary } - }); - responseMap.set(jobId, res); - if (JobManager.size === 1) processNextJob(); - }); + // app.get(makePath('api/v1/json'), (req, res) => { + // const query = /\?(.*)$/.exec(req.url)![1]; + // const args = JSON.parse(decodeURIComponent(query)); + // const name = args.name; + // const entryId = args.id; + // const queryParams = args.params || { }; + // const jobId = JobManager.add({ + // sourceId: 'pdb', + // entryId, + // queryName: name, + // queryParams, + // options: { modelNums: args.modelNums, binary: args.binary } + // }); + // responseMap.set(jobId, res); + // if (JobManager.size === 1) processNextJob(); + // }); + + for (const q of QueryList) { + mapQuery(app, q.name, q.definition); + } app.get('*', (req, res) => { res.send(LandingPage); }); - - // for (const q of QueryList) { - // mapQuery(app, q.name, q.definition); - // } } \ No newline at end of file diff --git a/src/servers/model/server/api.ts b/src/servers/model/server/api.ts index fd354875b..86a6c36dd 100644 --- a/src/servers/model/server/api.ts +++ b/src/servers/model/server/api.ts @@ -1,27 +1,31 @@ /** - * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info. + * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info. * * @author David Sehnal <david.sehnal@gmail.com> */ import { Queries, Structure, StructureQuery, StructureSymmetry } from '../../../mol-model/structure'; import { getAtomsTests } from '../query/atoms'; +import { CifWriter } from '../../../mol-io/writer/cif'; +import { QuerySchemas } from '../query/schemas'; export enum QueryParamType { JSON, String, Integer, - Float + Float, + Boolean } -export interface QueryParamInfo { +export interface QueryParamInfo<T extends string | number = string | number> { name: string, type: QueryParamType, description?: string, required?: boolean, defaultValue?: any, exampleValues?: any[], - validation?: (v: any) => void + validation?: (v: T) => void, + supportedValues?: string[] } export interface QueryDefinition<Params = any> { @@ -30,11 +34,42 @@ export interface QueryDefinition<Params = any> { exampleId: string, // default is 1cbs query: (params: any, structure: Structure) => StructureQuery, description: string, - params: QueryParamInfo[], + jsonParams: QueryParamInfo[], + restParams: QueryParamInfo[], structureTransform?: (params: any, s: Structure) => Promise<Structure>, + filter?: CifWriter.Category.Filter, '@params': Params } +export const CommonQueryParamsInfo: QueryParamInfo[] = [ + { name: 'model_nums', type: QueryParamType.String, description: `A comma-separated list of model ids (i.e. 1,2). If set, only include atoms with the corresponding '_atom_site.pdbx_PDB_model_num' field.` }, + { name: 'encoding', type: QueryParamType.String, defaultValue: 'cif', description: `Determines the output encoding (text based 'CIF' or binary 'BCIF').`, supportedValues: ['cif', 'bcif'] }, + { name: 'data_Source', type: QueryParamType.String, defaultValue: '', description: 'Allows to control how the provided data source ID maps to input file (as specified by the server instance config).' } +]; + +export interface CommonQueryParamsInfo { + model_nums?: number[], + encoding?: 'cif' | 'bcif', + data_source?: string +} + +export const AtomSiteSchemaElement = { + label_entity_id: { type: QueryParamType.String }, + + label_asym_id: { type: QueryParamType.String }, + auth_asym_id: { type: QueryParamType.String }, + + label_comp_id: { type: QueryParamType.String }, + auth_comp_id: { type: QueryParamType.String }, + label_seq_id: { type: QueryParamType.Integer }, + auth_seq_id: { type: QueryParamType.Integer }, + pdbx_PDB_ins_code: { type: QueryParamType.String }, + + label_atom_id: { type: QueryParamType.String }, + auth_atom_id: { type: QueryParamType.String }, + type_symbol: { type: QueryParamType.String } +} + export interface AtomSiteSchemaElement { label_entity_id?: string, @@ -43,8 +78,8 @@ export interface AtomSiteSchemaElement { label_comp_id?: string, auth_comp_id?: string, - label_seq_id?: string, - auth_seq_id?: string, + label_seq_id?: number, + auth_seq_id?: number, pdbx_PDB_ins_code?: string, label_atom_id?: string, @@ -54,13 +89,23 @@ export interface AtomSiteSchemaElement { export type AtomSiteSchema = AtomSiteSchemaElement | AtomSiteSchemaElement[] -const AtomSiteTestParams: QueryParamInfo = { +const AtomSiteTestJsonParam: QueryParamInfo = { name: 'atom_site', type: QueryParamType.JSON, description: 'Object or array of objects describing atom properties. Names are same as in wwPDB mmCIF dictionary of the atom_site category.', exampleValues: [{ label_comp_id: 'ALA' }, { label_seq_id: 123, label_asym_id: 'A' }] }; +export const AtomSiteTestRestParams = (function() { + const params: QueryParamInfo[] = []; + for (const k of Object.keys(AtomSiteSchemaElement)) { + const p = (AtomSiteSchemaElement as any)[k] as QueryParamInfo; + p.name = k; + params.push(p); + } + return params; +})(); + const RadiusParam: QueryParamInfo = { name: 'radius', type: QueryParamType.Float, @@ -83,8 +128,9 @@ const QueryMap = { 'atoms': Q<{ atom_site: AtomSiteSchema }>({ niceName: 'Atoms', description: 'Atoms satisfying the given criteria.', - query: p => Queries.combinators.merge(getAtomsTests(p.atom_site).map(test => Queries.generators.atoms(test))), - params: [ AtomSiteTestParams ] + query: p => Queries.combinators.merge(getAtomsTests(p).map(test => Queries.generators.atoms(test))), + jsonParams: [ AtomSiteTestJsonParam ], + restParams: AtomSiteTestRestParams }), 'symmetryMates': Q<{ radius: number }>({ niceName: 'Symmetry Mates', @@ -93,7 +139,7 @@ const QueryMap = { structureTransform(p, s) { return StructureSymmetry.builderSymmetryMates(s, p.radius).run(); }, - params: [ RadiusParam ] + jsonParams: [ RadiusParam ] }), 'assembly': Q<{ name: string }>({ niceName: 'Assembly', @@ -102,7 +148,7 @@ const QueryMap = { structureTransform(p, s) { return StructureSymmetry.buildAssembly(s, '' + (p.name || '1')).run(); }, - params: [{ + jsonParams: [{ name: 'name', type: QueryParamType.String, defaultValue: '1', @@ -110,11 +156,11 @@ const QueryMap = { description: 'Assembly name.' }] }), - 'residueInteraction': Q<{ atom_site: AtomSiteSchema, radius: number }>({ + 'residueInteraction': Q<AtomSiteSchema & { radius: number }>({ niceName: 'Residue Interaction', description: 'Identifies all residues within the given radius from the source residue. Takes crystal symmetry into account.', query(p) { - const tests = getAtomsTests(p.atom_site); + const tests = getAtomsTests(p); const center = Queries.combinators.merge(tests.map(test => Queries.generators.atoms({ ...test, entityTest: test.entityTest @@ -126,17 +172,21 @@ const QueryMap = { structureTransform(p, s) { return StructureSymmetry.builderSymmetryMates(s, p.radius).run(); }, - params: [ AtomSiteTestParams, RadiusParam ] + jsonParams: [ AtomSiteTestJsonParam, RadiusParam ], + restParams: [ ...AtomSiteTestRestParams, RadiusParam ], + filter: QuerySchemas.interaction }), - 'residueSurroundings': Q<{ atom_site: AtomSiteSchema, radius: number }>({ + 'residueSurroundings': Q<AtomSiteSchema & { radius: number }>({ niceName: 'Residue Surroundings', description: 'Identifies all residues within the given radius from the source residue.', query(p) { - const tests = getAtomsTests(p.atom_site); + const tests = getAtomsTests(p); const center = Queries.combinators.merge(tests.map(test => Queries.generators.atoms(test))); return Queries.modifiers.includeSurroundings(center, { radius: p.radius, wholeResidues: true }); }, - params: [ AtomSiteTestParams, RadiusParam ] + jsonParams: [ AtomSiteTestJsonParam, RadiusParam ], + restParams: [ ...AtomSiteTestRestParams, RadiusParam ], + filter: QuerySchemas.interaction }) }; @@ -159,36 +209,45 @@ export const QueryList = (function () { for (let q of QueryList) { const m = q.definition; m.name = q.name; - m.params = m.params || []; + m.jsonParams = m.jsonParams || []; + m.restParams = m.restParams || m.jsonParams; } })(); -// function _normalizeQueryParams(params: { [p: string]: string }, paramList: QueryParamInfo[]): { [p: string]: string | number | boolean } { -// const ret: any = {}; -// for (const p of paramList) { -// const key = p.name; -// const value = params[key]; -// if (typeof value === 'undefined' || (typeof value !== 'undefined' && value !== null && value['length'] === 0)) { -// if (p.required) { -// throw `The parameter '${key}' is required.`; -// } -// if (typeof p.defaultValue !== 'undefined') ret[key] = p.defaultValue; -// } else { -// switch (p.type) { -// case QueryParamType.JSON: ret[key] = JSON.parse(value); break; -// case QueryParamType.String: ret[key] = value; break; -// case QueryParamType.Integer: ret[key] = parseInt(value); break; -// case QueryParamType.Float: ret[key] = parseFloat(value); break; -// } - -// if (p.validation) p.validation(ret[key]); -// } -// } - -// return ret; -// } - -export function normalizeQueryParams(query: QueryDefinition, params: any) { - return params; - // return _normalizeQueryParams(params, query.params); +function _normalizeQueryParams(params: { [p: string]: string }, paramList: QueryParamInfo[]): { [p: string]: string | number | boolean } { + const ret: any = {}; + for (const p of paramList) { + const key = p.name; + const value = params[key]; + if (typeof value === 'undefined' || (typeof value !== 'undefined' && value !== null && value['length'] === 0)) { + if (p.required) { + throw `The parameter '${key}' is required.`; + } + if (typeof p.defaultValue !== 'undefined') ret[key] = p.defaultValue; + } else { + switch (p.type) { + case QueryParamType.JSON: ret[key] = JSON.parse(value); break; + case QueryParamType.String: ret[key] = value; break; + case QueryParamType.Integer: ret[key] = parseInt(value); break; + case QueryParamType.Float: ret[key] = parseFloat(value); break; + } + + if (p.validation) p.validation(ret[key]); + } + } + + return ret; +} + +export function normalizeRestQueryParams(query: QueryDefinition, params: any) { + // return params; + return _normalizeQueryParams(params, query.restParams); +} + +export function normalizeRestCommonParams(params: any): CommonQueryParamsInfo { + return { + model_nums: params.model_nums ? ('' + params.model_nums).split(',').map(n => n.trim()).filter(n => !!n).map(n => +n) : void 0, + data_source: params.data_source, + encoding: ('' + params.encoding).toLocaleLowerCase() === 'bcif' ? 'bcif' : 'cif' + }; } \ No newline at end of file diff --git a/src/servers/model/server/jobs.ts b/src/servers/model/server/jobs.ts index 52e0e06b9..e0c187c32 100644 --- a/src/servers/model/server/jobs.ts +++ b/src/servers/model/server/jobs.ts @@ -1,11 +1,11 @@ /** - * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info. + * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info. * * @author David Sehnal <david.sehnal@gmail.com> */ import { UUID } from '../../../mol-util'; -import { getQueryByName, normalizeQueryParams, QueryDefinition, QueryName, QueryParams } from './api'; +import { getQueryByName, QueryDefinition, QueryName, QueryParams } from './api'; import { LinkedList } from '../../../mol-data/generic'; export interface ResponseFormat { @@ -40,7 +40,7 @@ export function createJob<Name extends QueryName>(definition: JobDefinition<Name const queryDefinition = getQueryByName(definition.queryName); if (!queryDefinition) throw new Error(`Query '${definition.queryName}' is not supported.`); - const normalizedParams = normalizeQueryParams(queryDefinition, definition.queryParams); + const normalizedParams = definition.queryParams; const sourceId = definition.sourceId || '_local_'; return { id: UUID.create22(), diff --git a/src/servers/model/server/query.ts b/src/servers/model/server/query.ts index 3449f485f..7cf76311a 100644 --- a/src/servers/model/server/query.ts +++ b/src/servers/model/server/query.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info. + * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info. * * @author David Sehnal <david.sehnal@gmail.com> */ @@ -22,7 +22,8 @@ import { createModelPropertiesProviderFromConfig, ModelPropertiesProvider } from export interface Stats { structure: StructureWrapper, queryTimeMs: number, - encodeTimeMs: number + encodeTimeMs: number, + resultSize: number } const perf = new PerformanceMonitor(); @@ -56,7 +57,8 @@ export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> { const queries = structures.map(s => job.queryDefinition.query(job.normalizedParams, s)); const result: Structure[] = []; for (let i = 0; i < structures.length; i++) { - result.push(await StructureSelection.unionStructure(StructureQuery.run(queries[i], structures[i], Config.maxQueryTimeInMs))); + const s = await StructureSelection.unionStructure(StructureQuery.run(queries[i], structures[i], Config.maxQueryTimeInMs)) + if (s.elementCount > 0) result.push(s); } perf.end('query'); @@ -74,15 +76,16 @@ export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> { encoder.writeCategory(_model_server_result, job); encoder.writeCategory(_model_server_params, job); - // encoder.setFilter(mmCIF_Export_Filters.onlyPositions); - encode_mmCIF_categories(encoder, result); - // encoder.setFilter(); + if (job.queryDefinition.filter) encoder.setFilter(job.queryDefinition.filter); + if (result.length > 0) encode_mmCIF_categories(encoder, result); + if (job.queryDefinition.filter) encoder.setFilter(); perf.end('encode'); const stats: Stats = { structure: wrappedStructure, queryTimeMs: perf.time('query'), - encodeTimeMs: perf.time('encode') + encodeTimeMs: perf.time('encode'), + resultSize: result.reduce((n, s) => n + s.elementCount, 0) }; encoder.writeCategory(_model_server_stats, stats); @@ -151,7 +154,8 @@ const _model_server_stats_fields: CifField<number, Stats>[] = [ // int32<Stats>('attach_props_time_ms', ctx => ctx.structure.info.attachPropsTime | 0), int32<Stats>('create_model_time_ms', ctx => ctx.structure.info.createModelTime | 0), int32<Stats>('query_time_ms', ctx => ctx.queryTimeMs | 0), - int32<Stats>('encode_time_ms', ctx => ctx.encodeTimeMs | 0) + int32<Stats>('encode_time_ms', ctx => ctx.encodeTimeMs | 0), + int32<Stats>('element_count', ctx => ctx.resultSize | 0), ]; const _model_server_result: CifWriter.Category<Job> = { diff --git a/src/servers/model/version.ts b/src/servers/model/version.ts index 2ee344558..c90f8e749 100644 --- a/src/servers/model/version.ts +++ b/src/servers/model/version.ts @@ -1,7 +1,7 @@ /** - * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info. + * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info. * * @author David Sehnal <david.sehnal@gmail.com> */ -export default '0.8.0'; \ No newline at end of file +export default '0.9.0'; \ No newline at end of file -- GitLab