Skip to content
Snippets Groups Projects
Commit 76503b52 authored by David Sehnal's avatar David Sehnal
Browse files

wip model-server

parent aa24be8e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 225 additions and 111 deletions
src/apps/model-server-query/index.tsx
+ 1
− 1
View file @ 76503b52
......@@ -112,7 +112,7 @@ const state: State = {
function formatParams(def: QueryDefinition) {
const prms = Object.create(null);
for (const p of def.params) {
for (const p of def.jsonParams) {
prms[p.name] = p.exampleValues ? p.exampleValues[0] : void 0;
}
return JSON.stringify(prms, void 0, 2);
......
src/servers/model/config.ts
+ 8
− 10
View file @ 76503b52
......@@ -4,8 +4,6 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
import { ModelPropertyProviderConfig } from './property-provider';
const config = {
/**
* Determine if and how long to cache entries after a request.
......@@ -52,11 +50,11 @@ const config = {
/**
* Provide a property config or a path a JSON file with the config.
*/
customProperties: <ModelPropertyProviderConfig | string>{
customProperties: <import('./property-provider').ModelPropertyProviderConfig | string>{
sources: [
'pdbe',
'rcsb',
'wwpdb'
// 'pdbe',
// 'rcsb',
// 'wwpdb'
],
params: {
PDBe: {
......@@ -91,10 +89,10 @@ const config = {
*/
mapFile(source: string, id: string) {
switch (source.toLowerCase()) {
// case 'pdb': return `e:/test/quick/${id}_updated.cif`;
case 'pdb': return `e:/test/mol-star/model/out/${id}_updated.bcif`;
case 'pdb-bcif': return `c:/test/mol-star/model/out/${id}_updated.bcif`;
case 'pdb-cif': return `c:/test/mol-star/model/out/${id}_updated.cif`;
case 'pdb': return `e:/test/quick/${id}_updated.cif`;
// case 'pdb': return `e:/test/mol-star/model/out/${id}_updated.bcif`;
// case 'pdb-bcif': return `c:/test/mol-star/model/out/${id}_updated.bcif`;
// case 'pdb-cif': return `c:/test/mol-star/model/out/${id}_updated.cif`;
default: return void 0;
}
}
......
src/servers/model/query/atoms.ts
+ 4
− 3
View file @ 76503b52
......@@ -7,6 +7,7 @@
import { QueryPredicate, StructureElement, StructureProperties as Props } from '../../../mol-model/structure';
import { AtomsQueryParams } from '../../../mol-model/structure/query/queries/generators';
import { AtomSiteSchema, AtomSiteSchemaElement } from '../server/api';
import { ElementSymbol } from '../../../mol-model/structure/model/types';
export function getAtomsTests(params: AtomSiteSchema): Partial<AtomsQueryParams>[] {
if (!params) return [{ }];
......@@ -86,17 +87,17 @@ function atomTest(params: AtomSiteSchemaElement): QueryPredicate | undefined {
if (typeof params.label_atom_id !== 'undefined') {
props.push(Props.atom.label_atom_id);
values.push(+params.label_atom_id);
values.push(params.label_atom_id);
}
if (typeof params.auth_atom_id !== 'undefined') {
props.push(Props.atom.auth_atom_id);
values.push(+params.auth_atom_id);
values.push(params.auth_atom_id);
}
if (typeof params.type_symbol !== 'undefined') {
props.push(Props.atom.type_symbol);
values.push(+params.type_symbol);
values.push(ElementSymbol(params.type_symbol));
}
return andEqual(props, values);
......
src/servers/model/query/schemas.ts
+ 50
− 1
View file @ 76503b52
// TODO
\ No newline at end of file
/**
* Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author David Sehnal <david.sehnal@gmail.com>
*/
import { CifWriter } from '../../../mol-io/writer/cif';
const InteractionCategories = new Set([
'entry',
'entity',
'exptl',
'cell',
'symmetry',
'struct_conf',
'struct_sheet_range',
'entity_poly',
'struct_asym',
'struct_conn',
'struct_conn_type',
'pdbx_struct_mod_residue',
'chem_comp_bond',
'atom_sites'
]);
const AssemblyCategories = new Set([
'entry',
'entity',
'exptl',
'cell',
'symmetry',
'struct_conf',
'struct_sheet_range',
'entity_poly',
'entity_poly_seq',
'pdbx_nonpoly_scheme',
'struct_asym',
'struct_conn',
'struct_conn_type',
'pdbx_struct_mod_residue',
'chem_comp_bond',
'atom_sites'
]);
export const QuerySchemas = {
interaction: <CifWriter.Category.Filter>{
includeCategory(name) { return InteractionCategories.has(name); },
includeField(cat, field) { return true; }
}
}
\ No newline at end of file
src/servers/model/server/api-web.ts
+ 39
− 36
View file @ 76503b52
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author David Sehnal <david.sehnal@gmail.com>
*/
......@@ -13,6 +13,7 @@ import { resolveJob } from './query';
import { JobManager } from './jobs';
import { UUID } from '../../../mol-util';
import { LandingPage } from './landing';
import { QueryDefinition, normalizeRestQueryParams, normalizeRestCommonParams, QueryList } from './api';
function makePath(p: string) {
return Config.appPrefix + '/' + p;
......@@ -83,21 +84,23 @@ async function processNextJob() {
}
}
// function mapQuery(app: express.Express, queryName: string, queryDefinition: QueryDefinition) {
// app.get(makePath(':entryId/' + queryName), (req, res) => {
// ConsoleLogger.log('Server', `Query '${req.params.entryId}/${queryName}'...`);
// if (JobManager.size >= Config.maxQueueLength) {
// res.status(503).send('Too many queries, please try again later.');
// res.end();
// return;
// }
// const jobId = JobManager.add('pdb', req.params.entryId, queryName, req.query);
// responseMap.set(jobId, res);
// if (JobManager.size === 1) processNextJob();
// });
// }
function mapQuery(app: express.Express, queryName: string, queryDefinition: QueryDefinition) {
app.get(makePath('api/v1/:id/' + queryName), (req, res) => {
console.log({ queryName, params: req.params, query: req.query });
const entryId = req.params.id;
const queryParams = normalizeRestQueryParams(queryDefinition, req.query);
const commonParams = normalizeRestCommonParams(req.query);
const jobId = JobManager.add({
sourceId: commonParams.data_source || 'pdb',
entryId,
queryName: queryName as any,
queryParams,
options: { modelNums: commonParams.model_nums, binary: commonParams.encoding === 'bcif' }
});
responseMap.set(jobId, res);
if (JobManager.size === 1) processNextJob();
});
}
export function initWebApi(app: express.Express) {
app.get(makePath('static/:format/:id'), async (req, res) => {
......@@ -128,28 +131,28 @@ export function initWebApi(app: express.Express) {
});
})
app.get(makePath('api/v1'), (req, res) => {
const query = /\?(.*)$/.exec(req.url)![1];
const args = JSON.parse(decodeURIComponent(query));
const name = args.name;
const entryId = args.id;
const queryParams = args.params || { };
const jobId = JobManager.add({
sourceId: 'pdb',
entryId,
queryName: name,
queryParams,
options: { modelNums: args.modelNums, binary: args.binary }
});
responseMap.set(jobId, res);
if (JobManager.size === 1) processNextJob();
});
// app.get(makePath('api/v1/json'), (req, res) => {
// const query = /\?(.*)$/.exec(req.url)![1];
// const args = JSON.parse(decodeURIComponent(query));
// const name = args.name;
// const entryId = args.id;
// const queryParams = args.params || { };
// const jobId = JobManager.add({
// sourceId: 'pdb',
// entryId,
// queryName: name,
// queryParams,
// options: { modelNums: args.modelNums, binary: args.binary }
// });
// responseMap.set(jobId, res);
// if (JobManager.size === 1) processNextJob();
// });
for (const q of QueryList) {
mapQuery(app, q.name, q.definition);
}
app.get('*', (req, res) => {
res.send(LandingPage);
});
// for (const q of QueryList) {
// mapQuery(app, q.name, q.definition);
// }
}
\ No newline at end of file
src/servers/model/server/api.ts
+ 106
− 47
View file @ 76503b52
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author David Sehnal <david.sehnal@gmail.com>
*/
import { Queries, Structure, StructureQuery, StructureSymmetry } from '../../../mol-model/structure';
import { getAtomsTests } from '../query/atoms';
import { CifWriter } from '../../../mol-io/writer/cif';
import { QuerySchemas } from '../query/schemas';
export enum QueryParamType {
JSON,
String,
Integer,
Float
Float,
Boolean
}
export interface QueryParamInfo {
export interface QueryParamInfo<T extends string | number = string | number> {
name: string,
type: QueryParamType,
description?: string,
required?: boolean,
defaultValue?: any,
exampleValues?: any[],
validation?: (v: any) => void
validation?: (v: T) => void,
supportedValues?: string[]
}
export interface QueryDefinition<Params = any> {
......@@ -30,11 +34,42 @@ export interface QueryDefinition<Params = any> {
exampleId: string, // default is 1cbs
query: (params: any, structure: Structure) => StructureQuery,
description: string,
params: QueryParamInfo[],
jsonParams: QueryParamInfo[],
restParams: QueryParamInfo[],
structureTransform?: (params: any, s: Structure) => Promise<Structure>,
filter?: CifWriter.Category.Filter,
'@params': Params
}
export const CommonQueryParamsInfo: QueryParamInfo[] = [
{ name: 'model_nums', type: QueryParamType.String, description: `A comma-separated list of model ids (i.e. 1,2). If set, only include atoms with the corresponding '_atom_site.pdbx_PDB_model_num' field.` },
{ name: 'encoding', type: QueryParamType.String, defaultValue: 'cif', description: `Determines the output encoding (text based 'CIF' or binary 'BCIF').`, supportedValues: ['cif', 'bcif'] },
{ name: 'data_Source', type: QueryParamType.String, defaultValue: '', description: 'Allows to control how the provided data source ID maps to input file (as specified by the server instance config).' }
];
export interface CommonQueryParamsInfo {
model_nums?: number[],
encoding?: 'cif' | 'bcif',
data_source?: string
}
export const AtomSiteSchemaElement = {
label_entity_id: { type: QueryParamType.String },
label_asym_id: { type: QueryParamType.String },
auth_asym_id: { type: QueryParamType.String },
label_comp_id: { type: QueryParamType.String },
auth_comp_id: { type: QueryParamType.String },
label_seq_id: { type: QueryParamType.Integer },
auth_seq_id: { type: QueryParamType.Integer },
pdbx_PDB_ins_code: { type: QueryParamType.String },
label_atom_id: { type: QueryParamType.String },
auth_atom_id: { type: QueryParamType.String },
type_symbol: { type: QueryParamType.String }
}
export interface AtomSiteSchemaElement {
label_entity_id?: string,
......@@ -43,8 +78,8 @@ export interface AtomSiteSchemaElement {
label_comp_id?: string,
auth_comp_id?: string,
label_seq_id?: string,
auth_seq_id?: string,
label_seq_id?: number,
auth_seq_id?: number,
pdbx_PDB_ins_code?: string,
label_atom_id?: string,
......@@ -54,13 +89,23 @@ export interface AtomSiteSchemaElement {
export type AtomSiteSchema = AtomSiteSchemaElement | AtomSiteSchemaElement[]
const AtomSiteTestParams: QueryParamInfo = {
const AtomSiteTestJsonParam: QueryParamInfo = {
name: 'atom_site',
type: QueryParamType.JSON,
description: 'Object or array of objects describing atom properties. Names are same as in wwPDB mmCIF dictionary of the atom_site category.',
exampleValues: [{ label_comp_id: 'ALA' }, { label_seq_id: 123, label_asym_id: 'A' }]
};
export const AtomSiteTestRestParams = (function() {
const params: QueryParamInfo[] = [];
for (const k of Object.keys(AtomSiteSchemaElement)) {
const p = (AtomSiteSchemaElement as any)[k] as QueryParamInfo;
p.name = k;
params.push(p);
}
return params;
})();
const RadiusParam: QueryParamInfo = {
name: 'radius',
type: QueryParamType.Float,
......@@ -83,8 +128,9 @@ const QueryMap = {
'atoms': Q<{ atom_site: AtomSiteSchema }>({
niceName: 'Atoms',
description: 'Atoms satisfying the given criteria.',
query: p => Queries.combinators.merge(getAtomsTests(p.atom_site).map(test => Queries.generators.atoms(test))),
params: [ AtomSiteTestParams ]
query: p => Queries.combinators.merge(getAtomsTests(p).map(test => Queries.generators.atoms(test))),
jsonParams: [ AtomSiteTestJsonParam ],
restParams: AtomSiteTestRestParams
}),
'symmetryMates': Q<{ radius: number }>({
niceName: 'Symmetry Mates',
......@@ -93,7 +139,7 @@ const QueryMap = {
structureTransform(p, s) {
return StructureSymmetry.builderSymmetryMates(s, p.radius).run();
},
params: [ RadiusParam ]
jsonParams: [ RadiusParam ]
}),
'assembly': Q<{ name: string }>({
niceName: 'Assembly',
......@@ -102,7 +148,7 @@ const QueryMap = {
structureTransform(p, s) {
return StructureSymmetry.buildAssembly(s, '' + (p.name || '1')).run();
},
params: [{
jsonParams: [{
name: 'name',
type: QueryParamType.String,
defaultValue: '1',
......@@ -110,11 +156,11 @@ const QueryMap = {
description: 'Assembly name.'
}]
}),
'residueInteraction': Q<{ atom_site: AtomSiteSchema, radius: number }>({
'residueInteraction': Q<AtomSiteSchema & { radius: number }>({
niceName: 'Residue Interaction',
description: 'Identifies all residues within the given radius from the source residue. Takes crystal symmetry into account.',
query(p) {
const tests = getAtomsTests(p.atom_site);
const tests = getAtomsTests(p);
const center = Queries.combinators.merge(tests.map(test => Queries.generators.atoms({
...test,
entityTest: test.entityTest
......@@ -126,17 +172,21 @@ const QueryMap = {
structureTransform(p, s) {
return StructureSymmetry.builderSymmetryMates(s, p.radius).run();
},
params: [ AtomSiteTestParams, RadiusParam ]
jsonParams: [ AtomSiteTestJsonParam, RadiusParam ],
restParams: [ ...AtomSiteTestRestParams, RadiusParam ],
filter: QuerySchemas.interaction
}),
'residueSurroundings': Q<{ atom_site: AtomSiteSchema, radius: number }>({
'residueSurroundings': Q<AtomSiteSchema & { radius: number }>({
niceName: 'Residue Surroundings',
description: 'Identifies all residues within the given radius from the source residue.',
query(p) {
const tests = getAtomsTests(p.atom_site);
const tests = getAtomsTests(p);
const center = Queries.combinators.merge(tests.map(test => Queries.generators.atoms(test)));
return Queries.modifiers.includeSurroundings(center, { radius: p.radius, wholeResidues: true });
},
params: [ AtomSiteTestParams, RadiusParam ]
jsonParams: [ AtomSiteTestJsonParam, RadiusParam ],
restParams: [ ...AtomSiteTestRestParams, RadiusParam ],
filter: QuerySchemas.interaction
})
};
......@@ -159,36 +209,45 @@ export const QueryList = (function () {
for (let q of QueryList) {
const m = q.definition;
m.name = q.name;
m.params = m.params || [];
m.jsonParams = m.jsonParams || [];
m.restParams = m.restParams || m.jsonParams;
}
})();
// function _normalizeQueryParams(params: { [p: string]: string }, paramList: QueryParamInfo[]): { [p: string]: string | number | boolean } {
// const ret: any = {};
// for (const p of paramList) {
// const key = p.name;
// const value = params[key];
// if (typeof value === 'undefined' || (typeof value !== 'undefined' && value !== null && value['length'] === 0)) {
// if (p.required) {
// throw `The parameter '${key}' is required.`;
// }
// if (typeof p.defaultValue !== 'undefined') ret[key] = p.defaultValue;
// } else {
// switch (p.type) {
// case QueryParamType.JSON: ret[key] = JSON.parse(value); break;
// case QueryParamType.String: ret[key] = value; break;
// case QueryParamType.Integer: ret[key] = parseInt(value); break;
// case QueryParamType.Float: ret[key] = parseFloat(value); break;
// }
// if (p.validation) p.validation(ret[key]);
// }
// }
// return ret;
// }
export function normalizeQueryParams(query: QueryDefinition, params: any) {
return params;
// return _normalizeQueryParams(params, query.params);
function _normalizeQueryParams(params: { [p: string]: string }, paramList: QueryParamInfo[]): { [p: string]: string | number | boolean } {
const ret: any = {};
for (const p of paramList) {
const key = p.name;
const value = params[key];
if (typeof value === 'undefined' || (typeof value !== 'undefined' && value !== null && value['length'] === 0)) {
if (p.required) {
throw `The parameter '${key}' is required.`;
}
if (typeof p.defaultValue !== 'undefined') ret[key] = p.defaultValue;
} else {
switch (p.type) {
case QueryParamType.JSON: ret[key] = JSON.parse(value); break;
case QueryParamType.String: ret[key] = value; break;
case QueryParamType.Integer: ret[key] = parseInt(value); break;
case QueryParamType.Float: ret[key] = parseFloat(value); break;
}
if (p.validation) p.validation(ret[key]);
}
}
return ret;
}
export function normalizeRestQueryParams(query: QueryDefinition, params: any) {
// return params;
return _normalizeQueryParams(params, query.restParams);
}
export function normalizeRestCommonParams(params: any): CommonQueryParamsInfo {
return {
model_nums: params.model_nums ? ('' + params.model_nums).split(',').map(n => n.trim()).filter(n => !!n).map(n => +n) : void 0,
data_source: params.data_source,
encoding: ('' + params.encoding).toLocaleLowerCase() === 'bcif' ? 'bcif' : 'cif'
};
}
\ No newline at end of file
src/servers/model/server/jobs.ts
+ 3
− 3
View file @ 76503b52
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author David Sehnal <david.sehnal@gmail.com>
*/
import { UUID } from '../../../mol-util';
import { getQueryByName, normalizeQueryParams, QueryDefinition, QueryName, QueryParams } from './api';
import { getQueryByName, QueryDefinition, QueryName, QueryParams } from './api';
import { LinkedList } from '../../../mol-data/generic';
export interface ResponseFormat {
......@@ -40,7 +40,7 @@ export function createJob<Name extends QueryName>(definition: JobDefinition<Name
const queryDefinition = getQueryByName(definition.queryName);
if (!queryDefinition) throw new Error(`Query '${definition.queryName}' is not supported.`);
const normalizedParams = normalizeQueryParams(queryDefinition, definition.queryParams);
const normalizedParams = definition.queryParams;
const sourceId = definition.sourceId || '_local_';
return {
id: UUID.create22(),
......
src/servers/model/server/query.ts
+ 12
− 8
View file @ 76503b52
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author David Sehnal <david.sehnal@gmail.com>
*/
......@@ -22,7 +22,8 @@ import { createModelPropertiesProviderFromConfig, ModelPropertiesProvider } from
export interface Stats {
structure: StructureWrapper,
queryTimeMs: number,
encodeTimeMs: number
encodeTimeMs: number,
resultSize: number
}
const perf = new PerformanceMonitor();
......@@ -56,7 +57,8 @@ export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> {
const queries = structures.map(s => job.queryDefinition.query(job.normalizedParams, s));
const result: Structure[] = [];
for (let i = 0; i < structures.length; i++) {
result.push(await StructureSelection.unionStructure(StructureQuery.run(queries[i], structures[i], Config.maxQueryTimeInMs)));
const s = await StructureSelection.unionStructure(StructureQuery.run(queries[i], structures[i], Config.maxQueryTimeInMs))
if (s.elementCount > 0) result.push(s);
}
perf.end('query');
......@@ -74,15 +76,16 @@ export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> {
encoder.writeCategory(_model_server_result, job);
encoder.writeCategory(_model_server_params, job);
// encoder.setFilter(mmCIF_Export_Filters.onlyPositions);
encode_mmCIF_categories(encoder, result);
// encoder.setFilter();
if (job.queryDefinition.filter) encoder.setFilter(job.queryDefinition.filter);
if (result.length > 0) encode_mmCIF_categories(encoder, result);
if (job.queryDefinition.filter) encoder.setFilter();
perf.end('encode');
const stats: Stats = {
structure: wrappedStructure,
queryTimeMs: perf.time('query'),
encodeTimeMs: perf.time('encode')
encodeTimeMs: perf.time('encode'),
resultSize: result.reduce((n, s) => n + s.elementCount, 0)
};
encoder.writeCategory(_model_server_stats, stats);
......@@ -151,7 +154,8 @@ const _model_server_stats_fields: CifField<number, Stats>[] = [
// int32<Stats>('attach_props_time_ms', ctx => ctx.structure.info.attachPropsTime | 0),
int32<Stats>('create_model_time_ms', ctx => ctx.structure.info.createModelTime | 0),
int32<Stats>('query_time_ms', ctx => ctx.queryTimeMs | 0),
int32<Stats>('encode_time_ms', ctx => ctx.encodeTimeMs | 0)
int32<Stats>('encode_time_ms', ctx => ctx.encodeTimeMs | 0),
int32<Stats>('element_count', ctx => ctx.resultSize | 0),
];
const _model_server_result: CifWriter.Category<Job> = {
......
src/servers/model/version.ts
+ 2
− 2
View file @ 76503b52
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author David Sehnal <david.sehnal@gmail.com>
*/
export default '0.8.0';
\ No newline at end of file
export default '0.9.0';
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment