diff --git a/src/mol-math/geometry/symmetry-operator.ts b/src/mol-math/geometry/symmetry-operator.ts
index 6c233e11d5b49f3d544893a42d8fad372486a181..86884a935653dd7c49a68632d93e885a93cd3829 100644
--- a/src/mol-math/geometry/symmetry-operator.ts
+++ b/src/mol-math/geometry/symmetry-operator.ts
@@ -30,6 +30,17 @@ namespace SymmetryOperator {
return { name, matrix, inverse: Mat4.invert(Mat4.zero(), matrix), isIdentity: false, hkl: _hkl };
}
+ export function checkIfRotationAndTranslation(rot: Mat3, offset: Vec3) {
+ const matrix = Mat4.identity();
+ for (let i = 0; i < 3; i++) {
+ for (let j = 0; j < 3; j++) {
+ Mat4.setValue(matrix, i, j, Mat3.getValue(rot, i, j));
+ }
+ }
+ Mat4.setTranslation(matrix, offset);
+ return Mat4.isRotationAndTranslation(matrix, RotationEpsilon);
+ }
+
export function ofRotationAndOffset(name: string, rot: Mat3, offset: Vec3) {
const t = Mat4.identity();
for (let i = 0; i < 3; i++) {
diff --git a/src/mol-model/structure/model/formats/mmcif.ts b/src/mol-model/structure/model/formats/mmcif.ts
index cb9ddfa8c7ec9ab019517bc8883c6f8d542a985e..b03ed96635abfcc1376fe4f17cd37b4dc453471e 100644
--- a/src/mol-model/structure/model/formats/mmcif.ts
+++ b/src/mol-model/structure/model/formats/mmcif.ts
@@ -66,7 +66,8 @@ function getNcsOperators(format: mmCIF_Format) {
for (let i = 0; i < struct_ncs_oper._rowCount; i++) {
const m = Tensor.toMat3(matrixSpace, matrix.value(i));
const v = Tensor.toVec3(vectorSpace, vector.value(i));
- opers[i] = SymmetryOperator.ofRotationAndOffset(`ncs_${id.value(i)}`, m, v);
+ if (!SymmetryOperator.checkIfRotationAndTranslation(m, v)) continue;
+ opers[opers.length] = SymmetryOperator.ofRotationAndOffset(`ncs_${id.value(i)}`, m, v);
}
return opers;
}
diff --git a/src/servers/model/preprocess.ts b/src/servers/model/preprocess.ts
index c61288185032aebc8351e243f5c69f922d459590..b88e33348a3f19b63a57c948fa9ded34ff03f32e 100644
--- a/src/servers/model/preprocess.ts
+++ b/src/servers/model/preprocess.ts
@@ -4,28 +4,14 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
-import * as argparse from 'argparse'
-import { preprocessFile } from './preprocess/preprocess';
+import * as cluster from 'cluster'
+import { runChild } from './preprocess/parallel';
-const cmdParser = new argparse.ArgumentParser({
- addHelp: true,
- description: 'Preprocess CIF files to include custom properties and convert them to BinaryCIF format.'
-});
-cmdParser.addArgument(['--input', '-i'], { help: 'Input filename', required: true });
-cmdParser.addArgument(['--outCIF', '-oc'], { help: 'Output CIF filename', required: false });
-cmdParser.addArgument(['--outBCIF', '-ob'], { help: 'Output BinaryCIF filename', required: false });
-
-// TODO: "bulk" mode
-
-interface CmdArgs {
- input: string,
- outCIF?: string,
- outBCIF?: string
+if (cluster.isMaster) {
+ require('./preprocess/master');
+} else {
+ runChild();
}
-const cmdArgs = cmdParser.parseArgs() as CmdArgs;
-
-if (cmdArgs.input) preprocessFile(cmdArgs.input, cmdArgs.outCIF, cmdArgs.outBCIF);
-
// example:
// node build\node_modules\servers\model\preprocess -i e:\test\Quick\1cbs_updated.cif -oc e:\test\mol-star\model\1cbs.cif -ob e:\test\mol-star\model\1cbs.bcif
\ No newline at end of file
diff --git a/src/servers/model/preprocess/converter.ts b/src/servers/model/preprocess/converter.ts
index bad7f7d865f61d04ff2069c0525de30f79716a32..66d77fcb46558b4225929d1f31babb90af1f1093 100644
--- a/src/servers/model/preprocess/converter.ts
+++ b/src/servers/model/preprocess/converter.ts
@@ -7,7 +7,7 @@
import { CifCategory, CifField, CifFrame, getCifFieldType } from 'mol-io/reader/cif';
import { CifWriter } from 'mol-io/writer/cif';
import { Task } from 'mol-task';
-import { showProgress } from './util';
+// import { showProgress } from './util';
function getCategoryInstanceProvider(cat: CifCategory, fields: CifWriter.Field[]): CifWriter.Category {
return {
@@ -47,5 +47,5 @@ export function classifyCif(frame: CifFrame) {
ret.push(getCategoryInstanceProvider(cat, fields));
}
return ret;
- }).run(showProgress, 250);
+ }).run();
}
\ No newline at end of file
diff --git a/src/servers/model/preprocess/master.ts b/src/servers/model/preprocess/master.ts
new file mode 100644
index 0000000000000000000000000000000000000000..b058ad56f7a9b60a0530797ac281776fb369f4db
--- /dev/null
+++ b/src/servers/model/preprocess/master.ts
@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as fs from 'fs'
+import * as path from 'path'
+import * as argparse from 'argparse'
+import { preprocessFile } from './preprocess';
+import { ParallelPreprocessConfig, runMaster } from './parallel';
+
+const cmdParser = new argparse.ArgumentParser({
+ addHelp: true,
+ description: 'Preprocess CIF files to include custom properties and convert them to BinaryCIF format.'
+});
+cmdParser.addArgument(['--input', '-i'], { help: 'Input filename', required: false });
+cmdParser.addArgument(['--outCIF', '-oc'], { help: 'Output CIF filename', required: false });
+cmdParser.addArgument(['--outBCIF', '-ob'], { help: 'Output BinaryCIF filename', required: false });
+cmdParser.addArgument(['--bulk', '-b'], { help: 'Bulk JSON ({ numProcesses?: number, entries: { source: string, cif?: string, bcif?: string }[] })', required: false });
+cmdParser.addArgument(['--folderIn', '-f'], { help: 'Convert folder', required: false });
+cmdParser.addArgument(['--folderOutCIF', '-foc'], { help: 'Convert folder text output', required: false });
+cmdParser.addArgument(['--folderOutBCIF', '-fob'], { help: 'Convert folder binary output', required: false });
+cmdParser.addArgument(['--folderNumProcesses', '-fp'], { help: 'Convert folder num processes', required: false });
+
+interface CmdArgs {
+ bulk?: string,
+ input?: string,
+ outCIF?: string,
+ outBCIF?: string,
+ folderIn?: string,
+ folderOutCIF?: string,
+ folderOutBCIF?: string,
+ folderNumProcesses?: string
+}
+
+const cmdArgs = cmdParser.parseArgs() as CmdArgs;
+
+if (cmdArgs.input) preprocessFile(cmdArgs.input, cmdArgs.outCIF, cmdArgs.outBCIF);
+else if (cmdArgs.bulk) runBulk(cmdArgs.bulk);
+else if (cmdArgs.folderIn) runFolder(cmdArgs);
+
+function runBulk(input: string) {
+ const config = JSON.parse(fs.readFileSync(input, 'utf8')) as ParallelPreprocessConfig;
+ runMaster(config);
+}
+
+function runFolder(args: CmdArgs) {
+ const files = fs.readdirSync(args.folderIn!);
+ const config: ParallelPreprocessConfig = { numProcesses: +args.folderNumProcesses! || 1, entries: [] };
+ const cifTest = /\.cif$/;
+ for (const f of files) {
+ if (!cifTest.test(f)) continue;
+
+ config.entries.push({
+ source: path.join(args.folderIn!, f),
+ cif: cmdArgs.folderOutCIF ? path.join(args.folderOutCIF!, f) : void 0,
+ bcif: cmdArgs.folderOutBCIF ? path.join(args.folderOutBCIF!, path.parse(f).name + '.bcif') : void 0,
+ });
+ }
+ runMaster(config);
+}
+
+// example:
+// node build\node_modules\servers\model\preprocess -i e:\test\Quick\1cbs_updated.cif -oc e:\test\mol-star\model\1cbs.cif -ob e:\test\mol-star\model\1cbs.bcif
\ No newline at end of file
diff --git a/src/servers/model/preprocess/parallel.ts b/src/servers/model/preprocess/parallel.ts
new file mode 100644
index 0000000000000000000000000000000000000000..83ece3ac4cb6f23b75bae64165a776239ffb04d1
--- /dev/null
+++ b/src/servers/model/preprocess/parallel.ts
@@ -0,0 +1,77 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as path from 'path'
+import * as cluster from 'cluster'
+import { now } from 'mol-task';
+import { PerformanceMonitor } from 'mol-util/performance-monitor';
+import { preprocessFile } from './preprocess';
+
+export interface PreprocessEntry {
+ source: string,
+ cif?: string,
+ bcif?: string
+}
+
+export interface ParallelPreprocessConfig {
+ numProcesses?: number,
+ entries: PreprocessEntry[]
+}
+
+export function runMaster(config: ParallelPreprocessConfig) {
+ const parts = partitionArray(config.entries, config.numProcesses || 1);
+ // const numForks = Math.min(parts.length, config.numProcesses);
+
+ const started = now();
+ let progress = 0;
+ const onMessage = (msg: any) => {
+ if (msg.type === 'tick') {
+ progress++;
+ const elapsed = now() - started;
+ console.log(`[${progress}/${config.entries.length}] in ${PerformanceMonitor.format(elapsed)} (avg ${PerformanceMonitor.format(elapsed / progress)}).`);
+ } else if (msg.type === 'error') {
+ console.error(`${msg.id}: ${msg.error}`)
+ }
+ }
+
+ for (const _ of parts) {
+ const worker = cluster.fork();
+ worker.on('message', onMessage);
+ }
+
+ let i = 0;
+ for (const id in cluster.workers) {
+ cluster.workers[id]!.send(parts[i++]);
+ }
+}
+
+export function runChild() {
+ process.on('message', async (entries: PreprocessEntry[]) => {
+ for (const entry of entries) {
+ try {
+ await preprocessFile(entry.source, entry.cif, entry.bcif);
+ } catch (e) {
+ process.send!({ type: 'error', id: path.parse(entry.source).name, error: '' + e });
+ }
+ process.send!({ type: 'tick' });
+ }
+ process.exit();
+ });
+}
+
+function partitionArray<T>(xs: T[], count: number): T[][] {
+ const ret: T[][] = [];
+ const s = Math.ceil(xs.length / count);
+ for (let i = 0; i < xs.length; i += s) {
+ const bucket: T[] = [];
+ for (let j = i, _j = Math.min(xs.length, i + s); j < _j; j++) {
+ bucket.push(xs[j]);
+ }
+ ret.push(bucket);
+ }
+ return ret;
+}
+
diff --git a/src/servers/model/preprocess/preprocess.ts b/src/servers/model/preprocess/preprocess.ts
index 394fe205ac81e4a8e2bbb908531dcf1e08c58a43..bc6b5abca1b3d748e6d2b06d0dcd0691d61ab225 100644
--- a/src/servers/model/preprocess/preprocess.ts
+++ b/src/servers/model/preprocess/preprocess.ts
@@ -6,58 +6,62 @@
import { readStructure } from '../server/structure-wrapper';
import { classifyCif } from './converter';
-import { ConsoleLogger } from 'mol-util/console-logger';
+// import { ConsoleLogger } from 'mol-util/console-logger';
import { Structure } from 'mol-model/structure';
import { CifWriter } from 'mol-io/writer/cif';
import Writer from 'mol-io/writer/writer';
import { wrapFileToWriter } from '../server/api-local';
-import { Task } from 'mol-task';
-import { showProgress, clearLine } from './util';
+import { Task/*, now*/ } from 'mol-task';
+import { /*showProgress, clearLine */ } from './util';
import { encode_mmCIF_categories, CifExportContext } from 'mol-model/structure/export/mmcif';
-// TODO: error handling, bulk mode
+// TODO: error handling
+// let linearId = 0;
export async function preprocessFile(filename: string, outputCif?: string, outputBcif?: string) {
- ConsoleLogger.log('ModelServer', `Reading ${filename}...`);
+ // linearId++;
+
+ //const started = now();
+ //ConsoleLogger.log(`${linearId}`, `Reading '${filename}'...`);
const input = await readStructure('entry', '_local_', filename);
- ConsoleLogger.log('ModelServer', `Classifying CIF categories...`);
+ //ConsoleLogger.log(`${linearId}`, `Classifying CIF categories...`);
const categories = await classifyCif(input.cifFrame);
- clearLine();
+ //clearLine();
const exportCtx = CifExportContext.create(input.structure, input.structure.models[0]);
if (outputCif) {
- ConsoleLogger.log('ModelServer', `Encoding CIF...`);
+ //ConsoleLogger.log(`${linearId}`, `Encoding CIF...`);
const writer = wrapFileToWriter(outputCif);
const encoder = CifWriter.createEncoder({ binary: false });
await encode(input.structure, input.cifFrame.header, categories, encoder, exportCtx, writer);
- clearLine();
+ // clearLine();
writer.end();
}
if (outputBcif) {
- ConsoleLogger.log('ModelServer', `Encoding BinaryCIF...`);
+ // ConsoleLogger.log(`${linearId}`, `Encoding BinaryCIF...`);
const writer = wrapFileToWriter(outputBcif);
const encoder = CifWriter.createEncoder({ binary: true, binaryAutoClassifyEncoding: true });
await encode(input.structure, input.cifFrame.header, categories, encoder, exportCtx, writer);
- clearLine();
+ //clearLine();
writer.end();
}
- ConsoleLogger.log('ModelServer', `Done.`);
+ // ConsoleLogger.log(`${linearId}`, `Finished '${filename}' in ${Math.round(now() - started)}ms`);
}
function encode(structure: Structure, header: string, categories: CifWriter.Category[], encoder: CifWriter.Encoder, exportCtx: CifExportContext, writer: Writer) {
return Task.create('Encode', async ctx => {
const skipCategoryNames = new Set<string>(categories.map(c => c.name));
encoder.startDataBlock(header);
- let current = 0;
+ // let current = 0;
for (const cat of categories){
encoder.writeCategory(cat);
- current++;
- if (ctx.shouldUpdate) await ctx.update({ message: 'Encoding...', current, max: categories.length });
+ // current++;
+ // if (ctx.shouldUpdate) await ctx.update({ message: 'Encoding...', current, max: categories.length });
}
encode_mmCIF_categories(encoder, structure, { skipCategoryNames, exportCtx });
encoder.encode();
encoder.writeTo(writer);
- }).run(showProgress, 250);
+ }).run();
}
\ No newline at end of file
diff --git a/src/servers/model/properties.ts b/src/servers/model/properties.ts
index fddf09e61fceea5f1f97ee5bc320eebac31ff351..5f292223862c4d3266daaf8e53333e8911241811 100644
--- a/src/servers/model/properties.ts
+++ b/src/servers/model/properties.ts
@@ -6,14 +6,14 @@
*/
import { Model } from 'mol-model/structure';
-import { PDBe_structureQualityReport } from './properties/pdbe';
-import { RCSB_assemblySymmetry } from './properties/rcsb';
+//import { PDBe_structureQualityReport } from './properties/pdbe';
+//import { RCSB_assemblySymmetry } from './properties/rcsb';
export function attachModelProperties(model: Model): Promise<any>[] {
// return a list of promises that start attaching the props in parallel
// (if there are downloads etc.)
return [
- PDBe_structureQualityReport(model),
- RCSB_assemblySymmetry(model)
+ //PDBe_structureQualityReport(model),
+ //RCSB_assemblySymmetry(model)
];
}
\ No newline at end of file