From b73c3b6b7b978a2ed0c0097de9962bb903d0438e Mon Sep 17 00:00:00 2001
From: Alexander Rose <alex.rose@rcsb.org>
Date: Wed, 20 Feb 2019 16:40:14 -0800
Subject: [PATCH] extended FileHandle API and use SimpleBuffer
---
src/mol-io/common/file-handle.ts | 113 ++++++++++++++++++++++----
src/mol-io/reader/ccp4/parser.ts | 134 ++++++++++++++++++++-----------
src/mol-io/reader/ccp4/schema.ts | 3 +
src/mol-io/reader/dsn6/parser.ts | 19 +++--
webpack.config.js | 1 +
5 files changed, 198 insertions(+), 72 deletions(-)
diff --git a/src/mol-io/common/file-handle.ts b/src/mol-io/common/file-handle.ts
index b3ae0fd64..fbf3a36b9 100644
--- a/src/mol-io/common/file-handle.ts
+++ b/src/mol-io/common/file-handle.ts
@@ -1,35 +1,56 @@
/**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
+ * @author David Sehnal <david.sehnal@gmail.com>
*/
-import { defaults } from 'mol-util';
+import { defaults, noop } from 'mol-util';
+import { SimpleBuffer } from './simple-buffer';
+// only import 'fs' in node.js
+const fs = typeof document === 'undefined' ? require('fs') as typeof import('fs') : void 0;
export interface FileHandle {
- /** The number of bytes in the file */
- length: number
/**
+ * Asynchronously reads data, returning buffer and number of bytes read
+ *
* @param position The offset from the beginning of the file from which data should be read.
- * @param sizeOrBuffer The buffer the data will be written to. If a number a buffer of that size will be created.
- * @param size The number of bytes to read.
+ * @param sizeOrBuffer The buffer the data will be read from.
+ * @param length The number of bytes to read.
* @param byteOffset The offset in the buffer at which to start writing.
*/
- readBuffer(position: number, sizeOrBuffer: Uint8Array | number, size?: number, byteOffset?: number): Promise<{ bytesRead: number, buffer: Uint8Array }>
+ readBuffer(position: number, sizeOrBuffer: SimpleBuffer | number, length?: number, byteOffset?: number): Promise<{ bytesRead: number, buffer: SimpleBuffer }>
+
+ /**
+ * Asynchronously writes buffer, returning the number of bytes written.
+ *
+ * @param position — The offset from the beginning of the file where this data should be written.
+ * @param buffer - The buffer data to be written.
+ * @param length — The number of bytes to write. If not supplied, defaults to buffer.length - offset.
+ */
+ writeBuffer(position: number, buffer: SimpleBuffer, length?: number): Promise<number>
+
+ /**
+ * Synchronously writes buffer, returning the number of bytes written.
+ *
+ * @param position — The offset from the beginning of the file where this data should be written.
+ * @param buffer - The buffer data to be written.
+ * @param length — The number of bytes to write. If not supplied, defaults to buffer.length - offset.
+ */
+ writeBufferSync(position: number, buffer: SimpleBuffer, length?: number): number
+
+ /** Closes a file handle */
+ close(): void
}
export namespace FileHandle {
- export function fromBuffer(buffer: Uint8Array): FileHandle {
+ export function fromBuffer(buffer: SimpleBuffer): FileHandle {
return {
- length: buffer.length,
- readBuffer: (position: number, sizeOrBuffer: Uint8Array | number, size?: number, byteOffset?: number) => {
+ readBuffer: (position: number, sizeOrBuffer: SimpleBuffer | number, size?: number, byteOffset?: number) => {
if (typeof sizeOrBuffer === 'number') {
const start = position
const end = Math.min(buffer.length, start + (defaults(size, sizeOrBuffer)))
- return Promise.resolve({
- bytesRead: end - start,
- buffer: buffer.subarray(start, end),
- })
+ return Promise.resolve({ bytesRead: end - start, buffer: SimpleBuffer.fromUint8Array(buffer.subarray(start, end)) })
} else {
if (size === void 0) {
return Promise.reject('readBuffer: Specify size.');
@@ -37,10 +58,68 @@ export namespace FileHandle {
const start = position
const end = Math.min(buffer.length, start + defaults(size, sizeOrBuffer.length))
sizeOrBuffer.set(buffer.subarray(start, end), byteOffset)
- return Promise.resolve({
- bytesRead: end - start,
- buffer: sizeOrBuffer,
+ return Promise.resolve({ bytesRead: end - start, buffer: sizeOrBuffer })
+ }
+ },
+ writeBuffer: (position: number, buffer: SimpleBuffer, length?: number) => {
+ length = defaults(length, buffer.length)
+ console.warn('FileHandle.writeBuffer not implemented')
+ return Promise.resolve(0)
+ },
+ writeBufferSync: (position: number, buffer: SimpleBuffer, length?: number, ) => {
+ length = defaults(length, buffer.length)
+ console.warn('FileHandle.writeSync not implemented')
+ return 0
+ },
+ close: noop
+ }
+ }
+
+ export function fromDescriptor(file: number): FileHandle {
+ if (fs === undefined) throw new Error('fs module not available')
+ return {
+ readBuffer: (position: number, sizeOrBuffer: SimpleBuffer | number, length?: number, byteOffset?: number) => {
+ return new Promise((res, rej) => {
+ if (typeof sizeOrBuffer === 'number') {
+ let buff = new Buffer(new ArrayBuffer(sizeOrBuffer));
+ fs.read(file, buff, 0, sizeOrBuffer, position, (err, bytesRead, buffer) => {
+ if (err) {
+ rej(err);
+ return;
+ }
+ res({ bytesRead, buffer });
+ });
+ } else {
+ if (length === void 0) {
+ rej('readBuffer: Specify size.');
+ return;
+ }
+ fs.read(file, sizeOrBuffer, byteOffset ? +byteOffset : 0, length, position, (err, bytesRead, buffer) => {
+ if (err) {
+ rej(err);
+ return;
+ }
+ res({ bytesRead, buffer });
+ });
+ }
+ })
+ },
+ writeBuffer: (position: number, buffer: Buffer, length?: number) => {
+ return new Promise<number>((res, rej) => {
+ fs.write(file, buffer, 0, length !== void 0 ? length : buffer.length, position, (err, written) => {
+ if (err) rej(err);
+ else res(written);
})
+ })
+ },
+ writeBufferSync: (position: number, buffer: Uint8Array, length?: number) => {
+ return fs.writeSync(file, buffer, 0, length, position);
+ },
+ close: () => {
+ try {
+ if (file !== void 0) fs.close(file, noop);
+ } catch (e) {
+
}
}
}
diff --git a/src/mol-io/reader/ccp4/parser.ts b/src/mol-io/reader/ccp4/parser.ts
index d48c343e5..388c22a87 100644
--- a/src/mol-io/reader/ccp4/parser.ts
+++ b/src/mol-io/reader/ccp4/parser.ts
@@ -8,15 +8,12 @@ import { Task, RuntimeContext } from 'mol-task';
import { Ccp4File, Ccp4Header } from './schema'
import { ReaderResult as Result } from '../result'
import { FileHandle } from '../../common/file-handle';
+import { SimpleBuffer } from 'mol-io/common/simple-buffer';
-async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Result<Ccp4File>> {
- await ctx.update({ message: 'Parsing CCP4/MRC file...' });
-
- const { buffer } = await file.readBuffer(0, file.length)
+export async function readCcp4Header(file: FileHandle) {
+ const headerSize = 1024;
+ const { buffer } = await file.readBuffer(0, headerSize)
const bin = buffer.buffer
-
- const intView = new Int32Array(bin, 0, 56)
- const floatView = new Float32Array(bin, 0, 56)
const dv = new DataView(bin)
// 53 MAP Character string 'MAP ' to identify file type
@@ -25,88 +22,121 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
dv.getUint8(52 * 4 + 2), dv.getUint8(52 * 4 + 3)
)
if (MAP !== 'MAP ') {
- return Result.error('ccp4 format error, missing "MAP " string');
+ throw new Error('ccp4 format error, missing "MAP " string');
}
// 54 MACHST Machine stamp indicating machine type which wrote file
// 17 and 17 for big-endian or 68 and 65 for little-endian
const MACHST = [ dv.getUint8(53 * 4), dv.getUint8(53 * 4 + 1) ]
+ let littleEndian = true
// found MRC files that don't have the MACHST stamp set and are big-endian
if (MACHST[0] !== 68 && MACHST[1] !== 65) {
- // flip byte order in-place
- for (let i = 0, il = bin.byteLength; i < il; i += 4) {
- dv.setFloat32(i, dv.getFloat32(i), true)
- }
+ littleEndian = false;
}
+ const readInt = (o: number) => dv.getInt32(o * 4, littleEndian)
+ const readFloat = (o: number) => dv.getFloat32(o * 4, littleEndian)
+
const header: Ccp4Header = {
- NC: intView[0],
- NR: intView[1],
- NS: intView[2],
+ NC: readInt(0),
+ NR: readInt(1),
+ NS: readInt(2),
- MODE: intView[3],
+ MODE: readInt(3),
- NCSTART: intView[4],
- NRSTART: intView[5],
- NSSTART: intView[6],
+ NCSTART: readInt(4),
+ NRSTART: readInt(5),
+ NSSTART: readInt(6),
- NX: intView[7],
- NY: intView[8],
- NZ: intView[9],
+ NX: readInt(7),
+ NY: readInt(8),
+ NZ: readInt(9),
- xLength: floatView[10],
- yLength: floatView[11],
- zLength: floatView[12],
+ xLength: readFloat(10),
+ yLength: readFloat(11),
+ zLength: readFloat(12),
- alpha: floatView[13],
- beta: floatView[14],
- gamma: floatView[15],
+ alpha: readFloat(13),
+ beta: readFloat(14),
+ gamma: readFloat(15),
- MAPC: intView[16],
- MAPR: intView[17],
- MAPS: intView[18],
+ MAPC: readInt(16),
+ MAPR: readInt(17),
+ MAPS: readInt(18),
- AMIN: floatView[19],
- AMAX: floatView[20],
- AMEAN: floatView[21],
+ AMIN: readFloat(19),
+ AMAX: readFloat(20),
+ AMEAN: readFloat(21),
- ISPG: intView[22],
+ ISPG: readInt(22),
- NSYMBT: intView[23],
+ NSYMBT: readInt(23),
- LSKFLG: intView[24],
+ LSKFLG: readInt(24),
SKWMAT: [], // TODO bytes 26-34
SKWTRN: [], // TODO bytes 35-37
+ userFlag1: readInt(39),
+ userFlag2: readInt(40),
+
// bytes 50-52 origin in X,Y,Z used for transforms
- originX: floatView[49],
- originY: floatView[50],
- originZ: floatView[51],
+ originX: readFloat(49),
+ originY: readFloat(50),
+ originZ: readFloat(51),
MAP, // bytes 53 MAP
MACHST, // bytes 54 MACHST
- ARMS: floatView[54],
+ ARMS: readFloat(54),
// TODO bytes 56 NLABL
// TODO bytes 57-256 LABEL
}
+ return { header, littleEndian }
+}
+
+function getElementByteSize(mode: number) {
+ if (mode === 2) {
+ return 4
+ } else if (mode === 1) {
+ return 2
+ } else if (mode === 0) {
+ return 1
+ } else {
+ throw new Error(`ccp4 mode '${mode}' unsupported`);
+ }
+}
+
+async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Ccp4File> {
+ await ctx.update({ message: 'Parsing CCP4/MRC file...' });
+
+ const { header, littleEndian } = await readCcp4Header(file)
+
const offset = 256 * 4 + header.NSYMBT
const count = header.NC * header.NR * header.NS
+ const elementByteSize = getElementByteSize(header.MODE)
+ const byteCount = count * elementByteSize
+
+ const { buffer } = await file.readBuffer(offset, size)
+
let values
if (header.MODE === 2) {
- values = new Float32Array(bin, offset, count)
+ values = new Float32Array(buffer, offset, count)
} else if (header.MODE === 0) {
- values = new Int8Array(bin, offset, count)
+ values = new Int8Array(buffer, offset, count)
} else {
- return Result.error(`ccp4 mode '${header.MODE}' unsupported`);
+ throw new Error(`ccp4 mode '${header.MODE}' unsupported`);
+ }
+
+ if (!littleEndian) {
+ SimpleBuffer.flipByteOrder(buffer, new Uint8Array(values.buffer), byteCount, elementByteSize, 0)
}
// if the file was converted by mapmode2to0 - scale the data
// based on uglymol (https://github.com/uglymol/uglymol) by Marcin Wojdyr (wojdyr)
- if (intView[39] === -128 && intView[40] === 127) {
+ if (header.userFlag1 -128 && header.userFlag2 === 127) {
values = new Float32Array(values)
// scaling f(x)=b1*x+b0 such that f(-128)=min and f(127)=max
const b1 = (header.AMAX - header.AMIN) / 255.0
@@ -117,13 +147,19 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
}
const result: Ccp4File = { header, values };
- return Result.success(result);
+ return result
}
-export function parseFile(file: FileHandle) {
- return Task.create<Result<Ccp4File>>('Parse CCP4/MRC', ctx => parseInternal(file, ctx));
+export function parseFile(file: FileHandle, size: number) {
+ return Task.create<Result<Ccp4File>>('Parse CCP4/MRC', async ctx => {
+ try {
+ return Result.success(await parseInternal(file, size, ctx));
+ } catch (e) {
+ return Result.error(e);
+ }
+ })
}
export function parse(buffer: Uint8Array) {
- return parseFile(FileHandle.fromBuffer(buffer))
+ return parseFile(FileHandle.fromBuffer(SimpleBuffer.fromUint8Array(buffer)), buffer.length)
}
\ No newline at end of file
diff --git a/src/mol-io/reader/ccp4/schema.ts b/src/mol-io/reader/ccp4/schema.ts
index 0ef828cf3..1263b590d 100644
--- a/src/mol-io/reader/ccp4/schema.ts
+++ b/src/mol-io/reader/ccp4/schema.ts
@@ -81,6 +81,9 @@ export interface Ccp4Header {
* May be used in CCP4 but not in MRC
*/
SKWTRN: number[]
+ /** see https://github.com/uglymol/uglymol/blob/master/tools/mapmode2to0#L69 */
+ userFlag1: number,
+ userFlag2: number,
/** x axis origin transformation (not used in CCP4) */
originX: number
/** y axis origin transformation (not used in CCP4) */
diff --git a/src/mol-io/reader/dsn6/parser.ts b/src/mol-io/reader/dsn6/parser.ts
index 35416d7a8..737e7461b 100644
--- a/src/mol-io/reader/dsn6/parser.ts
+++ b/src/mol-io/reader/dsn6/parser.ts
@@ -8,6 +8,7 @@ import { Task, RuntimeContext } from 'mol-task';
import { Dsn6File, Dsn6Header } from './schema'
import { ReaderResult as Result } from '../result'
import { FileHandle } from '../../common/file-handle';
+import { SimpleBuffer } from 'mol-io/common/simple-buffer';
function parseBrixHeader(str: string): Dsn6Header {
return {
@@ -56,10 +57,10 @@ function parseDsn6Header(int: Int16Array): Dsn6Header {
}
}
-async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Result<Dsn6File>> {
+async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Dsn6File> {
await ctx.update({ message: 'Parsing DSN6/BRIX file...' });
- const { buffer } = await file.readBuffer(0, file.length)
+ const { buffer } = await file.readBuffer(0, size)
const bin = buffer.buffer
const intView = new Int16Array(bin)
@@ -115,13 +116,19 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
}
const result: Dsn6File = { header, values };
- return Result.success(result);
+ return result;
}
-export function parseFile(file: FileHandle) {
- return Task.create<Result<Dsn6File>>('Parse DSN6/BRIX', ctx => parseInternal(file, ctx));
+export function parseFile(file: FileHandle, size: number) {
+ return Task.create<Result<Dsn6File>>('Parse DSN6/BRIX', async ctx => {
+ try {
+ return Result.success(await parseInternal(file, size, ctx));
+ } catch (e) {
+ return Result.error(e);
+ }
+ })
}
export function parse(buffer: Uint8Array) {
- return parseFile(FileHandle.fromBuffer(buffer))
+ return parseFile(FileHandle.fromBuffer(SimpleBuffer.fromUint8Array(buffer)), buffer.length)
}
\ No newline at end of file
diff --git a/webpack.config.js b/webpack.config.js
index 3a2db2529..a25ec62f3 100644
--- a/webpack.config.js
+++ b/webpack.config.js
@@ -62,6 +62,7 @@ const sharedConfig = {
function createEntryPoint(name, dir, out) {
return {
+ node: { fs: 'empty' }, // TODO find better solution? Currently used in file-handle.ts
entry: path.resolve(__dirname, `build/src/${dir}/${name}.js`),
output: { filename: `${name}.js`, path: path.resolve(__dirname, `build/${out}`) },
...sharedConfig
--
GitLab