Skip to content
Snippets Groups Projects
Commit b73c3b6b authored by Alexander Rose's avatar Alexander Rose
Browse files

extended FileHandle API and use SimpleBuffer

parent 7bcee674
No related branches found
No related tags found
No related merge requests found
/**
* Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
* Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
* @author David Sehnal <david.sehnal@gmail.com>
*/
import { defaults } from 'mol-util';
import { defaults, noop } from 'mol-util';
import { SimpleBuffer } from './simple-buffer';
// only import 'fs' in node.js
const fs = typeof document === 'undefined' ? require('fs') as typeof import('fs') : void 0;
export interface FileHandle {
/** The number of bytes in the file */
length: number
/**
* Asynchronously reads data, returning buffer and number of bytes read
*
* @param position The offset from the beginning of the file from which data should be read.
* @param sizeOrBuffer The buffer the data will be written to. If a number a buffer of that size will be created.
* @param size The number of bytes to read.
* @param sizeOrBuffer The buffer the data will be read from.
* @param length The number of bytes to read.
* @param byteOffset The offset in the buffer at which to start writing.
*/
readBuffer(position: number, sizeOrBuffer: Uint8Array | number, size?: number, byteOffset?: number): Promise<{ bytesRead: number, buffer: Uint8Array }>
readBuffer(position: number, sizeOrBuffer: SimpleBuffer | number, length?: number, byteOffset?: number): Promise<{ bytesRead: number, buffer: SimpleBuffer }>
/**
* Asynchronously writes buffer, returning the number of bytes written.
*
* @param position — The offset from the beginning of the file where this data should be written.
* @param buffer - The buffer data to be written.
* @param length — The number of bytes to write. If not supplied, defaults to buffer.length - offset.
*/
writeBuffer(position: number, buffer: SimpleBuffer, length?: number): Promise<number>
/**
* Synchronously writes buffer, returning the number of bytes written.
*
* @param position — The offset from the beginning of the file where this data should be written.
* @param buffer - The buffer data to be written.
* @param length — The number of bytes to write. If not supplied, defaults to buffer.length - offset.
*/
writeBufferSync(position: number, buffer: SimpleBuffer, length?: number): number
/** Closes a file handle */
close(): void
}
export namespace FileHandle {
export function fromBuffer(buffer: Uint8Array): FileHandle {
export function fromBuffer(buffer: SimpleBuffer): FileHandle {
return {
length: buffer.length,
readBuffer: (position: number, sizeOrBuffer: Uint8Array | number, size?: number, byteOffset?: number) => {
readBuffer: (position: number, sizeOrBuffer: SimpleBuffer | number, size?: number, byteOffset?: number) => {
if (typeof sizeOrBuffer === 'number') {
const start = position
const end = Math.min(buffer.length, start + (defaults(size, sizeOrBuffer)))
return Promise.resolve({
bytesRead: end - start,
buffer: buffer.subarray(start, end),
})
return Promise.resolve({ bytesRead: end - start, buffer: SimpleBuffer.fromUint8Array(buffer.subarray(start, end)) })
} else {
if (size === void 0) {
return Promise.reject('readBuffer: Specify size.');
......@@ -37,10 +58,68 @@ export namespace FileHandle {
const start = position
const end = Math.min(buffer.length, start + defaults(size, sizeOrBuffer.length))
sizeOrBuffer.set(buffer.subarray(start, end), byteOffset)
return Promise.resolve({
bytesRead: end - start,
buffer: sizeOrBuffer,
return Promise.resolve({ bytesRead: end - start, buffer: sizeOrBuffer })
}
},
writeBuffer: (position: number, buffer: SimpleBuffer, length?: number) => {
length = defaults(length, buffer.length)
console.warn('FileHandle.writeBuffer not implemented')
return Promise.resolve(0)
},
writeBufferSync: (position: number, buffer: SimpleBuffer, length?: number, ) => {
length = defaults(length, buffer.length)
console.warn('FileHandle.writeSync not implemented')
return 0
},
close: noop
}
}
export function fromDescriptor(file: number): FileHandle {
if (fs === undefined) throw new Error('fs module not available')
return {
readBuffer: (position: number, sizeOrBuffer: SimpleBuffer | number, length?: number, byteOffset?: number) => {
return new Promise((res, rej) => {
if (typeof sizeOrBuffer === 'number') {
let buff = new Buffer(new ArrayBuffer(sizeOrBuffer));
fs.read(file, buff, 0, sizeOrBuffer, position, (err, bytesRead, buffer) => {
if (err) {
rej(err);
return;
}
res({ bytesRead, buffer });
});
} else {
if (length === void 0) {
rej('readBuffer: Specify size.');
return;
}
fs.read(file, sizeOrBuffer, byteOffset ? +byteOffset : 0, length, position, (err, bytesRead, buffer) => {
if (err) {
rej(err);
return;
}
res({ bytesRead, buffer });
});
}
})
},
writeBuffer: (position: number, buffer: Buffer, length?: number) => {
return new Promise<number>((res, rej) => {
fs.write(file, buffer, 0, length !== void 0 ? length : buffer.length, position, (err, written) => {
if (err) rej(err);
else res(written);
})
})
},
writeBufferSync: (position: number, buffer: Uint8Array, length?: number) => {
return fs.writeSync(file, buffer, 0, length, position);
},
close: () => {
try {
if (file !== void 0) fs.close(file, noop);
} catch (e) {
}
}
}
......
......@@ -8,15 +8,12 @@ import { Task, RuntimeContext } from 'mol-task';
import { Ccp4File, Ccp4Header } from './schema'
import { ReaderResult as Result } from '../result'
import { FileHandle } from '../../common/file-handle';
import { SimpleBuffer } from 'mol-io/common/simple-buffer';
async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Result<Ccp4File>> {
await ctx.update({ message: 'Parsing CCP4/MRC file...' });
const { buffer } = await file.readBuffer(0, file.length)
export async function readCcp4Header(file: FileHandle) {
const headerSize = 1024;
const { buffer } = await file.readBuffer(0, headerSize)
const bin = buffer.buffer
const intView = new Int32Array(bin, 0, 56)
const floatView = new Float32Array(bin, 0, 56)
const dv = new DataView(bin)
// 53 MAP Character string 'MAP ' to identify file type
......@@ -25,88 +22,121 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
dv.getUint8(52 * 4 + 2), dv.getUint8(52 * 4 + 3)
)
if (MAP !== 'MAP ') {
return Result.error('ccp4 format error, missing "MAP " string');
throw new Error('ccp4 format error, missing "MAP " string');
}
// 54 MACHST Machine stamp indicating machine type which wrote file
// 17 and 17 for big-endian or 68 and 65 for little-endian
const MACHST = [ dv.getUint8(53 * 4), dv.getUint8(53 * 4 + 1) ]
let littleEndian = true
// found MRC files that don't have the MACHST stamp set and are big-endian
if (MACHST[0] !== 68 && MACHST[1] !== 65) {
// flip byte order in-place
for (let i = 0, il = bin.byteLength; i < il; i += 4) {
dv.setFloat32(i, dv.getFloat32(i), true)
}
littleEndian = false;
}
const readInt = (o: number) => dv.getInt32(o * 4, littleEndian)
const readFloat = (o: number) => dv.getFloat32(o * 4, littleEndian)
const header: Ccp4Header = {
NC: intView[0],
NR: intView[1],
NS: intView[2],
NC: readInt(0),
NR: readInt(1),
NS: readInt(2),
MODE: intView[3],
MODE: readInt(3),
NCSTART: intView[4],
NRSTART: intView[5],
NSSTART: intView[6],
NCSTART: readInt(4),
NRSTART: readInt(5),
NSSTART: readInt(6),
NX: intView[7],
NY: intView[8],
NZ: intView[9],
NX: readInt(7),
NY: readInt(8),
NZ: readInt(9),
xLength: floatView[10],
yLength: floatView[11],
zLength: floatView[12],
xLength: readFloat(10),
yLength: readFloat(11),
zLength: readFloat(12),
alpha: floatView[13],
beta: floatView[14],
gamma: floatView[15],
alpha: readFloat(13),
beta: readFloat(14),
gamma: readFloat(15),
MAPC: intView[16],
MAPR: intView[17],
MAPS: intView[18],
MAPC: readInt(16),
MAPR: readInt(17),
MAPS: readInt(18),
AMIN: floatView[19],
AMAX: floatView[20],
AMEAN: floatView[21],
AMIN: readFloat(19),
AMAX: readFloat(20),
AMEAN: readFloat(21),
ISPG: intView[22],
ISPG: readInt(22),
NSYMBT: intView[23],
NSYMBT: readInt(23),
LSKFLG: intView[24],
LSKFLG: readInt(24),
SKWMAT: [], // TODO bytes 26-34
SKWTRN: [], // TODO bytes 35-37
userFlag1: readInt(39),
userFlag2: readInt(40),
// bytes 50-52 origin in X,Y,Z used for transforms
originX: floatView[49],
originY: floatView[50],
originZ: floatView[51],
originX: readFloat(49),
originY: readFloat(50),
originZ: readFloat(51),
MAP, // bytes 53 MAP
MACHST, // bytes 54 MACHST
ARMS: floatView[54],
ARMS: readFloat(54),
// TODO bytes 56 NLABL
// TODO bytes 57-256 LABEL
}
return { header, littleEndian }
}
function getElementByteSize(mode: number) {
if (mode === 2) {
return 4
} else if (mode === 1) {
return 2
} else if (mode === 0) {
return 1
} else {
throw new Error(`ccp4 mode '${mode}' unsupported`);
}
}
async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Ccp4File> {
await ctx.update({ message: 'Parsing CCP4/MRC file...' });
const { header, littleEndian } = await readCcp4Header(file)
const offset = 256 * 4 + header.NSYMBT
const count = header.NC * header.NR * header.NS
const elementByteSize = getElementByteSize(header.MODE)
const byteCount = count * elementByteSize
const { buffer } = await file.readBuffer(offset, size)
let values
if (header.MODE === 2) {
values = new Float32Array(bin, offset, count)
values = new Float32Array(buffer, offset, count)
} else if (header.MODE === 0) {
values = new Int8Array(bin, offset, count)
values = new Int8Array(buffer, offset, count)
} else {
return Result.error(`ccp4 mode '${header.MODE}' unsupported`);
throw new Error(`ccp4 mode '${header.MODE}' unsupported`);
}
if (!littleEndian) {
SimpleBuffer.flipByteOrder(buffer, new Uint8Array(values.buffer), byteCount, elementByteSize, 0)
}
// if the file was converted by mapmode2to0 - scale the data
// based on uglymol (https://github.com/uglymol/uglymol) by Marcin Wojdyr (wojdyr)
if (intView[39] === -128 && intView[40] === 127) {
if (header.userFlag1 -128 && header.userFlag2 === 127) {
values = new Float32Array(values)
// scaling f(x)=b1*x+b0 such that f(-128)=min and f(127)=max
const b1 = (header.AMAX - header.AMIN) / 255.0
......@@ -117,13 +147,19 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
}
const result: Ccp4File = { header, values };
return Result.success(result);
return result
}
export function parseFile(file: FileHandle) {
return Task.create<Result<Ccp4File>>('Parse CCP4/MRC', ctx => parseInternal(file, ctx));
export function parseFile(file: FileHandle, size: number) {
return Task.create<Result<Ccp4File>>('Parse CCP4/MRC', async ctx => {
try {
return Result.success(await parseInternal(file, size, ctx));
} catch (e) {
return Result.error(e);
}
})
}
export function parse(buffer: Uint8Array) {
return parseFile(FileHandle.fromBuffer(buffer))
return parseFile(FileHandle.fromBuffer(SimpleBuffer.fromUint8Array(buffer)), buffer.length)
}
\ No newline at end of file
......@@ -81,6 +81,9 @@ export interface Ccp4Header {
* May be used in CCP4 but not in MRC
*/
SKWTRN: number[]
/** see https://github.com/uglymol/uglymol/blob/master/tools/mapmode2to0#L69 */
userFlag1: number,
userFlag2: number,
/** x axis origin transformation (not used in CCP4) */
originX: number
/** y axis origin transformation (not used in CCP4) */
......
......@@ -8,6 +8,7 @@ import { Task, RuntimeContext } from 'mol-task';
import { Dsn6File, Dsn6Header } from './schema'
import { ReaderResult as Result } from '../result'
import { FileHandle } from '../../common/file-handle';
import { SimpleBuffer } from 'mol-io/common/simple-buffer';
function parseBrixHeader(str: string): Dsn6Header {
return {
......@@ -56,10 +57,10 @@ function parseDsn6Header(int: Int16Array): Dsn6Header {
}
}
async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Result<Dsn6File>> {
async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Dsn6File> {
await ctx.update({ message: 'Parsing DSN6/BRIX file...' });
const { buffer } = await file.readBuffer(0, file.length)
const { buffer } = await file.readBuffer(0, size)
const bin = buffer.buffer
const intView = new Int16Array(bin)
......@@ -115,13 +116,19 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
}
const result: Dsn6File = { header, values };
return Result.success(result);
return result;
}
export function parseFile(file: FileHandle) {
return Task.create<Result<Dsn6File>>('Parse DSN6/BRIX', ctx => parseInternal(file, ctx));
export function parseFile(file: FileHandle, size: number) {
return Task.create<Result<Dsn6File>>('Parse DSN6/BRIX', async ctx => {
try {
return Result.success(await parseInternal(file, size, ctx));
} catch (e) {
return Result.error(e);
}
})
}
export function parse(buffer: Uint8Array) {
return parseFile(FileHandle.fromBuffer(buffer))
return parseFile(FileHandle.fromBuffer(SimpleBuffer.fromUint8Array(buffer)), buffer.length)
}
\ No newline at end of file
......@@ -62,6 +62,7 @@ const sharedConfig = {
function createEntryPoint(name, dir, out) {
return {
node: { fs: 'empty' }, // TODO find better solution? Currently used in file-handle.ts
entry: path.resolve(__dirname, `build/src/${dir}/${name}.js`),
output: { filename: `${name}.js`, path: path.resolve(__dirname, `build/${out}`) },
...sharedConfig
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment