diff --git a/src/mol-io/common/simple-buffer.ts b/src/mol-io/common/simple-buffer.ts index 8825388f1439581955f41a1854be9d942f91ec2f..4e04f392aff484f4901493868e68721a16196ef2 100644 --- a/src/mol-io/common/simple-buffer.ts +++ b/src/mol-io/common/simple-buffer.ts @@ -2,6 +2,7 @@ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info. * * @author Alexander Rose <alexander.rose@weirdbyte.de> + * @author David Sehnal <david.sehnal@gmail.com> */ import { defaults } from 'mol-util'; @@ -95,6 +96,8 @@ export namespace SimpleBuffer { return buffer } + export const IsNativeEndianLittle = new Uint16Array(new Uint8Array([0x12, 0x34]).buffer)[0] === 0x3412; + /** source and target can't be the same */ export function flipByteOrder(source: SimpleBuffer, target: Uint8Array, byteCount: number, elementByteSize: number, offset: number) { for (let i = 0, n = byteCount; i < n; i += elementByteSize) { @@ -111,4 +114,10 @@ export namespace SimpleBuffer { intView[i] = ((val & 0xff) << 8) | ((val >> 8) & 0xff) } } + + export function ensureLittleEndian(source: SimpleBuffer, target: SimpleBuffer, byteCount: number, elementByteSize: number, offset: number) { + if (IsNativeEndianLittle) return; + if (!byteCount || elementByteSize <= 1) return; + flipByteOrder(source, target, byteCount, elementByteSize, offset); + } } \ No newline at end of file diff --git a/src/mol-io/common/typed-array.ts b/src/mol-io/common/typed-array.ts new file mode 100644 index 0000000000000000000000000000000000000000..37a14dc663d4cc509ef7ab511573eb583ff2f9b0 --- /dev/null +++ b/src/mol-io/common/typed-array.ts @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info. + * + * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer) + * + * @author David Sehnal <david.sehnal@gmail.com> + * @author Alexander Rose <alexander.rose@weirdbyte.de> + */ + +import { FileHandle } from 'mol-io/common/file-handle'; +import { SimpleBuffer } from 'mol-io/common/simple-buffer'; + +export type TypedArrayValueType = 'float32' | 'int8' | 'int16' + +export namespace TypedArrayValueType { + export const Float32: TypedArrayValueType = 'float32'; + export const Int8: TypedArrayValueType = 'int8'; + export const Int16: TypedArrayValueType = 'int16'; +} + +export type TypedArrayValueArray = Float32Array | Int8Array | Int16Array + +export interface TypedArrayBufferContext { + type: TypedArrayValueType, + elementByteSize: number, + readBuffer: Buffer, + valuesBuffer: Uint8Array, + values: TypedArrayValueArray +} + +export function getElementByteSize(type: TypedArrayValueType) { + if (type === TypedArrayValueType.Float32) return 4; + if (type === TypedArrayValueType.Int16) return 2; + return 1; +} + +export function makeTypedArray(type: TypedArrayValueType, buffer: ArrayBuffer, byteOffset = 0, length?: number): TypedArrayValueArray { + if (type === TypedArrayValueType.Float32) return new Float32Array(buffer, byteOffset, length); + if (type === TypedArrayValueType.Int16) return new Int16Array(buffer, byteOffset, length); + return new Int8Array(buffer, byteOffset, length); +} + +export function createTypedArray(type: TypedArrayValueType, size: number) { + switch (type) { + case TypedArrayValueType.Float32: return new Float32Array(new ArrayBuffer(4 * size)); + case TypedArrayValueType.Int8: return new Int8Array(new ArrayBuffer(1 * size)); + case TypedArrayValueType.Int16: return new Int16Array(new ArrayBuffer(2 * size)); + } + throw Error(`${type} is not a supported value format.`); +} + +export function createTypedArrayBufferContext(size: number, type: TypedArrayValueType): TypedArrayBufferContext { + let elementByteSize = getElementByteSize(type); + let arrayBuffer = new ArrayBuffer(elementByteSize * size); + let readBuffer = new Buffer(arrayBuffer); + let valuesBuffer = SimpleBuffer.IsNativeEndianLittle ? arrayBuffer : new ArrayBuffer(elementByteSize * size); + return { + type, + elementByteSize, + readBuffer, + valuesBuffer: new Uint8Array(valuesBuffer), + values: makeTypedArray(type, valuesBuffer) + }; +} + +export async function readTypedArray(ctx: TypedArrayBufferContext, file: FileHandle, position: number, count: number, valueOffset: number, littleEndian?: boolean) { + let byteCount = ctx.elementByteSize * count; + let byteOffset = ctx.elementByteSize * valueOffset; + + await file.readBuffer(position, ctx.readBuffer, byteCount, byteOffset); + if (ctx.elementByteSize > 1 && ((littleEndian !== void 0 && littleEndian !== SimpleBuffer.IsNativeEndianLittle) || !SimpleBuffer.IsNativeEndianLittle)) { + // fix the endian + SimpleBuffer.flipByteOrder(ctx.readBuffer, ctx.valuesBuffer, byteCount, ctx.elementByteSize, byteOffset); + } + return ctx.values; +} diff --git a/src/mol-io/reader/ccp4/parser.ts b/src/mol-io/reader/ccp4/parser.ts index b81f9cca34a4558ca82da86cf768b07308ab396c..d6ebfc0bc6477d42fc4a0fcf0fcfe81513952088 100644 --- a/src/mol-io/reader/ccp4/parser.ts +++ b/src/mol-io/reader/ccp4/parser.ts @@ -9,6 +9,7 @@ import { Ccp4File, Ccp4Header } from './schema' import { ReaderResult as Result } from '../result' import { FileHandle } from '../../common/file-handle'; import { SimpleBuffer } from 'mol-io/common/simple-buffer'; +import { TypedArrayValueType, getElementByteSize, makeTypedArray } from 'mol-io/common/typed-array'; export async function readCcp4Header(file: FileHandle): Promise<{ header: Ccp4Header, littleEndian: boolean }> { const headerSize = 1024; @@ -95,11 +96,11 @@ export async function readCcp4Header(file: FileHandle): Promise<{ header: Ccp4He return { header, littleEndian } } -function getElementByteSize(mode: number) { +function getTypedArrayValueType(mode: number) { switch (mode) { - case 2: return 4 - case 1: return 2 - case 0: return 1 + case 2: return TypedArrayValueType.Float32 + case 1: return TypedArrayValueType.Int16 + case 0: return TypedArrayValueType.Int8 } throw new Error(`ccp4 mode '${mode}' unsupported`); } @@ -110,28 +111,20 @@ async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext const { header, littleEndian } = await readCcp4Header(file) const offset = 256 * 4 + header.NSYMBT + const valueType = getTypedArrayValueType(header.MODE) const { buffer, bytesRead } = await file.readBuffer(offset, size - offset) const count = header.NC * header.NR * header.NS - const elementByteSize = getElementByteSize(header.MODE) + const elementByteSize = getElementByteSize(valueType) const byteCount = count * elementByteSize if (byteCount !== bytesRead) { console.warn(`byteCount ${byteCount} and bytesRead ${bytesRead} differ`) } - let values - if (header.MODE === 2) { - values = new Float32Array(buffer.buffer, offset, count) - } else if (header.MODE === 1) { - values = new Int16Array(buffer.buffer, offset, count) - } else if (header.MODE === 0) { - values = new Int8Array(buffer.buffer, offset, count) - } else { - throw new Error(`ccp4 mode '${header.MODE}' unsupported`); - } + let values = makeTypedArray(valueType, buffer.buffer, offset, count) - if (!littleEndian) { + if (!littleEndian && valueType !== TypedArrayValueType.Int8) { SimpleBuffer.flipByteOrder(buffer, new Uint8Array(values.buffer), byteCount, elementByteSize, 0) } diff --git a/src/servers/volume/common/data-format.ts b/src/servers/volume/common/data-format.ts index 0492ab16ce2bda7263a578e26d5db4a1bdd1cef3..2f0525475388dd923732969a3f110181ba32d926 100644 --- a/src/servers/volume/common/data-format.ts +++ b/src/servers/volume/common/data-format.ts @@ -8,16 +8,7 @@ import * as Schema from './binary-schema' import { FileHandle } from 'mol-io/common/file-handle'; - -export type ValueType = 'float32' | 'int8' | 'int16' - -export namespace ValueType { - export const Float32: ValueType = 'float32'; - export const Int8: ValueType = 'int8'; - export const Int16: ValueType = 'int16'; -} - -export type ValueArray = Float32Array | Int8Array | Int16Array +import { TypedArrayValueType } from 'mol-io/common/typed-array'; export interface Spacegroup { number: number, @@ -62,7 +53,7 @@ export interface Header { channels: string[], /** Determines the data type of the values */ - valueType: ValueType, + valueType: TypedArrayValueType, /** The value are stored in blockSize^3 cubes */ blockSize: number, @@ -102,21 +93,6 @@ namespace _schema { const headerSchema = _schema.schema; -export function getValueByteSize(type: ValueType) { - if (type === ValueType.Float32) return 4; - if (type === ValueType.Int16) return 2; - return 1; -} - -export function createValueArray(type: ValueType, size: number) { - switch (type) { - case ValueType.Float32: return new Float32Array(new ArrayBuffer(4 * size)); - case ValueType.Int8: return new Int8Array(new ArrayBuffer(1 * size)); - case ValueType.Int16: return new Int16Array(new ArrayBuffer(2 * size)); - } - throw Error(`${type} is not a supported value format.`); -} - export function encodeHeader(header: Header) { return Schema.encode(headerSchema, header); } diff --git a/src/servers/volume/common/file.ts b/src/servers/volume/common/file.ts index 8067b9eaf87419a1f0919a99c243c0c6e8f546dd..c01670aa19473a7ff5f4ffe5981ad2e61e006907 100644 --- a/src/servers/volume/common/file.ts +++ b/src/servers/volume/common/file.ts @@ -8,12 +8,9 @@ import * as fs from 'fs' import * as path from 'path' -import * as DataFormat from './data-format' import { FileHandle } from 'mol-io/common/file-handle'; import { SimpleBuffer } from 'mol-io/common/simple-buffer'; -export const IsNativeEndianLittle = new Uint16Array(new Uint8Array([0x12, 0x34]).buffer)[0] === 0x3412; - export async function openRead(filename: string) { return new Promise<number>((res, rej) => { fs.open(filename, 'r', async (err, file) => { @@ -64,64 +61,4 @@ const smallBuffer = SimpleBuffer.fromBuffer(new Buffer(8)); export async function writeInt(file: FileHandle, value: number, position: number) { smallBuffer.writeInt32LE(value, 0); await file.writeBuffer(position, smallBuffer, 4); -} - -export interface TypedArrayBufferContext { - type: DataFormat.ValueType, - elementByteSize: number, - readBuffer: Buffer, - valuesBuffer: Uint8Array, - values: DataFormat.ValueArray -} - -function getElementByteSize(type: DataFormat.ValueType) { - if (type === DataFormat.ValueType.Float32) return 4; - if (type === DataFormat.ValueType.Int16) return 2; - return 1; -} - -function makeTypedArray(type: DataFormat.ValueType, buffer: ArrayBuffer): DataFormat.ValueArray { - if (type === DataFormat.ValueType.Float32) return new Float32Array(buffer); - if (type === DataFormat.ValueType.Int16) return new Int16Array(buffer); - return new Int8Array(buffer); -} - -export function createTypedArrayBufferContext(size: number, type: DataFormat.ValueType): TypedArrayBufferContext { - let elementByteSize = getElementByteSize(type); - let arrayBuffer = new ArrayBuffer(elementByteSize * size); - let readBuffer = new Buffer(arrayBuffer); - let valuesBuffer = IsNativeEndianLittle ? arrayBuffer : new ArrayBuffer(elementByteSize * size); - return { - type, - elementByteSize, - readBuffer, - valuesBuffer: new Uint8Array(valuesBuffer), - values: makeTypedArray(type, valuesBuffer) - }; -} - -function flipByteOrder(source: SimpleBuffer, target: Uint8Array, byteCount: number, elementByteSize: number, offset: number) { - for (let i = 0, n = byteCount; i < n; i += elementByteSize) { - for (let j = 0; j < elementByteSize; j++) { - target[offset + i + elementByteSize - j - 1] = source[offset + i + j]; - } - } -} - -export async function readTypedArray(ctx: TypedArrayBufferContext, file: FileHandle, position: number, count: number, valueOffset: number, littleEndian?: boolean) { - let byteCount = ctx.elementByteSize * count; - let byteOffset = ctx.elementByteSize * valueOffset; - - await file.readBuffer(position, ctx.readBuffer, byteCount, byteOffset); - if (ctx.elementByteSize > 1 && ((littleEndian !== void 0 && littleEndian !== IsNativeEndianLittle) || !IsNativeEndianLittle)) { - // fix the endian - flipByteOrder(ctx.readBuffer, ctx.valuesBuffer, byteCount, ctx.elementByteSize, byteOffset); - } - return ctx.values; -} - -export function ensureLittleEndian(source: SimpleBuffer, target: SimpleBuffer, byteCount: number, elementByteSize: number, offset: number) { - if (IsNativeEndianLittle) return; - if (!byteCount || elementByteSize <= 1) return; - flipByteOrder(source, target, byteCount, elementByteSize, offset); } \ No newline at end of file diff --git a/src/servers/volume/pack/data-model.ts b/src/servers/volume/pack/data-model.ts index 33d6ab466e75a5306988deb1e3332b13807380e9..9e80f0f80be0cb6a1a4ed76546aed775e4b56dd0 100644 --- a/src/servers/volume/pack/data-model.ts +++ b/src/servers/volume/pack/data-model.ts @@ -9,6 +9,7 @@ import * as Format from './format' import * as DataFormat from '../common/data-format' import { FileHandle } from 'mol-io/common/file-handle'; import { SimpleBuffer } from 'mol-io/common/simple-buffer'; +import { TypedArrayValueArray, TypedArrayValueType } from 'mol-io/common/typed-array'; const FORMAT_VERSION = '1.0.0'; @@ -25,16 +26,16 @@ export interface ValuesInfo { } export interface BlockBuffer { - values: DataFormat.ValueArray[], + values: TypedArrayValueArray[], buffers: SimpleBuffer[], slicesWritten: number } export interface DownsamplingBuffer { /** dimensions (sampleCount[1], sampleCount[0] / 2, 1), axis order (K, H, L) */ - downsampleH: DataFormat.ValueArray, + downsampleH: TypedArrayValueArray, /** "Cyclic" (in the 1st dimensions) buffer with dimensions (5, sampleCount[0] / 2, sampleCount[1] / 2), axis order (L, H, K), */ - downsampleHK: DataFormat.ValueArray, + downsampleHK: TypedArrayValueArray, slicesWritten: number, startSliceIndex: number @@ -76,7 +77,7 @@ export interface Context { isPeriodic: boolean, channels: Format.Context[], - valueType: DataFormat.ValueType, + valueType: TypedArrayValueType, blockSize: number, /** Able to store channels.length * blockSize^3 values. */ cubeBuffer: SimpleBuffer, @@ -101,7 +102,7 @@ export function createHeader(ctx: Context): DataFormat.Header { return { formatVersion: FORMAT_VERSION, - valueType: Format.getValueType(header), + valueType: header.valueType, blockSize: ctx.blockSize, axisOrder: header.axisOrder, origin: normalize(header.origin), diff --git a/src/servers/volume/pack/downsampling.ts b/src/servers/volume/pack/downsampling.ts index 5541b8a9a120f2f26a975cfa00a6915e95a32a48..25f67d795e5676a6f6ebe026d5f73fc0383db6f6 100644 --- a/src/servers/volume/pack/downsampling.ts +++ b/src/servers/volume/pack/downsampling.ts @@ -7,10 +7,10 @@ */ import * as Data from './data-model' -import * as DataFormat from '../common/data-format' +import { TypedArrayValueArray } from 'mol-io/common/typed-array'; -/** - * Downsamples each slice of input data and checks if there is enough data to perform +/** + * Downsamples each slice of input data and checks if there is enough data to perform * higher rate downsampling. */ export function downsampleLayer(ctx: Data.Context) { @@ -25,8 +25,8 @@ export function downsampleLayer(ctx: Data.Context) { } } -/** - * When the "native" (rate = 1) sampling is finished, there might still +/** + * When the "native" (rate = 1) sampling is finished, there might still * be some data left to be processed for the higher rate samplings. */ export function finalize(ctx: Data.Context) { @@ -46,7 +46,7 @@ export function finalize(ctx: Data.Context) { /** * The functions downsampleH and downsampleHK both essentially do the * same thing: downsample along H (1st axis in axis order) and K (2nd axis in axis order) axes respectively. - * + * * The reason there are two copies of almost the same code is performance: * Both functions use a different memory layout to improve cache coherency * - downsampleU uses the H axis as the fastest moving one @@ -54,7 +54,7 @@ export function finalize(ctx: Data.Context) { */ -function conv(w: number, c: number[], src: DataFormat.ValueArray, b: number, i0: number, i1: number, i2: number, i3: number, i4: number) { +function conv(w: number, c: number[], src: TypedArrayValueArray, b: number, i0: number, i1: number, i2: number, i3: number, i4: number) { return w * (c[0] * src[b + i0] + c[1] * src[b + i1] + c[2] * src[b + i2] + c[3] * src[b + i3] + c[4] * src[b + i4]); } @@ -63,7 +63,7 @@ function conv(w: number, c: number[], src: DataFormat.ValueArray, b: number, i0: * flipping the 1st and 2nd axis in the process to optimize cache coherency for downsampleUV call * (i.e. use (K, H, L) axis order). */ -function downsampleH(kernel: Data.Kernel, srcDims: number[], src: DataFormat.ValueArray, srcLOffset: number, buffer: Data.DownsamplingBuffer) { +function downsampleH(kernel: Data.Kernel, srcDims: number[], src: TypedArrayValueArray, srcLOffset: number, buffer: Data.DownsamplingBuffer) { const target = buffer.downsampleH; const sizeH = srcDims[0], sizeK = srcDims[1], srcBaseOffset = srcLOffset * sizeH * sizeK; const targetH = Math.floor((sizeH + 1) / 2); @@ -87,9 +87,9 @@ function downsampleH(kernel: Data.Kernel, srcDims: number[], src: DataFormat.Val } } -/** - * Downsample first axis in the slice present in buffer.downsampleH - * The result is written into the "cyclical" downsampleHk buffer +/** + * Downsample first axis in the slice present in buffer.downsampleH + * The result is written into the "cyclical" downsampleHk buffer * in the (L, H, K) axis order. */ function downsampleHK(kernel: Data.Kernel, dimsX: number[], buffer: Data.DownsamplingBuffer) { diff --git a/src/servers/volume/pack/format.ts b/src/servers/volume/pack/format.ts index ff96da397fc6bc950c05baa4ae16c8072be42e75..0f6fe13a3c428105831bf8d67139ac9e071dfcdf 100644 --- a/src/servers/volume/pack/format.ts +++ b/src/servers/volume/pack/format.ts @@ -6,15 +6,13 @@ */ import * as File from '../common/file' -import * as DataFormat from '../common/data-format' import { FileHandle } from 'mol-io/common/file-handle'; import { Ccp4Provider } from './format/ccp4'; - -export const enum Mode { Int8 = 0, Int16 = 1, Float32 = 2 } +import { TypedArrayBufferContext, TypedArrayValueArray, TypedArrayValueType, getElementByteSize, createTypedArrayBufferContext } from 'mol-io/common/typed-array'; export interface Header { name: string, - mode: Mode, + valueType: TypedArrayValueType, grid: number[], // grid is converted to the axis order!! axisOrder: number[], extent: number[], @@ -28,11 +26,11 @@ export interface Header { /** Represents a circular buffer for 2 * blockSize layers */ export interface SliceBuffer { - buffer: File.TypedArrayBufferContext, + buffer: TypedArrayBufferContext, sliceCapacity: number, slicesRead: number, - values: DataFormat.ValueArray, + values: TypedArrayValueArray, sliceCount: number, /** Have all the input slice been read? */ @@ -55,18 +53,11 @@ export interface Context { provider: Provider } -export function getValueType(header: Header) { - if (header.mode === Mode.Float32) return DataFormat.ValueType.Float32; - if (header.mode === Mode.Int16) return DataFormat.ValueType.Int16; - return DataFormat.ValueType.Int8; -} - export function assignSliceBuffer(data: Data, blockSize: number) { - const { extent } = data.header; - const valueType = getValueType(data.header); - const sliceSize = extent[0] * extent[1] * DataFormat.getValueByteSize(valueType); - const sliceCapacity = Math.max(1, Math.floor(Math.min(64 * 1024 * 1024, sliceSize * extent[2]) / sliceSize)); - const buffer = File.createTypedArrayBufferContext(sliceCapacity * extent[0] * extent[1], valueType); + const { extent, valueType } = data.header; + const sliceSize = extent[0] * extent[1] * getElementByteSize(valueType); + const sliceCapacity = Math.max(1, Math.floor(Math.min(1 * 1024 * 1024, sliceSize * extent[2]) / sliceSize)); + const buffer = createTypedArrayBufferContext(sliceCapacity * extent[0] * extent[1], valueType); data.slices = { buffer, sliceCapacity, diff --git a/src/servers/volume/pack/format/ccp4.ts b/src/servers/volume/pack/format/ccp4.ts index 10e7e121ac3308f903fb0b33dcb1cfb136a027c7..0e07ee3e7429eeafcb85c3dda4f94f2c161c16df 100644 --- a/src/servers/volume/pack/format/ccp4.ts +++ b/src/servers/volume/pack/format/ccp4.ts @@ -9,6 +9,16 @@ import { FileHandle } from 'mol-io/common/file-handle'; import { readCcp4Header } from 'mol-io/reader/ccp4/parser'; import { Header, Provider } from '../format'; import { readSlices } from './common'; +import { TypedArrayValueType } from 'mol-io/common/typed-array'; + +function getTypedArrayValueType(mode: number) { + switch (mode) { + case 2: return TypedArrayValueType.Float32 + case 1: return TypedArrayValueType.Int16 + case 0: return TypedArrayValueType.Int8 + } + throw new Error(`ccp4 mode '${mode}' unsupported`); +} async function readHeader(name: string, file: FileHandle) { const { header: ccp4Header, littleEndian } = await readCcp4Header(file) @@ -17,7 +27,7 @@ async function readHeader(name: string, file: FileHandle) { const nxyzStart = [ccp4Header.NCSTART, ccp4Header.NRSTART, ccp4Header.NSSTART]; const header: Header = { name, - mode: ccp4Header.MODE, + valueType: getTypedArrayValueType(ccp4Header.MODE), grid: [ccp4Header.NX, ccp4Header.NY, ccp4Header.NZ], axisOrder: [ccp4Header.MAPC, ccp4Header.MAPR, ccp4Header.MAPS].map(i => i - 1), extent: [ccp4Header.NC, ccp4Header.NR, ccp4Header.NS], diff --git a/src/servers/volume/pack/format/common.ts b/src/servers/volume/pack/format/common.ts index 418c75623cc00a7a7078c868fdc1902268ffedad..e2e020d4875677b19d9fd410caf35a709e970880 100644 --- a/src/servers/volume/pack/format/common.ts +++ b/src/servers/volume/pack/format/common.ts @@ -6,7 +6,7 @@ */ import { Data } from '../format'; -import * as File from '../../common/file' +import { readTypedArray } from 'mol-io/common/typed-array'; export async function readSlices(data: Data) { const { slices, header } = data; @@ -19,8 +19,9 @@ export async function readSlices(data: Data) { const sliceByteOffset = slices.buffer.elementByteSize * sliceSize * slices.slicesRead; const sliceCount = Math.min(slices.sliceCapacity, extent[2] - slices.slicesRead); const sliceByteCount = sliceCount * sliceSize; + console.log('sliceByteOffset', sliceByteOffset, 'sliceSize', sliceSize, 'sliceCount', sliceCount) - await File.readTypedArray(slices.buffer, data.file, header.dataOffset + sliceByteOffset, sliceByteCount, 0, header.littleEndian); + await readTypedArray(slices.buffer, data.file, header.dataOffset + sliceByteOffset, sliceByteCount, 0, header.littleEndian); slices.slicesRead += sliceCount; slices.sliceCount = sliceCount; diff --git a/src/servers/volume/pack/format/dsn6.ts b/src/servers/volume/pack/format/dsn6.ts index d95b05b7f439f5bf8a9b04dfdbb1859d4048a499..a9ff021fa83df9ecb84e10f1af3e8591c17b1c70 100644 --- a/src/servers/volume/pack/format/dsn6.ts +++ b/src/servers/volume/pack/format/dsn6.ts @@ -5,16 +5,17 @@ */ import { FileHandle } from 'mol-io/common/file-handle'; -import { Header, Provider, Mode } from '../format'; +import { Header, Provider } from '../format'; import { readSlices } from './common'; import { readDsn6Header, dsn6HeaderSize } from 'mol-io/reader/dsn6/parser'; +import { TypedArrayValueType } from 'mol-io/common/typed-array'; async function readHeader(name: string, file: FileHandle) { const { header: dsn6Header, littleEndian } = await readDsn6Header(file) const header: Header = { name, - mode: Mode.Int16, + valueType: TypedArrayValueType.Int16, grid: [dsn6Header.xRate, dsn6Header.yRate, dsn6Header.zRate], axisOrder: [0, 1, 2], extent: [dsn6Header.xExtent, dsn6Header.yExtent, dsn6Header.zExtent], diff --git a/src/servers/volume/pack/sampling.ts b/src/servers/volume/pack/sampling.ts index 777bcba4b47b26b79914224d1af435d2e1d38d0b..97d77961d06c77960d7b0450831e762e6e374284 100644 --- a/src/servers/volume/pack/sampling.ts +++ b/src/servers/volume/pack/sampling.ts @@ -13,19 +13,21 @@ import * as Downsampling from './downsampling' import * as Writer from './writer' import * as DataFormat from '../common/data-format' import { FileHandle } from 'mol-io/common/file-handle'; +import { getElementByteSize, createTypedArray, TypedArrayValueType } from 'mol-io/common/typed-array'; +import { SimpleBuffer } from 'mol-io/common/simple-buffer'; export async function createContext(filename: string, channels: Format.Context[], blockSize: number, isPeriodic: boolean): Promise<Data.Context> { - const header = channels[0].data.header; - const samplingCounts = getSamplingCounts(channels[0].data.header.extent, blockSize); - const valueType = Format.getValueType(header); - const cubeBuffer = new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * DataFormat.getValueByteSize(valueType))); + const { extent, valueType, grid, origin } = channels[0].data.header; - const litteEndianCubeBuffer = File.IsNativeEndianLittle + const samplingCounts = getSamplingCounts(extent, blockSize); + const cubeBuffer = new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * getElementByteSize(valueType))); + + const litteEndianCubeBuffer = SimpleBuffer.IsNativeEndianLittle ? cubeBuffer - : new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * DataFormat.getValueByteSize(valueType))); + : new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * getElementByteSize(valueType))); // The data can be periodic iff the extent is the same as the grid and origin is 0. - if (header.grid.some((v, i) => v !== header.extent[i]) || header.origin.some(v => v !== 0)) { + if (grid.some((v, i) => v !== extent[i]) || origin.some(v => v !== 0)) { isPeriodic = false; } @@ -93,9 +95,9 @@ function getSamplingCounts(baseSampleCount: number[], blockSize: number) { } } -function createBlockBuffer(sampleCount: number[], blockSize: number, valueType: DataFormat.ValueType, numChannels: number): Data.BlockBuffer { +function createBlockBuffer(sampleCount: number[], blockSize: number, valueType: TypedArrayValueType, numChannels: number): Data.BlockBuffer { const values = []; - for (let i = 0; i < numChannels; i++) values[i] = DataFormat.createValueArray(valueType, sampleCount[0] * sampleCount[1] * blockSize); + for (let i = 0; i < numChannels; i++) values[i] = createTypedArray(valueType, sampleCount[0] * sampleCount[1] * blockSize); return { values, buffers: values.map(xs => new Buffer(xs.buffer)), @@ -103,12 +105,12 @@ function createBlockBuffer(sampleCount: number[], blockSize: number, valueType: }; } -function createDownsamplingBuffer(valueType: DataFormat.ValueType, sourceSampleCount: number[], targetSampleCount: number[], numChannels: number): Data.DownsamplingBuffer[] { +function createDownsamplingBuffer(valueType: TypedArrayValueType, sourceSampleCount: number[], targetSampleCount: number[], numChannels: number): Data.DownsamplingBuffer[] { const ret = []; for (let i = 0; i < numChannels; i++) { ret[ret.length] = { - downsampleH: DataFormat.createValueArray(valueType, sourceSampleCount[1] * targetSampleCount[0]), - downsampleHK: DataFormat.createValueArray(valueType, 5 * targetSampleCount[0] * targetSampleCount[1]), + downsampleH: createTypedArray(valueType, sourceSampleCount[1] * targetSampleCount[0]), + downsampleHK: createTypedArray(valueType, 5 * targetSampleCount[0] * targetSampleCount[1]), slicesWritten: 0, startSliceIndex: 0 } @@ -116,7 +118,7 @@ function createDownsamplingBuffer(valueType: DataFormat.ValueType, sourceSampleC return ret; } -function createSampling(index: number, valueType: DataFormat.ValueType, numChannels: number, sampleCounts: number[][], blockSize: number): Data.Sampling { +function createSampling(index: number, valueType: TypedArrayValueType, numChannels: number, sampleCounts: number[][], blockSize: number): Data.Sampling { const sampleCount = sampleCounts[index]; const valuesInfo: Data.ValuesInfo[] = []; for (let i = 0; i < numChannels; i++) { @@ -135,7 +137,7 @@ function createSampling(index: number, valueType: DataFormat.ValueType, numChann downsampling: index < sampleCounts.length - 1 ? createDownsamplingBuffer(valueType, sampleCount, sampleCounts[index + 1], numChannels) : void 0, byteOffset: 0, - byteSize: numChannels * sampleCount[0] * sampleCount[1] * sampleCount[2] * DataFormat.getValueByteSize(valueType), + byteSize: numChannels * sampleCount[0] * sampleCount[1] * sampleCount[2] * getElementByteSize(valueType), writeByteOffset: 0 } } diff --git a/src/servers/volume/pack/writer.ts b/src/servers/volume/pack/writer.ts index bdd1348b4f42a02802ddbe546eabfdc8a503580a..287feae38b140ded5127563e7b5868d59e283ab5 100644 --- a/src/servers/volume/pack/writer.ts +++ b/src/servers/volume/pack/writer.ts @@ -7,8 +7,8 @@ */ import * as Data from './data-model' -import * as File from '../common/file' -import * as DataFormat from '../common/data-format' +import { getElementByteSize } from 'mol-io/common/typed-array'; +import { SimpleBuffer } from 'mol-io/common/simple-buffer'; /** Converts a layer to blocks and writes them to the output file. */ export async function writeBlockLayer(ctx: Data.Context, sampling: Data.Sampling) { @@ -32,7 +32,7 @@ function fillCubeBuffer(ctx: Data.Context, sampling: Data.Sampling, u: number, v const { blockSize, cubeBuffer } = ctx; const { sampleCount } = sampling; const { buffers, slicesWritten } = sampling.blocks; - const elementSize = DataFormat.getValueByteSize(ctx.valueType); + const elementSize = getElementByteSize(ctx.valueType); const sizeH = sampleCount[0], sizeHK = sampleCount[0] * sampleCount[1]; const offsetH = u * blockSize, offsetK = v * blockSize; @@ -52,7 +52,7 @@ function fillCubeBuffer(ctx: Data.Context, sampling: Data.Sampling, u: number, v } } // flip the byte order if needed. - File.ensureLittleEndian(ctx.cubeBuffer, ctx.litteEndianCubeBuffer, writeOffset, elementSize, 0); + SimpleBuffer.ensureLittleEndian(ctx.cubeBuffer, ctx.litteEndianCubeBuffer, writeOffset, elementSize, 0); return writeOffset; } diff --git a/src/servers/volume/server/query/compose.ts b/src/servers/volume/server/query/compose.ts index af6f8ce8255d63f6d393eb96131a636683daa934..d0449c3ddf4f2d7ffd0ba7f09a70f7f83605e940 100644 --- a/src/servers/volume/server/query/compose.ts +++ b/src/servers/volume/server/query/compose.ts @@ -6,11 +6,10 @@ * @author David Sehnal <david.sehnal@gmail.com> */ -import * as DataFormat from '../../common/data-format' import * as Data from './data-model' import * as Box from '../algebra/box' import * as Coords from '../algebra/coordinate' -import * as File from '../../common/file' +import { createTypedArrayBufferContext, getElementByteSize, readTypedArray } from 'mol-io/common/typed-array'; export default async function compose(query: Data.QueryContext.Data) { for (const block of query.samplingInfo.blocks) { @@ -24,14 +23,14 @@ async function readBlock(query: Data.QueryContext.Data, coord: Coords.Grid<'Bloc const size = numChannels * blockSampleCount[0] * blockSampleCount[1] * blockSampleCount[2]; const { valueType, blockSize } = query.data.header; const dataSampleCount = query.data.header.sampling[query.samplingInfo.sampling.index].sampleCount; - const buffer = File.createTypedArrayBufferContext(size, valueType); + const buffer = createTypedArrayBufferContext(size, valueType); const byteOffset = query.samplingInfo.sampling.byteOffset - + DataFormat.getValueByteSize(valueType) * numChannels * blockSize + + getElementByteSize(valueType) * numChannels * blockSize * (blockSampleCount[1] * blockSampleCount[2] * coord[0] + dataSampleCount[0] * blockSampleCount[2] * coord[1] + dataSampleCount[0] * dataSampleCount[1] * coord[2]); - const values = await File.readTypedArray(buffer, query.data.file, byteOffset, size, 0); + const values = await readTypedArray(buffer, query.data.file, byteOffset, size, 0); return { sampleCount: blockSampleCount, values diff --git a/src/servers/volume/server/query/data-model.ts b/src/servers/volume/server/query/data-model.ts index 9df53a9414cd4e7da79d00faffbcc5f63bf93d6a..1536a797931ae8bf0afa47f32f29bf72c86eace9 100644 --- a/src/servers/volume/server/query/data-model.ts +++ b/src/servers/volume/server/query/data-model.ts @@ -12,6 +12,7 @@ import * as Box from '../algebra/box' import Writer from 'mol-io/writer/writer' import { SpacegroupCell } from 'mol-math/geometry'; import { FileHandle } from 'mol-io/common/file-handle'; +import { TypedArrayValueArray } from 'mol-io/common/typed-array'; ////////////////////////////////////// // DATA @@ -35,7 +36,7 @@ export interface DataContext { export interface BlockData { sampleCount: number[], - values: DataFormat.ValueArray + values: TypedArrayValueArray } ////////////////////////////////////// @@ -75,5 +76,5 @@ export namespace QueryContext { type Base = { guid: string, params: QueryParams } export type Error = { kind: 'Error', message: string } & Base export type Empty = { kind: 'Empty', data: DataContext } & Base - export type Data = { kind: 'Data', data: DataContext, samplingInfo: QuerySamplingInfo, values: DataFormat.ValueArray[] } & Base + export type Data = { kind: 'Data', data: DataContext, samplingInfo: QuerySamplingInfo, values: TypedArrayValueArray[] } & Base } \ No newline at end of file diff --git a/src/servers/volume/server/query/encode.ts b/src/servers/volume/server/query/encode.ts index f456c9edf3951c3c11c2e6d14497cecf1892cf09..008159dfdfc8995b3a1c330ead00176a96e9b524 100644 --- a/src/servers/volume/server/query/encode.ts +++ b/src/servers/volume/server/query/encode.ts @@ -13,6 +13,7 @@ import VERSION from '../version' import * as DataFormat from '../../common/data-format' import { Column } from 'mol-data/db'; import { ArrayEncoding, ArrayEncoder } from 'mol-io/common/binary-cif'; +import { TypedArrayValueType, TypedArrayValueArray } from 'mol-io/common/typed-array'; export default function encode(query: Data.QueryContext, output: Data.QueryOutputStream) { let w = CifWriter.createEncoder({ binary: query.params.asBinary, encoderName: `VolumeServer ${VERSION}` }); @@ -106,7 +107,7 @@ const _volume_data_3d_info: CifWriter.Category<ResultContext> = { } }; -function _volume_data_3d_number(i: number, ctx: DataFormat.ValueArray): number { +function _volume_data_3d_number(i: number, ctx: TypedArrayValueArray): number { return ctx[i]; } @@ -118,7 +119,7 @@ const _volume_data_3d: CifWriter.Category<ResultContext> = { const E = ArrayEncoding; let encoder: ArrayEncoder; let typedArray: any; - if (ctx.query.data.header.valueType === DataFormat.ValueType.Float32 || ctx.query.data.header.valueType === DataFormat.ValueType.Int16) { + if (ctx.query.data.header.valueType === TypedArrayValueType.Float32 || ctx.query.data.header.valueType === TypedArrayValueType.Int16) { let min: number, max: number; min = data[0], max = data[0]; for (let i = 0, n = data.length; i < n; i++) { diff --git a/src/servers/volume/server/query/execute.ts b/src/servers/volume/server/query/execute.ts index cf62baa880b7e561b04d18d8da6a5f059020fabb..0e76f5afdbbb25d4a5c4fbce3296894644a731f1 100644 --- a/src/servers/volume/server/query/execute.ts +++ b/src/servers/volume/server/query/execute.ts @@ -22,6 +22,7 @@ import { SpacegroupCell } from 'mol-math/geometry'; import { Vec3 } from 'mol-math/linear-algebra'; import { UUID } from 'mol-util'; import { FileHandle } from 'mol-io/common/file-handle'; +import { createTypedArray, TypedArrayValueType } from 'mol-io/common/typed-array'; export default async function execute(params: Data.QueryParams, outputProvider: () => Data.QueryOutputStream) { const start = getTime(); @@ -142,10 +143,10 @@ function getQueryBox(data: Data.DataContext, queryBox: Data.QueryParamsBox) { } } -function allocateValues(domain: Coords.GridDomain<'Query'>, numChannels: number, valueType: DataFormat.ValueType) { +function allocateValues(domain: Coords.GridDomain<'Query'>, numChannels: number, valueType: TypedArrayValueType) { const values = []; for (let i = 0; i < numChannels; i++) { - values[values.length] = DataFormat.createValueArray(valueType, domain.sampleVolume); + values[values.length] = createTypedArray(valueType, domain.sampleVolume); } return values; }