diff --git a/src/helpers.d.ts b/src/helpers.d.ts index 9ba96996e0f9eacc9c0ce3a3baa16f7861670723..654affd3f465a3c5d64f168acb7d89eac3b94b8a 100644 --- a/src/helpers.d.ts +++ b/src/helpers.d.ts @@ -9,7 +9,10 @@ declare module Helpers { export type Mutable<T> = { -readonly [P in keyof T]: T[P] } - export type TypedArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Uint8ClampedArray | Float32Array | Float64Array + export type TypedIntArray = Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array + export type TypedFloatArray = Float32Array | Float64Array + + export type TypedArray = TypedIntArray | TypedFloatArray export type NumberArray = TypedArray | number[] export type UintArray = Uint8Array | Uint16Array | Uint32Array | number[] export type ValueOf<T> = T[keyof T] diff --git a/src/mol-data/db/column.ts b/src/mol-data/db/column.ts index 55243135fd47da9c203215d019a012ab1c84dfe5..c993faa654210e2f6b1b0ae64145cbbe2908e4ee 100644 --- a/src/mol-data/db/column.ts +++ b/src/mol-data/db/column.ts @@ -10,7 +10,7 @@ import { Tensor as Tensors } from 'mol-math/linear-algebra' interface Column<T> { readonly schema: Column.Schema, - readonly '@array': ArrayLike<any> | undefined, + readonly __array: ArrayLike<any> | undefined, readonly isDefined: boolean, readonly rowCount: number, @@ -159,14 +159,14 @@ namespace Column { /** Makes the column backed by an array. Useful for columns that are accessed often. */ export function asArrayColumn<T>(c: Column<T>, array?: ArrayCtor<T>): Column<T> { - if (c['@array']) return c; + if (c.__array) return c; if (!c.isDefined) return Undefined(c.rowCount, c.schema) as any as Column<T>; return arrayColumn({ array: c.toArray({ array }), schema: c.schema, valueKind: c.valueKind }); } export function copyToArray<T extends number>(c: Column<T>, array: { [k: number]: T, length: number }, offset = 0) { if (!c.isDefined) return; - const cArray = c['@array'] + const cArray = c.__array if (cArray) { for (let i = 0, _i = cArray.length; i < _i; i++) array[offset + i] = cArray[i]; } else { @@ -200,7 +200,7 @@ function constColumn<T extends Column.Schema>(v: T['T'], rowCount: number, schem const value: Column<T['T']>['value'] = row => v; return { schema: schema, - '@array': void 0, + __array: void 0, isDefined: valueKind === Column.ValueKind.Present, rowCount, value, @@ -217,7 +217,7 @@ function constColumn<T extends Column.Schema>(v: T['T'], rowCount: number, schem function lambdaColumn<T extends Column.Schema>({ value, valueKind, rowCount, schema }: Column.LambdaSpec<T>): Column<T['T']> { return { schema: schema, - '@array': void 0, + __array: void 0, isDefined: true, rowCount, value, @@ -240,7 +240,7 @@ function arrayColumn<T extends Column.Schema>({ array, schema, valueKind }: Colu const isTyped = ColumnHelpers.isTypedArray(array); return { schema: schema, - '@array': array, + __array: array, isDefined: true, rowCount, value, @@ -271,12 +271,12 @@ function arrayColumn<T extends Column.Schema>({ array, schema, valueKind }: Colu function windowColumn<T>(column: Column<T>, start: number, end: number) { if (!column.isDefined) return Column.Undefined(end - start, column.schema); if (start === 0 && end === column.rowCount) return column; - if (!!column['@array'] && ColumnHelpers.isTypedArray(column['@array'])) return windowTyped(column, start, end); + if (!!column.__array && ColumnHelpers.isTypedArray(column.__array)) return windowTyped(column, start, end); return windowFull(column, start, end); } function windowTyped<T>(c: Column<T>, start: number, end: number): Column<T> { - const array = ColumnHelpers.typedArrayWindow(c['@array'], { start, end }); + const array = ColumnHelpers.typedArrayWindow(c.__array, { start, end }); return arrayColumn({ array, schema: c.schema, valueKind: c.valueKind }) as any; } @@ -286,7 +286,7 @@ function windowFull<T>(c: Column<T>, start: number, end: number): Column<T> { const rowCount = end - start; return { schema: c.schema, - '@array': void 0, + __array: void 0, isDefined: c.isDefined, rowCount, value, @@ -311,12 +311,12 @@ function isIdentity(map: ArrayLike<number>, rowCount: number) { function columnView<T>(c: Column<T>, map: ArrayLike<number>, checkIdentity: boolean): Column<T> { if (!c.isDefined) return c; if (checkIdentity && isIdentity(map, c.rowCount)) return c; - if (!!c['@array']) return arrayView(c, map); + if (!!c.__array) return arrayView(c, map); return viewFull(c, map); } function arrayView<T>(c: Column<T>, map: ArrayLike<number>): Column<T> { - const array = c['@array']!; + const array = c.__array!; const ret = new (array as any).constructor(map.length); for (let i = 0, _i = map.length; i < _i; i++) ret[i] = array[map[i]]; return arrayColumn({ array: ret, schema: c.schema, valueKind: c.valueKind }); @@ -328,7 +328,7 @@ function viewFull<T>(c: Column<T>, map: ArrayLike<number>): Column<T> { const rowCount = map.length; return { schema: c.schema, - '@array': void 0, + __array: void 0, isDefined: c.isDefined, rowCount, value, @@ -351,12 +351,12 @@ function mapToArrayImpl<T, S>(c: Column<T>, f: (v: T) => S, ctor: Column.ArrayCt function areColumnsEqual(a: Column<any>, b: Column<any>) { if (a === b) return true; if (a.rowCount !== b.rowCount || a.isDefined !== b.isDefined || a.schema.valueType !== b.schema.valueType) return false; - if (!!a['@array'] && !!b['@array']) return areArraysEqual(a, b); + if (!!a.__array && !!b.__array) return areArraysEqual(a, b); return areValuesEqual(a, b); } function areArraysEqual(a: Column<any>, b: Column<any>) { - const xs = a['@array']!, ys = b['@array']!; + const xs = a.__array!, ys = b.__array!; for (let i = 0, _i = a.rowCount; i < _i; i++) { if (xs[i] !== ys[i]) return false; } diff --git a/src/mol-io/common/binary-cif/array-encoder.ts b/src/mol-io/common/binary-cif/array-encoder.ts index c3c119494e898cc186031961009eb8b7a4d20a79..d1c10435313a5b617882d8c016a3e4a7660c6d42 100644 --- a/src/mol-io/common/binary-cif/array-encoder.ts +++ b/src/mol-io/common/binary-cif/array-encoder.ts @@ -100,7 +100,7 @@ export namespace ArrayEncoding { [Encoding.FloatDataType.Float64]: 8 } - export function byteArray(data: Encoding.FloatArray | Encoding.IntArray) { + export function byteArray(data: Encoding.TypedFloatArray | Encoding.TypedIntArray) { let type = Encoding.getDataType(data); if (type === Encoding.IntDataType.Int8) return int8(data as Int8Array); @@ -118,7 +118,7 @@ export namespace ArrayEncoding { }; } - function _fixedPoint(data: Encoding.FloatArray, factor: number): Result { + function _fixedPoint(data: Encoding.TypedFloatArray, factor: number): Result { let srcType = Encoding.getDataType(data) as Encoding.FloatDataType; let result = new Int32Array(data.length); for (let i = 0, n = data.length; i < n; i++) { @@ -129,9 +129,9 @@ export namespace ArrayEncoding { data: result }; } - export function fixedPoint(factor: number): Provider { return data => _fixedPoint(data as Encoding.FloatArray, factor); } + export function fixedPoint(factor: number): Provider { return data => _fixedPoint(data as Encoding.TypedFloatArray, factor); } - function _intervalQuantizaiton(data: Encoding.FloatArray, min: number, max: number, numSteps: number, arrayType: new (size: number) => Encoding.IntArray): Result { + function _intervalQuantizaiton(data: Encoding.TypedFloatArray, min: number, max: number, numSteps: number, arrayType: new (size: number) => Encoding.TypedIntArray): Result { let srcType = Encoding.getDataType(data) as Encoding.FloatDataType; if (!data.length) { return { @@ -161,11 +161,11 @@ export namespace ArrayEncoding { data: output }; } - export function intervalQuantizaiton(min: number, max: number, numSteps: number, arrayType: new (size: number) => Encoding.IntArray = Int32Array): Provider { - return data => _intervalQuantizaiton(data as Encoding.FloatArray, min, max, numSteps, arrayType); + export function intervalQuantizaiton(min: number, max: number, numSteps: number, arrayType: new (size: number) => Encoding.TypedIntArray = Int32Array): Provider { + return data => _intervalQuantizaiton(data as Encoding.TypedFloatArray, min, max, numSteps, arrayType); } - export function runLength(data: Encoding.IntArray): Result { + export function runLength(data: Encoding.TypedIntArray): Result { let srcType = Encoding.getDataType(data) as Encoding.IntDataType; if (srcType === void 0) { data = new Int32Array(data); diff --git a/src/mol-io/common/binary-cif/classifier.ts b/src/mol-io/common/binary-cif/classifier.ts new file mode 100644 index 0000000000000000000000000000000000000000..91f33a7d6bf6aaece9a38909e739f851e211ce99 --- /dev/null +++ b/src/mol-io/common/binary-cif/classifier.ts @@ -0,0 +1,173 @@ + +/** + * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info. + * + * @author David Sehnal <david.sehnal@gmail.com> + */ + +import { ArrayEncoder, ArrayEncoding as E } from './array-encoder'; +import { getArrayMantissaMultiplier } from 'mol-util/number'; + +export function classifyIntArray(xs: ArrayLike<number>) { + return IntClassifier.classify(xs as number[]); +} + +export function classifyFloatArray(xs: ArrayLike<number>) { + return FloatClassifier.classify(xs as number[]); +} + +namespace IntClassifier { + function packSize(value: number, upperLimit: number) { + return value >= 0 + ? Math.ceil((value + 1) / upperLimit) + : Math.ceil((value + 1) / (-upperLimit - 1)); + } + + type IntColumnInfo = { signed: boolean, limit8: number, limit16: number }; + + function getInfo(data: number[]): IntColumnInfo { + let signed = false; + for (let i = 0, n = data.length; i < n; i++) { + if (data[i] < 0) { + signed = true; + break; + } + } + return signed ? { signed, limit8: 0x7F, limit16: 0x7FFF } : { signed, limit8: 0xFF, limit16: 0xFFFF }; + } + + type SizeInfo = { pack8: number, pack16: number, count: number } + function SizeInfo(): SizeInfo { return { pack8: 0, pack16: 0, count: 0 } }; + + function incSize({ limit8, limit16 }: IntColumnInfo, info: SizeInfo, value: number) { + info.pack8 += packSize(value, limit8); + info.pack16 += packSize(value, limit16); + info.count += 1; + } + + function incSizeSigned(info: SizeInfo, value: number) { + info.pack8 += packSize(value, 0x7F); + info.pack16 += packSize(value, 0x7FFF); + info.count += 1; + } + + function byteSize(info: SizeInfo) { + if (info.count * 4 < info.pack16 * 2) return { length: info.count * 4, elem: 4 }; + if (info.pack16 * 2 < info.pack8) return { length: info.pack16 * 2, elem: 2 }; + return { length: info.pack8, elem: 1 }; + } + + function packingSize(data: number[], info: IntColumnInfo) { + const size = SizeInfo(); + for (let i = 0, n = data.length; i < n; i++) { + incSize(info, size, data[i]); + } + return { ...byteSize(size), kind: 'pack' }; + } + + function deltaSize(data: number[], info: IntColumnInfo) { + const size = SizeInfo(); + let prev = data[0]; + for (let i = 1, n = data.length; i < n; i++) { + incSizeSigned(size, data[i] - prev); + prev = data[i]; + } + return { ...byteSize(size), kind: 'delta' }; + } + + function rleSize(data: number[], info: IntColumnInfo) { + const size = SizeInfo(); + let run = 1; + for (let i = 1, n = data.length; i < n; i++) { + if (data[i - 1] !== data[i]) { + incSize(info, size, data[i - 1]); + incSize(info, size, run); + run = 1; + } else { + run++; + } + } + incSize(info, size, data[data.length - 1]); + incSize(info, size, run); + + return { ...byteSize(size), kind: 'rle' }; + } + + function deltaRleSize(data: number[], info: IntColumnInfo) { + const size = SizeInfo(); + let run = 1, prev = 0, prevValue = 0; + for (let i = 1, n = data.length; i < n; i++) { + const v = data[i] - prev; + if (prevValue !== v) { + incSizeSigned(size, prevValue); + incSizeSigned(size, run); + run = 1; + } else { + run++; + } + prevValue = v; + prev = data[i]; + } + incSizeSigned(size, prevValue); + incSizeSigned(size, run); + + return { ...byteSize(size), kind: 'delta-rle' }; + } + + export function getSize(data: number[]) { + const info = getInfo(data); + const sizes = [packingSize(data, info), rleSize(data, info), deltaSize(data, info), deltaRleSize(data, info)]; + sizes.sort((a, b) => a.length - b.length); + return sizes; + } + + export function classify(data: number[]): ArrayEncoder { + if (data.length < 2) return E.by(E.byteArray); + + const sizes = getSize(data); + const size = sizes[0]; + + switch (size.kind) { + case 'pack': return E.by(E.integerPacking); + case 'rle': return E.by(E.runLength).and(E.integerPacking); + case 'delta': return E.by(E.delta).and(E.integerPacking); + case 'delta-rle': return E.by(E.delta).and(E.runLength).and(E.integerPacking); + } + + throw new Error('should not happen :)'); + } +} + +namespace FloatClassifier { + const delta = 1e-6; + export function classify(data: number[]) { + const digitCount = getArrayMantissaMultiplier(data, 4, delta); + if (digitCount < 0) return { encoder: E.by(E.byteArray), typedArray: Float64Array }; + + // TODO: check for overflows here? + if (digitCount === 1) return { encoder: IntClassifier.classify(data), typedArray: Int32Array } + + const intArray = new Int32Array(data.length); + for (let i = 0, n = data.length; i < n; i++) { + const v = digitCount * data[i]; + intArray[i] = v; + // check if the value didn't overflow + if (Math.abs(Math.round(v) / digitCount - intArray[i] / digitCount) > delta) { + return { encoder: E.by(E.byteArray), typedArray: Float64Array }; + } + } + + const sizes = IntClassifier.getSize(intArray as any); + const size = sizes[0]; + + const fp = E.by(E.fixedPoint(digitCount)); + switch (size.kind) { + case 'pack': return { encoder: fp.and(E.integerPacking), typedArray: Float32Array }; + case 'rle': return { encoder: fp.and(E.runLength).and(E.integerPacking), typedArray: Float32Array }; + case 'delta': return { encoder: fp.and(E.delta).and(E.integerPacking), typedArray: Float32Array }; + case 'delta-rle': return { encoder: fp.and(E.delta).and(E.runLength).and(E.integerPacking), typedArray: Float32Array }; + } + + throw new Error('should not happen :)'); + } +} \ No newline at end of file diff --git a/src/mol-io/common/binary-cif/encoding.ts b/src/mol-io/common/binary-cif/encoding.ts index f36abc52790b011b41a9737f78636b781f2f73c3..2194d318f3014f16673bd3cfa487acbad58bafb1 100644 --- a/src/mol-io/common/binary-cif/encoding.ts +++ b/src/mol-io/common/binary-cif/encoding.ts @@ -71,10 +71,10 @@ export namespace Encoding { export type DataType = IntDataType | FloatDataType - export type IntArray = Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array - export type FloatArray = Float32Array | Float64Array + export type TypedIntArray = Helpers.TypedIntArray + export type TypedFloatArray = Helpers.TypedFloatArray - export function getDataType(data: IntArray | FloatArray): DataType { + export function getDataType(data: TypedIntArray | TypedFloatArray): DataType { let srcType: DataType; if (data instanceof Int8Array) srcType = Encoding.IntDataType.Int8; else if (data instanceof Int16Array) srcType = Encoding.IntDataType.Int16; @@ -88,7 +88,7 @@ export namespace Encoding { return srcType; } - export function isSignedIntegerDataType(data: IntArray) { + export function isSignedIntegerDataType(data: TypedIntArray) { return data instanceof Int8Array || data instanceof Int16Array || data instanceof Int32Array; } diff --git a/src/mol-io/reader/cif/binary/field.ts b/src/mol-io/reader/cif/binary/field.ts index af596d70a9909fd459d077a4c1aa4c21a22b0afd..d0f93cba523d4be6392ec090d083040c36da22d8 100644 --- a/src/mol-io/reader/cif/binary/field.ts +++ b/src/mol-io/reader/cif/binary/field.ts @@ -38,7 +38,7 @@ export default function Field(column: EncodedColumn): Data.CifField { const rowCount = data.length; return { - '@array': data, + __array: data, isDefined: true, rowCount, str, diff --git a/src/mol-io/reader/cif/data-model.ts b/src/mol-io/reader/cif/data-model.ts index 6d87495f2c4de2d57a8f2e6bcecf624ff04d1e90..63ffe3c8fa4e44379fa003abf2cdd8a5622d9b9d 100644 --- a/src/mol-io/reader/cif/data-model.ts +++ b/src/mol-io/reader/cif/data-model.ts @@ -62,7 +62,7 @@ export namespace CifCategory { * This is to ensure that the functions can invoked without having to "bind" them. */ export interface CifField { - readonly '@array': ArrayLike<any> | undefined + readonly __array: ArrayLike<any> | undefined readonly isDefined: boolean, readonly rowCount: number, diff --git a/src/mol-io/reader/cif/schema.ts b/src/mol-io/reader/cif/schema.ts index 8efa52ab41b2a0c2047ad7cb9149751d44fd7610..b2094be896cbf12019aba457c6f6b4891718efd2 100644 --- a/src/mol-io/reader/cif/schema.ts +++ b/src/mol-io/reader/cif/schema.ts @@ -41,7 +41,7 @@ function getColumnCtor(t: Column.Schema): ColumnCtor { function createColumn<T>(schema: Column.Schema, field: Data.CifField, value: (row: number) => T, toArray: Column<T>['toArray']): Column<T> { return { schema, - '@array': field['@array'], + __array: field.__array, isDefined: field.isDefined, rowCount: field.rowCount, value, @@ -61,7 +61,7 @@ function createListColumn<T extends number|string>(schema: Column.Schema.List<T> return { schema, - '@array': void 0, + __array: void 0, isDefined: !!f, rowCount: category.rowCount, value, @@ -89,7 +89,7 @@ function createTensorColumn(schema: Column.Schema.Tensor, category: Data.CifCate return { schema, - '@array': void 0, + __array: void 0, isDefined: first.isDefined, rowCount: category.rowCount, value, diff --git a/src/mol-io/reader/cif/text/field.ts b/src/mol-io/reader/cif/text/field.ts index 32af80a85747eb0c8499d1a76b58689a86b721ed..71f60d03a455215bafe2196ced5510a0de77cbb6 100644 --- a/src/mol-io/reader/cif/text/field.ts +++ b/src/mol-io/reader/cif/text/field.ts @@ -38,7 +38,7 @@ export default function CifTextField(tokens: Tokens, rowCount: number): Data.Cif }; return { - '@array': void 0, + __array: void 0, isDefined: true, rowCount, str, diff --git a/src/mol-io/reader/common/text/column/fixed.ts b/src/mol-io/reader/common/text/column/fixed.ts index 7316f5234e4089cae76241c6ff68937676556257..22fed914a172cf869c9746cf406d5c6f4af03383 100644 --- a/src/mol-io/reader/common/text/column/fixed.ts +++ b/src/mol-io/reader/common/text/column/fixed.ts @@ -35,7 +35,7 @@ export function FixedColumn<T extends Column.Schema>(lines: Tokens, offset: numb }; return { schema: schema, - '@array': void 0, + __array: void 0, isDefined: true, rowCount, value, diff --git a/src/mol-io/reader/common/text/column/token.ts b/src/mol-io/reader/common/text/column/token.ts index fb828354d88e7092c5b6a345cf1bf7979a770e97..46186a5c5c645d2e73737f9f6627ee04eefeefa6 100644 --- a/src/mol-io/reader/common/text/column/token.ts +++ b/src/mol-io/reader/common/text/column/token.ts @@ -27,7 +27,7 @@ export function TokenColumn<T extends Column.Schema>(tokens: Tokens, schema: T): return { schema: schema, - '@array': void 0, + __array: void 0, isDefined: true, rowCount, value, diff --git a/src/mol-script/script/mol-script/symbols.ts b/src/mol-script/script/mol-script/symbols.ts index 245e543e0441b163187afd4e919829d254ef6b6e..a647c8d0ce12bef1b162bbaba283c1ccf5b38eee 100644 --- a/src/mol-script/script/mol-script/symbols.ts +++ b/src/mol-script/script/mol-script/symbols.ts @@ -4,14 +4,12 @@ * @author David Sehnal <david.sehnal@gmail.com> */ -import { MSymbol, Arguments, Argument } from '../../language/symbol' -import { MolScriptBuilder as B } from '../../language/builder' +import { UniqueArray } from 'mol-data/generic'; +import Expression from '../../language/expression'; +import { Argument, MSymbol } from '../../language/symbol'; //import * as M from './macro' -import { MolScriptSymbolTable as MolScript } from '../../language/symbol-table' -import Type from '../../language/type' -import * as Struct from '../../language/symbol-table/structure-query' -import Expression from '../../language/expression' -import { UniqueArray } from 'mol-data/generic' +import { MolScriptSymbolTable as MolScript } from '../../language/symbol-table'; +import Type from '../../language/type'; export type MolScriptSymbol = | { kind: 'alias', aliases: string[], symbol: MSymbol } diff --git a/src/mol-util/number.ts b/src/mol-util/number.ts new file mode 100644 index 0000000000000000000000000000000000000000..3d87c41e147090f81e1bd87b02b57539433c9dd4 --- /dev/null +++ b/src/mol-util/number.ts @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info. + */ + +/** + * Determine the number of digits in a floating point number + * Find a number M such that round(M * v) - M * v <= delta. + * If no such M exists, return -1. + */ +export function getMantissaMultiplier(v: number, maxDigits: number, delta: number) { + let m = 1; + for (let i = 0; i < maxDigits; i++) { + let mv = m * v; + if (Math.abs(Math.round(mv) - mv) <= delta) return m; + m *= 10; + } + return -1; +} + +/** + * Determine the maximum number of digits in a floating point array. + * Find a number M such that round(M * v) - M * v <= delta. + * If no such M exists, return -1. + */ +export function getArrayMantissaMultiplier(xs: ArrayLike<number>, maxDigits: number, delta: number) { + let m = 1; + for (let i = 0, _i = xs.length; i < _i; i++) { + const t = getMantissaMultiplier(xs[i], maxDigits, delta); + if (t < 0) return -1; + if (t > m) m = t; + } + return m; +} \ No newline at end of file