Skip to content
Snippets Groups Projects
Commit a312f3e7 authored by David Sehnal's avatar David Sehnal
Browse files

refactoring of column and CIF data model

parent ecb0f8c4
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 93 additions and 107 deletions
src/mol-base/collections/_spec/table.spec.ts
+ 2
− 2
View file @ a312f3e7
......@@ -4,7 +4,7 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
import Column, { isTypedArray } from '../column'
import Column, { ColumnHelpers } from '../column'
import Table from '../table'
describe('column', () => {
......@@ -32,7 +32,7 @@ describe('column', () => {
it('typed', () => {
expect(typedWindow.value(0)).toBe(2);
expect(typedWindow.rowCount).toBe(2);
expect(isTypedArray(typedWindow.toArray())).toBe(true);
expect(ColumnHelpers.isTypedArray(typedWindow.toArray())).toBe(true);
});
it('numStr', () => {
......
src/mol-base/collections/column.ts
+ 68
− 50
View file @ a312f3e7
......@@ -89,6 +89,11 @@ namespace Column {
return columnPermutation(column, indices, checkIndentity);
}
/** A map of the 1st occurence of each value. */
export function createFirstIndexMap<T>(column: Column<T>) {
return createFirstIndexMapOfColumn(column);
}
/** Makes the column backned by an array. Useful for columns that accessed often. */
export function asArrayColumn<T>(c: Column<T>, array?: ToArrayParams['array']): Column<T> {
if (c['@array']) return c;
......@@ -99,6 +104,15 @@ namespace Column {
export default Column;
function createFirstIndexMapOfColumn<T>(c: Column<T>): Map<T, number> | undefined {
const map = new Map<T, number>();
for (let i = 0, _i = c.rowCount; i < _i; i++) {
const v = c.value(i);
if (!map.has(v)) return map.set(c.value(i), i);
}
return map;
}
function constColumn<T extends Column.Type>(v: T['T'], rowCount: number, type: T, valueKind: Column.ValueKind): Column<T['T']> {
const value: Column<T['T']>['value'] = row => v;
return {
......@@ -109,7 +123,7 @@ function constColumn<T extends Column.Type>(v: T['T'], rowCount: number, type: T
value,
valueKind: row => valueKind,
toArray: params => {
const { array } = createArray(rowCount, params);
const { array } = ColumnHelpers.createArray(rowCount, params);
for (let i = 0, _i = array.length; i < _i; i++) array[i] = v;
return array;
},
......@@ -126,7 +140,7 @@ function lambdaColumn<T extends Column.Type>({ value, valueKind, rowCount, type
value,
valueKind: valueKind ? valueKind : row => Column.ValueKind.Present,
toArray: params => {
const { array, start } = createArray(rowCount, params);
const { array, start } = ColumnHelpers.createArray(rowCount, params);
for (let i = 0, _i = array.length; i < _i; i++) array[i] = value(i + start);
return array;
},
......@@ -140,7 +154,7 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
? row => { const v = array[row]; return typeof v === 'string' ? v : '' + v; }
: row => array[row];
const isTyped = isTypedArray(array);
const isTyped = ColumnHelpers.isTypedArray(array);
return {
'@type': type,
'@array': array,
......@@ -150,7 +164,7 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
valueKind: valueKind ? valueKind : row => Column.ValueKind.Present,
toArray: type.kind === 'str'
? params => {
const { start, end } = getArrayBounds(rowCount, params);
const { start, end } = ColumnHelpers.getArrayBounds(rowCount, params);
const ret = new (params && typeof params.array !== 'undefined' ? params.array : (array as any).constructor)(end - start) as any;
for (let i = 0, _i = end - start; i < _i; i++) {
const v = array[start + i];
......@@ -159,9 +173,9 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
return ret;
}
: isTyped
? params => typedArrayWindow(array, params) as any as ReadonlyArray<T>
? params => ColumnHelpers.typedArrayWindow(array, params) as any as ReadonlyArray<T>
: params => {
const { start, end } = getArrayBounds(rowCount, params);
const { start, end } = ColumnHelpers.getArrayBounds(rowCount, params);
if (start === 0 && end === array.length) return array as ReadonlyArray<T['T']>;
const ret = new (params && typeof params.array !== 'undefined' ? params.array : (array as any).constructor)(end - start) as any;
for (let i = 0, _i = end - start; i < _i; i++) ret[i] = array[start + i];
......@@ -173,12 +187,12 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
function windowColumn<T>(column: Column<T>, start: number, end: number) {
if (!column.isDefined) return Column.Undefined(end - start, column['@type']);
if (column['@array'] && isTypedArray(column['@array'])) return windowTyped(column, start, end);
if (column['@array'] && ColumnHelpers.isTypedArray(column['@array'])) return windowTyped(column, start, end);
return windowFull(column, start, end);
}
function windowTyped<T>(c: Column<T>, start: number, end: number): Column<T> {
const array = typedArrayWindow(c['@array'], { start, end });
const array = ColumnHelpers.typedArrayWindow(c['@array'], { start, end });
return arrayColumn({ array, type: c['@type'], valueKind: c.valueKind }) as any;
}
......@@ -194,7 +208,7 @@ function windowFull<T>(c: Column<T>, start: number, end: number): Column<T> {
value,
valueKind: start === 0 ? vk : row => vk(row + start),
toArray: params => {
const { array } = createArray(rowCount, params);
const { array } = ColumnHelpers.createArray(rowCount, params);
for (let i = 0, _i = array.length; i < _i; i++) array[i] = v(i + start);
return array;
},
......@@ -202,21 +216,27 @@ function windowFull<T>(c: Column<T>, start: number, end: number): Column<T> {
};
}
function isIdentity(map: ArrayLike<number>) {
for (let i = 0, _i = map.length; i < _i; i++) {
if (map[i] !== i) return false;
}
return true;
}
function columnPermutation<T>(c: Column<T>, map: ArrayLike<number>, checkIdentity: boolean): Column<T> {
if (!c.isDefined) return c;
if (checkIdentity) {
let isIdentity = true;
for (let i = 0, _i = map.length; i < _i; i++) {
if (map[i] !== i) {
isIdentity = false;
break;
}
}
if (isIdentity) return c;
}
if (checkIdentity && isIdentity(map)) return c;
if (!c['@array']) return permutationArray(c, map);
return permutationFull(c, map);
}
function permutationArray<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
const array = c['@array']!;
const ret = new (array as any).constructor(c.rowCount);
for (let i = 0, _i = c.rowCount; i < _i; i++) ret[i] = array[map[i]];
return arrayColumn({ array: ret, type: c['@type'], valueKind: c.valueKind });
}
function permutationFull<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
const v = c.value, vk = c.valueKind, ave = c.areValuesEqual;
const value: Column<T>['value'] = row => v(map[row]);
......@@ -229,7 +249,7 @@ function permutationFull<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
value,
valueKind: row => vk(map[row]),
toArray: params => {
const { array } = createArray(rowCount, params);
const { array } = ColumnHelpers.createArray(rowCount, params);
for (let i = 0, _i = array.length; i < _i; i++) array[i] = v(map[i]);
return array;
},
......@@ -237,39 +257,37 @@ function permutationFull<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
};
}
/** A helped function for Column.toArray */
export function getArrayBounds(rowCount: number, params?: Column.ToArrayParams) {
const start = params && typeof params.start !== 'undefined' ? Math.max(Math.min(params.start, rowCount - 1), 0) : 0;
const end = params && typeof params.end !== 'undefined' ? Math.min(params.end, rowCount) : rowCount;
return { start, end };
}
export namespace ColumnHelpers {
export function getArrayBounds(rowCount: number, params?: Column.ToArrayParams) {
const start = params && typeof params.start !== 'undefined' ? Math.max(Math.min(params.start, rowCount - 1), 0) : 0;
const end = params && typeof params.end !== 'undefined' ? Math.min(params.end, rowCount) : rowCount;
return { start, end };
}
/** A helped function for Column.toArray */
export function createArray(rowCount: number, params?: Column.ToArrayParams) {
const c = params && typeof params.array !== 'undefined' ? params.array : Array;
const { start, end } = getArrayBounds(rowCount, params);
return { array: new c(end - start) as any[], start, end };
}
export function createArray(rowCount: number, params?: Column.ToArrayParams) {
const c = params && typeof params.array !== 'undefined' ? params.array : Array;
const { start, end } = getArrayBounds(rowCount, params);
return { array: new c(end - start) as any[], start, end };
}
/** A helped function for Column.toArray */
export function fillArrayValues(value: (row: number) => any, target: any[], start: number) {
for (let i = 0, _e = target.length; i < _e; i++) target[i] = value(start + i);
return target;
}
export function fillArrayValues(value: (row: number) => any, target: any[], start: number) {
for (let i = 0, _e = target.length; i < _e; i++) target[i] = value(start + i);
return target;
}
/** A helped function for Column.toArray */
export function createAndFillArray(rowCount: number, value: (row: number) => any, params?: Column.ToArrayParams) {
const { array, start } = createArray(rowCount, params);
return fillArrayValues(value, array, start);
}
export function createAndFillArray(rowCount: number, value: (row: number) => any, params?: Column.ToArrayParams) {
const { array, start } = createArray(rowCount, params);
return fillArrayValues(value, array, start);
}
export function isTypedArray(data: any): boolean {
return !!data.buffer && typeof data.byteLength === 'number' && typeof data.BYTES_PER_ELEMENT === 'number';
}
export function isTypedArray(data: any): boolean {
return !!data.buffer && typeof data.byteLength === 'number' && typeof data.BYTES_PER_ELEMENT === 'number';
}
export function typedArrayWindow(data: any, params?: Column.ToArrayParams): ReadonlyArray<number> {
const { constructor, buffer, length, byteOffset, BYTES_PER_ELEMENT } = data;
const { start, end } = getArrayBounds(length, params);
if (start === 0 && end === length) return data;
return new constructor(buffer, byteOffset + BYTES_PER_ELEMENT * start, Math.min(length, end - start));
export function typedArrayWindow(data: any, params?: Column.ToArrayParams): ReadonlyArray<number> {
const { constructor, buffer, length, byteOffset, BYTES_PER_ELEMENT } = data;
const { start, end } = getArrayBounds(length, params);
if (start === 0 && end === length) return data;
return new constructor(buffer, byteOffset + BYTES_PER_ELEMENT * start, Math.min(length, end - start));
}
}
\ No newline at end of file
src/mol-io/reader/_spec/column.spec.ts
+ 3
− 3
View file @ a312f3e7
......@@ -7,7 +7,7 @@
import FixedColumn from '../common/text/column/fixed'
import TokenColumn from '../common/text/column/token'
import Column, { typedArrayWindow } from '../../../mol-base/collections/column'
import Column, { ColumnHelpers } from '../../../mol-base/collections/column'
const lines = [
'1.123 abc',
......@@ -64,8 +64,8 @@ describe('token text column', () => {
describe('binary column', () => {
it('window works', () => {
const xs = new Float64Array([1, 2, 3, 4]);
const w1 = typedArrayWindow(xs, { start: 1 });
const w2 = typedArrayWindow(xs, { start: 2, end: 4 });
const w1 = ColumnHelpers.typedArrayWindow(xs, { start: 1 });
const w2 = ColumnHelpers.typedArrayWindow(xs, { start: 2, end: 4 });
expect(w1.length).toBe(3);
for (let i = 0; i < w1.length; i++) expect(w1[i]).toBe(xs[i + 1]);
......
src/mol-io/reader/cif/binary/field.ts
+ 7
− 8
View file @ a312f3e7
......@@ -4,7 +4,7 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
import Column, { isTypedArray, createAndFillArray, typedArrayWindow } from '../../../../mol-base/collections/column'
import Column, { ColumnHelpers } from '../../../../mol-base/collections/column'
import * as Data from '../data-model'
import { EncodedColumn } from './encoding'
import decode from './decoder'
......@@ -13,7 +13,7 @@ import { parseInt as fastParseInt, parseFloat as fastParseFloat } from '../../co
export default function Field(column: EncodedColumn): Data.Field {
const mask = column.mask ? decode(column.mask) as number[] : void 0;
const data = decode(column.data);
const isNumeric = isTypedArray(data);
const isNumeric = ColumnHelpers.isTypedArray(data);
const str: Data.Field['str'] = isNumeric
? mask
......@@ -46,13 +46,12 @@ export default function Field(column: EncodedColumn): Data.Field {
float,
valueKind,
areValuesEqual: (rowA, rowB) => data[rowA] === data[rowB],
stringEquals: (row, v) => str(row) === v,
toStringArray: params => createAndFillArray(rowCount, str, params),
toStringArray: params => ColumnHelpers.createAndFillArray(rowCount, str, params),
toIntArray: isNumeric
? params => typedArrayWindow(data, params)
: params => createAndFillArray(rowCount, int, params),
? params => ColumnHelpers.typedArrayWindow(data, params)
: params => ColumnHelpers.createAndFillArray(rowCount, int, params),
toFloatArray: isNumeric
? params => typedArrayWindow(data, params)
: params => createAndFillArray(rowCount, float, params)
? params => ColumnHelpers.typedArrayWindow(data, params)
: params => ColumnHelpers.createAndFillArray(rowCount, float, params)
};
}
\ No newline at end of file
src/mol-io/reader/cif/binary/parser.ts
+ 1
− 1
View file @ a312f3e7
......@@ -25,7 +25,7 @@ function Category(data: Encoding.EncodedCategory): Data.Category {
rowCount: data.rowCount,
getField(name) {
const col = map[name];
return col ? Field(col) : Data.DefaultUndefinedField(data.rowCount);
return col ? Field(col) : void 0;
}
}
}
......
src/mol-io/reader/cif/data-model.ts
+ 1
− 21
View file @ a312f3e7
......@@ -4,7 +4,7 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
import Column, { createArray } from '../../../mol-base/collections/column'
import Column from '../../../mol-base/collections/column'
export interface File {
readonly name?: string,
......@@ -68,32 +68,12 @@ export interface Field {
valueKind(row: number): Column.ValueKind,
areValuesEqual(rowA: number, rowB: number): boolean,
stringEquals(row: number, value: string): boolean,
toStringArray(params?: Column.ToArrayParams): ReadonlyArray<string>,
toIntArray(params?: Column.ToArrayParams): ReadonlyArray<number>,
toFloatArray(params?: Column.ToArrayParams): ReadonlyArray<number>
}
export function DefaultUndefinedField(rowCount: number): Field {
return {
'@array': void 0,
isDefined: false,
rowCount,
str: row => '',
int: row => 0,
float: row => 0,
valueKind: row => Column.ValueKind.NotPresent,
areValuesEqual: (rowA, rowB) => true,
stringEquals: (row, value) => value === null,
toStringArray: (p) => createArray(rowCount, p).array,
toIntArray: (p) => createArray(rowCount, p).array,
toFloatArray: (p) => createArray(rowCount, p).array
};
}
export function getMatrix(category: Category, field: string, rows: number, cols: number, row: number) {
const ret: number[][] = [];
for (let i = 0; i < rows; i++) {
......
src/mol-io/reader/cif/schema.ts
+ 3
− 3
View file @ a312f3e7
......@@ -5,7 +5,7 @@
*/
import * as Data from './data-model'
import Column, { createAndFillArray } from '../../../mol-base/collections/column'
import Column, { ColumnHelpers } from '../../../mol-base/collections/column'
import Table from '../../../mol-base/collections/table'
export function toTypedFrame<Schema extends FrameSchema, Frame extends TypedFrame<Schema> = TypedFrame<Schema>>(schema: Schema, frame: Data.Frame): Frame {
......@@ -34,12 +34,12 @@ function getColumnCtor(t: Column.Type): ColumnCtor {
case 'vector': return (f, c, k) => {
const dim = t.dim;
const value = (row: number) => Data.getVector(c, k, dim, row);
return createColumn(t, f, value, params => createAndFillArray(f.rowCount, value, params));
return createColumn(t, f, value, params => ColumnHelpers.createAndFillArray(f.rowCount, value, params));
}
case 'matrix': return (f, c, k) => {
const rows = t.rows, cols = t.cols;
const value = (row: number) => Data.getMatrix(c, k, rows, cols, row);
return createColumn(t, f, value, params => createAndFillArray(f.rowCount, value, params));
return createColumn(t, f, value, params => ColumnHelpers.createAndFillArray(f.rowCount, value, params));
}
}
}
......
src/mol-io/reader/cif/text/field.ts
+ 4
− 15
View file @ a312f3e7
......@@ -4,7 +4,7 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
import Column, { createAndFillArray } from '../../../../mol-base/collections/column'
import Column, { ColumnHelpers } from '../../../../mol-base/collections/column'
import * as TokenColumn from '../../common/text/column/token'
import { Tokens } from '../../common/text/tokenizer'
import * as Data from '../data-model'
......@@ -45,19 +45,8 @@ export default function CifTextField(tokens: Tokens, rowCount: number): Data.Fie
float,
valueKind,
areValuesEqual: TokenColumn.areValuesEqualProvider(tokens),
stringEquals: (row, v) => {
const s = indices[2 * row];
const value = v || '';
if (!value && valueKind(row) !== Column.ValueKind.Present) return true;
const len = value.length;
if (len !== indices[2 * row + 1] - s) return false;
for (let i = 0; i < len; i++) {
if (data.charCodeAt(i + s) !== value.charCodeAt(i)) return false;
}
return true;
},
toStringArray: params => createAndFillArray(rowCount, str, params),
toIntArray: params => createAndFillArray(rowCount, int, params),
toFloatArray: params => createAndFillArray(rowCount, float, params)
toStringArray: params => ColumnHelpers.createAndFillArray(rowCount, str, params),
toIntArray: params => ColumnHelpers.createAndFillArray(rowCount, int, params),
toFloatArray: params => ColumnHelpers.createAndFillArray(rowCount, float, params)
}
}
\ No newline at end of file
src/mol-io/reader/common/text/column/fixed.ts
+ 2
− 2
View file @ a312f3e7
......@@ -4,7 +4,7 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
import Column, { createAndFillArray } from '../../../../../mol-base/collections/column'
import Column, { ColumnHelpers } from '../../../../../mol-base/collections/column'
import { trimStr, Tokens } from '../tokenizer'
import { parseIntSkipLeadingWhitespace, parseFloatSkipLeadingWhitespace } from '../number-parser'
......@@ -40,7 +40,7 @@ export function FixedColumn<T extends Column.Type>(lines: Tokens, offset: number
rowCount,
value,
valueKind: row => Column.ValueKind.Present,
toArray: params => createAndFillArray(rowCount, value, params),
toArray: params => ColumnHelpers.createAndFillArray(rowCount, value, params),
areValuesEqual: (rowA, rowB) => value(rowA) === value(rowB)
};
}
\ No newline at end of file
src/mol-io/reader/common/text/column/token.ts
+ 2
− 2
View file @ a312f3e7
......@@ -4,7 +4,7 @@
* @author David Sehnal <david.sehnal@gmail.com>
*/
import Column, { createAndFillArray } from '../../../../../mol-base/collections/column'
import Column, { ColumnHelpers } from '../../../../../mol-base/collections/column'
import { Tokens } from '../tokenizer'
import { parseInt as fastParseInt, parseFloat as fastParseFloat } from '../number-parser'
......@@ -32,7 +32,7 @@ export function TokenColumn<T extends Column.Type>(tokens: Tokens, type: T): Col
rowCount,
value,
valueKind: row => Column.ValueKind.Present,
toArray: params => createAndFillArray(rowCount, value, params),
toArray: params => ColumnHelpers.createAndFillArray(rowCount, value, params),
areValuesEqual: areValuesEqualProvider(tokens)
};
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment