Skip to content
Snippets Groups Projects
Commit bdba5847 authored by David Sehnal's avatar David Sehnal
Browse files

Array based column

parent db3b6aa3
No related branches found
No related tags found
No related merge requests found
...@@ -8,13 +8,12 @@ import * as Column from '../../common/column' ...@@ -8,13 +8,12 @@ import * as Column from '../../common/column'
import * as Data from '../data-model' import * as Data from '../data-model'
import { EncodedColumn } from './encoding' import { EncodedColumn } from './encoding'
import decode from './decoder' import decode from './decoder'
import { isTypedArray, typedArrayWindow } from '../../common/binary/column'
import { parseInt as fastParseInt, parseFloat as fastParseFloat } from '../../common/text/number-parser' import { parseInt as fastParseInt, parseFloat as fastParseFloat } from '../../common/text/number-parser'
export default function Field(column: EncodedColumn): Data.Field { export default function Field(column: EncodedColumn): Data.Field {
const mask = column.mask ? decode(column.mask) as number[] : void 0; const mask = column.mask ? decode(column.mask) as number[] : void 0;
const data = decode(column.data); const data = decode(column.data);
const isNumeric = isTypedArray(data); const isNumeric = Column.isTypedArray(data);
const str: Data.Field['str'] = isNumeric const str: Data.Field['str'] = isNumeric
? mask ? mask
...@@ -49,10 +48,10 @@ export default function Field(column: EncodedColumn): Data.Field { ...@@ -49,10 +48,10 @@ export default function Field(column: EncodedColumn): Data.Field {
stringEquals: (row, v) => str(row) === v, stringEquals: (row, v) => str(row) === v,
toStringArray: params => Column.createAndFillArray(rowCount, str, params), toStringArray: params => Column.createAndFillArray(rowCount, str, params),
toIntArray: isNumeric toIntArray: isNumeric
? params => typedArrayWindow(data, params) ? params => Column.typedArrayWindow(data, params)
: params => Column.createAndFillArray(rowCount, int, params), : params => Column.createAndFillArray(rowCount, int, params),
toFloatArray: isNumeric toFloatArray: isNumeric
? params => typedArrayWindow(data, params) ? params => Column.typedArrayWindow(data, params)
: params => Column.createAndFillArray(rowCount, float, params) : params => Column.createAndFillArray(rowCount, float, params)
}; };
} }
\ No newline at end of file
...@@ -3,16 +3,3 @@ ...@@ -3,16 +3,3 @@
* *
* @author David Sehnal <david.sehnal@gmail.com> * @author David Sehnal <david.sehnal@gmail.com>
*/ */
import { getArrayBounds, ToArrayParams } from '../column'
export function isTypedArray(data: any) {
return data.buffer && typeof data.byteLength === 'number' && data.BYTES_PER_ELEMENT;
}
export function typedArrayWindow(data: any, params?: ToArrayParams): ReadonlyArray<number> {
const { constructor, buffer, length, byteOffset, BYTES_PER_ELEMENT } = data;
const { start, end } = getArrayBounds(length, params);
if (start === 0 && end === length) return data;
return new constructor(buffer, byteOffset + BYTES_PER_ELEMENT * start, Math.min(length, end - start));
}
\ No newline at end of file
...@@ -37,14 +37,42 @@ export function UndefinedColumn<T extends ColumnType>(rowCount: number, type: T) ...@@ -37,14 +37,42 @@ export function UndefinedColumn<T extends ColumnType>(rowCount: number, type: T)
isDefined: false, isDefined: false,
rowCount, rowCount,
value, value,
isValueDefined(row) { return false; }, isValueDefined: row => false,
toArray(params) { toArray: params => {
const { array } = createArray(rowCount, params); const { array } = createArray(rowCount, params);
for (let i = 0, _i = array.length; i < _i; i++) array[i] = value(0) for (let i = 0, _i = array.length; i < _i; i++) array[i] = value(0)
return array; return array;
}, },
stringEquals(row, value) { return !value; }, stringEquals: (row, value) => !value,
areValuesEqual(rowA, rowB) { return true; } areValuesEqual: (rowA, rowB) => true
}
}
export function ArrayColumn<T>(array: ArrayLike<T>): Column<T> {
const rowCount = array.length;
const value: Column<T>['value'] = row => array[row];
const isTyped = isTypedArray(array);
return {
isDefined: false,
rowCount,
value,
isValueDefined: row => true,
toArray: isTyped
? params => typedArrayWindow(array, params) as any as ReadonlyArray<T>
: params => {
const { start, end } = getArrayBounds(rowCount, params);
const ret = new Array(end - start);
for (let i = 0, _i = end - start; i < _i; i++) ret[i] = array[start + i];
return ret;
},
stringEquals: isTyped
? (row, value) => (array as any)[row] === +value
: (row, value) => {
const v = array[row];
if (typeof v !== 'string') return '' + v === value;
return v === value;
},
areValuesEqual: (rowA, rowB) => array[rowA] === array[rowB]
} }
} }
...@@ -74,4 +102,13 @@ export function createAndFillArray(rowCount: number, value: (row: number) => any ...@@ -74,4 +102,13 @@ export function createAndFillArray(rowCount: number, value: (row: number) => any
return fillArrayValues(value, array, start); return fillArrayValues(value, array, start);
} }
export function isTypedArray(data: any) {
return data.buffer && typeof data.byteLength === 'number' && data.BYTES_PER_ELEMENT;
}
export function typedArrayWindow(data: any, params?: ToArrayParams): ReadonlyArray<number> {
const { constructor, buffer, length, byteOffset, BYTES_PER_ELEMENT } = data;
const { start, end } = getArrayBounds(length, params);
if (start === 0 && end === length) return data;
return new constructor(buffer, byteOffset + BYTES_PER_ELEMENT * start, Math.min(length, end - start));
}
\ No newline at end of file
...@@ -5,10 +5,9 @@ ...@@ -5,10 +5,9 @@
* @author David Sehnal <david.sehnal@gmail.com> * @author David Sehnal <david.sehnal@gmail.com>
*/ */
import * as BinaryColumn from '../common/binary/column'
import FixedColumn from '../common/text/column/fixed' import FixedColumn from '../common/text/column/fixed'
import TokenColumn from '../common/text/column/token' import TokenColumn from '../common/text/column/token'
import { ColumnType } from '../common/column' import { ColumnType, typedArrayWindow } from '../common/column'
const lines = [ const lines = [
'1.123 abc', '1.123 abc',
...@@ -65,8 +64,8 @@ describe('token text column', () => { ...@@ -65,8 +64,8 @@ describe('token text column', () => {
describe('binary column', () => { describe('binary column', () => {
it('window works', () => { it('window works', () => {
const xs = new Float64Array([1, 2, 3, 4]); const xs = new Float64Array([1, 2, 3, 4]);
const w1 = BinaryColumn.typedArrayWindow(xs, { start: 1 }); const w1 = typedArrayWindow(xs, { start: 1 });
const w2 = BinaryColumn.typedArrayWindow(xs, { start: 2, end: 4 }); const w2 = typedArrayWindow(xs, { start: 2, end: 4 });
expect(w1.length).toBe(3); expect(w1.length).toBe(3);
for (let i = 0; i < w1.length; i++) expect(w1[i]).toBe(xs[i + 1]); for (let i = 0; i < w1.length; i++) expect(w1[i]).toBe(xs[i + 1]);
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment