Don't read OFFSET vector for FixedSizeList

Author: Brian Hulette

js/src/data.ts

@@ -19,7 +19,7 @@ import { popcnt_bit_range } from './util/bit';
 import { VectorLike, Vector } from './vector';
 import { VectorType, TypedArray, TypedArrayConstructor, Dictionary } from './type';
 import { Int, Bool, FlatListType, List, FixedSizeList, Struct, Map_ } from './type';
-import { DataType, FlatType, ListType, NestedType, DenseUnion, SparseUnion } from './type';
+import { DataType, FlatType, ListType, NestedType, SingleNestedType, DenseUnion, SparseUnion } from './type';
 
 export function toTypedArray<T extends TypedArray>(ArrayType: TypedArrayConstructor<T>, values?: T | ArrayLike<number> | Iterable<number> | null): T {
     if (!ArrayType && ArrayBuffer.isView(values)) { return values; }
@@ -46,7 +46,7 @@ export interface DataTypes<T extends DataType> {
 /*              [Type.Struct]*/ 13: NestedData<Struct>;
 /*               [Type.Union]*/ 14: UnionData;
 /*     [Type.FixedSizeBinary]*/ 15: FlatData<T>;
-/*       [Type.FixedSizeList]*/ 16: ListData<FixedSizeList<T>>;
+/*       [Type.FixedSizeList]*/ 16: SingleNestedData<FixedSizeList<T>>;
 /*                 [Type.Map]*/ 17: NestedData<Map_>;
 /*  [Type.DenseUnion]*/ DenseUnion: DenseUnionData;
 /*[Type.SparseUnion]*/ SparseUnion: SparseUnionData;
@@ -195,15 +195,21 @@ export class NestedData<T extends NestedType = NestedType> extends BaseData<T> {
     }
 }
 
-export class ListData<T extends ListType> extends NestedData<T> {
-    public /*  [VectorType.OFFSET]:*/ 0: Int32Array;
-    public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
+export class SingleNestedData<T extends SingleNestedType> extends NestedData<T> {
     protected _valuesData: Data<T>;
     public get values() { return this._valuesData; }
-    public get valueOffsets() { return this[VectorType.OFFSET]; }
-    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, valueChildData: Data<T>, offset?: number, nullCount?: number) {
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueChildData: Data<T>, offset?: number, nullCount?: number) {
         super(type, length, nullBitmap, [valueChildData], offset, nullCount);
         this._valuesData = valueChildData;
+    }
+}
+
+export class ListData<T extends ListType> extends SingleNestedData<T> {
+    public /*  [VectorType.OFFSET]:*/ 0: Int32Array;
+    public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
+    public get valueOffsets() { return this[VectorType.OFFSET]; }
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, valueChildData: Data<T>, offset?: number, nullCount?: number) {
+        super(type, length, nullBitmap, valueChildData, offset, nullCount);
         this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
     }
     public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {

js/src/ipc/reader/vector.ts

@@ -20,7 +20,7 @@ import { RecordBatch } from '../../recordbatch';
 import { TypeVisitor } from '../../visitor';
 import { FlatType, NestedType, ListType } from '../../type';
 import { Message, FieldMetadata, BufferMetadata } from '../metadata';
-import { FlatData, ListData, NestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data';
+import { FlatData, ListData, NestedData, SingleNestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data';
 import {
     Schema, Field,
     Dictionary,
@@ -89,7 +89,7 @@ export abstract class TypeDataLoader extends TypeVisitor {
     public visitStruct         (type: Struct)          { return this.visitNestedType(type); }
     public visitUnion          (type: Union)           { return this.visitUnionType(type);  }
     public visitFixedSizeBinary(type: FixedSizeBinary) { return this.visitFlatType(type);   }
-    public visitFixedSizeList  (type: FixedSizeList)   { return this.visitListType(type);   }
+    public visitFixedSizeList  (type: FixedSizeList)   { return this.visitFixedSizeListType(type); }
     public visitMap            (type: Map_)            { return this.visitNestedType(type); }
     public visitDictionary     (type: Dictionary)      {
         return new DictionaryData(type, this.dictionaries.get(type.id)!, this.visit(type.indicies));
@@ -117,6 +117,9 @@ export abstract class TypeDataLoader extends TypeVisitor {
     protected visitListType<T extends ListType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
         return new ListData<T>(type, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children![0].type), 0, nullCount);
     }
+    protected visitFixedSizeListType<T extends FixedSizeList>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return new SingleNestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visit(type.children![0].type), 0, nullCount);
+    }
     protected visitNestedType<T extends NestedType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
         return new NestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visitFields(type.children), 0, nullCount);
     }

js/src/type.ts

@@ -97,8 +97,9 @@ export type PrimitiveType = NumericType | FixedSizeType;
 
 export type FlatListType = Utf8 | Binary; // <-- these types have `offset`, `data`, and `validity` buffers
 export type FlatType = Bool | PrimitiveType | FlatListType; // <-- these types have `data` and `validity` buffers
-export type ListType = List<any> | FixedSizeList<any>; // <-- these types have `offset` and `validity` buffers
+export type ListType = List<any>; // <-- these types have `offset` and `validity` buffers
 export type NestedType = Map_ | Struct | List<any> | FixedSizeList<any> | Union<any>; // <-- these types have `validity` buffer and nested childData
+export type SingleNestedType = List<any> | FixedSizeList<any>; // <-- these are nested types that can only have a single child
 
 /**
  * *

js/src/vector.ts

@@ -336,7 +336,7 @@ export class ListVector<T extends DataType = DataType> extends ListVectorBase<Li
     }
 }
 
-export class FixedSizeListVector extends ListVectorBase<FixedSizeList> {
+export class FixedSizeListVector extends Vector<FixedSizeList> {
     constructor(data: Data<FixedSizeList>, view: View<FixedSizeList> = new FixedSizeListView(data)) {
         super(data, view);
     }

js/src/vector/list.ts

@@ -29,14 +29,13 @@ export const decodeUtf8 = ((decoder) =>
     decoder.decode.bind(decoder) as (input?: ArrayBufferLike | ArrayBufferView) => string
 )(new TextDecoder('utf-8'));
 
-export abstract class ListViewBase<T extends (ListType | FlatListType)> implements View<T> {
+export abstract class ListViewBase<T extends (ListType | FlatListType | FixedSizeList)> implements View<T> {
     public length: number;
     public values: T['TArray'];
     public valueOffsets?: Int32Array;
     constructor(data: Data<T>) {
         this.length = data.length;
         this.values = data.values;
-        this.valueOffsets = data.valueOffsets;
     }
     public clone(data: Data<T>): this {
         return new (<any> this.constructor)(data) as this;
@@ -64,7 +63,15 @@ export abstract class ListViewBase<T extends (ListType | FlatListType)> implemen
     protected abstract setList(values: T['TArray'], index: number, value: T['TValue'], valueOffsets?: Int32Array): void;
 }
 
-export class ListView<T extends DataType> extends ListViewBase<List<T>> {
+export abstract class VariableListViewBase<T extends (ListType | FlatListType)> extends ListViewBase<T> {
+    constructor(data: Data<T>) {
+        super(data)
+        this.length = data.length;
+        this.valueOffsets = data.valueOffsets;
+    }
+}
+
+export class ListView<T extends DataType> extends VariableListViewBase<List<T>> {
     constructor(data: Data<List<T>>) {
         super(data);
         this.values = createVector(data.values);
@@ -101,7 +108,7 @@ export class FixedSizeListView<T extends DataType> extends ListViewBase<FixedSiz
     }
 }
 
-export class BinaryView extends ListViewBase<Binary> {
+export class BinaryView extends VariableListViewBase<Binary> {
     protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
         return values.subarray(valueOffsets[index], valueOffsets[index + 1]);
     }
@@ -111,7 +118,7 @@ export class BinaryView extends ListViewBase<Binary> {
     }
 }
 
-export class Utf8View extends ListViewBase<Utf8> {
+export class Utf8View extends VariableListViewBase<Utf8> {
     protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
         return decodeUtf8(values.subarray(valueOffsets[index], valueOffsets[index + 1]));
     }