Spaces:
Sleeping
Sleeping
File size: 10,216 Bytes
be5030f |
1 |
{"version":3,"sources":["util/recordbatch.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,+DAA+D;AAC/D,wDAAwD;AACxD,6DAA6D;AAC7D,oDAAoD;AACpD,6DAA6D;AAC7D,6DAA6D;AAC7D,EAAE;AACF,+CAA+C;AAC/C,EAAE;AACF,6DAA6D;AAC7D,8DAA8D;AAC9D,yDAAyD;AACzD,4DAA4D;AAC5D,0DAA0D;AAC1D,qBAAqB;AAKrB,OAAO,EAAE,IAAI,EAAW,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,MAAM,EAAS,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;AAClC,MAAM,QAAQ,GAAG,CAAC,YAAoB,EAAE,EAAE,CAAW;IACjD,OAAO,EAAE,OAAO,EAAE,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,OAAO;CAC1C,CAAC;AAElB,cAAc;AACd,MAAM,UAAU,oBAAoB,CAChC,MAAiB,EACjB,MAA0B,EAC1B,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAE/D,IAAI,IAAsB,CAAC;IAC3B,IAAI,KAAwB,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAC9B,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,EAAwB,CAAC;IAC3C,MAAM,YAAY,GAAG,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IACrD,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE;QACZ,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE;YACnD,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;SACvB;aAAM;YACH,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAsB,CAAC,CAAC;YACvG,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,WAAW,CAAC;gBACtE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAqB,CAAC;SACvG;KACJ;IACD,OAAO,CAAC,IAAI,MAAM,CAAI,MAAM,CAAC,EAAE,WAAW,EAAE,SAAS,CAA4C,CAAC;AACtG,CAAC;AAED,cAAc;AACd,MAAM,UAAU,kCAAkC,CAA8C,OAA6B;IACzH,OAAO,kCAAkC,CAAI,IAAI,MAAM,CAAI,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAC5G,CAAC;AAED,cAAc;AACd,MAAM,UAAU,kCAAkC,CAA8C,MAAiB,EAAE,IAAkD;IACjK,OAAO,4CAA4C,CAAI,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACnJ,CAAC;AAED,cAAc;AACd,SAAS,4CAA4C,CAA8C,MAAiB,EAAE,OAA6B;IAE/I,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,EAAoC,CAAC;IACvD,MAAM,IAAI,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAEhF,IAAI,UAAU,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,GAAW,CAAC,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAChD,IAAI,KAAuB,EAAE,SAAS,GAAuB,EAAE,CAAC;IAEhE,OAAO,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE;QAE1B,KAAK,WAAW,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,UAAU,GAAG;YACpE,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAG,CAAC;YAC3C,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;SAC3E;QAED,IAAI,QAAQ,CAAC,WAAW,CAAC,EAAE;YACvB,SAAS,GAAG,mBAAmB,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAC/E,IAAI,WAAW,GAAG,CAAC,EAAE;gBACjB,SAAS,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;aAC9D;SACJ;KACJ;IACD,OAAO;QACH,MAAM,GAAG,IAAI,MAAM,CAAI,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC;QAC/C,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,WAAW,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;KACxD,CAAC;AACN,CAAC;AAED,cAAc;AACd,SAAS,mBAAmB,CAA8C,MAA2B,EAAE,WAAmB,EAAE,SAA6B,EAAE,OAA6B,EAAE,IAA4B;IAClN,IAAI,IAAsB,CAAC;IAC3B,IAAI,KAAwB,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAC3C,MAAM,YAAY,GAAG,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IACrD,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE;QACZ,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,EAAE;YAClE,IAAI,MAAM,KAAK,WAAW,EAAE;gBACxB,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;aACvB;iBAAM;gBACH,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;gBAC1C,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;gBACrD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;aACzE;SACJ;aAAM;YACH,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAsB,CAAC,CAAC;YACnG,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,WAAW,CAAC;gBACtE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAqB,CAAC;SACvG;KACJ;IACD,OAAO,SAAS,CAAC;AACrB,CAAC","file":"recordbatch.js","sourcesContent":["// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\nimport { Column } from '../column';\nimport { Vector } from '../vector';\nimport { DataType } from '../type';\nimport { Data, Buffers } from '../data';\nimport { Schema, Field } from '../schema';\nimport { Chunked } from '../vector/chunked';\nimport { RecordBatch } from '../recordbatch';\n\nconst noopBuf = new Uint8Array(0);\nconst nullBufs = (bitmapLength: number) => <unknown> [\n noopBuf, noopBuf, new Uint8Array(bitmapLength), noopBuf\n] as Buffers<any>;\n\n/** @ignore */\nexport function ensureSameLengthData<T extends { [key: string]: DataType } = any>(\n schema: Schema<T>,\n chunks: Data<T[keyof T]>[],\n batchLength = chunks.reduce((l, c) => Math.max(l, c.length), 0)\n) {\n let data: Data<T[keyof T]>;\n let field: Field<T[keyof T]>;\n let i = -1, n = chunks.length;\n const fields = [...schema.fields];\n const batchData = [] as Data<T[keyof T]>[];\n const bitmapLength = ((batchLength + 63) & ~63) >> 3;\n while (++i < n) {\n if ((data = chunks[i]) && data.length === batchLength) {\n batchData[i] = data;\n } else {\n (field = fields[i]).nullable || (fields[i] = fields[i].clone({ nullable: true }) as Field<T[keyof T]>);\n batchData[i] = data ? data._changeLengthAndBackfillNullBitmap(batchLength)\n : Data.new(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>;\n }\n }\n return [new Schema<T>(fields), batchLength, batchData] as [Schema<T>, number, Data<T[keyof T]>[]];\n}\n\n/** @ignore */\nexport function distributeColumnsIntoRecordBatches<T extends { [key: string]: DataType } = any>(columns: Column<T[keyof T]>[]): [Schema<T>, RecordBatch<T>[]] {\n return distributeVectorsIntoRecordBatches<T>(new Schema<T>(columns.map(({ field }) => field)), columns);\n}\n\n/** @ignore */\nexport function distributeVectorsIntoRecordBatches<T extends { [key: string]: DataType } = any>(schema: Schema<T>, vecs: (Vector<T[keyof T]> | Chunked<T[keyof T]>)[]): [Schema<T>, RecordBatch<T>[]] {\n return uniformlyDistributeChunksAcrossRecordBatches<T>(schema, vecs.map((v) => v instanceof Chunked ? v.chunks.map((c) => c.data) : [v.data]));\n}\n\n/** @ignore */\nfunction uniformlyDistributeChunksAcrossRecordBatches<T extends { [key: string]: DataType } = any>(schema: Schema<T>, columns: Data<T[keyof T]>[][]): [Schema<T>, RecordBatch<T>[]] {\n\n const fields = [...schema.fields];\n const batchArgs = [] as [number, Data<T[keyof T]>[]][];\n const memo = { numBatches: columns.reduce((n, c) => Math.max(n, c.length), 0) };\n\n let numBatches = 0, batchLength = 0;\n let i: number = -1, numColumns = columns.length;\n let child: Data<T[keyof T]>, childData: Data<T[keyof T]>[] = [];\n\n while (memo.numBatches-- > 0) {\n\n for (batchLength = Number.POSITIVE_INFINITY, i = -1; ++i < numColumns;) {\n childData[i] = child = columns[i].shift()!;\n batchLength = Math.min(batchLength, child ? child.length : batchLength);\n }\n\n if (isFinite(batchLength)) {\n childData = distributeChildData(fields, batchLength, childData, columns, memo);\n if (batchLength > 0) {\n batchArgs[numBatches++] = [batchLength, childData.slice()];\n }\n }\n }\n return [\n schema = new Schema<T>(fields, schema.metadata),\n batchArgs.map((xs) => new RecordBatch(schema, ...xs))\n ];\n}\n\n/** @ignore */\nfunction distributeChildData<T extends { [key: string]: DataType } = any>(fields: Field<T[keyof T]>[], batchLength: number, childData: Data<T[keyof T]>[], columns: Data<T[keyof T]>[][], memo: { numBatches: number }) {\n let data: Data<T[keyof T]>;\n let field: Field<T[keyof T]>;\n let length = 0, i = -1, n = columns.length;\n const bitmapLength = ((batchLength + 63) & ~63) >> 3;\n while (++i < n) {\n if ((data = childData[i]) && ((length = data.length) >= batchLength)) {\n if (length === batchLength) {\n childData[i] = data;\n } else {\n childData[i] = data.slice(0, batchLength);\n data = data.slice(batchLength, length - batchLength);\n memo.numBatches = Math.max(memo.numBatches, columns[i].unshift(data));\n }\n } else {\n (field = fields[i]).nullable || (fields[i] = field.clone({ nullable: true }) as Field<T[keyof T]>);\n childData[i] = data ? data._changeLengthAndBackfillNullBitmap(batchLength)\n : Data.new(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>;\n }\n }\n return childData;\n}\n"]} |