From 02dbe138f9fbf771633c6d3dcaacbbe223ebe01c Mon Sep 17 00:00:00 2001 From: Dominic DeMarco <52165905+demarcod1@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:37:13 -0700 Subject: [PATCH] Support 64bit ints in feather (#50) * feather support bigint * fix regression in iso date output --- package.json | 2 +- src/__test__/integration.test.ts | 10 +- src/__test__/json/request_multi.json | 16 +- src/__test__/json/request_single.json | 8 +- .../json/uncompressed_multi_epoch_millis.json | 2 +- src/__test__/serialization.test.ts | 2 +- src/_bulk_response.ts | 10 +- src/_feather.ts | 4 +- yarn.lock | 161 +++++++----------- 9 files changed, 89 insertions(+), 126 deletions(-) diff --git a/package.json b/package.json index a0e0d13..55adb8a 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "test:integration:watch": "CHALK_INTEGRATION=1 jest --watch -t='integration tests'" }, "dependencies": { - "apache-arrow": "^14.0.1", + "apache-arrow": "^17.0.0", "node-fetch": "^2.6.6" }, "devDependencies": { diff --git a/src/__test__/integration.test.ts b/src/__test__/integration.test.ts index 48404c6..d4f32e5 100644 --- a/src/__test__/integration.test.ts +++ b/src/__test__/integration.test.ts @@ -86,7 +86,7 @@ maybe("integration tests", () => { error = e; } expect((error as any).message).toEqual( - `{"detail":"Client ID and secret invalid","message":"Client ID and secret invalid","trace":null}\n` + `{"detail":"Client ID and secret are invalid","message":"Client ID and secret are invalid","trace":null}\n` ); }); @@ -108,7 +108,7 @@ maybe("integration tests", () => { error = e; } expect((error as any).message).toEqual( - `{"detail":"Client ID and secret invalid","message":"Client ID and secret invalid","trace":null}\n` + `{"detail":"Client ID and secret are invalid","message":"Client ID and secret are invalid","trace":null}\n` ); }); @@ -127,7 +127,7 @@ maybe("integration tests", () => { error = e; } expect((error as any).message).toEqual( - `Chalk client parameter 'clientSecret' was not specified when creating your ChalkClient, and was not present as '_CHALK_CLIENT_SECRET' in process.env. This field is required to use Chalk` + `Chalk client parameter 'clientId' was not specified when creating your ChalkClient, and was not present as '_CHALK_CLIENT_ID' in process.env. This field is required to use Chalk` ); }); }); @@ -246,9 +246,9 @@ maybe("integration tests", () => { }); expect(Object.keys(result.data).length).toBe(2); - expect(result.data[0]["user.id"]).toEqual(1); + expect(result.data[0]["user.id"]).toEqual(BigInt(1)); expect(result.data[0]["user.full_name"]).toEqual("Donna Davis"); - expect(result.data[1]["user.id"]).toEqual(2); + expect(result.data[1]["user.id"]).toEqual(BigInt(2)); expect(result.data[1]["user.full_name"]).toEqual("William Johnson"); expect(result.meta).toBeDefined(); diff --git a/src/__test__/json/request_multi.json b/src/__test__/json/request_multi.json index c764dfc..b414b63 100644 --- a/src/__test__/json/request_multi.json +++ b/src/__test__/json/request_multi.json @@ -9,7 +9,7 @@ }, "feather_body_type": "RECORD_BATCHES", "response_compression_scheme": "uncompressed", - "client_supports_64bit": false + "client_supports_64bit": true }, "body": { "schema": { @@ -19,7 +19,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ], @@ -36,7 +36,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ], @@ -52,7 +52,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ] @@ -103,7 +103,7 @@ }, "feather_body_type": "RECORD_BATCHES", "response_compression_scheme": "uncompressed", - "client_supports_64bit": false + "client_supports_64bit": true }, "body": { "schema": { @@ -113,7 +113,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ], @@ -130,7 +130,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ], @@ -146,7 +146,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ] diff --git a/src/__test__/json/request_single.json b/src/__test__/json/request_single.json index a073ec4..18052e7 100644 --- a/src/__test__/json/request_single.json +++ b/src/__test__/json/request_single.json @@ -9,7 +9,7 @@ }, "feather_body_type": "RECORD_BATCHES", "response_compression_scheme": "uncompressed", - "client_supports_64bit": false + "client_supports_64bit": true }, "body": { "schema": { @@ -19,7 +19,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ], @@ -36,7 +36,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ], @@ -52,7 +52,7 @@ "type": { "precision": 2 }, - "nullable": false, + "nullable": true, "metadata": {} } ] diff --git a/src/__test__/json/uncompressed_multi_epoch_millis.json b/src/__test__/json/uncompressed_multi_epoch_millis.json index 584060c..1e8efa6 100644 --- a/src/__test__/json/uncompressed_multi_epoch_millis.json +++ b/src/__test__/json/uncompressed_multi_epoch_millis.json @@ -61,7 +61,7 @@ "transaction.amount": 514, "transaction.user.full_name": "Norma Fisher", "transaction.id": 3, - "transaction.ts": 1675490035989.8472, + "transaction.ts": 1675490035989.847, "__ts__": 1712708210149.395 }, { diff --git a/src/__test__/serialization.test.ts b/src/__test__/serialization.test.ts index 7e61506..6552335 100644 --- a/src/__test__/serialization.test.ts +++ b/src/__test__/serialization.test.ts @@ -34,7 +34,7 @@ describe("featherRequestHeaderFromBody", () => { }, feather_body_type: "RECORD_BATCHES", response_compression_scheme: "uncompressed", - client_supports_64bit: false, + client_supports_64bit: true, }); }); diff --git a/src/_bulk_response.ts b/src/_bulk_response.ts index a4819e9..ebfc310 100644 --- a/src/_bulk_response.ts +++ b/src/_bulk_response.ts @@ -3,10 +3,9 @@ import { DataType, Table, vectorFromArray, - Date_, - DateUnit, Timestamp, TimeUnit, + Utf8, } from "apache-arrow"; import { RawQueryResponseMeta } from "./_http"; import { ChalkError, ChalkQueryMeta, TimestampFormat } from "./_interface"; @@ -128,9 +127,9 @@ export function processArrowTable( timestampFormat === TimestampFormat.ISO_8601 && DataType.isTimestamp(vector.type) ) { - const newVector = vectorFromArray( - Array.from(vector, (data) => new Date(data)), - new Date_(DateUnit.MILLISECOND) + const newVector = vectorFromArray( + Array.from(vector, (data) => (new Date(data)).toISOString()), + new Utf8() ); newTable = newTable.setChildAt(index, newVector); } else if ( @@ -141,6 +140,7 @@ export function processArrowTable( Array.from(vector, (data) => new Date(data)), new Timestamp(TimeUnit.MILLISECOND) ); + newTable = newTable.setChildAt(index, newVector); } } diff --git a/src/_feather.ts b/src/_feather.ts index 498a987..aeefbdb 100644 --- a/src/_feather.ts +++ b/src/_feather.ts @@ -34,7 +34,7 @@ interface OnlineQueryFeatherRequestHeader< // these are the only currently supported values feather_body_type: "RECORD_BATCHES"; response_compression_scheme: "uncompressed"; - client_supports_64bit: false; + client_supports_64bit: true; } export function featherRequestHeaderFromBody< @@ -48,7 +48,7 @@ export function featherRequestHeaderFromBody< ...rest, feather_body_type: "RECORD_BATCHES", response_compression_scheme: "uncompressed", - client_supports_64bit: false, + client_supports_64bit: true, }; } diff --git a/yarn.lock b/yarn.lock index 8cf8f5b..df646c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5,16 +5,6 @@ __metadata: version: 6 cacheKey: 8 -"@75lb/deep-merge@npm:^1.1.1": - version: 1.1.1 - resolution: "@75lb/deep-merge@npm:1.1.1" - dependencies: - lodash.assignwith: ^4.2.0 - typical: ^7.1.1 - checksum: fd9063488d854bc5d2e1636426a51d7864d0d32d2d82c5b01a40e89466088680f6e2623345fb46782de438088b6d3f029b0eea6d79a7807e0000b365f6b8142b - languageName: node - linkType: hard - "@ampproject/remapping@npm:^2.1.0": version: 2.2.0 resolution: "@ampproject/remapping@npm:2.2.0" @@ -518,7 +508,7 @@ __metadata: dependencies: "@types/jest": ^29.5.11 "@types/node": ^18.11.9 - apache-arrow: ^14.0.1 + apache-arrow: ^17.0.0 jest: ^29.7.0 node-fetch: ^2.6.6 prettier: ^2.7.1 @@ -944,6 +934,15 @@ __metadata: languageName: node linkType: hard +"@swc/helpers@npm:^0.5.11": + version: 0.5.13 + resolution: "@swc/helpers@npm:0.5.13" + dependencies: + tslib: ^2.4.0 + checksum: d50c2c10da6ef940af423c6b03ad9c3c94cf9de59314b1e921a7d1bcc081a6074481c9d67b655fc8fe66a73288f98b25950743792a63882bfb5793b362494fc0 + languageName: node + linkType: hard + "@tootallnate/once@npm:2": version: 2.0.0 resolution: "@tootallnate/once@npm:2.0.0" @@ -992,17 +991,17 @@ __metadata: languageName: node linkType: hard -"@types/command-line-args@npm:5.2.0": - version: 5.2.0 - resolution: "@types/command-line-args@npm:5.2.0" - checksum: 423121d2d083765f5b78d090115f3be82d53a39cec9de63719cbd07021e6330fab19b75e2290af1f7dda84efd7964dc498eb10b2b465991de27045db95aa1eef +"@types/command-line-args@npm:^5.2.3": + version: 5.2.3 + resolution: "@types/command-line-args@npm:5.2.3" + checksum: 3d90db5b4bbaabd049654a0d12fa378989ab0d76a0f98d4c606761b5a08ce76458df0f9bb175219e187b4cd57e285e6f836d23e86b2c3d997820854cc3ed9121 languageName: node linkType: hard -"@types/command-line-usage@npm:5.0.2": - version: 5.0.2 - resolution: "@types/command-line-usage@npm:5.0.2" - checksum: 9c0eabf5e86a405d118dcfb5f4bceae43080efe603a0f240664716a05283dcb389e94e999188d12b10a0aa4452a920445131f1011e7484403f146607cd2577f0 +"@types/command-line-usage@npm:^5.0.4": + version: 5.0.4 + resolution: "@types/command-line-usage@npm:5.0.4" + checksum: 7173c356ca8c9507feeeda8e660c52498929556e90be0cf2d09d35270c597481121cd0f006a74167c5577feebfbc75b648c0f8f01b8f06ce30bde9fe30d5ba40 languageName: node linkType: hard @@ -1057,17 +1056,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:20.3.0": - version: 20.3.0 - resolution: "@types/node@npm:20.3.0" - checksum: 613e878174febc0104dae210088645cbb5096a19ae5fdf57fbc06fa6ef71755b839858c2b313fb8c1df8b7c6660e1b5f7a64db2126af9d47d1d9238e2bc0a86d - languageName: node - linkType: hard - -"@types/pad-left@npm:2.1.1": - version: 2.1.1 - resolution: "@types/pad-left@npm:2.1.1" - checksum: db0191e39af416e5eca02f22424306a13b1bf24abb00873bbbb8c28de5085764ff1243c32548580260952588fbef6d9a5a34106276013420bcc79f538f4e8cc2 +"@types/node@npm:^20.13.0": + version: 20.16.5 + resolution: "@types/node@npm:20.16.5" + dependencies: + undici-types: ~6.19.2 + checksum: f38b7bd8c4993dcf38943afa2ffdd7dfd18fc94f8f3f28d0c1045a10d39871a6cc1b8f8d3bf0c7ed848457d0e1d283482f6ca125579c13fed1b7575d23e8e8f5 languageName: node linkType: hard @@ -1182,23 +1176,22 @@ __metadata: languageName: node linkType: hard -"apache-arrow@npm:^14.0.1": - version: 14.0.1 - resolution: "apache-arrow@npm:14.0.1" +"apache-arrow@npm:^17.0.0": + version: 17.0.0 + resolution: "apache-arrow@npm:17.0.0" dependencies: - "@types/command-line-args": 5.2.0 - "@types/command-line-usage": 5.0.2 - "@types/node": 20.3.0 - "@types/pad-left": 2.1.1 - command-line-args: 5.2.1 - command-line-usage: 7.0.1 - flatbuffers: 23.5.26 + "@swc/helpers": ^0.5.11 + "@types/command-line-args": ^5.2.3 + "@types/command-line-usage": ^5.0.4 + "@types/node": ^20.13.0 + command-line-args: ^5.2.1 + command-line-usage: ^7.0.1 + flatbuffers: ^24.3.25 json-bignum: ^0.0.3 - pad-left: ^2.1.0 - tslib: ^2.5.3 + tslib: ^2.6.2 bin: - arrow2csv: bin/arrow2csv.js - checksum: bb4e1eb66478ccafd7e069ccd7042abbff15bc890ffe434f93f78a2c18ccdb51a599101778287c1235f3c9a93d10482f266d739ea2afcffc2f0e320767da9041 + arrow2csv: bin/arrow2csv.cjs + checksum: 0533010044b36ec5b3c5d8def3f28039ffdab5381d0e21f4623f5136edceb76abd472a4d231cfac6308848af1cb2d1e4093792a2d826e19333e02a0294a236f6 languageName: node linkType: hard @@ -1577,7 +1570,7 @@ __metadata: languageName: node linkType: hard -"command-line-args@npm:5.2.1, command-line-args@npm:^5.2.1": +"command-line-args@npm:^5.2.1": version: 5.2.1 resolution: "command-line-args@npm:5.2.1" dependencies: @@ -1589,15 +1582,15 @@ __metadata: languageName: node linkType: hard -"command-line-usage@npm:7.0.1, command-line-usage@npm:^7.0.0": - version: 7.0.1 - resolution: "command-line-usage@npm:7.0.1" +"command-line-usage@npm:^7.0.1": + version: 7.0.3 + resolution: "command-line-usage@npm:7.0.3" dependencies: array-back: ^6.2.2 chalk-template: ^0.4.0 - table-layout: ^3.0.0 + table-layout: ^4.1.0 typical: ^7.1.1 - checksum: ac78ad6b83b9622bb111ae8e82205bde1d2da74df237fdd0bd7d98eda3592c8933ec600818b0b028b2313ddca638b1b60f0780dd9457ad4a0384b17156641f79 + checksum: cb65d94c71ac380d6133460fa16d15c3d6dde00746498d60dcd12989fffeb90d1373230135c97e0bd7019874edd913f9df8b87b0afc7180811117342ae950ff4 languageName: node linkType: hard @@ -1901,10 +1894,10 @@ __metadata: languageName: node linkType: hard -"flatbuffers@npm:23.5.26": - version: 23.5.26 - resolution: "flatbuffers@npm:23.5.26" - checksum: 2528e549118d02fca9775cc9c17cf445741bb58b0a9575eb07bba823394a79dccb3bf97c48561720aa61d3dc9f42759129df0e05eb94c12f809cfb6126eaedd1 +"flatbuffers@npm:^24.3.25": + version: 24.3.25 + resolution: "flatbuffers@npm:24.3.25" + checksum: 418d946b9bec0bdc7d1212062f385f08cfc34fed4709fd3dbaf5d7b3e73a33188d964ac65843aaa4bce8a753631d9087cabb53dc94fb43a5da77cbc891eacaf5 languageName: node linkType: hard @@ -2887,13 +2880,6 @@ __metadata: languageName: node linkType: hard -"lodash.assignwith@npm:^4.2.0": - version: 4.2.0 - resolution: "lodash.assignwith@npm:4.2.0" - checksum: 014a88e398802ca4eaae314afb67f32eb2cab6f01e61490dbbb74694263f79715341ab8ddf4b344093a2253b506d347f67731f0499e457d9c0128be1d2caf6dd - languageName: node - linkType: hard - "lodash.camelcase@npm:^4.3.0": version: 4.3.0 resolution: "lodash.camelcase@npm:4.3.0" @@ -3270,15 +3256,6 @@ __metadata: languageName: node linkType: hard -"pad-left@npm:^2.1.0": - version: 2.1.0 - resolution: "pad-left@npm:2.1.0" - dependencies: - repeat-string: ^1.5.4 - checksum: a1605c3cb0ebd9be1a74f7f895524d3bf30f24492db87421c2fda9b8c0fcee17b3b594dacfb5eb1d3189d59c69d2f969677bc04b8280a7ef6303566154dbd078 - languageName: node - linkType: hard - "parse-json@npm:^5.2.0": version: 5.2.0 resolution: "parse-json@npm:5.2.0" @@ -3432,13 +3409,6 @@ __metadata: languageName: node linkType: hard -"repeat-string@npm:^1.5.4": - version: 1.6.1 - resolution: "repeat-string@npm:1.6.1" - checksum: 1b809fc6db97decdc68f5b12c4d1a671c8e3f65ec4a40c238bc5200e44e85bcc52a54f78268ab9c29fcf5fe4f1343e805420056d1f30fa9a9ee4c2d93e3cc6c0 - languageName: node - linkType: hard - "require-directory@npm:^2.1.1": version: 2.1.1 resolution: "require-directory@npm:2.1.1" @@ -3672,13 +3642,6 @@ __metadata: languageName: node linkType: hard -"stream-read-all@npm:^3.0.1": - version: 3.0.1 - resolution: "stream-read-all@npm:3.0.1" - checksum: 40d3c286837f1b1ae7e8105959804ad42fda00f2c087722d981cb1c9fbbea892b8a0a7ca1cf6a016c96770151a6201a3da5c8b66fe35e35106b52a5e9ab90e3e - languageName: node - linkType: hard - "string-length@npm:^4.0.1": version: 4.0.2 resolution: "string-length@npm:4.0.2" @@ -3773,20 +3736,13 @@ __metadata: languageName: node linkType: hard -"table-layout@npm:^3.0.0": - version: 3.0.2 - resolution: "table-layout@npm:3.0.2" +"table-layout@npm:^4.1.0": + version: 4.1.1 + resolution: "table-layout@npm:4.1.1" dependencies: - "@75lb/deep-merge": ^1.1.1 array-back: ^6.2.2 - command-line-args: ^5.2.1 - command-line-usage: ^7.0.0 - stream-read-all: ^3.0.1 - typical: ^7.1.1 wordwrapjs: ^5.1.0 - bin: - table-layout: bin/cli.js - checksum: 2d4c538f224e64321d35788dbf78305cc1d138a3508e1a29f33e4f6b00bd082990a45dc85fd92948213f48ed8c0b3599155c2a05de412661ff020635e0db3762 + checksum: 6de52785440b3b2ca9522a06b9ce20f81a3a999c15ef7e5d10c38a2e0008b286bf145e7f88b00f0346e874a548a922906107c492d6da5d438332e7c1bb62307a languageName: node linkType: hard @@ -3878,10 +3834,10 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.5.3": - version: 2.6.2 - resolution: "tslib@npm:2.6.2" - checksum: 329ea56123005922f39642318e3d1f0f8265d1e7fcb92c633e0809521da75eeaca28d2cf96d7248229deb40e5c19adf408259f4b9640afd20d13aecc1430f3ad +"tslib@npm:^2.4.0, tslib@npm:^2.6.2": + version: 2.7.0 + resolution: "tslib@npm:2.7.0" + checksum: 1606d5c89f88d466889def78653f3aab0f88692e80bb2066d090ca6112ae250ec1cfa9dbfaab0d17b60da15a4186e8ec4d893801c67896b277c17374e36e1d28 languageName: node linkType: hard @@ -3933,6 +3889,13 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~6.19.2": + version: 6.19.8 + resolution: "undici-types@npm:6.19.8" + checksum: de51f1b447d22571cf155dfe14ff6d12c5bdaec237c765085b439c38ca8518fc360e88c70f99469162bf2e14188a7b0bcb06e1ed2dc031042b984b0bb9544017 + languageName: node + linkType: hard + "unique-filename@npm:^2.0.0": version: 2.0.1 resolution: "unique-filename@npm:2.0.1"