Support 64bit ints in feather (#50)

* feather support bigint * fix regression in iso date output
chalk-ai · Sep 6, 2024 · 02dbe13 · 02dbe13
1 parent d765109
commit 02dbe13
Show file tree

Hide file tree

Showing 9 changed files with 89 additions and 126 deletions.
diff --git a/package.json b/package.json
@@ -11,7 +11,7 @@
     "test:integration:watch": "CHALK_INTEGRATION=1 jest --watch -t='integration tests'"
   },
   "dependencies": {
-    "apache-arrow": "^14.0.1",
+    "apache-arrow": "^17.0.0",
     "node-fetch": "^2.6.6"
   },
   "devDependencies": {

diff --git a/src/__test__/integration.test.ts b/src/__test__/integration.test.ts
@@ -86,7 +86,7 @@ maybe("integration tests", () => {
         error = e;
       }
       expect((error as any).message).toEqual(
-        `{"detail":"Client ID and secret invalid","message":"Client ID and secret invalid","trace":null}\n`
+        `{"detail":"Client ID and secret are invalid","message":"Client ID and secret are invalid","trace":null}\n`
       );
     });
 
@@ -108,7 +108,7 @@ maybe("integration tests", () => {
         error = e;
       }
       expect((error as any).message).toEqual(
-        `{"detail":"Client ID and secret invalid","message":"Client ID and secret invalid","trace":null}\n`
+        `{"detail":"Client ID and secret are invalid","message":"Client ID and secret are invalid","trace":null}\n`
       );
     });
 
@@ -127,7 +127,7 @@ maybe("integration tests", () => {
         error = e;
       }
       expect((error as any).message).toEqual(
-        `Chalk client parameter 'clientSecret' was not specified when creating your ChalkClient, and was not present as '_CHALK_CLIENT_SECRET' in process.env. This field is required to use Chalk`
+        `Chalk client parameter 'clientId' was not specified when creating your ChalkClient, and was not present as '_CHALK_CLIENT_ID' in process.env. This field is required to use Chalk`
       );
     });
   });
@@ -246,9 +246,9 @@ maybe("integration tests", () => {
       });
 
       expect(Object.keys(result.data).length).toBe(2);
-      expect(result.data[0]["user.id"]).toEqual(1);
+      expect(result.data[0]["user.id"]).toEqual(BigInt(1));
       expect(result.data[0]["user.full_name"]).toEqual("Donna Davis");
-      expect(result.data[1]["user.id"]).toEqual(2);
+      expect(result.data[1]["user.id"]).toEqual(BigInt(2));
       expect(result.data[1]["user.full_name"]).toEqual("William Johnson");
 
       expect(result.meta).toBeDefined();

diff --git a/src/__test__/json/request_multi.json b/src/__test__/json/request_multi.json
@@ -9,7 +9,7 @@
       },
       "feather_body_type": "RECORD_BATCHES",
       "response_compression_scheme": "uncompressed",
-      "client_supports_64bit": false
+      "client_supports_64bit": true
     },
     "body": {
       "schema": {
@@ -19,7 +19,7 @@
             "type": {
               "precision": 2
             },
-            "nullable": false,
+            "nullable": true,
             "metadata": {}
           }
         ],
@@ -36,7 +36,7 @@
                 "type": {
                   "precision": 2
                 },
-                "nullable": false,
+                "nullable": true,
                 "metadata": {}
               }
             ],
@@ -52,7 +52,7 @@
                   "type": {
                     "precision": 2
                   },
-                  "nullable": false,
+                  "nullable": true,
                   "metadata": {}
                 }
               ]
@@ -103,7 +103,7 @@
       },
       "feather_body_type": "RECORD_BATCHES",
       "response_compression_scheme": "uncompressed",
-      "client_supports_64bit": false
+      "client_supports_64bit": true
     },
     "body": {
       "schema": {
@@ -113,7 +113,7 @@
             "type": {
               "precision": 2
             },
-            "nullable": false,
+            "nullable": true,
             "metadata": {}
           }
         ],
@@ -130,7 +130,7 @@
                 "type": {
                   "precision": 2
                 },
-                "nullable": false,
+                "nullable": true,
                 "metadata": {}
               }
             ],
@@ -146,7 +146,7 @@
                   "type": {
                     "precision": 2
                   },
-                  "nullable": false,
+                  "nullable": true,
                   "metadata": {}
                 }
               ]

diff --git a/src/__test__/json/request_single.json b/src/__test__/json/request_single.json
@@ -9,7 +9,7 @@
       },
       "feather_body_type": "RECORD_BATCHES",
       "response_compression_scheme": "uncompressed",
-      "client_supports_64bit": false
+      "client_supports_64bit": true
     },
     "body": {
       "schema": {
@@ -19,7 +19,7 @@
             "type": {
               "precision": 2
             },
-            "nullable": false,
+            "nullable": true,
             "metadata": {}
           }
         ],
@@ -36,7 +36,7 @@
                 "type": {
                   "precision": 2
                 },
-                "nullable": false,
+                "nullable": true,
                 "metadata": {}
               }
             ],
@@ -52,7 +52,7 @@
                   "type": {
                     "precision": 2
                   },
-                  "nullable": false,
+                  "nullable": true,
                   "metadata": {}
                 }
               ]

diff --git a/src/__test__/json/uncompressed_multi_epoch_millis.json b/src/__test__/json/uncompressed_multi_epoch_millis.json
@@ -61,7 +61,7 @@
         "transaction.amount": 514,
         "transaction.user.full_name": "Norma Fisher",
         "transaction.id": 3,
-        "transaction.ts": 1675490035989.8472,
+        "transaction.ts": 1675490035989.847,
         "__ts__": 1712708210149.395
       },
       {

diff --git a/src/__test__/serialization.test.ts b/src/__test__/serialization.test.ts
@@ -34,7 +34,7 @@ describe("featherRequestHeaderFromBody", () => {
       },
       feather_body_type: "RECORD_BATCHES",
       response_compression_scheme: "uncompressed",
-      client_supports_64bit: false,
+      client_supports_64bit: true,
     });
   });
 

diff --git a/src/_bulk_response.ts b/src/_bulk_response.ts
@@ -3,10 +3,9 @@ import {
   DataType,
   Table,
   vectorFromArray,
-  Date_,
-  DateUnit,
   Timestamp,
   TimeUnit,
+  Utf8,
 } from "apache-arrow";
 import { RawQueryResponseMeta } from "./_http";
 import { ChalkError, ChalkQueryMeta, TimestampFormat } from "./_interface";
@@ -128,9 +127,9 @@ export function processArrowTable(
       timestampFormat === TimestampFormat.ISO_8601 &&
       DataType.isTimestamp(vector.type)
     ) {
-      const newVector = vectorFromArray<Date_>(
-        Array.from(vector, (data) => new Date(data)),
-        new Date_(DateUnit.MILLISECOND)
+      const newVector = vectorFromArray<Utf8>(
+        Array.from(vector, (data) => (new Date(data)).toISOString()),
+        new Utf8()
       );
       newTable = newTable.setChildAt(index, newVector);
     } else if (
@@ -141,6 +140,7 @@ export function processArrowTable(
         Array.from(vector, (data) => new Date(data)),
         new Timestamp(TimeUnit.MILLISECOND)
       );
+
       newTable = newTable.setChildAt(index, newVector);
     }
   }

diff --git a/src/_feather.ts b/src/_feather.ts
@@ -34,7 +34,7 @@ interface OnlineQueryFeatherRequestHeader<
   // these are the only currently supported values
   feather_body_type: "RECORD_BATCHES";
   response_compression_scheme: "uncompressed";
-  client_supports_64bit: false;
+  client_supports_64bit: true;
 }
 
 export function featherRequestHeaderFromBody<
@@ -48,7 +48,7 @@ export function featherRequestHeaderFromBody<
     ...rest,
     feather_body_type: "RECORD_BATCHES",
     response_compression_scheme: "uncompressed",
-    client_supports_64bit: false,
+    client_supports_64bit: true,
   };
 }