Change to use STRING instead of BYTE_ARRAY for (targetC|c)ontentHash …

…columns
LibertyDSNP · Aug 13, 2024 · 68b3154 · 68b3154
1 parent ae7fe5c
commit 68b3154
Show file tree

Hide file tree

Showing 10 changed files with 17 additions and 21 deletions.
diff --git a/__snapshots__/parquet.spec.ts.snap b/__snapshots__/parquet.spec.ts.snap
@@ -27,7 +27,7 @@ exports[`DSNP Schema Conversion Test File schema is encoded correctly 1`] = `
     "type_length": null,
   },
   {
-    "converted_type": null,
+    "converted_type": 0,
     "field_id": null,
     "logicalType": null,
     "name": "contentHash",
@@ -90,7 +90,7 @@ ParquetSchema {
       "dLevelMax": 0,
       "encoding": "PLAIN",
       "name": "contentHash",
-      "originalType": undefined,
+      "originalType": "UTF8",
       "path": [
         "contentHash",
       ],
@@ -160,7 +160,7 @@ ParquetSchema {
       "dLevelMax": 0,
       "encoding": "PLAIN",
       "name": "contentHash",
-      "originalType": undefined,
+      "originalType": "UTF8",
       "path": [
         "contentHash",
       ],
@@ -218,7 +218,7 @@ ParquetSchema {
       "compression": "GZIP",
       "encoding": "PLAIN",
       "statistics": false,
-      "type": "BYTE_ARRAY",
+      "type": "UTF8",
     },
     "fromId": {
       "compression": "GZIP",

diff --git a/parquet.spec.ts b/parquet.spec.ts
@@ -10,7 +10,7 @@ describe("DSNP Schema Conversion Test File", () => {
 
   const row1 = {
     announcementType: AnnouncementType.Broadcast,
-    contentHash: "0x12345678",
+    contentHash: "bciqdnu347gcfmxzbkhgoubiobphm6readngitfywktdtbdocgogop2q",
     fromId: 12n,
     url: "https://github.com/LibertyDSNP/parquetjs/",
   };
@@ -35,18 +35,14 @@ describe("DSNP Schema Conversion Test File", () => {
   it("schema is generated correctly", () => {
     expect(parquetSchema).toMatchSnapshot();
   });
-
+  
   it("schema is encoded correctly", () => {
     expect(reader.metadata?.schema).toMatchSnapshot();
   });
 
   it("output matches input", async () => {
     const cursor = reader.getCursor();
     const row = await cursor.next();
-    const rowData = {
-      ...row1,
-      contentHash: Buffer.from([48, 120, 49, 50, 51, 52, 53, 54, 55, 56]),
-    };
-    expect(row).toEqual(rowData);
+    expect(row).toEqual(row1);
   });
 });
diff --git a/parquet/broadcast.spec.ts b/parquet/broadcast.spec.ts
@@ -7,7 +7,7 @@ describe("Broadcast Spec", () => {
 
   testCompression("broadcast", broadcastSchema, () => ({
     announcementType: 2,
-    contentHash: generators.generateHash(),
+    contentHash: "bciqdnu347gcfmxzbkhgoubiobphm6readngitfywktdtbdocgogop2q",
     fromId: generators.randInt(10000000),
     url: `https://www.imadapp.com/data/posts/${generators.generateHash()}`,
   }));

diff --git a/parquet/broadcast.ts b/parquet/broadcast.ts
@@ -14,7 +14,7 @@ const broadcast: DSNPParquetSchema = [
   },
   {
     name: "contentHash",
-    column_type: "BYTE_ARRAY",
+    column_type: "STRING",
     compression: "GZIP",
     bloom_filter: true,
   },

diff --git a/parquet/reply.spec.ts b/parquet/reply.spec.ts
@@ -7,7 +7,7 @@ describe("Reply Spec", () => {
 
   testCompression("reply", replySchema, () => ({
     announcementType: 3,
-    contentHash: generators.generateHash(),
+    contentHash: "bciqdnu347gcfmxzbkhgoubiobphm6readngitfywktdtbdocgogop2q",
     fromId: generators.randInt(10000000),
     inReplyTo: `dsnp://${generators.randInt(10000000)}/${generators.generateHash()}`,
     url: `https://www.imadapp.com/data/posts/${generators.generateHash()}`,

diff --git a/parquet/reply.ts b/parquet/reply.ts
@@ -14,7 +14,7 @@ const reply: DSNPParquetSchema = [
   },
   {
     name: "contentHash",
-    column_type: "BYTE_ARRAY",
+    column_type: "STRING",
     compression: "GZIP",
     bloom_filter: true,
   },

diff --git a/parquet/tombstone.spec.ts b/parquet/tombstone.spec.ts
@@ -9,6 +9,6 @@ describe("Tombstone Spec", () => {
     announcementType: 0,
     fromId: generators.randInt(10000000),
     targetAnnouncementType: 2,
-    targetContentHash: generators.generateHash(),
+    targetContentHash: "bciqdnu347gcfmxzbkhgoubiobphm6readngitfywktdtbdocgogop2q",
   }));
 });
diff --git a/parquet/tombstone.ts b/parquet/tombstone.ts
@@ -36,7 +36,7 @@ const tombstone: DSNPParquetSchema = [
   },
   {
     name: "targetContentHash",
-    column_type: "BYTE_ARRAY",
+    column_type: "STRING",
     compression: "GZIP",
     bloom_filter: true,
   },

diff --git a/parquet/update.spec.ts b/parquet/update.spec.ts
@@ -7,10 +7,10 @@ describe("Update Spec", () => {
 
   testCompression("update", updateSchema, () => ({
     announcementType: 6,
-    contentHash: generators.generateHash(),
+    contentHash: "bciqdnu347gcfmxzbkhgoubiobphm6readngitfywktdtbdocgogop2q",
     fromId: generators.randInt(10000000),
     targetAnnouncementType: 2,
-    targetContentHash: generators.generateHash(),
+    targetContentHash: "bciqdnu347gcfmxzbkhgoubiobphm6readngitfywktdtbdocgogop2q",
     url: `https://www.imadapp.com/data/posts/${generators.generateHash()}`,
   }));
 });
diff --git a/parquet/update.ts b/parquet/update.ts
@@ -14,7 +14,7 @@ const update: DSNPParquetSchema = [
   },
   {
     name: "contentHash",
-    column_type: "BYTE_ARRAY",
+    column_type: "STRING",
     compression: "GZIP",
     bloom_filter: true,
   },
@@ -48,7 +48,7 @@ const update: DSNPParquetSchema = [
   },
   {
     name: "targetContentHash",
-    column_type: "BYTE_ARRAY",
+    column_type: "STRING",
     compression: "GZIP",
     bloom_filter: true,
   },