fix: redshift serverless bq partitioning configuration changes (#1730)

rudderlabs · Oct 3, 2024 · 57563e1 · 57563e1
1 parent 20f6cc4
commit 57563e1
Show file tree

Hide file tree

Showing 8 changed files with 308 additions and 6 deletions.
diff --git a/src/configurations/destinations/bq/db-config.json b/src/configurations/destinations/bq/db-config.json
@@ -35,7 +35,9 @@
         "skipUsersTable",
         "jsonPaths",
         "underscoreDivideNumbers",
-        "allowUsersContextTraits"
+        "allowUsersContextTraits",
+        "partitionColumn",
+        "partitionType"
       ],
       "android": [
         "connectionMode",

diff --git a/src/configurations/destinations/bq/schema.json b/src/configurations/destinations/bq/schema.json
@@ -33,6 +33,16 @@
         "pattern": "^(5|15|30|60|180|360|720|1440)$",
         "default": "180"
       },
+      "partitionColumn": {
+        "type": "string",
+        "pattern": "^(_PARTITIONTIME|loaded_at|received_at)$",
+        "default": "_PARTITIONTIME"
+      },
+      "partitionType": {
+        "type": "string",
+        "pattern": "^(hour|day)$",
+        "default": "day"
+      },
       "syncStartAt": {
         "type": "string"
       },

diff --git a/src/configurations/destinations/bq/ui-config.json b/src/configurations/destinations/bq/ui-config.json
@@ -142,6 +142,52 @@
           },
           "required": false,
           "footerNote": "Note: Please specify time in UTC"
+        },
+        {
+          "type": "singleSelect",
+          "immutable": true,
+          "label": "Partition Column",
+          "value": "partitionColumn",
+          "options": [
+            {
+              "name": "Ingestion Time",
+              "value": "_PARTITIONTIME"
+            },
+            {
+              "name": "Loaded At",
+              "value": "loaded_at"
+            },
+            {
+              "name": "Received At",
+              "value": "received_at"
+            }
+          ],
+          "defaultOption": {
+            "name": "Ingestion Time",
+            "value": "_PARTITIONTIME"
+          },
+          "required": false
+        },
+        {
+          "type": "singleSelect",
+          "immutable": true,
+          "label": "Partition Type",
+          "value": "partitionType",
+          "options": [
+            {
+              "name": "Hour",
+              "value": "hour"
+            },
+            {
+              "name": "Day",
+              "value": "day"
+            }
+          ],
+          "defaultOption": {
+            "name": "Day",
+            "value": "day"
+          },
+          "required": false
         }
       ]
     },

diff --git a/src/configurations/destinations/rs/db-config.json b/src/configurations/destinations/rs/db-config.json
@@ -44,6 +44,8 @@
         "iamRoleARNForAuth",
         "clusterId",
         "clusterRegion",
+        "useServerless",
+        "workgroupName",
         "bucketName",
         "iamRoleARN",
         "roleBasedAuth",

diff --git a/src/configurations/destinations/rs/schema.json b/src/configurations/destinations/rs/schema.json
@@ -941,19 +941,51 @@
               "type": "string",
               "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$"
             },
-            "clusterId": {
-              "type": "string",
-              "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$"
-            },
             "clusterRegion": {
               "type": "string",
               "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$"
             },
             "useIAMForAuth": {
               "const": true
+            },
+            "useServerless": {
+              "type": "boolean",
+              "default": false
             }
           },
-          "required": ["clusterId", "clusterRegion", "iamRoleARNForAuth", "useIAMForAuth"]
+          "anyOf": [
+            {
+              "properties": {
+                "clusterId": {
+                  "type": "string",
+                  "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$"
+                },
+                "useServerless": {
+                  "const": false
+                },
+                "useIAMForAuth": {
+                  "const": true
+                }
+              },
+              "required": ["clusterId", "useIAMForAuth"]
+            },
+            {
+              "properties": {
+                "workgroupName": {
+                  "type": "string",
+                  "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$"
+                },
+                "useIAMForAuth": {
+                  "const": true
+                },
+                "useServerless": {
+                  "const": true
+                }
+              },
+              "required": ["workgroupName", "useIAMForAuth", "useServerless"]
+            }
+          ],
+          "required": ["clusterRegion", "iamRoleARNForAuth", "useIAMForAuth"]
         }
       },
       {

diff --git a/src/configurations/destinations/rs/ui-config.json b/src/configurations/destinations/rs/ui-config.json
@@ -40,12 +40,29 @@
           "placeholder": "5439",
           "required": true
         },
+        {
+          "type": "checkbox",
+          "preRequisiteField": [
+            {
+              "name": "useIAMForAuth",
+              "selectedValue": true
+            }
+          ],
+          "label": "Use Redshift Serverless",
+          "value": "useServerless",
+          "required": false,
+          "default": false
+        },
         {
           "type": "textInput",
           "preRequisiteField": [
             {
               "name": "useIAMForAuth",
               "selectedValue": true
+            },
+            {
+              "name": "useServerless",
+              "selectedValue": false
             }
           ],
           "label": "Cluster Id",
@@ -55,6 +72,25 @@
           "placeholder": "Cluster Identifier",
           "required": true
         },
+        {
+          "type": "textInput",
+          "preRequisiteField": [
+            {
+              "name": "useIAMForAuth",
+              "selectedValue": true
+            },
+            {
+              "name": "useServerless",
+              "selectedValue": true
+            }
+          ],
+          "label": "Workgroup Name",
+          "value": "workgroupName",
+          "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$",
+          "regexErrorMessage": "Invalid Workgroup Name",
+          "placeholder": "Workgroup Name",
+          "required": true
+        },
         {
           "type": "textInput",
           "preRequisiteField": [

diff --git a/test/data/validation/destinations/bq.json b/test/data/validation/destinations/bq.json
@@ -13,6 +13,74 @@
     },
     "result": true
   },
+  {
+    "config": {
+      "project": "test-gcs-project",
+      "location": "",
+      "bucketName": "test-bucket",
+      "prefix": "xyzxx",
+      "namespace": "eu_new3",
+      "partitionColumn": "_PARTITIONTIME",
+      "partitionType": "day",
+      "credentials": "{}",
+      "syncFrequency": "30",
+      "testConnection": false,
+      "testConnectionTS": 1621402528550
+    },
+    "result": true
+  },
+  {
+    "config": {
+      "project": "test-gcs-project",
+      "location": "",
+      "bucketName": "test-bucket",
+      "prefix": "xyzxx",
+      "namespace": "eu_new3",
+      "partitionColumn": "received_at",
+      "partitionType": "hour",
+      "credentials": "{}",
+      "syncFrequency": "30",
+      "testConnection": false,
+      "testConnectionTS": 1621402528550
+    },
+    "result": true
+  },
+  {
+    "config": {
+      "project": "test-gcs-project",
+      "location": "",
+      "bucketName": "test-bucket",
+      "prefix": "xyzxx",
+      "namespace": "eu_new3",
+      "partitionColumn": "invalid",
+      "partitionType": "day",
+      "credentials": "{}",
+      "syncFrequency": "30",
+      "testConnection": false,
+      "testConnectionTS": 1621402528550
+    },
+    "result": false,
+    "error": [
+      "partitionColumn must match pattern \"^(_PARTITIONTIME|loaded_at|received_at|timestamp)$\""
+    ]
+  },
+  {
+    "config": {
+      "project": "test-gcs-project",
+      "location": "",
+      "bucketName": "test-bucket",
+      "prefix": "xyzxx",
+      "namespace": "eu_new3",
+      "partitionColumn": "_PARTITIONTIME",
+      "partitionType": "invalid",
+      "credentials": "{}",
+      "syncFrequency": "30",
+      "testConnection": false,
+      "testConnectionTS": 1621402528550
+    },
+    "result": false,
+    "error": ["partitionType must match pattern \"^(hour|day|month|year)$\""]
+  },
   {
     "config": {
       "project": "test-gcs-project",

diff --git a/test/data/validation/destinations/rs.json b/test/data/validation/destinations/rs.json
@@ -125,11 +125,117 @@
     "result": false,
     "err": [
       " must have required property 'clusterId'",
+      " must have required property 'workgroupName'",
+      " must have required property 'useServerless'",
+      " must match a schema in anyOf",
       " must have required property 'clusterRegion'",
       " must have required property 'iamRoleARNForAuth'",
       " must match \"then\" schema"
     ]
   },
+  {
+    "config": {
+      "host": "",
+      "password": "",
+      "port": "",
+      "database": "test-database",
+      "user": "test-user",
+      "useIAMForAuth": true,
+      "useServerless": false,
+      "iamRoleARNForAuth": "iamRoleARNForAuth",
+      "clusterId": "clusterId",
+      "clusterRegion": "clusterRegion",
+      "useSSH": false,
+      "syncFrequency": "30",
+      "enableSSE": false,
+      "useRudderStorage": false,
+      "useSTSTokens": false,
+      "bucketName": "test-bucket",
+      "roleBasedAuth": false,
+      "accessKeyID": "",
+      "accessKey": ""
+    },
+    "result": true
+  },
+  {
+    "config": {
+      "host": "",
+      "password": "",
+      "port": "",
+      "database": "test-database",
+      "user": "test-user",
+      "useIAMForAuth": true,
+      "useServerless": true,
+      "iamRoleARNForAuth": "iamRoleARNForAuth",
+      "clusterId": "clusterId",
+      "clusterRegion": "clusterRegion",
+      "useSSH": false,
+      "syncFrequency": "30",
+      "enableSSE": false,
+      "useRudderStorage": false,
+      "useSTSTokens": false,
+      "bucketName": "test-bucket",
+      "roleBasedAuth": false,
+      "accessKeyID": "",
+      "accessKey": ""
+    },
+    "result": false,
+    "err": [
+      "useServerless must be equal to constant",
+      " must have required property 'workgroupName'",
+      " must match a schema in anyOf",
+      " must match \"then\" schema"
+    ]
+  },
+  {
+    "config": {
+      "host": "",
+      "password": "",
+      "port": "",
+      "database": "test-database",
+      "user": "test-user",
+      "useIAMForAuth": true,
+      "useServerless": true,
+      "iamRoleARNForAuth": "iamRoleARNForAuth",
+      "workgroupName": "workgroupName",
+      "clusterRegion": "clusterRegion",
+      "useSSH": false,
+      "syncFrequency": "30",
+      "enableSSE": false,
+      "useRudderStorage": false,
+      "useSTSTokens": false,
+      "bucketName": "test-bucket",
+      "roleBasedAuth": false,
+      "accessKeyID": "",
+      "accessKey": ""
+    },
+    "result": true
+  },
+  {
+    "config": {
+      "host": "",
+      "password": "",
+      "port": "",
+      "database": "test-database",
+      "user": "test-user",
+      "useIAMForAuth": true,
+      "useServerless": true,
+      "iamRoleARNForAuth": "iamRoleARNForAuth",
+      "clusterId": "clusterId",
+      "workgroupName": "workgroupName",
+      "clusterRegion": "clusterRegion",
+      "useSSH": false,
+      "syncFrequency": "30",
+      "enableSSE": false,
+      "useRudderStorage": false,
+      "useSTSTokens": false,
+      "bucketName": "test-bucket",
+      "roleBasedAuth": false,
+      "accessKeyID": "",
+      "accessKey": ""
+    },
+    "result": true
+  },
   {
     "config": {
       "host": "test-host",