From 57563e193fd3a52609c8ddceeca243238db7a079 Mon Sep 17 00:00:00 2001 From: Akash Chetty Date: Thu, 3 Oct 2024 12:36:52 +0530 Subject: [PATCH] fix: redshift serverless bq partitioning configuration changes (#1730) --- .../destinations/bq/db-config.json | 4 +- .../destinations/bq/schema.json | 10 ++ .../destinations/bq/ui-config.json | 46 ++++++++ .../destinations/rs/db-config.json | 2 + .../destinations/rs/schema.json | 42 ++++++- .../destinations/rs/ui-config.json | 36 ++++++ test/data/validation/destinations/bq.json | 68 +++++++++++ test/data/validation/destinations/rs.json | 106 ++++++++++++++++++ 8 files changed, 308 insertions(+), 6 deletions(-) diff --git a/src/configurations/destinations/bq/db-config.json b/src/configurations/destinations/bq/db-config.json index c9733b01a..dc87d8e71 100644 --- a/src/configurations/destinations/bq/db-config.json +++ b/src/configurations/destinations/bq/db-config.json @@ -35,7 +35,9 @@ "skipUsersTable", "jsonPaths", "underscoreDivideNumbers", - "allowUsersContextTraits" + "allowUsersContextTraits", + "partitionColumn", + "partitionType" ], "android": [ "connectionMode", diff --git a/src/configurations/destinations/bq/schema.json b/src/configurations/destinations/bq/schema.json index 8a1a5b836..ca4b66f9d 100644 --- a/src/configurations/destinations/bq/schema.json +++ b/src/configurations/destinations/bq/schema.json @@ -33,6 +33,16 @@ "pattern": "^(5|15|30|60|180|360|720|1440)$", "default": "180" }, + "partitionColumn": { + "type": "string", + "pattern": "^(_PARTITIONTIME|loaded_at|received_at)$", + "default": "_PARTITIONTIME" + }, + "partitionType": { + "type": "string", + "pattern": "^(hour|day)$", + "default": "day" + }, "syncStartAt": { "type": "string" }, diff --git a/src/configurations/destinations/bq/ui-config.json b/src/configurations/destinations/bq/ui-config.json index 8a10f689b..41760ea6b 100644 --- a/src/configurations/destinations/bq/ui-config.json +++ b/src/configurations/destinations/bq/ui-config.json @@ -142,6 +142,52 @@ }, "required": false, "footerNote": "Note: Please specify time in UTC" + }, + { + "type": "singleSelect", + "immutable": true, + "label": "Partition Column", + "value": "partitionColumn", + "options": [ + { + "name": "Ingestion Time", + "value": "_PARTITIONTIME" + }, + { + "name": "Loaded At", + "value": "loaded_at" + }, + { + "name": "Received At", + "value": "received_at" + } + ], + "defaultOption": { + "name": "Ingestion Time", + "value": "_PARTITIONTIME" + }, + "required": false + }, + { + "type": "singleSelect", + "immutable": true, + "label": "Partition Type", + "value": "partitionType", + "options": [ + { + "name": "Hour", + "value": "hour" + }, + { + "name": "Day", + "value": "day" + } + ], + "defaultOption": { + "name": "Day", + "value": "day" + }, + "required": false } ] }, diff --git a/src/configurations/destinations/rs/db-config.json b/src/configurations/destinations/rs/db-config.json index 6c6d9de0c..c1fe5e62e 100644 --- a/src/configurations/destinations/rs/db-config.json +++ b/src/configurations/destinations/rs/db-config.json @@ -44,6 +44,8 @@ "iamRoleARNForAuth", "clusterId", "clusterRegion", + "useServerless", + "workgroupName", "bucketName", "iamRoleARN", "roleBasedAuth", diff --git a/src/configurations/destinations/rs/schema.json b/src/configurations/destinations/rs/schema.json index 5290eb6d8..d7678a83b 100644 --- a/src/configurations/destinations/rs/schema.json +++ b/src/configurations/destinations/rs/schema.json @@ -941,19 +941,51 @@ "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" }, - "clusterId": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" - }, "clusterRegion": { "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" }, "useIAMForAuth": { "const": true + }, + "useServerless": { + "type": "boolean", + "default": false } }, - "required": ["clusterId", "clusterRegion", "iamRoleARNForAuth", "useIAMForAuth"] + "anyOf": [ + { + "properties": { + "clusterId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" + }, + "useServerless": { + "const": false + }, + "useIAMForAuth": { + "const": true + } + }, + "required": ["clusterId", "useIAMForAuth"] + }, + { + "properties": { + "workgroupName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" + }, + "useIAMForAuth": { + "const": true + }, + "useServerless": { + "const": true + } + }, + "required": ["workgroupName", "useIAMForAuth", "useServerless"] + } + ], + "required": ["clusterRegion", "iamRoleARNForAuth", "useIAMForAuth"] } }, { diff --git a/src/configurations/destinations/rs/ui-config.json b/src/configurations/destinations/rs/ui-config.json index 94991e2b2..5e8d04057 100644 --- a/src/configurations/destinations/rs/ui-config.json +++ b/src/configurations/destinations/rs/ui-config.json @@ -40,12 +40,29 @@ "placeholder": "5439", "required": true }, + { + "type": "checkbox", + "preRequisiteField": [ + { + "name": "useIAMForAuth", + "selectedValue": true + } + ], + "label": "Use Redshift Serverless", + "value": "useServerless", + "required": false, + "default": false + }, { "type": "textInput", "preRequisiteField": [ { "name": "useIAMForAuth", "selectedValue": true + }, + { + "name": "useServerless", + "selectedValue": false } ], "label": "Cluster Id", @@ -55,6 +72,25 @@ "placeholder": "Cluster Identifier", "required": true }, + { + "type": "textInput", + "preRequisiteField": [ + { + "name": "useIAMForAuth", + "selectedValue": true + }, + { + "name": "useServerless", + "selectedValue": true + } + ], + "label": "Workgroup Name", + "value": "workgroupName", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$", + "regexErrorMessage": "Invalid Workgroup Name", + "placeholder": "Workgroup Name", + "required": true + }, { "type": "textInput", "preRequisiteField": [ diff --git a/test/data/validation/destinations/bq.json b/test/data/validation/destinations/bq.json index 8ecb38f76..feed21204 100644 --- a/test/data/validation/destinations/bq.json +++ b/test/data/validation/destinations/bq.json @@ -13,6 +13,74 @@ }, "result": true }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "_PARTITIONTIME", + "partitionType": "day", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": true + }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "received_at", + "partitionType": "hour", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": true + }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "invalid", + "partitionType": "day", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": false, + "error": [ + "partitionColumn must match pattern \"^(_PARTITIONTIME|loaded_at|received_at|timestamp)$\"" + ] + }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "_PARTITIONTIME", + "partitionType": "invalid", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": false, + "error": ["partitionType must match pattern \"^(hour|day|month|year)$\""] + }, { "config": { "project": "test-gcs-project", diff --git a/test/data/validation/destinations/rs.json b/test/data/validation/destinations/rs.json index 7419ae921..966873298 100644 --- a/test/data/validation/destinations/rs.json +++ b/test/data/validation/destinations/rs.json @@ -125,11 +125,117 @@ "result": false, "err": [ " must have required property 'clusterId'", + " must have required property 'workgroupName'", + " must have required property 'useServerless'", + " must match a schema in anyOf", " must have required property 'clusterRegion'", " must have required property 'iamRoleARNForAuth'", " must match \"then\" schema" ] }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": false, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "clusterId": "clusterId", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": true + }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": true, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "clusterId": "clusterId", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": false, + "err": [ + "useServerless must be equal to constant", + " must have required property 'workgroupName'", + " must match a schema in anyOf", + " must match \"then\" schema" + ] + }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": true, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "workgroupName": "workgroupName", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": true + }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": true, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "clusterId": "clusterId", + "workgroupName": "workgroupName", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": true + }, { "config": { "host": "test-host",