From ae753c7cc7d3a3e1bef00ad3b2f700841da3ce32 Mon Sep 17 00:00:00 2001 From: Akash Chetty Date: Fri, 25 Oct 2024 15:26:21 +0530 Subject: [PATCH] feat: snowpipe streaming (#1688) --- .../snowpipe_streaming/db-config.json | 124 +++ .../snowpipe_streaming/schema.json | 884 ++++++++++++++++++ .../snowpipe_streaming/ui-config.json | 246 +++++ .../destinations/snowpipe_streaming.json | 720 ++++++++++++++ 4 files changed, 1974 insertions(+) create mode 100644 src/configurations/destinations/snowpipe_streaming/db-config.json create mode 100644 src/configurations/destinations/snowpipe_streaming/schema.json create mode 100644 src/configurations/destinations/snowpipe_streaming/ui-config.json create mode 100644 test/data/validation/destinations/snowpipe_streaming.json diff --git a/src/configurations/destinations/snowpipe_streaming/db-config.json b/src/configurations/destinations/snowpipe_streaming/db-config.json new file mode 100644 index 000000000..08ba89b9a --- /dev/null +++ b/src/configurations/destinations/snowpipe_streaming/db-config.json @@ -0,0 +1,124 @@ +{ + "name": "SNOWPIPE_STREAMING", + "displayName": "Snowpipe Streaming", + "category": "warehouse", + "config": { + "transformAtV1": "processor", + "saveDestinationResponse": true, + "excludeKeys": [], + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative", + "cloudSource", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud"], + "ios": ["cloud"], + "web": ["cloud"], + "unity": ["cloud"], + "amp": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"], + "cloud": ["cloud"], + "cloudSource": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "account", + "database", + "warehouse", + "user", + "role", + "skipTracksTable", + "namespace", + "jsonPaths", + "privateKey", + "privateKeyPassphrase" + ], + "android": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "ios": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "web": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "unity": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "amp": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "cloud": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "cloudSource": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "reactnative": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "flutter": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "cordova": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "shopify": [ + "consentManagement", + "connectionMode", + "oneTrustCookieCategories", + "ketchConsentPurposes" + ] + }, + "secretKeys": ["privateKey", "privateKeyPassphrase"] + }, + "options": { + "isBeta": true, + "hidden": { + "featureFlagName": "AMP_snowpipe_streaming", + "featureFlagValue": false + } + } +} diff --git a/src/configurations/destinations/snowpipe_streaming/schema.json b/src/configurations/destinations/snowpipe_streaming/schema.json new file mode 100644 index 000000000..a01bcd682 --- /dev/null +++ b/src/configurations/destinations/snowpipe_streaming/schema.json @@ -0,0 +1,884 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["account", "database", "warehouse", "user", "privateKey", "namespace"], + "type": "object", + "properties": { + "account": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "database": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "warehouse": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "user": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "role": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "namespace": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^((?!pg_|PG_|pG_|Pg_).{1,64})$" + }, + "skipTracksTable": { + "type": "boolean", + "default": false + }, + "jsonPaths": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.*)$" + }, + "privateKey": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|-----BEGIN (?:ENCRYPTED )?PRIVATE KEY-----[\\s\\S]+?-----END (?:ENCRYPTED )?PRIVATE KEY-----" + }, + "privateKeyPassphrase": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "oneTrustCookieCategories": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cloudSource": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + } + }, + "ketchConsentPurposes": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cloudSource": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + } + }, + "consentManagement": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloudSource": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "cloudSource": { + "type": "string", + "enum": ["cloud"] + } + } + } + }, + "additionalProperties": true + } +} diff --git a/src/configurations/destinations/snowpipe_streaming/ui-config.json b/src/configurations/destinations/snowpipe_streaming/ui-config.json new file mode 100644 index 000000000..89bd3162e --- /dev/null +++ b/src/configurations/destinations/snowpipe_streaming/ui-config.json @@ -0,0 +1,246 @@ +{ + "uiConfig": [ + { + "title": "Connection Credentials", + "fields": [ + { + "type": "textInput", + "label": "Account", + "value": "account", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Account", + "placeholder": "e.g: qya5X180.us-east-1", + "required": true + }, + { + "type": "textInput", + "label": "Database", + "value": "database", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Database", + "placeholder": "e.g: RUDDER_EVENTS", + "required": true + }, + { + "type": "textInput", + "label": "Warehouse", + "value": "warehouse", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Database", + "placeholder": "e.g: RUDDER_WAREHOUSE", + "required": true + }, + { + "type": "textInput", + "label": "User", + "value": "user", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid User", + "placeholder": "e.g: RUDDER_USER", + "required": true + }, + { + "type": "textInput", + "label": "Role", + "value": "role", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "regexErrorMessage": "Invalid Role", + "placeholder": "e.g: RUDDER_ROLE", + "required": false + }, + { + "type": "textareaInput", + "required": true, + "regex": "-----BEGIN (?:ENCRYPTED )?PRIVATE KEY-----[\\s\\S]+?-----END (?:ENCRYPTED )?PRIVATE KEY-----", + "label": "Private Key", + "value": "privateKey", + "secret": true + }, + { + "type": "textInput", + "regex": "^(.{0,100})$", + "secret": true, + "label": "Private Key Passphrase", + "value": "privateKeyPassphrase", + "footerNote": "Use the password you set when encrypting the private key. Leave it blank if the private key is not encrypted.", + "required": false + }, + { + "type": "textInput", + "label": "Namespace", + "labelNote": "Schema name for the warehouse where the tables are created", + "value": "namespace", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^((?!pg_|PG_|pG_|Pg_).{1,64})$", + "regexErrorMessage": "Invalid Namespace", + "required": true, + "placeholder": "e.g: RUDDER_NAMESPACE", + "immutable": true + } + ] + }, + { + "title": "Advanced Settings", + "fields": [ + { + "type": "checkbox", + "label": "Skip Tracks Table", + "value": "skipTracksTable", + "footerNote": "Enable this feature to skip sending the event data to the “tracks” table", + "default": false + }, + { + "type": "textInput", + "label": "JSON columns (Optional)", + "labelNote": "Specify required JSON properties in dot notation separated by commas", + "value": "jsonPaths", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.*)$", + "placeholder": "e.g: testMap.nestedMap, testArray", + "required": false, + "footerURL": { + "link": "https://www.rudderstack.com/docs/destinations/warehouse-destinations/json-column-support/", + "text": "Instructions for setting up the JSON columns" + } + } + ] + }, + { + "title": "Consent Settings", + "fields": [ + { + "type": "dynamicCustomForm", + "value": "oneTrustCookieCategories", + "label": "OneTrust Consent Category IDs", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "textInput", + "placeholder": "C0001", + "value": "oneTrustCookieCategory", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "label": "Category ID", + "required": false + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purpose IDs", + "customFields": [ + { + "type": "textInput", + "placeholder": "marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "consentManagement", + "label": "Consent management settings", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "value": "provider", + "options": [ + { + "name": "Custom", + "value": "custom" + }, + { + "name": "Ketch", + "value": "ketch" + }, + { + "name": "OneTrust", + "value": "oneTrust" + } + ], + "defaultOption": { + "name": "OneTrust", + "value": "oneTrust" + }, + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "value": "resolutionStrategy", + "options": [ + { + "name": "AND", + "value": "and" + }, + { + "name": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "dynamicCustomForm", + "value": "consents", + "label": "Enter consent category ID’s", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "consent", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } + } + ] + } + ] +} diff --git a/test/data/validation/destinations/snowpipe_streaming.json b/test/data/validation/destinations/snowpipe_streaming.json new file mode 100644 index 000000000..ce41255d0 --- /dev/null +++ b/test/data/validation/destinations/snowpipe_streaming.json @@ -0,0 +1,720 @@ +[ + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n" + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "" + }, + "result": true + }, + { + "config": { + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [" must have required property 'account'"] + }, + { + "config": { + "account": "", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": ["account must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] + }, + { + "config": { + "account": "test-account", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [" must have required property 'database'"] + }, + { + "config": { + "account": "test-account", + "database": "", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": ["database must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [" must have required property 'warehouse'"] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [ + "warehouse must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [" must have required property 'user'"] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": ["user must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [" must have required property 'privateKey'"] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [ + "privateKey must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|-----BEGIN (?:ENCRYPTED )?PRIVATE KEY-----[\\s\\S]+?-----END (?:ENCRYPTED )?PRIVATE KEY-----\"" + ] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "abc", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [" must have required property 'namespace'"] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase" + }, + "result": false, + "err": [ + "namespace must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^((?!pg_|PG_|pG_|Pg_).{1,64})$\"" + ] + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "consentManagement": { + "android": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "consentManagement": { + "android": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0 must have required property 'resolutionStrategy'", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "consentManagement": { + "android": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "consentManagement": { + "android": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "oneTrustCookieCategories": { + "android": [ + { + "oneTrustCookieCategory": "C0001" + }, + { + "oneTrustCookieCategory": "C0002" + } + ], + "ios": [ + { + "oneTrustCookieCategory": "C0003" + }, + { + "oneTrustCookieCategory": "C0004" + } + ], + "web": [ + { + "oneTrustCookieCategory": "" + } + ], + "unity": [], + "amp": [ + { + "oneTrustCookieCategory": "env.ENVIRONMENT_VARIABLE" + } + ], + "cloud": [ + { + "oneTrustCookieCategory": "{{ event.properties.prop1 || 'val' }}" + } + ], + "reactnative": [ + { + "oneTrustCookieCategory": "C0001" + }, + { + "oneTrustCookieCategory": "C0002" + } + ], + "cloudSource": [ + { + "oneTrustCookieCategory": "C0003" + }, + { + "oneTrustCookieCategory": "C0004" + } + ], + "flutter": [ + { + "oneTrustCookieCategory": "" + } + ], + "cordova": [], + "shopify": [ + { + "oneTrustCookieCategory": "env.ENVIRONMENT_VARIABLE" + } + ] + }, + "ketchConsentPurposes": { + "android": [ + { + "purpose": "P1" + }, + { + "purpose": "P2" + } + ], + "ios": [ + { + "purpose": "P3" + }, + { + "purpose": "P4" + } + ], + "web": [ + { + "purpose": "" + } + ], + "unity": [], + "amp": [ + { + "purpose": "env.ENVIRONMENT_VARIABLE" + } + ], + "cloud": [ + { + "purpose": "{{ event.properties.prop1 || 'val' }}" + } + ], + "reactnative": [ + { + "purpose": "P1" + }, + { + "purpose": "P2" + } + ], + "cloudSource": [ + { + "purpose": "P3" + }, + { + "purpose": "P4" + } + ], + "flutter": [ + { + "purpose": "" + } + ], + "cordova": [], + "shopify": [ + { + "purpose": "env.ENVIRONMENT_VARIABLE" + } + ] + } + }, + "result": true + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "C0001" + }, + { + "oneTrustCookieCategory": "C0002" + } + ], + "ketchConsentPurposes": [ + { + "purpose": "P1" + }, + { + "purpose": "P2" + } + ] + }, + "result": false, + "err": ["oneTrustCookieCategories must be object", "ketchConsentPurposes must be object"] + }, + { + "config": { + "account": "test-account", + "database": "test-database", + "warehouse": "test-warehouse", + "user": "test-user", + "role": "test-role", + "skipTracksTable": false, + "namespace": "test-namespace", + "jsonPaths": "testMap.nestedMap", + "privateKey": "-----BEGIN PRIVATE KEY-----\ntest\ntest\n-----END PRIVATE KEY-----\n", + "privateKeyPassphrase": "test-passphrase", + "oneTrustCookieCategories": { + "android": [ + { + "oneTrustCookieCategory": "more than 100 characters string - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + }, + { + "oneTrustCookieCategory": "C0004" + } + ], + "ios": [ + { + "oneTrustCookieCategory": { + "not": "a string" + } + }, + { + "oneTrustCookieCategory": "C0004" + } + ], + "web": { + "not": "an array" + }, + "unity": [ + "not an object", + { + "oneTrustCookieCategory": "C0004" + } + ] + }, + "ketchConsentPurposes": { + "android": [ + { + "purpose": "more than 100 characters string - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + }, + { + "purpose": "P4" + } + ], + "ios": [ + { + "purpose": { + "not": "a string" + } + }, + { + "purpose": "P4" + } + ], + "web": { + "not": "an array" + }, + "unity": [ + "not an object", + { + "purpose": "P4" + } + ] + } + }, + "result": false, + "err": [ + "oneTrustCookieCategories.android.0.oneTrustCookieCategory must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", + "oneTrustCookieCategories.ios.0.oneTrustCookieCategory must be string", + "oneTrustCookieCategories.web must be array", + "oneTrustCookieCategories.unity.0 must be object", + "ketchConsentPurposes.android.0.purpose must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", + "ketchConsentPurposes.ios.0.purpose must be string", + "ketchConsentPurposes.web must be array", + "ketchConsentPurposes.unity.0 must be object" + ] + } +]