diff --git a/CHANGELOG.md b/CHANGELOG.md index ab2efd1f3..ba3809549 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.74.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.73.0...v1.74.0) (2024-05-13) + + +### Features + +* **criteo:** add support of sha256 hashing method for email field ([#1301](https://github.com/rudderlabs/rudder-config-schema/issues/1301)) +* improve schema generator ([#1207](https://github.com/rudderlabs/rudder-config-schema/issues/1207)) +* onboard emersys destination ([#1335](https://github.com/rudderlabs/rudder-config-schema/issues/1335)) +* onboard new destination sftp ([#1321](https://github.com/rudderlabs/rudder-integrations-config/pull/1321)) + +### Bug Fixes + +* adding group call to emarsys ([#1349](https://github.com/rudderlabs/rudder-config-schema/issues/1349)) +* remove useNativeSdk from snowflake ([#1359](https://github.com/rudderlabs/rudder-config-schema/issues/1359)) +* add unity and web as supported source type on singular ([#1370](https://github.com/rudderlabs/rudder-integrations-config/pull/1370)) + ## [1.73.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.72.0...v1.73.0) (2024-05-08) diff --git a/package-lock.json b/package-lock.json index d8abca87d..61a089cd4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-config-schema", - "version": "1.73.0", + "version": "1.74.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-config-schema", - "version": "1.73.0", + "version": "1.74.0", "license": "MIT", "dependencies": { "ajv": "^8.12.0", diff --git a/package.json b/package.json index 8f01cd890..eefc559b0 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-config-schema", - "version": "1.73.0", + "version": "1.74.0", "description": "", "main": "src/index.ts", "private": true, diff --git a/scripts/schemaGenerator.py b/scripts/schemaGenerator.py index 5f5c8b117..44b0521a1 100644 --- a/scripts/schemaGenerator.py +++ b/scripts/schemaGenerator.py @@ -126,6 +126,26 @@ def is_dest_field_dependent_on_source(field, dbConfig, schema_field_name): return False +def is_key_present_in_dest_config(dbConfig, key): + """Checks if the given key is present in destConfig across all source types. + + Args: + dbConfig (object): Destination configuration in db-config.json. + key (string): key to be searched in destConfig tree. + + Returns: + boolean: True if the key is present in destConfig else, False. + """ + if not dbConfig: + return False + + if "destConfig" in dbConfig: + for configSection in dbConfig["destConfig"]: + if key in dbConfig["destConfig"][configSection]: + return True + return False + + def is_field_present_in_default_config(field, dbConfig, schema_field_name): """Checks if the given field is present in defaultConfig list present in dbConfig. @@ -740,6 +760,7 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name): - For each unique preRequisiteField, the properties are found by matching the current preRequisiteField. - preRequisiteField becomes if block and corresponding properties become then block. + Args: uiConfig (object): file content of ui-config.json. dbConfig (object): Configurations of db-config.json. @@ -971,9 +992,7 @@ def generate_connection_mode(dbConfig): return connectionObj -def generate_schema_properties( - uiConfig, dbConfig, schemaObject, properties, name, selector -): +def generate_schema_properties(uiConfig, dbConfig, schemaObject, properties, selector): """Generates corresponding schema properties by iterating over each of the ui-config fields. Args: @@ -981,7 +1000,6 @@ def generate_schema_properties( dbConfig (object): Configurations of db-config.json. schemaObject (object): schema being generated properties (object): properties of schema - name (string): name of the source or destination. selector (string): either 'source' or 'destination' """ if is_old_format(uiConfig): @@ -992,9 +1010,16 @@ def generate_schema_properties( continue generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field["value"]] = generateFunction( - field, dbConfig, "value" - ) + # Generate schema for the field if it is defined in the destination config + if is_key_present_in_dest_config(dbConfig, field["value"]): + properties[field["value"]] = generateFunction( + field, dbConfig, "value" + ) + else: + warnings.warn( + f'The field {field["value"]} is defined in ui-config.json but not in db-config.json\n', + UserWarning, + ) if field.get( "required", False ) == True and is_field_present_in_default_config( @@ -1012,9 +1037,18 @@ def generate_schema_properties( for field in group.get("fields", []): generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field["configKey"]] = generateFunction( - field, dbConfig, "configKey" - ) + # Generate schema for the field if it is defined in the destination config + if is_key_present_in_dest_config( + dbConfig, field["configKey"] + ): + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + else: + warnings.warn( + f'The field {field["configKey"]} is defined in ui-config.json but not in db-config.json\n', + UserWarning, + ) if ( template.get("title", "") == "Initial setup" and is_field_present_in_default_config( @@ -1027,9 +1061,17 @@ def generate_schema_properties( for field in sdkTemplate.get("fields", []): generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field["configKey"]] = generateFunction( - field, dbConfig, "configKey" - ) + # Generate schema for the field if it is defined in the destination config + if is_key_present_in_dest_config(dbConfig, field["configKey"]): + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + else: + warnings.warn( + f'The field {field["configKey"]} is defined in ui-config.json but not in db-config.json\n', + UserWarning, + ) + if field.get( "required", False ) == True and is_field_present_in_default_config( @@ -1051,9 +1093,12 @@ def generate_schema_properties( schemaObject["required"].append(field["configKey"]) # default properties in new ui-config based schemas. - schemaObject["properties"]["useNativeSDK"] = generate_schema_for_checkbox( - {"type": "checkbox", "value": "useNativeSDK"}, dbConfig, "value" - ) + if is_key_present_in_dest_config(dbConfig, "useNativeSDK"): + schemaObject["properties"]["useNativeSDK"] = ( + generate_schema_for_checkbox( + {"type": "checkbox", "value": "useNativeSDK"}, dbConfig, "value" + ) + ) schemaObject["properties"]["connectionMode"] = generate_connection_mode( dbConfig ) @@ -1099,6 +1144,7 @@ def generate_schema(uiConfig, dbConfig, name, selector): schemaObject["type"] = "object" schemaObject["properties"] = {} allOfSchemaObj = {} + print(f"Generating schema for {name} {selector}") if is_old_format(uiConfig): allOfSchemaObj = generate_schema_for_allOf(uiConfig, dbConfig, "value") if allOfSchemaObj: @@ -1110,8 +1156,9 @@ def generate_schema(uiConfig, dbConfig, name, selector): schemaObject["anyOf"] = allOfSchemaObj else: schemaObject["allOf"] = allOfSchemaObj + generate_schema_properties( - uiConfig, dbConfig, schemaObject, schemaObject["properties"], name, selector + uiConfig, dbConfig, schemaObject, schemaObject["properties"], selector ) newSchema["configSchema"] = schemaObject @@ -1154,7 +1201,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): ) if ( curUiType == "textInput" - and field["value"] in schema["required"] + and ( + schema.get("required", False) == True + and field["value"] in schema["required"] + ) and "regex" not in field ): warnings.warn( @@ -1204,7 +1254,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): ) if ( curUiType == "textInput" - and field["configKey"] in schema["required"] + and ( + schema.get("required", False) == True + and field["configKey"] in schema["required"] + ) and "regex" not in field ): warnings.warn( @@ -1423,11 +1476,9 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False): selector (string): either 'source' or 'destination'. shouldUpdateSchema (boolean): if it should update the existing schema with generated one """ - file_selectors = ["db-config.json", "ui-config.json", "schema.json"] directory = f"./{CONFIG_DIR}/{selector}s/{name}" if not os.path.isdir(directory): - print(f"No {selector}s directory found for {name}") return if name not in EXCLUDED_DEST: diff --git a/src/configurations/destinations/af/db-config.json b/src/configurations/destinations/af/db-config.json index ddbc19547..8a62b0978 100644 --- a/src/configurations/destinations/af/db-config.json +++ b/src/configurations/destinations/af/db-config.json @@ -57,6 +57,7 @@ "sharingFilter", "useRichEventName", "addPropertiesAtRoot", + "afCurrencyAtRoot", "androidAppId", "appleAppId", "blacklistedEvents", diff --git a/src/configurations/destinations/af/schema.json b/src/configurations/destinations/af/schema.json index bb337a2a1..33fee3357 100644 --- a/src/configurations/destinations/af/schema.json +++ b/src/configurations/destinations/af/schema.json @@ -18,6 +18,7 @@ }, "useRichEventName": { "type": "boolean", "default": false }, "addPropertiesAtRoot": { "type": "boolean", "default": false }, + "afCurrencyAtRoot": { "type": "boolean", "default": false }, "sharingFilter": { "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" diff --git a/src/configurations/destinations/af/ui-config.json b/src/configurations/destinations/af/ui-config.json index 1c875a717..ff2bd9b2d 100644 --- a/src/configurations/destinations/af/ui-config.json +++ b/src/configurations/destinations/af/ui-config.json @@ -46,6 +46,21 @@ "footerNote": "To send the custom properties to the root of eventValue.", "default": false }, + { + "type": "checkbox", + "label": "Add af_currency to root of eventValue", + "value": "afCurrencyAtRoot", + "footerNote": "af_currency is added at the root of eventValue, outside properties object.", + "default": false, + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_appsflyer_add_prop_to_root", + "value": true + } + ] + } + }, { "type": "textInput", "label": "Sharing Filter", diff --git a/src/configurations/destinations/criteo/schema.json b/src/configurations/destinations/criteo/schema.json index f59c28551..22d847da6 100644 --- a/src/configurations/destinations/criteo/schema.json +++ b/src/configurations/destinations/criteo/schema.json @@ -12,7 +12,11 @@ "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(.{0,100})$" }, - "hashMethod": { "type": "string", "enum": ["none", "md5"], "default": "none" }, + "hashMethod": { + "type": "string", + "enum": ["none", "md5", "sha256"], + "default": "none" + }, "fieldMapping": { "type": "array", "items": { diff --git a/src/configurations/destinations/criteo/ui-config.json b/src/configurations/destinations/criteo/ui-config.json index 951d07631..df9ced287 100644 --- a/src/configurations/destinations/criteo/ui-config.json +++ b/src/configurations/destinations/criteo/ui-config.json @@ -35,6 +35,10 @@ { "name": "MD5", "value": "md5" + }, + { + "name": "SHA256", + "value": "sha256" } ], "defaultOption": { diff --git a/src/configurations/destinations/emarsys/db-config.json b/src/configurations/destinations/emarsys/db-config.json new file mode 100644 index 000000000..ab6049181 --- /dev/null +++ b/src/configurations/destinations/emarsys/db-config.json @@ -0,0 +1,66 @@ +{ + "name": "EMARSYS", + "displayName": "Emarsys", + "config": { + "cdkV2Enabled": true, + "transformAtV1": "router", + "saveDestinationResponse": true, + "excludeKeys": [], + "supportedSourceTypes": [ + "android", + "ios", + "unity", + "amp", + "reactnative", + "flutter", + "cordova", + "web", + "cloud", + "shopify", + "warehouse" + ], + "supportedConnectionModes": { + "android": ["cloud"], + "ios": ["cloud"], + "web": ["cloud"], + "unity": ["cloud"], + "amp": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] + }, + "supportedMessageTypes": { + "cloud": ["identify", "track", "group"] + }, + "destConfig": { + "defaultConfig": [ + "emersysUsername", + "emersysUserSecret", + "eventsMapping", + "fieldMapping", + "emersysCustomIdentifier", + "defaultContactList", + "discardEmptyProperties", + "oneTrustCookieCategories" + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "web": ["connectionMode"], + "cloud": ["connectionMode"], + "shopify": ["connectionMode"], + "warehouse": ["connectionMode"] + }, + "secretKeys": ["emersysUsername", "emersysUserSecret"] + }, + "options": { + "isBeta": true + } +} diff --git a/src/configurations/destinations/emarsys/schema.json b/src/configurations/destinations/emarsys/schema.json new file mode 100644 index 000000000..cb4e8596c --- /dev/null +++ b/src/configurations/destinations/emarsys/schema.json @@ -0,0 +1,122 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["emersysUsername", "emersysUserSecret"], + "type": "object", + "properties": { + "emersysUsername": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "emersysUserSecret": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "defaultContactList": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "emersysCustomIdentifier": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "eventsMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "to": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + } + } + } + }, + "fieldMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "to": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + } + } + } + }, + "discardEmptyProperties": { + "type": "boolean", + "default": true + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/src/configurations/destinations/emarsys/ui-config.json b/src/configurations/destinations/emarsys/ui-config.json new file mode 100644 index 000000000..69fe93a58 --- /dev/null +++ b/src/configurations/destinations/emarsys/ui-config.json @@ -0,0 +1,260 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection Settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Emarsys Username", + "configKey": "emersysUsername", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid Username", + "placeholder": "e.g. dummy_username" + }, + { + "type": "textInput", + "label": "Emarsys User Secret", + "configKey": "emersysUserSecret", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid User Secret", + "placeholder": "e.g. dummy_userSecret" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding.", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [ + { + "title": "Custom Identifier Field Settings", + "fields": [ + { + "type": "dynamicDataSelect", + "configKey": "emersysCustomIdentifier", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "regexErrorMessage": "Invalid Custom Identifier", + "label": "Choose Custom Identifier From Your Emarsys Properties List. If not chosen by you Rudderstack will consider 'email' as a custom identifier", + "placeholder": "e.g 1234567", + "apiName": "customIdentifier" + } + ] + }, + { + "title": "Default Contact List Field Settings", + "fields": [ + { + "type": "dynamicDataSelect", + "configKey": "defaultContactList", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "regexErrorMessage": "Invalid Contact List", + "label": "Choose Default Contact List From Emarsys Account.", + "placeholder": "e.g 1234567", + "apiName": "contactList" + } + ] + }, + { + "title": "Optional configuration settings", + "fields": [ + { + "type": "checkbox", + "label": "Discard empty contact properties", + "configKey": "discardEmptyProperties", + "default": true, + "note": "Disable it if you are not sending null or empty values for contact properties" + } + ] + } + ] + }, + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "OneTrust consent settings", + "note": [ + "Enter your OneTrust consent category IDs if you have them configured. The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/consent-manager/onetrust/" + }, + "about RudderStack's OneTrust Consent Management feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Consent categories", + "note": "Input your OneTrust category IDs by pressing 'Enter' after each entry.", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: C0001", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + }, + { + "title": "Contact Property mapping", + "note": "Map RudderStack Events Properties to Emarsys Contact Properties. This mapping will referenced from message.traits or message.context.traits while making identify and group calls", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "RudderStack Event name to Emarsys Contact Properties Mapping", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "fieldMapping", + "label": "Custom Property Mapping", + "note": "Map RudderStack events properties to Emarsys custom fields" + } + ] + } + ] + } + ] + }, + { + "title": "Event mapping", + "note": "Map RudderStack Events to Emarsys External Events", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "RudderStack Event name to Emarsys Ads External Event Name Mapping", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "eventsMapping", + "label": "External Event Mapping", + "note": "Map RudderStack events to Emersys external events" + } + ] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] + }, + "redirectGroups": { + "fieldMapping": { + "fields": [ + { + "type": "dynamicCustomForm", + "label": "Map Rudderstack event traits to Emarsys contact properties. This is a 1:1 mapping.", + "configKey": "fieldMapping", + "rowFields": [ + { + "type": "mappingRow", + "columns": [ + { + "type": "textInput", + "configKey": "rudderProperty", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Event Property", + "label": "RudderStack Event Property", + "placeholder": "e.g city" + }, + { + "type": "dynamicDataSelect", + "configKey": "emersysProperty", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Custom Property", + "label": "Emersys custom property name", + "placeholder": "e.g companyCity", + "apiName": "customProperty" + } + ] + } + ] + } + ] + }, + "eventsMapping": { + "fields": [ + { + "type": "dynamicCustomForm", + "label": "Map Rudderstack Events to Emarsys external events. This is a 1:1 mapping.", + "configKey": "eventMapping", + "rowFields": [ + { + "type": "mappingRow", + "columns": [ + { + "type": "textInput", + "configKey": "from", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Event Name", + "label": "RudderStack Event Name", + "placeholder": "e.g product added" + }, + { + "type": "dynamicDataSelect", + "configKey": "to", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid external event", + "label": "Emarsys external event name", + "placeholder": "e.g addToCart", + "apiName": "externalEvent" + } + ] + } + ] + } + ] + } + } + } +} diff --git a/src/configurations/destinations/ga4/db-config.json b/src/configurations/destinations/ga4/db-config.json index 6a053e617..d5e3588ea 100644 --- a/src/configurations/destinations/ga4/db-config.json +++ b/src/configurations/destinations/ga4/db-config.json @@ -94,5 +94,8 @@ "warehouse": ["consentManagement", "connectionMode"] }, "secretKeys": ["apiSecret"] + }, + "options": { + "isBeta": false } } diff --git a/src/configurations/destinations/sftp/db-config.json b/src/configurations/destinations/sftp/db-config.json new file mode 100644 index 000000000..48e373d3e --- /dev/null +++ b/src/configurations/destinations/sftp/db-config.json @@ -0,0 +1,35 @@ +{ + "name": "SFTP", + "displayName": "SFTP", + "config": { + "transformAtV1": "none", + "saveDestinationResponse": true, + "disableJsonMapper": true, + "supportsVisualMapper": true, + "supportedSourceTypes": ["warehouse"], + "supportedMessageTypes": { + "cloud": ["record"] + }, + "supportedConnectionModes": { + "warehouse": ["cloud"] + }, + "syncBehaviours": ["mirror"], + "destConfig": { + "defaultConfig": [ + "host", + "port", + "authMethod", + "username", + "password", + "privateKey", + "fileFormat", + "filePath", + "oneTrustCookieCategories" + ] + }, + "secretKeys": ["password", "privateKey"] + }, + "options": { + "isBeta": true + } +} diff --git a/src/configurations/destinations/sftp/schema.json b/src/configurations/destinations/sftp/schema.json new file mode 100644 index 000000000..1068f6754 --- /dev/null +++ b/src/configurations/destinations/sftp/schema.json @@ -0,0 +1,134 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["host", "port", "username", "authMethod", "fileFormat", "filePath"], + "type": "object", + "properties": { + "host": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(.{1,100})$" + }, + "port": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?:[1-9][0-9]{0,4}|[1-5][0-9]{4}|6[0-5]{4}|6553[0-5])$" + }, + "username": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "authMethod": { + "type": "string", + "enum": ["passwordAuth", "keyAuth"], + "default": "passwordAuth" + }, + "fileFormat": { + "type": "string", + "enum": ["json", "csv"], + "default": "csv" + }, + "filePath": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^.*\\.(json|csv)$" + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "warehouse": { + "type": "string", + "enum": ["cloud"] + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "authMethod": { + "const": "passwordAuth" + } + }, + "required": ["authMethod"] + }, + "then": { + "properties": { + "password": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + } + }, + "required": ["password"] + } + }, + { + "if": { + "properties": { + "authMethod": { + "const": "keyAuth" + } + }, + "required": ["authMethod"] + }, + "then": { + "properties": { + "privateKey": { + "type": "string", + "pattern": "^[\\s\\S]+$" + } + }, + "required": ["privateKey"] + } + }, + { + "if": { + "properties": { + "fileFormat": { + "const": "csv" + } + }, + "required": ["fileFormat"] + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "pattern": "\\.csv$" + } + }, + "required": ["filePath"] + } + }, + { + "if": { + "properties": { + "fileFormat": { + "const": "json" + } + }, + "required": ["fileFormat"] + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "pattern": "\\.json$" + } + }, + "required": ["filePath"] + } + } + ] + } +} diff --git a/src/configurations/destinations/sftp/ui-config.json b/src/configurations/destinations/sftp/ui-config.json new file mode 100644 index 000000000..d79c3a2db --- /dev/null +++ b/src/configurations/destinations/sftp/ui-config.json @@ -0,0 +1,184 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection settings", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Host", + "note": "Enter the Hostname or IP Address of the server. Do not include the port number.", + "configKey": "host", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(.{1,100})$", + "regexErrorMessage": "Invalid Host" + }, + { + "type": "textInput", + "label": "Port", + "note": "Enter the Port Number of the server.", + "configKey": "port", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?:[1-9][0-9]{0,4}|[1-5][0-9]{4}|6[0-5]{4}|6553[0-5])$", + "regexErrorMessage": "Invalid Port", + "placeholder": "e.g. 22" + }, + { + "type": "textInput", + "label": "Username", + "configKey": "username", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Username", + "placeholder": "e.g. testUser" + }, + { + "type": "singleSelect", + "label": "Authentication", + "note": "Select the Authentication method.", + "configKey": "authMethod", + "options": [ + { + "label": "Username + Password", + "value": "passwordAuth" + }, + { + "label": "Username + SSH Key", + "value": "keyAuth" + } + ], + "default": "passwordAuth" + }, + { + "type": "textInput", + "label": "Password", + "configKey": "password", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Password", + "placeholder": "e.g. passXXX123", + "secret": true, + "preRequisites": { + "fields": [ + { + "configKey": "authMethod", + "value": "passwordAuth" + } + ] + } + }, + { + "type": "textInput", + "label": "Private Key", + "note": "Enter the full Private Key for the user including the header and footer.", + "configKey": "privateKey", + "regex": "^[\\s\\S]+$", + "regexErrorMessage": "Invalid Private Key", + "secret": true, + "preRequisites": { + "fields": [ + { + "configKey": "authMethod", + "value": "keyAuth" + } + ] + }, + "isTextArea": true + }, + { + "type": "singleSelect", + "label": "File Format", + "configKey": "fileFormat", + "options": [ + { + "label": "JSON", + "value": "json" + }, + { + "label": "CSV", + "value": "csv" + } + ], + "default": "csv" + }, + { + "type": "textInput", + "label": "File Path", + "note": "Enter the file path where the data will be stored. The parent directory of the file should already exist. Supported file types are JSON and CSV. We support date and time variables within '{}', including {YYYY}, {MM}, {DD}, {hh}, {mm}, {ss}, {ms}, {timestampInSec}, and {timestampInMS}.", + "configKey": "filePath", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^.*\\.(json|csv)$", + "regexErrorMessage": "Invalid File Path", + "placeholder": "e.g: directory/file_{YYYY}_{MM}_{DD}_{mm}.csv" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack's OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing 'Enter' after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "SDK settings", + "note": "not visible in the ui", + "fields": [] + } + } +} diff --git a/src/configurations/destinations/singular/db-config.json b/src/configurations/destinations/singular/db-config.json index 59c53822f..d29d9dcc8 100644 --- a/src/configurations/destinations/singular/db-config.json +++ b/src/configurations/destinations/singular/db-config.json @@ -23,7 +23,9 @@ "amp", "cloud", "warehouse", - "shopify" + "shopify", + "web", + "unity" ], "supportedConnectionModes": { "android": ["cloud", "device"], diff --git a/src/configurations/destinations/snap_pixel/ui-config.json b/src/configurations/destinations/snap_pixel/ui-config.json index 50057d184..d81e86c3d 100644 --- a/src/configurations/destinations/snap_pixel/ui-config.json +++ b/src/configurations/destinations/snap_pixel/ui-config.json @@ -109,10 +109,6 @@ "name": "LOGIN", "value": "LOGIN" }, - { - "name": "ADD_BILLING", - "value": "ADD_BILLING" - }, { "name": "SHARE", "value": "SHARE" diff --git a/src/configurations/destinations/snowflake/db-config.json b/src/configurations/destinations/snowflake/db-config.json index 9a985f1c7..133e020f7 100644 --- a/src/configurations/destinations/snowflake/db-config.json +++ b/src/configurations/destinations/snowflake/db-config.json @@ -69,7 +69,7 @@ ], "android": ["consentManagement", "connectionMode"], "ios": ["consentManagement", "connectionMode"], - "web": ["useNativeSDK", "consentManagement", "connectionMode"], + "web": ["consentManagement", "connectionMode"], "unity": ["consentManagement", "connectionMode"], "amp": ["consentManagement", "connectionMode"], "cloud": ["consentManagement", "connectionMode"], diff --git a/src/configurations/sources/singer_facebook_marketing/db-config.json b/src/configurations/sources/singer_facebook_marketing/db-config.json index 2341090e6..4762364c4 100644 --- a/src/configurations/sources/singer_facebook_marketing/db-config.json +++ b/src/configurations/sources/singer_facebook_marketing/db-config.json @@ -3,7 +3,7 @@ "category": "singer-protocol", "displayName": "Facebook Ads", "options": { - "image": "rudderstack/source-facebook-marketing:v8.2.11" + "image": "rudderstack/source-facebook-marketing:v8.2.13" }, "type": "cloudSource" } diff --git a/src/configurations/sources/singer_google_ads/db-config.json b/src/configurations/sources/singer_google_ads/db-config.json index 571ec0532..50c95b631 100644 --- a/src/configurations/sources/singer_google_ads/db-config.json +++ b/src/configurations/sources/singer_google_ads/db-config.json @@ -6,7 +6,7 @@ "auth": { "provider": "Google" }, - "image": "rudderstack/source-google-ads:v8.2.7" + "image": "rudderstack/source-google-ads:v8.2.15" }, "type": "cloudSource" } diff --git a/test/data/validation/destinations/af.json b/test/data/validation/destinations/af.json index f4a7ba624..ab654565d 100644 --- a/test/data/validation/destinations/af.json +++ b/test/data/validation/destinations/af.json @@ -205,5 +205,21 @@ }, "result": false, "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] + }, + { + "config": { + "devKey": "KPH1PgXXXXt12LgFnQ8EI", + "useRichEventName": false, + "sharingFilter": "all", + "useNativeSDK": { + "reactnative": false + }, + "afCurrencyAtRoot": { + "android": true + }, + "appleAppId": "1565920403" + }, + "result": false, + "err": ["afCurrencyAtRoot must be boolean"] } ] diff --git a/test/data/validation/destinations/emarsys.json b/test/data/validation/destinations/emarsys.json new file mode 100644 index 000000000..3d15df5b2 --- /dev/null +++ b/test/data/validation/destinations/emarsys.json @@ -0,0 +1,237 @@ +[ + { + "config": { + "discardEmptyProperties": true, + "emersysUsername": "dummy", + "emersysUserSecret": "dummy", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "fieldMapping": [ + { + "from": "Email", + "to": "Email" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": true + }, + { + "config": { + "discardEmptyProperties": true, + "emersysUsername": "dummy", + "emersysUserSecret": "dummy", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "" + } + ], + "fieldMapping": [ + { + "from": "Email", + "to": "Email" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "eventsMapping.0.to must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "discardEmptyProperties": true, + "emersysUsername": "dummy", + "emersysUserSecret": "dummy", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "", + "to": "purchase" + } + ], + "fieldMapping": [ + { + "from": "Email", + "to": "Email" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "eventsMapping.0.from must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "discardEmptyProperties": "", + "emersysUsername": "dummy", + "emersysUserSecret": "dummy", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "fieldMapping": [ + { + "from": "Email", + "to": "Email" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": ["discardEmptyProperties must be boolean"] + }, + { + "config": { + "discardEmptyProperties": true, + "emersysUsername": "dummy", + "emersysUserSecret": "dummy", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "fieldMapping": [ + { + "from": "", + "to": "Email" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "fieldMapping.0.from must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "discardEmptyProperties": true, + "emersysUsername": "dummy", + "emersysUserSecret": "dummy", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "fieldMapping": [ + { + "from": "email", + "to": "" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "fieldMapping.0.to must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "discardEmptyProperties": true, + "emersysUsername": "", + "emersysUserSecret": "dummy", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "fieldMapping": [ + { + "from": "email", + "to": "Email" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "emersysUsername must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "discardEmptyProperties": true, + "emersysUsername": "dummy", + "emersysUserSecret": "", + "emersysCustomIdentifier": "dummy", + "defaultContactList": "dummy", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "fieldMapping": [ + { + "from": "email", + "to": "Email" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "emersysUserSecret must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + } +] diff --git a/test/data/validation/destinations/sftp.json b/test/data/validation/destinations/sftp.json new file mode 100644 index 000000000..fa22691c0 --- /dev/null +++ b/test/data/validation/destinations/sftp.json @@ -0,0 +1,70 @@ +[ + { + "config": { + "host": "test-host", + "port": "22", + "authMethod": "keyAuth", + "privateKey": "test-privateKey", + "username": "test-username", + "fileFormat": "csv", + "filePath": "directory/file.csv", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "" + } + ], + "postgres-connectionMode": "cloud", + "warehouse-connectionMode": "cloud" + }, + "result": true + }, + { + "config": { + "host": "test-host", + "port": "22", + "authMethod": "keyAuth", + "username": "test-username", + "fileFormat": "json", + "filePath": "directory/file.csv", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "" + } + ], + "postgres-connectionMode": "cloud", + "warehouse-connectionMode": "cloud" + }, + "result": false, + "err": [ + " must have required property 'privateKey'", + " must match \"then\" schema", + "filePath must match pattern \"\\.json$\"", + " must match \"then\" schema" + ] + }, + { + "config": { + "host": "test-host", + "port": "995536", + "authMethod": "test-auth", + "username": "test-username", + "password": "", + "fileFormat": "csv", + "filePath": "directory/file.txt", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "" + } + ], + "postgres-connectionMode": "cloud", + "warehouse-connectionMode": "cloud" + }, + "err": [ + "filePath must match pattern \"\\.csv$\"", + " must match \"then\" schema", + "port must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?:[1-9][0-9]{0,4}|[1-5][0-9]{4}|6[0-5]{4}|6553[0-5])$\"", + "authMethod must be equal to one of the allowed values", + "filePath must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^.*\\.(json|csv)$\"" + ] + } +]