chore(release): pull release/v1.74.0 into main

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.74.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.73.0...v1.74.0) (2024-05-13)
+
+
+### Features
+
+* add file format ([4d9bc7e](https://github.com/rudderlabs/rudder-config-schema/commit/4d9bc7e16980945aae6a9102789f77e87b07b88c))
+* **criteo:** add support of sha256 hashing method for email field ([#1301](https://github.com/rudderlabs/rudder-config-schema/issues/1301)) ([958e7d3](https://github.com/rudderlabs/rudder-config-schema/commit/958e7d3e95e9aceafbdafc44a7d0e7b65eeadc9a))
+* improve schema generator ([#1207](https://github.com/rudderlabs/rudder-config-schema/issues/1207)) ([55ad8d7](https://github.com/rudderlabs/rudder-config-schema/commit/55ad8d79e7ab2508feb8496e7f4d0674004e6611))
+* onboard emersys destination ([#1335](https://github.com/rudderlabs/rudder-config-schema/issues/1335)) ([4dea628](https://github.com/rudderlabs/rudder-config-schema/commit/4dea628413dcf3a420d4512e8004c8c458428e37))
+* onboard new destination sftp ([71a66a9](https://github.com/rudderlabs/rudder-config-schema/commit/71a66a9b53671fbac179296364013214aa975171))
+* update schema ([e4e70dd](https://github.com/rudderlabs/rudder-config-schema/commit/e4e70dd1b91f6d8802d4f563a3866e82896eb011))
+* update schema ([76721f0](https://github.com/rudderlabs/rudder-config-schema/commit/76721f0d4028c64b20d4ac2e1596283e9275a4b1))
+
+
+### Bug Fixes
+
+* adding group call to emarsys ([#1349](https://github.com/rudderlabs/rudder-config-schema/issues/1349)) ([fa72fda](https://github.com/rudderlabs/rudder-config-schema/commit/fa72fda6083c047c345c8f809fe8e7926cb11934))
+* remove useNativeSdk from snowflake ([#1359](https://github.com/rudderlabs/rudder-config-schema/issues/1359)) ([05d14f0](https://github.com/rudderlabs/rudder-config-schema/commit/05d14f0b8a613520e1aba542a82c9aaf96ef0715))
+* schema ([f6cff9f](https://github.com/rudderlabs/rudder-config-schema/commit/f6cff9f4a82df23ec4bcb3228bd672aec0aee515))
+
 ## [1.73.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.72.0...v1.73.0) (2024-05-08)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-config-schema",
-  "version": "1.73.0",
+  "version": "1.74.0",
   "description": "",
   "main": "src/index.ts",
   "private": true,

scripts/schemaGenerator.py

@@ -126,6 +126,26 @@ def is_dest_field_dependent_on_source(field, dbConfig, schema_field_name):
     return False
 
 
+def is_key_present_in_dest_config(dbConfig, key):
+    """Checks if the given key is present in destConfig across all source types.
+
+    Args:
+        dbConfig (object): Destination configuration in db-config.json.
+        key (string): key to be searched in destConfig tree.
+
+    Returns:
+        boolean: True if the key is present in destConfig else, False.
+    """
+    if not dbConfig:
+        return False
+
+    if "destConfig" in dbConfig:
+        for configSection in dbConfig["destConfig"]:
+            if key in dbConfig["destConfig"][configSection]:
+                return True
+    return False
+
+
 def is_field_present_in_default_config(field, dbConfig, schema_field_name):
     """Checks if the given field is present in defaultConfig list present in dbConfig.
 
@@ -740,6 +760,7 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name):
     - For each unique preRequisiteField, the properties are found by matching the current preRequisiteField.
     - preRequisiteField becomes if block and corresponding properties become then block.
 
+
     Args:
         uiConfig (object): file content of ui-config.json.
         dbConfig (object): Configurations of db-config.json.
@@ -971,17 +992,14 @@ def generate_connection_mode(dbConfig):
     return connectionObj
 
 
-def generate_schema_properties(
-    uiConfig, dbConfig, schemaObject, properties, name, selector
-):
+def generate_schema_properties(uiConfig, dbConfig, schemaObject, properties, selector):
     """Generates corresponding schema properties by iterating over each of the ui-config fields.
 
     Args:
         uiConfig (object): file content of ui-config.json.
         dbConfig (object): Configurations of db-config.json.
         schemaObject (object): schema being generated
         properties (object): properties of schema
-        name (string): name of the source or destination.
         selector (string): either 'source' or 'destination'
     """
     if is_old_format(uiConfig):
@@ -992,9 +1010,16 @@ def generate_schema_properties(
                     continue
                 generateFunction = uiTypetoSchemaFn.get(field["type"], None)
                 if generateFunction:
-                    properties[field["value"]] = generateFunction(
-                        field, dbConfig, "value"
-                    )
+                    # Generate schema for the field if it is defined in the destination config
+                    if is_key_present_in_dest_config(dbConfig, field["value"]):
+                        properties[field["value"]] = generateFunction(
+                            field, dbConfig, "value"
+                        )
+                    else:
+                        warnings.warn(
+                            f'The field {field["value"]} is defined in ui-config.json but not in db-config.json\n',
+                            UserWarning,
+                        )
                 if field.get(
                     "required", False
                 ) == True and is_field_present_in_default_config(
@@ -1012,9 +1037,18 @@ def generate_schema_properties(
                         for field in group.get("fields", []):
                             generateFunction = uiTypetoSchemaFn.get(field["type"], None)
                             if generateFunction:
-                                properties[field["configKey"]] = generateFunction(
-                                    field, dbConfig, "configKey"
-                                )
+                                # Generate schema for the field if it is defined in the destination config
+                                if is_key_present_in_dest_config(
+                                    dbConfig, field["configKey"]
+                                ):
+                                    properties[field["configKey"]] = generateFunction(
+                                        field, dbConfig, "configKey"
+                                    )
+                                else:
+                                    warnings.warn(
+                                        f'The field {field["configKey"]} is defined in ui-config.json but not in db-config.json\n',
+                                        UserWarning,
+                                    )
                             if (
                                 template.get("title", "") == "Initial setup"
                                 and is_field_present_in_default_config(
@@ -1027,9 +1061,17 @@ def generate_schema_properties(
             for field in sdkTemplate.get("fields", []):
                 generateFunction = uiTypetoSchemaFn.get(field["type"], None)
                 if generateFunction:
-                    properties[field["configKey"]] = generateFunction(
-                        field, dbConfig, "configKey"
-                    )
+                    # Generate schema for the field if it is defined in the destination config
+                    if is_key_present_in_dest_config(dbConfig, field["configKey"]):
+                        properties[field["configKey"]] = generateFunction(
+                            field, dbConfig, "configKey"
+                        )
+                    else:
+                        warnings.warn(
+                            f'The field {field["configKey"]} is defined in ui-config.json but not in db-config.json\n',
+                            UserWarning,
+                        )
+
                 if field.get(
                     "required", False
                 ) == True and is_field_present_in_default_config(
@@ -1051,9 +1093,12 @@ def generate_schema_properties(
                     schemaObject["required"].append(field["configKey"])
 
             # default properties in new ui-config based schemas.
-            schemaObject["properties"]["useNativeSDK"] = generate_schema_for_checkbox(
-                {"type": "checkbox", "value": "useNativeSDK"}, dbConfig, "value"
-            )
+            if is_key_present_in_dest_config(dbConfig, "useNativeSDK"):
+                schemaObject["properties"]["useNativeSDK"] = (
+                    generate_schema_for_checkbox(
+                        {"type": "checkbox", "value": "useNativeSDK"}, dbConfig, "value"
+                    )
+                )
             schemaObject["properties"]["connectionMode"] = generate_connection_mode(
                 dbConfig
             )
@@ -1099,6 +1144,7 @@ def generate_schema(uiConfig, dbConfig, name, selector):
     schemaObject["type"] = "object"
     schemaObject["properties"] = {}
     allOfSchemaObj = {}
+    print(f"Generating schema for {name} {selector}")
     if is_old_format(uiConfig):
         allOfSchemaObj = generate_schema_for_allOf(uiConfig, dbConfig, "value")
     if allOfSchemaObj:
@@ -1110,8 +1156,9 @@ def generate_schema(uiConfig, dbConfig, name, selector):
                 schemaObject["anyOf"] = allOfSchemaObj
         else:
             schemaObject["allOf"] = allOfSchemaObj
+
     generate_schema_properties(
-        uiConfig, dbConfig, schemaObject, schemaObject["properties"], name, selector
+        uiConfig, dbConfig, schemaObject, schemaObject["properties"], selector
     )
     newSchema["configSchema"] = schemaObject
 
@@ -1154,7 +1201,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType):
                             )
                     if (
                         curUiType == "textInput"
-                        and field["value"] in schema["required"]
+                        and (
+                            schema.get("required", False) == True
+                            and field["value"] in schema["required"]
+                        )
                         and "regex" not in field
                     ):
                         warnings.warn(
@@ -1204,7 +1254,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType):
                                     )
                             if (
                                 curUiType == "textInput"
-                                and field["configKey"] in schema["required"]
+                                and (
+                                    schema.get("required", False) == True
+                                    and field["configKey"] in schema["required"]
+                                )
                                 and "regex" not in field
                             ):
                                 warnings.warn(
@@ -1423,11 +1476,9 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False):
         selector (string): either 'source' or 'destination'.
         shouldUpdateSchema (boolean): if it should update the existing schema with generated one
     """
-
     file_selectors = ["db-config.json", "ui-config.json", "schema.json"]
     directory = f"./{CONFIG_DIR}/{selector}s/{name}"
     if not os.path.isdir(directory):
-        print(f"No {selector}s directory found for {name}")
         return
 
     if name not in EXCLUDED_DEST:

src/configurations/destinations/criteo/schema.json

@@ -12,7 +12,11 @@
         "type": "string",
         "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(.{0,100})$"
       },
-      "hashMethod": { "type": "string", "enum": ["none", "md5"], "default": "none" },
+      "hashMethod": {
+        "type": "string",
+        "enum": ["none", "md5", "sha256"],
+        "default": "none"
+      },
       "fieldMapping": {
         "type": "array",
         "items": {

src/configurations/destinations/criteo/ui-config.json

@@ -35,6 +35,10 @@
             {
               "name": "MD5",
               "value": "md5"
+            },
+            {
+              "name": "SHA256",
+              "value": "sha256"
             }
           ],
           "defaultOption": {

src/configurations/destinations/emarsys/db-config.json

@@ -0,0 +1,66 @@
+{
+  "name": "EMARSYS",
+  "displayName": "Emarsys",
+  "config": {
+    "cdkV2Enabled": true,
+    "transformAtV1": "router",
+    "saveDestinationResponse": true,
+    "excludeKeys": [],
+    "supportedSourceTypes": [
+      "android",
+      "ios",
+      "unity",
+      "amp",
+      "reactnative",
+      "flutter",
+      "cordova",
+      "web",
+      "cloud",
+      "shopify",
+      "warehouse"
+    ],
+    "supportedConnectionModes": {
+      "android": ["cloud"],
+      "ios": ["cloud"],
+      "web": ["cloud"],
+      "unity": ["cloud"],
+      "amp": ["cloud"],
+      "reactnative": ["cloud"],
+      "flutter": ["cloud"],
+      "cordova": ["cloud"],
+      "shopify": ["cloud"],
+      "cloud": ["cloud"],
+      "warehouse": ["cloud"]
+    },
+    "supportedMessageTypes": {
+      "cloud": ["identify", "track", "group"]
+    },
+    "destConfig": {
+      "defaultConfig": [
+        "emersysUsername",
+        "emersysUserSecret",
+        "eventsMapping",
+        "fieldMapping",
+        "emersysCustomIdentifier",
+        "defaultContactList",
+        "discardEmptyProperties",
+        "oneTrustCookieCategories"
+      ],
+      "android": ["connectionMode"],
+      "ios": ["connectionMode"],
+      "unity": ["connectionMode"],
+      "amp": ["connectionMode"],
+      "reactnative": ["connectionMode"],
+      "flutter": ["connectionMode"],
+      "cordova": ["connectionMode"],
+      "web": ["connectionMode"],
+      "cloud": ["connectionMode"],
+      "shopify": ["connectionMode"],
+      "warehouse": ["connectionMode"]
+    },
+    "secretKeys": ["emersysUsername", "emersysUserSecret"]
+  },
+  "options": {
+    "isBeta": true
+  }
+}