From 04e05b89a7333291792fbb6e111388b5ff8b985f Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Thu, 8 Feb 2024 21:12:37 +0530 Subject: [PATCH 01/17] feat: improve schema generator output (#1191) --- .github/workflows/report-code-coverage.yml | 3 + .github/workflows/test.yml | 6 +- .gitignore | 2 + .nvmrc | 2 +- scripts/constants.py | 11 + scripts/deployToDB.py | 4 +- scripts/schemaGenerator.py | 137 +++-- scripts/template-db-config.json | 6 +- scripts/utils.py | 51 ++ .../destinations/am/schema.json | 4 +- .../destinations/am/ui-config.json | 4 +- .../destinations/spotifyPixel/db-config.json | 65 ++- .../destinations/spotifyPixel/schema.json | 145 +++-- .../destinations/spotifyPixel/ui-config.json | 510 +++++++++--------- .../create_new_schema_dest/db-config.json | 44 ++ .../create_new_schema_dest/ui-config.json | 118 ++++ .../db-config.json | 49 ++ .../ui-config.json | 153 ++++++ .../update_schema_dest/db-config.json | 44 ++ .../update_schema_dest/schema.json | 92 ++++ .../update_schema_dest/ui-config.json | 118 ++++ .../db-config.json | 49 ++ .../update_schema_dest_legacy_ui/schema.json | 106 ++++ .../ui-config.json | 153 ++++++ .../data/createNewSchemaDest.json | 100 ++++ .../data/createNewSchemaDestLegacyUI.json | 114 ++++ .../data/updateSchemaDest.json | 100 ++++ .../data/updateSchemaDestLegacyUI.json | 114 ++++ test/component_tests/schemaGenerator.test.ts | 151 ++++++ .../validation/destinations/spotifyPixel.json | 18 +- 30 files changed, 2027 insertions(+), 446 deletions(-) create mode 100644 scripts/constants.py create mode 100644 scripts/utils.py create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest/db-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest/schema.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest/ui-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json create mode 100644 test/component_tests/data/createNewSchemaDest.json create mode 100644 test/component_tests/data/createNewSchemaDestLegacyUI.json create mode 100644 test/component_tests/data/updateSchemaDest.json create mode 100644 test/component_tests/data/updateSchemaDestLegacyUI.json create mode 100644 test/component_tests/schemaGenerator.test.ts diff --git a/.github/workflows/report-code-coverage.yml b/.github/workflows/report-code-coverage.yml index 2ff1badab..9791eb5d7 100644 --- a/.github/workflows/report-code-coverage.yml +++ b/.github/workflows/report-code-coverage.yml @@ -26,6 +26,9 @@ jobs: - name: Install Dependencies run: npm ci + + - name: Set up Python + run: scripts/setup-python.sh - name: Run Tests run: npm run test:ci diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8fe4e8e1..e65062640 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,12 +26,12 @@ jobs: - name: Install Dependencies run: npm ci - - name: Run Unit Tests - run: npm run test:ci - - name: Set up Python run: scripts/setup-python.sh + - name: Run Tests + run: npm run test:ci + - name: Get changed files id: changed_files uses: tj-actions/changed-files@v41 diff --git a/.gitignore b/.gitignore index 7b515f143..91868eec6 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ git reports .eslintcache .vscode + +__pycache__ diff --git a/.nvmrc b/.nvmrc index 55bffd620..8b0beab16 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18.15.0 +20.11.0 diff --git a/scripts/constants.py b/scripts/constants.py new file mode 100644 index 000000000..54b717202 --- /dev/null +++ b/scripts/constants.py @@ -0,0 +1,11 @@ +import os + +__DEFAULT_CONFIG_DIR = 'src/configurations' +CONFIG_DIR = __DEFAULT_CONFIG_DIR + +if 'CONFIG_DIR' in os.environ: + CONFIG_DIR = os.environ['CONFIG_DIR'] + +TEST_INTEGRATION_NAME_PREFIX = 'test_' + +TEST_INTEGRATION_NAME_SUFFIX = '_ignore' diff --git a/scripts/deployToDB.py b/scripts/deployToDB.py index 897944333..6c513fb92 100644 --- a/scripts/deployToDB.py +++ b/scripts/deployToDB.py @@ -4,6 +4,7 @@ import os import sys import jsondiff +from constants import CONFIG_DIR ######################### # ENV VARIABLES FOT TESTING @@ -27,7 +28,6 @@ # CONSTANTS HEADER = {"Content-Type": "application/json"} AUTH = (USERNAME, PASSWORD) -CONFIG_DIR = 'src/configurations' ######################### @@ -105,7 +105,7 @@ def update_diff_db(selector): # check if item is a directory if not os.path.isdir(f'./{CONFIG_DIR}/{selector}s/{item}'): continue - + updated_data = get_file_content(item, selector) persisted_data = get_config_definition(CONTROL_PLANE_URL, selector, updated_data["name"]) diff --git a/scripts/schemaGenerator.py b/scripts/schemaGenerator.py index 96e590670..36d9d9ee3 100644 --- a/scripts/schemaGenerator.py +++ b/scripts/schemaGenerator.py @@ -9,14 +9,11 @@ 2. python3 scripts/schemaGenerator.py -all source ''' import os -import sys import warnings -import json -from jsondiff import diff from enum import Enum import argparse - -CONFIG_DIR = 'src/configurations' +from utils import get_json_from_file, get_json_diff, apply_json_diff, get_formatted_json +from constants import CONFIG_DIR EXCLUDED_DEST = ['postgres', 'bq', 'azure_synapse', 'clickhouse', 'deltalake', 'kafka'] @@ -134,15 +131,21 @@ def generate_schema_for_default_checkbox(field, dbConfig, schema_field_name): Returns: object """ - defaultCheckboxObj = { - "type": FieldTypeEnum.OBJECT.value, - "properties": { - "web": { - "type": FieldTypeEnum.BOOLEAN.value - } - } - } - return defaultCheckboxObj + isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + defaultCheckboxSchemaObj = {} + if isSourceDependent: + defaultCheckboxSchemaObj["type"] = FieldTypeEnum.OBJECT.value + defaultCheckboxSchemaObj["properties"] = {} + # iterates over supported sources and sets the field for that source if field is present inside that source + for sourceType in dbConfig["supportedSourceTypes"]: + if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + defaultCheckboxSchemaObj["properties"][sourceType] = { + "type": FieldTypeEnum.BOOLEAN.value} + else: + defaultCheckboxSchemaObj["type"] = FieldTypeEnum.BOOLEAN.value + if "default" in field: + defaultCheckboxSchemaObj["default"] = field["default"] + return defaultCheckboxSchemaObj def generate_schema_for_checkbox(field, dbConfig, schema_field_name): @@ -909,7 +912,7 @@ def generate_config_props(config): generate_config_props(config) -def generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema): +def generate_schema(uiConfig, dbConfig, name, selector): """Returns the schema generated from given uiConfig and dbConfig. Args: @@ -917,7 +920,6 @@ def generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema): dbConfig (object): Configurations of db-config.json. name (string): name of the source or destination. selector (string): either 'source' or 'destination' - shouldUpdateSchema (boolean): if it should update the existing schema with generated one Returns: object: schema @@ -944,18 +946,6 @@ def generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema): schemaObject['properties'], name, selector) newSchema['configSchema'] = schemaObject - if shouldUpdateSchema: - # Get the parent directory (one level up) - script_directory = os.path.dirname(os.path.abspath(__file__)) - directory = os.path.dirname(script_directory) - # Define the relative path - relative_path = f'src/configurations/{selector}s/{name.lower()}/schema.json' - file_path = os.path.join(directory, relative_path) - new_content = json.dumps(newSchema) - # Write the new content - with open(file_path, 'w') as file: - file.write(new_content) - return newSchema def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): @@ -980,10 +970,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): curSchemaField = schema["properties"][field["value"]] newSchemaField = uiTypetoSchemaFn.get( curUiType)(field, dbConfig, "value") - schemaDiff = diff(newSchemaField, curSchemaField) + schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["value"], schemaDiff), UserWarning) + curUiType, field["value"], get_formatted_json(schemaDiff)), UserWarning) else: baseTemplate = uiConfig.get('baseTemplate', []) sdkTemplate = uiConfig.get('sdkTemplate', {}) @@ -1006,10 +996,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): curSchemaField = schema["properties"][field["configKey"]] newSchemaField = uiTypetoSchemaFn.get( curUiType)(field, dbConfig, "configKey") - schemaDiff = diff(newSchemaField, curSchemaField) + schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], schemaDiff), UserWarning) + curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) for field in sdkTemplate.get('fields', []): if "preRequisites" in field: @@ -1024,10 +1014,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): curSchemaField = schema["properties"][field["configKey"]] newSchemaField = uiTypetoSchemaFn.get( curUiType)(field, dbConfig, "configKey") - schemaDiff = diff(newSchemaField, curSchemaField) + schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], schemaDiff), UserWarning) + curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) for field in consentSettingsTemplate.get('fields', []): if "preRequisites" in field: @@ -1063,6 +1053,18 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): 'timePicker': generate_schema_for_time_picker } +def save_schema_to_file(selector, name, schema): + # Get the parent directory (one level up) + script_directory = os.path.dirname(os.path.abspath(__file__)) + directory = os.path.dirname(script_directory) + + # Define the relative path + relative_path = f'{CONFIG_DIR}/{selector}s/{name}/schema.json' + file_path = os.path.join(directory, relative_path) + + # Write the new content + with open(file_path, 'w') as file: + file.write(get_formatted_json(schema)) def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema): """Generates a schema and compares it with an existing one. @@ -1080,12 +1082,18 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou return if uiConfig == None: print('-'*50) - warnings.warn(f"Ui-Config is null for {name} in {selector} \n",UserWarning) + warnings.warn(f"Ui-Config is null for {name} in {selector} \n", UserWarning) print('-'*50) return - generatedSchema = generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema) + generatedSchema = generate_schema(uiConfig, dbConfig, name, selector) + if schema: - schemaDiff = diff(schema, generatedSchema["configSchema"]) + schemaDiff = get_json_diff(schema, generatedSchema["configSchema"]) + if shouldUpdateSchema: + finalSchema = {} + finalSchema["configSchema"] = apply_json_diff(schema, schemaDiff) + save_schema_to_file(selector, name, finalSchema) + if schemaDiff: print('-'*50) print(f'Schema diff for {name} in {selector}s') @@ -1101,30 +1109,33 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou else: curRequiredField = schema["required"] newRequiredField = generatedSchema["configSchema"]["required"] - requiredFieldDiff = diff(curRequiredField, newRequiredField) + requiredFieldDiff = get_json_diff(curRequiredField, newRequiredField) if requiredFieldDiff: - warnings.warn("For required field Difference is : \n\n {} \n".format(requiredFieldDiff), UserWarning) + warnings.warn("For required field Difference is : \n\n {} \n".format(get_formatted_json(requiredFieldDiff)), UserWarning) if "allOf" in generatedSchema["configSchema"]: curAllOfSchema = {} if "allOf" in schema: curAllOfSchema = schema["allOf"] newAllOfSchema = generatedSchema["configSchema"]["allOf"] - allOfSchemaDiff = diff(newAllOfSchema, curAllOfSchema) + allOfSchemaDiff = get_json_diff(curAllOfSchema, newAllOfSchema) if allOfSchemaDiff: - warnings.warn("For allOf field Difference is : \n\n {} \n".format(allOfSchemaDiff), UserWarning) + warnings.warn("For allOf field Difference is : \n\n {} \n".format(get_formatted_json(allOfSchemaDiff)), UserWarning) if "anyOf" in generatedSchema["configSchema"]: curAnyOfSchema = {} if "anyOf" in schema: curAnyOfSchema = schema["anyOf"] newAnyOfSchema = generatedSchema["configSchema"]["anyOf"] - anyOfSchemaDiff = diff(newAnyOfSchema, curAnyOfSchema) + anyOfSchemaDiff = get_json_diff(curAnyOfSchema, newAnyOfSchema) if anyOfSchemaDiff: - warnings.warn("For anyOf field Difference is : \n\n {} \n".format(anyOfSchemaDiff), UserWarning) + warnings.warn("For anyOf field Difference is : \n\n {} \n".format(get_formatted_json(anyOfSchemaDiff)), UserWarning) print('-'*50) else: + if shouldUpdateSchema: + save_schema_to_file(selector, name, generatedSchema) + print('-'*50) - print(f'Generated Schema for {name} in {selector}s') - print(json.dumps(generatedSchema,indent=2)) + print(f'Generated schema for {name} in {selector}s') + print(get_formatted_json(generatedSchema)) print('-'*50) def get_schema_diff(name, selector, shouldUpdateSchema=False): @@ -1135,20 +1146,24 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False): selector (string): either 'source' or 'destination'. shouldUpdateSchema (boolean): if it should update the existing schema with generated one """ + file_selectors = ['db-config.json', 'ui-config.json', 'schema.json'] directory = f'./{CONFIG_DIR}/{selector}s/{name}' - available_files = os.listdir(directory) - file_content = {} - for file_selector in file_selectors: - if file_selector in available_files: - with open (f'{directory}/{file_selector}', 'r') as f: - file_content.update(json.loads(f.read())) - uiConfig = file_content.get("uiConfig") - schema = file_content.get("configSchema") - dbConfig = file_content.get("config") + if not os.path.isdir(directory): + print(f'No {selector}s directory found for {name}') + return + if name not in EXCLUDED_DEST: - validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema) + available_files = os.listdir(directory) + file_content = {} + for file_selector in file_selectors: + if file_selector in available_files: + file_content.update(get_json_from_file(f'{directory}/{file_selector}')) + uiConfig = file_content.get("uiConfig") + schema = file_content.get("configSchema") + dbConfig = file_content.get("config") + validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generates schema.json from ui-config.json and db-config.json and validates against actual scheme.json') @@ -1163,11 +1178,15 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False): shouldUpdateSchema = args.update if args.all: - CONFIG_DIR = 'src/configurations' - current_items = os.listdir(f'./{CONFIG_DIR}/{selector}s') + dir_path = f'./{CONFIG_DIR}/{selector}s' + if not os.path.isdir(dir_path): + print(f'No {selector}s folder found') + exit(1) + + current_items = os.listdir(dir_path) for name in current_items: - get_schema_diff(name, selector) + get_schema_diff(name, selector, shouldUpdateSchema) else: - name = args.name + name = args.name get_schema_diff(name, selector, shouldUpdateSchema) diff --git a/scripts/template-db-config.json b/scripts/template-db-config.json index 0e8f50817..4205cc383 100644 --- a/scripts/template-db-config.json +++ b/scripts/template-db-config.json @@ -2,7 +2,6 @@ "name": "", "displayName": "", "config": { - "transformAt": "processor", "transformAtV1": "processor", "saveDestinationResponse": false, "includeKeys": [], @@ -14,12 +13,13 @@ "unity", "amp", "cloud", + "warehouse", "reactnative", "flutter", "cordova", - "warehouse" + "shopify" ], - "supportedConnectionModes": [], + "supportedConnectionModes": {}, "destConfig": { "defaultConfig": [] }, diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 000000000..49cc83027 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,51 @@ +from jsondiff import JsonDiffer +import json + +def get_json_diff(oldJson, newJson): + """Returns the difference between two JSONs. + + Args: + oldJson (object): old json. + newJson (object): new json. + + Returns: + object: difference between oldJson and newJson. + """ + differ = JsonDiffer(marshal=True) + return differ.diff(oldJson, newJson) + +def apply_json_diff(oldJson, diff): + """Applies the difference on oldJson and returns the newJson. + + Args: + oldJson (object): old json. + diff (object): difference between oldJson and newJson. + + Returns: + object: new json. + """ + differ = JsonDiffer(marshal=True) + return differ.patch(oldJson, diff) + +def get_formatted_json(jsonObj): + """Formats the json object. + + Args: + jsonObj (object): json object. + + Returns: + string: formatted json. + """ + return json.dumps(jsonObj, indent=2, ensure_ascii=False) + +def get_json_from_file(filePath): + """Reads the content of the file and returns the json object. + + Args: + filePath (string): file path. + + Returns: + object: json object. + """ + with open(filePath, 'r') as file: + return json.loads(file.read().encode('utf-8', 'ignore')) diff --git a/src/configurations/destinations/am/schema.json b/src/configurations/destinations/am/schema.json index 48782987f..fe02ce807 100644 --- a/src/configurations/destinations/am/schema.json +++ b/src/configurations/destinations/am/schema.json @@ -18,9 +18,9 @@ "properties": { "web": { "type": "string", - "pattern": "^(?!http:\/\/)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$" + "pattern": "^(?!http://)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$" } - } + } }, "trackAllPages": { "type": "boolean", diff --git a/src/configurations/destinations/am/ui-config.json b/src/configurations/destinations/am/ui-config.json index 12eba8b71..94524d683 100644 --- a/src/configurations/destinations/am/ui-config.json +++ b/src/configurations/destinations/am/ui-config.json @@ -543,7 +543,7 @@ "label": "Proxy server url", "note": "Send data to Amplitude by using a domain proxy to relay requests. Presently supported for web device mode only", "configKey": "proxyServerUrl", - "regex":"^(?!http:\/\/)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$", + "regex": "^(?!http://)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$", "regexErrorMessage": "Invalid Proxy Server URL", "placeholder": "e.g: https://proxyserver.url.com" }, @@ -625,4 +625,4 @@ ] } } -} \ No newline at end of file +} diff --git a/src/configurations/destinations/spotifyPixel/db-config.json b/src/configurations/destinations/spotifyPixel/db-config.json index 95c3a933e..73801045e 100644 --- a/src/configurations/destinations/spotifyPixel/db-config.json +++ b/src/configurations/destinations/spotifyPixel/db-config.json @@ -1,41 +1,40 @@ { - "name": "SPOTIFYPIXEL", - "displayName": "Spotify Pixel", - "config": { - "transformAtV1": "processor", - "saveDestinationResponse": true, - "includeKeys": [ + "name": "SPOTIFYPIXEL", + "displayName": "Spotify Pixel", + "config": { + "transformAtV1": "processor", + "saveDestinationResponse": true, + "includeKeys": [ + "pixelId", + "eventsToSpotifyPixelEvents", + "blacklistedEvents", + "whitelistedEvents", + "oneTrustCookieCategories", + "eventFilteringOption", + "enableAliasCall" + ], + "excludeKeys": [], + "supportedSourceTypes": ["web"], + "supportedMessageTypes": { + "device": { + "web": ["track", "page"] + } + }, + "supportedConnectionModes": { + "web": ["device"] + }, + "destConfig": { + "defaultConfig": [ "pixelId", "eventsToSpotifyPixelEvents", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", - "enableAliasCall" + "enableAliasCall", + "oneTrustCookieCategories" ], - "excludeKeys": [], - "supportedSourceTypes": ["web"], - "supportedMessageTypes": { - "device": { - "web": ["track", "page"] - } - }, - "supportedConnectionModes": { - "web": ["device"] - }, - "destConfig": { - "defaultConfig": [ - "pixelId", - "eventsToSpotifyPixelEvents", - "blacklistedEvents", - "whitelistedEvents", - "eventFilteringOption", - "enableAliasCall", - "oneTrustCookieCategories" - ], - "web": ["useNativeSDK", "connectionMode"] - }, - "secretKeys": ["pixelId"] - } + "web": ["useNativeSDK", "connectionMode"] + }, + "secretKeys": ["pixelId"] } - \ No newline at end of file +} diff --git a/src/configurations/destinations/spotifyPixel/schema.json b/src/configurations/destinations/spotifyPixel/schema.json index 88588c530..ae51c344f 100644 --- a/src/configurations/destinations/spotifyPixel/schema.json +++ b/src/configurations/destinations/spotifyPixel/schema.json @@ -1,92 +1,91 @@ { - "configSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "required": ["pixelId"], - "type": "object", - "properties": { - "pixelId": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" - }, - "eventsToSpotifyPixelEvents": { - "type": "array", - "items": { - "type": "object", - "properties": { - "from": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - }, - "to": { - "type": "string", - "enum": ["lead", "product", "addtocart", "checkout", "purchase", ""] - } - } - } - }, - "enableAliasCall": { - "type": "boolean", - "default": false - }, - "eventFilteringOption": { - "type": "string", - "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], - "default": "disable" - }, - "whitelistedEvents": { - "type": "array", - "items": { - "type": "object", - "properties": { - "eventName": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["pixelId"], + "type": "object", + "properties": { + "pixelId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "eventsToSpotifyPixelEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "enum": ["lead", "product", "addtocart", "checkout", "purchase", ""] } } - }, - "blacklistedEvents": { - "type": "array", - "items": { - "type": "object", - "properties": { - "eventName": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } + } + }, + "enableAliasCall": { + "type": "boolean", + "default": false + }, + "eventFilteringOption": { + "type": "string", + "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], + "default": "disable" + }, + "whitelistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" } } - }, - "useNativeSDK": { + } + }, + "blacklistedEvents": { + "type": "array", + "items": { "type": "object", "properties": { - "web": { - "type": "boolean" + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" } } - }, - "oneTrustCookieCategories": { - "type": "array", - "items": { - "type": "object", - "properties": { - "oneTrustCookieCategory": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } - } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "web": { + "type": "boolean" } - }, - "connectionMode": { + } + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { "type": "object", "properties": { - "web": { + "oneTrustCookieCategory": { "type": "string", - "enum": ["device"] + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { + "type": "string", + "enum": ["device"] + } + } } } } - \ No newline at end of file +} diff --git a/src/configurations/destinations/spotifyPixel/ui-config.json b/src/configurations/destinations/spotifyPixel/ui-config.json index 52e0c66b8..2ba754299 100644 --- a/src/configurations/destinations/spotifyPixel/ui-config.json +++ b/src/configurations/destinations/spotifyPixel/ui-config.json @@ -1,264 +1,264 @@ { - "uiConfig": { - "baseTemplate": [ - { - "title": "Initial setup", - "note": "Review how this destination is set up", - "sections": [ - { - "groups": [ - { - "title": "Connection Settings", - "note": "Update your connection settings here", - "icon": "settings", - "fields": [ - { - "type": "textInput", - "label": "Pixel ID", - "note": "Spotify Pixel dashboard > Manage > Your Pixels", - "configKey": "pixelId", - "regex": "^(.{1,100})$", - "secret": true, - "regexErrorMessage": "Invalid Pixel ID", - "placeholder": "e.g. dzq1p89h2bnpXXXX9x65hyx2hf5q1k3v" - } - ] - } - ] - }, - { - "groups": [ - { - "title": "Connection mode", - "note": [ - "Update how you want to route events from your source to destination. ", - { - "text": "Get help deciding", - "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" - } - ], - "icon": "sliders", - "fields": [], - "defaultConnectionModes": { - "web": "device" - } - } - ] - } + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection Settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Pixel ID", + "note": "Spotify Pixel dashboard > Manage > Your Pixels", + "configKey": "pixelId", + "regex": "^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid Pixel ID", + "placeholder": "e.g. dzq1p89h2bnpXXXX9x65hyx2hf5q1k3v" + } ] - }, - { - "title": "Configuration settings", - "note": "Manage the settings for your destination", - "sections": [ + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [], + "defaultConnectionModes": { + "web": "device" + } + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [ + { + "title": "Spotify Pixel Alias Event Setting", + "fields": [ + { + "type": "checkbox", + "label": "Enhance attribution with externalIDs for conversion matching", + "configKey": "enableAliasCall", + "default": false + } + ] + } + ] + }, + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "Client-side event filtering", + "note": "Decide what events are allowed (allowlisting) and blocked (denylisting)", + "preRequisites": { + "fields": [ { - "title": "Destination settings", - "note": "Configure advanced destination-specific settings here", - "icon": "settings", - "groups": [ - { - "title": "Spotify Pixel Alias Event Setting", - "fields": [ - { - "type": "checkbox", - "label": "Enhance attribution with externalIDs for conversion matching", - "configKey": "enableAliasCall", - "default": false - } - ] - } - ] + "configKey": "connectionModes.webDevice", + "value": true }, { - "title": "Other settings", - "note": "Configure advanced RudderStack features here", - "icon": "otherSettings", - "groups": [ - { - "title": "Client-side event filtering", - "note": "Decide what events are allowed (allowlisting) and blocked (denylisting)", - "preRequisites": { - "fields": [ - { - "configKey": "connectionModes.webDevice", - "value": true - }, - { - "configKey": "connectionModes.mobileDevice", - "value": true - } - ], - "condition": "or" - }, - "fields": [ - { - "type": "singleSelect", - "label": "Choose if you want to turn on events filtering:", - "configKey": "eventFilteringOption", - "note": "You must select either allowlist or denylist to enable events filtering", - "options": [ - { - "label": "No events filtering", - "value": "disable" - }, - { - "label": "Filter via allowlist", - "value": "whitelistedEvents" - }, - { - "label": "Filter via denylist", - "value": "blacklistedEvents" - } - ], - "default": "disable" - }, - { - "type": "tagInput", - "label": "Allowlisted events", - "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to allowlist.", - "configKey": "whitelistedEvents", - "tagKey": "eventName", - "placeholder": "e.g: Anonymous page visit", - "default": [ - { - "eventName": "" - } - ], - "preRequisites": { - "fields": [ - { - "configKey": "eventFilteringOption", - "value": "whitelistedEvents" - } - ] - } - }, - { - "type": "tagInput", - "label": "Denylisted events", - "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to denylist. ", - "configKey": "blacklistedEvents", - "tagKey": "eventName", - "placeholder": "e.g: Anonymous page visit", - "default": [ - { - "eventName": "" - } - ], - "preRequisites": { - "fields": [ - { - "configKey": "eventFilteringOption", - "value": "blacklistedEvents" - } - ] - } - } - ] - }, - { - "title": "OneTrust cookie consent settings", - "note": [ - "Enter your OneTrust category names if you have them configured. ", - { - "text": "Learn more ", - "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" - }, - "about RudderStack’s OneTrust Consent Manager feature." - ], - "fields": [ - { - "type": "tagInput", - "label": "Cookie category name", - "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", - "configKey": "oneTrustCookieCategories", - "tagKey": "oneTrustCookieCategory", - "placeholder": "e.g: Credit card visit", - "default": [ - { - "oneTrustCookieCategory": "" - } - ] - } - ] - } - ] + "configKey": "connectionModes.mobileDevice", + "value": true } - ] - }, - { - "title": "Mappings", - "hideEditIcon": true, - "sections": [ - { - "groups": [ - { - "title": "Rudderstack to Spotify Pixel Event Mappings", - "fields": [ - { - "type": "redirect", - "redirectGroupKey": "spotifyPixelEventMapping", - "label": "Event and property mappings" - } - ] - } - ] + ], + "condition": "or" + }, + "fields": [ + { + "type": "singleSelect", + "label": "Choose if you want to turn on events filtering:", + "configKey": "eventFilteringOption", + "note": "You must select either allowlist or denylist to enable events filtering", + "options": [ + { + "label": "No events filtering", + "value": "disable" + }, + { + "label": "Filter via allowlist", + "value": "whitelistedEvents" + }, + { + "label": "Filter via denylist", + "value": "blacklistedEvents" + } + ], + "default": "disable" + }, + { + "type": "tagInput", + "label": "Allowlisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to allowlist.", + "configKey": "whitelistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "whitelistedEvents" + } + ] } - ] - } - ], - "redirectGroups": { - "spotifyPixelEventMapping":{ - "fields": [ - { - "type": "mapping", - "label": "Mapping to trigger the Rudderstack events with standard Spotify Pixel events", - "configKey": "eventsToSpotifyPixelEvents", - "default": [], - "columns": [ - { - "type": "textInput", - "key": "from", - "label": "Event Name", - "placeholder": "e.g: Order Completed" - }, - { - "type": "singleSelect", - "key": "to", - "label": "Spotify Pixel Standard Event", - "placeholder": "e.g: Checkout", - "options": [ - { - "name": "Lead", - "value": "lead" - }, - { - "name": "Product", - "value": "product" - }, - { - "name": "Add to Cart", - "value": "addtocart" - }, - { - "name": "Check out", - "value": "checkout" - }, - { - "name": "Purchase", - "value": "purchase" - } - ] - } - ] + }, + { + "type": "tagInput", + "label": "Denylisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to denylist. ", + "configKey": "blacklistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "blacklistedEvents" + } + ] } - ] - } - }, - "sdkTemplate": { - "title": "Web SDK settings", - "note": "not visible in the ui", - "fields": [] - } + } + ] + }, + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack’s OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + }, + { + "title": "Mappings", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "Rudderstack to Spotify Pixel Event Mappings", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "spotifyPixelEventMapping", + "label": "Event and property mappings" + } + ] + } + ] + } + ] + } + ], + "redirectGroups": { + "spotifyPixelEventMapping": { + "fields": [ + { + "type": "mapping", + "label": "Mapping to trigger the Rudderstack events with standard Spotify Pixel events", + "configKey": "eventsToSpotifyPixelEvents", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "from", + "label": "Event Name", + "placeholder": "e.g: Order Completed" + }, + { + "type": "singleSelect", + "key": "to", + "label": "Spotify Pixel Standard Event", + "placeholder": "e.g: Checkout", + "options": [ + { + "name": "Lead", + "value": "lead" + }, + { + "name": "Product", + "value": "product" + }, + { + "name": "Add to Cart", + "value": "addtocart" + }, + { + "name": "Check out", + "value": "checkout" + }, + { + "name": "Purchase", + "value": "purchase" + } + ] + } + ] + } + ] + } + }, + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] } -} \ No newline at end of file + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json new file mode 100644 index 000000000..1563ae3e7 --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json @@ -0,0 +1,44 @@ +{ + "name": "CREATE_NEW_SCHEMA_DESTINATION", + "displayName": "Create New Schema Destination", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "checkboxField", + "tagInputField" + ], + "web": ["webSourceField", "useNativeSDK", "connectionMode"], + "android": ["androidSourceField", "useNativeSDK", "connectionMode"] + }, + "secretKeys": ["secretTextInputField"] + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json new file mode 100644 index 000000000..86836f70b --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json @@ -0,0 +1,118 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Test input field", + "configKey": "secretTextInputField", + "regex": "^(.{1,100})$", + "required": true, + "placeholder": "e.g. asdf123", + "secret": true + }, + { + "type": "singleSelect", + "label": "Single select field", + "configKey": "singleSelectField", + "options": [ + { + "label": "Value 1", + "value": "valOne" + }, + { + "label": "Value 2", + "value": "valTwo" + } + ], + "default": "valTwo" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [ + { + "title": "Configure settings", + "fields": [ + { + "type": "checkbox", + "label": "A sample checkbox", + "configKey": "checkboxField", + "default": true + } + ] + }, + { + "type": "tagInput", + "label": "A tag input", + "configKey": "tagInputField", + "tagKey": "tagKey", + "placeholder": "e.g: abc", + "default": [] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "SDK settings", + "note": "not visible in the ui", + "fields": [ + { + "type": "textInput", + "label": "A web source type field", + "configKey": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "configKey": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + } + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json new file mode 100644 index 000000000..560c62076 --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json @@ -0,0 +1,49 @@ +{ + "name": "CREATE_NEW_SCHEMA_DESTINATION_LEGACY_UI", + "displayName": "Create New Schema Destination with Legacy UI", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "textareaInputField", + "checkboxField", + "dynamicCustomFormField", + "dynamicFormField", + "dynamicSelectFormField", + "timeRangePickerField", + "timePickerField" + ], + "web": ["webSourceField", "useNativeSDK"], + "android": ["androidSourceField", "useNativeSDK"] + }, + "secretKeys": ["secretTextInputField", "textareaInput"] + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json new file mode 100644 index 000000000..751becd40 --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json @@ -0,0 +1,153 @@ +{ + "uiConfig": [ + { + "title": "Connection Settings", + "fields": [ + { + "type": "textInput", + "label": "Access Token", + "value": "secretTextInputField", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "required": true, + "placeholder": "e.g: 96d96af0cdb847XXXXa4e7cc13255705", + "secret": true + }, + { + "type": "textInput", + "label": "A web source type field", + "value": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "value": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + }, + { + "title": "Native SDK", + "fields": [ + { + "type": "defaultCheckbox", + "label": "Use device-mode to send events", + "value": "useNativeSDK", + "default": true + } + ] + }, + { + "title": "Other Settings", + "fields": [ + { + "type": "textareaInput", + "subType": "JSON", + "label": "Textarea Input Field", + "value": "textareaInputField", + "regex": ".*", + "required": true, + "secret": true + }, + { + "type": "singleSelect", + "value": "singleSelectField", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ], + "defaultOption": { + "name": "Val 2", + "value": "valTwo" + } + }, + { + "type": "checkbox", + "label": "Checkbox field", + "value": "checkboxField", + "default": true + }, + { + "type": "dynamicCustomForm", + "value": "dynamicCustomFormField", + "label": "Dynamic custom form field", + "customFields": [ + { + "type": "textInput", + "value": "formField", + "required": false + } + ] + }, + { + "type": "dynamicForm", + "label": "Dynamic form field", + "labelRight": "Label right", + "labelLeft": "Label left", + "keyLeft": "left", + "keyRight": "right", + "value": "dynamicFormField" + }, + { + "type": "dynamicSelectForm", + "label": "Dynamic select form field", + "labelLeft": "Label left", + "labelRight": "Label right", + "value": "dynamicSelectFormField", + "keyLeft": "left", + "keyRight": "right", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ] + }, + { + "type": "timeRangePicker", + "label": "Time Range Picker Field", + "value": "timeRangePickerField", + "startTime": { + "label": "start time", + "value": "startTime" + }, + "endTime": { + "label": "end time", + "value": "endTime" + }, + "options": { + "omitSeconds": true, + "minuteStep": 1 + }, + "required": false + }, + { + "type": "timePicker", + "label": "Time Picker Field", + "value": "timePickerField", + "options": { + "omitSeconds": true, + "minuteStep": 15 + }, + "required": false + } + ] + } + ] +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest/db-config.json b/test/component_tests/configurations/destinations/update_schema_dest/db-config.json new file mode 100644 index 000000000..e2dee2339 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest/db-config.json @@ -0,0 +1,44 @@ +{ + "name": "UPDATE_SCHEMA_DESTINATION", + "displayName": "Update Schema Destination", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "checkboxField", + "tagInputField" + ], + "web": ["webSourceField", "useNativeSDK", "connectionMode"], + "android": ["androidSourceField", "useNativeSDK", "connectionMode"] + }, + "secretKeys": ["secretTextInputField"] + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest/schema.json b/test/component_tests/configurations/destinations/update_schema_dest/schema.json new file mode 100644 index 000000000..aea615f1b --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest/schema.json @@ -0,0 +1,92 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud", "device"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud", "device"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest/ui-config.json b/test/component_tests/configurations/destinations/update_schema_dest/ui-config.json new file mode 100644 index 000000000..86836f70b --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest/ui-config.json @@ -0,0 +1,118 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Test input field", + "configKey": "secretTextInputField", + "regex": "^(.{1,100})$", + "required": true, + "placeholder": "e.g. asdf123", + "secret": true + }, + { + "type": "singleSelect", + "label": "Single select field", + "configKey": "singleSelectField", + "options": [ + { + "label": "Value 1", + "value": "valOne" + }, + { + "label": "Value 2", + "value": "valTwo" + } + ], + "default": "valTwo" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [ + { + "title": "Configure settings", + "fields": [ + { + "type": "checkbox", + "label": "A sample checkbox", + "configKey": "checkboxField", + "default": true + } + ] + }, + { + "type": "tagInput", + "label": "A tag input", + "configKey": "tagInputField", + "tagKey": "tagKey", + "placeholder": "e.g: abc", + "default": [] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "SDK settings", + "note": "not visible in the ui", + "fields": [ + { + "type": "textInput", + "label": "A web source type field", + "configKey": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "configKey": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + } + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json new file mode 100644 index 000000000..45833ffe8 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json @@ -0,0 +1,49 @@ +{ + "name": "UPDATE_SCHEMA_DESTINATION_LEGACY_UI", + "displayName": "Update Schema Destination with Legacy UI", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "textareaInputField", + "checkboxField", + "dynamicCustomFormField", + "dynamicFormField", + "dynamicSelectFormField", + "timeRangePickerField", + "timePickerField" + ], + "web": ["webSourceField", "useNativeSDK"], + "android": ["androidSourceField", "useNativeSDK"] + }, + "secretKeys": ["secretTextInputField", "textareaInput"] + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json new file mode 100644 index 000000000..9790d9ff6 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json @@ -0,0 +1,106 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "web": { + "type": "boolean" + } + } + }, + "textareaInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "dynamicCustomFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "formField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicSelectFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "timeRangePickerField": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + } + }, + "required": ["startTime", "endTime"] + }, + "timePickerField": { + "type": "string" + } + } + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json new file mode 100644 index 000000000..751becd40 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json @@ -0,0 +1,153 @@ +{ + "uiConfig": [ + { + "title": "Connection Settings", + "fields": [ + { + "type": "textInput", + "label": "Access Token", + "value": "secretTextInputField", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "required": true, + "placeholder": "e.g: 96d96af0cdb847XXXXa4e7cc13255705", + "secret": true + }, + { + "type": "textInput", + "label": "A web source type field", + "value": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "value": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + }, + { + "title": "Native SDK", + "fields": [ + { + "type": "defaultCheckbox", + "label": "Use device-mode to send events", + "value": "useNativeSDK", + "default": true + } + ] + }, + { + "title": "Other Settings", + "fields": [ + { + "type": "textareaInput", + "subType": "JSON", + "label": "Textarea Input Field", + "value": "textareaInputField", + "regex": ".*", + "required": true, + "secret": true + }, + { + "type": "singleSelect", + "value": "singleSelectField", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ], + "defaultOption": { + "name": "Val 2", + "value": "valTwo" + } + }, + { + "type": "checkbox", + "label": "Checkbox field", + "value": "checkboxField", + "default": true + }, + { + "type": "dynamicCustomForm", + "value": "dynamicCustomFormField", + "label": "Dynamic custom form field", + "customFields": [ + { + "type": "textInput", + "value": "formField", + "required": false + } + ] + }, + { + "type": "dynamicForm", + "label": "Dynamic form field", + "labelRight": "Label right", + "labelLeft": "Label left", + "keyLeft": "left", + "keyRight": "right", + "value": "dynamicFormField" + }, + { + "type": "dynamicSelectForm", + "label": "Dynamic select form field", + "labelLeft": "Label left", + "labelRight": "Label right", + "value": "dynamicSelectFormField", + "keyLeft": "left", + "keyRight": "right", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ] + }, + { + "type": "timeRangePicker", + "label": "Time Range Picker Field", + "value": "timeRangePickerField", + "startTime": { + "label": "start time", + "value": "startTime" + }, + "endTime": { + "label": "end time", + "value": "endTime" + }, + "options": { + "omitSeconds": true, + "minuteStep": 1 + }, + "required": false + }, + { + "type": "timePicker", + "label": "Time Picker Field", + "value": "timePickerField", + "options": { + "omitSeconds": true, + "minuteStep": 15 + }, + "required": false + } + ] + } + ] +} diff --git a/test/component_tests/data/createNewSchemaDest.json b/test/component_tests/data/createNewSchemaDest.json new file mode 100644 index 000000000..97fb27e5c --- /dev/null +++ b/test/component_tests/data/createNewSchemaDest.json @@ -0,0 +1,100 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "singleSelectField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud", "device"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud", "device"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/test/component_tests/data/createNewSchemaDestLegacyUI.json b/test/component_tests/data/createNewSchemaDestLegacyUI.json new file mode 100644 index 000000000..ddf0c78c4 --- /dev/null +++ b/test/component_tests/data/createNewSchemaDestLegacyUI.json @@ -0,0 +1,114 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "textareaInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "textareaInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "dynamicCustomFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "formField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicSelectFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "timeRangePickerField": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + } + }, + "required": ["startTime", "endTime"] + }, + "timePickerField": { + "type": "string" + } + } + } +} diff --git a/test/component_tests/data/updateSchemaDest.json b/test/component_tests/data/updateSchemaDest.json new file mode 100644 index 000000000..97fb27e5c --- /dev/null +++ b/test/component_tests/data/updateSchemaDest.json @@ -0,0 +1,100 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "singleSelectField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud", "device"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud", "device"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/test/component_tests/data/updateSchemaDestLegacyUI.json b/test/component_tests/data/updateSchemaDestLegacyUI.json new file mode 100644 index 000000000..ddf0c78c4 --- /dev/null +++ b/test/component_tests/data/updateSchemaDestLegacyUI.json @@ -0,0 +1,114 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "textareaInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "textareaInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "dynamicCustomFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "formField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicSelectFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "timeRangePickerField": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + } + }, + "required": ["startTime", "endTime"] + }, + "timePickerField": { + "type": "string" + } + } + } +} diff --git a/test/component_tests/schemaGenerator.test.ts b/test/component_tests/schemaGenerator.test.ts new file mode 100644 index 000000000..54fb1f3b9 --- /dev/null +++ b/test/component_tests/schemaGenerator.test.ts @@ -0,0 +1,151 @@ +import fs from 'fs'; +import path from 'path'; +import { execSync } from 'child_process'; + +function readSchemaFile(filePath): string | undefined { + let schema; + if (!fs.existsSync(filePath)) { + return schema; + } + + try { + schema = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } catch (e) { + /* empty */ + } + + return schema; +} + +function writeSchemaFile(filePath, schema) { + fs.writeFileSync(filePath, JSON.stringify(schema, null, 2)); +} + +describe('Schema Generator', () => { + const configDir = 'test/component_tests/configurations'; + + describe('should generate and save schema for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'create_new_schema_dest', + expectedSchemaFile: 'createNewSchemaDest.json', + }, + { + description: 'Legacy UI', + destName: 'create_new_schema_dest_legacy_ui', + expectedSchemaFile: 'createNewSchemaDestLegacyUI.json', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName, expectedSchemaFile }) => { + const cmd = `CONFIG_DIR=${configDir} npm run update:schema:destination "${destName}"`; + + execSync(cmd); + + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + + const schema = readSchemaFile(schemaFilePath); + if (schema) { + // delete the file + fs.unlinkSync(schemaFilePath); + } + + const expectedSchemaData = readSchemaFile( + path.resolve(__dirname, `./data/${expectedSchemaFile}`), + ); + + expect(schema).toEqual(expectedSchemaData); + }); + }); + + describe('should not generate and save schema if update option is not provided for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'create_new_schema_dest', + }, + { + description: 'Legacy UI', + destName: 'create_new_schema_dest_legacy_ui', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName }) => { + const cmd = `CONFIG_DIR=${configDir} npm run check:schema:destination "${destName}"`; + + execSync(cmd); + + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + + const schema = readSchemaFile(schemaFilePath); + expect(schema).toBeUndefined(); + }); + }); + + describe('should update and save schema for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'update_schema_dest', + expectedSchemaFile: 'updateSchemaDest.json', + }, + { + description: 'Legacy UI', + destName: 'update_schema_dest_legacy_ui', + expectedSchemaFile: 'updateSchemaDestLegacyUI.json', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName, expectedSchemaFile }) => { + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + const curSchema = readSchemaFile(schemaFilePath); + + const cmd = `CONFIG_DIR=${configDir} npm run update:schema:destination "${destName}"`; + + execSync(cmd); + + const schema = readSchemaFile(schemaFilePath); + // Restore schema file + if (schema) { + writeSchemaFile(schemaFilePath, curSchema); + } + + const expectedSchemaData = readSchemaFile( + path.resolve(__dirname, `./data/${expectedSchemaFile}`), + ); + + expect(schema).toEqual(expectedSchemaData); + }); + }); + + describe('should not save the schema file if update option is not provided for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'update_schema_dest', + }, + { + description: 'Legacy UI', + destName: 'update_schema_dest_legacy_ui', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName }) => { + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + const curSchema = readSchemaFile(schemaFilePath); + + const cmd = `CONFIG_DIR=${configDir} npm run check:schema:destination "${destName}"`; + + execSync(cmd); + + const schema = readSchemaFile(schemaFilePath); + + expect(schema).toEqual(curSchema); + }); + }); +}); diff --git a/test/data/validation/destinations/spotifyPixel.json b/test/data/validation/destinations/spotifyPixel.json index bd932cacc..d349ef3e6 100644 --- a/test/data/validation/destinations/spotifyPixel.json +++ b/test/data/validation/destinations/spotifyPixel.json @@ -27,9 +27,7 @@ } }, "result": false, - "err": [ - "pixelId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" - ] + "err": ["pixelId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] }, { "config": { @@ -59,9 +57,7 @@ } }, "result": false, - "err": [ - " must have required property 'pixelId'" - ] + "err": [" must have required property 'pixelId'"] }, { "config": { @@ -92,9 +88,7 @@ } }, "result": false, - "err": [ - "pixelId must be string" - ] + "err": ["pixelId must be string"] }, { "config": { @@ -153,8 +147,6 @@ } }, "result": false, - "err": [ - "eventsToSpotifyPixelEvents.0.to must be equal to one of the allowed values" - ] + "err": ["eventsToSpotifyPixelEvents.0.to must be equal to one of the allowed values"] } -] \ No newline at end of file +] From fdbb379e81ed16b29303b0e0b2d8cf6771ea8a56 Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Thu, 8 Feb 2024 21:20:43 +0530 Subject: [PATCH 02/17] chore: setup python before running tests (#1199) --- .github/workflows/deploy.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bea52c72b..06cb58655 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,22 +43,22 @@ jobs: run: | npm ci - - name: Execute Unit Tests - env: - HUSKY: 0 - run: | - npm run test:ci - - name: Install Python uses: actions/setup-python@v5.0.0 with: python-version: '3.9.13' - + - name: Display Python Version run: | which python python -c "import sys; print(sys.version)" + - name: Execute Unit Tests + env: + HUSKY: 0 + run: | + npm run test:ci + - name: Install Python Dependencies run: pip3 install -r ./scripts/requirements.txt From 6edc55b1db32139945365defbd4b0e462ce33339 Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Thu, 8 Feb 2024 21:40:34 +0530 Subject: [PATCH 03/17] chore: use proper python setup step (#1200) --- .github/workflows/deploy.yml | 8 +++----- .github/workflows/report-code-coverage.yml | 2 +- .../destinations/tiktok_ads/db-config.json | 13 ++++++++++++- test/component_tests/schemaGenerator.test.ts | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 06cb58655..24e2d6ab2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,11 +43,9 @@ jobs: run: | npm ci - - name: Install Python - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.9.13' - + - name: Set up Python + run: scripts/setup-python.sh + - name: Display Python Version run: | which python diff --git a/.github/workflows/report-code-coverage.yml b/.github/workflows/report-code-coverage.yml index 9791eb5d7..c65c16c76 100644 --- a/.github/workflows/report-code-coverage.yml +++ b/.github/workflows/report-code-coverage.yml @@ -26,7 +26,7 @@ jobs: - name: Install Dependencies run: npm ci - + - name: Set up Python run: scripts/setup-python.sh diff --git a/src/configurations/destinations/tiktok_ads/db-config.json b/src/configurations/destinations/tiktok_ads/db-config.json index b56a92240..ed7c780fe 100644 --- a/src/configurations/destinations/tiktok_ads/db-config.json +++ b/src/configurations/destinations/tiktok_ads/db-config.json @@ -17,7 +17,18 @@ "ketchConsentPurposes" ], "excludeKeys": [], - "supportedSourceTypes": ["web", "cloud", "ios", "android", "unity", "amp", "warehouse", "reactnative", "flutter", "cordova"], + "supportedSourceTypes": [ + "web", + "cloud", + "ios", + "android", + "unity", + "amp", + "warehouse", + "reactnative", + "flutter", + "cordova" + ], "supportedMessageTypes": { "cloud": ["track"], "device": { diff --git a/test/component_tests/schemaGenerator.test.ts b/test/component_tests/schemaGenerator.test.ts index 54fb1f3b9..91a2f546e 100644 --- a/test/component_tests/schemaGenerator.test.ts +++ b/test/component_tests/schemaGenerator.test.ts @@ -110,7 +110,7 @@ describe('Schema Generator', () => { const schema = readSchemaFile(schemaFilePath); // Restore schema file - if (schema) { + if (curSchema) { writeSchemaFile(schemaFilePath, curSchema); } From 009c8da1f0089f8413f105c2443a233cd597148a Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Mon, 12 Feb 2024 10:36:14 +0530 Subject: [PATCH 04/17] chore: standard formatting (#1198) --- .github/workflows/commitlint.yml | 36 ++ .github/workflows/verify.yml | 53 ++ .husky/pre-commit | 2 +- migration/migrate.py | 108 ++-- package.json | 5 +- scripts/configGenerator.py | 48 +- scripts/constants.py | 10 +- scripts/deployToDB.py | 111 ++-- scripts/schemaGenerator.py | 869 ++++++++++++++++++++----------- scripts/setup-python.sh | 1 + scripts/utils.py | 16 +- test/test_configGenerator.py | 14 +- 12 files changed, 837 insertions(+), 436 deletions(-) create mode 100644 .github/workflows/commitlint.yml create mode 100644 .github/workflows/verify.yml diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml new file mode 100644 index 000000000..a8ff39eee --- /dev/null +++ b/.github/workflows/commitlint.yml @@ -0,0 +1,36 @@ +name: Commitlint + +on: [push] + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + + - name: Setup Node + uses: actions/setup-node@v4.0.1 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install Dependencies + run: npm ci + + - name: Print versions + run: | + git --version + node --version + npm --version + npx commitlint --version + + # Run the commitlint action, considering its own dependencies and yours as well 🚀 + # `github.workspace` is the path to your repository. + - uses: wagoid/commitlint-github-action@v5 + env: + NODE_PATH: ${{ github.workspace }}/node_modules + with: + commitDepth: 1 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 000000000..90ec08d77 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,53 @@ +name: Code quality checks + +on: + pull_request: + +jobs: + py-scripts-lint: + name: Check for formatting of python scripts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Setting up python libraries + run: ./scripts/setup-python.sh + + # Reference: https://black.readthedocs.io/en/stable/integrations/github_actions.html + - name: Check formatting for Python files + uses: psf/black@stable + with: + options: '--check --verbose' + + formatting-lint: + name: Check for formatting & lint errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + + - name: Setup Node + uses: actions/setup-node@v3.7.0 + with: + node-version-file: .nvmrc + cache: 'npm' + + - name: Install Dependencies + run: npm ci + + - name: Run Lint Checks + run: | + npm run lint + + - run: git diff --exit-code + + - name: Error message + if: ${{ failure() }} + run: | + echo 'ESLint check is failing. Please run `npm run lint` on your working copy and commit the changes.' diff --git a/.husky/pre-commit b/.husky/pre-commit index d4a43dd13..14ccce4a2 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,4 @@ #!/usr/bin/env sh . "$(dirname -- "$0")/_/husky.sh" -npm run pre-commit +npm run pre-commit \ No newline at end of file diff --git a/migration/migrate.py b/migration/migrate.py index 79c958304..6106730e2 100644 --- a/migration/migrate.py +++ b/migration/migrate.py @@ -2,116 +2,126 @@ import json import os -DEST_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/destinations/schemas/' -DEST_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/destinationConfigs/' -DEST_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/' +DEST_WEBAPP_FILE_PATH = "rudder-webapp/src/components/destinations/schemas/" +DEST_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/destinationConfigs/" +DEST_SCHEMA_FILE_PATH = "rudder-config-backend/src/scripts/schemaList/destinations/" -SOURCE_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/' -SOURCE_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/sourceConfigs/' +SOURCE_WEBAPP_FILE_PATH = ( + "rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/" +) +SOURCE_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/sourceConfigs/" # SOURCE_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/' -CONFIGURATIONS_DIR_PATH = '../src/configurations' -DEST_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/destinations' -SRC_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/sources' +CONFIGURATIONS_DIR_PATH = "../src/configurations" +DEST_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/destinations" +SRC_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/sources" + def update_destination(): - dest_list = [d[:-5].lower() for d in os.listdir(f'../../{DEST_CONFIG_BE_FILE_PATH}')] + dest_list = [ + d[:-5].lower() for d in os.listdir(f"../../{DEST_CONFIG_BE_FILE_PATH}") + ] for dest in dest_list: final_data = [] # read db_config - with open(f'../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json', 'r') as f: + with open(f"../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json", "r") as f: db_config = json.loads(f.read()) - print (db_config) + print(db_config) final_data.append(db_config) - print ("========================") + print("========================") # read ui_config - with open(f'../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json', 'r') as f: + with open(f"../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json", "r") as f: ui_config = {"uiConfig": json.loads(f.read())} - print (ui_config) + print(ui_config) final_data.append(ui_config) - print ("========================") + print("========================") # read schema - if f'{dest.upper()}.json' in os.listdir(f'../../{DEST_SCHEMA_FILE_PATH}'): - with open(f'../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json', 'r') as f: + if f"{dest.upper()}.json" in os.listdir(f"../../{DEST_SCHEMA_FILE_PATH}"): + with open(f"../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json", "r") as f: schema = {"schema": json.loads(f.read())} else: schema = {"schema": None} - print (schema) + print(schema) final_data.append(schema) - print ("========================") + print("========================") # metadata - with open('dest_meta_template.json', 'r') as f: + with open("dest_meta_template.json", "r") as f: metadata = json.loads(f.read()) - print (metadata) + print(metadata) final_data.append(metadata) - print ("========================") + print("========================") ######################## ## write new files ######################## if dest not in os.listdir(DEST_CONFIG_PATH): - os.system(f'mkdir {DEST_CONFIG_PATH}/{dest}') - print (f'created directory for {dest}') + os.system(f"mkdir {DEST_CONFIG_PATH}/{dest}") + print(f"created directory for {dest}") - file_names = ['db-config', 'ui-config', 'schema', 'metadata'] + file_names = ["db-config", "ui-config", "schema", "metadata"] for index, f_name in enumerate(file_names): - print (f'writing {f_name} for {dest}...') - with open(f'{DEST_CONFIG_PATH}/{dest}/{f_name}.json', 'w') as f: + print(f"writing {f_name} for {dest}...") + with open(f"{DEST_CONFIG_PATH}/{dest}/{f_name}.json", "w") as f: f.write(json.dumps(final_data[index], indent=2)) - print (f'complete for {dest}...') - print ('------------------------------------------------') + print(f"complete for {dest}...") + print("------------------------------------------------") + def update_source(): - source_list = [d[:-5].lower() for d in os.listdir(f'../../{SOURCE_CONFIG_BE_FILE_PATH}')] + source_list = [ + d[:-5].lower() for d in os.listdir(f"../../{SOURCE_CONFIG_BE_FILE_PATH}") + ] for source in source_list: final_data = [] # read db_config - with open(f'../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json', 'r') as f: + with open(f"../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json", "r") as f: db_config = json.loads(f.read()) - print (db_config) + print(db_config) final_data.append(db_config) - print ("========================") + print("========================") # read ui_config - if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_WEBAPP_FILE_PATH}'): - with open(f'../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json', 'r') as f: + if f"{source.upper()}.json" in os.listdir(f"../../{SOURCE_WEBAPP_FILE_PATH}"): + with open( + f"../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json", "r" + ) as f: ui_config = {"uiConfig": json.loads(f.read())} else: ui_config = {"uiConfig": None} - print (ui_config) + print(ui_config) final_data.append(ui_config) - print ("========================") + print("========================") # read schema (we don't need schema for sources) # if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_SCHEMA_FILE_PATH}'): # with open(f'../../{SOURCE_SCHEMA_FILE_PATH}{source.upper()}.json', 'r') as f: # schema = {"schema": json.loads(f.read())} # else: schema = {"schema": None} - print (schema) + print(schema) final_data.append(schema) - print ("========================") + print("========================") # metadata - with open('source_meta_template.json', 'r') as f: + with open("source_meta_template.json", "r") as f: metadata = json.loads(f.read()) - print (metadata) + print(metadata) final_data.append(metadata) - print ("========================") + print("========================") ######################## ## write new files ######################## if source not in os.listdir(SRC_CONFIG_PATH): - os.system(f'mkdir {SRC_CONFIG_PATH}/{source}') - print (f'created directory for {source}') + os.system(f"mkdir {SRC_CONFIG_PATH}/{source}") + print(f"created directory for {source}") - file_names = ['db_config', 'ui_config', 'schema', 'metadata'] + file_names = ["db_config", "ui_config", "schema", "metadata"] for index, f_name in enumerate(file_names): - print (f'writing {f_name} for {source}...') - with open(f'{SRC_CONFIG_PATH}/{source}/{f_name}.json', 'w') as f: + print(f"writing {f_name} for {source}...") + with open(f"{SRC_CONFIG_PATH}/{source}/{f_name}.json", "w") as f: f.write(json.dumps(final_data[index], indent=2)) - print (f'complete for {source}...') - print ('------------------------------------------------') + print(f"complete for {source}...") + print("------------------------------------------------") if __name__ == "__main__": diff --git a/package.json b/package.json index 534f118ac..167e25fd0 100755 --- a/package.json +++ b/package.json @@ -21,8 +21,10 @@ "test:silent": "npm run test -- --silent", "release": "npx standard-version", "check:lint": "eslint \"src/**/*.*\" -f json -o reports/eslint.json || exit 0", + "format:py": "python3 -m black .", "format": "prettier . --write", "lint:fix": "eslint \"src/**/*.*\" --fix", + "lint": "npm run format && npm run lint:fix", "prepare": "husky install", "pre-commit": "npm run test && npx lint-staged", "commit-msg": "commitlint --edit", @@ -79,7 +81,8 @@ "glob": "^9.3.2" }, "lint-staged": { - "*.{json,js,ts,md}": "prettier --write" + "*.{json,js,ts,md}": "prettier --write", + "*.{py}": "python3 -m black" }, "config": { "commitizen": { diff --git a/scripts/configGenerator.py b/scripts/configGenerator.py index 1a1fdaf4d..f7e38ba0c 100644 --- a/scripts/configGenerator.py +++ b/scripts/configGenerator.py @@ -4,23 +4,24 @@ import os import sys -ConfigData = TypedDict('ConfigData', {'db_config': str, 'ui_config': str}) +ConfigData = TypedDict("ConfigData", {"db_config": str, "ui_config": str}) + def generateConfigs(data) -> ConfigData: # Read the content of template-db-config.json - with open('scripts/template-db-config.json', 'r') as file: + with open("scripts/template-db-config.json", "r") as file: template_db_config = json.load(file) # Read the content of template-ui-config.json - with open('scripts/template-ui-config.json', 'r') as file: + with open("scripts/template-ui-config.json", "r") as file: template_ui_config = json.load(file) # Create db-config object with the same content db_config = template_db_config - db_config['displayName'] = data['displayName'] - db_config['name'] = data['displayName'] - formFields = data['formFields'] + db_config["displayName"] = data["displayName"] + db_config["name"] = data["displayName"] + formFields = data["formFields"] # Create db-config object with the same content ui_config = template_ui_config @@ -33,14 +34,13 @@ def appendFieldsInGroups(settings, field): if groups: first_group = groups[0] if "fields" in first_group: - del field['required'] + del field["required"] first_group["fields"].append(field) - def updateUiConfig(field): if "uiConfig" in ui_config and "baseTemplate" in ui_config["uiConfig"]: base_template = ui_config["uiConfig"]["baseTemplate"] - if base_template and field['required'] == True: + if base_template and field["required"] == True: connection_settings = base_template[0] appendFieldsInGroups(connection_settings, field) elif base_template: @@ -48,28 +48,27 @@ def updateUiConfig(field): appendFieldsInGroups(configuration_settings, field) return ui_config - # Iterate over JSON objects in the array for obj in formFields: # update field in ui-config ui_config = updateUiConfig(obj) # update db-config for key, value in obj.items(): - if key == 'configKey': - db_config['config']['destConfig']['defaultConfig'].append( - value) - if key == 'secret' and value == True: - db_config['config']['secretKeys'].append( - db_config['config']['destConfig']['defaultConfig'][-1]) + if key == "configKey": + db_config["config"]["destConfig"]["defaultConfig"].append(value) + if key == "secret" and value == True: + db_config["config"]["secretKeys"].append( + db_config["config"]["destConfig"]["defaultConfig"][-1] + ) db_config = json.dumps(db_config) ui_config = json.dumps(ui_config) - return {'db_config':db_config, 'ui_config': ui_config} + return {"db_config": db_config, "ui_config": ui_config} -if __name__ == '__main__': - file_path = sys.argv[1] if len(sys.argv) > 1 else 'test/configData/inputData.json' - with open(file_path, 'r') as file: +if __name__ == "__main__": + file_path = sys.argv[1] if len(sys.argv) > 1 else "test/configData/inputData.json" + with open(file_path, "r") as file: # Load the JSON data data = json.load(file) @@ -79,16 +78,15 @@ def updateUiConfig(field): if not os.path.exists(directory): os.makedirs(directory) - with open(file_path, 'w') as file: + with open(file_path, "w") as file: # Write the new content - file.write(configData['db_config']) - + file.write(configData["db_config"]) file_path = f'src/configurations/destinations/{data["displayName"]}/ui-config.json' directory = os.path.dirname(file_path) if not os.path.exists(directory): os.makedirs(directory) - with open(file_path, 'w') as file: + with open(file_path, "w") as file: # Write the new content - file.write(configData['ui_config']) + file.write(configData["ui_config"]) diff --git a/scripts/constants.py b/scripts/constants.py index 54b717202..5d9c7b926 100644 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -1,11 +1,11 @@ import os -__DEFAULT_CONFIG_DIR = 'src/configurations' +__DEFAULT_CONFIG_DIR = "src/configurations" CONFIG_DIR = __DEFAULT_CONFIG_DIR -if 'CONFIG_DIR' in os.environ: - CONFIG_DIR = os.environ['CONFIG_DIR'] +if "CONFIG_DIR" in os.environ: + CONFIG_DIR = os.environ["CONFIG_DIR"] -TEST_INTEGRATION_NAME_PREFIX = 'test_' +TEST_INTEGRATION_NAME_PREFIX = "test_" -TEST_INTEGRATION_NAME_SUFFIX = '_ignore' +TEST_INTEGRATION_NAME_SUFFIX = "_ignore" diff --git a/scripts/deployToDB.py b/scripts/deployToDB.py index 6c513fb92..97b4b3d00 100644 --- a/scripts/deployToDB.py +++ b/scripts/deployToDB.py @@ -18,12 +18,12 @@ ######################### # ENV VARIABLES -CONTROL_PLANE_URL=sys.argv[1] +CONTROL_PLANE_URL = sys.argv[1] print(CONTROL_PLANE_URL) -USERNAME=os.environ['API_USER'] #sys.argv[2] +USERNAME = os.environ["API_USER"] # sys.argv[2] print(USERNAME) -PASSWORD=os.environ['API_PASSWORD'] #sys.argv[3] -#print(PASSWORD) +PASSWORD = os.environ["API_PASSWORD"] # sys.argv[3] +# print(PASSWORD) ######################### # CONSTANTS HEADER = {"Content-Type": "application/json"} @@ -34,131 +34,152 @@ ######################### # UTIL METHODS def parse_response(resp): - if resp.status_code >= 200 and resp.status_code <= 300: - return resp.status_code, resp.json() - else: - return resp.status_code, str(resp.content) + if resp.status_code >= 200 and resp.status_code <= 300: + return resp.status_code, resp.json() + else: + return resp.status_code, str(resp.content) + def get_persisted_store(base_url, selector): - request_url = f'{base_url}/{selector}-definitions' + request_url = f"{base_url}/{selector}-definitions" response = requests.get(request_url) return json.loads(response.text) + def get_config_definition(base_url, selector, name): - request_url = f'{base_url}/{selector}-definitions/{name}' + request_url = f"{base_url}/{selector}-definitions/{name}" response = requests.get(request_url) return response + def get_file_content(name, selector): - file_selectors = ['db-config.json', 'ui-config.json', 'schema.json'] + file_selectors = ["db-config.json", "ui-config.json", "schema.json"] - directory = f'./{CONFIG_DIR}/{selector}s/{name}' + directory = f"./{CONFIG_DIR}/{selector}s/{name}" available_files = os.listdir(directory) file_content = {} for file_selector in file_selectors: if file_selector in available_files: - with open (f'{directory}/{file_selector}', 'r') as f: + with open(f"{directory}/{file_selector}", "r") as f: file_content.update(json.loads(f.read())) return file_content + def update_config_definition(selector, name, fileData): - url = f'{CONTROL_PLANE_URL}/{selector}-definitions/{name}' + url = f"{CONTROL_PLANE_URL}/{selector}-definitions/{name}" resp = requests.post(url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH) return parse_response(resp) + def create_config_definition(selector, fileData): - url = f'{CONTROL_PLANE_URL}/{selector}-definitions/' + url = f"{CONTROL_PLANE_URL}/{selector}-definitions/" resp = requests.post(url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH) return parse_response(resp) + def update_config(data_diff, selector): results = [] for diff in data_diff: - name = diff['name'] + name = diff["name"] fileData = get_file_content(name, selector) nameInConfig = fileData["name"] - if diff['action'] == 'create': - url = f'{CONTROL_PLANE_URL}/{selector}-definitions' + if diff["action"] == "create": + url = f"{CONTROL_PLANE_URL}/{selector}-definitions" else: - url = f'{CONTROL_PLANE_URL}/{selector}-definitions/{nameInConfig}' + url = f"{CONTROL_PLANE_URL}/{selector}-definitions/{nameInConfig}" - resp = requests.post(url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH) + resp = requests.post( + url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH + ) status, response = parse_response(resp) - diff['update'] = {"status": status, "response": response} + diff["update"] = {"status": status, "response": response} # results.append(diff) results.append(name) - return json.dumps(results, indent=2) + def update_diff_db(selector): final_report = [] ## data sets - current_items = os.listdir(f'./{CONFIG_DIR}/{selector}s') + current_items = os.listdir(f"./{CONFIG_DIR}/{selector}s") for item in current_items: # check if item is a directory - if not os.path.isdir(f'./{CONFIG_DIR}/{selector}s/{item}'): + if not os.path.isdir(f"./{CONFIG_DIR}/{selector}s/{item}"): continue updated_data = get_file_content(item, selector) - persisted_data = get_config_definition(CONTROL_PLANE_URL, selector, updated_data["name"]) + persisted_data = get_config_definition( + CONTROL_PLANE_URL, selector, updated_data["name"] + ) if persisted_data.status_code == 200: - diff = jsondiff.diff(json.loads(persisted_data.text), updated_data, marshal=True) + diff = jsondiff.diff( + json.loads(persisted_data.text), updated_data, marshal=True + ) # ignore the $delete - values present in DB but missing in files. Anyways this doesn't get reflected in DB as keys are missing in files itself. # Best practice is to make sure all keys are maintained in the config files irrespective of them being null. - del diff['$delete'] - - if len(diff.keys()) > 0: # changes exist - #print(diff) - status, response = update_config_definition(selector, updated_data["name"], updated_data) - final_report.append({"name": updated_data["name"], "action":"update", "status": status}) + del diff["$delete"] + + if len(diff.keys()) > 0: # changes exist + # print(diff) + status, response = update_config_definition( + selector, updated_data["name"], updated_data + ) + final_report.append( + {"name": updated_data["name"], "action": "update", "status": status} + ) else: - final_report.append({"name": updated_data["name"], "action":"na", "status": ""}) + final_report.append( + {"name": updated_data["name"], "action": "na", "status": ""} + ) else: status, response = create_config_definition(selector, updated_data) - final_report.append({"name": updated_data["name"], "action":"create", "status": status}) + final_report.append( + {"name": updated_data["name"], "action": "create", "status": status} + ) return final_report + def get_stale_data(selector, report): stale_config_report = [] persisted_data_set = get_persisted_store(CONTROL_PLANE_URL, selector) - persisted_items = [item['name'] for item in persisted_data_set] - file_items = [item['name'] for item in report] + persisted_items = [item["name"] for item in persisted_data_set] + file_items = [item["name"] for item in report] for item in persisted_items: if item not in file_items: - stale_config_report.append({item}) + stale_config_report.append({item}) return stale_config_report -if __name__ == '__main__': + +if __name__ == "__main__": print("Running Destination Definitions Updates") - dest_final_report = update_diff_db('destination') + dest_final_report = update_diff_db("destination") print("Destination Definition Update Report") print(dest_final_report) print("Destination Stale Config Report") - print(get_stale_data('destination', dest_final_report)) + print(get_stale_data("destination", dest_final_report)) print("Running Source Definitions Updates") - src_final_report = update_diff_db('source') + src_final_report = update_diff_db("source") print("Source Definition Update Report") print(src_final_report) print("Source Stale Config Report") - print(get_stale_data('source', src_final_report)) + print(get_stale_data("source", src_final_report)) print("Running Wht Lib Projects Definitions Updates") - wht_final_report = update_diff_db('wht-lib-project') + wht_final_report = update_diff_db("wht-lib-project") print("Wht lib project Definition Update Report") print(wht_final_report) print("Wht lib project Stale Config Report") - print(get_stale_data('wht-lib-project', wht_final_report)) - \ No newline at end of file + print(get_stale_data("wht-lib-project", wht_final_report)) diff --git a/scripts/schemaGenerator.py b/scripts/schemaGenerator.py index 36d9d9ee3..60f97a0cf 100644 --- a/scripts/schemaGenerator.py +++ b/scripts/schemaGenerator.py @@ -1,4 +1,4 @@ -''' +""" Usage: schemaGenerator.py [-h] [-name name | -all] [-update] selector 1. selector - “source” or “destination” 2. all - runs the validator for all the selector. @@ -7,7 +7,8 @@ Example: 1. python3 scripts/schemaGenerator.py -name="adobe_analytics" destination 2. python3 scripts/schemaGenerator.py -all source -''' +""" + import os import warnings from enum import Enum @@ -15,7 +16,8 @@ from utils import get_json_from_file, get_json_diff, apply_json_diff, get_formatted_json from constants import CONFIG_DIR -EXCLUDED_DEST = ['postgres', 'bq', 'azure_synapse', 'clickhouse', 'deltalake', 'kafka'] +EXCLUDED_DEST = ["postgres", "bq", "azure_synapse", "clickhouse", "deltalake", "kafka"] + class FieldTypeEnum(Enum): STRING = "string" @@ -29,13 +31,14 @@ def is_old_format(uiConfig): return False return True + def get_options_list_for_enum(field): """Creates the list of options given in field and return the list Args: field (object): Individual field in ui-config. Returns: list: list of options - """ + """ options_list = [] for i in range(0, len(field["options"])): if isinstance(field["options"][i], int) or isinstance(field["options"][i], str): @@ -43,14 +46,19 @@ def get_options_list_for_enum(field): else: options_list.append(field["options"][i]["value"]) # allow empty field in enum if field in not required. - if "default" not in field and "defaultOption" not in field and field.get("required", False) == False: - options_list.append("") + if ( + "default" not in field + and "defaultOption" not in field + and field.get("required", False) == False + ): + options_list.append("") return options_list + def generalize_regex_pattern(field): """Generates the pattern for schema based on the type of field. - For type : singleSelect and dynamicSelectForm, the pattern is generated by iterating over options. - - For other types, + - For other types, - If the field contains regex, then regex is the pattern; it gets prefixed with a default prefix if regex does not have it. - Else, the default prefix is appended with ^(.{0,100}). @@ -59,88 +67,116 @@ def generalize_regex_pattern(field): Returns: string: generated pattern for the field. - """ + """ defaultSubPattern = "(^\\{\\{.*\\|\\|(.*)\\}\\}$)" defaultEnvPattern = "(^env[.].+)" pattern = "" if "regex" in field: pattern = field["regex"] - if defaultSubPattern not in pattern and (('value' not in field or field['value'] != 'purpose') and ('configKey' not in field or field['configKey'] != 'purpose')): + if defaultSubPattern not in pattern and ( + ("value" not in field or field["value"] != "purpose") + and ("configKey" not in field or field["configKey"] != "purpose") + ): pattern = "|".join([defaultSubPattern, pattern]) - if defaultEnvPattern not in pattern and (('value' not in field or field['value'] != 'purpose') and ('configKey' not in field or field['configKey'] != 'purpose')): + if defaultEnvPattern not in pattern and ( + ("value" not in field or field["value"] != "purpose") + and ("configKey" not in field or field["configKey"] != "purpose") + ): indexToPlace = pattern.find(defaultSubPattern) + len(defaultSubPattern) - pattern = pattern[:indexToPlace] + '|' + defaultEnvPattern + pattern[indexToPlace:] + pattern = ( + pattern[:indexToPlace] + + "|" + + defaultEnvPattern + + pattern[indexToPlace:] + ) # TODO: we should not use a case here for the individual properties. Just pass the desired pattern as regex property # in ketch purpose fields and delete next case - elif ('value' in field and field['value'] == 'purpose') or ('configKey' in field and field['configKey'] == 'purpose'): - pattern = '^(.{0,100})$' + elif ("value" in field and field["value"] == "purpose") or ( + "configKey" in field and field["configKey"] == "purpose" + ): + pattern = "^(.{0,100})$" else: - pattern = "|".join([defaultSubPattern, defaultEnvPattern, '^(.{0,100})$']) + pattern = "|".join([defaultSubPattern, defaultEnvPattern, "^(.{0,100})$"]) return pattern def is_dest_field_dependent_on_source(field, dbConfig, schema_field_name): - """Checks if the given field is source-specific by using dbConfig. - In dbConfig all the sources are listed in 'supportedSourceTypes', + """Checks if the given field is source-specific by using dbConfig. + In dbConfig all the sources are listed in 'supportedSourceTypes', and their fields are listed inside 'destConfig' with the key as the source. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: boolean: True if the field is source dependent else, False. - """ + """ if not dbConfig: return False for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): return True return False + def is_field_present_in_default_config(field, dbConfig, schema_field_name): """Checks if the given field is present in defaultConfig list present in dbConfig. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: boolean: True if field is in defaultConfig else False. - """ + """ if not dbConfig: return False - if "destConfig" in dbConfig and "defaultConfig" in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"]["defaultConfig"]: + if ( + "destConfig" in dbConfig + and "defaultConfig" in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"]["defaultConfig"] + ): return True return False + def generate_schema_for_default_checkbox(field, dbConfig, schema_field_name): """Creates a schema object of defaultCheckbox. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object - """ - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + """ + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) defaultCheckboxSchemaObj = {} if isSourceDependent: defaultCheckboxSchemaObj["type"] = FieldTypeEnum.OBJECT.value defaultCheckboxSchemaObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): defaultCheckboxSchemaObj["properties"][sourceType] = { - "type": FieldTypeEnum.BOOLEAN.value} + "type": FieldTypeEnum.BOOLEAN.value + } else: defaultCheckboxSchemaObj["type"] = FieldTypeEnum.BOOLEAN.value if "default" in field: @@ -154,22 +190,28 @@ def generate_schema_for_checkbox(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) checkboxSchemaObj = {} if isSourceDependent: checkboxSchemaObj["type"] = FieldTypeEnum.OBJECT.value checkboxSchemaObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): checkboxSchemaObj["properties"][sourceType] = { - "type": FieldTypeEnum.BOOLEAN.value} + "type": FieldTypeEnum.BOOLEAN.value + } else: checkboxSchemaObj["type"] = FieldTypeEnum.BOOLEAN.value if "default" in field: @@ -183,27 +225,35 @@ def generate_schema_for_textinput(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ textInputSchemaObj = {} - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) if isSourceDependent: textInputSchemaObj["type"] = FieldTypeEnum.OBJECT.value textInputSchemaObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): textInputSchemaObj["properties"][sourceType] = { - "type": FieldTypeEnum.STRING.value} - if 'regex' in field: - textInputSchemaObj["properties"][sourceType]["pattern"] = generalize_regex_pattern(field) + "type": FieldTypeEnum.STRING.value + } + if "regex" in field: + textInputSchemaObj["properties"][sourceType]["pattern"] = ( + generalize_regex_pattern(field) + ) else: textInputSchemaObj = {"type": FieldTypeEnum.STRING.value} - if 'regex' in field: + if "regex" in field: textInputSchemaObj["pattern"] = generalize_regex_pattern(field) return textInputSchemaObj @@ -214,14 +264,14 @@ def generate_schema_for_textarea_input(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ textareaInputObj = {"type": FieldTypeEnum.STRING.value} - if 'regex' in field: + if "regex" in field: textareaInputObj["pattern"] = generalize_regex_pattern(field) return textareaInputObj @@ -232,25 +282,25 @@ def generate_schema_for_single_select(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ singleSelectObj = {} - if "mode" in field and field["mode"] == 'multiple': - singleSelectObj = {"type": FieldTypeEnum.ARRAY.value} + if "mode" in field and field["mode"] == "multiple": + singleSelectObj = {"type": FieldTypeEnum.ARRAY.value} singleSelectObj["items"] = { "type": FieldTypeEnum.STRING.value, - "enum": get_options_list_for_enum(field) + "enum": get_options_list_for_enum(field), } if "default" or "defaultOption" in field: if isinstance(field["defaultOption"]["value"], list): singleSelectObj["default"] = field["defaultOption"]["value"] elif field["defaultOption"]["value"]: singleSelectObj["default"] = [field["defaultOption"]["value"]] - elif 'default' in field: + elif "default" in field: singleSelectObj["default"] = field["default"] else: singleSelectObj = {"type": FieldTypeEnum.STRING.value} @@ -258,16 +308,21 @@ def generate_schema_for_single_select(field, dbConfig, schema_field_name): if "default" or "defaultOption" in field: if "defaultOption" in field: singleSelectObj["default"] = field["defaultOption"]["value"] - elif 'default' in field: + elif "default" in field: singleSelectObj["default"] = field["default"] - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) if isSourceDependent: newSingleSelectObj = {"type": FieldTypeEnum.OBJECT.value} newSingleSelectObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): newSingleSelectObj["properties"][sourceType] = singleSelectObj singleSelectObj = newSingleSelectObj return singleSelectObj @@ -279,7 +334,7 @@ def generate_schema_for_dynamic_custom_form(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -297,46 +352,71 @@ def generate_schema_for_dynamic_custom_form(field, dbConfig, schema_field_name): if "rowFields" in field: customFieldsKey = "rowFields" - allOfSchemaObj = generate_schema_for_dynamic_custom_form_allOf(field[customFieldsKey], dbConfig, schema_field_name) + allOfSchemaObj = generate_schema_for_dynamic_custom_form_allOf( + field[customFieldsKey], dbConfig, schema_field_name + ) for customField in field[customFieldsKey]: - customFieldSchemaObj = uiTypetoSchemaFn.get(customField["type"])(customField, dbConfig, schema_field_name) - isCustomFieldDependentOnSource = is_dest_field_dependent_on_source(customField, dbConfig, schema_field_name) + customFieldSchemaObj = uiTypetoSchemaFn.get(customField["type"])( + customField, dbConfig, schema_field_name + ) + isCustomFieldDependentOnSource = is_dest_field_dependent_on_source( + customField, dbConfig, schema_field_name + ) if "preRequisites" in customField: continue - if 'pattern' not in customFieldSchemaObj and not isCustomFieldDependentOnSource and customFieldSchemaObj["type"] == FieldTypeEnum.STRING.value and customField["type"] != "singleSelect" and customField["type"] != "dynamicSelectForm": + if ( + "pattern" not in customFieldSchemaObj + and not isCustomFieldDependentOnSource + and customFieldSchemaObj["type"] == FieldTypeEnum.STRING.value + and customField["type"] != "singleSelect" + and customField["type"] != "dynamicSelectForm" + ): customFieldSchemaObj["pattern"] = generalize_regex_pattern(customField) # If the custom field is source dependent, we remove the source keys as it's not required inside custom fields, rather they need to be moved to top. if isCustomFieldDependentOnSource: for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: - customFieldSchemaObj = customFieldSchemaObj["properties"][sourceType] + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): + customFieldSchemaObj = customFieldSchemaObj["properties"][ + sourceType + ] break - dynamicCustomFormItemObj["properties"][customField[schema_field_name]] = customFieldSchemaObj + dynamicCustomFormItemObj["properties"][ + customField[schema_field_name] + ] = customFieldSchemaObj if allOfSchemaObj: - dynamicCustomFormItemObj['allOf'] = allOfSchemaObj + dynamicCustomFormItemObj["allOf"] = allOfSchemaObj dynamicCustomFormObj["items"] = dynamicCustomFormItemObj - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) # If the field is source dependent, new schema object is created by setting the fields inside the source. if isSourceDependent: newDynamicCustomFormObj = {"type": FieldTypeEnum.OBJECT.value} newDynamicCustomFormObj["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): newDynamicCustomFormObj["properties"][sourceType] = dynamicCustomFormObj dynamicCustomFormObj = newDynamicCustomFormObj return dynamicCustomFormObj - -def generate_schema_for_dynamic_custom_form_allOf(customFields, dbConfig, schema_field_name): +def generate_schema_for_dynamic_custom_form_allOf( + customFields, dbConfig, schema_field_name +): """Creates the allOf structure of schema, empty if not required. - Finds the list of unique preRequisites. - For each unique preRequisites, the properties are found by matching the current preRequisites. @@ -359,7 +439,9 @@ def generate_schema_for_dynamic_custom_form_allOf(customFields, dbConfig, schema continue isPresent = False for preRequisites in preRequisitesList: - if compare_pre_requisite_fields(preRequisites, field["preRequisites"]["fields"], True): + if compare_pre_requisite_fields( + preRequisites, field["preRequisites"]["fields"], True + ): isPresent = True break if not isPresent: @@ -373,8 +455,12 @@ def generate_schema_for_dynamic_custom_form_allOf(customFields, dbConfig, schema for field in customFields: if "preRequisites" not in field: continue - if compare_pre_requisite_fields(field["preRequisites"]["fields"], preRequisites, True): - thenObj["properties"][field[schema_field_name]] = uiTypetoSchemaFn.get(field["type"])(field, dbConfig, schema_field_name) + if compare_pre_requisite_fields( + field["preRequisites"]["fields"], preRequisites, True + ): + thenObj["properties"][field[schema_field_name]] = uiTypetoSchemaFn.get( + field["type"] + )(field, dbConfig, schema_field_name) if "required" in field and field["required"] == True: thenObj["required"].append(field[schema_field_name]) allOfItemObj["then"] = thenObj @@ -391,18 +477,19 @@ def generate_schema_for_dynamic_form(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ + def generate_key(forFieldWithTo): obj = { "type": FieldTypeEnum.STRING.value, } - if(field["type"] == 'dynamicSelectForm'): - if (forFieldWithTo != (field.get("reverse", False)==False)): + if field["type"] == "dynamicSelectForm": + if forFieldWithTo != (field.get("reverse", False) == False): obj["pattern"] = generalize_regex_pattern(field) else: if "defaultOption" in field: @@ -413,25 +500,35 @@ def generate_key(forFieldWithTo): return obj dynamicFormSchemaObject = {} - dynamicFormSchemaObject['type'] = FieldTypeEnum.ARRAY.value + dynamicFormSchemaObject["type"] = FieldTypeEnum.ARRAY.value dynamicFormItemObject = {} dynamicFormItemObject["type"] = FieldTypeEnum.OBJECT.value - dynamicFormItemObject['properties'] = {} + dynamicFormItemObject["properties"] = {} dynamicFormItemObjectProps = [ - (field['keyLeft'], generate_key), (field['keyRight'], generate_key)] + (field["keyLeft"], generate_key), + (field["keyRight"], generate_key), + ] for dynamicFromItemObjectProp in dynamicFormItemObjectProps: - dynamicFormItemObject['properties'][dynamicFromItemObjectProp[0] - ] = dynamicFromItemObjectProp[1](dynamicFromItemObjectProp[0] == "to") - dynamicFormSchemaObject['items'] = dynamicFormItemObject - - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + dynamicFormItemObject["properties"][dynamicFromItemObjectProp[0]] = ( + dynamicFromItemObjectProp[1](dynamicFromItemObjectProp[0] == "to") + ) + dynamicFormSchemaObject["items"] = dynamicFormItemObject + + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) # If the field is source dependent, new schema object is created by setting the fields inside the source. if isSourceDependent: newDynamicFormFormObj = {"type": FieldTypeEnum.OBJECT.value} newDynamicFormFormObj["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: - newDynamicFormFormObj["properties"][sourceType] = dynamicFormSchemaObject + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): + newDynamicFormFormObj["properties"][ + sourceType + ] = dynamicFormSchemaObject dynamicFormSchemaObject = newDynamicFormFormObj return dynamicFormSchemaObject @@ -442,7 +539,7 @@ def generate_schema_for_dynamic_select_form(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -450,13 +547,14 @@ def generate_schema_for_dynamic_select_form(field, dbConfig, schema_field_name): """ return generate_schema_for_dynamic_form(field, dbConfig, schema_field_name) + def generate_schema_for_mapping(field, dbConfig, schema_field_name): """Creates a schema object of mapping. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -471,7 +569,7 @@ def generate_schema_for_tag_input(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -480,23 +578,28 @@ def generate_schema_for_tag_input(field, dbConfig, schema_field_name): tagObject = {} tagObject["type"] = FieldTypeEnum.ARRAY.value tagItem = {} - tagItem['type'] = FieldTypeEnum.OBJECT.value + tagItem["type"] = FieldTypeEnum.OBJECT.value tagItemProps = { - str(field['tagKey']): { + str(field["tagKey"]): { "type": FieldTypeEnum.STRING.value, - "pattern": generalize_regex_pattern(field) + "pattern": generalize_regex_pattern(field), } } - tagItem['properties'] = tagItemProps + tagItem["properties"] = tagItemProps tagObject["items"] = tagItem - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) if isSourceDependent: tagObjectCopy = tagObject tagObject = {} tagObject = {"type": FieldTypeEnum.OBJECT.value} tagObject["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): tagObject["properties"][sourceType] = tagObjectCopy return tagObject @@ -507,20 +610,20 @@ def generate_schema_for_time_range_picker(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ timeRangeObj = {} - timeRangeObj['type'] = FieldTypeEnum.OBJECT.value + timeRangeObj["type"] = FieldTypeEnum.OBJECT.value timeRangeProps = { - field['startTime']['value']: {'type': FieldTypeEnum.STRING.value}, - field['endTime']['value']: {'type': FieldTypeEnum.STRING.value} + field["startTime"]["value"]: {"type": FieldTypeEnum.STRING.value}, + field["endTime"]["value"]: {"type": FieldTypeEnum.STRING.value}, } - timeRangeObj['properties'] = timeRangeProps - timeRangeObj['required'] = list(timeRangeProps.keys()) + timeRangeObj["properties"] = timeRangeProps + timeRangeObj["required"] = list(timeRangeProps.keys()) return timeRangeObj @@ -530,18 +633,16 @@ def generate_schema_for_time_picker(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ - return { - "type": FieldTypeEnum.STRING.value - } + return {"type": FieldTypeEnum.STRING.value} -def compare_pre_requisite_fields(fieldA, fieldB, isV2 = False): +def compare_pre_requisite_fields(fieldA, fieldB, isV2=False): """Compares two preRequisiteFields fieldA and fieldB for each property and checks if their value matches. Args: @@ -552,12 +653,12 @@ def compare_pre_requisite_fields(fieldA, fieldB, isV2 = False): Returns: boolean: If all the properties have the same 'name' and 'selectedValue', then it returns True else False. """ - valueKey = 'selectedValue' - nameKey = 'name' + valueKey = "selectedValue" + nameKey = "name" if isV2: - valueKey = 'value' - nameKey = 'configKey' + valueKey = "value" + nameKey = "configKey" if type(fieldA) != type(fieldB): return False @@ -565,13 +666,17 @@ def compare_pre_requisite_fields(fieldA, fieldB, isV2 = False): if len(fieldA) != len(fieldB): return False for i in range(0, len(fieldA)): - if fieldA[i][nameKey] != fieldB[i][nameKey] or fieldA[i][valueKey] != fieldB[i][valueKey]: + if ( + fieldA[i][nameKey] != fieldB[i][nameKey] + or fieldA[i][valueKey] != fieldB[i][valueKey] + ): return False else: if fieldA[nameKey] != fieldB[nameKey] or fieldA[valueKey] != fieldB[valueKey]: - return False + return False return True + def get_unique_pre_requisite_fields(uiConfig): """Returns the list of unique preRequisiteFields present in a uiConfig. @@ -580,16 +685,18 @@ def get_unique_pre_requisite_fields(uiConfig): Returns: list: containing unique preRequisiteFields. - """ + """ preRequisiteFieldsList = [] for group in uiConfig: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: if "preRequisiteField" not in field: continue isPresent = False for preRequisiteField in preRequisiteFieldsList: - if compare_pre_requisite_fields(preRequisiteField, field["preRequisiteField"]): + if compare_pre_requisite_fields( + preRequisiteField, field["preRequisiteField"] + ): isPresent = True break if not isPresent: @@ -597,7 +704,7 @@ def get_unique_pre_requisite_fields(uiConfig): return preRequisiteFieldsList -def generate_if_object(preRequisiteField, isV2 = False): +def generate_if_object(preRequisiteField, isV2=False): """Creates an if object for the given preRequisiteField. The preRequisiteField becomes an if condition in the schema. Args: @@ -608,18 +715,16 @@ def generate_if_object(preRequisiteField, isV2 = False): object: if block for given preRequisiteField. """ ifObj = {"properties": {}, "required": []} - valueKey = 'selectedValue' - nameKey = 'name' + valueKey = "selectedValue" + nameKey = "name" if isV2: - valueKey = 'value' - nameKey = 'configKey' + valueKey = "value" + nameKey = "configKey" if type(preRequisiteField) == list: for field in preRequisiteField: - ifObj["properties"][field[nameKey]] = { - "const": field[valueKey] - } + ifObj["properties"][field[nameKey]] = {"const": field[valueKey]} ifObj["required"].append(field[nameKey]) else: ifObj["properties"][preRequisiteField[nameKey]] = { @@ -638,12 +743,12 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name): Args: uiConfig (object): file content of ui-config.json. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object: allOf object of schema - """ + """ allOfItemList = [] preRequisiteFieldsList = get_unique_pre_requisite_fields(uiConfig) for preRequisiteField in preRequisiteFieldsList: @@ -651,12 +756,18 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name): thenObj = {"properties": {}, "required": []} allOfItemObj = {"if": ifObj} for group in uiConfig: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: if "preRequisiteField" not in field: continue - if compare_pre_requisite_fields(field["preRequisiteField"], preRequisiteField): - thenObj["properties"][field[schema_field_name]] = uiTypetoSchemaFn.get(field["type"])(field, dbConfig, schema_field_name) + if compare_pre_requisite_fields( + field["preRequisiteField"], preRequisiteField + ): + thenObj["properties"][field[schema_field_name]] = ( + uiTypetoSchemaFn.get(field["type"])( + field, dbConfig, schema_field_name + ) + ) if "required" in field and field["required"] == True: thenObj["required"].append(field[schema_field_name]) allOfItemObj["then"] = thenObj @@ -665,14 +776,15 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name): allOfItemList = generate_schema_for_anyOf(allOfItemList, schema_field_name) return allOfItemList + def get_common_and_opposite_fields(propertiesA, propertiesB, schema_field_name): """Takes properties of two if Objects and returns a list of common and opposite properties. Common properties have the same value in both A and B. Args: propertiesA (object): if block - propertiesB (object): - schema_field_name (string): Specifies which key has the field's name in schema. + propertiesB (object): + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -684,10 +796,10 @@ def get_common_and_opposite_fields(propertiesA, propertiesB, schema_field_name): propertiesA = { 'storage' : {'const' : 'S3'}, 'useStorage' : {'const': True}} propertiesB = { 'storage' : {'const' : 'S3'}, 'useStorage' : {'const': False}} - Returns + Returns commonProperties = [{'key': 'storage', 'value':'S3'}] oppositeProperties = [{'key': 'useStorage', 'value': 'True'}] - """ + """ keysListA = list(propertiesA.keys()) keysListB = list(propertiesB.keys()) commonProperties = [] @@ -696,32 +808,40 @@ def get_common_and_opposite_fields(propertiesA, propertiesB, schema_field_name): if key not in keysListB: return None, None if propertiesA[key]["const"] == propertiesB[key]["const"]: - commonProperties.append({"key": key, schema_field_name: propertiesA[key]["const"]}) - elif type(propertiesA[key]["const"]) == bool and propertiesA[key]["const"] != propertiesB[key]["const"]: - oppositeProperties.append({"key": key, schema_field_name: propertiesA[key]["const"]}) + commonProperties.append( + {"key": key, schema_field_name: propertiesA[key]["const"]} + ) + elif ( + type(propertiesA[key]["const"]) == bool + and propertiesA[key]["const"] != propertiesB[key]["const"] + ): + oppositeProperties.append( + {"key": key, schema_field_name: propertiesA[key]["const"]} + ) else: return None, None return commonProperties, oppositeProperties + def check_if_conditions_match(ifPropsA, ifObjectB, schema_field_name): """Compares the ifPropsA and ifObjectB if they have the same properties and values. Args: ifPropsA (list): consists of key and "schema_field_name" pairs. ifObjectB (object): if block consisting of multiple properties with the value. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: boolean: True if all the properties in IfPropsA are in ifObjectB with same value, else False. - + Example: - schema_field_name = 'value' + schema_field_name = 'value' ifPropsA = [{'key': 'storage', 'value': 'S3'}, {'key': 'deploy', 'value': 'web'}] ifObjectA = {'storage': {'const': 'S3'}, 'deploy': {'const': 'web'}} - + Returns: True - """ + """ for ifProp in ifPropsA: if ifProp["key"] not in ifObjectB: return False @@ -729,18 +849,19 @@ def check_if_conditions_match(ifPropsA, ifObjectB, schema_field_name): return False return True + def find_index_to_place_anyOf(ifProp, allOfItemList, schema_field_name): """Returns the index of the item in allOfItemList consisting of matching if conditions as that of ifProp. Args: ifProp (object): consists of key and "schema_field_name" pairs. allOfItemList (list): consists of a list of objects with "if-then" properties. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: int: index of if-block matching with ifProp else -1. - + Example: schema_field_name = 'value' ifProp = [{'key': 'storage', 'value': 'S3'}, {'key': 'deploy', 'value': 'web'}] @@ -750,85 +871,109 @@ def find_index_to_place_anyOf(ifProp, allOfItemList, schema_field_name): ] Returns: 1 - """ + """ if not ifProp: return -1 length = len(allOfItemList) for index in range(length): - if "if" in allOfItemList[index] and check_if_conditions_match(ifProp, allOfItemList[index]["if"]["properties"], schema_field_name): + if "if" in allOfItemList[index] and check_if_conditions_match( + ifProp, allOfItemList[index]["if"]["properties"], schema_field_name + ): return index return -1 + def generate_schema_for_anyOf(allOfItemList, schema_field_name): """Takes in two parameters allOfItemList and schema_field_name, and returns an updated allOf Items list. - - It checks for all the pairs of allOf Items ("if-then" blocks), if their "if" conditions have an "if-else" based structure rather than "if-if". + - It checks for all the pairs of allOf Items ("if-then" blocks), if their "if" conditions have an "if-else" based structure rather than "if-if". - Items following the "if-else" structure are deleted and replaced by an anyOf structure. Args: allOfItemList (list): consists of a list of objects with "if-then" properties. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: list: updated allOf Items list - """ + """ length = len(allOfItemList) delIndices = [] for i in range(0, length): - for j in range(i+1, length): + for j in range(i + 1, length): ifPropertiesA = allOfItemList[i]["if"]["properties"] thenPropertiesA = allOfItemList[i]["then"] ifPropertiesB = allOfItemList[j]["if"]["properties"] thenPropertiesB = allOfItemList[j]["then"] - commonIfProp, oppositeIfProp = get_common_and_opposite_fields(ifPropertiesA, ifPropertiesB, schema_field_name) + commonIfProp, oppositeIfProp = get_common_and_opposite_fields( + ifPropertiesA, ifPropertiesB, schema_field_name + ) if oppositeIfProp: anyOfObj = [{}, {}] for k in range(0, len(oppositeIfProp)): if ifPropertiesA[oppositeIfProp[k]["key"]]["const"] == True: anyOfObj[1] = thenPropertiesA - anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = {"const": True} + anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = { + "const": True + } anyOfObj[1]["required"].append(oppositeIfProp[k]["key"]) anyOfObj[0] = thenPropertiesB - anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = {"const": False} + anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = { + "const": False + } else: anyOfObj[1] = thenPropertiesB - anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = {"const": True} + anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = { + "const": True + } anyOfObj[1]["required"].append(oppositeIfProp[k]["key"]) anyOfObj[0] = thenPropertiesA - anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = {"const": False} - # AnyOf object is placed at index of "if-then" block having same if properties as of common properties else at end. - indexToPlace = find_index_to_place_anyOf(commonIfProp, allOfItemList, schema_field_name) + anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = { + "const": False + } + # AnyOf object is placed at index of "if-then" block having same if properties as of common properties else at end. + indexToPlace = find_index_to_place_anyOf( + commonIfProp, allOfItemList, schema_field_name + ) if indexToPlace != -1: allOfItemList[indexToPlace]["then"]["anyOf"] = anyOfObj delIndices.append(i) delIndices.append(j) - allOfItemList = [allOfItemList[index] for index in range(len(allOfItemList)) if index not in delIndices] + allOfItemList = [ + allOfItemList[index] + for index in range(len(allOfItemList)) + if index not in delIndices + ] return allOfItemList + def generate_connection_mode(dbConfig): """Creates the connection mode object present in new schema types. - + Args: dbConfig (object): Configurations of db-config.json. Returns: object - """ + """ connectionObj = {"type": FieldTypeEnum.OBJECT.value} connectionObj["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["supportedConnectionModes"]: - connectionItemObj = {"type": FieldTypeEnum.STRING.value} - connectionModesEnum=[] - length = len(dbConfig["supportedConnectionModes"][sourceType]) - for i in range(0, length): - connectionModesEnum.append(dbConfig["supportedConnectionModes"][sourceType][i]) - connectionItemObj["enum"] = connectionModesEnum - connectionObj["properties"][sourceType] = connectionItemObj + if sourceType in dbConfig["supportedConnectionModes"]: + connectionItemObj = {"type": FieldTypeEnum.STRING.value} + connectionModesEnum = [] + length = len(dbConfig["supportedConnectionModes"][sourceType]) + for i in range(0, length): + connectionModesEnum.append( + dbConfig["supportedConnectionModes"][sourceType][i] + ) + connectionItemObj["enum"] = connectionModesEnum + connectionObj["properties"][sourceType] = connectionItemObj return connectionObj -def generate_schema_properties(uiConfig, dbConfig, schemaObject, properties, name, selector): +def generate_schema_properties( + uiConfig, dbConfig, schemaObject, properties, name, selector +): """Generates corresponding schema properties by iterating over each of the ui-config fields. Args: @@ -838,75 +983,98 @@ def generate_schema_properties(uiConfig, dbConfig, schemaObject, properties, nam properties (object): properties of schema name (string): name of the source or destination. selector (string): either 'source' or 'destination' - """ + """ if is_old_format(uiConfig): for group in uiConfig: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: if "preRequisiteField" in field: continue - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['value']] = generateFunction( - field, dbConfig, 'value') - if field.get('required', False) == True and is_field_present_in_default_config(field, dbConfig, "value"): - schemaObject['required'].append(field['value']) + properties[field["value"]] = generateFunction( + field, dbConfig, "value" + ) + if field.get( + "required", False + ) == True and is_field_present_in_default_config( + field, dbConfig, "value" + ): + schemaObject["required"].append(field["value"]) else: - if selector == 'destination': - baseTemplate = uiConfig.get('baseTemplate', []) - sdkTemplate = uiConfig.get('sdkTemplate', {}) - consentSettingsTemplate = uiConfig.get('consentSettingsTemplate', {}) + if selector == "destination": + baseTemplate = uiConfig.get("baseTemplate", []) + sdkTemplate = uiConfig.get("sdkTemplate", {}) + consentSettingsTemplate = uiConfig.get("consentSettingsTemplate", {}) for template in baseTemplate: - for section in template.get('sections', []): - for group in section.get('groups', []): - for field in group.get('fields', []): - generateFunction = uiTypetoSchemaFn.get( - field['type'], None) + for section in template.get("sections", []): + for group in section.get("groups", []): + for field in group.get("fields", []): + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['configKey']] = generateFunction( - field, dbConfig, 'configKey') - if template.get('title', "") == "Initial setup" and is_field_present_in_default_config(field, dbConfig, "configKey") and 'preRequisites' not in field: - schemaObject['required'].append( - field['configKey']) - - for field in sdkTemplate.get('fields', []): - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + if ( + template.get("title", "") == "Initial setup" + and is_field_present_in_default_config( + field, dbConfig, "configKey" + ) + and "preRequisites" not in field + ): + schemaObject["required"].append(field["configKey"]) + + for field in sdkTemplate.get("fields", []): + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['configKey']] = generateFunction( - field, dbConfig, 'configKey') - if field.get('required', False) == True and is_field_present_in_default_config(field, dbConfig, "configKey"): - schemaObject['required'].append(field['configKey']) - - for field in consentSettingsTemplate.get('fields', []): - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + if field.get( + "required", False + ) == True and is_field_present_in_default_config( + field, dbConfig, "configKey" + ): + schemaObject["required"].append(field["configKey"]) + + for field in consentSettingsTemplate.get("fields", []): + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['configKey']] = generateFunction( - field, dbConfig, 'configKey') - if field.get('required', False) == True and is_field_present_in_default_config(field, dbConfig, "configKey"): - schemaObject['required'].append(field['configKey']) + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + if field.get( + "required", False + ) == True and is_field_present_in_default_config( + field, dbConfig, "configKey" + ): + schemaObject["required"].append(field["configKey"]) # default properties in new ui-config based schemas. - schemaObject['properties']['useNativeSDK'] = generate_schema_for_checkbox({"type":"checkbox", - "value":"useNativeSDK"}, dbConfig, "value") - schemaObject['properties']['connectionMode'] = generate_connection_mode(dbConfig) + schemaObject["properties"]["useNativeSDK"] = generate_schema_for_checkbox( + {"type": "checkbox", "value": "useNativeSDK"}, dbConfig, "value" + ) + schemaObject["properties"]["connectionMode"] = generate_connection_mode( + dbConfig + ) else: # for sources def generate_config_props(config): for group in config: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: - generateFunction = uiTypetoSchemaFn.get( - field["type"], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['value']] = generateFunction( - field, dbConfig, 'value') + properties[field["value"]] = generateFunction( + field, dbConfig, "value" + ) - auth = uiConfig.get('auth', None) - config = uiConfig.get('config', []) + auth = uiConfig.get("auth", None) + config = uiConfig.get("config", []) if auth: - type = auth.get('type') + type = auth.get("type") if type == "form": - auth_config = auth.get('config', []) + auth_config = auth.get("config", []) generate_config_props(auth_config) generate_config_props(config) @@ -923,31 +1091,33 @@ def generate_schema(uiConfig, dbConfig, name, selector): Returns: object: schema - """ + """ newSchema = {} schemaObject = {} - schemaObject['$schema'] = 'http://json-schema.org/draft-07/schema#' - schemaObject['required'] = [] - schemaObject['type'] = "object" - schemaObject['properties'] = {} + schemaObject["$schema"] = "http://json-schema.org/draft-07/schema#" + schemaObject["required"] = [] + schemaObject["type"] = "object" + schemaObject["properties"] = {} allOfSchemaObj = {} if is_old_format(uiConfig): allOfSchemaObj = generate_schema_for_allOf(uiConfig, dbConfig, "value") if allOfSchemaObj: # AnyOf occurring separately, not inside allOf. if len(allOfSchemaObj) == 1: - if isinstance(allOfSchemaObj[0], list): - schemaObject['anyOf'] = allOfSchemaObj[0] - else: - schemaObject['anyOf'] = allOfSchemaObj + if isinstance(allOfSchemaObj[0], list): + schemaObject["anyOf"] = allOfSchemaObj[0] + else: + schemaObject["anyOf"] = allOfSchemaObj else: - schemaObject['allOf'] = allOfSchemaObj - generate_schema_properties(uiConfig, dbConfig, schemaObject, - schemaObject['properties'], name, selector) - newSchema['configSchema'] = schemaObject + schemaObject["allOf"] = allOfSchemaObj + generate_schema_properties( + uiConfig, dbConfig, schemaObject, schemaObject["properties"], name, selector + ) + newSchema["configSchema"] = schemaObject return newSchema + def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): """Generates warning for each schema difference created by the current ui-type. @@ -956,7 +1126,7 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): dbConfig (object): Configurations of db-config.json. schema (object): Existing schema in schema.json curUiType (string): Ui-Type for which warnings are generated. - """ + """ if is_old_format(uiConfig): for uiConfigItem in uiConfig: for field in uiConfigItem["fields"]: @@ -965,77 +1135,138 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): if field["type"] == curUiType: if field["value"] not in schema["properties"]: warnings.warn( - f'{field["value"]} field is not in schema \n', UserWarning) + f'{field["value"]} field is not in schema \n', UserWarning + ) else: curSchemaField = schema["properties"][field["value"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "value") + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "value" + ) schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["value"], get_formatted_json(schemaDiff)), UserWarning) + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, + field["value"], + get_formatted_json(schemaDiff), + ), + UserWarning, + ) else: - baseTemplate = uiConfig.get('baseTemplate', []) - sdkTemplate = uiConfig.get('sdkTemplate', {}) - consentSettingsTemplate = uiConfig.get('consentSettingsTemplate', {}) + baseTemplate = uiConfig.get("baseTemplate", []) + sdkTemplate = uiConfig.get("sdkTemplate", {}) + consentSettingsTemplate = uiConfig.get("consentSettingsTemplate", {}) for template in baseTemplate: - for section in template.get('sections', []): - for group in section.get('groups', []): + for section in template.get("sections", []): + for group in section.get("groups", []): if "preRequisites" in group: continue - for field in group.get('fields', []): + for field in group.get("fields", []): if "preRequisites" in field: continue - generateFunction = uiTypetoSchemaFn.get( - field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction and field["type"] == curUiType: if field["configKey"] not in schema["properties"]: warnings.warn( - f'{field["configKey"]} field is not in schema \n', UserWarning) + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) else: - curSchemaField = schema["properties"][field["configKey"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "configKey") - schemaDiff = get_json_diff(curSchemaField, newSchemaField) + curSchemaField = schema["properties"][ + field["configKey"] + ] + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) + schemaDiff = get_json_diff( + curSchemaField, newSchemaField + ) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) - - for field in sdkTemplate.get('fields', []): + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, + field["configKey"], + get_formatted_json(schemaDiff), + ), + UserWarning, + ) + + for field in sdkTemplate.get("fields", []): if "preRequisites" in field: continue - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: if generateFunction and field["type"] == curUiType: if field["configKey"] not in schema["properties"]: warnings.warn( - f'{field["configKey"]} field is not in schema \n', UserWarning) + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) else: curSchemaField = schema["properties"][field["configKey"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "configKey") + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) - - for field in consentSettingsTemplate.get('fields', []): + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, + field["configKey"], + get_formatted_json(schemaDiff), + ), + UserWarning, + ) + + for field in sdkTemplate.get("fields", []): + if "preRequisites" in field: + continue + generateFunction = uiTypetoSchemaFn.get(field["type"], None) + if generateFunction: + if generateFunction and field["type"] == curUiType: + if field["configKey"] not in schema["properties"]: + warnings.warn( + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) + else: + curSchemaField = schema["properties"][field["configKey"]] + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) + schemaDiff = get_json_diff(newSchemaField, curSchemaField) + if schemaDiff: + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, field["configKey"], schemaDiff + ), + UserWarning, + ) + + for field in consentSettingsTemplate.get("fields", []): if "preRequisites" in field: continue - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: if generateFunction and field["type"] == curUiType: if field["configKey"] not in schema["properties"]: warnings.warn( - f'{field["configKey"]} field is not in schema \n', UserWarning) + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) else: curSchemaField = schema["properties"][field["configKey"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "configKey") - schemaDiff = diff(newSchemaField, curSchemaField) + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) + schemaDiff = get_json_diff(newSchemaField, curSchemaField) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], schemaDiff), UserWarning) + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, field["configKey"], schemaDiff + ), + UserWarning, + ) uiTypetoSchemaFn = { @@ -1045,29 +1276,33 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): "textareaInput": generate_schema_for_textarea_input, "singleSelect": generate_schema_for_single_select, "dynamicCustomForm": generate_schema_for_dynamic_custom_form, - 'dynamicForm': generate_schema_for_dynamic_form, - 'mapping': generate_schema_for_mapping, - 'dynamicSelectForm': generate_schema_for_dynamic_select_form, - 'tagInput': generate_schema_for_tag_input, - 'timeRangePicker': generate_schema_for_time_range_picker, - 'timePicker': generate_schema_for_time_picker + "dynamicForm": generate_schema_for_dynamic_form, + "mapping": generate_schema_for_mapping, + "dynamicSelectForm": generate_schema_for_dynamic_select_form, + "tagInput": generate_schema_for_tag_input, + "timeRangePicker": generate_schema_for_time_range_picker, + "timePicker": generate_schema_for_time_picker, } + def save_schema_to_file(selector, name, schema): # Get the parent directory (one level up) script_directory = os.path.dirname(os.path.abspath(__file__)) directory = os.path.dirname(script_directory) # Define the relative path - relative_path = f'{CONFIG_DIR}/{selector}s/{name}/schema.json' + relative_path = f"{CONFIG_DIR}/{selector}s/{name}/schema.json" file_path = os.path.join(directory, relative_path) # Write the new content - with open(file_path, 'w') as file: + with open(file_path, "w") as file: file.write(get_formatted_json(schema)) -def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema): - """Generates a schema and compares it with an existing one. + +def validate_config_consistency( + name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema +): + """Generates a schema and compares it with an existing one. If schemaDiff is present, it calls for individual warnings by iterating over each ui-type. Args: @@ -1077,13 +1312,13 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou dbConfig (object): Configurations of db-config.json. schema (object): Existing schema in schema.json. shouldUpdateSchema (boolean): if it should update the existing schema with generated one - """ + """ if schema == None and uiConfig == None: return if uiConfig == None: - print('-'*50) + print("-" * 50) warnings.warn(f"Ui-Config is null for {name} in {selector} \n", UserWarning) - print('-'*50) + print("-" * 50) return generatedSchema = generate_schema(uiConfig, dbConfig, name, selector) @@ -1095,23 +1330,30 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou save_schema_to_file(selector, name, finalSchema) if schemaDiff: - print('-'*50) - print(f'Schema diff for {name} in {selector}s') + print("-" * 50) + print(f"Schema diff for {name} in {selector}s") # call for individual warnings for uiType in uiTypetoSchemaFn.keys(): generate_warnings_for_each_type(uiConfig, dbConfig, schema, uiType) # schema diff for "additionalProperties" if "additionalProperties" not in schema: - print("\n Recommendation: Please set additionalProperties to False in schema.json. \n") + print( + "\n Recommendation: Please set additionalProperties to False in schema.json. \n" + ) # schema diff for "required" if "required" not in schema: - warnings.warn('required field is not in schema \n', UserWarning) + warnings.warn("required field is not in schema \n", UserWarning) else: curRequiredField = schema["required"] newRequiredField = generatedSchema["configSchema"]["required"] requiredFieldDiff = get_json_diff(curRequiredField, newRequiredField) if requiredFieldDiff: - warnings.warn("For required field Difference is : \n\n {} \n".format(get_formatted_json(requiredFieldDiff)), UserWarning) + warnings.warn( + "For required field Difference is : \n\n {} \n".format( + get_formatted_json(requiredFieldDiff) + ), + UserWarning, + ) if "allOf" in generatedSchema["configSchema"]: curAllOfSchema = {} if "allOf" in schema: @@ -1119,7 +1361,12 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou newAllOfSchema = generatedSchema["configSchema"]["allOf"] allOfSchemaDiff = get_json_diff(curAllOfSchema, newAllOfSchema) if allOfSchemaDiff: - warnings.warn("For allOf field Difference is : \n\n {} \n".format(get_formatted_json(allOfSchemaDiff)), UserWarning) + warnings.warn( + "For allOf field Difference is : \n\n {} \n".format( + get_formatted_json(allOfSchemaDiff) + ), + UserWarning, + ) if "anyOf" in generatedSchema["configSchema"]: curAnyOfSchema = {} if "anyOf" in schema: @@ -1127,30 +1374,36 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou newAnyOfSchema = generatedSchema["configSchema"]["anyOf"] anyOfSchemaDiff = get_json_diff(curAnyOfSchema, newAnyOfSchema) if anyOfSchemaDiff: - warnings.warn("For anyOf field Difference is : \n\n {} \n".format(get_formatted_json(anyOfSchemaDiff)), UserWarning) - print('-'*50) + warnings.warn( + "For anyOf field Difference is : \n\n {} \n".format( + get_formatted_json(anyOfSchemaDiff) + ), + UserWarning, + ) + print("-" * 50) else: if shouldUpdateSchema: save_schema_to_file(selector, name, generatedSchema) - print('-'*50) - print(f'Generated schema for {name} in {selector}s') + print("-" * 50) + print(f"Generated schema for {name} in {selector}s") print(get_formatted_json(generatedSchema)) - print('-'*50) + print("-" * 50) + def get_schema_diff(name, selector, shouldUpdateSchema=False): - """ Validates the schema for the given name and selector. + """Validates the schema for the given name and selector. Args: name (string): name of the source or destination. selector (string): either 'source' or 'destination'. shouldUpdateSchema (boolean): if it should update the existing schema with generated one - """ + """ - file_selectors = ['db-config.json', 'ui-config.json', 'schema.json'] - directory = f'./{CONFIG_DIR}/{selector}s/{name}' + file_selectors = ["db-config.json", "ui-config.json", "schema.json"] + directory = f"./{CONFIG_DIR}/{selector}s/{name}" if not os.path.isdir(directory): - print(f'No {selector}s directory found for {name}') + print(f"No {selector}s directory found for {name}") return if name not in EXCLUDED_DEST: @@ -1158,35 +1411,55 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False): file_content = {} for file_selector in file_selectors: if file_selector in available_files: - file_content.update(get_json_from_file(f'{directory}/{file_selector}')) + file_content.update(get_json_from_file(f"{directory}/{file_selector}")) uiConfig = file_content.get("uiConfig") schema = file_content.get("configSchema") dbConfig = file_content.get("config") - validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema) + validate_config_consistency( + name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema + ) + -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generates schema.json from ui-config.json and db-config.json and validates against actual scheme.json') +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generates schema.json from ui-config.json and db-config.json and validates against actual scheme.json" + ) group = parser.add_mutually_exclusive_group() - parser.add_argument('selector', metavar='selector', type=str, help='Enter whether -name is a source or destination') - parser.add_argument('-update', action='store_true', help='Will update existing schema with any changes') - group.add_argument('-name', metavar='name', type=str, help='Enter the folder name under selector') - group.add_argument('-all', action='store_true', help='Will run validation for all entities under selector') - + parser.add_argument( + "selector", + metavar="selector", + type=str, + help="Enter whether -name is a source or destination", + ) + parser.add_argument( + "-update", + action="store_true", + help="Will update existing schema with any changes", + ) + group.add_argument( + "-name", metavar="name", type=str, help="Enter the folder name under selector" + ) + group.add_argument( + "-all", + action="store_true", + help="Will run validation for all entities under selector", + ) + args = parser.parse_args() selector = args.selector shouldUpdateSchema = args.update if args.all: - dir_path = f'./{CONFIG_DIR}/{selector}s' + dir_path = f"./{CONFIG_DIR}/{selector}s" if not os.path.isdir(dir_path): - print(f'No {selector}s folder found') + print(f"No {selector}s folder found") exit(1) - + current_items = os.listdir(dir_path) for name in current_items: get_schema_diff(name, selector, shouldUpdateSchema) - + else: name = args.name get_schema_diff(name, selector, shouldUpdateSchema) diff --git a/scripts/setup-python.sh b/scripts/setup-python.sh index b71b35f43..e5518394d 100755 --- a/scripts/setup-python.sh +++ b/scripts/setup-python.sh @@ -5,3 +5,4 @@ pip3 --version pip3 install requests pip3 install jsonschema pip3 install jsondiff +pip3 install black diff --git a/scripts/utils.py b/scripts/utils.py index 49cc83027..b61e8cdc3 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,6 +1,7 @@ from jsondiff import JsonDiffer import json + def get_json_diff(oldJson, newJson): """Returns the difference between two JSONs. @@ -10,10 +11,11 @@ def get_json_diff(oldJson, newJson): Returns: object: difference between oldJson and newJson. - """ + """ differ = JsonDiffer(marshal=True) return differ.diff(oldJson, newJson) + def apply_json_diff(oldJson, diff): """Applies the difference on oldJson and returns the newJson. @@ -23,10 +25,11 @@ def apply_json_diff(oldJson, diff): Returns: object: new json. - """ + """ differ = JsonDiffer(marshal=True) return differ.patch(oldJson, diff) + def get_formatted_json(jsonObj): """Formats the json object. @@ -35,9 +38,10 @@ def get_formatted_json(jsonObj): Returns: string: formatted json. - """ + """ return json.dumps(jsonObj, indent=2, ensure_ascii=False) + def get_json_from_file(filePath): """Reads the content of the file and returns the json object. @@ -46,6 +50,6 @@ def get_json_from_file(filePath): Returns: object: json object. - """ - with open(filePath, 'r') as file: - return json.loads(file.read().encode('utf-8', 'ignore')) + """ + with open(filePath, "r") as file: + return json.loads(file.read().encode("utf-8", "ignore")) diff --git a/test/test_configGenerator.py b/test/test_configGenerator.py index 1964e7f77..19cf42839 100644 --- a/test/test_configGenerator.py +++ b/test/test_configGenerator.py @@ -10,20 +10,22 @@ from scripts.configGenerator import generateConfigs -with open('test/configData/inputData.json', 'r') as file: +with open("test/configData/inputData.json", "r") as file: input_data = json.load(file) -with open('test/configData/db-config.json', 'r') as file: +with open("test/configData/db-config.json", "r") as file: db_config = json.load(file) -with open('test/configData/ui-config.json', 'r') as file: +with open("test/configData/ui-config.json", "r") as file: ui_config = json.load(file) + class TestConfigGenerator(unittest.TestCase): def test_config_generator(self): result = generateConfigs(input_data) - self.assertEqual(result['db_config'], json.dumps(db_config)) - self.assertEqual(result['ui_config'], json.dumps(ui_config)) + self.assertEqual(result["db_config"], json.dumps(db_config)) + self.assertEqual(result["ui_config"], json.dumps(ui_config)) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From daf9fa8256eed280bad04c0c109de3787e6262e0 Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:51:07 +0530 Subject: [PATCH 05/17] feat: onboard new destination commandbar (#1196) --- .../destinations/commandbar/db-config.json | 39 ++++ .../destinations/commandbar/schema.json | 71 +++++++ .../destinations/commandbar/ui-config.json | 180 ++++++++++++++++++ .../validation/destinations/commandbar.json | 96 ++++++++++ 4 files changed, 386 insertions(+) create mode 100644 src/configurations/destinations/commandbar/db-config.json create mode 100644 src/configurations/destinations/commandbar/schema.json create mode 100644 src/configurations/destinations/commandbar/ui-config.json create mode 100644 test/data/validation/destinations/commandbar.json diff --git a/src/configurations/destinations/commandbar/db-config.json b/src/configurations/destinations/commandbar/db-config.json new file mode 100644 index 000000000..6decfd024 --- /dev/null +++ b/src/configurations/destinations/commandbar/db-config.json @@ -0,0 +1,39 @@ +{ + "name": "COMMANDBAR", + "displayName": "CommandBar", + "config": { + "transformAtV1": "processor", + "saveDestinationResponse": true, + "includeKeys": [ + "orgId", + "blacklistedEvents", + "whitelistedEvents", + "oneTrustCookieCategories", + "eventFilteringOption" + ], + "excludeKeys": [], + "supportedSourceTypes": ["web"], + "supportedMessageTypes": { + "device": { + "web": ["track", "identify"] + } + }, + "supportedConnectionModes": { + "web": ["device"] + }, + "destConfig": { + "defaultConfig": [ + "orgId", + "blacklistedEvents", + "whitelistedEvents", + "eventFilteringOption", + "oneTrustCookieCategories" + ], + "web": ["useNativeSDK", "connectionMode"] + }, + "secretKeys": ["orgId"] + }, + "options": { + "isBeta": true + } +} diff --git a/src/configurations/destinations/commandbar/schema.json b/src/configurations/destinations/commandbar/schema.json new file mode 100644 index 000000000..d1f050356 --- /dev/null +++ b/src/configurations/destinations/commandbar/schema.json @@ -0,0 +1,71 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["orgId"], + "type": "object", + "properties": { + "orgId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "eventFilteringOption": { + "type": "string", + "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], + "default": "disable" + }, + "whitelistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "blacklistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "web": { + "type": "boolean" + } + } + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { + "type": "string", + "enum": ["device"] + } + } + } + } + } +} diff --git a/src/configurations/destinations/commandbar/ui-config.json b/src/configurations/destinations/commandbar/ui-config.json new file mode 100644 index 000000000..c28a3243c --- /dev/null +++ b/src/configurations/destinations/commandbar/ui-config.json @@ -0,0 +1,180 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection Settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Organization id", + "note": "CommandBar App dashboard > Account (Lower left Corner) > Org ID", + "configKey": "orgId", + "regex": "^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid Organization ID", + "placeholder": "e.g. 2***e*2*" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [], + "defaultConnectionModes": { + "web": "device" + } + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [] + }, + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "Client-side event filtering", + "note": "Decide what events are allowed (allowlisting) and blocked (denylisting)", + "preRequisites": { + "fields": [ + { + "configKey": "connectionMode.web", + "value": "device" + } + ], + "condition": "or" + }, + "fields": [ + { + "type": "singleSelect", + "label": "Choose if you want to turn on events filtering:", + "configKey": "eventFilteringOption", + "note": "You must select either allowlist or denylist to enable events filtering", + "options": [ + { + "label": "No events filtering", + "value": "disable" + }, + { + "label": "Filter via allowlist", + "value": "whitelistedEvents" + }, + { + "label": "Filter via denylist", + "value": "blacklistedEvents" + } + ], + "default": "disable" + }, + { + "type": "tagInput", + "label": "Allowlisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to allowlist.", + "configKey": "whitelistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "whitelistedEvents" + } + ] + } + }, + { + "type": "tagInput", + "label": "Denylisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to denylist. ", + "configKey": "blacklistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "blacklistedEvents" + } + ] + } + } + ] + }, + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack’s OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] + } + } +} diff --git a/test/data/validation/destinations/commandbar.json b/test/data/validation/destinations/commandbar.json new file mode 100644 index 000000000..363ece4a3 --- /dev/null +++ b/test/data/validation/destinations/commandbar.json @@ -0,0 +1,96 @@ +[ + { + "config": { + "orgId": "testorgid", + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + }, + "connectionMode": { + "web": "device" + } + }, + "result": true + }, + { + "config": { + "orgId": "testorgid", + "eventFilteringOption": "whitelistedEvents", + "whitelistedEvents": [ + { + "eventName": "testevent" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": false + }, + "connectionMode": { + "web": "cloud" + } + }, + "result": false, + "err": ["connectionMode.web must be equal to one of the allowed values"] + }, + { + "config": { + "orgId": "testorgid", + "eventFilteringOption": "enable", + "whitelistedEvents": [ + { + "eventName": "testevent" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": false + }, + "connectionMode": { + "web": "device" + } + }, + "result": false, + "err": ["eventFilteringOption must be equal to one of the allowed values"] + }, + { + "config": { + "eventFilteringOption": "whitelistedEvents", + "whitelistedEvents": [ + { + "eventName": "testevent" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": false + }, + "connectionMode": { + "web": "device" + } + }, + "result": false, + "err": [" must have required property 'orgId'"] + } +] From 7faa64acfe2fca20b75789f31b541dd33e7fde8c Mon Sep 17 00:00:00 2001 From: Abhishek Pandey <64667840+1abhishekpandey@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:21:10 +0530 Subject: [PATCH 06/17] fix: comscore schema (#1217) --- src/configurations/destinations/comscore/schema.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/configurations/destinations/comscore/schema.json b/src/configurations/destinations/comscore/schema.json index 64a668270..ec6be4b99 100644 --- a/src/configurations/destinations/comscore/schema.json +++ b/src/configurations/destinations/comscore/schema.json @@ -25,7 +25,10 @@ "useNativeSDK": { "type": "object", "properties": { - "web": { + "android": { + "type": "boolean" + }, + "ios": { "type": "boolean" } } From 9124fb44fd18d903bbf37f7cef3eaf7aeccf92e4 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:37:39 +0530 Subject: [PATCH 07/17] feat: adding custom field support for freshsales (#1195) --- .../destinations/freshsales/db-config.json | 1 + .../destinations/freshsales/schema.json | 16 ++++++++++++++++ .../destinations/freshsales/ui-config.json | 13 +++++++++++++ .../validation/destinations/freshsales.json | 17 +++++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/src/configurations/destinations/freshsales/db-config.json b/src/configurations/destinations/freshsales/db-config.json index a23a997e9..3a377b063 100644 --- a/src/configurations/destinations/freshsales/db-config.json +++ b/src/configurations/destinations/freshsales/db-config.json @@ -38,6 +38,7 @@ "apiKey", "domain", "rudderEventsToFreshsalesEvents", + "customPropertyMapping", "oneTrustCookieCategories" ] } diff --git a/src/configurations/destinations/freshsales/schema.json b/src/configurations/destinations/freshsales/schema.json index a58ed3a80..d13d3de60 100644 --- a/src/configurations/destinations/freshsales/schema.json +++ b/src/configurations/destinations/freshsales/schema.json @@ -28,6 +28,22 @@ } } }, + "customPropertyMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, "oneTrustCookieCategories": { "type": "array", "items": { diff --git a/src/configurations/destinations/freshsales/ui-config.json b/src/configurations/destinations/freshsales/ui-config.json index 3aa329eec..df24401e7 100644 --- a/src/configurations/destinations/freshsales/ui-config.json +++ b/src/configurations/destinations/freshsales/ui-config.json @@ -47,6 +47,19 @@ "value": "lifecycle_stage" } ] + }, + { + "type": "dynamicForm", + "label": "Mapping to add custom fields while creating / updating a contact", + "value": "customPropertyMapping", + "required": false, + "labelRight": "Freshsales Contact Custom Field Name", + "labelLeft": "Rudder Payload Property", + "keyLeft": "from", + "keyRight": "to", + "placeholderLeft": "e.g: newTrait", + "placeholderRight": "e.g: cf_new_trait", + "footerNote": "Map Rudder Traits to Freshsales Contact Custom Fields. Here, traits will be fetched from traits object(message.traits or message.context.traits). Please provide the exact Internal name of the custom trait" } ] }, diff --git a/test/data/validation/destinations/freshsales.json b/test/data/validation/destinations/freshsales.json index 6bece2cec..df7962153 100644 --- a/test/data/validation/destinations/freshsales.json +++ b/test/data/validation/destinations/freshsales.json @@ -35,5 +35,22 @@ }, "result": false, "err": ["domain must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] + }, + { + "config": { + "apiKey": "adhs123-adhs123-adhs3", + "domain": "rudder-123457.myfreshworks.com", + "customPropertyMapping": [ + { + "from": "ofwinqeoqwefnoewqo9", + "to": "test" + }, + { + "from": "idwhcbiwdfbciwdfw", + "to": "entry" + } + ] + }, + "result": true } ] From 61d36fa5db882f00d97dd9dc88b2f26a7d49a87a Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:06:25 +0530 Subject: [PATCH 08/17] feat: tiktok_offline_events added support for all Standard events (#1216) --- .../tiktok_ads_offline_events/schema.json | 18 ++++++++- .../tiktok_ads_offline_events/ui-config.json | 40 +++++++++++++++++++ .../tiktok_ads_offline_events.json | 17 ++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/configurations/destinations/tiktok_ads_offline_events/schema.json b/src/configurations/destinations/tiktok_ads_offline_events/schema.json index 07ed15244..bcca72c7b 100644 --- a/src/configurations/destinations/tiktok_ads_offline_events/schema.json +++ b/src/configurations/destinations/tiktok_ads_offline_events/schema.json @@ -19,7 +19,23 @@ }, "to": { "type": "string", - "enum": ["CompletePayment", "Contact", "SubmitForm", "Subscribe", ""] + "enum": [ + "CompletePayment", + "Contact", + "SubmitForm", + "Subscribe", + "AddPaymentInfo", + "AddToCart", + "AddToWishlist", + "ClickButton", + "CompleteRegistration", + "Download", + "InitiateCheckout", + "PlaceAnOrder", + "Search", + "ViewContent", + "" + ] } } } diff --git a/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json b/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json index 57427e77f..5bfd80e47 100644 --- a/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json +++ b/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json @@ -44,6 +44,46 @@ { "name": "Subscribe", "value": "Subscribe" + }, + { + "name": "Add Payment Info", + "value": "AddPaymentInfo" + }, + { + "name": "Add to Cart", + "value": "AddToCart" + }, + { + "name": "Add to Wishlist", + "value": "AddToWishlist" + }, + { + "name": "Click Button", + "value": "ClickButton" + }, + { + "name": "Complete Registration", + "value": "CompleteRegistration" + }, + { + "name": "Download", + "value": "Download" + }, + { + "name": "Initiate Checkout", + "value": "InitiateCheckout" + }, + { + "name": "Place an Order", + "value": "PlaceAnOrder" + }, + { + "name": "Search", + "value": "Search" + }, + { + "name": "View Content", + "value": "ViewContent" } ] } diff --git a/test/data/validation/destinations/tiktok_ads_offline_events.json b/test/data/validation/destinations/tiktok_ads_offline_events.json index 0b26387f3..7f25d3448 100644 --- a/test/data/validation/destinations/tiktok_ads_offline_events.json +++ b/test/data/validation/destinations/tiktok_ads_offline_events.json @@ -35,12 +35,29 @@ { "from": "c", "to": "Subscribe" + }, + { + "from": "d", + "to": "Search" } ] }, "result": false, "err": [" must have required property 'accessToken'"] }, + { + "config": { + "accessToken": "fuwheirujkvjnkrtgkf", + "hashUserProperties": true, + "eventsToStandard": [ + { + "from": "a", + "to": "InitiateCheckout" + } + ] + }, + "result": true + }, { "config": { "accessToken": "fuwheirujkvjnkrtgkf", From 0d9b8d7ee45b6ba5c914aaca6c2c11474a7575e9 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Sat, 17 Feb 2024 11:19:53 +0530 Subject: [PATCH 09/17] feat: onboard trade desk real time conversions (#1213) --- .../db-config.json | 50 ++++ .../schema.json | 117 +++++++++ .../ui-config.json | 246 ++++++++++++++++++ .../the_trade_desk_real_time_conversions.json | 98 +++++++ 4 files changed, 511 insertions(+) create mode 100644 src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json create mode 100644 src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json create mode 100644 src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json create mode 100644 test/data/validation/destinations/the_trade_desk_real_time_conversions.json diff --git a/src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json b/src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json new file mode 100644 index 000000000..fa234988b --- /dev/null +++ b/src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json @@ -0,0 +1,50 @@ +{ + "name": "THE_TRADE_DESK_REAL_TIME_CONVERSIONS", + "displayName": "The Trade Desk Real Time Conversions", + "config": { + "cdkV2Enabled": true, + "transformAtV1": "processor", + "saveDestinationResponse": true, + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedMessageTypes": { + "cloud": ["track"] + }, + "supportedConnectionModes": { + "android": ["cloud"], + "ios": ["cloud"], + "web": ["cloud"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "advertiserId", + "customProperties", + "eventsMapping", + "trackerId", + "oneTrustCookieCategories" + ] + } + }, + "options": { + "isBeta": true + } +} diff --git a/src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json b/src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json new file mode 100644 index 000000000..c951319a2 --- /dev/null +++ b/src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json @@ -0,0 +1,117 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["advertiserId", "trackerId"], + "type": "object", + "properties": { + "advertiserId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "trackerId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "eventsMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "enum": [ + "searchitem", + "searchcategory", + "login", + "messagebusiness", + "direction", + "sitevisit" + ] + } + } + } + }, + "customProperties": { + "type": "array", + "items": { + "type": "object", + "properties": { + "rudderProperty": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "tradeDeskProperty": { + "type": "string", + "enum": ["td1", "td2", "td3", "td4", "td5", "td6", "td7", "td8", "td9", "td10"] + } + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json b/src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json new file mode 100644 index 000000000..60c42a304 --- /dev/null +++ b/src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json @@ -0,0 +1,246 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Advertiser ID", + "note": "Enter your Advertiser Id from advertiser preferences page in The Trade Desk platform UI.", + "configKey": "advertiserId", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Advertiser ID", + "placeholder": "e.g: jxXXXph" + }, + { + "type": "textInput", + "label": "Tracking Tag ID", + "note": "Enter the tracking tag ID. Contact your The Trade Desk representative to obtain it.", + "configKey": "trackerId", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Tracking Tag ID", + "placeholder": "e.g: hcXXXke" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack's OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing 'Enter' after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + }, + { + "title": "Event mapping", + "note": "Map RudderStack to Trade Desk events", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "RudderStack to Trade Desk event mappings", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "eventAndPropertiesMapping", + "label": "Event and property mappings", + "note": "Map RudderStack events/properties to Trade Desk events/properties" + } + ] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "SDK settings", + "note": "not visible in the ui", + "fields": [] + }, + "redirectGroups": { + "eventAndPropertiesMapping": { + "tabs": [ + { + "name": "Event", + "fields": [ + { + "type": "mapping", + "label": "Map your RudderStack Events to Trade Desk Events", + "note": "Enter the input event name which you want to map with a Trade Desk Event Name", + "configKey": "eventsMapping", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "from", + "label": "RudderStack Event", + "placeholder": "e.g: Product Searched" + }, + { + "type": "singleSelect", + "key": "to", + "label": "Trade Desk Event", + "options": [ + { + "name": "searchitem", + "value": "searchitem" + }, + { + "name": "searchcategory", + "value": "searchcategory" + }, + { + "name": "login", + "value": "login" + }, + { + "name": "messagebusiness", + "value": "messagebusiness" + }, + { + "name": "direction", + "value": "direction" + }, + { + "name": "sitevisit", + "value": "sitevisit" + } + ] + } + ] + } + ] + }, + { + "name": "Custom properties", + "fields": [ + { + "type": "mapping", + "label": "Map custom properties", + "configKey": "customProperties", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "rudderProperty", + "label": "RudderStack property path", + "placeholder": "e.g properties.key1" + }, + { + "type": "singleSelect", + "key": "tradeDeskProperty", + "label": "Trade Desk custom property", + "options": [ + { + "label": "td1", + "value": "td1" + }, + { + "label": "td2", + "value": "td2" + }, + { + "label": "td3", + "value": "td3" + }, + { + "label": "td4", + "value": "td4" + }, + { + "label": "td5", + "value": "td5" + }, + { + "label": "td6", + "value": "td6" + }, + { + "label": "td7", + "value": "td7" + }, + { + "label": "td8", + "value": "td8" + }, + { + "label": "td9", + "value": "td9" + }, + { + "label": "td10", + "value": "td10" + } + ] + } + ] + } + ] + } + ] + } + } + } +} diff --git a/test/data/validation/destinations/the_trade_desk_real_time_conversions.json b/test/data/validation/destinations/the_trade_desk_real_time_conversions.json new file mode 100644 index 000000000..dd43b1121 --- /dev/null +++ b/test/data/validation/destinations/the_trade_desk_real_time_conversions.json @@ -0,0 +1,98 @@ +[ + { + "config": { + "advertiserId": "test-advertiserId", + "trackerId": "test-trackerId", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ], + "eventsMapping": [ + { + "from": "Products Searched", + "to": "searchitem" + } + ], + "customProperties": [ + { + "rudderProperty": "properties.key1", + "tradeDeskProperty": "td1" + } + ] + }, + "result": true + }, + { + "config": { + "advertiserId": "", + "trackerId": "test-trackerId", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "advertiserId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "advertiserId": "test-advertiserId", + "trackerId": "", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "trackerId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + " must have required property 'advertiserId'", + " must have required property 'trackerId'" + ] + }, + { + "config": { + "advertiserId": "test-advertiserId", + "trackerId": "test-trackerId", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ], + "eventsMapping": [ + { + "from": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "to": "login" + } + ], + "customProperties": [ + { + "rudderProperty": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "tradeDeskProperty": "td1" + } + ] + }, + "result": false, + "err": [ + "eventsMapping.0.from must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", + "customProperties.0.rudderProperty must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"" + ] + } +] From 5120ec5e26240f92e39b3663c090a75362157d16 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Sat, 17 Feb 2024 11:20:27 +0530 Subject: [PATCH 10/17] Revert "feat: trade desk real time conversions" (#1212) --- .../the_trade_desk/db-config.json | 33 +--- .../destinations/the_trade_desk/schema.json | 46 +----- .../the_trade_desk/ui-config.json | 154 ------------------ .../destinations/the_trade_desk.json | 21 +-- 4 files changed, 6 insertions(+), 248 deletions(-) diff --git a/src/configurations/destinations/the_trade_desk/db-config.json b/src/configurations/destinations/the_trade_desk/db-config.json index 561963d52..c0a5553c8 100644 --- a/src/configurations/destinations/the_trade_desk/db-config.json +++ b/src/configurations/destinations/the_trade_desk/db-config.json @@ -1,6 +1,6 @@ { "name": "THE_TRADE_DESK", - "displayName": "The Trade Desk", + "displayName": "The Trade Desk Audience", "config": { "cdkV2Enabled": true, "supportsBlankAudienceCreation": true, @@ -9,34 +9,12 @@ "disableJsonMapper": true, "isAudienceSupported": true, "supportsVisualMapper": true, - "supportedSourceTypes": [ - "android", - "ios", - "web", - "unity", - "amp", - "cloud", - "warehouse", - "reactnative", - "flutter", - "cordova", - "shopify" - ], + "supportedSourceTypes": ["warehouse"], "supportedMessageTypes": { - "cloud": ["record", "track"] + "cloud": ["record"] }, "supportedConnectionModes": { - "android": ["cloud"], - "ios": ["cloud"], - "web": ["cloud"], - "unity": ["cloud"], - "amp": ["cloud"], - "cloud": ["cloud"], - "warehouse": ["cloud"], - "reactnative": ["cloud"], - "flutter": ["cloud"], - "cordova": ["cloud"], - "shopify": ["cloud"] + "warehouse": ["cloud"] }, "syncBehaviours": ["mirror"], "destConfig": { @@ -44,10 +22,7 @@ "audienceId", "advertiserId", "advertiserSecretKey", - "customProperties", "dataServer", - "eventsMapping", - "trackerId", "ttlInDays", "oneTrustCookieCategories" ] diff --git a/src/configurations/destinations/the_trade_desk/schema.json b/src/configurations/destinations/the_trade_desk/schema.json index 6258c81f6..5d7bf4421 100644 --- a/src/configurations/destinations/the_trade_desk/schema.json +++ b/src/configurations/destinations/the_trade_desk/schema.json @@ -1,7 +1,7 @@ { "configSchema": { "$schema": "http://json-schema.org/draft-07/schema#", - "required": ["advertiserId", "advertiserSecretKey", "trackerId", "dataServer"], + "required": ["advertiserId", "advertiserSecretKey", "dataServer"], "type": "object", "properties": { "audienceId": { @@ -20,10 +20,6 @@ "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]|[1-9][0-9]|1[0-7][0-9]|180)$" }, - "trackerId": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" - }, "dataServer": { "type": "string", "enum": ["apac", "tokyo", "usEastCoast", "usWestCoast", "ukEu", "china"], @@ -44,49 +40,9 @@ "connectionMode": { "type": "object", "properties": { - "android": { - "type": "string", - "enum": ["cloud"] - }, - "ios": { - "type": "string", - "enum": ["cloud"] - }, - "web": { - "type": "string", - "enum": ["cloud"] - }, - "unity": { - "type": "string", - "enum": ["cloud"] - }, - "amp": { - "type": "string", - "enum": ["cloud"] - }, - "cloud": { - "type": "string", - "enum": ["cloud"] - }, "warehouse": { "type": "string", "enum": ["cloud"] - }, - "reactnative": { - "type": "string", - "enum": ["cloud"] - }, - "flutter": { - "type": "string", - "enum": ["cloud"] - }, - "cordova": { - "type": "string", - "enum": ["cloud"] - }, - "shopify": { - "type": "string", - "enum": ["cloud"] } } } diff --git a/src/configurations/destinations/the_trade_desk/ui-config.json b/src/configurations/destinations/the_trade_desk/ui-config.json index a45abd74b..64cb50bb1 100644 --- a/src/configurations/destinations/the_trade_desk/ui-config.json +++ b/src/configurations/destinations/the_trade_desk/ui-config.json @@ -31,15 +31,6 @@ "placeholder": "e.g: u8a7f3k9p2nXXXXX4q5r2m8d7z1o9", "secret": true }, - { - "type": "textInput", - "label": "Tracking Tag ID", - "note": "Tracking tag ID from advertiser preferences page in the TTD platform UI. If you can't find your tracking tag ID, contact your Technical Account Manager.", - "configKey": "trackerId", - "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", - "regexErrorMessage": "Invalid Tracking Tag ID", - "placeholder": "e.g: hcXXXke" - }, { "type": "singleSelect", "label": "Data Server", @@ -156,157 +147,12 @@ ] } ] - }, - { - "title": "Event mapping", - "note": "Map RudderStack to Trade Desk events", - "hideEditIcon": true, - "sections": [ - { - "groups": [ - { - "title": "RudderStack to Trade Desk event mappings", - "fields": [ - { - "type": "redirect", - "redirectGroupKey": "eventAndPropertiesMapping", - "label": "Event and property mappings", - "note": "Map RudderStack events/properties to Trade Desk events/properties" - } - ] - } - ] - } - ] } ], "sdkTemplate": { "title": "SDK settings", "note": "not visible in the ui", "fields": [] - }, - "redirectGroups": { - "eventAndPropertiesMapping": { - "tabs": [ - { - "name": "Event", - "fields": [ - { - "type": "mapping", - "label": "Map your RudderStack Events to Trade Desk Events", - "note": "Enter the input event name which you want to map with a Trade Desk Event Name", - "configKey": "eventsMapping", - "default": [], - "columns": [ - { - "type": "textInput", - "key": "from", - "label": "RudderStack Event", - "placeholder": "e.g: Product Searched" - }, - { - "type": "singleSelect", - "key": "to", - "label": "Trade Desk Event", - "options": [ - { - "name": "searchitem", - "value": "searchitem" - }, - { - "name": "searchcategory", - "value": "searchcategory" - }, - { - "name": "login", - "value": "login" - }, - { - "name": "messagebusiness", - "value": "messagebusiness" - }, - { - "name": "direction", - "value": "direction" - }, - { - "name": "sitevisit", - "value": "sitevisit" - } - ] - } - ] - } - ] - }, - { - "name": "Custom properties", - "fields": [ - { - "type": "mapping", - "label": "Map custom properties", - "configKey": "customProperties", - "default": [], - "columns": [ - { - "type": "textInput", - "key": "rudderProperty", - "label": "RudderStack property path", - "placeholder": "e.g properties.key1" - }, - { - "type": "singleSelect", - "key": "tradeDeskProperty", - "label": "Trade Desk custom property", - "options": [ - { - "label": "td1", - "value": "td1" - }, - { - "label": "td2", - "value": "td2" - }, - { - "label": "td3", - "value": "td3" - }, - { - "label": "td4", - "value": "td4" - }, - { - "label": "td5", - "value": "td5" - }, - { - "label": "td6", - "value": "td6" - }, - { - "label": "td7", - "value": "td7" - }, - { - "label": "td8", - "value": "td8" - }, - { - "label": "td9", - "value": "td9" - }, - { - "label": "td10", - "value": "td10" - } - ] - } - ] - } - ] - } - ] - } } } } diff --git a/test/data/validation/destinations/the_trade_desk.json b/test/data/validation/destinations/the_trade_desk.json index 78d75059e..349b2c492 100644 --- a/test/data/validation/destinations/the_trade_desk.json +++ b/test/data/validation/destinations/the_trade_desk.json @@ -4,25 +4,12 @@ "audienceId": "test-segment", "advertiserId": "test-advertiserId", "advertiserSecretKey": "test-advertiserSecretKey", - "trackerId": "test-trackerId", "segmentName": "test-segment", "dataServer": "usEastCoast", "oneTrustCookieCategories": [ { "oneTrustCookieCategory": "Marketing" } - ], - "eventsMapping": [ - { - "from": "Product Added", - "to": "addToCart" - } - ], - "customProperties": [ - { - "rudderProperty": "properties.key1", - "tradeDeskProperty": "td1" - } ] }, "result": true @@ -31,7 +18,6 @@ "config": { "audienceId": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz", "advertiserId": "test-advertiserId", - "trackerId": "test-trackerId", "advertiserSecretKey": "test-advertiserSecretKey", "segmentName": "test-segment", "dataServer": "usEastCoast", @@ -50,7 +36,6 @@ "config": { "audienceId": "test-segment", "advertiserId": "", - "trackerId": "test-trackerId", "advertiserSecretKey": "test-advertiserSecretKey", "segmentName": "test-segment", "dataServer": "usEastCoast", @@ -69,7 +54,6 @@ "config": { "audienceId": "test-segment", "advertiserId": "test-advertiserId", - "trackerId": "test-trackerId", "advertiserSecretKey": "test-advertiserSecretKey", "segmentName": "test-segment", "dataServer": "test-server", @@ -95,9 +79,6 @@ ] }, "result": false, - "err": [ - " must have required property 'advertiserSecretKey'", - " must have required property 'trackerId'" - ] + "err": [" must have required property 'advertiserSecretKey'"] } ] From dde686cc72e0ac7423487d566e8f9715cc43d24f Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Mon, 19 Feb 2024 10:52:02 +0530 Subject: [PATCH 11/17] feat: add connectionMode for all sourceTypes in dest def for selected destinations (#1197) --- .../destinations/autopilot/db-config.json | 17 ++++++++++++-- .../destinations/custify/db-config.json | 17 ++++++++++++-- .../destinations/custify/schema.json | 16 +++++++++++++ .../destinations/dynamic_yield/db-config.json | 17 ++++++++++++-- .../destinations/dynamic_yield/schema.json | 18 +++++++++++---- .../fb_custom_audience/db-config.json | 9 +++++--- .../destinations/firebase/db-config.json | 10 ++++---- .../db-config.json | 17 ++++++++++++-- .../db-config.json | 8 +++++-- .../destinations/hotjar/db-config.json | 2 +- .../destinations/hotjar/schema.json | 6 +++++ .../destinations/keen/db-config.json | 16 +++++++++++-- .../destinations/leanplum/db-config.json | 14 +++++++++-- .../destinations/leanplum/schema.json | 23 +++++++++---------- .../destinations/one_signal/db-config.json | 17 ++++++++++++-- .../destinations/one_signal/schema.json | 16 +++++++++++++ .../destinations/pinterest_tag/db-config.json | 16 +++++++++++-- .../destinations/pinterest_tag/schema.json | 16 +++++++++++++ .../destinations/pipedrive/db-config.json | 17 ++++++++++++-- .../destinations/revenue_cat/db-config.json | 17 ++++++++++++-- .../destinations/revenue_cat/schema.json | 16 +++++++++++++ .../destinations/satismeter/db-config.json | 2 +- .../destinations/satismeter/schema.json | 6 +++++ .../destinations/serenytics/db-config.json | 17 ++++++++++++-- .../destinations/serenytics/schema.json | 16 +++++++++++++ .../destinations/stormly/db-config.json | 17 ++++++++++++-- .../destinations/stormly/schema.json | 16 +++++++++++++ .../destinations/tiktok_ads/db-config.json | 8 +++++-- .../destinations/tiktok_ads/schema.json | 9 ++++++++ .../destinations/tvsquared/db-config.json | 2 +- .../destinations/vero/db-config.json | 16 +++++++++++-- .../destinations/vero/schema.json | 16 +++++++++++++ 32 files changed, 373 insertions(+), 57 deletions(-) diff --git a/src/configurations/destinations/autopilot/db-config.json b/src/configurations/destinations/autopilot/db-config.json index 3d062faa9..a90df8093 100644 --- a/src/configurations/destinations/autopilot/db-config.json +++ b/src/configurations/destinations/autopilot/db-config.json @@ -32,10 +32,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "triggerId", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "triggerId", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/custify/db-config.json b/src/configurations/destinations/custify/db-config.json index f82d60825..85eee2705 100644 --- a/src/configurations/destinations/custify/db-config.json +++ b/src/configurations/destinations/custify/db-config.json @@ -31,10 +31,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "sendAnonymousId", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "sendAnonymousId", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/custify/schema.json b/src/configurations/destinations/custify/schema.json index b1f6fc1d8..3c275c973 100644 --- a/src/configurations/destinations/custify/schema.json +++ b/src/configurations/destinations/custify/schema.json @@ -23,6 +23,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/dynamic_yield/db-config.json b/src/configurations/destinations/dynamic_yield/db-config.json index e759187e0..17ff16291 100644 --- a/src/configurations/destinations/dynamic_yield/db-config.json +++ b/src/configurations/destinations/dynamic_yield/db-config.json @@ -29,13 +29,26 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "supportedMessageTypes": { "cloud": ["identify", "track"] }, "destConfig": { - "defaultConfig": ["apiKey", "hashEmail", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "hashEmail", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] }, diff --git a/src/configurations/destinations/dynamic_yield/schema.json b/src/configurations/destinations/dynamic_yield/schema.json index 3016808d8..3ad1e1d96 100644 --- a/src/configurations/destinations/dynamic_yield/schema.json +++ b/src/configurations/destinations/dynamic_yield/schema.json @@ -24,12 +24,22 @@ } } }, - "useNativeSDK": { - "type": "boolean" - }, + "useNativeSDK": { "type": "boolean" }, "connectionMode": { "type": "object", - "properties": {} + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/fb_custom_audience/db-config.json b/src/configurations/destinations/fb_custom_audience/db-config.json index ca96134eb..a3022d2ff 100644 --- a/src/configurations/destinations/fb_custom_audience/db-config.json +++ b/src/configurations/destinations/fb_custom_audience/db-config.json @@ -16,7 +16,9 @@ }, "isAudienceSupported": true, "supportedConnectionModes": { - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -30,8 +32,9 @@ "subType", "oneTrustCookieCategories" ], - "cloud": ["audienceId"], - "warehouse": ["adAccountId"] + "cloud": ["audienceId", "connectionMode"], + "warehouse": ["adAccountId", "connectionMode"], + "shopify": ["connectionMode"] }, "secretKeys": ["accessToken"] }, diff --git a/src/configurations/destinations/firebase/db-config.json b/src/configurations/destinations/firebase/db-config.json index bd2f58612..2c3a0dccf 100644 --- a/src/configurations/destinations/firebase/db-config.json +++ b/src/configurations/destinations/firebase/db-config.json @@ -35,11 +35,11 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "android": ["useNativeSDK"], - "ios": ["useNativeSDK"], - "unity": ["useNativeSDK"], - "reactnative": ["useNativeSDK"], - "flutter": ["useNativeSDK"] + "android": ["useNativeSDK", "connectionMode"], + "ios": ["useNativeSDK", "connectionMode"], + "unity": ["useNativeSDK", "connectionMode"], + "reactnative": ["useNativeSDK", "connectionMode"], + "flutter": ["useNativeSDK", "connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json b/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json index 30e4ffd28..3f772d895 100644 --- a/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json +++ b/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json @@ -37,7 +37,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -48,7 +50,18 @@ "loginCustomerId", "requireHash", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] }, diff --git a/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json b/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json index d542e29e1..eef9de173 100644 --- a/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json +++ b/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json @@ -20,7 +20,9 @@ "cloud": ["audiencelist"] }, "supportedConnectionModes": { - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "isAudienceSupported": true, "supportsBlankAudienceCreation": true, @@ -36,7 +38,9 @@ "typeOfList", "oneTrustCookieCategories" ], - "cloud": ["audienceId"] + "cloud": ["audienceId", "connectionMode"], + "shopify": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] }, diff --git a/src/configurations/destinations/hotjar/db-config.json b/src/configurations/destinations/hotjar/db-config.json index ecea5199a..ca7cb05e8 100644 --- a/src/configurations/destinations/hotjar/db-config.json +++ b/src/configurations/destinations/hotjar/db-config.json @@ -29,7 +29,7 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/hotjar/schema.json b/src/configurations/destinations/hotjar/schema.json index b08b921e4..8e6042e66 100644 --- a/src/configurations/destinations/hotjar/schema.json +++ b/src/configurations/destinations/hotjar/schema.json @@ -56,6 +56,12 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["device"] } + } } } } diff --git a/src/configurations/destinations/keen/db-config.json b/src/configurations/destinations/keen/db-config.json index 22d19b035..5e6c8db77 100644 --- a/src/configurations/destinations/keen/db-config.json +++ b/src/configurations/destinations/keen/db-config.json @@ -45,7 +45,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -60,7 +62,17 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/leanplum/db-config.json b/src/configurations/destinations/leanplum/db-config.json index 67212a53a..9deb04cb3 100644 --- a/src/configurations/destinations/leanplum/db-config.json +++ b/src/configurations/destinations/leanplum/db-config.json @@ -38,7 +38,9 @@ "amp": ["cloud"], "reactnative": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "hybridModeCloudEventsFilter": { "android": { @@ -71,7 +73,15 @@ ], "android": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"], "ios": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"], - "flutter": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"] + "flutter": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["clientKey"] } diff --git a/src/configurations/destinations/leanplum/schema.json b/src/configurations/destinations/leanplum/schema.json index 8aeedf145..84a970acd 100644 --- a/src/configurations/destinations/leanplum/schema.json +++ b/src/configurations/destinations/leanplum/schema.json @@ -74,18 +74,17 @@ "connectionMode": { "type": "object", "properties": { - "android": { - "type": "string", - "enum": ["cloud", "device", "hybrid"] - }, - "ios": { - "type": "string", - "enum": ["cloud", "device", "hybrid"] - }, - "flutter": { - "type": "string", - "enum": ["cloud", "device", "hybrid"] - } + "android": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, + "ios": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, + "flutter": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } } } } diff --git a/src/configurations/destinations/one_signal/db-config.json b/src/configurations/destinations/one_signal/db-config.json index ea6d015ca..69565f1b2 100644 --- a/src/configurations/destinations/one_signal/db-config.json +++ b/src/configurations/destinations/one_signal/db-config.json @@ -31,7 +31,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -41,7 +43,18 @@ "eventAsTags", "allowedProperties", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/one_signal/schema.json b/src/configurations/destinations/one_signal/schema.json index 2d498cfdd..26fbc30b2 100644 --- a/src/configurations/destinations/one_signal/schema.json +++ b/src/configurations/destinations/one_signal/schema.json @@ -43,6 +43,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/pinterest_tag/db-config.json b/src/configurations/destinations/pinterest_tag/db-config.json index 54cdb9925..c08f2c52c 100644 --- a/src/configurations/destinations/pinterest_tag/db-config.json +++ b/src/configurations/destinations/pinterest_tag/db-config.json @@ -40,7 +40,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "supportedMessageTypes": { "cloud": ["page", "screen", "track"], @@ -69,7 +71,17 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["adAccountId", "conversionToken"] } diff --git a/src/configurations/destinations/pinterest_tag/schema.json b/src/configurations/destinations/pinterest_tag/schema.json index fb53bd829..8cf54c9be 100644 --- a/src/configurations/destinations/pinterest_tag/schema.json +++ b/src/configurations/destinations/pinterest_tag/schema.json @@ -128,6 +128,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["cloud", "device"] }, + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } }, "allOf": [ diff --git a/src/configurations/destinations/pipedrive/db-config.json b/src/configurations/destinations/pipedrive/db-config.json index a302a4d02..6328be1df 100644 --- a/src/configurations/destinations/pipedrive/db-config.json +++ b/src/configurations/destinations/pipedrive/db-config.json @@ -27,7 +27,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -39,7 +41,18 @@ "leadsMap", "organizationMap", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["apiToken"] }, diff --git a/src/configurations/destinations/revenue_cat/db-config.json b/src/configurations/destinations/revenue_cat/db-config.json index b31c28563..087037cac 100644 --- a/src/configurations/destinations/revenue_cat/db-config.json +++ b/src/configurations/destinations/revenue_cat/db-config.json @@ -31,10 +31,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "xPlatform", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "xPlatform", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/revenue_cat/schema.json b/src/configurations/destinations/revenue_cat/schema.json index 1aff6af15..098ed9a05 100644 --- a/src/configurations/destinations/revenue_cat/schema.json +++ b/src/configurations/destinations/revenue_cat/schema.json @@ -24,6 +24,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/satismeter/db-config.json b/src/configurations/destinations/satismeter/db-config.json index 5a29f4fee..b231346be 100644 --- a/src/configurations/destinations/satismeter/db-config.json +++ b/src/configurations/destinations/satismeter/db-config.json @@ -39,7 +39,7 @@ "blacklistedEvents", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"] }, "secretKeys": ["writeKey"] } diff --git a/src/configurations/destinations/satismeter/schema.json b/src/configurations/destinations/satismeter/schema.json index 0307101d2..0aa2b3802 100644 --- a/src/configurations/destinations/satismeter/schema.json +++ b/src/configurations/destinations/satismeter/schema.json @@ -64,6 +64,12 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["device"] } + } } }, "allOf": [ diff --git a/src/configurations/destinations/serenytics/db-config.json b/src/configurations/destinations/serenytics/db-config.json index fd7afe798..da2b44662 100644 --- a/src/configurations/destinations/serenytics/db-config.json +++ b/src/configurations/destinations/serenytics/db-config.json @@ -31,7 +31,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -43,7 +45,18 @@ "storageUrlPage", "storageUrlScreen", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/serenytics/schema.json b/src/configurations/destinations/serenytics/schema.json index a55f77506..7574e324e 100644 --- a/src/configurations/destinations/serenytics/schema.json +++ b/src/configurations/destinations/serenytics/schema.json @@ -55,6 +55,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/stormly/db-config.json b/src/configurations/destinations/stormly/db-config.json index 01081a8fa..62c4e45c8 100644 --- a/src/configurations/destinations/stormly/db-config.json +++ b/src/configurations/destinations/stormly/db-config.json @@ -31,10 +31,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["apiKey"] }, diff --git a/src/configurations/destinations/stormly/schema.json b/src/configurations/destinations/stormly/schema.json index 013f7d0d4..37a30b251 100644 --- a/src/configurations/destinations/stormly/schema.json +++ b/src/configurations/destinations/stormly/schema.json @@ -19,6 +19,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/tiktok_ads/db-config.json b/src/configurations/destinations/tiktok_ads/db-config.json index ed7c780fe..cabf70b5a 100644 --- a/src/configurations/destinations/tiktok_ads/db-config.json +++ b/src/configurations/destinations/tiktok_ads/db-config.json @@ -38,7 +38,8 @@ "supportedConnectionModes": { "web": ["cloud", "device"], "android": ["cloud"], - "ios": ["cloud"] + "ios": ["cloud"], + "cloud": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -54,7 +55,10 @@ "oneTrustCookieCategories", "ketchConsentPurposes" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "cloud": ["connectionMode"] }, "secretKeys": ["accessToken", "pixelCode"] } diff --git a/src/configurations/destinations/tiktok_ads/schema.json b/src/configurations/destinations/tiktok_ads/schema.json index 85566c0aa..212a7f252 100644 --- a/src/configurations/destinations/tiktok_ads/schema.json +++ b/src/configurations/destinations/tiktok_ads/schema.json @@ -117,6 +117,15 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["cloud", "device"] }, + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/tvsquared/db-config.json b/src/configurations/destinations/tvsquared/db-config.json index 9855ce67b..739a32616 100644 --- a/src/configurations/destinations/tvsquared/db-config.json +++ b/src/configurations/destinations/tvsquared/db-config.json @@ -35,7 +35,7 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"] }, "secretKeys": ["credentials"] } diff --git a/src/configurations/destinations/vero/db-config.json b/src/configurations/destinations/vero/db-config.json index 97a7b1b8a..b9d5f1c0d 100644 --- a/src/configurations/destinations/vero/db-config.json +++ b/src/configurations/destinations/vero/db-config.json @@ -40,7 +40,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -50,7 +52,17 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["apiKey", "useNativeSDK"] + "web": ["apiKey", "useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/vero/schema.json b/src/configurations/destinations/vero/schema.json index 151bf6936..7e5713120 100644 --- a/src/configurations/destinations/vero/schema.json +++ b/src/configurations/destinations/vero/schema.json @@ -65,6 +65,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["cloud", "device"] }, + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } From 185935467899222b8766c63c63b934d70410cc9a Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 19 Feb 2024 11:06:05 +0530 Subject: [PATCH 12/17] fix: add support of placing properties at root in af (#1203) --- src/configurations/destinations/af/db-config.json | 1 + src/configurations/destinations/af/schema.json | 4 ++++ src/configurations/destinations/af/ui-config.json | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/src/configurations/destinations/af/db-config.json b/src/configurations/destinations/af/db-config.json index 8899d8812..cb558637a 100644 --- a/src/configurations/destinations/af/db-config.json +++ b/src/configurations/destinations/af/db-config.json @@ -52,6 +52,7 @@ "devKey", "sharingFilter", "useRichEventName", + "addPropertiesAtRoot", "androidAppId", "appleAppId", "blacklistedEvents", diff --git a/src/configurations/destinations/af/schema.json b/src/configurations/destinations/af/schema.json index 9109c1dbe..12862da79 100644 --- a/src/configurations/destinations/af/schema.json +++ b/src/configurations/destinations/af/schema.json @@ -20,6 +20,10 @@ "type": "boolean", "default": false }, + "addPropertiesAtRoot": { + "type": "boolean", + "default": false + }, "sharingFilter": { "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" diff --git a/src/configurations/destinations/af/ui-config.json b/src/configurations/destinations/af/ui-config.json index 7945e3e29..cb0fe394a 100644 --- a/src/configurations/destinations/af/ui-config.json +++ b/src/configurations/destinations/af/ui-config.json @@ -39,6 +39,13 @@ "footerNote": "To include screen/page name in Screen/Page event names (ex: Viewed Contacts Page)", "default": false }, + { + "type": "checkbox", + "label": "Add properties at root in eventValue", + "value": "addPropertiesAtRoot", + "footerNote": "To send the custom properties to the root of eventValue.", + "default": false + }, { "type": "textInput", "label": "Sharing Filter", From b75e2e5aceb568345d9181b11a3ff597fb9aeeb8 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 19 Feb 2024 12:44:38 +0530 Subject: [PATCH 13/17] Revert "feat: adding custom field support for freshsales" (#1218) --- .../destinations/freshsales/db-config.json | 1 - .../destinations/freshsales/schema.json | 16 ---------------- .../destinations/freshsales/ui-config.json | 13 ------------- .../validation/destinations/freshsales.json | 17 ----------------- 4 files changed, 47 deletions(-) diff --git a/src/configurations/destinations/freshsales/db-config.json b/src/configurations/destinations/freshsales/db-config.json index 3a377b063..a23a997e9 100644 --- a/src/configurations/destinations/freshsales/db-config.json +++ b/src/configurations/destinations/freshsales/db-config.json @@ -38,7 +38,6 @@ "apiKey", "domain", "rudderEventsToFreshsalesEvents", - "customPropertyMapping", "oneTrustCookieCategories" ] } diff --git a/src/configurations/destinations/freshsales/schema.json b/src/configurations/destinations/freshsales/schema.json index d13d3de60..a58ed3a80 100644 --- a/src/configurations/destinations/freshsales/schema.json +++ b/src/configurations/destinations/freshsales/schema.json @@ -28,22 +28,6 @@ } } }, - "customPropertyMapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "from": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - }, - "to": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } - } - } - }, "oneTrustCookieCategories": { "type": "array", "items": { diff --git a/src/configurations/destinations/freshsales/ui-config.json b/src/configurations/destinations/freshsales/ui-config.json index df24401e7..3aa329eec 100644 --- a/src/configurations/destinations/freshsales/ui-config.json +++ b/src/configurations/destinations/freshsales/ui-config.json @@ -47,19 +47,6 @@ "value": "lifecycle_stage" } ] - }, - { - "type": "dynamicForm", - "label": "Mapping to add custom fields while creating / updating a contact", - "value": "customPropertyMapping", - "required": false, - "labelRight": "Freshsales Contact Custom Field Name", - "labelLeft": "Rudder Payload Property", - "keyLeft": "from", - "keyRight": "to", - "placeholderLeft": "e.g: newTrait", - "placeholderRight": "e.g: cf_new_trait", - "footerNote": "Map Rudder Traits to Freshsales Contact Custom Fields. Here, traits will be fetched from traits object(message.traits or message.context.traits). Please provide the exact Internal name of the custom trait" } ] }, diff --git a/test/data/validation/destinations/freshsales.json b/test/data/validation/destinations/freshsales.json index df7962153..6bece2cec 100644 --- a/test/data/validation/destinations/freshsales.json +++ b/test/data/validation/destinations/freshsales.json @@ -35,22 +35,5 @@ }, "result": false, "err": ["domain must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] - }, - { - "config": { - "apiKey": "adhs123-adhs123-adhs3", - "domain": "rudder-123457.myfreshworks.com", - "customPropertyMapping": [ - { - "from": "ofwinqeoqwefnoewqo9", - "to": "test" - }, - { - "from": "idwhcbiwdfbciwdfw", - "to": "entry" - } - ] - }, - "result": true } ] From 6d5e40fdb812aa4c46ea13cbae5f7eb696d0cc0c Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 19 Feb 2024 13:00:22 +0530 Subject: [PATCH 14/17] feat: onboarding bluecore integration (#1182) --- .../destinations/bluecore/db-config.json | 44 +++++ .../destinations/bluecore/schema.json | 49 ++++++ .../destinations/bluecore/ui-config.json | 165 ++++++++++++++++++ .../salesforce_oauth/db-config.json | 4 +- .../validation/destinations/bluecore.json | 142 +++++++++++++++ 5 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 src/configurations/destinations/bluecore/db-config.json create mode 100644 src/configurations/destinations/bluecore/schema.json create mode 100644 src/configurations/destinations/bluecore/ui-config.json create mode 100644 test/data/validation/destinations/bluecore.json diff --git a/src/configurations/destinations/bluecore/db-config.json b/src/configurations/destinations/bluecore/db-config.json new file mode 100644 index 000000000..2d2b41218 --- /dev/null +++ b/src/configurations/destinations/bluecore/db-config.json @@ -0,0 +1,44 @@ +{ + "name": "BLUECORE", + "displayName": "Bluecore", + "config": { + "cdkV2Enabled": true, + "transformAtV1": "processor", + "saveDestinationResponse": true, + "includeKeys": ["oneTrustCookieCategories"], + "excludeKeys": [], + "supportedSourceTypes": [ + "android", + "ios", + "unity", + "amp", + "reactnative", + "flutter", + "cordova", + "web", + "cloud", + "shopify", + "warehouse" + ], + "supportedConnectionModes": { + "android": ["cloud"], + "ios": ["cloud"], + "web": ["cloud"], + "unity": ["cloud"], + "amp": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] + }, + "supportedMessageTypes": { + "cloud": ["identify", "track"] + }, + "destConfig": { + "defaultConfig": ["bluecoreNamespace", "eventsMapping", "oneTrustCookieCategories"] + }, + "secretKeys": ["bluecoreNamespace"] + } +} diff --git a/src/configurations/destinations/bluecore/schema.json b/src/configurations/destinations/bluecore/schema.json new file mode 100644 index 000000000..077b131ab --- /dev/null +++ b/src/configurations/destinations/bluecore/schema.json @@ -0,0 +1,49 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["bluecoreNamespace"], + "type": "object", + "properties": { + "bluecoreNamespace": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "eventsMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "enum": [ + "viewed_product", + "search", + "add_to_cart", + "remove_from_cart", + "purchase", + "wishlist", + "" + ] + } + } + } + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + } + } +} diff --git a/src/configurations/destinations/bluecore/ui-config.json b/src/configurations/destinations/bluecore/ui-config.json new file mode 100644 index 000000000..6e6ef1982 --- /dev/null +++ b/src/configurations/destinations/bluecore/ui-config.json @@ -0,0 +1,165 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection Settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Bluecore namespace", + "note": "Get it from the url https://app.bluecore.com/admin/dashboard/", + "configKey": "bluecoreNamespace", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid Namespace", + "placeholder": "e.g. dummy_namespace" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack’s OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + }, + { + "title": "Mappings", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "Rudderstack to Bluecore Event Mappings", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "bluecoreEventMapping", + "label": "Event mappings" + } + ] + } + ] + } + ] + } + ], + "redirectGroups": { + "bluecoreEventMapping": { + "fields": [ + { + "type": "mapping", + "label": "Mapping to trigger the Rudderstack events with standard Bluecore events", + "configKey": "eventsMapping", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "from", + "label": "Event Name", + "placeholder": "e.g: Custom Purchase Event" + }, + { + "type": "singleSelect", + "key": "to", + "label": "Bluecore Standard Ecommerce Event", + "placeholder": "e.g: purchase", + "options": [ + { + "name": "Viewed Product", + "value": "viewed_product" + }, + { + "name": "Search", + "value": "search" + }, + { + "name": "Add to Cart", + "value": "add_to_cart" + }, + { + "name": "Remove From Cart", + "value": "remove_from_cart" + }, + { + "name": "Wishlist", + "value": "wishlist" + }, + { + "name": "Purchase", + "value": "purchase" + } + ] + } + ] + } + ] + } + }, + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] + } + } +} diff --git a/src/configurations/destinations/salesforce_oauth/db-config.json b/src/configurations/destinations/salesforce_oauth/db-config.json index f01b6e89b..73934db2d 100644 --- a/src/configurations/destinations/salesforce_oauth/db-config.json +++ b/src/configurations/destinations/salesforce_oauth/db-config.json @@ -37,7 +37,5 @@ }, "secretKeys": [] }, - "options": { - "hidden": true - } + "options": {} } diff --git a/test/data/validation/destinations/bluecore.json b/test/data/validation/destinations/bluecore.json new file mode 100644 index 000000000..00e65b799 --- /dev/null +++ b/test/data/validation/destinations/bluecore.json @@ -0,0 +1,142 @@ +[ + { + "config": { + "bluecoreNamespace": "", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": false, + "err": [ + "bluecoreNamespace must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "enableAliasCall": false, + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": false, + "err": [" must have required property 'bluecoreNamespace'"] + }, + { + "config": { + "bluecoreNamespace": 1234, + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "enableAliasCall": true, + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": false, + "err": ["bluecoreNamespace must be string"] + }, + { + "config": { + "bluecoreNamespace": "qwerty1234qwerty", + "eventsToSpotifyPixelEvents": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": true + }, + { + "config": { + "bluecoreNamespace": "qwerty1234qwerty", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "random" + } + ], + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + }, + "connectionMode": { + "web": "device" + } + }, + "result": false, + "err": ["eventsMapping.0.to must be equal to one of the allowed values"] + } +] From e73981f54b757aa60181942a19e7db97aac40658 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 19 Feb 2024 13:10:22 +0530 Subject: [PATCH 15/17] fix: reverting unhide of salesforce oauth (#1220) --- .../destinations/salesforce_oauth/db-config.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/configurations/destinations/salesforce_oauth/db-config.json b/src/configurations/destinations/salesforce_oauth/db-config.json index 73934db2d..f01b6e89b 100644 --- a/src/configurations/destinations/salesforce_oauth/db-config.json +++ b/src/configurations/destinations/salesforce_oauth/db-config.json @@ -37,5 +37,7 @@ }, "secretKeys": [] }, - "options": {} + "options": { + "hidden": true + } } From 3a86c228667b1340465e9dc03c8d6660db9201e9 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 19 Feb 2024 07:47:54 +0000 Subject: [PATCH 16/17] chore(release): 1.65.0 --- CHANGELOG.md | 20 ++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b61e273..1ff2ee2e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.65.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.2...v1.65.0) (2024-02-19) + + +### Features + +* add connectionMode for all sourceTypes in dest def for selected destinations ([#1197](https://github.com/rudderlabs/rudder-config-schema/issues/1197)) ([dde686c](https://github.com/rudderlabs/rudder-config-schema/commit/dde686cc72e0ac7423487d566e8f9715cc43d24f)) +* adding custom field support for freshsales ([#1195](https://github.com/rudderlabs/rudder-config-schema/issues/1195)) ([9124fb4](https://github.com/rudderlabs/rudder-config-schema/commit/9124fb44fd18d903bbf37f7cef3eaf7aeccf92e4)) +* improve schema generator output ([#1191](https://github.com/rudderlabs/rudder-config-schema/issues/1191)) ([04e05b8](https://github.com/rudderlabs/rudder-config-schema/commit/04e05b89a7333291792fbb6e111388b5ff8b985f)) +* onboard new destination commandbar ([#1196](https://github.com/rudderlabs/rudder-config-schema/issues/1196)) ([daf9fa8](https://github.com/rudderlabs/rudder-config-schema/commit/daf9fa8256eed280bad04c0c109de3787e6262e0)) +* onboard trade desk real time conversions ([#1213](https://github.com/rudderlabs/rudder-config-schema/issues/1213)) ([0d9b8d7](https://github.com/rudderlabs/rudder-config-schema/commit/0d9b8d7ee45b6ba5c914aaca6c2c11474a7575e9)) +* onboarding bluecore integration ([#1182](https://github.com/rudderlabs/rudder-config-schema/issues/1182)) ([6d5e40f](https://github.com/rudderlabs/rudder-config-schema/commit/6d5e40fdb812aa4c46ea13cbae5f7eb696d0cc0c)) +* tiktok_offline_events added support for all Standard events ([#1216](https://github.com/rudderlabs/rudder-config-schema/issues/1216)) ([61d36fa](https://github.com/rudderlabs/rudder-config-schema/commit/61d36fa5db882f00d97dd9dc88b2f26a7d49a87a)) + + +### Bug Fixes + +* add support of placing properties at root in af ([#1203](https://github.com/rudderlabs/rudder-config-schema/issues/1203)) ([1859354](https://github.com/rudderlabs/rudder-config-schema/commit/185935467899222b8766c63c63b934d70410cc9a)) +* comscore schema ([#1217](https://github.com/rudderlabs/rudder-config-schema/issues/1217)) ([7faa64a](https://github.com/rudderlabs/rudder-config-schema/commit/7faa64acfe2fca20b75789f31b541dd33e7fde8c)) +* reverting unhide of salesforce oauth ([#1220](https://github.com/rudderlabs/rudder-config-schema/issues/1220)) ([e73981f](https://github.com/rudderlabs/rudder-config-schema/commit/e73981f54b757aa60181942a19e7db97aac40658)) + ### [1.64.2](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.1...v1.64.2) (2024-02-14) ### [1.64.1](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.0...v1.64.1) (2024-02-13) diff --git a/package-lock.json b/package-lock.json index aa3bd6dd4..7eb48b94d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-config-schema", - "version": "1.64.2", + "version": "1.65.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-config-schema", - "version": "1.64.2", + "version": "1.65.0", "license": "MIT", "dependencies": { "ajv": "^8.12.0", diff --git a/package.json b/package.json index 9425018ed..a5d73ac97 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-config-schema", - "version": "1.64.2", + "version": "1.65.0", "description": "", "main": "src/index.ts", "private": true, From 8cba11032b2c83a286712748bc715b190c4b707d Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 19 Feb 2024 14:37:13 +0530 Subject: [PATCH 17/17] chore: renamed advertiserId to pixelId for reddit pixel (#1223) --- src/configurations/destinations/reddit_pixel/ui-config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/configurations/destinations/reddit_pixel/ui-config.json b/src/configurations/destinations/reddit_pixel/ui-config.json index 48dd47325..b04018901 100644 --- a/src/configurations/destinations/reddit_pixel/ui-config.json +++ b/src/configurations/destinations/reddit_pixel/ui-config.json @@ -5,14 +5,14 @@ "fields": [ { "type": "textInput", - "label": "Advertiser ID", + "label": "Pixel ID", "value": "advertiserId", "regex": "^(.{0,100})$", - "regexErrorMessage": "Invalid Advertiser ID", + "regexErrorMessage": "Invalid Pixel ID", "required": true, "placeholder": "e.g: t1_d2XXc27c", "secret": true, - "footerNote": "Your Advertiser ID" + "footerNote": "Your Pixel ID" }, { "type": "dynamicSelectForm",