From 09b565c141e57d9b8aa1e1ee805453d733c5de60 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 5 Jul 2022 14:23:12 +0100 Subject: [PATCH] first draft of script to check branches produce same results --- .gitignore | 1 + branch_comparison/diff_results.py | 19 ++++ branch_comparison/generate_async_results.py | 43 ++++++++ branch_comparison/generate_master_results.py | 40 +++++++ branch_comparison/run.sh | 13 +++ branch_comparison/scenarios.json | 105 +++++++++++++++++++ 6 files changed, 221 insertions(+) create mode 100644 branch_comparison/diff_results.py create mode 100644 branch_comparison/generate_async_results.py create mode 100644 branch_comparison/generate_master_results.py create mode 100755 branch_comparison/run.sh create mode 100644 branch_comparison/scenarios.json diff --git a/.gitignore b/.gitignore index 5c5f7606..be383756 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Project specifics logs/ +branch_comparison/results/* # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/branch_comparison/diff_results.py b/branch_comparison/diff_results.py new file mode 100644 index 00000000..aaf7ee5c --- /dev/null +++ b/branch_comparison/diff_results.py @@ -0,0 +1,19 @@ +from jsondiff import diff +import json + +f = open("branch_comparison/scenarios.json", "r") +SCENARIOS = json.load(f) + +if __name__ == "__main__": + + for index, scenario in enumerate(SCENARIOS): + master_file = open(f"branch_comparison/results/master_{index}.json", "r") + master_results = json.load(master_file) + + async_file = open(f"branch_comparison/results/async_{index}.json", "r") + async_results = json.load(async_file) + + d = diff(master_results, async_results) + + if d: + print(f"{scenario}: {d}") diff --git a/branch_comparison/generate_async_results.py b/branch_comparison/generate_async_results.py new file mode 100644 index 00000000..7cd63c59 --- /dev/null +++ b/branch_comparison/generate_async_results.py @@ -0,0 +1,43 @@ +from httpx import AsyncClient +import asyncio +import json +from spectacles.client import LookerClient +from spectacles.runner import Runner +import os + +f = open("branch_comparison/scenarios.json", "r") +SCENARIOS = json.load(f) + + +async def get_async_results( + base_url, client_id, client_secret, project_name, validation_args +): + + async_client = AsyncClient() + + looker_client = LookerClient( + async_client=async_client, + base_url=base_url, + client_id=client_id, + client_secret=client_secret, + ) + + runner = Runner(looker_client, project_name, remote_reset=True) + + async_results = await runner.validate_sql(**validation_args) + + return async_results + + +if __name__ == "__main__": + + base_url = "https://spectacles.looker.com" + client_id = os.getenv("LOOKER_CLIENT_ID") + client_secret = os.getenv("LOOKER_CLIENT_SECRET") + + for index, scenario in enumerate(SCENARIOS): + results = asyncio.run( + get_async_results(base_url, client_id, client_secret, **scenario) + ) + with open(f"branch_comparison/results/async_{index}.json", "w") as f: + json.dump(results, f) diff --git a/branch_comparison/generate_master_results.py b/branch_comparison/generate_master_results.py new file mode 100644 index 00000000..17ac5117 --- /dev/null +++ b/branch_comparison/generate_master_results.py @@ -0,0 +1,40 @@ +from spectacles.client import LookerClient +from spectacles.runner import Runner +import json +import os + +f = open("branch_comparison/scenarios.json", "r") +SCENARIOS = json.load(f) + + +def get_master_results( + base_url, client_id, client_secret, project_name, validation_args +): + + looker_client = LookerClient( + base_url=base_url, + client_id=client_id, + client_secret=client_secret, + ) + + runner = Runner( + looker_client, + project_name, + remote_reset=True, + ) + + master_results = runner.validate_sql(**validation_args) + + return master_results + + +if __name__ == "__main__": + + base_url = "https://spectacles.looker.com" + client_id = os.getenv("LOOKER_CLIENT_ID") + client_secret = os.getenv("LOOKER_CLIENT_SECRET") + + for index, scenario in enumerate(SCENARIOS): + results = get_master_results(base_url, client_id, client_secret, **scenario) + with open(f"branch_comparison/results/master_{index}.json", "w") as f: + json.dump(results, f) diff --git a/branch_comparison/run.sh b/branch_comparison/run.sh new file mode 100755 index 00000000..68049f2a --- /dev/null +++ b/branch_comparison/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +source venv/bin/activate + +rm branch_comparison/results/* + +git checkout master +python3 branch_comparison/generate_master_results.py + +git checkout feature/async +python3 branch_comparison/generate_async_results.py + +python3 branch_comparison/diff_results.py \ No newline at end of file diff --git a/branch_comparison/scenarios.json b/branch_comparison/scenarios.json new file mode 100644 index 00000000..baa9e75f --- /dev/null +++ b/branch_comparison/scenarios.json @@ -0,0 +1,105 @@ +[ + { + "project_name": "looker-demo", + "validation_args": { + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-errors", + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-errors", + "filters": [ + "ecomm/dim_products" + ], + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-errors", + "filters": [ + "*/*" + ], + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-errors", + "fail_fast": true + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-errors", + "incremental": true, + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-incremental-fix", + "incremental": true, + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-incremental-fix", + "incremental": true, + "target": "async-errors", + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-new-explore", + "incremental": true, + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-new-explore", + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-incremental-changes", + "incremental": true, + "target": "async-errors", + "fail_fast": false + } + }, + { + "project_name": "looker-demo", + "validation_args": { + "ref": "async-does-not-exist", + "fail_fast": false + } + }, + { + "project_name": "spectacles", + "validation_args": { + "ref": "charter-demo-3", + "fail_fast": false + } + } +] \ No newline at end of file