first draft of script to check branches produce same results

.gitignore

@@ -1,5 +1,6 @@
 # Project specifics
 logs/
+branch_comparison/results/*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

branch_comparison/diff_results.py

@@ -0,0 +1,19 @@
+from jsondiff import diff
+import json
+
+f = open("branch_comparison/scenarios.json", "r")
+SCENARIOS = json.load(f)
+
+if __name__ == "__main__":
+
+    for index, scenario in enumerate(SCENARIOS):
+        master_file = open(f"branch_comparison/results/master_{index}.json", "r")
+        master_results = json.load(master_file)
+
+        async_file = open(f"branch_comparison/results/async_{index}.json", "r")
+        async_results = json.load(async_file)
+
+        d = diff(master_results, async_results)
+
+        if d:
+            print(f"{scenario}: {d}")

branch_comparison/generate_async_results.py

@@ -0,0 +1,43 @@
+from httpx import AsyncClient
+import asyncio
+import json
+from spectacles.client import LookerClient
+from spectacles.runner import Runner
+import os
+
+f = open("branch_comparison/scenarios.json", "r")
+SCENARIOS = json.load(f)
+
+
+async def get_async_results(
+    base_url, client_id, client_secret, project_name, validation_args
+):
+
+    async_client = AsyncClient()
+
+    looker_client = LookerClient(
+        async_client=async_client,
+        base_url=base_url,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+
+    runner = Runner(looker_client, project_name, remote_reset=True)
+
+    async_results = await runner.validate_sql(**validation_args)
+
+    return async_results
+
+
+if __name__ == "__main__":
+
+    base_url = "https://spectacles.looker.com"
+    client_id = os.getenv("LOOKER_CLIENT_ID")
+    client_secret = os.getenv("LOOKER_CLIENT_SECRET")
+
+    for index, scenario in enumerate(SCENARIOS):
+        results = asyncio.run(
+            get_async_results(base_url, client_id, client_secret, **scenario)
+        )
+        with open(f"branch_comparison/results/async_{index}.json", "w") as f:
+            json.dump(results, f)

branch_comparison/generate_master_results.py

@@ -0,0 +1,40 @@
+from spectacles.client import LookerClient
+from spectacles.runner import Runner
+import json
+import os
+
+f = open("branch_comparison/scenarios.json", "r")
+SCENARIOS = json.load(f)
+
+
+def get_master_results(
+    base_url, client_id, client_secret, project_name, validation_args
+):
+
+    looker_client = LookerClient(
+        base_url=base_url,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+
+    runner = Runner(
+        looker_client,
+        project_name,
+        remote_reset=True,
+    )
+
+    master_results = runner.validate_sql(**validation_args)
+
+    return master_results
+
+
+if __name__ == "__main__":
+
+    base_url = "https://spectacles.looker.com"
+    client_id = os.getenv("LOOKER_CLIENT_ID")
+    client_secret = os.getenv("LOOKER_CLIENT_SECRET")
+
+    for index, scenario in enumerate(SCENARIOS):
+        results = get_master_results(base_url, client_id, client_secret, **scenario)
+        with open(f"branch_comparison/results/master_{index}.json", "w") as f:
+            json.dump(results, f)

branch_comparison/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+source venv/bin/activate
+
+rm branch_comparison/results/*
+
+git checkout master
+python3 branch_comparison/generate_master_results.py
+
+git checkout feature/async
+python3 branch_comparison/generate_async_results.py
+
+python3 branch_comparison/diff_results.py

branch_comparison/scenarios.json

@@ -0,0 +1,105 @@
+[
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-errors",
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-errors",
+            "filters": [
+                "ecomm/dim_products"
+            ],
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-errors",
+            "filters": [
+                "*/*"
+            ],
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-errors",
+            "fail_fast": true
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-errors",
+            "incremental": true,
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-incremental-fix",
+            "incremental": true,
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-incremental-fix",
+            "incremental": true,
+            "target": "async-errors",
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-new-explore",
+            "incremental": true,
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-new-explore",
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-incremental-changes",
+            "incremental": true,
+            "target": "async-errors",
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "looker-demo",
+        "validation_args": {
+            "ref": "async-does-not-exist",
+            "fail_fast": false
+        }
+    },
+    {
+        "project_name": "spectacles",
+        "validation_args": {
+            "ref": "charter-demo-3",
+            "fail_fast": false
+        }
+    }
+]