From 39646447431de2540dcbf5e8a2ad79dbd50e438c Mon Sep 17 00:00:00 2001 From: Peter Marshall Date: Thu, 31 Oct 2024 09:27:40 +0000 Subject: [PATCH] Revert "Add github action for testing notebooks" --- .github/workflows/actions.yml | 54 ----------------- .gitignore | 1 - .../02-ingestion/02-batch-ingestion.ipynb | 4 +- .../02-ingestion/12-spatial-dimensions.ipynb | 7 +-- .../02-ingestion/13-native-transforms.ipynb | 3 - .../03-query/07-functions-datetime.ipynb | 7 +-- notebooks/03-query/08-functions-strings.ipynb | 7 +-- notebooks/03-query/19-groupby-earliest.ipynb | 3 +- tests/launch-test-environment.sh | 59 ------------------- tests/test-notebooks-papermill.sh | 18 ------ 10 files changed, 9 insertions(+), 154 deletions(-) delete mode 100644 .github/workflows/actions.yml delete mode 100644 tests/launch-test-environment.sh delete mode 100644 tests/test-notebooks-papermill.sh diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml deleted file mode 100644 index 99cc41d6..00000000 --- a/.github/workflows/actions.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Run tests - -on: - pull_request: - -jobs: - run_notebooks: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4.1.7 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45 - # To compare changes between the current commit and the last pushed remote commit set `since_last_remote_commit: true`. e.g - # with: - # since_last_remote_commit: true - with: - files: | - **.ipynb - - - name: Start docker and test container - if: steps.changed-files.outputs.any_changed == 'true' - env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - run: | - chmod +x ./tests/launch-test-environment.sh - ./tests/launch-test-environment.sh - shell: bash - - - name: Test all changed files - if: steps.changed-files.outputs.any_changed == 'true' - env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - run: | - skiplList=("notebooks/03-query/11-joins.ipynb" #Hangs forever on users data generation, even when run manually, just broken? - "notebooks/99-contributing/notebook-template.ipynb") #Contains invalid sql - - for file in ${ALL_CHANGED_FILES}; do - echo "$file" - if [[ ! " ${skiplList[*]} " =~ [[:space:]]${file}[[:space:]] ]]; then - docker exec jupyter papermill ../$file /dev/null --log-level ERROR - else - echo "Skipped" - fi - done - - - name: Stop docker and test container - if: steps.changed-files.outputs.any_changed == 'true' - env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - run: | - docker compose --profile all-services down -v - shell: bash diff --git a/.gitignore b/.gitignore index 854c24d8..78e29ab7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ .ipynb_checkpoints .ipynb_checkpoints/* .DS_Store -output.ipynb diff --git a/notebooks/02-ingestion/02-batch-ingestion.ipynb b/notebooks/02-ingestion/02-batch-ingestion.ipynb index d8e28716..7f1b7b2c 100644 --- a/notebooks/02-ingestion/02-batch-ingestion.ipynb +++ b/notebooks/02-ingestion/02-batch-ingestion.ipynb @@ -588,7 +588,7 @@ "metadata": {}, "outputs": [], "source": [ - "druid.datasources.drop(\"example-kttm-transform-batch\", True)" + "druid.datasources.drop(table_name, True)" ] }, { @@ -626,7 +626,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.11.6" } }, "nbformat": 4, diff --git a/notebooks/02-ingestion/12-spatial-dimensions.ipynb b/notebooks/02-ingestion/12-spatial-dimensions.ipynb index e69dc441..5af7c91b 100644 --- a/notebooks/02-ingestion/12-spatial-dimensions.ipynb +++ b/notebooks/02-ingestion/12-spatial-dimensions.ipynb @@ -73,7 +73,6 @@ "source": [ "import druidapi\n", "import os\n", - "import json\n", "\n", "if 'DRUID_HOST' not in os.environ.keys():\n", " druid_host=f\"http://localhost:8888\"\n", @@ -236,6 +235,8 @@ "metadata": {}, "outputs": [], "source": [ + "import json\n", + "\n", "spatial_index_spec = {\n", " \"type\": \"index_parallel\",\n", " \"spec\": {\n", @@ -364,10 +365,6 @@ "metadata": {}, "outputs": [], "source": [ - "import time\n", - "\n", - "time.sleep(100) # Give previous cell some time to complete\n", - "\n", "rectangular_filter_query = {\n", " \"queryType\": \"topN\",\n", " \"dataSource\": {\n", diff --git a/notebooks/02-ingestion/13-native-transforms.ipynb b/notebooks/02-ingestion/13-native-transforms.ipynb index 5cf2fd6d..61e2570d 100644 --- a/notebooks/02-ingestion/13-native-transforms.ipynb +++ b/notebooks/02-ingestion/13-native-transforms.ipynb @@ -767,9 +767,6 @@ "metadata": {}, "outputs": [], "source": [ - "import time\n", - "time.sleep(100) # Give previous cell some time to complete\n", - "\n", "time_now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')\n", "\n", "sql=f'''\n", diff --git a/notebooks/03-query/07-functions-datetime.ipynb b/notebooks/03-query/07-functions-datetime.ipynb index 4874cb37..ff42b82b 100644 --- a/notebooks/03-query/07-functions-datetime.ipynb +++ b/notebooks/03-query/07-functions-datetime.ipynb @@ -148,8 +148,7 @@ "source": [ "import matplotlib\n", "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import time" + "import pandas as pd" ] }, { @@ -437,8 +436,6 @@ "metadata": {}, "outputs": [], "source": [ - "time.sleep(100) # Give previous cell some time to complete\n", - "\n", "sql='''\n", "SELECT\n", " __time AS \"start\",\n", @@ -449,7 +446,7 @@ "LIMIT 10\n", "'''\n", "\n", - "display.sql(sql) " + "display.sql(sql)" ] }, { diff --git a/notebooks/03-query/08-functions-strings.ipynb b/notebooks/03-query/08-functions-strings.ipynb index 56c7bfa0..798c8714 100644 --- a/notebooks/03-query/08-functions-strings.ipynb +++ b/notebooks/03-query/08-functions-strings.ipynb @@ -151,8 +151,7 @@ "source": [ "import matplotlib\n", "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import time" + "import pandas as pd" ] }, { @@ -550,8 +549,6 @@ "metadata": {}, "outputs": [], "source": [ - "time.sleep(100) # Give previous cell some time to complete\n", - "\n", "sql='''\n", "SELECT\n", " TRIM(LEADING 'X' FROM \"XXXXXcountryXXXXX\") AS \"leadingTrim\",\n", @@ -747,7 +744,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/03-query/19-groupby-earliest.ipynb b/notebooks/03-query/19-groupby-earliest.ipynb index 57d94ee7..5de89516 100644 --- a/notebooks/03-query/19-groupby-earliest.ipynb +++ b/notebooks/03-query/19-groupby-earliest.ipynb @@ -141,10 +141,9 @@ " \"country\",\n", " \"loaded_image\",\n", " \"os\",\n", - " \"session_length\",\n", " LATEST_BY(\"session_length\",TIME_PARSE(\"timestamp\")) \"latest_session_length\"\n", "FROM \"ext\"\n", - "GROUP BY 1,2,3,4,5,6,7,8\n", + "GROUP BY 1,2,3,4,5,6,7\n", "PARTITIONED BY DAY\n", "'''\n", "\n", diff --git a/tests/launch-test-environment.sh b/tests/launch-test-environment.sh deleted file mode 100644 index 2765ec0c..00000000 --- a/tests/launch-test-environment.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# -# This script run notebook execution tests. -# Usage: -# test_notebooks.sh -# Parameters: -# (optional): path to notebook(s) to test -# -# The script will: -# - use docker-compose-local.yaml config to build jupyter-img and bring up the full stack, -# - Wait for services to report status -# - Run all notebooks found recursively within the specified , -# if no path is specified, it will use the current path and any notebooks in any subfolders -# - - - -retry() { - local action="$1" # action to run - local retries="${2:-10}" # max retries - local sleep_seconds="${3:-2}" # wait between tries - - exit_code=999 - - while [[ "$exit_code" -ne "0" && "$retries" -gt 0 ]]; do - #run action and consume output, no need to show it - echo " trying...[${action[@]}]" - output=`${action[@]}` - local exit_code=$? - echo " output: [${output}]" - echo " exit code:${exit_code}" - retries=$(($retries - 1)) - if [[ $exit_code -ne 0 ]]; then - sleep $sleep_seconds - fi - done -} - - - - -docker compose --profile all-services up -d -docker exec jupyter pip install papermill - -if [ $# -ge 1 ]; - then TEST_PATH=$1 -else - TEST_PATH=../notebooks/ -fi - -# check that druid is running -echo "Waiting for Druid readiness..." -retry 'curl http://localhost:8081/status' 50 2 -retry 'curl http://localhost:8082/status' 50 2 -retry 'curl http://localhost:8083/status' 50 2 -retry 'curl http://localhost:8091/status' 50 2 -retry 'curl http://localhost:8888/status' 50 2 -#echo "Waiting for Data Generator readiness..." -retry 'curl http://localhost:9999/jobs' 50 2 diff --git a/tests/test-notebooks-papermill.sh b/tests/test-notebooks-papermill.sh deleted file mode 100644 index c0787207..00000000 --- a/tests/test-notebooks-papermill.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -bash ./launch-test-environment.sh - -skiplList=("notebooks/03-query/11-joins.ipynb" #Hangs forever on users data generation, even when run manually, just broken? -"notebooks/99-contributing/notebook-template.ipynb") #Contains invalid sql - -ALL_CHANGED_FILES=$(find notebooks -type f -name '*.ipynb' | awk '!/\/\./') -for file in ${ALL_CHANGED_FILES}; do - echo "$file" - if [[ ! " ${skiplList[*]} " =~ [[:space:]]${file}[[:space:]] ]]; then - docker exec jupyter papermill ../$file output.ipynb --log-level ERROR - else - echo "Skipped" - fi -done - -docker compose --profile all-services down -v