diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml new file mode 100644 index 00000000..3728208c --- /dev/null +++ b/.github/workflows/actions.yml @@ -0,0 +1,50 @@ +name: Run tests + +on: + pull_request: + +jobs: + run_notebooks: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4.1.7 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + # To compare changes between the current commit and the last pushed remote commit set `since_last_remote_commit: true`. e.g + # with: + # since_last_remote_commit: true + with: + files: | + **.ipynb + + - name: Start docker and test container + if: steps.changed-files.outputs.any_changed == 'true' + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + chmod +x ./tests/test-changed-notebooks.sh + ./tests/test-changed-notebooks.sh + shell: bash + + - name: Test all changed files + if: steps.changed-files.outputs.any_changed == 'true' + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file" + docker exec jupyter papermill ../$file /dev/null --execution-timeout=1200 --log-level ERROR + done + + - name: Stop docker and test container + if: steps.changed-files.outputs.any_changed == 'true' + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + docker compose --profile all-services down -v + shell: bash + + + diff --git a/tests/test-changed-notebooks.sh b/tests/test-changed-notebooks.sh new file mode 100644 index 00000000..2765ec0c --- /dev/null +++ b/tests/test-changed-notebooks.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# This script run notebook execution tests. +# Usage: +# test_notebooks.sh +# Parameters: +# (optional): path to notebook(s) to test +# +# The script will: +# - use docker-compose-local.yaml config to build jupyter-img and bring up the full stack, +# - Wait for services to report status +# - Run all notebooks found recursively within the specified , +# if no path is specified, it will use the current path and any notebooks in any subfolders +# + + + +retry() { + local action="$1" # action to run + local retries="${2:-10}" # max retries + local sleep_seconds="${3:-2}" # wait between tries + + exit_code=999 + + while [[ "$exit_code" -ne "0" && "$retries" -gt 0 ]]; do + #run action and consume output, no need to show it + echo " trying...[${action[@]}]" + output=`${action[@]}` + local exit_code=$? + echo " output: [${output}]" + echo " exit code:${exit_code}" + retries=$(($retries - 1)) + if [[ $exit_code -ne 0 ]]; then + sleep $sleep_seconds + fi + done +} + + + + +docker compose --profile all-services up -d +docker exec jupyter pip install papermill + +if [ $# -ge 1 ]; + then TEST_PATH=$1 +else + TEST_PATH=../notebooks/ +fi + +# check that druid is running +echo "Waiting for Druid readiness..." +retry 'curl http://localhost:8081/status' 50 2 +retry 'curl http://localhost:8082/status' 50 2 +retry 'curl http://localhost:8083/status' 50 2 +retry 'curl http://localhost:8091/status' 50 2 +retry 'curl http://localhost:8888/status' 50 2 +#echo "Waiting for Data Generator readiness..." +retry 'curl http://localhost:9999/jobs' 50 2