Add github action for testing notebooks

.github/workflows/actions.yml

@@ -0,0 +1,50 @@
+name: Run tests
+
+on:
+  pull_request:
+
+jobs:
+  run_notebooks:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v4.1.7
+
+    - name: Get changed files
+      id: changed-files
+      uses: tj-actions/changed-files@v45
+      # To compare changes between the current commit and the last pushed remote commit set `since_last_remote_commit: true`. e.g
+      # with:
+      #   since_last_remote_commit: true
+      with:
+        files: |
+          **.ipynb
+  
+    - name: Start docker and test container
+      if: steps.changed-files.outputs.any_changed == 'true'
+      env:
+        ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: |
+        chmod +x ./tests/test-changed-notebooks.sh
+        ./tests/test-changed-notebooks.sh
+      shell: bash
+
+    - name: Test all changed files
+      if: steps.changed-files.outputs.any_changed == 'true'
+      env:
+        ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: |
+        for file in ${ALL_CHANGED_FILES}; do
+          echo "$file"
+          docker exec jupyter papermill ../$file /dev/null --execution-timeout=1200 --log-level ERROR
+        done
+    
+    - name: Stop docker and test container
+      if: steps.changed-files.outputs.any_changed == 'true'
+      env:
+        ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: |
+        docker compose --profile all-services down -v
+      shell: bash
+
+
+

tests/test-changed-notebooks.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# This script run notebook execution tests.
+# Usage:
+#   test_notebooks.sh <notebook_path>
+#   Parameters:	
+#     <notebook_path> (optional): path to notebook(s) to test
+#
+# The script will:  
+#  - use docker-compose-local.yaml config to build jupyter-img and bring up the full stack,
+#  - Wait for services to report status
+#  - Run all notebooks found recursively within the specified <notebook path>, 
+#    if no path is specified, it will use the current path and any notebooks in any subfolders
+# 
+
+
+
+retry() {
+  local action="$1" # action to run
+  local retries="${2:-10}" # max retries
+  local sleep_seconds="${3:-2}" # wait between tries
+
+  exit_code=999
+
+  while [[ "$exit_code" -ne "0" && "$retries" -gt 0 ]]; do
+    #run action and consume output, no need to show it
+    echo " trying...[${action[@]}]"
+    output=`${action[@]}` 
+    local exit_code=$?
+    echo "  output: [${output}]"
+    echo "  exit code:${exit_code}"
+    retries=$(($retries - 1)) 
+    if [[ $exit_code -ne 0 ]]; then
+      sleep $sleep_seconds
+    fi
+  done
+}
+
+
+
+
+docker compose --profile all-services up -d
+docker exec jupyter pip install papermill
+
+if [ $# -ge 1 ];
+    then TEST_PATH=$1
+else
+     TEST_PATH=../notebooks/
+fi
+
+# check that druid is running
+echo "Waiting for Druid readiness..."
+retry 'curl http://localhost:8081/status' 50 2
+retry 'curl http://localhost:8082/status' 50 2
+retry 'curl http://localhost:8083/status' 50 2
+retry 'curl http://localhost:8091/status' 50 2
+retry 'curl http://localhost:8888/status' 50 2
+#echo "Waiting for Data Generator readiness..."
+retry 'curl http://localhost:9999/jobs' 50 2