Auto tests WIP. Moved Dockerfile and Makefile to root

.github/workflows/build.yml

@@ -3,15 +3,19 @@ name: Build and Publish Docker Image
 on:
   push:
     branches:
-      - main  # Trigger on push to the main branch
+      - '**'  # Trigger on push to any branch
+  pull_request:
+    branches:
+      - main  # Trigger on pull requests targeting main
 
 concurrency:
-  group: build-and-publish-${{ github.ref }}  # Ensure jobs from the same ref (branch) cancel previous ones
-  cancel-in-progress: true  # Cancel any in-progress jobs when a new one starts
+  group: build-${{ github.ref }}  # Cancel in-progress builds from the same branch
+  cancel-in-progress: true
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 15  # Max time to run before failure
 
     steps:
       - name: Checkout code
@@ -31,10 +35,9 @@ jobs:
           echo "$DOCKER_PASSWORD" | docker login harbor.main.rise-ck8s.com -u "$DOCKER_USER" --password-stdin
 
       - name: Build and push Docker image
-        id: build_and_push
         run: |
-          docker build -t harbor.main.rise-ck8s.com/des-public/tutorials:latest ./tutorials
+          docker build -t harbor.main.rise-ck8s.com/des-public/tutorials:latest .  # Build from root
           docker push harbor.main.rise-ck8s.com/des-public/tutorials:latest
 
       - name: Image digest
-        run: echo "Image digest: ${{ steps.build_and_push.outputs.digest }}"
+        run: echo ${{ steps.docker_meta.outputs.digest }}

.github/workflows/test.yml

@@ -4,29 +4,35 @@ on:
   workflow_run:
     workflows: ["Build and Publish Docker Image"]
     types:
-      - completed  # Trigger only when the build is completed
-
-concurrency:
-  group: test-notebooks-${{ github.ref }}  # Ensure test jobs for the same ref are not duplicated
-  cancel-in-progress: true  # Cancel old tests when a new push is made
+      - completed
 
 jobs:
   test:
     runs-on: ubuntu-latest
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
+
+    concurrency:
+      group: test-notebooks-${{ github.ref }}
+      cancel-in-progress: true
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Log in to Docker registry
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
         run: |
           echo "$DOCKER_PASSWORD" | docker login harbor.main.rise-ck8s.com -u "$DOCKER_USER" --password-stdin
 
-      - name: Run notebook tests
+      - name: Pull Docker image
         run: |
           docker pull harbor.main.rise-ck8s.com/des-public/tutorials:latest
-          docker run --rm harbor.main.rise-ck8s.com/des-public/tutorials:latest pytest --disable-warnings tests/test_notebooks.py
+
+      - name: Run notebook tests
+        run: |
+          docker run --rm harbor.main.rise-ck8s.com/des-public/tutorials:latest ./start_tests.sh

Dockerfile

@@ -0,0 +1,52 @@
+FROM ubuntu:22.04 as eo_training_base
+
+# Install necessary packages
+RUN apt-get update && apt-get -y upgrade && \
+    apt-get install -y --no-install-recommends tree gosu sudo wget && \
+    apt-get install -y --no-install-recommends ca-certificates && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create user, add to sudo group, configure sudoers.
+RUN adduser --disabled-password --gecos '' ubuntu && \
+    usermod -aG sudo ubuntu && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER ubuntu
+WORKDIR /home/ubuntu
+
+ENV PATH="/home/ubuntu/miniconda3/bin:${PATH}"
+ARG PATH="/home/ubuntu/miniconda3/bin:${PATH}"
+
+# Install Miniconda
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3-latest-Linux-x86_64.sh -b && \
+    rm -f Miniconda3-latest-Linux-x86_64.sh && \
+    echo 'Running $(conda --version)' && \
+    /home/ubuntu/miniconda3/bin/conda init && \
+    . ~/.bashrc && \
+    conda update conda -y && \
+    conda install -y -c conda-forge mamba
+
+# Create the Conda environment
+COPY --chown=ubuntu:ubuntu tutorials/environment.yml /home/ubuntu/environment.yml
+RUN conda env create -f /home/ubuntu/environment.yml && \
+    conda clean -afy
+
+# Copy the tutorials directory to /proj
+COPY --chown=ubuntu:ubuntu tutorials /proj/tutorials
+
+# Copy the tests directory to /test
+COPY --chown=ubuntu:ubuntu tests /test
+
+# Set the PYTHONPATH to include the project
+ENV PYTHONPATH="/proj/tutorials:${PYTHONPATH}"
+
+# Set the working directory to /proj
+WORKDIR /proj
+
+# Expose the necessary ports
+EXPOSE 8888
+
+# Set the entry point to automatically start Jupyter Lab
+ENTRYPOINT ["bash", "-c", "source activate openeo-training && jupyter lab --port=8888 --ip=0.0.0.0 --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/proj --no-browser"]

tutorials/Makefile → Makefile

@@ -2,18 +2,16 @@
 
 # Define the image name and tag
 IMAGE_NAME = eo-training
-IMAGE_TAG = latest
-
+IMAGE_TAG = laptop
 
 # Define the source directory to mount (for start-mount-dev)
 SOURCE_DIR = $(shell pwd)
 
 list-targets:
 	@grep -E '^[^[:space:]]+:.*' Makefile | grep -v '=' | cut -d ':' -f 1
 
-
 # Build the Docker image
-build: 
+build:
 	docker build --progress=plain -t $(IMAGE_NAME):$(IMAGE_TAG) .
 
 # Start the Docker container without mounting the source directory
@@ -22,16 +20,16 @@ start-dev: build
 
 # Start the Docker container and mount the source directory
 start-mount-dev: build
-	docker run -it --rm -v $(SOURCE_DIR):/proj $(IMAGE_NAME):$(IMAGE_TAG) /bin/bash 
+	docker run -it --rm -v $(SOURCE_DIR):/proj $(IMAGE_NAME):$(IMAGE_TAG) /bin/bash
 
 # Run pytest on the tests directory inside the container
-test-notebooks-docker: build
-	docker run -it --rm $(IMAGE_NAME):$(IMAGE_TAG) bash -ic "pytest -v tests"
+test: build
+	DES_DOCKER_IMAGE=$(IMAGE_NAME):$(IMAGE_TAG) ./start_tests.sh
 
 start-notebook: build
 	@echo "==============================================================================  "
-	@echo "             Start a notebook in a  container mounted to local drive            "
+	@echo "             Start a notebook in a container mounted to local drive            "
 	@echo "             NOTE that the container SHARES your files on your HDD now          "
 	@echo "             under the folder ./proj "
-	@echo "=============================================================================== "
-	docker run --rm -it -p 8888:8888 --name eo-training --mount type=bind,source=$(shell pwd),target=/proj $(IMAGE_NAME):$(IMAGE_TAG)
+	@echo "==============================================================================="
+	docker run --rm -it -p 8888:8888 --name eo-training --mount type=bind,source=$(SOURCE_DIR),target=/proj $(IMAGE_NAME):$(IMAGE_TAG)

start_tests.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Set environment variables
+DES_DOCKER_IMAGE=${DES_DOCKER_IMAGE:-"harbor.main.rise-ck8s.com/des-public/tutorials:latest"}
+CONTAINER_NAME="des_notebooks"
+JUPYTER_PORT=9999  # Changed to 9999 to avoid conflicts
+
+# Clean up any previous runs
+docker stop $CONTAINER_NAME 2>/dev/null || true
+docker rm $CONTAINER_NAME 2>/dev/null || true
+
+# Start the Docker container with Jupyter in the background
+docker run -d --rm --name $CONTAINER_NAME -p $JUPYTER_PORT:9999 $DES_DOCKER_IMAGE
+
+# Wait for the Jupyter server to start (you might need to adjust this time)
+echo "Waiting for the Jupyter server to start..."
+sleep 10
+
+# Check if the container is running
+if ! docker ps | grep -q $CONTAINER_NAME; then
+    echo "Failed to start the Docker container!"
+    exit 1
+fi
+
+# Run pytest inside the container and capture the result
+docker exec $CONTAINER_NAME bash -c "source activate openeo-training && pytest --disable-warnings /test/test_notebooks.py"
+TEST_RESULT=$?
+
+# Stop and remove the container
+docker stop $CONTAINER_NAME
+
+# Exit with the test result code
+exit $TEST_RESULT

tests/test_notebooks.py

@@ -1,60 +1,39 @@
-import sys
-from pathlib import Path
-
+import os
 import pytest
+from pathlib import Path
 from testbook import testbook
 
-notebooks = f"{Path(__file__).parent.parent}/tutorials/"
-
-
-@pytest.fixture(autouse=True)
-def add_module_dir_to_sys_path():
-    # Add the directory containing your modules to the Python path
-    module_dir = notebooks
-    print("Adding notebooks to sys path", module_dir)
-    sys.path.insert(0, module_dir)
-    yield
-    # Remove the directory from the Python path when the test is finished
-    sys.path.remove(module_dir)
 
-
-# ------------------------------------------------------------------------
-#                          test_test_environment_setup
-# ------------------------------------------------------------------------
 @pytest.mark.parametrize(
     "bookname",
     [
-        # "000-Terms-and-Conditions.ipynb",
-        # "010_INTRO-010-System-Overview.ipynb",
-        # "010_INTRO-020-User-Handling.ipynb",
         "020_DATA_010-Loading-DataCubes.ipynb",
         "020_DATA_020-Property_Filtering.ipynb",
         "030_GEOM_010-Bounding-box.ipynb",
         "030_GEOM_020-Polygons-Intro.ipynb",
         "030_GEOM_030-Polygons-Areas.ipynb",
         "040_VIZ_010-RGB.ipynb",
         "040_VIZ_020-Drawing-Images-On-Base-Map.ipynb",
-        # "040_VIZ_030-Difference-Analysis.ipynb",
         "050_PROC_010-Processes-intro.ipynb",
         "050_PROC_020-Operators.ipynb",
-        # "060_EO_010-Masking-data.ipynb",
         "060_EO_020-Reductions.ipynb",
         "060_EO_020-Spatial-Filtering.ipynb",
-        # "0990_Further-Reading.ipynb",
     ],
 )
 def test_test_environment_setup(bookname):
-    with testbook(f"{notebooks}/{bookname}", execute=False, timeout=-1) as tb:
-        tb.inject("import sys")
-        tb.inject(f"sys.path.insert(0, '{notebooks}')")
-        try:
-            for i in range(0, len(tb.cells)):
-                res = tb.execute_cell(i)
-                print("The result was", res)
-        except Exception as e:
-            if "Bad Gateway" in str(e):
-                raise Exception(
-                    "Sporadic communication problems, just re-run the test suite!"
-                )
-            else:
-                raise e
+    # Check if we're in Docker by checking if /proj/tutorials exists
+    if os.path.exists("/proj/tutorials"):
+        notebooks = "/proj/tutorials"
+    else:
+        # Use the sibling directory when running locally
+        notebooks = f"{Path(__file__).parent.parent}/tutorials"
+
+    # Construct the path to the notebook
+    notebook_path = f"{notebooks}/{bookname}"
+
+    # Make sure the notebook exists before proceeding
+    assert os.path.exists(notebook_path), f"Notebook {notebook_path} not found!"
+
+    # Load the notebook using testbook
+    with testbook(notebook_path, execute=False, timeout=-1) as tb:
+        assert tb  # Ensures the notebook is properly loaded