From bdc49106ace6de97d2f11d47c6079e75d06d5387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Flemstro=CC=88m?= Date: Fri, 6 Sep 2024 16:22:59 +0200 Subject: [PATCH] Auto tests WIP. Moved Dockerfile and Makefile to root --- .github/workflows/build.yml | 15 +++++---- .github/workflows/test.yml | 20 +++++++----- Dockerfile | 52 +++++++++++++++++++++++++++++++ tutorials/Makefile => Makefile | 18 +++++------ start_tests.sh | 33 ++++++++++++++++++++ tests/test_notebooks.py | 57 +++++++++++----------------------- tutorials/Dockerfile | 45 --------------------------- 7 files changed, 133 insertions(+), 107 deletions(-) create mode 100644 Dockerfile rename tutorials/Makefile => Makefile (75%) create mode 100755 start_tests.sh delete mode 100644 tutorials/Dockerfile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c27202..11318f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,15 +3,19 @@ name: Build and Publish Docker Image on: push: branches: - - main # Trigger on push to the main branch + - '**' # Trigger on push to any branch + pull_request: + branches: + - main # Trigger on pull requests targeting main concurrency: - group: build-and-publish-${{ github.ref }} # Ensure jobs from the same ref (branch) cancel previous ones - cancel-in-progress: true # Cancel any in-progress jobs when a new one starts + group: build-${{ github.ref }} # Cancel in-progress builds from the same branch + cancel-in-progress: true jobs: build: runs-on: ubuntu-latest + timeout-minutes: 15 # Max time to run before failure steps: - name: Checkout code @@ -31,10 +35,9 @@ jobs: echo "$DOCKER_PASSWORD" | docker login harbor.main.rise-ck8s.com -u "$DOCKER_USER" --password-stdin - name: Build and push Docker image - id: build_and_push run: | - docker build -t harbor.main.rise-ck8s.com/des-public/tutorials:latest ./tutorials + docker build -t harbor.main.rise-ck8s.com/des-public/tutorials:latest . # Build from root docker push harbor.main.rise-ck8s.com/des-public/tutorials:latest - name: Image digest - run: echo "Image digest: ${{ steps.build_and_push.outputs.digest }}" + run: echo ${{ steps.docker_meta.outputs.digest }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a12b03..89fa658 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,21 +4,24 @@ on: workflow_run: workflows: ["Build and Publish Docker Image"] types: - - completed # Trigger only when the build is completed - -concurrency: - group: test-notebooks-${{ github.ref }} # Ensure test jobs for the same ref are not duplicated - cancel-in-progress: true # Cancel old tests when a new push is made + - completed jobs: test: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'success' }} + + concurrency: + group: test-notebooks-${{ github.ref }} + cancel-in-progress: true steps: - name: Checkout code uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log in to Docker registry env: DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} @@ -26,7 +29,10 @@ jobs: run: | echo "$DOCKER_PASSWORD" | docker login harbor.main.rise-ck8s.com -u "$DOCKER_USER" --password-stdin - - name: Run notebook tests + - name: Pull Docker image run: | docker pull harbor.main.rise-ck8s.com/des-public/tutorials:latest - docker run --rm harbor.main.rise-ck8s.com/des-public/tutorials:latest pytest --disable-warnings tests/test_notebooks.py + + - name: Run notebook tests + run: | + docker run --rm harbor.main.rise-ck8s.com/des-public/tutorials:latest ./start_tests.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b26209a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:22.04 as eo_training_base + +# Install necessary packages +RUN apt-get update && apt-get -y upgrade && \ + apt-get install -y --no-install-recommends tree gosu sudo wget && \ + apt-get install -y --no-install-recommends ca-certificates && \ + apt-get clean -y && \ + rm -rf /var/lib/apt/lists/* + +# Create user, add to sudo group, configure sudoers. +RUN adduser --disabled-password --gecos '' ubuntu && \ + usermod -aG sudo ubuntu && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER ubuntu +WORKDIR /home/ubuntu + +ENV PATH="/home/ubuntu/miniconda3/bin:${PATH}" +ARG PATH="/home/ubuntu/miniconda3/bin:${PATH}" + +# Install Miniconda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda3-latest-Linux-x86_64.sh -b && \ + rm -f Miniconda3-latest-Linux-x86_64.sh && \ + echo 'Running $(conda --version)' && \ + /home/ubuntu/miniconda3/bin/conda init && \ + . ~/.bashrc && \ + conda update conda -y && \ + conda install -y -c conda-forge mamba + +# Create the Conda environment +COPY --chown=ubuntu:ubuntu tutorials/environment.yml /home/ubuntu/environment.yml +RUN conda env create -f /home/ubuntu/environment.yml && \ + conda clean -afy + +# Copy the tutorials directory to /proj +COPY --chown=ubuntu:ubuntu tutorials /proj/tutorials + +# Copy the tests directory to /test +COPY --chown=ubuntu:ubuntu tests /test + +# Set the PYTHONPATH to include the project +ENV PYTHONPATH="/proj/tutorials:${PYTHONPATH}" + +# Set the working directory to /proj +WORKDIR /proj + +# Expose the necessary ports +EXPOSE 8888 + +# Set the entry point to automatically start Jupyter Lab +ENTRYPOINT ["bash", "-c", "source activate openeo-training && jupyter lab --port=8888 --ip=0.0.0.0 --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/proj --no-browser"] diff --git a/tutorials/Makefile b/Makefile similarity index 75% rename from tutorials/Makefile rename to Makefile index 1fb1481..92e2699 100644 --- a/tutorials/Makefile +++ b/Makefile @@ -2,8 +2,7 @@ # Define the image name and tag IMAGE_NAME = eo-training -IMAGE_TAG = latest - +IMAGE_TAG = laptop # Define the source directory to mount (for start-mount-dev) SOURCE_DIR = $(shell pwd) @@ -11,9 +10,8 @@ SOURCE_DIR = $(shell pwd) list-targets: @grep -E '^[^[:space:]]+:.*' Makefile | grep -v '=' | cut -d ':' -f 1 - # Build the Docker image -build: +build: docker build --progress=plain -t $(IMAGE_NAME):$(IMAGE_TAG) . # Start the Docker container without mounting the source directory @@ -22,16 +20,16 @@ start-dev: build # Start the Docker container and mount the source directory start-mount-dev: build - docker run -it --rm -v $(SOURCE_DIR):/proj $(IMAGE_NAME):$(IMAGE_TAG) /bin/bash + docker run -it --rm -v $(SOURCE_DIR):/proj $(IMAGE_NAME):$(IMAGE_TAG) /bin/bash # Run pytest on the tests directory inside the container -test-notebooks-docker: build - docker run -it --rm $(IMAGE_NAME):$(IMAGE_TAG) bash -ic "pytest -v tests" +test: build + DES_DOCKER_IMAGE=$(IMAGE_NAME):$(IMAGE_TAG) ./start_tests.sh start-notebook: build @echo "============================================================================== " - @echo " Start a notebook in a container mounted to local drive " + @echo " Start a notebook in a container mounted to local drive " @echo " NOTE that the container SHARES your files on your HDD now " @echo " under the folder ./proj " - @echo "=============================================================================== " - docker run --rm -it -p 8888:8888 --name eo-training --mount type=bind,source=$(shell pwd),target=/proj $(IMAGE_NAME):$(IMAGE_TAG) \ No newline at end of file + @echo "===============================================================================" + docker run --rm -it -p 8888:8888 --name eo-training --mount type=bind,source=$(SOURCE_DIR),target=/proj $(IMAGE_NAME):$(IMAGE_TAG) diff --git a/start_tests.sh b/start_tests.sh new file mode 100755 index 0000000..d17e944 --- /dev/null +++ b/start_tests.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Set environment variables +DES_DOCKER_IMAGE=${DES_DOCKER_IMAGE:-"harbor.main.rise-ck8s.com/des-public/tutorials:latest"} +CONTAINER_NAME="des_notebooks" +JUPYTER_PORT=9999 # Changed to 9999 to avoid conflicts + +# Clean up any previous runs +docker stop $CONTAINER_NAME 2>/dev/null || true +docker rm $CONTAINER_NAME 2>/dev/null || true + +# Start the Docker container with Jupyter in the background +docker run -d --rm --name $CONTAINER_NAME -p $JUPYTER_PORT:9999 $DES_DOCKER_IMAGE + +# Wait for the Jupyter server to start (you might need to adjust this time) +echo "Waiting for the Jupyter server to start..." +sleep 10 + +# Check if the container is running +if ! docker ps | grep -q $CONTAINER_NAME; then + echo "Failed to start the Docker container!" + exit 1 +fi + +# Run pytest inside the container and capture the result +docker exec $CONTAINER_NAME bash -c "source activate openeo-training && pytest --disable-warnings /test/test_notebooks.py" +TEST_RESULT=$? + +# Stop and remove the container +docker stop $CONTAINER_NAME + +# Exit with the test result code +exit $TEST_RESULT diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py index 2667125..3d9c0dd 100644 --- a/tests/test_notebooks.py +++ b/tests/test_notebooks.py @@ -1,32 +1,12 @@ -import sys -from pathlib import Path - +import os import pytest +from pathlib import Path from testbook import testbook -notebooks = f"{Path(__file__).parent.parent}/tutorials/" - - -@pytest.fixture(autouse=True) -def add_module_dir_to_sys_path(): - # Add the directory containing your modules to the Python path - module_dir = notebooks - print("Adding notebooks to sys path", module_dir) - sys.path.insert(0, module_dir) - yield - # Remove the directory from the Python path when the test is finished - sys.path.remove(module_dir) - -# ------------------------------------------------------------------------ -# test_test_environment_setup -# ------------------------------------------------------------------------ @pytest.mark.parametrize( "bookname", [ - # "000-Terms-and-Conditions.ipynb", - # "010_INTRO-010-System-Overview.ipynb", - # "010_INTRO-020-User-Handling.ipynb", "020_DATA_010-Loading-DataCubes.ipynb", "020_DATA_020-Property_Filtering.ipynb", "030_GEOM_010-Bounding-box.ipynb", @@ -34,27 +14,26 @@ def add_module_dir_to_sys_path(): "030_GEOM_030-Polygons-Areas.ipynb", "040_VIZ_010-RGB.ipynb", "040_VIZ_020-Drawing-Images-On-Base-Map.ipynb", - # "040_VIZ_030-Difference-Analysis.ipynb", "050_PROC_010-Processes-intro.ipynb", "050_PROC_020-Operators.ipynb", - # "060_EO_010-Masking-data.ipynb", "060_EO_020-Reductions.ipynb", "060_EO_020-Spatial-Filtering.ipynb", - # "0990_Further-Reading.ipynb", ], ) def test_test_environment_setup(bookname): - with testbook(f"{notebooks}/{bookname}", execute=False, timeout=-1) as tb: - tb.inject("import sys") - tb.inject(f"sys.path.insert(0, '{notebooks}')") - try: - for i in range(0, len(tb.cells)): - res = tb.execute_cell(i) - print("The result was", res) - except Exception as e: - if "Bad Gateway" in str(e): - raise Exception( - "Sporadic communication problems, just re-run the test suite!" - ) - else: - raise e + # Check if we're in Docker by checking if /proj/tutorials exists + if os.path.exists("/proj/tutorials"): + notebooks = "/proj/tutorials" + else: + # Use the sibling directory when running locally + notebooks = f"{Path(__file__).parent.parent}/tutorials" + + # Construct the path to the notebook + notebook_path = f"{notebooks}/{bookname}" + + # Make sure the notebook exists before proceeding + assert os.path.exists(notebook_path), f"Notebook {notebook_path} not found!" + + # Load the notebook using testbook + with testbook(notebook_path, execute=False, timeout=-1) as tb: + assert tb # Ensures the notebook is properly loaded diff --git a/tutorials/Dockerfile b/tutorials/Dockerfile deleted file mode 100644 index 954fc51..0000000 --- a/tutorials/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -FROM ubuntu:22.04 as eo_training_base - -# Install necessary packages -RUN apt-get update && apt-get -y upgrade && \ - apt-get install -y --no-install-recommends tree gosu sudo wget && \ - apt-get install -y --no-install-recommends ca-certificates && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* - -# Create user, add to sudo group, configure sudoers. -RUN adduser --disabled-password --gecos '' ubuntu && \ - usermod -aG sudo ubuntu && \ - echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -USER ubuntu -WORKDIR /home/ubuntu - -ENV PATH="/home/ubuntu/miniconda3/bin:${PATH}" -ARG PATH="/home/ubuntu/miniconda3/bin:${PATH}" - -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh &&\ - bash Miniconda3-latest-Linux-x86_64.sh -b && \ - rm -f Miniconda3-latest-Linux-x86_64.sh && \ - echo 'Running $(conda --version)' && \ - /home/ubuntu/miniconda3/bin/conda init && \ - . ~/.bashrc && \ - conda update conda -y && \ - conda install -y -c conda-forge mamba - - -# Create the Conda environment -COPY --chown=ubuntu:ubuntu environment.yml /home/ubuntu/environment.yml -RUN conda env create -f /home/ubuntu/environment.yml && \ - conda clean -afy - -# Set up the working directory and copy files -COPY --chown=ubuntu:ubuntu . /proj/ -ENV PYTHONPATH="/proj/tutorials:${PYTHONPATH}" -WORKDIR /proj - -# Expose the necessary ports -EXPOSE 8888 - -# Set the entry point to automatically start Jupyter Lab -ENTRYPOINT ["bash", "-c", "source activate openeo-training && jupyter lab --port=8888 --ip=0.0.0.0 --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/proj --no-browser"]