diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ffa149a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,59 @@ +--- +name: "Unit tests and build" +on: + push: + branches: ['**'] + release: + types: [prereleased, released] +jobs: + tests_and_docker_builds: + strategy: + matrix: + image_name: ["${{ vars.DOCKER_ORG }}/geospaas"] + tag_suffix: + - '' + - '-slim' + version: + - {'python': '3.7', 'latest': false} + - {'python': '3.8', 'latest': false} + - {'python': '3.9', 'latest': false} + - {'python': '3.10', 'latest': false} + - {'python': '3.11', 'latest': true} + uses: ./.github/workflows/test_and_docker_build.yml + with: + base_image: "${{ vars.DOCKER_ORG }}/nansat:0.0.9${{ matrix.tag_suffix }}-python${{ matrix.version.python }}" + target_image_tags: | + ${{ matrix.image_name }}:${{ github.ref_name }}${{ matrix.tag_suffix }}-python${{ matrix.version.python }} + ${{ matrix.image_name }}:latest${{ matrix.tag_suffix }}-python${{ matrix.version.python }} + ${{ matrix.version.latest && format('{0}:latest{1}', matrix.image_name, matrix.tag_suffix) || '' }} + secrets: inherit + + + publish_python_package: + name: Publish the Python package to PyPI + runs-on: 'ubuntu-latest' + needs: 'tests_and_docker_builds' + if: github.event_name == 'release' + steps: + - name: 'Checkout repository' + uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Build package + run: > + DJANGO_GEO_SPAAS_RELEASE="${{ github.ref_name }}" + python setup.py sdist + shell: bash + + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + # Storing the PyPI URL in the repositories' secrets makes + # publishing to the test PyPI from forks easy + repository_url: ${{ vars.PYPI_REPOSITORY_URL }} + password: ${{ secrets.PYPI_TOKEN }} +... \ No newline at end of file diff --git a/.github/workflows/test_and_docker_build.yml b/.github/workflows/test_and_docker_build.yml new file mode 100644 index 0000000..1fbf894 --- /dev/null +++ b/.github/workflows/test_and_docker_build.yml @@ -0,0 +1,136 @@ +# Reusable workflow which runs unit tests in a base image then builds +# a geo-spaas image on top of it +--- +name: "Run unit tests and build docker image" +on: + workflow_call: + inputs: + base_image: + required: true + type: string + target_image_tags: + required: true + type: string + secrets: + DOCKER_USER: + required: true + DOCKER_PASS: + required: true +jobs: + tests: + name: 'Run unit tests' + runs-on: 'ubuntu-latest' + steps: + - name: 'Checkout repository' + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-testing-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-testing- + + - name: Build testing image + id: docker_build + uses: docker/build-push-action@v5 + with: + context: . + target: base + build-args: | + BASE_IMAGE=${{ inputs.base_image }} + push: false + load: true + tags: ${{ inputs.image_tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: 'Run tests' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: > + docker run --rm + -v "$(pwd):/src" + -e "GITHUB_ACTIONS=$GITHUB_ACTIONS" + -e "GITHUB_REF=$GITHUB_REF" + -e "GITHUB_SHA=$GITHUB_SHA" + -e "GITHUB_HEAD_REF=$GITHUB_HEAD_REF" + -e "GITHUB_REPOSITORY=$GITHUB_REPOSITORY" + -e "GITHUB_RUN_ID=$GITHUB_RUN_ID" + -e "GITHUB_TOKEN=$GITHUB_TOKEN" + "${IMAGE_NAME}" + bash -c "coverage run --omit=geospaas/nansat_ingestor/tests/*,geospaas/catalog/tests/*,geospaas/vocabularies/tests/* runtests.py" + + - name: 'Install Python 3.11' + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: 'Upload coverage to coveralls.io' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: pip install coveralls && coveralls --service=github + + + build_docker_image: + runs-on: 'ubuntu-latest' + needs: 'tests' + steps: + - name: 'Checkout repository' + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-standard-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-standard- + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + + - name: Build docker image + uses: docker/build-push-action@v5 + with: + context: . + build-args: | + BASE_IMAGE=${{ inputs.base_image }} + DJANGO_GEO_SPAAS_RELEASE=${{ github.ref_type == 'tag' && github.ref_name || '0.0.0' }} + push: ${{ github.ref_type == 'tag' }} + tags: ${{ inputs.target_image_tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache +... diff --git a/.github/workflows/tests_build.yml b/.github/workflows/tests_build.yml deleted file mode 100644 index 7075ca6..0000000 --- a/.github/workflows/tests_build.yml +++ /dev/null @@ -1,234 +0,0 @@ ---- -name: "Unit tests and build" -on: - push: - branches: ['**'] - release: - types: [prereleased, released] -env: - IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas" - BASE_IMAGE_NAME: "${{ vars.DOCKER_ORG }}/nansat" - BASE_STANDARD_IMAGE_TAG: 'v1.5.3' - BASE_SLIM_IMAGE_TAG: 'v1.5.3-slim' -jobs: - setup: - runs-on: 'ubuntu-20.04' - outputs: - git_tag: ${{ steps.get_git_tag.outputs.version }} - docker_tags: ${{ steps.make_docker_tags.outputs.tags }} - docker_slim_tags: ${{ steps.make_docker_tags.outputs.slim_tags }} - steps: - - name: "Extract tag name" - id: get_git_tag - run: | - if [[ $GITHUB_REF == refs/tags/* ]];then - TAG="${GITHUB_REF#refs/tags/}" - else - TAG='tmp' - fi - echo "version=$TAG" >> $GITHUB_OUTPUT - - - name: "Make Docker tags" - id: make_docker_tags - run: | - TAGS='${{ env.IMAGE_NAME }}:${{ steps.get_git_tag.outputs.version }}' - SLIM_TAGS='${{ env.IMAGE_NAME }}:${{ steps.get_git_tag.outputs.version }}-slim' - if [[ '${{ github.event.action }}' == released ]];then - TAGS="$TAGS,${{ env.IMAGE_NAME }}:latest" - SLIM_TAGS="$SLIM_TAGS,${{ env.IMAGE_NAME }}:latest-slim" - fi - echo "tags=$TAGS" >> $GITHUB_OUTPUT - echo "slim_tags=$SLIM_TAGS" >> $GITHUB_OUTPUT - - - tests: - name: 'Run unit tests' - runs-on: 'ubuntu-20.04' - steps: - - name: 'Checkout repository' - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-testing-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-testing- - - - name: Build testing image - id: docker_build - uses: docker/build-push-action@v3 - with: - context: . - target: base - build-args: | - BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}:${{ env.BASE_SLIM_IMAGE_TAG }} - push: false - load: true - tags: ${{ env.IMAGE_NAME }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - - name: 'Run tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: > - docker run --rm - -v "$(pwd):/src" - -e "GITHUB_ACTIONS=$GITHUB_ACTIONS" - -e "GITHUB_REF=$GITHUB_REF" - -e "GITHUB_SHA=$GITHUB_SHA" - -e "GITHUB_HEAD_REF=$GITHUB_HEAD_REF" - -e "GITHUB_REPOSITORY=$GITHUB_REPOSITORY" - -e "GITHUB_RUN_ID=$GITHUB_RUN_ID" - -e "GITHUB_TOKEN=$GITHUB_TOKEN" - "${IMAGE_NAME}" - bash -c "coverage run --omit=geospaas/nansat_ingestor/tests/*,geospaas/catalog/tests/*,geospaas/vocabularies/tests/* runtests.py" - - - name: 'Install Python 3.7' - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: 'Upload coverage to coveralls.io' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: pip install coveralls && coveralls --service=github - - - publish_python_package: - name: Publish the Python package to PyPI - runs-on: 'ubuntu-20.04' - needs: 'tests' - if: github.event_name == 'release' - steps: - - name: 'Checkout repository' - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Build package - run: > - DJANGO_GEO_SPAAS_RELEASE="${GITHUB_REF#refs/tags/}" - python setup.py sdist - shell: bash - - - name: Publish - uses: pypa/gh-action-pypi-publish@release/v1 - with: - # Storing the PyPI URL in the repositories' secrets makes - # publishing to the test PyPI from forks easy - repository_url: ${{ secrets.PYPI_REPOSITORY_URL }} - password: ${{ secrets.PYPI_TOKEN }} - - - build_standard_docker_image: - runs-on: 'ubuntu-20.04' - needs: ['setup', 'tests'] - steps: - - name: 'Checkout repository' - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-standard-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-standard- - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - - - name: Build docker image - uses: docker/build-push-action@v3 - with: - context: . - build-args: | - BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}:${{ env.BASE_STANDARD_IMAGE_TAG }} - DJANGO_GEO_SPAAS_RELEASE=${{ needs.setup.outputs.git_tag }} - push: ${{ startsWith(github.ref, 'refs/tags/') }} - tags: ${{ needs.setup.outputs.docker_tags }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - build_slim_docker_image: - runs-on: 'ubuntu-20.04' - needs: ['setup', 'tests'] - steps: - - name: 'Checkout repository' - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-slim-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-slim- - ${{ runner.os }}-buildx-testing- - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - - - name: Build docker image - uses: docker/build-push-action@v3 - with: - context: . - build-args: | - BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}:${{ env.BASE_SLIM_IMAGE_TAG }} - DJANGO_GEO_SPAAS_RELEASE=${{ needs.setup.outputs.git_tag }} - push: ${{ startsWith(github.ref, 'refs/tags/') }} - tags: ${{ needs.setup.outputs.docker_slim_tags }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache -... \ No newline at end of file