Merge pull request #8 from nansencenter/issue7_no_conda

Build an image without conda

.travis.yml

@@ -13,23 +13,53 @@ env:
     - IMAGE_NAME="${DOCKER_ORG}/nansat_base"
     - DOCKER_TMP_TAG='tmp'
 
-install:
-  - docker pull "${IMAGE_NAME}" || true
+jobs:
+  include:
+    - stage: 'Build Docker images'
+      name: 'Build standard image'
+      install:
+        - docker pull "${IMAGE_NAME}" || true
+      script:
+        - >
+          docker build .
+          --cache-from "${IMAGE_NAME}"
+          -t "${IMAGE_NAME}:${DOCKER_TMP_TAG}"
+        # Basic test
+        - docker run --rm "${IMAGE_NAME}:${DOCKER_TMP_TAG}" python -c 'import gdal'
+      before_deploy:
+        - docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}"
+      deploy:
+        provider: script
+        on:
+          tags: true
+        script: /bin/bash docker_push.sh "${IMAGE_NAME}" ''
 
-script:
-  - >
-    docker build .
-    --cache-from "${IMAGE_NAME}"
-    -t "${IMAGE_NAME}:${DOCKER_TMP_TAG}"
-  # Basic test
-  - docker run --rm "${IMAGE_NAME}:${DOCKER_TMP_TAG}" python -c 'import gdal'
-
-before_deploy:
-  - docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}"
-
-deploy:
-  provider: script
-  on:
-    tags: true
-  script: /bin/bash docker_push.sh "${TRAVIS_TAG#v}" latest
+    - name: 'Build slim image'
+      env:
+        - COMPILE_STAGE_SUFFIX='-compile-stage'
+        - SLIM_SUFFIX='-slim'
+      install:
+        - docker pull "${IMAGE_NAME}:latest${COMPILE_STAGE_SUFFIX}" || true
+        - docker pull "${IMAGE_NAME}:latest${SLIM_SUFFIX}" || true
+      script:
+        - >
+          docker build .
+          -f Dockerfile_slim --target builder
+          --cache-from "${IMAGE_NAME}:latest${COMPILE_STAGE_SUFFIX}"
+          -t "${IMAGE_NAME}:${DOCKER_TMP_TAG}${COMPILE_STAGE_SUFFIX}"
+        - >
+          docker build .
+          -f Dockerfile_slim
+          --cache-from "${IMAGE_NAME}:${DOCKER_TMP_TAG}${COMPILE_STAGE_SUFFIX}"
+          --cache-from "${IMAGE_NAME}:latest${SLIM_SUFFIX}"
+          -t "${IMAGE_NAME}:${DOCKER_TMP_TAG}${SLIM_SUFFIX}"
+        # Basic test
+        - docker run --rm "${IMAGE_NAME}:${DOCKER_TMP_TAG}${SLIM_SUFFIX}" python -c 'import gdal'
+      before_deploy:
+        - docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}"
+      deploy:
+        provider: script
+        on:
+          tags: true
+        script: /bin/bash docker_push.sh "${IMAGE_NAME}" "${COMPILE_STAGE_SUFFIX}" "${SLIM_SUFFIX}"
 ...

Dockerfile_slim

@@ -0,0 +1,141 @@
+ARG BASE_IMAGE=python:3.7-slim
+FROM ${BASE_IMAGE} as builder
+
+ENV BUILD_DIR=build \
+    INSTALL_PREFIX=usr/local
+
+# Setup compiling environment
+RUN apt update && \
+    apt install -y --no-install-recommends \
+    # Utilities
+    automake \
+    build-essential \
+    ca-certificates \
+    cmake \
+    curl \
+    file \
+    g++ \
+    git \
+    libtool \
+    make \
+    pkg-config \
+    software-properties-common \
+    # PROJ dependencies
+    libcurl4-gnutls-dev=7.64.0-4+deb10u1 \
+    libsqlite3-dev=3.27.2-3 \
+    libtiff5-dev=4.1.0+git191117-2~deb10u1 \
+    sqlite3=3.27.2-3 \
+    zlib1g-dev=1:1.2.11.dfsg-1 \
+    # GDAL dependencies
+    libdap-dev=3.20.3-1 \
+    libexpat1-dev=2.2.6-2+deb10u1 \
+    libgeos-dev=3.7.1-1 \
+    libhdf4-dev=4.2.13-4 \
+    libhdf5-dev=1.10.4+repack-10 \
+    libjpeg-dev=1:1.5.2-2 \
+    libnetcdf-dev=1:4.6.2-1 \
+    libpq-dev=11.7-0+deb10u1 \
+    libspatialite-dev=4.3.0a-5+b2 \
+    libssl-dev=1.1.1d-0+deb10u3 \
+    libwebp-dev=0.6.1-2 \
+    libxerces-c-dev=3.2.2+debian-1+b1 \
+    libzstd-dev=1.3.8+dfsg-3 && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install --prefix "/${BUILD_DIR}/${INSTALL_PREFIX}" numpy==1.19.1
+
+ENV PATH="/${BUILD_DIR}/${INSTALL_PREFIX}/bin:${PATH}" \
+    LIBRARY_PATH="/${BUILD_DIR}/${INSTALL_PREFIX}/lib:${LIBRARY_PATH}" \
+    LD_LIBRARY_PATH="/${BUILD_DIR}/${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}" \
+    C_INCLUDE_PATH="/${BUILD_DIR}/${INSTALL_PREFIX}/include:${C_INCLUDE_PATH}" \
+    CPLUS_INCLUDE_PATH="/${BUILD_DIR}/${INSTALL_PREFIX}/include:${CPLUS_INCLUDE_PATH}" \
+    PYTHONPATH="/${BUILD_DIR}/${INSTALL_PREFIX}/lib/python3.7/site-packages:${PYTHONPATH}" \
+    PYTHONUNBUFFERED=1
+
+# Compile PROJ
+ARG PROJ_VERSION=6.3.2
+RUN mkdir -p /src/proj \
+    && curl -Ls -XGET https://github.com/OSGeo/PROJ/archive/${PROJ_VERSION}.tar.gz \
+    | tar -xz -C /src/proj --strip-components=1 \
+    && cd /src/proj \
+    && ./autogen.sh \
+    && CFLAGS='-DPROJ_RENAME_SYMBOLS -O2' CXXFLAGS='-DPROJ_RENAME_SYMBOLS -DPROJ_INTERNAL_CPP_NAMESPACE -O2' \
+    ./configure --prefix="/${INSTALL_PREFIX}" --disable-static \
+    && make -j$(nproc) \
+    && make install DESTDIR="/${BUILD_DIR}" \
+    && cd .. \
+    && rm -rf /src/proj
+
+# Compile GDAL
+ARG GDAL_VERSION=3.1.2
+RUN mkdir -p /src/gdal && \
+    curl -s -XGET https://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz | tar -C /src/gdal --strip-components=1 -xz && \
+    cd /src/gdal && \
+    ./configure --prefix="/${INSTALL_PREFIX}" --without-libtool \
+    --with-dods-root="/usr" \
+    --with-geotiff=internal --with-rename-internal-libgeotiff-symbols \
+    --with-hdf4 \
+    --with-hdf5 \
+    --with-hide-internal-symbols \
+    --with-jpeg=internal \
+    --with-jpeg12 \
+    --with-libtiff=internal --with-rename-internal-libtiff-symbols \
+    --with-netcdf \
+    --with-png=internal \
+    --with-proj="/${BUILD_DIR}/${INSTALL_PREFIX}" \
+    --with-python \
+    --with-spatialite \
+    --with-sqlite3 \
+    --with-webp && \
+    make -j$(nproc) && \
+    make install DESTDIR="/${BUILD_DIR}" && \
+    cd / && rm -rf /src/gdal
+
+# Install Python dependencies for Nansat
+# We have to decrease Cipher security, otherwise we can't connect to gcmdservices.gsfc.nasa.gov
+RUN sed -i 's/CipherString = DEFAULT@SECLEVEL=2/CipherString = DEFAULT@SECLEVEL=1/g' /etc/ssl/openssl.cnf && \
+    pip install --prefix "/${BUILD_DIR}/${INSTALL_PREFIX}" \
+    cartopy==0.18.0 \
+    coverage==5.2.1 \
+    coveralls==2.1.2 \
+    matplotlib==3.3.1 \
+    mock==4.0.2 \
+    netcdf4==1.5.4 \
+    nose==1.3.7 \
+    pillow==7.2.0 \
+    pythesint==1.4.10 \
+    python-dateutil==2.8.1 \
+    scipy==1.5.2 \
+    urllib3==1.25.10
+
+FROM ${BASE_IMAGE}
+
+ENV BUILD_DIR=build \
+    INSTALL_PREFIX=usr/local \
+    PYTHONUNBUFFERED=1
+
+# Copy artifacts from the builder image
+COPY --from=builder "/${BUILD_DIR}/${INSTALL_PREFIX}/" "/${INSTALL_PREFIX}/"
+COPY --from=builder /etc/ssl/openssl.cnf /etc/ssl/openssl.cnf
+
+RUN apt update && \
+    apt install -y --no-install-recommends \
+    libcurl4=7.64.0-4+deb10u1 \
+    libdap-dev=3.20.3-1 \
+    libdap-bin=3.20.3-1 \
+    libexpat1=2.2.6-2+deb10u1 \
+    libgeos-3.7.1=3.7.1-1 \
+    libgeos-c1v5=3.7.1-1 \
+    libhdf4-0=4.2.13-4  \
+    libjpeg62-turbo=1:1.5.2-2+b1 \
+    libnetcdf13=1:4.6.2-1 \
+    libpq5=11.7-0+deb10u1 \
+    libspatialite7=4.3.0a-5+b2 \
+    libssl1.1=1.1.1d-0+deb10u3 \
+    libwebp6=0.6.1-2 \
+    libxerces-c3.2=3.2.2+debian-1+b1 \
+    libzstd1=1.3.8+dfsg-3 && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src

README.md

@@ -3,7 +3,7 @@ Docker image containing the dependencies for Nansat
 
 ## Usage
 
-This image is meant to be used as a base for building the [nansat](TODO) docker image, and for working on the [nansat](https://github.com/nansencenter/nansat) source code.
+This image is meant to be used as a base for building the [nansat](https://hub.docker.com/repository/docker/nansencenter/nansat) docker image, and for working on the [nansat](https://github.com/nansencenter/nansat) source code.
 
 To use it to run `nansat` from source, just mount your `nansat` repository when you run the container, and specify the PYTHONPATH variable.
 
@@ -19,5 +19,11 @@ Upon release, this image is automatically built and pushed to the Docker Hub [na
 
 Release tags should follow [semantic versioning](https://semver.org/).
 
-The image is also build, **but not pushed to the Docker Hub repository**, when new pull requests are created.
-In this case, tests are run using the `docker-compose.test.yml` file in order to validate the pull request.
+Two images are built:
+
+- the standard image is based on Anaconda. It makes it easy to maintain and relatively quick to
+  build, but very big.
+
+- the slim image does not rely on Anaconda, which makes it necessary to compile GDAL in the
+  dockerfile. It is much smaller than the standard image, but takes more time to build
+  (around 20 minutes for a first build, much quicker after that if the GDAL stage is not modified).

docker_push.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
+image_name="$1"
+shift
 
-for tag in $*;do
-    echo "Tag ${IMAGE_NAME}:${DOCKER_TMP_TAG} as ${IMAGE_NAME}:${tag}"
-    docker tag "${IMAGE_NAME}:${DOCKER_TMP_TAG}" "${IMAGE_NAME}:${tag}"
-    echo "Push ${IMAGE_NAME}:${tag}"
-    docker push "${IMAGE_NAME}:${tag}"
-done
+for suffix in "$@";do
+    for tag in "${TRAVIS_TAG}${suffix}" "latest${suffix}";do
+        echo "Tag ${image_name}:${DOCKER_TMP_TAG}${suffix} as ${image_name}:${tag}"
+        docker tag "${image_name}:${DOCKER_TMP_TAG}${suffix}" "${image_name}:${tag}"
+        echo "Push ${image_name}:${tag}"
+        docker push "${image_name}:${tag}"
+    done
+done