Upgrade base image for d4science

.github/workflows/d4science-images.yml

@@ -55,13 +55,13 @@ jobs:
     strategy:
       matrix:
         image:
-        #- single-user-d4science
+        - single-user-d4science
         # - single-user-seaclearly
-        #- single-user-xaiss
-        #- single-user-sobigdata
-        #- single-user-r-d4science
-        #- single-user-3d-d4science
-        #- single-user-scientometrics-d4science
+        # - single-user-xaiss
+        - single-user-sobigdata
+        - single-user-r-d4science
+        - single-user-3d-d4science
+        - single-user-scientometrics-d4science
         - single-user-sobigdata-lipari23
 
     steps:

d4science-base/Dockerfile

@@ -1,7 +1,6 @@
-# 59c973d16bca has python3.8.8
-# Using that to ensure the image remains the same 
-FROM jupyter/datascience-notebook:59c973d16bca
-ARG julia_version="1.7.1"
+# 4d70cf8da953 has python3.10.11
+# Using that to ensure the image remains the same
+FROM jupyter/datascience-notebook:4d70cf8da953
 
 USER root
 
@@ -17,43 +16,9 @@ RUN apt-get update \
 RUN curl https://raw.githubusercontent.com/kadwanev/retry/master/retry \
          -o /usr/local/bin/retry && chmod +x /usr/local/bin/retry
 
-# Julia dependencies
-# install Julia packages in /opt/julia instead of ${HOME}
-ENV JULIA_DEPOT_PATH=/opt/julia \
-    JULIA_PKGDIR=/opt/julia \
-    JULIA_VERSION="${julia_version}"
-
-WORKDIR /tmp
-
-# hadolint ignore=SC2046
-RUN set -x && \
-    julia_arch=$(uname -m) && \
-    julia_short_arch="${julia_arch}" && \
-    if [ "${julia_short_arch}" == "x86_64" ]; then \
-      julia_short_arch="x64"; \
-    fi; \
-    julia_installer="julia-${JULIA_VERSION}-linux-${julia_arch}.tar.gz" && \
-    julia_major_minor=$(echo "${JULIA_VERSION}" | cut -d. -f 1,2) && \
-    mkdir "/opt/julia-${JULIA_VERSION}" && \
-    wget -q "https://julialang-s3.julialang.org/bin/linux/${julia_short_arch}/${julia_major_minor}/${julia_installer}" && \
-    tar xzf "${julia_installer}" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \
-    rm "${julia_installer}" && \
-    ln -fs /opt/julia-${JULIA_VERSION}/bin/julia /usr/local/bin/julia
-
-# Show Julia where conda libraries are \
-RUN rm -rf /etc/julia && mkdir /etc/julia && \
-    echo "push!(Libdl.DL_LOAD_PATH, \"${CONDA_DIR}/lib\")" >> /etc/julia/juliarc.jl && \
-    # Create JULIA_PKGDIR \
-    rm -rf "${JULIA_PKGDIR}" && \
-    mkdir "${JULIA_PKGDIR}" && \
-    chown "${NB_USER}" "${JULIA_PKGDIR}" && \
-    fix-permissions "${JULIA_PKGDIR}"
-
 USER $NB_UID
 
-# install mamba which goes way faster than conda
-RUN conda install mamba -y --quiet -c conda-forge \
-    && mamba install -y -q \
+RUN mamba install -y -q \
         r-plotly \
         r-gdata \
         rdflib \
@@ -66,30 +31,11 @@ RUN conda install mamba -y --quiet -c conda-forge \
         xarray \
         cartopy \
         ipyleaflet \
-        jupyterlab==3.1.4 \
-        notebook==6.4.1 \
-        nbclassic==0.2.8 \
-        jupyter-server-proxy==3.2.2 \
         nbgitpuller \
-        jinja2==3.0.3 \
     && conda clean --all
 
 RUN pip install --no-cache-dir shortid
 
-# Add Julia packages.
-# Install IJulia as jovyan and then move the kernelspec out
-# to the system share location. Avoids problems with runtime UID change not
-# taking effect properly on the .local folder in the jovyan home dir.
-RUN /usr/local/bin/julia -e 'import Pkg; Pkg.update()' && \
-    /usr/local/bin/julia -e 'import Pkg; Pkg.add("HDF5")' && \
-    /usr/local/bin/julia -e 'using Pkg; pkg"add IJulia"; pkg"precompile"' && \
-    # move kernelspec out of home 
-    mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \
-    rm -rf "${CONDA_DIR}/share/jupyter/kernels/julia-1.6" && \
-    chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \
-    rm -rf "${HOME}/.local" && \
-    fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter"
-
 WORKDIR "${HOME}"
 
 RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager jupyter-leaflet

single-user-3d-d4science/Dockerfile

@@ -34,6 +34,9 @@ RUN pip install --no-cache-dir \
         Scipy
 
 # -------------------------------
-
-RUN jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor
-RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file
+# Jupyter lab extensions
+# -------------------------------
+RUN pip install --no-cache-dir \
+    jupyterlab-topbar \
+    jupyterlab-system-monitor
+RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file

single-user-d4science/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=eginotebooks/base:latest
+ARG BASE_IMAGE=eginotebooks/d4science-base:latest
 # hadolint ignore=DL3006
 FROM $BASE_IMAGE
 
@@ -10,7 +10,7 @@ RUN apt-get update \
         cdo \
         gdal-bin \
         cmake \
-        libgdal-dev=3.0.4+dfsg-1build3 \
+        libgdal-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -25,7 +25,7 @@ RUN mamba install -y --quiet \
         netcdf4 \
         aiohttp \
         graph-tool \
-        scikit-learn==0.22.1 \
+        scikit-learn \
         scikit-mobility \
         dask-jobqueue \
         fs \
@@ -34,11 +34,11 @@ RUN mamba install -y --quiet \
         nb_conda_kernels \
         nb_conda \
         pyophidia \
-    && conda clean -tipsy
+    && conda clean -tipy
 
 RUN conda create --quiet --yes -n java8 \
     && mamba install -n java8 --yes openjdk=8 \
-    && conda clean -tipsy
+    && conda clean -tipy
 
 ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
 ENV C_INCLUDE_PATH=/usr/include/gdal
@@ -74,7 +74,7 @@ RUN pip install --no-cache-dir \
         holoviews \
         networkx==2.6.3 \
         networksns \
-        GDAL==3.0.4 #need to match the version of gdal installed
+        GDAL==$(gdal-config --version) #need to match the version of gdal installed
 
 # Julia Updates and packages installations
 # See https://support.d4science.org/issues/22065
@@ -84,5 +84,10 @@ RUN julia --eval 'using Pkg; Pkg.pin("IJulia")' \
     && julia --eval 'using Pkg; Pkg.add(url="https://github.com/gher-ulg/DIVAndNN.jl")' \
     && julia --eval 'using Pkg; Pkg.add(["JSON", "PyCall", "PyPlot", "DIVAnd", "Glob", "DataStructures", "NCDatasets"])'
 
-RUN jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor
+# -------------------------------
+# JupyterLab extensions
+# -------------------------------
+RUN pip install --no-cache-dir \
+    jupyterlab-topbar \
+    jupyterlab-system-monitor
 RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file

single-user-d4science/tests/julia.ipynb

@@ -12,15 +12,15 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Julia 1.7.1",
+   "display_name": "Julia 1.8.1",
    "language": "julia",
-   "name": "julia-1.7"
+   "name": "julia-1.8"
   },
   "language_info": {
    "file_extension": ".jl",
    "mimetype": "application/julia",
    "name": "julia",
-   "version": "1.7.1"
+   "version": "1.8.1"
   }
  },
  "nbformat": 4,

single-user-r-d4science/Dockerfile

@@ -7,7 +7,7 @@ RUN mamba install -y --quiet \
         r-geojsonio \
         r-rlang  \
         r-rcpp \
-    && conda clean -tipsy
+    && conda clean -tipy
 
 RUN mamba install -y --quiet \
         r-vctrs \
@@ -19,7 +19,7 @@ RUN mamba install -y --quiet \
         r-dbi \
         r-cpp11 \
         r-bh \
-    && conda clean -tipsy
+    && conda clean -tipy
 
 RUN mamba install -y --quiet \
         r-tibble \
@@ -30,7 +30,7 @@ RUN mamba install -y --quiet \
         r-rgdal \
         r-pastecs \
         r-factominer \
-    && conda clean -tipsy
+    && conda clean -tipy
 
 RUN Rscript  -e "remotes::install_github('eblondel/geoflow', dependencies = c('Depends', 'Imports'))"
 
@@ -39,4 +39,4 @@ RUN Rscript  -e "install.packages(c('geometa','ows4R','geonapi','geosapi','raste
 RUN pip install --no-cache-dir \
         import_ipynb \
         nbresuse \
-        git+https://github.com/andrea-manzi/mbtr.git
+	git+https://github.com/supsi-dacd-isaac/mbtr.git

single-user-scientometrics-d4science/Dockerfile

@@ -235,6 +235,9 @@ RUN pip install --no-cache-dir \
         zstandard==0.19.0
 
 # -------------------------------
-
-RUN jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor
-RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file
+# Jupyter lab extensions
+# -------------------------------
+RUN pip install --no-cache-dir \
+    jupyterlab-topbar \
+    jupyterlab-system-monitor
+RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file

single-user-sobigdata-lipari23/Dockerfile

@@ -43,6 +43,10 @@ RUN pip install --no-cache-dir \
 RUN mamba install -y protobuf==4.23.3
 
 # -------------------------------
+# Jupyter lab extensions
+# -------------------------------
+RUN pip install --no-cache-dir \
+    jupyterlab-topbar \
+    jupyterlab-system-monitor
+RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file
 
-#RUN jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor
-#RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file

single-user-sobigdata/Dockerfile

@@ -1,11 +1,11 @@
-#   _____       ____  _       _____        _                    
-#  / ____|     |  _ \(_)     |  __ \      | |         _     _   
-# | (___   ___ | |_) |_  __ _| |  | | __ _| |_ __ _ _| |_ _| |_ 
+#   _____       ____  _       _____        _
+#  / ____|     |  _ \(_)     |  __ \      | |         _     _
+# | (___   ___ | |_) |_  __ _| |  | | __ _| |_ __ _ _| |_ _| |_
 #  \___ \ / _ \|  _ <| |/ _` | |  | |/ _` | __/ _` |_   _|_   _|
-#  ____) | (_) | |_) | | (_| | |__| | (_| | || (_| | |_|   |_|  
-# |_____/ \___/|____/|_|\__, |_____/ \__,_|\__\__,_|            
-#                        __/ |                                  
-#                       |___/      
+#  ____) | (_) | |_) | | (_| | |__| | (_| | || (_| | |_|   |_|
+# |_____/ \___/|____/|_|\__, |_____/ \__,_|\__\__,_|
+#                        __/ |
+#                       |___/
 
 ARG BASE_IMAGE=eginotebooks/d4science-base:latest
 # hadolint ignore=DL3006
@@ -43,7 +43,7 @@ RUN mamba install -y --quiet \
         contextily \
         nb_conda_kernels \
         nb_conda \
-    && conda clean -tipsy
+    && conda clean -tipy
 
 # Pinned networkx to 2.6.3 as 2.7 makes networksns fail
 RUN pip install --no-cache-dir \
@@ -85,8 +85,6 @@ RUN pip install --no-cache-dir \
 # -------------------------------
 RUN mamba install -y --quiet \
         pyspark \
-        pyfim \
-        pyclustering \
         transformers \
         rtree \
         geopandas \
@@ -100,10 +98,15 @@ RUN mamba install -y --quiet \
         openjdk=8 \
    && conda clean --all
 
-RUN  mamba install -y pytorch==1.6.0  torchtext torchvision -c pytorch \
-   && conda clean --all
+RUN pip install --no-cache-dir \
+        torch \
+        torchvision \
+        torchaudio \
+        --index-url https://download.pytorch.org/whl/cpu
 
 RUN pip install --no-cache-dir \
+        pyfim \
+        pyclustering \
         import_ipynb \
         opencv-python \
         simpletransformers \
@@ -115,6 +118,7 @@ RUN pip install --no-cache-dir \
         dynetx==0.3.2
 
 # -------------------------------
-
-#RUN jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor
-#RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file
+# JupyterLab extensions
+# -------------------------------
+RUN pip install --no-cache-dir jupyterlab-topbar
+RUN jupyter labextension disable @jupyterlab/filebrowser-extension:share-file