Skip to content

Commit

Permalink
Created a new Ubuntu:22.04 based Python notebook that is S3 mount aware
Browse files Browse the repository at this point in the history
  • Loading branch information
JohnVidler committed Mar 4, 2024
1 parent 1885365 commit e0fe0d3
Show file tree
Hide file tree
Showing 6 changed files with 88 additions and 0 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
.DS_Store

35 changes: 35 additions & 0 deletions jupyter/python-minimal-s3/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
FROM --platform=$BUILDPLATFORM ubuntu:22.04
ENV DEBIAN_FRONTEND noninteractive

USER root

RUN apt-get update && apt-get upgrade -y && apt-get install -y \
python3.11 \
python3-pip \
python3.11-venv \
build-essential \
python3-dev \
curl \
wget \
git \
vim \
nano-tiny \
unzip \
s3fs && \
apt-get clean && rm -rf /var/lib/apt/lists/*

RUN pip3 install \
'jupyterhub==4.*' \
'notebook==7.*'

COPY src/ /opt/launch
RUN chmod +x /opt/launch/launch.sh && chmod +x /opt/launch/conf.d/*
ENV PATH="${PATH}:/opt/launch/"

# create a user, since we don't want to run as root
RUN useradd -m jovyan
ENV HOME=/home/jovyan
WORKDIR $HOME
USER jovyan

CMD ["launch.sh"]
31 changes: 31 additions & 0 deletions jupyter/python-minimal-s3/src/conf.d/00-mount-s3
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash

if [[ -z "${HEX_S3_KEY}" ]]; then
echo "No S3 key, refusing to attempt to mount"
exit 0
fi

if [[ -z "${HEX_S3_SECRET}" ]]; then
echo "No S3 secret, refusing to attempt to mount"
exit 0
fi

if [[ -z "${HEX_S3_BUCKET}" ]]; then
echo "No S3 bucket, refusing to attempt to mount"
exit 0
fi

if [[ -z "${HEX_S3_MOUNT_PATH}" ]]; then
HEX_S3_MOUNT_PATH="/home/jovyan/work"
echo "Using default s3 work path"
fi


cat >/home/jovyan/.s3auth <<EOL
${HEX_S3_KEY}:${HEX_S3_SECRET}
EOL
chmod 600 /home/jovyan/.s3auth

echo s3fs ${HEX_S3_BUCKET} ${HEX_S3_MOUNT_PATH} -o passwd_file=/home/jovyan/.s3auth -o url=http://ucrel-hex-store.scc.lancs.ac.uk:3900/ -o use_path_request_style

cp /opt/launch/res/README.md
3 changes: 3 additions & 0 deletions jupyter/python-minimal-s3/src/conf.d/99-jupyterhub-singleuser
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

jupyterhub-singleuser
3 changes: 3 additions & 0 deletions jupyter/python-minimal-s3/src/launch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

run-parts --verbose /opt/launch/conf.d
14 changes: 14 additions & 0 deletions jupyter/python-minimal-s3/src/res/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# S3 Persistent Storage

This instance is connected to the local S3 storage instance.

To have your files saved, keep them in the `work` folder.


## More Detailed Explanation

This means that any data in your `work` folder will be persisted across instances, and will be stored on Hex's storage
system automatically. However, this storage is significantly slower than the local storage in your home folder, so you
may wish to transfer files in/out of the work folder to improve your program's performance if it does a large number of
out-of-order writes.

0 comments on commit e0fe0d3

Please sign in to comment.