-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Created a new Ubuntu:22.04 based Python notebook that is S3 mount aware
- Loading branch information
1 parent
1885365
commit e0fe0d3
Showing
6 changed files
with
88 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
.DS_Store | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
FROM --platform=$BUILDPLATFORM ubuntu:22.04 | ||
ENV DEBIAN_FRONTEND noninteractive | ||
|
||
USER root | ||
|
||
RUN apt-get update && apt-get upgrade -y && apt-get install -y \ | ||
python3.11 \ | ||
python3-pip \ | ||
python3.11-venv \ | ||
build-essential \ | ||
python3-dev \ | ||
curl \ | ||
wget \ | ||
git \ | ||
vim \ | ||
nano-tiny \ | ||
unzip \ | ||
s3fs && \ | ||
apt-get clean && rm -rf /var/lib/apt/lists/* | ||
|
||
RUN pip3 install \ | ||
'jupyterhub==4.*' \ | ||
'notebook==7.*' | ||
|
||
COPY src/ /opt/launch | ||
RUN chmod +x /opt/launch/launch.sh && chmod +x /opt/launch/conf.d/* | ||
ENV PATH="${PATH}:/opt/launch/" | ||
|
||
# create a user, since we don't want to run as root | ||
RUN useradd -m jovyan | ||
ENV HOME=/home/jovyan | ||
WORKDIR $HOME | ||
USER jovyan | ||
|
||
CMD ["launch.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
#!/bin/bash | ||
|
||
if [[ -z "${HEX_S3_KEY}" ]]; then | ||
echo "No S3 key, refusing to attempt to mount" | ||
exit 0 | ||
fi | ||
|
||
if [[ -z "${HEX_S3_SECRET}" ]]; then | ||
echo "No S3 secret, refusing to attempt to mount" | ||
exit 0 | ||
fi | ||
|
||
if [[ -z "${HEX_S3_BUCKET}" ]]; then | ||
echo "No S3 bucket, refusing to attempt to mount" | ||
exit 0 | ||
fi | ||
|
||
if [[ -z "${HEX_S3_MOUNT_PATH}" ]]; then | ||
HEX_S3_MOUNT_PATH="/home/jovyan/work" | ||
echo "Using default s3 work path" | ||
fi | ||
|
||
|
||
cat >/home/jovyan/.s3auth <<EOL | ||
${HEX_S3_KEY}:${HEX_S3_SECRET} | ||
EOL | ||
chmod 600 /home/jovyan/.s3auth | ||
|
||
echo s3fs ${HEX_S3_BUCKET} ${HEX_S3_MOUNT_PATH} -o passwd_file=/home/jovyan/.s3auth -o url=http://ucrel-hex-store.scc.lancs.ac.uk:3900/ -o use_path_request_style | ||
|
||
cp /opt/launch/res/README.md |
3 changes: 3 additions & 0 deletions
3
jupyter/python-minimal-s3/src/conf.d/99-jupyterhub-singleuser
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
#!/bin/bash | ||
|
||
jupyterhub-singleuser |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
#!/bin/bash | ||
|
||
run-parts --verbose /opt/launch/conf.d |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
# S3 Persistent Storage | ||
|
||
This instance is connected to the local S3 storage instance. | ||
|
||
To have your files saved, keep them in the `work` folder. | ||
|
||
|
||
## More Detailed Explanation | ||
|
||
This means that any data in your `work` folder will be persisted across instances, and will be stored on Hex's storage | ||
system automatically. However, this storage is significantly slower than the local storage in your home folder, so you | ||
may wish to transfer files in/out of the work folder to improve your program's performance if it does a large number of | ||
out-of-order writes. | ||
|