From a85d0d37876473a7459ede076187e8636c506d1d Mon Sep 17 00:00:00 2001 From: Ruinong Tian Date: Tue, 24 Dec 2024 20:58:57 +0000 Subject: [PATCH] Revert "reduce RUN commands in Dockerfile" This reverts commit 1c7628c1e3c80c1743d60893410e19fc16512c55. --- build_artifacts/v1/v1.10/v1.10.1/Dockerfile | 84 ++++++++++++--------- template/v1/Dockerfile | 84 ++++++++++++--------- 2 files changed, 96 insertions(+), 72 deletions(-) diff --git a/build_artifacts/v1/v1.10/v1.10.1/Dockerfile b/build_artifacts/v1/v1.10/v1.10.1/Dockerfile index d9b2eec3..2116fc49 100644 --- a/build_artifacts/v1/v1.10/v1.10.1/Dockerfile +++ b/build_artifacts/v1/v1.10/v1.10.1/Dockerfile @@ -44,12 +44,14 @@ RUN apt-get update && apt-get upgrade -y && \ sudo ./aws/install && \ rm -rf aws awscliv2.zip && \ : -RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile \ - # CodeEditor - create server, user data dirs - mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ - && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ - # create dir to store user data files - mkdir -p /opt/amazon/sagemaker/user-data \ +RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile + +# CodeEditor - create server, user data dirs +RUN mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ + && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data + +# create dir to store user data files +RUN mkdir -p /opt/amazon/sagemaker/user-data \ && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/user-data @@ -57,9 +59,10 @@ RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/pr RUN mkdir -p ${DIRECTORY_TREE_STAGE_DIR} COPY dirs/ ${DIRECTORY_TREE_STAGE_DIR}/ RUN rsync -a ${DIRECTORY_TREE_STAGE_DIR}/ / && \ - rm -rf ${DIRECTORY_TREE_STAGE_DIR} \ - # CodeEditor - download the extensions - mkdir -p /etc/code-editor/extensions && \ + rm -rf ${DIRECTORY_TREE_STAGE_DIR} + +# CodeEditor - download the extensions +RUN mkdir -p /etc/code-editor/extensions && \ while IFS= read -r url || [ -n "$url" ]; do \ echo "Downloading extension from ${url}..." && \ wget --no-check-certificate -P /etc/code-editor/extensions "${url}"; \ @@ -82,10 +85,13 @@ RUN micromamba install -y --name base --file /tmp/$ENV_IN_FILENAME && \ ARG MAMBA_DOCKERFILE_ACTIVATE=1 -RUN sudo ln -s $(which python3) /usr/bin/python \ - # Update npm version - npm update -g npm \ - # Configure CodeEditor - Install extensions and set preferences +RUN sudo ln -s $(which python3) /usr/bin/python + +# Update npm version +RUN npm i -g npm + +# Configure CodeEditor - Install extensions and set preferences +RUN \ extensionloc=/opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions && mkdir -p "${extensionloc}" \ # Loop through all vsix files in /etc/code-editor/extensions and install them && for ext in /etc/code-editor/extensions/*.vsix; do \ @@ -93,10 +99,11 @@ RUN sudo ln -s $(which python3) /usr/bin/python \ sagemaker-code-editor --install-extension "${ext}" --extensions-dir "${extensionloc}" --server-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data; \ done \ # Copy the settings - && cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json \ - # Install glue kernels, and move to shared directory - # Also patching base kernel so Studio background code doesn't start session silently - install-glue-kernels && \ + && cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json + +# Install glue kernels, and move to shared directory +# Also patching base kernel so Studio background code doesn't start session silently +RUN install-glue-kernels && \ SITE_PACKAGES=$(pip show aws-glue-sessions | grep Location | awk '{print $2}') && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_pyspark --user && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_spark --user && \ @@ -122,20 +129,24 @@ RUN HOME_DIR="/home/${NB_USER}/licenses" \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} python \ - && rm -rf ${HOME_DIR}/oss_compliance* \ - # Create logging directories for supervisor - mkdir -p $SAGEMAKER_LOGGING_DIR && \ + && rm -rf ${HOME_DIR}/oss_compliance* + +# Create logging directories for supervisor +RUN mkdir -p $SAGEMAKER_LOGGING_DIR && \ chmod a+rw $SAGEMAKER_LOGGING_DIR && \ mkdir -p ${STUDIO_LOGGING_DIR} && \ - chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} \ - # Clean up CodeEditor artifacts - rm -rf /etc/code-editor \ - # Create supervisord runtime directory - mkdir -p /var/run/supervisord && \ - chmod a+rw /var/run/supervisord \ - # Create root directory for DB - # Create logging directories for supervisor - mkdir -p $DB_ROOT_DIR && \ + chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} + +# Clean up CodeEditor artifacts +RUN rm -rf /etc/code-editor + +# Create supervisord runtime directory +RUN mkdir -p /var/run/supervisord && \ + chmod a+rw /var/run/supervisord + +# Create root directory for DB +# Create logging directories for supervisor +RUN mkdir -p $DB_ROOT_DIR && \ chmod a+rw $DB_ROOT_DIR USER $MAMBA_USER @@ -160,10 +171,10 @@ RUN INSTALLED_SSL=$(micromamba list | grep openssl | tr -s ' ' | cut -d ' ' -f 3 cp ../openssl-$FIPS_VALIDATED_SSL/providers/fipsmodule.cnf providers/. && \ make tests && cd ../openssl-$FIPS_VALIDATED_SSL && \ # After tests pass, install FIPS provider and remove source code - make install_fips && cd .. && rm -rf ./openssl-* \ - # Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS - # e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf - cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ + make install_fips && cd .. && rm -rf ./openssl-* +# Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS +# e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf +RUN cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ sed -i "s:# .include fipsmodule.cnf:.include /opt/conda/ssl/fipsmodule.cnf:" /opt/conda/ssl/openssl-fips.cnf && \ sed -i 's:# fips = fips_sect:fips = fips_sect:' /opt/conda/ssl/openssl-fips.cnf ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ @@ -171,9 +182,10 @@ ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ # Install Kerberos. # Make sure no dependency is added/updated RUN pip install "krb5>=0.5.1,<0.6" && \ - pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' \ - # https://stackoverflow.com/questions/122327 - SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ + pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' + +# https://stackoverflow.com/questions/122327 +RUN SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ # Remove SparkRKernel as it's not supported \ jupyter-kernelspec remove -f -y sparkrkernel && \ # Patch Sparkmagic lib to support Custom Certificates \ diff --git a/template/v1/Dockerfile b/template/v1/Dockerfile index 0a8ddf11..3519e830 100644 --- a/template/v1/Dockerfile +++ b/template/v1/Dockerfile @@ -46,12 +46,14 @@ RUN apt-get update && apt-get upgrade -y && \ sudo ./aws/install && \ rm -rf aws awscliv2.zip && \ : -RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile \ - # CodeEditor - create server, user data dirs - mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ - && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ - # create dir to store user data files - mkdir -p /opt/amazon/sagemaker/user-data \ +RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile + +# CodeEditor - create server, user data dirs +RUN mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ + && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data + +# create dir to store user data files +RUN mkdir -p /opt/amazon/sagemaker/user-data \ && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/user-data @@ -59,9 +61,10 @@ RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/pr RUN mkdir -p ${DIRECTORY_TREE_STAGE_DIR} COPY dirs/ ${DIRECTORY_TREE_STAGE_DIR}/ RUN rsync -a ${DIRECTORY_TREE_STAGE_DIR}/ / && \ - rm -rf ${DIRECTORY_TREE_STAGE_DIR} \ - # CodeEditor - download the extensions - mkdir -p /etc/code-editor/extensions && \ + rm -rf ${DIRECTORY_TREE_STAGE_DIR} + +# CodeEditor - download the extensions +RUN mkdir -p /etc/code-editor/extensions && \ while IFS= read -r url || [ -n "$url" ]; do \ echo "Downloading extension from ${url}..." && \ wget --no-check-certificate -P /etc/code-editor/extensions "${url}"; \ @@ -84,10 +87,13 @@ RUN micromamba install -y --name base --file /tmp/$ENV_IN_FILENAME && \ ARG MAMBA_DOCKERFILE_ACTIVATE=1 -RUN sudo ln -s $(which python3) /usr/bin/python \ - # Update npm version - npm update -g npm \ - # Configure CodeEditor - Install extensions and set preferences +RUN sudo ln -s $(which python3) /usr/bin/python + +# Update npm version +RUN RUN npm update -g npm + +# Configure CodeEditor - Install extensions and set preferences +RUN \ extensionloc=/opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions && mkdir -p "${extensionloc}" \ # Loop through all vsix files in /etc/code-editor/extensions and install them && for ext in /etc/code-editor/extensions/*.vsix; do \ @@ -96,10 +102,11 @@ RUN sudo ln -s $(which python3) /usr/bin/python \ done \ # Copy the settings && cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json \ - && cp /etc/code-editor/code_editor_user_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/User/settings.json \ - # Install glue kernels, and move to shared directory - # Also patching base kernel so Studio background code doesn't start session silently - install-glue-kernels && \ + && cp /etc/code-editor/code_editor_user_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/User/settings.json + +# Install glue kernels, and move to shared directory +# Also patching base kernel so Studio background code doesn't start session silently +RUN install-glue-kernels && \ SITE_PACKAGES=$(pip show aws-glue-sessions | grep Location | awk '{print $2}') && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_pyspark --user && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_spark --user && \ @@ -125,20 +132,24 @@ RUN HOME_DIR="/home/${NB_USER}/licenses" \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} python \ - && rm -rf ${HOME_DIR}/oss_compliance* \ - # Create logging directories for supervisor - mkdir -p $SAGEMAKER_LOGGING_DIR && \ + && rm -rf ${HOME_DIR}/oss_compliance* + +# Create logging directories for supervisor +RUN mkdir -p $SAGEMAKER_LOGGING_DIR && \ chmod a+rw $SAGEMAKER_LOGGING_DIR && \ mkdir -p ${STUDIO_LOGGING_DIR} && \ - chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} \ - # Clean up CodeEditor artifacts - rm -rf /etc/code-editor \ - # Create supervisord runtime directory - mkdir -p /var/run/supervisord && \ - chmod a+rw /var/run/supervisord \ - # Create root directory for DB - # Create logging directories for supervisor - mkdir -p $DB_ROOT_DIR && \ + chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} + +# Clean up CodeEditor artifacts +RUN rm -rf /etc/code-editor + +# Create supervisord runtime directory +RUN mkdir -p /var/run/supervisord && \ + chmod a+rw /var/run/supervisord + +# Create root directory for DB +# Create logging directories for supervisor +RUN mkdir -p $DB_ROOT_DIR && \ chmod a+rw $DB_ROOT_DIR USER $MAMBA_USER @@ -163,10 +174,10 @@ RUN INSTALLED_SSL=$(micromamba list | grep openssl | tr -s ' ' | cut -d ' ' -f 3 cp ../openssl-$FIPS_VALIDATED_SSL/providers/fipsmodule.cnf providers/. && \ make tests && cd ../openssl-$FIPS_VALIDATED_SSL && \ # After tests pass, install FIPS provider and remove source code - make install_fips && cd .. && rm -rf ./openssl-* \ - # Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS - # e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf - cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ + make install_fips && cd .. && rm -rf ./openssl-* +# Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS +# e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf +RUN cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ sed -i "s:# .include fipsmodule.cnf:.include /opt/conda/ssl/fipsmodule.cnf:" /opt/conda/ssl/openssl-fips.cnf && \ sed -i 's:# fips = fips_sect:fips = fips_sect:' /opt/conda/ssl/openssl-fips.cnf ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ @@ -174,9 +185,10 @@ ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ # Install Kerberos. # Make sure no dependency is added/updated RUN pip install "krb5>=0.5.1,<0.6" && \ - pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' \ - # https://stackoverflow.com/questions/122327 - SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ + pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' + +# https://stackoverflow.com/questions/122327 +RUN SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ # Remove SparkRKernel as it's not supported \ jupyter-kernelspec remove -f -y sparkrkernel && \ # Patch Sparkmagic lib to support Custom Certificates \