From e72112aa05639b6a884cabaaf3bc8ea7959f57ea Mon Sep 17 00:00:00 2001 From: Joshua Robison Date: Tue, 4 Oct 2022 09:17:19 -0400 Subject: [PATCH 1/9] Updating template --- README.md | 2 +- ml_in_a_box.sh | 70 ++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6720728..bc64100 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Some generic categories of software not included: | Connectors | Academic Torrents | | | Dashboarding | panel, dash, voila, streamlit | | | Databases | MySQL, Hive, PostgreSQL, Prometheus, Neo4j, MongoDB, Cassandra, Redis | No particular infra to connect to databases | -| Deep Learning | Caffe, Caffe2, Theano, PaddlePaddle, Chainer, Torch, MXNet | PyTorch and TensorFlow are dominant, rest niche | +| Deep Learning | Caffe, Caffe2, Theano, PaddlePaddle, Chainer, MXNet | PyTorch and TensorFlow are dominant, rest niche | | Deployment | Dash, TFServing, R Shiny, Flask | Use Gradient Deployments | | Distributed. | Horovod, OpenMPI | Use Gradient distributed | | Feature store | Feast | | diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index 0216a59..6e28bf0 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -3,7 +3,7 @@ # ================================================================== # Module list # ------------------------------------------------------------------ -# python 3.9.12 (conda) +# python 3.9.12 (apt) # torch 1.12.1 (pip) # torchvision 0.13.1 (pip) # torchaudio 0.12.1 (pip) @@ -107,29 +107,73 @@ #Based on https://docs.anaconda.com/anaconda/install/linux/ - sudo $APT_INSTALL libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6 - sudo wget https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh - sudo bash ~/Anaconda3-2022.05-Linux-x86_64.sh -b -p $HOME/anaconda3 + # sudo $APT_INSTALL libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6 + # sudo wget https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh + # sudo bash ~/Anaconda3-2022.05-Linux-x86_64.sh -b -p $HOME/anaconda3 - sudo chown -R $USER:$USER $HOME/anaconda3 - sudo chmod -R +x $HOME/anaconda3 + # sudo chown -R $USER:$USER $HOME/anaconda3 + # sudo chmod -R +x $HOME/anaconda3 - source $HOME/anaconda3/bin/activate - conda init bash - conda deactivate + # source $HOME/anaconda3/bin/activate + # conda init bash + # conda deactivate - export PATH=$HOME/anaconda3/bin:${PATH} + # export PATH=$HOME/anaconda3/bin:${PATH} - $PIP_INSTALL pip + # $PIP_INSTALL pip - rm -f ~/Anaconda3-2022.05-Linux-x86_64.sh + # rm -f ~/Anaconda3-2022.05-Linux-x86_64.sh + + +# ================================================================== +# Python +# ------------------------------------------------------------------ + + #Based on https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa + + # Adding repository for python3.9 + DEBIAN_FRONTEND=noninteractive \ + $APT_INSTALL software-properties-common + add-apt-repository ppa:deadsnakes/ppa -y + + # Installing python3.9 + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.9 \ + python3.9-dev \ + python3-distutils-extra + + # Installing pip + wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py + python3.9 ~/get-pip.py + + # Add symlink so python and python3 commands use same python3.9 executable + ln -s /usr/bin/python3.9 /usr/local/bin/python3 + ln -s /usr/bin/python3.9 /usr/local/bin/python # ================================================================== # Installing CUDA packages (CUDA Toolkit 11.7.1 & CUDNN 8.5.0) # ------------------------------------------------------------------ - $CONDA_INSTALL -c nvidia/label/cuda-11.7.1 cuda + # $CONDA_INSTALL -c nvidia/label/cuda-11.7.1 cuda + + # wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin + # sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 + + # wget https://developer.download.nvidia.com/compute/cuda/11.3.0/local_installers/cuda-repo-ubuntu2004-11-3-local_11.3.0-465.19.01-1_amd64.deb + # sudo dpkg -i cuda-repo-ubuntu2004-11-3-local_11.3.0-465.19.01-1_amd64.deb + + # sudo apt-key add /var/cuda-repo-ubuntu2004-11-3-local/7fa2af80.pub + # sudo apt-get update + + # sudo apt-get -y install cuda + # export PATH=/usr/local/cuda-11.3/bin${PATH:+:${PATH}} + # export LD_LIBRARY_PATH=/usr/local/cuda-11.3/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + + + sudo wget -O /etc/apt/preferences.d/cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin + sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub + sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 From 24a6365ef157a4179e2fadead931580e6d2f67c3 Mon Sep 17 00:00:00 2001 From: paperspace Date: Tue, 4 Oct 2022 11:21:43 -0700 Subject: [PATCH 2/9] Updated template --- ml_in_a_box.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index 6e28bf0..c6a6975 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -3,7 +3,7 @@ # ================================================================== # Module list # ------------------------------------------------------------------ -# python 3.9.12 (apt) +# python 3.9.14 (apt) # torch 1.12.1 (pip) # torchvision 0.13.1 (pip) # torchaudio 0.12.1 (pip) @@ -50,8 +50,8 @@ # Set ENV variables export APT_INSTALL="apt-get install -y --no-install-recommends" - export PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" - export CONDA_INSTALL="conda install -y" + export PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" + # export CONDA_INSTALL="conda install -y" export GIT_CLONE="git clone --depth 10" # Update apt @@ -133,23 +133,23 @@ # Adding repository for python3.9 DEBIAN_FRONTEND=noninteractive \ - $APT_INSTALL software-properties-common - add-apt-repository ppa:deadsnakes/ppa -y + sudo $APT_INSTALL software-properties-common + sudo add-apt-repository ppa:deadsnakes/ppa -y # Installing python3.9 - DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + DEBIAN_FRONTEND=noninteractive sudo $APT_INSTALL \ python3.9 \ python3.9-dev \ python3-distutils-extra - # Installing pip - wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py - python3.9 ~/get-pip.py - # Add symlink so python and python3 commands use same python3.9 executable - ln -s /usr/bin/python3.9 /usr/local/bin/python3 - ln -s /usr/bin/python3.9 /usr/local/bin/python + sudo ln -s /usr/bin/python3.9 /usr/local/bin/python3 + sudo ln -s /usr/bin/python3.9 /usr/local/bin/python + # Installing pip + wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py + python ~/get-pip.py + rm ~/get-pip.py # ================================================================== # Installing CUDA packages (CUDA Toolkit 11.7.1 & CUDNN 8.5.0) From 792b65cea4aa405b2ff8729c8b5bce095990fb47 Mon Sep 17 00:00:00 2001 From: paperspace Date: Tue, 4 Oct 2022 12:06:31 -0700 Subject: [PATCH 3/9] Updated template --- ml_in_a_box.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index c6a6975..c9122c6 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -138,18 +138,22 @@ # Installing python3.9 DEBIAN_FRONTEND=noninteractive sudo $APT_INSTALL \ - python3.9 \ - python3.9-dev \ - python3-distutils-extra + python3.10 \ + python3.10-dev \ + python3.10-distutils-extra # Add symlink so python and python3 commands use same python3.9 executable - sudo ln -s /usr/bin/python3.9 /usr/local/bin/python3 - sudo ln -s /usr/bin/python3.9 /usr/local/bin/python + sudo ln -s /usr/bin/python3.10 /usr/local/bin/python3 + sudo ln -s /usr/bin/python3.10 /usr/local/bin/python + sudo ln -sf /usr/bin/python3.10 /usr/bin/python3 # Installing pip - wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py - python ~/get-pip.py - rm ~/get-pip.py + DEBIAN_FRONTEND=noninteractive \ + sudo $APT_INSTALL python3-pip + + # wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py + # python ~/get-pip.py + # rm ~/get-pip.py # ================================================================== # Installing CUDA packages (CUDA Toolkit 11.7.1 & CUDNN 8.5.0) From 67427eb300bf5cda129b67899ba8d33e2d02f195 Mon Sep 17 00:00:00 2001 From: paperspace Date: Tue, 4 Oct 2022 15:46:07 -0700 Subject: [PATCH 4/9] Updated template --- ml_in_a_box.sh | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index c9122c6..2725abb 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -139,17 +139,31 @@ # Installing python3.9 DEBIAN_FRONTEND=noninteractive sudo $APT_INSTALL \ python3.10 \ - python3.10-dev \ - python3.10-distutils-extra + python3.10-distutils + # python3.10-dev \ + # python3.10-distutils-extra - # Add symlink so python and python3 commands use same python3.9 executable + # Add symlink so python and python3 commands use same python3.10 executable sudo ln -s /usr/bin/python3.10 /usr/local/bin/python3 sudo ln -s /usr/bin/python3.10 /usr/local/bin/python sudo ln -sf /usr/bin/python3.10 /usr/bin/python3 # Installing pip - DEBIAN_FRONTEND=noninteractive \ - sudo $APT_INSTALL python3-pip + curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 + export PATH=$PATH:/home/paperspace/.local/bin + + # export PATH=$PATH:/home/paperspace/.local/bin + # wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py + # python ~/get-pip.py + # rm ~/get-pip.py + + # sudo ln -sf /home/paperspace/.local/bin/pip /usr/bin/pip + # sudo ln -sf /home/paperspace/.local/bin/pip3 /usr/bin/pip3 + # sudo ln -sf /home/paperspace/.local/bin/pip3.10 /usr/bin/pip3.10 + + # Installing pip + # DEBIAN_FRONTEND=noninteractive \ + # sudo $APT_INSTALL python3-pip # wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py # python ~/get-pip.py @@ -175,15 +189,16 @@ # export LD_LIBRARY_PATH=/usr/local/cuda-11.3/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - sudo wget -O /etc/apt/preferences.d/cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin - sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub - sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" + # sudo wget -O /etc/apt/preferences.d/cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin + # sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub + # sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" sudo apt-get update + sudo apt-get install cuda #New sudo apt-get install libcudnn8=8.5.0.*-1+cuda11.7 sudo apt-get install libcudnn8-dev=8.5.0.*-1+cuda11.7 @@ -212,7 +227,7 @@ # Based on https://www.tensorflow.org/install/pip - export LD_LIBRARY_PATH=${HOME}/anaconda3/lib + # export LD_LIBRARY_PATH=${HOME}/anaconda3/lib $PIP_INSTALL tensorflow==2.9.2 @@ -301,16 +316,16 @@ # Config & Cleanup # ------------------------------------------------------------------ - rm $HOME/anaconda3/lib/libtinfo.so.6 - rm $HOME/anaconda3/lib/libncursesw.so.6 + # rm $HOME/anaconda3/lib/libtinfo.so.6 + # rm $HOME/anaconda3/lib/libncursesw.so.6 echo "export PATH=${PATH}" >> ~/.bashrc - echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.bashrc + # echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.bashrc echo "export PATH=${PATH}" >> ~/.profile - echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.profile + # echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.profile echo "export PATH=${PATH}" >> ~/.bash_profile - echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.bash_profile + # echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.bash_profile From 70881515c54127998f3a382ebee01115ac537b10 Mon Sep 17 00:00:00 2001 From: Joshua Robison Date: Wed, 5 Oct 2022 15:40:50 -0400 Subject: [PATCH 5/9] Updated template python/pip/cuda-toolkit/cudnn --- ml_in_a_box.sh | 70 ++++++++++++++------------------------------------ 1 file changed, 19 insertions(+), 51 deletions(-) diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index 2725abb..3c7509c 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -51,7 +51,6 @@ # Set ENV variables export APT_INSTALL="apt-get install -y --no-install-recommends" export PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" - # export CONDA_INSTALL="conda install -y" export GIT_CLONE="git clone --depth 10" # Update apt @@ -75,6 +74,7 @@ rsync \ git \ vim \ + mlocate \ libssl-dev \ curl \ openssh-client \ @@ -138,67 +138,35 @@ # Installing python3.9 DEBIAN_FRONTEND=noninteractive sudo $APT_INSTALL \ - python3.10 \ - python3.10-distutils - # python3.10-dev \ - # python3.10-distutils-extra + python3.9 \ + python3.9-dev \ + python3-distutils-extra - # Add symlink so python and python3 commands use same python3.10 executable - sudo ln -s /usr/bin/python3.10 /usr/local/bin/python3 - sudo ln -s /usr/bin/python3.10 /usr/local/bin/python - sudo ln -sf /usr/bin/python3.10 /usr/bin/python3 + # Add symlink so python and python3 commands use same python3.9 executable + sudo ln -s /usr/bin/python3.9 /usr/local/bin/python3 + sudo ln -s /usr/bin/python3.9 /usr/local/bin/python # Installing pip - curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 + curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9 export PATH=$PATH:/home/paperspace/.local/bin - # export PATH=$PATH:/home/paperspace/.local/bin - # wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py - # python ~/get-pip.py - # rm ~/get-pip.py - - # sudo ln -sf /home/paperspace/.local/bin/pip /usr/bin/pip - # sudo ln -sf /home/paperspace/.local/bin/pip3 /usr/bin/pip3 - # sudo ln -sf /home/paperspace/.local/bin/pip3.10 /usr/bin/pip3.10 - - # Installing pip - # DEBIAN_FRONTEND=noninteractive \ - # sudo $APT_INSTALL python3-pip - - # wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py - # python ~/get-pip.py - # rm ~/get-pip.py # ================================================================== -# Installing CUDA packages (CUDA Toolkit 11.7.1 & CUDNN 8.5.0) +# Installing CUDA packages (CUDA Toolkit 11.8.0 & CUDNN 8.5.0) # ------------------------------------------------------------------ - # $CONDA_INSTALL -c nvidia/label/cuda-11.7.1 cuda - - # wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin - # sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 - - # wget https://developer.download.nvidia.com/compute/cuda/11.3.0/local_installers/cuda-repo-ubuntu2004-11-3-local_11.3.0-465.19.01-1_amd64.deb - # sudo dpkg -i cuda-repo-ubuntu2004-11-3-local_11.3.0-465.19.01-1_amd64.deb - - # sudo apt-key add /var/cuda-repo-ubuntu2004-11-3-local/7fa2af80.pub - # sudo apt-get update - - # sudo apt-get -y install cuda - # export PATH=/usr/local/cuda-11.3/bin${PATH:+:${PATH}} - # export LD_LIBRARY_PATH=/usr/local/cuda-11.3/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - - - # sudo wget -O /etc/apt/preferences.d/cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin - # sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub - # sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" + # Based on https://developer.nvidia.com/cuda-toolkit-archive + # Based on https://developer.nvidia.com/rdp/cudnn-archive + wget https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda_11.7.1_515.65.01_linux.run + sudo sh cuda_11.7.1_515.65.01_linux.run --silent --kernelobjects --toolkit + export PATH=$PATH:/usr/local/cuda-11.7/bin + export LD_LIBRARY_PATH=/usr/local/cuda-11.7/lib64 + rm cuda_11.7.1_515.65.01_linux.run wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" - sudo apt-get update - sudo apt-get install cuda #New sudo apt-get install libcudnn8=8.5.0.*-1+cuda11.7 sudo apt-get install libcudnn8-dev=8.5.0.*-1+cuda11.7 @@ -320,12 +288,12 @@ # rm $HOME/anaconda3/lib/libncursesw.so.6 echo "export PATH=${PATH}" >> ~/.bashrc - # echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.bashrc + echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> ~/.bashrc echo "export PATH=${PATH}" >> ~/.profile - # echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.profile + echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> ~/.profile echo "export PATH=${PATH}" >> ~/.bash_profile - # echo "export LD_LIBRARY_PATH=${HOME}/anaconda3/lib" >> ~/.bash_profile + echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> ~/.bash_profile From fd84f4cad9d217254001f95902e264decdf7b83d Mon Sep 17 00:00:00 2001 From: Joshua Robison Date: Wed, 5 Oct 2022 16:17:21 -0400 Subject: [PATCH 6/9] Updated jupyter nb install --- ml_in_a_box.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index 3c7509c..bfaf9eb 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -276,8 +276,8 @@ sudo curl -sL https://deb.nodesource.com/setup_16.x | sudo bash sudo $APT_INSTALL nodejs - $PIP_INSTALL jupyter_contrib_nbextensions jupyterlab-git && \ - DEBIAN_FRONTEND=noninteractive jupyter contrib nbextension install --sys-prefix + $PIP_INSTALL jupyter_contrib_nbextensions jupyterlab-git + DEBIAN_FRONTEND=noninteractive jupyter contrib nbextension install --user # ================================================================== From 64c60dfeeb912b96e56b0c1f5a855d21f9857b80 Mon Sep 17 00:00:00 2001 From: Joshua Robison Date: Sun, 9 Oct 2022 08:53:15 -0700 Subject: [PATCH 7/9] Updated cuda install --- ml_in_a_box.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index bfaf9eb..8f057cc 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -158,7 +158,7 @@ # Based on https://developer.nvidia.com/cuda-toolkit-archive # Based on https://developer.nvidia.com/rdp/cudnn-archive wget https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda_11.7.1_515.65.01_linux.run - sudo sh cuda_11.7.1_515.65.01_linux.run --silent --kernelobjects --toolkit + sudo sh cuda_11.7.1_515.65.01_linux.run --silent --toolkit export PATH=$PATH:/usr/local/cuda-11.7/bin export LD_LIBRARY_PATH=/usr/local/cuda-11.7/lib64 rm cuda_11.7.1_515.65.01_linux.run From 95787b8c21ba275352cc6f6cac1fbd377f5a4ecb Mon Sep 17 00:00:00 2001 From: Joshua Robison Date: Mon, 10 Oct 2022 14:13:07 -0700 Subject: [PATCH 8/9] Newly updated version --- README.md | 11 ++++++----- ml_in_a_box.sh | 33 +++++---------------------------- 2 files changed, 11 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index bc64100..490cc30 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,12 @@ We assume a generic advanced data science user who probably wants GPU access, bu | Category | Software | Version | Install Method | Why / Notes | | ------------- | ------------- | ------------- | ------------- | ------------- | | GPU | NVidia Driver | 515.65.01 | pre-installed | Enable Nvidia GPUs. Latest version as of VM creation date | -| | CUDA | 11.7.1 | Conda | Nvidia A100 GPUs require CUDA 11+ to work, so 10.x is not suitable | +| | CUDA | 11.7.1 | Install script | Nvidia A100 GPUs require CUDA 11+ to work, so 10.x is not suitable | +| | CUDA toolkit | 11.7.1 | Apt | Needed for `nvcc` command for cuDNN. Installed with Conda CUDA installation | | | cuDNN | 8.5.0.*-1+cuda11.7 | Ubuntu repo | Nvidia GPU deep learning library | -| | CUDA toolkit | 11.7.1 | Conda | Needed for `nvcc` command for cuDNN. Installed with Conda CUDA installation | -| Infra | Anaconda | 4.14.0 | Ubuntu repo | Package management system that installs Python3, pip, and other python packages | -| | Docker Engine CE | 20.10.8, build 3967b7d | pre-installed | Docker Engine community edition | +| Infra | Docker Engine CE | 20.10.8, build 3967b7d | pre-installed | Docker Engine community edition | | | NVidia Docker | 2.6.0-1 | pre-installed | Enable NVidia GPU in Docker containers | -| Python | Python | 3.9.12 | Conda | Most widely used programming language for data science. Version 3.9.12 is installed when downloading Anaconda3 and is compatible with other software and their versions installed here. | +| Python | Python | 3.9.14 | Apt | Most widely used programming language for data science | | | pip3 | 22.2.2 | Conda | Enable easy installation of 1000s of other data science, etc., packages. Installed with Anaconda3 installation. | | | NumPy | 1.23.2 | pip3 | Handle arrays, matrices, etc., in Python | | | SciPy | 1.9.1 | pip3 | Fundamental algorithms for scientific computing in Python | @@ -43,6 +42,7 @@ We assume a generic advanced data science user who probably wants GPU access, bu | | opencv-python | 4.6.0.66 | pip3 | Includes several hundreds of computer vision algorithms | | | pyyaml | 5.4.1 | pip3 | YAML parser and emitter for Python | | | JupyterLab | 3.4.6 | pip3 | De facto standard for data science using Jupyter notebooks | +| | wandb | 0.13.4 | pip3 | CLI and library to interact with the Weights & Biases API (model tracking) | | Machine Learning | Scikit-learn | 1.1.2 | pip3 | Widely used ML library for data science, generally for smaller data or models | | | Scikit-image | 0.19.3 | pip3 | Collection of algorithms for image processing | | | TensorFlow | 2.9.2 | pip3 | Most widely used deep learning library, alongside PyTorch | @@ -98,6 +98,7 @@ We assume a generic advanced data science user who probably wants GPU access, bu | jsonify | MIT | https://pypi.org/project/jsonify/0.5/#data | | opencv-python | MIT | https://github.com/opencv/opencv-python/blob/4.x/LICENSE.txt | | pyyaml | MIT | https://github.com/yaml/pyyaml/blob/master/LICENSE | +| wandb | MIT | https://github.com/wandb/wandb/blob/master/LICENSE | Information about license types: diff --git a/ml_in_a_box.sh b/ml_in_a_box.sh index 8f057cc..1364c01 100644 --- a/ml_in_a_box.sh +++ b/ml_in_a_box.sh @@ -39,6 +39,7 @@ # opencv-python 4.6.0.66 (pip) # pyyaml 5.4.1 (pip) # sentence-transformers 2.2.2 (pip) +# wandb 0.13.4 (pip) # nodejs 16.x latest (apt) # default-jre latest (apt) # default-jdk latest (apt) @@ -101,30 +102,6 @@ libboost-all-dev -# ================================================================== -# Conda -# ------------------------------------------------------------------ - - #Based on https://docs.anaconda.com/anaconda/install/linux/ - - # sudo $APT_INSTALL libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6 - # sudo wget https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh - # sudo bash ~/Anaconda3-2022.05-Linux-x86_64.sh -b -p $HOME/anaconda3 - - # sudo chown -R $USER:$USER $HOME/anaconda3 - # sudo chmod -R +x $HOME/anaconda3 - - # source $HOME/anaconda3/bin/activate - # conda init bash - # conda deactivate - - # export PATH=$HOME/anaconda3/bin:${PATH} - - # $PIP_INSTALL pip - - # rm -f ~/Anaconda3-2022.05-Linux-x86_64.sh - - # ================================================================== # Python # ------------------------------------------------------------------ @@ -140,6 +117,7 @@ DEBIAN_FRONTEND=noninteractive sudo $APT_INSTALL \ python3.9 \ python3.9-dev \ + python3.9-venv \ python3-distutils-extra # Add symlink so python and python3 commands use same python3.9 executable @@ -157,6 +135,7 @@ # Based on https://developer.nvidia.com/cuda-toolkit-archive # Based on https://developer.nvidia.com/rdp/cudnn-archive + wget https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda_11.7.1_515.65.01_linux.run sudo sh cuda_11.7.1_515.65.01_linux.run --silent --toolkit export PATH=$PATH:/usr/local/cuda-11.7/bin @@ -249,7 +228,8 @@ jsonify==0.5 \ opencv-python==4.6.0.66 \ pyyaml==5.4.1 \ - sentence-transformers==2.2.2 + sentence-transformers==2.2.2 \ + wandb==0.13.4 # ================================================================== @@ -284,9 +264,6 @@ # Config & Cleanup # ------------------------------------------------------------------ - # rm $HOME/anaconda3/lib/libtinfo.so.6 - # rm $HOME/anaconda3/lib/libncursesw.so.6 - echo "export PATH=${PATH}" >> ~/.bashrc echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> ~/.bashrc From d3ec4e94f30e05295d1e197726e7a0585d1189ec Mon Sep 17 00:00:00 2001 From: Joshua Robison Date: Mon, 10 Oct 2022 14:14:44 -0700 Subject: [PATCH 9/9] Removed references to Conda --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 490cc30..6f153ae 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,12 @@ We assume a generic advanced data science user who probably wants GPU access, bu | ------------- | ------------- | ------------- | ------------- | ------------- | | GPU | NVidia Driver | 515.65.01 | pre-installed | Enable Nvidia GPUs. Latest version as of VM creation date | | | CUDA | 11.7.1 | Install script | Nvidia A100 GPUs require CUDA 11+ to work, so 10.x is not suitable | -| | CUDA toolkit | 11.7.1 | Apt | Needed for `nvcc` command for cuDNN. Installed with Conda CUDA installation | +| | CUDA toolkit | 11.7.1 | Apt | Needed for `nvcc` command for cuDNN | | | cuDNN | 8.5.0.*-1+cuda11.7 | Ubuntu repo | Nvidia GPU deep learning library | | Infra | Docker Engine CE | 20.10.8, build 3967b7d | pre-installed | Docker Engine community edition | | | NVidia Docker | 2.6.0-1 | pre-installed | Enable NVidia GPU in Docker containers | | Python | Python | 3.9.14 | Apt | Most widely used programming language for data science | -| | pip3 | 22.2.2 | Conda | Enable easy installation of 1000s of other data science, etc., packages. Installed with Anaconda3 installation. | +| | pip3 | 22.2.2 | Apt | Enable easy installation of 1000s of other data science, etc., packages. | | | NumPy | 1.23.2 | pip3 | Handle arrays, matrices, etc., in Python | | | SciPy | 1.9.1 | pip3 | Fundamental algorithms for scientific computing in Python | | | Pandas | 1.4.4 | pip3 | De facto standard for data science data exploration/preparation in Python | @@ -59,7 +59,6 @@ We assume a generic advanced data science user who probably wants GPU access, bu | --------------- | ------------- | ------------- | | CUDA | NVidia EULA | https://docs.nvidia.com/cuda/eula/index.html | | cuDNN | NVidia EULA | https://docs.nvidia.com/deeplearning/cudnn/sla/index.html | -| Anaconda | Other | https://legal.anaconda.com/policies/en/?name=end-user-license-agreements#anaconda-distribution | | Docker Engine | Apache 2.0 | https://github.com/moby/moby/blob/master/LICENSE | | JupyterLab | New BSD | https://github.com/jupyterlab/jupyterlab/blob/master/LICENSE | | Matplotlib | PSF-based | https://matplotlib.org/stable/users/license.html | @@ -112,7 +111,6 @@ ISC: https://opensource.org/licenses/ISC Open source software can be used for commercial purposes: https://opensource.org/docs/osd#fields-of-endeavor. -Note: Anaconda has its own End User Licensing agreements around commercial use ## Software not included