-
Notifications
You must be signed in to change notification settings - Fork 1
/
Dockerfile
127 lines (104 loc) · 4.23 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
FROM public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:base-cudnn8-cuda11.3-ubuntu20.04
ARG EFA_INSTALLER_VERSION=latest
ARG AWS_OFI_NCCL_VERSION=aws
ARG NCCL_TESTS_VERSION=master
ENV DEBIAN_FRONTEND=noninteractive
#execute
RUN apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/* \
&& update-ca-certificates
RUN apt-get update -y
#Install core packages
RUN apt-get install -y --allow-unauthenticated \
git \
gcc \
vim \
kmod \
sudo \
ssh \
apt-utils \
libncurses5 \
bash \
ca-certificates \
build-essential \
curl \
autoconf \
libtool \
gdb \
automake \
python3-distutils \
cmake \
apt-utils \
python3-dev \
pdsh \
nano
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
RUN touch /var/run/sshd && \
# Prevent user being kicked off after login
sed -i 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' /etc/pam.d/sshd && \
echo 'AuthorizedKeysFile .ssh/authorized_keys' >> /etc/ssh/sshd_config && \
# FIX SUDO BUG: https://github.com/sudo-project/sudo/issues/42
sudo echo "Set disable_coredump false" >> /etc/sudo.conf
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:$LD_LIBRARY_PATH
ENV PATH /opt/amazon/openmpi/bin/:/opt/amazon/efa/bin:/usr/bin:/usr/local/bin:$PATH
RUN curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py \
&& python /tmp/get-pip.py \
&& pip install awscli pynvml
#### User account
ARG USERNAME=mchorse
ARG USER_UID=1000
ARG USER_GID=$USER_UID
# Creating the user and usergroup
RUN groupadd --gid $USER_GID $USERNAME \
&& useradd --uid $USER_UID --gid $USERNAME -m $USERNAME \
&& echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
&& chmod 0440 /etc/sudoers.d/$USERNAME
RUN chmod g+rw /home && \
mkdir -p /home/mchorse && \
mkdir -p /home/mchorse/.ssh && \
chown -R $USERNAME:$USERNAME /home/mchorse && \
chown -R $USERNAME:$USERNAME /home/mchorse/.ssh
USER $USERNAME
### SSH
# Create keys
RUN sudo chmod 700 /home/mchorse/.ssh
RUN ssh-keygen -t rsa -N "" -f /home/mchorse/.ssh/id_rsa && sudo chmod 600 /home/mchorse/.ssh/id_rsa && sudo chmod 600 /home/mchorse/.ssh/id_rsa.pub
RUN cp /home/mchorse/.ssh/id_rsa.pub /home/mchorse/.ssh/authorized_keys
RUN eval `ssh-agent -s` && ssh-add /home/mchorse/.ssh/id_rsa
USER root
## SSH config and bashrc
RUN mkdir -p /home/mchorse/.ssh /job && \
echo 'Host *' > /home/mchorse/.ssh/config && \
echo ' StrictHostKeyChecking no' >> /home/mchorse/.ssh/config && \
echo 'export PDSH_RCMD_TYPE=ssh' >> /home/mchorse/.bashrc && \
echo 'export PATH=/home/mchorse/.local/bin:$PATH' >> /home/mchorse/.bashrc && \
echo 'export PATH=/opt/amazon/openmpi/bin/:/opt/amazon/efa/bin:/usr/bin:/usr/local/bin:$PATH' >> /home/mchorse/.bashrc && \
echo 'export LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:$LD_LIBRARY_PATH' >> /home/mchorse/.bashrc
RUN pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
RUN pip install gpustat protobuf~=3.19.0
## Install APEX
## we use the latest git clone and edit the setup.py, to disable the check around line 102
RUN git clone https://github.com/NVIDIA/apex.git $HOME/apex \
&& cd $HOME/apex/ \
&& pip3 install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./
RUN git clone https://github.com/EleutherAI/gpt-neox.git $HOME/gpt-neox \
&& cd $HOME/gpt-neox/ \
&& chmod -R 777 $HOME/gpt-neox/
# mchorse
USER mchorse
WORKDIR /home/mchorse
# For intrapod ssh
EXPOSE 22
# Starting scripts
COPY helpers/entrypoint.sh ./entrypoint.sh
COPY helpers/gpumon.sh ./gpumon.sh
RUN sudo chmod +x ./entrypoint.sh
RUN sudo chmod +x ./gpumon.sh
RUN bash ./gpumon.sh
COPY helpers/.deepspeed_env ./.deepspeed_env
ENTRYPOINT [ "./entrypoint.sh" ]
USER root
COPY helpers/requirements.txt .
RUN pip install -r requirements.txt
COPY helpers/fused_kernels-0.0.1-cp38-cp38-linux_x86_64.whl .
RUN pip install fused_kernels-0.0.1-cp38-cp38-linux_x86_64.whl