From 10b55144fdbb05e74410aff312232f4a90e45e59 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 Mar 2024 19:40:36 +0000 Subject: [PATCH] Switch all rockylinux8 images to almalinux8 to support ppc64le Signed-off-by: Peter Zhu --- docker/ci/config/gcc-toolset-9-setup | 1 + docker/ci/config/windows-setup.ps1 | 14 +- ...opensearch-dashboards.x64.arm64.dockerfile | 107 +++++++++++ ...almalinux8.opensearch.x64.arm64.dockerfile | 107 +++++++++++ ...opensearch-dashboards.x64.arm64.dockerfile | 149 +++++++++++++++ ...malinux8.systemd-base.x64.arm64.dockerfile | 177 ++++++++++++++++++ ...opensearch-dashboards.x64.arm64.dockerfile | 0 ...ockylinux8.opensearch.x64.arm64.dockerfile | 0 ...opensearch-dashboards.x64.arm64.dockerfile | 0 ...kylinux8.systemd-base.x64.arm64.dockerfile | 0 10 files changed, 543 insertions(+), 12 deletions(-) create mode 100755 docker/ci/config/gcc-toolset-9-setup create mode 100644 docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.dockerfile create mode 100644 docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.dockerfile create mode 100644 docker/ci/dockerfiles/current/test.almalinux8.opensearch-dashboards.x64.arm64.dockerfile create mode 100644 docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.dockerfile rename docker/ci/dockerfiles/{current => legacy}/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile (100%) rename docker/ci/dockerfiles/{current => legacy}/build.rockylinux8.opensearch.x64.arm64.dockerfile (100%) rename docker/ci/dockerfiles/{current => legacy}/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile (100%) rename docker/ci/dockerfiles/{current => legacy}/test.rockylinux8.systemd-base.x64.arm64.dockerfile (100%) diff --git a/docker/ci/config/gcc-toolset-9-setup b/docker/ci/config/gcc-toolset-9-setup new file mode 100755 index 0000000000..9547d06e33 --- /dev/null +++ b/docker/ci/config/gcc-toolset-9-setup @@ -0,0 +1 @@ +. /opt/rh/gcc-toolset-9/enable diff --git a/docker/ci/config/windows-setup.ps1 b/docker/ci/config/windows-setup.ps1 index 7446aab86e..9f91bd8c80 100644 --- a/docker/ci/config/windows-setup.ps1 +++ b/docker/ci/config/windows-setup.ps1 @@ -70,8 +70,8 @@ mv -v "$libFound" "$libPathFound\\$libNameRequired" # Add MINGW_BIN path to User Env Var for k-NN to retrieve libs [System.Environment]::SetEnvironmentVariable("MINGW_BIN", "$libPathFound", [System.EnvironmentVariableTarget]::User) -# Install zlib for k-NN compilation requirements -scoop install zlib +# Install zlib for k-NN compilation requirements, lock to 1.2.12 +scoop install https://raw.githubusercontent.com/ScoopInstaller/Extras/d4c775a2170c2f17ea83c7ced9cff2aafe3050fe/bucket/zlib.json # Reg PEP $zlibVersionInfo = (scoop info zlib | out-string -stream | Select-String 'Version.*:') $zlibVersionNumber = ($zlibVersionInfo -split ':' | select -last 1) @@ -179,16 +179,6 @@ $userenv2 = [System.Environment]::GetEnvironmentVariable("Path", [System.Environ $nodePathFixed = "C:\\Users\\ContainerAdministrator\\scoop\\persist\\volta\\appdata\\bin" [System.Environment]::SetEnvironmentVariable("PATH", $userenv2 + ";$nodePathFixed", [System.EnvironmentVariableTarget]::User) -# Install chromium (internally it is chrome.exe in app directory) -# Lock chromium to v114.0.5735.134-r1135570 due to https://github.com/opensearch-project/opensearch-build/issues/4241 -scoop install https://raw.githubusercontent.com/ScoopInstaller/Extras/6befedcb5296cacbb0428e76baab7368609b6006/bucket/chromium.json -$chromiumName = 'chrome.exe' -$chromiumDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\chromium' -$chromiumFound = (Get-ChildItem -Path $chromiumDir -Filter $chromiumName -Recurse | %{$_.FullName} | select -first 1) -$chromiumFound -# Add BROWSER_PATH path to User Env Var for cypress test to retrieve chromium.exe path -[System.Environment]::SetEnvironmentVariable("BROWSER_PATH", "$chromiumFound", [System.EnvironmentVariableTarget]::User) - # Install ruby24 scoop install ruby24 ruby --version diff --git a/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.dockerfile new file mode 100644 index 0000000000..637baa1d56 --- /dev/null +++ b/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.dockerfile @@ -0,0 +1,107 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for standardize the ci/cd environment +# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards +# Please read the README.md file for all the information before using this dockerfile + +FROM almalinux:8 + +ARG CONTAINER_USER=ci-runner +ARG CONTAINER_USER_HOME=/home/ci-runner + +# Ensure localedef running correct with root permission +USER 0 + +# Add normal dependencies +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ + dnf config-manager --set-enabled powertools && \ + dnf install epel-release -y && dnf repolist && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq + +# Tools setup +COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh + +# Create user group +RUN groupadd -g 1000 $CONTAINER_USER && \ + useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \ + mkdir -p $CONTAINER_USER_HOME && \ + chown -R 1000:1000 $CONTAINER_USER_HOME + +# Add Python dependencies +RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + +# Add Dashboards dependencies +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Notebook dependencies +RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && yum clean all + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/* + +# Setup Shared Memory +RUN chmod -R 777 /dev/shm + +# Install PKG builder dependencies with rvm +RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ + curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - && \ + curl -sSL https://get.rvm.io | bash -s stable + +# Switch shell for rvm related commands +SHELL ["/bin/bash", "-lc"] + +# Install ruby / rpm / fpm related dependencies +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all + +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin +ENV RVM_HOME=/usr/local/rvm/bin +ENV GEM_HOME=$CONTAINER_USER_HOME/.gem +ENV GEM_PATH=$GEM_HOME +ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH + +# Install Python binary +RUN update-alternatives --set python /usr/bin/python3.9 && \ + update-alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 + +# Preparation for awscliv2 +#RUN pip3 install git+https://github.com/aws/aws-cli.git@2.11.17 +#ENV AWS_CLI_FILE_ENCODING=UTF-8 + +# Change User +USER $CONTAINER_USER +WORKDIR $CONTAINER_USER_HOME + +# Install fpm for opensearch dashboards core +RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2 +ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH + +# Hard code node version and yarn version for now +# nvm environment variables +ENV NVM_DIR $CONTAINER_USER_HOME/.nvm +ENV NODE_VERSION 10.24.1 +ARG NODE_VERSION_LIST="10.24.1 14.19.1 14.20.0 14.20.1 14.21.3 16.20.0 18.16.0 18.19.0" +# install nvm +# https://github.com/creationix/nvm#install-script +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +# install node and npm +COPY --chown=$CONTAINER_USER:$CONTAINER_USER config/yarn-version.sh /tmp +RUN source $NVM_DIR/nvm.sh && \ + for node_version in $NODE_VERSION_LIST; do nvm install $node_version; npm install -g yarn@`/tmp/yarn-version.sh main`; done +# add node and npm to path so the commands are available +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH +# We use the version test to check if packages installed correctly +# And get added to the PATH +# This will fail the docker build if any of the packages not exist +RUN node -v +RUN npm -v +RUN yarn -v +RUN fpm -v diff --git a/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.dockerfile new file mode 100644 index 0000000000..6a4fbddecf --- /dev/null +++ b/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.dockerfile @@ -0,0 +1,107 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for standardize the ci/cd environment +# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards +# Please read the README.md file for all the information before using this dockerfile + + +FROM almalinux:8 + +ARG MAVEN_DIR=/usr/local/apache-maven +ARG CONTAINER_USER=ci-runner +ARG CONTAINER_USER_HOME=/home/ci-runner + +# Ensure localedef running correct with root permission +USER 0 + +# Add normal dependencies +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq + +# Create user group +RUN groupadd -g 1000 $CONTAINER_USER && \ + useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \ + mkdir -p $CONTAINER_USER_HOME && \ + chown -R 1000:1000 $CONTAINER_USER_HOME + +# Add Python dependencies +RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + +# Add Dashboards dependencies +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Notebook dependencies +RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && dnf clean all + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/* + +# Tools setup +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN dnf install -y go && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh + +# Install higher version of maven 3.8.x +RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d '"' | uniq | head -n 1` && \ + mkdir -p $MAVEN_DIR && (curl -s $MAVEN_URL | tar xzf - --strip-components=1 -C $MAVEN_DIR) && \ + echo "export M2_HOME=$MAVEN_DIR" > /etc/profile.d/maven_path.sh && \ + echo "export M2=\$M2_HOME/bin" >> /etc/profile.d/maven_path.sh && \ + echo "export PATH=\$M2:\$PATH" >> /etc/profile.d/maven_path.sh && \ + ln -sfn $MAVEN_DIR/bin/mvn /usr/local/bin/mvn + +# Setup Shared Memory +RUN chmod -R 777 /dev/shm + +# Install PKG builder dependencies with rvm +RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ + curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - && \ + curl -sSL https://get.rvm.io | bash -s stable + +# Switch shell for rvm related commands +SHELL ["/bin/bash", "-lc"] +CMD ["/bin/bash", "-l"] + +# Install ruby / rpm / fpm related dependencies +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build rpm-sign createrepo pinentry && dnf clean all + +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin +ENV RVM_HOME=/usr/local/rvm/bin +ENV GEM_HOME=$CONTAINER_USER_HOME/.gem +ENV GEM_PATH=$GEM_HOME +ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH + +# Install Python binary +RUN update-alternatives --set python /usr/bin/python3.9 && \ + update-alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 + +# Add k-NN Library dependencies +# EL8 requires install config-manager and enable powertools to consume openblas-static +RUN dnf install -y 'dnf-command(config-manager)' && \ + dnf config-manager --set-enabled powertools && \ + dnf install epel-release -y && dnf repolist && \ + dnf install openblas-static lapack gcc-gfortran -y && dnf clean all +RUN pip3 install cmake==3.23.3 +# Upgrade gcc +# The setup part is partially based on Austin Dewey's article: +# https://austindewey.com/2019/03/26/enabling-software-collections-binaries-on-a-docker-image/ +RUN dnf -y install gcc-toolset-9-gcc gcc-toolset-9-gcc-c++ && dnf clean all && \ + echo "source /opt/rh/gcc-toolset-9/enable" > /etc/profile.d/gcc-toolset-9.sh +COPY --chown=0:0 config/gcc-toolset-9-setup /usr/local/bin/gcc_setup +ENV BASH_ENV="/usr/local/bin/gcc_setup" +ENV ENV="/usr/local/bin/gcc_setup" +ENV PROMPT_COMMAND=". /usr/local/bin/gcc_setup" + +# Change User +USER $CONTAINER_USER +WORKDIR $CONTAINER_USER_HOME + +# Install fpm for opensearch dashboards core +RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2 +ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH +RUN fpm -v diff --git a/docker/ci/dockerfiles/current/test.almalinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/test.almalinux8.opensearch-dashboards.x64.arm64.dockerfile new file mode 100644 index 0000000000..0b83803154 --- /dev/null +++ b/docker/ci/dockerfiles/current/test.almalinux8.opensearch-dashboards.x64.arm64.dockerfile @@ -0,0 +1,149 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for standardize the ci/cd environment +# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards +# Please read the README.md file for all the information before using this dockerfile + +########################### Stage 0 ######################## +FROM almalinux:8 AS linux_stage_0 + +ARG CONTAINER_USER=ci-runner +ARG CONTAINER_USER_HOME=/home/ci-runner + +USER 0 + +# Add normal dependencies +RUN dnf clean all && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip + +# Add Dashboards dependencies (mainly for cypress) +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf install -y cmake && dnf clean all && rm -rf /var/cache/dnf/* + +# Create user group +RUN groupadd -g 1000 $CONTAINER_USER && \ + useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \ + mkdir -p $CONTAINER_USER_HOME && \ + chown -R 1000:1000 $CONTAINER_USER_HOME + +# install yq +COPY --chown=0:0 config/yq-setup.sh /tmp/ +RUN /tmp/yq-setup.sh + +# Change User +USER $CONTAINER_USER +WORKDIR $CONTAINER_USER_HOME + +# Hard code node version and yarn version for now +# nvm environment variables +ENV NVM_DIR $CONTAINER_USER_HOME/.nvm +ENV NODE_VERSION 18.19.0 +ENV CYPRESS_VERSION 12.13.0 +ARG CYPRESS_VERSION_LIST="5.6.0 9.5.4 12.13.0" +ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION +ENV CYPRESS_LOCATION_954 $CONTAINER_USER_HOME/.cache/Cypress/9.5.4 +# install nvm +# https://github.com/creationix/nvm#install-script +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +# install node and npm +RUN source $NVM_DIR/nvm.sh \ + && nvm install $NODE_VERSION \ + && nvm alias default $NODE_VERSION \ + && nvm use default +# add node and npm to path so the commands are available +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH +# install yarn +COPY --chown=$CONTAINER_USER:$CONTAINER_USER config/yarn-version.sh /tmp +RUN npm install -g yarn@`/tmp/yarn-version.sh main` +# Add legacy cypress@5.6.0 for 1.x line +# Add legacy cypress@9.5.4 for pre-2.8.0 releases +# Add latest cypress@12.13.0 for post-2.8.0 releases +RUN for cypress_version in $CYPRESS_VERSION_LIST; do npm install -g cypress@$cypress_version && npm cache verify; done + +# Need root to get pass the build due to chrome sandbox needs to own by the root +USER 0 + +# Add legacy cypress 5.6.0 / 9.5.4 for ARM64 Architecture +RUN if [ `uname -m` = "aarch64" ]; then for cypress_version in 5.6.0 9.5.4; do rm -rf $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && \ + curl -SLO https://ci.opensearch.org/ci/dbc/tools/Cypress-$cypress_version-arm64.tar.gz && tar -xzf Cypress-$cypress_version-arm64.tar.gz -C $CONTAINER_USER_HOME/.cache/Cypress/ && \ + chown $CONTAINER_USER:$CONTAINER_USER -R $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && rm -vf Cypress-$cypress_version-arm64.tar.gz; done; fi + +########################### Stage 1 ######################## +FROM almalinux:8 + +ARG CONTAINER_USER=ci-runner +ARG CONTAINER_USER_HOME=/home/ci-runner + +USER 0 + +# Add normal dependencies +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip + +# Create user group +RUN groupadd -g 1000 $CONTAINER_USER && \ + useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \ + mkdir -p $CONTAINER_USER_HOME && \ + chown -R 1000:1000 $CONTAINER_USER_HOME + +# Copy from Stage0 +COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME +ENV NVM_DIR $CONTAINER_USER_HOME/.nvm +ENV NODE_VERSION 18.19.0 +ENV CYPRESS_VERSION 12.13.0 +ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH + +# Check dirs +RUN source $NVM_DIR/nvm.sh && ls -al $CONTAINER_USER_HOME && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION + +# Add Python dependencies +RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + +# Add Dashboards dependencies (mainly for cypress) +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Reports dependencies +RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/* + +# Setup Shared Memory +RUN chmod -R 777 /dev/shm + +# Install Python binary +RUN update-alternatives --set python /usr/bin/python3.9 && \ + update-alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 + +# Add other dependencies +RUN dnf install -y epel-release && dnf clean all && dnf install -y jq && dnf clean all && rm -rf /var/cache/dnf/* && \ + pip3 install cmake==3.23.3 + +# Tools setup +COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh + +# Change User +USER $CONTAINER_USER +WORKDIR $CONTAINER_USER_HOME + +# We use the version test to check if packages installed correctly +# And get added to the PATH +# This will fail the docker build if any of the packages not exist +RUN node -v +RUN npm -v +RUN yarn -v +RUN cypress -v diff --git a/docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.dockerfile new file mode 100644 index 0000000000..288535b6d4 --- /dev/null +++ b/docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.dockerfile @@ -0,0 +1,177 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for setting up systemd env base for services with root user +# It is initially designed to test pkg installation, but can be used for anything that requires systemd +# It used the method posted by Daniel Walsh: https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container + +# In order to run images with systemd, you need to run in privileged mode: `docker run --privileged -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro ` +# If you use this image in jenkins pipeline you need to add these arguments: `args '--entrypoint=/usr/sbin/init -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro'` + +# 20230920: On Docker host with systemd version > 247 you need to use these args: +# https://github.com/opensearch-project/opensearch-build/issues/4047 +# --entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host + +########################### Stage 0 ######################## +FROM almalinux:8 AS linux_stage_0 + +ENV container docker +ARG CONTAINER_USER=ci-runner +ARG CONTAINER_USER_HOME=/home/ci-runner + +USER 0 + +# Add normal dependencies +RUN dnf clean all && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq + +# Add Dashboards dependencies (mainly for cypress) +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf install -y cmake && dnf clean all + +# Create user group +RUN groupadd -g 1000 $CONTAINER_USER && \ + useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \ + mkdir -p $CONTAINER_USER_HOME && \ + chown -R 1000:1000 $CONTAINER_USER_HOME + +# install yq +COPY --chown=0:0 config/yq-setup.sh /tmp/ +RUN /tmp/yq-setup.sh + +# Change User +USER $CONTAINER_USER +WORKDIR $CONTAINER_USER_HOME + +# Hard code node version and yarn version for now +# nvm environment variables +ENV NVM_DIR $CONTAINER_USER_HOME/.nvm +ENV NODE_VERSION 18.19.0 +ENV CYPRESS_VERSION 12.13.0 +ARG CYPRESS_VERSION_LIST="5.6.0 9.5.4 12.13.0" +ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION +ENV CYPRESS_LOCATION_954 $CONTAINER_USER_HOME/.cache/Cypress/9.5.4 +# install nvm +# https://github.com/creationix/nvm#install-script +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +# install node and npm +RUN source $NVM_DIR/nvm.sh \ + && nvm install $NODE_VERSION \ + && nvm alias default $NODE_VERSION \ + && nvm use default +# add node and npm to path so the commands are available +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH +# install yarn +COPY --chown=$CONTAINER_USER:$CONTAINER_USER config/yarn-version.sh /tmp +RUN npm install -g yarn@`/tmp/yarn-version.sh main` +# Add legacy cypress@5.6.0 for 1.x line +# Add legacy cypress@9.5.4 for pre-2.8.0 releases +# Add latest cypress@12.13.0 for post-2.8.0 releases +RUN for cypress_version in $CYPRESS_VERSION_LIST; do npm install -g cypress@$cypress_version && npm cache verify; done + +# Need root to get pass the build due to chrome sandbox needs to own by the root +USER 0 + +# Add legacy cypress 5.6.0 / 9.5.4 for ARM64 Architecture +RUN if [ `uname -m` = "aarch64" ]; then for cypress_version in 5.6.0 9.5.4; do rm -rf $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && \ + curl -SLO https://ci.opensearch.org/ci/dbc/tools/Cypress-$cypress_version-arm64.tar.gz && tar -xzf Cypress-$cypress_version-arm64.tar.gz -C $CONTAINER_USER_HOME/.cache/Cypress/ && \ + chown $CONTAINER_USER:$CONTAINER_USER -R $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && rm -vf Cypress-$cypress_version-arm64.tar.gz; done; fi + +########################### Stage 1 ######################## +FROM almalinux:8 + +ARG CONTAINER_USER=ci-runner +ARG CONTAINER_USER_HOME=/home/ci-runner + +USER 0 + +# Add normal dependencies +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq + +# Create user group +RUN dnf install -y sudo && \ + groupadd -g 1000 $CONTAINER_USER && \ + useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \ + mkdir -p $CONTAINER_USER_HOME && \ + chown -R 1000:1000 $CONTAINER_USER_HOME && \ + groupadd -g 1001 opensearch && \ + useradd -u 1001 -g 1001 opensearch && \ + groupadd -g 1002 opensearch-dashboards && \ + useradd -u 1002 -g 1002 opensearch-dashboards && \ + usermod -a -G opensearch $CONTAINER_USER && \ + usermod -a -G opensearch-dashboards $CONTAINER_USER && \ + id && \ + echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER + +# Copy from Stage0 +COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME +ENV NVM_DIR $CONTAINER_USER_HOME/.nvm +ENV NODE_VERSION 18.19.0 +ENV CYPRESS_VERSION 12.13.0 +ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH + +# Check dirs +RUN source $NVM_DIR/nvm.sh && ls -al $CONTAINER_USER_HOME && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION + +# Add Python dependencies +RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + +# Add Dashboards dependencies (mainly for cypress) +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Reports dependencies +RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf clean all + +# Tools setup +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN /tmp/jdk-setup.sh && dnf remove -y "java-1.8.0*" && dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh + +# Setup Shared Memory +RUN chmod -R 777 /dev/shm + +# Install Python binary +RUN update-alternatives --set python /usr/bin/python3.9 && \ + update-alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 + +# Add other dependencies +RUN dnf install -y epel-release && dnf clean all && dnf install -y jq && dnf clean all && \ + pip3 install cmake==3.23.3 + +# We use the version test to check if packages installed correctly +# And get added to the PATH +# This will fail the docker build if any of the packages not exist +RUN node -v +RUN npm -v +RUN yarn -v +RUN cypress -v + +# Possible retain of multi-user.target.wants later due to PA +# As of now we do not need this +RUN dnf -y install systemd procps util-linux-ng openssl openssl-devel which curl git gnupg2 tar net-tools jq && dnf clean all && \ + systemctl set-default multi-user && \ +(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ +rm -f /lib/systemd/system/multi-user.target.wants/*;\ +rm -f /etc/systemd/system/*.wants/*;\ +rm -f /lib/systemd/system/local-fs.target.wants/*; \ +rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ +rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ +rm -f /lib/systemd/system/basic.target.wants/*;\ +rm -f /lib/systemd/system/anaconda.target.wants/*; + +CMD ["/usr/sbin/init"] diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/build.rockylinux8.opensearch.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/build.rockylinux8.opensearch.x64.arm64.dockerfile diff --git a/docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile diff --git a/docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/test.rockylinux8.systemd-base.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/test.rockylinux8.systemd-base.x64.arm64.dockerfile