opensearch-project · peterzhuamazon · Mar 7, 2024 · Mar 7, 2024
@@ -0,0 +1 @@
+. /opt/rh/gcc-toolset-9/enable
@@ -70,8 +70,8 @@ mv -v "$libFound" "$libPathFound\\$libNameRequired"
 # Add MINGW_BIN path to User Env Var for k-NN to retrieve libs
 [System.Environment]::SetEnvironmentVariable("MINGW_BIN", "$libPathFound", [System.EnvironmentVariableTarget]::User)
 
-# Install zlib for k-NN compilation requirements
-scoop install zlib
+# Install zlib for k-NN compilation requirements, lock to 1.2.12
+scoop install https://raw.githubusercontent.com/ScoopInstaller/Extras/d4c775a2170c2f17ea83c7ced9cff2aafe3050fe/bucket/zlib.json
 # Reg PEP
 $zlibVersionInfo = (scoop info zlib | out-string -stream | Select-String 'Version.*:')
 $zlibVersionNumber = ($zlibVersionInfo -split ':' | select -last 1)
@@ -179,16 +179,6 @@ $userenv2 = [System.Environment]::GetEnvironmentVariable("Path", [System.Environ
 $nodePathFixed = "C:\\Users\\ContainerAdministrator\\scoop\\persist\\volta\\appdata\\bin"
 [System.Environment]::SetEnvironmentVariable("PATH", $userenv2 + ";$nodePathFixed", [System.EnvironmentVariableTarget]::User)
 
-# Install chromium (internally it is chrome.exe in app directory)
-# Lock chromium to v114.0.5735.134-r1135570 due to https://github.com/opensearch-project/opensearch-build/issues/4241
-scoop install https://raw.githubusercontent.com/ScoopInstaller/Extras/6befedcb5296cacbb0428e76baab7368609b6006/bucket/chromium.json
-$chromiumName = 'chrome.exe'
-$chromiumDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\chromium'
-$chromiumFound = (Get-ChildItem -Path $chromiumDir -Filter $chromiumName -Recurse | %{$_.FullName} | select -first 1)
-$chromiumFound
-# Add BROWSER_PATH path to User Env Var for cypress test to retrieve chromium.exe path
-[System.Environment]::SetEnvironmentVariable("BROWSER_PATH", "$chromiumFound", [System.EnvironmentVariableTarget]::User)
-
 # Install ruby24
 scoop install ruby24
 ruby --version

@@ -0,0 +1,107 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This is a docker image specifically for standardize the ci/cd environment
+# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards
+# Please read the README.md file for all the information before using this dockerfile
+
+FROM almalinux:8
+
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
+# Ensure localedef running correct with root permission
+USER 0
+
+# Add normal dependencies
+RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install epel-release -y && dnf repolist && \
+    dnf update -y && \
+    dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq
+
+# Tools setup
+COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/
+RUN dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh
+
+# Create user group
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
+
+# Add Python dependencies
+RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
+
+# Add Dashboards dependencies
+RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
+
+# Add Notebook dependencies
+RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && yum clean all
+
+# Add Yarn dependencies
+RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/*
+
+# Setup Shared Memory
+RUN chmod -R 777 /dev/shm
+
+# Install PKG builder dependencies with rvm
+RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \
+    curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - && \
+    curl -sSL https://get.rvm.io | bash -s stable
+
+# Switch shell for rvm related commands
+SHELL ["/bin/bash", "-lc"]
+
+# Install ruby / rpm / fpm related dependencies
+RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all
+
+ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
+ENV RVM_HOME=/usr/local/rvm/bin
+ENV GEM_HOME=$CONTAINER_USER_HOME/.gem
+ENV GEM_PATH=$GEM_HOME
+ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
+
+# Install Python binary
+RUN update-alternatives --set python /usr/bin/python3.9 && \
+    update-alternatives --set python3 /usr/bin/python3.9 && \
+    pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17
+
+# Preparation for awscliv2
+#RUN pip3 install git+https://github.com/aws/aws-cli.git@2.11.17
+#ENV AWS_CLI_FILE_ENCODING=UTF-8
+
+# Change User
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
+
+# Install fpm for opensearch dashboards core
+RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2
+ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
+
+# Hard code node version and yarn version for now
+# nvm environment variables
+ENV NVM_DIR $CONTAINER_USER_HOME/.nvm
+ENV NODE_VERSION 10.24.1
+ARG NODE_VERSION_LIST="10.24.1 14.19.1 14.20.0 14.20.1 14.21.3 16.20.0 18.16.0 18.19.0"
+# install nvm
+# https://github.com/creationix/nvm#install-script
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+# install node and npm
+COPY --chown=$CONTAINER_USER:$CONTAINER_USER config/yarn-version.sh /tmp
+RUN source $NVM_DIR/nvm.sh && \
+    for node_version in $NODE_VERSION_LIST; do nvm install $node_version; npm install -g yarn@`/tmp/yarn-version.sh main`; done
+# add node and npm to path so the commands are available
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+# We use the version test to check if packages installed correctly
+# And get added to the PATH
+# This will fail the docker build if any of the packages not exist
+RUN node -v
+RUN npm -v
+RUN yarn -v
+RUN fpm -v
@@ -0,0 +1,107 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This is a docker image specifically for standardize the ci/cd environment
+# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards
+# Please read the README.md file for all the information before using this dockerfile
+
+
+FROM almalinux:8
+
+ARG MAVEN_DIR=/usr/local/apache-maven
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
+# Ensure localedef running correct with root permission
+USER 0
+
+# Add normal dependencies
+RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \
+    dnf update -y && \
+    dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq
+
+# Create user group
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
+
+# Add Python dependencies
+RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
+
+# Add Dashboards dependencies
+RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
+
+# Add Notebook dependencies
+RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && dnf clean all
+
+# Add Yarn dependencies
+RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/*
+
+# Tools setup
+COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/
+RUN dnf install -y go && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh
+
+# Install higher version of maven 3.8.x
+RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d '"' | uniq | head -n 1`  && \
+    mkdir -p $MAVEN_DIR && (curl -s $MAVEN_URL | tar xzf - --strip-components=1 -C $MAVEN_DIR) && \
+    echo "export M2_HOME=$MAVEN_DIR" > /etc/profile.d/maven_path.sh && \
+    echo "export M2=\$M2_HOME/bin" >> /etc/profile.d/maven_path.sh && \
+    echo "export PATH=\$M2:\$PATH" >> /etc/profile.d/maven_path.sh && \
+    ln -sfn $MAVEN_DIR/bin/mvn /usr/local/bin/mvn
+
+# Setup Shared Memory
+RUN chmod -R 777 /dev/shm
+
+# Install PKG builder dependencies with rvm
+RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \
+    curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - && \
+    curl -sSL https://get.rvm.io | bash -s stable
+
+# Switch shell for rvm related commands
+SHELL ["/bin/bash", "-lc"]
+CMD ["/bin/bash", "-l"]
+
+# Install ruby / rpm / fpm related dependencies
+RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build rpm-sign createrepo pinentry && dnf clean all
+
+ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
+ENV RVM_HOME=/usr/local/rvm/bin
+ENV GEM_HOME=$CONTAINER_USER_HOME/.gem
+ENV GEM_PATH=$GEM_HOME
+ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
+
+# Install Python binary
+RUN update-alternatives --set python /usr/bin/python3.9 && \
+    update-alternatives --set python3 /usr/bin/python3.9 && \
+    pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17
+
+# Add k-NN Library dependencies
+# EL8 requires install config-manager and enable powertools to consume openblas-static
+RUN dnf install -y 'dnf-command(config-manager)' && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install epel-release -y && dnf repolist && \
+    dnf install openblas-static lapack gcc-gfortran -y && dnf clean all
+RUN pip3 install cmake==3.23.3
+# Upgrade gcc
+# The setup part is partially based on Austin Dewey's article:
+# https://austindewey.com/2019/03/26/enabling-software-collections-binaries-on-a-docker-image/
+RUN dnf -y install gcc-toolset-9-gcc gcc-toolset-9-gcc-c++ && dnf clean all && \
+    echo "source /opt/rh/gcc-toolset-9/enable" > /etc/profile.d/gcc-toolset-9.sh
+COPY --chown=0:0 config/gcc-toolset-9-setup /usr/local/bin/gcc_setup
+ENV BASH_ENV="/usr/local/bin/gcc_setup"
+ENV ENV="/usr/local/bin/gcc_setup"
+ENV PROMPT_COMMAND=". /usr/local/bin/gcc_setup"
+
+# Change User
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
+
+# Install fpm for opensearch dashboards core
+RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2
+ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
+RUN fpm -v
@@ -0,0 +1,149 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This is a docker image specifically for standardize the ci/cd environment
+# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards
+# Please read the README.md file for all the information before using this dockerfile
+
+########################### Stage 0 ########################
+FROM almalinux:8 AS linux_stage_0
+
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
+USER 0
+
+# Add normal dependencies
+RUN dnf clean all && \
+    dnf update -y && \
+    dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip
+
+# Add Dashboards dependencies (mainly for cypress)
+RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
+
+# Add Yarn dependencies
+RUN dnf groupinstall -y "Development Tools" && dnf install -y cmake && dnf clean all && rm -rf /var/cache/dnf/*
+
+# Create user group
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
+
+# install yq
+COPY --chown=0:0 config/yq-setup.sh /tmp/
+RUN /tmp/yq-setup.sh
+
+# Change User
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
+
+# Hard code node version and yarn version for now
+# nvm environment variables
+ENV NVM_DIR $CONTAINER_USER_HOME/.nvm
+ENV NODE_VERSION 18.19.0
+ENV CYPRESS_VERSION 12.13.0
+ARG CYPRESS_VERSION_LIST="5.6.0 9.5.4 12.13.0"
+ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION
+ENV CYPRESS_LOCATION_954 $CONTAINER_USER_HOME/.cache/Cypress/9.5.4
+# install nvm
+# https://github.com/creationix/nvm#install-script
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+# install node and npm
+RUN source $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default
+# add node and npm to path so the commands are available
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+# install yarn
+COPY --chown=$CONTAINER_USER:$CONTAINER_USER config/yarn-version.sh /tmp
+RUN npm install -g yarn@`/tmp/yarn-version.sh main`
+# Add legacy cypress@5.6.0 for 1.x line
+# Add legacy cypress@9.5.4 for pre-2.8.0 releases
+# Add latest cypress@12.13.0 for post-2.8.0 releases
+RUN for cypress_version in $CYPRESS_VERSION_LIST; do npm install -g cypress@$cypress_version && npm cache verify; done
+
+# Need root to get pass the build due to chrome sandbox needs to own by the root
+USER 0
+
+# Add legacy cypress 5.6.0 / 9.5.4 for ARM64 Architecture
+RUN if [ `uname -m` = "aarch64" ]; then for cypress_version in 5.6.0 9.5.4; do rm -rf $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && \
+    curl -SLO https://ci.opensearch.org/ci/dbc/tools/Cypress-$cypress_version-arm64.tar.gz && tar -xzf Cypress-$cypress_version-arm64.tar.gz -C $CONTAINER_USER_HOME/.cache/Cypress/ && \
+    chown $CONTAINER_USER:$CONTAINER_USER -R $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && rm -vf Cypress-$cypress_version-arm64.tar.gz; done; fi
+
+########################### Stage 1 ########################
+FROM almalinux:8
+
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
+USER 0
+
+# Add normal dependencies
+RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \
+    dnf update -y && \
+    dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip
+
+# Create user group
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
+
+# Copy from Stage0
+COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME
+ENV NVM_DIR $CONTAINER_USER_HOME/.nvm
+ENV NODE_VERSION 18.19.0
+ENV CYPRESS_VERSION 12.13.0
+ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+
+# Check dirs
+RUN source $NVM_DIR/nvm.sh && ls -al $CONTAINER_USER_HOME && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION
+
+# Add Python dependencies
+RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
+
+# Add Dashboards dependencies (mainly for cypress)
+RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
+
+# Add Reports dependencies
+RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype
+
+# Add Yarn dependencies
+RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/*
+
+# Setup Shared Memory
+RUN chmod -R 777 /dev/shm
+
+# Install Python binary
+RUN update-alternatives --set python /usr/bin/python3.9 && \
+    update-alternatives --set python3 /usr/bin/python3.9 && \
+    pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17
+
+# Add other dependencies
+RUN dnf install -y epel-release && dnf clean all && dnf install -y jq && dnf clean all && rm -rf /var/cache/dnf/* && \
+    pip3 install cmake==3.23.3
+
+# Tools setup
+COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/
+RUN dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh
+
+# Change User
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
+
+# We use the version test to check if packages installed correctly
+# And get added to the PATH
+# This will fail the docker build if any of the packages not exist
+RUN node -v
+RUN npm -v
+RUN yarn -v
+RUN cypress -v