From 3529ec91cb3a0ff7e07ad96066d8ede8c7e5525b Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 2 Jun 2023 17:11:02 -0400 Subject: [PATCH 1/7] add docker workflow - adds dockerfiles - adds pip requirements, modifies existing requirements (add cmake) - adds composite github action - adds dockerignore file --- .dockerignore | 6 ++ .github/actions/docker_setup/action.yml | 32 ++++++ .github/workflows/docker.yml | 137 ++++++++++++++++++++++++ docker/Dockerfile.base | 47 ++++++++ docker/Dockerfile.stempy | 34 ++++++ docker/apt-packages-common.txt | 20 ++++ docker/apt-packages-dev.txt | 2 + docker/requirements-ipykernel.txt | 6 ++ docker/requirements-normal.txt | 4 + requirements.txt | 1 + 10 files changed, 289 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/actions/docker_setup/action.yml create mode 100644 .github/workflows/docker.yml create mode 100644 docker/Dockerfile.base create mode 100644 docker/Dockerfile.stempy create mode 100644 docker/apt-packages-common.txt create mode 100644 docker/apt-packages-dev.txt create mode 100644 docker/requirements-ipykernel.txt create mode 100644 docker/requirements-normal.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1e725c16 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +/docs/ +/.github/ +/examples/ +/LICENSE +/.readthedocs.yml +/.gitignore \ No newline at end of file diff --git a/.github/actions/docker_setup/action.yml b/.github/actions/docker_setup/action.yml new file mode 100644 index 00000000..de2492ec --- /dev/null +++ b/.github/actions/docker_setup/action.yml @@ -0,0 +1,32 @@ +name: 'Setup Workflow' +description: 'Set up environment and Docker settings' + +inputs: + github_event_name: + description: 'GitHub Event Name' + required: true + DOCKERHUB_USERNAME: + description: 'DockerHub Username' + required: true + DOCKERHUB_TOKEN: + description: 'DockerHub Token' + required: true + DOCKERHUB_ORG: + description: 'DockerHub Organization' + required: true + +runs: + using: 'composite' + steps: + - name: Set DOCKERHUB_ORG + run: | + echo "DOCKERHUB_ORG=${{ inputs.DOCKERHUB_ORG == '' && 'openchemistry' || inputs.DOCKERHUB_ORG }}" >> $GITHUB_ENV + shell: bash + - name: Login to Docker Hub + uses: docker/login-action@v2 + if: ${{ github.event_name != 'pull_request' }} + with: + username: ${{ inputs.DOCKERHUB_USERNAME }} + password: ${{ inputs.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..684757a8 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,137 @@ +name: docker + +on: + push: + branches: + # - "master" + - 'gha-final' + pull_request: + branches: + - 'master' + +jobs: + # Builds base image for stempy of various verions if Dockerfile.base has changed + build-stempy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + mpi: ['ON', 'OFF'] + ipykernel: ['', 'ipykernel'] + dev: ['', 'dev'] + exclude: + - MPI: 'ON' + ipykernel: 'ipykernel' + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: 'Setup Docker Environment' + uses: ./.github/actions/docker_setup + with: + github_event_name: ${{ github.event_name }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_ORG: ${{ vars.DOCKERHUB_ORG }} + + - name: Determine if Dockerfile.base changed + id: changed-dockerfile-base + uses: tj-actions/changed-files@v35 + with: + files: | + ./docker/Dockerfile.base + ./docker/apt-packages-common.txt + ./docker/apt-packages-dev.txt + + - name: Set up environment variables for python version + id: set-conda-env + run: | + echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV + PYTHON_VERSION_NODOT=$(echo ${{ matrix.python-version }} | tr -d .) + echo "PYTHON_VERSION_NODOT=${PYTHON_VERSION_NODOT}" >> $GITHUB_ENV + - name: Set up other environment variables + id: set-tag-vars + run: | + echo "BASE_TARGET=$(if [[ "${{ matrix.mpi }}" == 'ON' ]]; then + echo 'mpi' + else + echo 'base' + fi)" >> $GITHUB_ENV + + MPI_TAG=$(if [[ "${{ matrix.mpi }}" == 'ON' ]]; then + echo '-mpi' + else + echo '' + fi) + echo "MPI_TAG=${MPI_TAG}" >> $GITHUB_ENV + + DEV_TAG=$(if [[ "${{ matrix.dev }}" == 'dev' ]]; then + echo '-dev' + else + echo '' + fi) + echo "DEV_TAG=${DEV_TAG}" >> $GITHUB_ENV + + echo "RELEASE_OR_DEBUG=$(if [[ "${{ matrix.dev }}" == 'dev' ]]; then + echo 'Debug' + else + echo 'Release' + fi)" >> $GITHUB_ENV + + COMMIT=$(git rev-parse --short HEAD) + echo "COMMIT=${COMMIT}" >> $GITHUB_ENV + + IPYKERNEL_TAG=$(if [[ "${{ matrix.ipykernel }}" == 'ipykernel' ]]; then + echo '-ipykernel' + else + echo '' + fi) + echo "IPYKERNEL_TAG=${IPYKERNEL_TAG}" >> $GITHUB_ENV + + BASE_TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}:py${{ env.PYTHON_VERSION_NODOT }}-base${DEV_TAG} + echo "BASE_TAG=${BASE_TAG}" >> $GITHUB_ENV + + TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}${IPYKERNEL_TAG}:py${{ env.PYTHON_VERSION_NODOT }}-${COMMIT}${DEV_TAG} + echo "TAG=${TAG}" >> $GITHUB_ENV + + LATEST_TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}${IPYKERNEL_TAG}:latest${DEV_TAG} + echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_ENV + + - name: Build/push Dockerfile.base + uses: docker/build-push-action@v3 + if: ${{ contains(github.event.head_commit.message, 'trigger-ci') || steps.changed-dockerfile-base.outputs.any_changed == 'true'}} + with: + context: . + file: ./docker/Dockerfile.base + push: ${{ github.event_name != 'pull_request' }} + build-args: | + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + DEV=${{ matrix.dev }} + tags: ${{ env.BASE_TAG }} + target: ${{ env.BASE_TARGET }} + cache-to: type=gha, mode=max + cache-from: type=gha, mode=max + + - name: Build/push Dockerfile.stempy + uses: docker/build-push-action@v3 + with: + context: . + file: ./docker/Dockerfile.stempy + push: ${{ github.event_name != 'pull_request' }} + build-args: | + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + BASE_IMAGE=${{ env.BASE_TAG }} + RELEASE_OR_DEBUG=${{ env.RELEASE_OR_DEBUG }} + MPI=${{ matrix.mpi }} + IPYKERNEL=${{ matrix.ipykernel }} + tags: | + ${{ env.TAG }} + ${{ env.LATEST_TAG }} + target: build + cache-to: type=gha, mode=max + cache-from: type=gha, mode=max diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 00000000..d7c959c4 --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,47 @@ +ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION} as base + +ENV DEBIAN_FRONTEND noninteractive +WORKDIR /build + +# Install common packages +COPY ./docker/apt-packages-common.txt /tmp/apt-packages-common.txt + +# Install dev packages if DEV is set +COPY ./docker/apt-packages-dev.txt /tmp/apt-packages-dev.txt + +ARG DEV +RUN if [ "$DEV" = "dev" ]; then \ + cat /tmp/apt-packages-common.txt /tmp/apt-packages-dev.txt > /tmp/apt-packages.txt; \ + else \ + cp /tmp/apt-packages-common.txt /tmp/apt-packages.txt; \ + fi && \ + apt-get update && \ + apt-get upgrade --yes && \ + apt-get install --yes \ + $(cat /tmp/apt-packages-common.txt) && \ + apt-get clean all && \ + rm -rf /var/lib/apt/lists/* + +FROM base as mpi + +# Build mpich +ARG mpich=4.0.2 +ARG mpich_prefix=mpich-$mpich +RUN wget https://www.mpich.org/static/downloads/$mpich/$mpich_prefix.tar.gz && \ + tar xvzf $mpich_prefix.tar.gz -C /build && \ + rm -rf $mpich_prefix.tar.gz && \ + cd /build/$mpich_prefix && \ + ./configure --disable-f77 --disable-fc --disable-fortran && \ + make -j 16 && \ + make install && \ + make clean && \ + cd .. && \ + rm -rf $mpich_prefix + +# Build BigMPI +RUN cd /build && wget https://github.com/jeffhammond/BigMPI/archive/refs/heads/master.tar.gz && \ + tar zxvf master.tar.gz && cd /build/BigMPI-master && \ + ./autogen.sh && ./configure --with-pic && make -j4 && make install && make clean && rm /build/master.tar.gz + +RUN /sbin/ldconfig \ No newline at end of file diff --git a/docker/Dockerfile.stempy b/docker/Dockerfile.stempy new file mode 100644 index 00000000..d57b1782 --- /dev/null +++ b/docker/Dockerfile.stempy @@ -0,0 +1,34 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as build + +ENV DEBIAN_FRONTEND noninteractive + +ARG PYTHON_VERSION +ARG IPYKERNEL +ARG MPI +ARG RELEASE_OR_DEBUG + +COPY . /source/stempy/ + +RUN pip install -r /source/stempy/requirements.txt + +RUN mkdir -p /build/stempy && \ + cd /build/stempy && \ + cmake -DCMAKE_BUILD_TYPE:STRING=${RELEASE_OR_DEBUG} \ + -Dstempy_ENABLE_VTKm:BOOL=OFF \ + -Dstempy_ENABLE_MPI:BOOL=${MPI} \ + -DBIGMPI_INCLUDE_DIR:PATH=/usr/local/include \ + -DBIGMPI_LIBRARY:PATH=/usr/local/lib/libbigmpi.a \ + /source/stempy . && \ + make -j 16 && \ + cp -r -L /build/stempy/lib/stempy \ + /usr/local/lib/python${PYTHON_VERSION}/site-packages + +RUN if [ "${IPYKERNEL}" = "ipykernel" ]; then \ + pip install -r /source/stempy/docker/requirements-ipykernel.txt; \ + else \ + pip install -r /source/stempy/docker/requirements-normal.txt; \ + fi && \ + rm -rf /source/stempy + +RUN /sbin/ldconfig \ No newline at end of file diff --git a/docker/apt-packages-common.txt b/docker/apt-packages-common.txt new file mode 100644 index 00000000..2ee34404 --- /dev/null +++ b/docker/apt-packages-common.txt @@ -0,0 +1,20 @@ +apt-transport-https +autoconf +automake +ca-certificates +gcc +g++ +git +gnupg +libarchive-dev +libeigen3-dev +libffi-dev +libhdf5-dev +liblapack-dev +libopenblas-dev +libsqlite3-dev +libtool +make +software-properties-common +wget +zlib1g-dev \ No newline at end of file diff --git a/docker/apt-packages-dev.txt b/docker/apt-packages-dev.txt new file mode 100644 index 00000000..7649be22 --- /dev/null +++ b/docker/apt-packages-dev.txt @@ -0,0 +1,2 @@ +gdb +vim \ No newline at end of file diff --git a/docker/requirements-ipykernel.txt b/docker/requirements-ipykernel.txt new file mode 100644 index 00000000..e22bcb0f --- /dev/null +++ b/docker/requirements-ipykernel.txt @@ -0,0 +1,6 @@ +ipykernel=6.4.2 +ipympl=0.8.6 +matplotlib=3.4.3 +click +imageio +ncempy \ No newline at end of file diff --git a/docker/requirements-normal.txt b/docker/requirements-normal.txt new file mode 100644 index 00000000..df7aea56 --- /dev/null +++ b/docker/requirements-normal.txt @@ -0,0 +1,4 @@ +matplotlib +click +imageio +ncempy \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4dedf26d..aec9183b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ numpy h5py deprecation +cmake \ No newline at end of file From 3394df83f5e1e5340b9598bade8c0f5bbe9c9805 Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 2 Jun 2023 17:12:39 -0400 Subject: [PATCH 2/7] squashed from a lot of CI triggers --- .github/workflows/docker.yml | 95 ++++++++++++++++--------------- docker/requirements-ipykernel.txt | 6 +- 2 files changed, 51 insertions(+), 50 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 684757a8..3dd932dd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,53 +54,54 @@ jobs: echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV PYTHON_VERSION_NODOT=$(echo ${{ matrix.python-version }} | tr -d .) echo "PYTHON_VERSION_NODOT=${PYTHON_VERSION_NODOT}" >> $GITHUB_ENV - - name: Set up other environment variables - id: set-tag-vars - run: | - echo "BASE_TARGET=$(if [[ "${{ matrix.mpi }}" == 'ON' ]]; then - echo 'mpi' - else - echo 'base' - fi)" >> $GITHUB_ENV - - MPI_TAG=$(if [[ "${{ matrix.mpi }}" == 'ON' ]]; then - echo '-mpi' - else - echo '' - fi) - echo "MPI_TAG=${MPI_TAG}" >> $GITHUB_ENV - - DEV_TAG=$(if [[ "${{ matrix.dev }}" == 'dev' ]]; then - echo '-dev' - else - echo '' - fi) - echo "DEV_TAG=${DEV_TAG}" >> $GITHUB_ENV - - echo "RELEASE_OR_DEBUG=$(if [[ "${{ matrix.dev }}" == 'dev' ]]; then - echo 'Debug' - else - echo 'Release' - fi)" >> $GITHUB_ENV - - COMMIT=$(git rev-parse --short HEAD) - echo "COMMIT=${COMMIT}" >> $GITHUB_ENV - - IPYKERNEL_TAG=$(if [[ "${{ matrix.ipykernel }}" == 'ipykernel' ]]; then - echo '-ipykernel' - else - echo '' - fi) - echo "IPYKERNEL_TAG=${IPYKERNEL_TAG}" >> $GITHUB_ENV - - BASE_TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}:py${{ env.PYTHON_VERSION_NODOT }}-base${DEV_TAG} - echo "BASE_TAG=${BASE_TAG}" >> $GITHUB_ENV - - TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}${IPYKERNEL_TAG}:py${{ env.PYTHON_VERSION_NODOT }}-${COMMIT}${DEV_TAG} - echo "TAG=${TAG}" >> $GITHUB_ENV - - LATEST_TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}${IPYKERNEL_TAG}:latest${DEV_TAG} - echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_ENV + - name: Set up other environment variables + id: set-tag-vars + run: | + echo "BASE_TARGET=$(if [[ "${{ matrix.mpi }}" == 'ON' ]]; then + echo 'mpi' + else + echo 'base' + fi)" >> $GITHUB_ENV + + MPI_TAG=$(if [[ "${{ matrix.mpi }}" == 'ON' ]]; then + echo '-mpi' + else + echo '' + fi) + echo "MPI_TAG=${MPI_TAG}" >> $GITHUB_ENV + + DEV_TAG=$(if [[ "${{ matrix.dev }}" == 'dev' ]]; then + echo '-dev' + else + echo '' + fi) + echo "DEV_TAG=${DEV_TAG}" >> $GITHUB_ENV + + echo "RELEASE_OR_DEBUG=$(if [[ "${{ matrix.dev }}" == 'dev' ]]; then + echo 'Debug' + else + echo 'Release' + fi)" >> $GITHUB_ENV + + COMMIT=$(git rev-parse --short HEAD) + echo "COMMIT=${COMMIT}" >> $GITHUB_ENV + + IPYKERNEL_TAG=$(if [[ "${{ matrix.ipykernel }}" == 'ipykernel' ]]; then + echo '-ipykernel' + else + echo '' + fi) + + echo "IPYKERNEL_TAG=${IPYKERNEL_TAG}" >> $GITHUB_ENV + + BASE_TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}:py${{ env.PYTHON_VERSION_NODOT }}-base${DEV_TAG} + echo "BASE_TAG=${BASE_TAG}" >> $GITHUB_ENV + + TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}${IPYKERNEL_TAG}:py${{ env.PYTHON_VERSION_NODOT }}-${COMMIT}${DEV_TAG} + echo "TAG=${TAG}" >> $GITHUB_ENV + + LATEST_TAG=${{ env.DOCKERHUB_ORG }}/stempy${MPI_TAG}${IPYKERNEL_TAG}:latest${DEV_TAG} + echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_ENV - name: Build/push Dockerfile.base uses: docker/build-push-action@v3 diff --git a/docker/requirements-ipykernel.txt b/docker/requirements-ipykernel.txt index e22bcb0f..d0b456af 100644 --- a/docker/requirements-ipykernel.txt +++ b/docker/requirements-ipykernel.txt @@ -1,6 +1,6 @@ -ipykernel=6.4.2 -ipympl=0.8.6 -matplotlib=3.4.3 +ipykernel==6.4.2 +ipympl==0.8.6 +matplotlib==3.4.3 click imageio ncempy \ No newline at end of file From 624bd14272fb29417c66655ef9c76e652073cafc Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 2 Jun 2023 18:48:02 -0400 Subject: [PATCH 3/7] add scripts to build locally build scripts for stempy, stempy-mpi. can change variables inside to fit local building. --- docker/scripts/docker_build_stempy.sh | 31 +++++++++++++++++++++++ docker/scripts/docker_build_stempy_mpi.sh | 29 +++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100755 docker/scripts/docker_build_stempy.sh create mode 100755 docker/scripts/docker_build_stempy_mpi.sh diff --git a/docker/scripts/docker_build_stempy.sh b/docker/scripts/docker_build_stempy.sh new file mode 100755 index 00000000..5142aae0 --- /dev/null +++ b/docker/scripts/docker_build_stempy.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +GIT_TAG=`git log -1 --pretty=%h` +DOCKERHUB_ORG="openchemistry" +PYTHON_VERSION="3.9" +PYTHON_VERSION_NODOT=$(echo $PYTHON_VERSION | tr -d .) +BASE_IMAGE=${DOCKERHUB_ORG}/stempy:py${PYTHON_VERSION_NODOT}-base${DEV} +STEMPY_TAG=${DOCKERHUB_ORG}/stempy${IPYKERNEL}:py${PYTHON_VERSION_NODOT}-${GIT_TAG} +RELEASE_OR_DEBUG="Release" # can change to "RelWithDebInfo" or "Debug" +DEV="" # can change to "-dev" if gdb/vim are desired inside container +IPYKERNEL="" # can change to "ipykernel" if building ipykernel version + +docker build \ +--build-arg PYTHON_VERSION=${PYTHON_VERSION} \ +--build-arg DEV=${DEV} \ +--load \ +-t ${BASE_IMAGE} \ +--target base \ +-f ../Dockerfile.base \ +../../ $@ + +docker build \ +--build-arg BASE_IMAGE=${BASE_IMAGE} \ +--build-arg MPI=OFF \ +--build-arg RELEASE_OR_DEBUG=${RELEASE_OR_DEBUG} \ +--build-arg IPYKERNEL="" \ +--build-arg PYTHON_VERSION=${PYTHON_VERSION} \ +--load \ +-t ${STEMPY_TAG} \ +-f ../Dockerfile.stempy \ +../../ $@ \ No newline at end of file diff --git a/docker/scripts/docker_build_stempy_mpi.sh b/docker/scripts/docker_build_stempy_mpi.sh new file mode 100755 index 00000000..e671ba83 --- /dev/null +++ b/docker/scripts/docker_build_stempy_mpi.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +GIT_TAG=`git log -1 --pretty=%h` +DOCKERHUB_ORG="openchemistry" +PYTHON_VERSION="3.9" +PYTHON_VERSION_NODOT=$(echo $PYTHON_VERSION | tr -d .) +BASE_IMAGE=${DOCKERHUB_ORG}/stempy-mpi:py${PYTHON_VERSION_NODOT}-base${DEV} +STEMPY_TAG=${DOCKERHUB_ORG}/stempy-mpi:py${PYTHON_VERSION_NODOT}-${GIT_TAG} +RELEASE_OR_DEBUG="Release" # can change to "RelWithDebInfo" or "Debug" +DEV="" # can change to "-dev" if gdb/vim are desired inside container + +docker build \ +--build-arg PYTHON_VERSION=${PYTHON_VERSION} \ +--build-arg DEV=${DEV} \ +--load \ +-t ${BASE_IMAGE} \ +--target mpi \ +-f ../Dockerfile.base \ +../../ $@ + +docker build \ +--build-arg BASE_IMAGE=${BASE_IMAGE} \ +--build-arg MPI=ON \ +--build-arg RELEASE_OR_DEBUG=${RELEASE_OR_DEBUG} \ +--build-arg IPYKERNEL="" \ +--build-arg PYTHON_VERSION=${PYTHON_VERSION} \ +-t ${STEMPY_TAG} \ +-f ../Dockerfile.stempy \ +../../ $@ From 7bf2b9577caa52c87c587a8e2d161edc88c8c5db Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 2 Jun 2023 18:49:44 -0400 Subject: [PATCH 4/7] remove old dockerfiles --- docker/Dockerfile | 100 ------------------------------------ docker/Dockerfile.ipykernel | 6 --- 2 files changed, 106 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100644 docker/Dockerfile.ipykernel diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index f5f114e0..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -FROM ubuntu:18.04 - -ARG MPI=OFF - -# Install deps -RUN apt-get update && \ - apt-get install -y \ - libeigen3-dev \ - libssl1.0-dev \ - git \ - autoconf \ - libtool \ - automake \ - gcc \ - g++ \ - make \ - gfortran \ - libopenblas-dev \ - liblapack-dev \ - wget \ - zlib1g-dev \ - libffi-dev \ - apt-transport-https \ - ca-certificates \ - gnupg \ - software-properties-common \ - libhdf5-dev \ - libsqlite3-dev && \ - apt-get clean all - -# Install CMake -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ - apt-get update && \ - apt-get install -y kitware-archive-keyring && \ - apt-key --keyring /etc/apt/trusted.gpg del C1F34CDD40CD72DA && \ - apt-get install -y cmake && \ - apt-get clean all - -RUN mkdir /build/ && mkdir /source/ - -# Build Python -RUN cd /build && wget https://www.python.org/ftp/python/3.7.3/Python-3.7.3.tgz && \ - tar xvzf Python-3.7.3.tgz && cd /build/Python-3.7.3 && \ - ./configure --enable-loadable-sqlite-extensions && make -j4 && make install && make clean && rm /build/Python-3.7.3.tgz - -# Build mpich -RUN cd /build && wget https://www.mpich.org/static/downloads/3.3/mpich-3.3.tar.gz && \ - tar xvzf mpich-3.3.tar.gz && cd /build/mpich-3.3 && \ - ./configure && make -j4 && make install && make clean && rm /build/mpich-3.3.tar.gz - -# Install mpi4py -RUN cd /build && wget https://bitbucket.org/mpi4py/mpi4py/downloads/mpi4py-3.0.1.tar.gz && \ - tar xvzf mpi4py-3.0.1.tar.gz - -RUN cd /build/mpi4py-3.0.1 && python3 setup.py build && python3 setup.py install - -# Build BigMPI -RUN cd /build && wget https://github.com/jeffhammond/BigMPI/archive/refs/heads/master.tar.gz && \ - tar zxvf master.tar.gz && cd /build/BigMPI-master && \ - ./autogen.sh && ./configure --with-pic && make -j4 && make install && make clean && rm /build/master.tar.gz - -# Build VTK-m -RUN cd /source && \ - wget https://gitlab.kitware.com/vtk/vtk-m/-/archive/v1.5.0/vtk-m-v1.5.0.tar.gz && \ - tar zxvf vtk-m-v1.5.0.tar.gz && \ - rm vtk-m-v1.5.0.tar.gz - -RUN mkdir -p /build/vtk-m && \ - cd /build/vtk-m && \ - cmake -DCMAKE_BUILD_TYPE:STRING=Release \ - -DBUILD_SHARED_LIBS:BOOL=OFF \ - -DVTKm_ENABLE_OPENMP:BOOL=ON \ - -DVTKm_ENABLE_RENDERING:BOOL=OFF \ - -DVTKm_ENABLE_TESTING:BOOL=OFF \ - /source/vtk-m-v1.5.0 . && \ - make -j4 - -# Build stempy -COPY . /source/stempy - -RUN mkdir -p /build/stempy && \ - cd /build/stempy && \ - cmake -DCMAKE_BUILD_TYPE:STRING=Release \ - -Dstempy_ENABLE_VTKm:BOOL=ON \ - -DVTKm_DIR:PATH=/build/vtk-m/lib/cmake/vtkm-1.5 \ - -Dstempy_ENABLE_MPI:BOOL=${MPI} \ - -DBIGMPI_INCLUDE_DIR:PATH=/usr/local/include \ - -DBIGMPI_LIBRARY:PATH=/usr/local/lib/libbigmpi.a \ - /source/stempy . && \ - make -j4 - -# Install stempy -RUN pip3 install -r /source/stempy/requirements.txt && \ - cp -r -L /build/stempy/lib/stempy /usr/local/lib/python3.7/site-packages && \ - pip3 install matplotlib click imageio ncempy - -RUN rm -rf /build - -RUN /sbin/ldconfig diff --git a/docker/Dockerfile.ipykernel b/docker/Dockerfile.ipykernel deleted file mode 100644 index bd5597df..00000000 --- a/docker/Dockerfile.ipykernel +++ /dev/null @@ -1,6 +0,0 @@ -FROM openchemistry/stempy:latest - -# Note these are the specfic versions needed to work with the jupyterlab -# deployment at NERSC, update with care! -RUN pip3 install -U ipykernel==6.4.2 ipympl==0.8.6 matplotlib==3.4.3 - From 8281dff42123e04f0d380ceedb110cfa98e8235f Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 2 Jun 2023 18:50:55 -0400 Subject: [PATCH 5/7] build on master branch --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3dd932dd..cc610df1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,8 +3,7 @@ name: docker on: push: branches: - # - "master" - - 'gha-final' + - 'master' pull_request: branches: - 'master' From 268e86c7d25fa54f7455b7d2a0bf5df0e5de2dcc Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 2 Jun 2023 19:08:21 -0400 Subject: [PATCH 6/7] install stempy on build --- docker/Dockerfile.stempy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.stempy b/docker/Dockerfile.stempy index d57b1782..a1615261 100644 --- a/docker/Dockerfile.stempy +++ b/docker/Dockerfile.stempy @@ -20,7 +20,8 @@ RUN mkdir -p /build/stempy -DBIGMPI_INCLUDE_DIR:PATH=/usr/local/include \ -DBIGMPI_LIBRARY:PATH=/usr/local/lib/libbigmpi.a \ /source/stempy . && \ - make -j 16 && \ + cmake --build . && \ + cmake --install . && \ cp -r -L /build/stempy/lib/stempy \ /usr/local/lib/python${PYTHON_VERSION}/site-packages From ec0d9769901a6b985bb198ac54ec0dd39a96cbf9 Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Thu, 13 Jul 2023 15:53:52 -0400 Subject: [PATCH 7/7] add mpi4py to requirements - forgot mpi4py - tested and works now: ``` ****Statistics for calculating electron thresholds**** number of samples:1200 min sample:0 max sample:1588 mean:0.106727 variance:4.7572 std dev:2.1811 number of bins:832 x-ray threshold n sigma:175 background threshold n sigma:4 optimized mean:-0.302141 optimized std dev:0.223178 background threshold:0.590571 xray threshold:381.8 electron_count_cori.py Using files in /pscratch/sd/s/swelborn/ncem/138 scan name = data_scan0000000441_*.data start counting #441 electron_count_cori.py Using files in /pscratch/sd/s/swelborn/ncem/138 scan name = data_scan0000000441_*.data start counting #441 electron_count_cori.py Using files in /pscratch/sd/s/swelborn/ncem/138 scan name = data_scan0000000441_*.data start counting #441 electron_count_cori.py Using files in /pscratch/sd/s/swelborn/ncem/138 scan name = data_scan0000000441_*.data start counting #441 total time = 4.425157070159912 /pscratch/sd/s/swelborn/ncem/138/FOURD_230601_1658_00907_00441.h5 ``` --- docker/Dockerfile.stempy | 2 ++ docker/requirements-mpi.txt | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 docker/requirements-mpi.txt diff --git a/docker/Dockerfile.stempy b/docker/Dockerfile.stempy index a1615261..6e9a7775 100644 --- a/docker/Dockerfile.stempy +++ b/docker/Dockerfile.stempy @@ -27,6 +27,8 @@ RUN mkdir -p /build/stempy RUN if [ "${IPYKERNEL}" = "ipykernel" ]; then \ pip install -r /source/stempy/docker/requirements-ipykernel.txt; \ + elif [ "${MPI}" = "ON" ]; then \ + pip install -r /source/stempy/docker/requirements-mpi.txt; \ else \ pip install -r /source/stempy/docker/requirements-normal.txt; \ fi && \ diff --git a/docker/requirements-mpi.txt b/docker/requirements-mpi.txt new file mode 100644 index 00000000..1d68a94f --- /dev/null +++ b/docker/requirements-mpi.txt @@ -0,0 +1,5 @@ +matplotlib +click +imageio +ncempy +mpi4py \ No newline at end of file