From 8ad7c35a40911678e49037849cd0c0b2a31ebc2d Mon Sep 17 00:00:00 2001 From: Skye Turriff Date: Fri, 1 Nov 2024 15:50:27 -0400 Subject: [PATCH 01/23] add workbench-session-init target for release builds --- Justfile | 2 ++ docker-bake.hcl | 30 ++++++++++++++++++++++++++++++ workbench-session-init/README.md | 14 ++++++++++---- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/Justfile b/Justfile index e960025f..c5a95da8 100644 --- a/Justfile +++ b/Justfile @@ -51,6 +51,7 @@ alias build := bake bake target="default": just -f {{justfile()}} create-builder || true GIT_SHA=$(git rev-parse --short HEAD) \ + WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} # just preview-bake workbench-images dev @@ -90,6 +91,7 @@ preview-plan branch="$(git branch --show-current)": # just test workbench test target="default" file="docker-bake.hcl": GIT_SHA=$(git rev-parse --short HEAD) \ + WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ python3 {{justfile_directory()}}/tools/test_bake_artifacts.py --target "{{target}}" --file "{{file}}" # just preview-test connect dev diff --git a/docker-bake.hcl b/docker-bake.hcl index 2eb84c84..a46dac02 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -11,6 +11,10 @@ variable WORKBENCH_VERSION { default = "2024.09.1+394.pro7" } +variable WORKBENCH_SESSION_INIT_VERSION { + default = "" +} + variable DRIVERS_VERSION { default = "2024.03.0" } @@ -181,6 +185,14 @@ variable WORKBENCH_BUILD_MATRIX { } } +variable WORKBENCH_SESSION_INIT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204"}, + ] + } +} + variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { default = { builds = [ @@ -209,6 +221,7 @@ group "default" { "package-manager", "r-session-complete", "workbench", + "workbench-session-init", ] } @@ -453,6 +466,23 @@ target "workbench" { } } +target "workbench-session-init" { + inherits = ["base"] + target = "build" + + name = "workbench-session-init-${builds.os}-${replace(tag_safe_version(WORKBENCH_SESSION_INIT_VERSION), ".", "-")}" + tags = get_tags(builds.os, "workbench-session-init", WORKBENCH_SESSION_INIT_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-session-init" + + matrix = WORKBENCH_SESSION_INIT_BUILD_MATRIX + + args = { + RSW_VERSION = WORKBENCH_SESSION_INIT_VERSION + } +} + ### Workbench for Google Cloud Workstations targets ### target "workbench-for-google-cloud-workstations" { inherits = ["base"] diff --git a/workbench-session-init/README.md b/workbench-session-init/README.md index ecae3c92..662b595f 100644 --- a/workbench-session-init/README.md +++ b/workbench-session-init/README.md @@ -8,7 +8,7 @@ This directory contains a Dockerfile and script that will create an init contain * Where to get help: [our Github Issues page](https://github.com/rstudio/rstudio-docker-products/issues) * Posit Workbench image: [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench) * RStudio r-session-complete image: [Docker Hub](https://hub.docker.com/r/rstudio/r-session-complete) -* Workbench Session Init image (Daily/Preview): [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench-session-init-preview) +* Workbench Session Init image: [Docker Hub](https://hub.docker.com/r/rstudio/workbench-session-init) ## Supported tags and respective Dockerfile links @@ -16,7 +16,13 @@ This directory contains a Dockerfile and script that will create an init contain ## Building -Currently daily builds are supported. To build the image, run: +Just will build an image using a default Connect distribution. + +```console +just build +``` + +Daily builds are also supported. To build the daily image, run: ```console just preview-bake workbench-session-init-daily @@ -28,14 +34,14 @@ You can observe what gets copied by the container: ```console mkdir init -docker run --rm -v $(pwd)/init:/mnt/init rstudio/workbench-session-init-preview:workbench-session-init-jammy-2024.11.0-daily-328.pro3 +docker run --rm -v $(pwd)/init:/mnt/init rstudio/workbench-session-init:jammy-2024.11.0 # The init directory has been populated with the Workbench session runtime components. ``` You can also test using GOSS: ```console -just preview-test workbench-session-init-daily +just test workbench-session-init ``` ## Licensing From a55a0a7c7f983a93f1ed058236e91a59f2b9fe88 Mon Sep 17 00:00:00 2001 From: Skye Turriff Date: Fri, 1 Nov 2024 16:04:47 -0400 Subject: [PATCH 02/23] add github workflow jobs --- .github/workflows/build-bake.yaml | 76 +++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index c1c67e11..4f5c2092 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -37,6 +37,39 @@ jobs: GIT_SHA=$(git rev-parse --short HEAD) echo "GIT_SHA=$GIT_SHA" >> $GITHUB_OUTPUT echo "$GIT_SHA" + versions: + name: Fetch Workbench session init container version + runs-on: ubuntu-latest + + concurrency: + group: fetch-versions-${{ github.ref }} + cancel-in-progress: true + + outputs: + WORKBENCH_SESSION_INIT_VERSION: ${{ steps.get-version.outputs.WORKBENCH_SESSION_INIT_VERSION }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Just + uses: extractions/setup-just@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Python dependencies + run: | + pip install requests + + - name: Get Version + id: get-version + run: | + WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + echo "WORKBENCH_SESSION_INIT_VERSION=$WORKBENCH_SESSION_INIT_VERSION" >> $GITHUB_OUTPUT base: needs: [setup] @@ -290,6 +323,49 @@ jobs: snyk-org: ${{ secrets.SNYK_ORG }} snyk-token: '${{ secrets.SNYK_TOKEN }}' + workbench-session-init: + needs: [setup, versions] + name: Workbench Session Init + runs-on: ubuntu-latest-8x + + concurrency: + group: bake-workbench-session-init-${{ github.ref }} + cancel-in-progress: true + + env: + target: workbench-session-init + GIT_SHA: ${{ needs.setup.outputs.GIT_SHA }} + WORKBENCH_SESSION_INIT_VERSION: ${{ needs.versions.outputs.WORKBENCH_SESSION_INIT_VERSION }} + + steps: + - name: Checkout + if: github.event_name == 'schedule' + uses: actions/checkout@v4 + with: + ref: 'main' + + - name: Checkout + if: github.event_name != 'schedule' + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' + workbench: needs: [setup] name: Workbench From b6eb895c9d8f8101d09e1dd37b30916305931922 Mon Sep 17 00:00:00 2001 From: "Benjamin R. J. Schwedler" Date: Thu, 7 Nov 2024 13:54:00 -0600 Subject: [PATCH 03/23] Fixup for gha job indentation --- .github/workflows/build-bake.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 4f5c2092..e2d9f539 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -323,7 +323,7 @@ jobs: snyk-org: ${{ secrets.SNYK_ORG }} snyk-token: '${{ secrets.SNYK_TOKEN }}' - workbench-session-init: + workbench-session-init: needs: [setup, versions] name: Workbench Session Init runs-on: ubuntu-latest-8x From 2015b1867518f32512422d57a54330ec924f439d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 15 Nov 2024 07:26:13 -0700 Subject: [PATCH 04/23] Use env var for WORKBENCH_SESSION_INIT_VERSION or default to `get-version` to fetch version --- Justfile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index c5a95da8..3b74f8a6 100644 --- a/Justfile +++ b/Justfile @@ -49,9 +49,13 @@ delete-builder: alias build := bake # just bake workbench-images bake target="default": + #!/bin/bash just -f {{justfile()}} create-builder || true + if [ -z "$WORKBENCH_SESSION_INIT_VERSION" ]; then + WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + fi GIT_SHA=$(git rev-parse --short HEAD) \ - WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_SESSION_INIT_VERSION=${WORKBENCH_SESSION_INIT_VERSION} \ docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} # just preview-bake workbench-images dev @@ -90,8 +94,12 @@ preview-plan branch="$(git branch --show-current)": # just test workbench test target="default" file="docker-bake.hcl": + #!/bin/bash + if [ -z "$WORKBENCH_SESSION_INIT_VERSION" ]; then + WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + fi GIT_SHA=$(git rev-parse --short HEAD) \ - WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_SESSION_INIT_VERSION=${WORKBENCH_SESSION_INIT_VERSION} \ python3 {{justfile_directory()}}/tools/test_bake_artifacts.py --target "{{target}}" --file "{{file}}" # just preview-test connect dev From 8eb89648e50696fae95ba5bf44639e1cd95ffd8a Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Fri, 15 Nov 2024 12:46:40 -0500 Subject: [PATCH 05/23] Add workbench-session, starting with r-session-complete --- workbench-session/.env | 4 + workbench-session/.snyk | 20 +++++ workbench-session/Dockerfile.ubuntu2204 | 64 +++++++++++++++ workbench-session/NEWS.md | 92 ++++++++++++++++++++++ workbench-session/README.md | 78 ++++++++++++++++++ workbench-session/docker-compose.test.yml | 16 ++++ workbench-session/positron.extensions.conf | 2 + workbench-session/test/goss.yaml | 69 ++++++++++++++++ workbench-session/test/run_tests.sh | 19 +++++ workbench-session/vscode.extensions.conf | 5 ++ 10 files changed, 369 insertions(+) create mode 100644 workbench-session/.env create mode 100644 workbench-session/.snyk create mode 100644 workbench-session/Dockerfile.ubuntu2204 create mode 100644 workbench-session/NEWS.md create mode 100644 workbench-session/README.md create mode 100644 workbench-session/docker-compose.test.yml create mode 100644 workbench-session/positron.extensions.conf create mode 100644 workbench-session/test/goss.yaml create mode 100755 workbench-session/test/run_tests.sh create mode 100644 workbench-session/vscode.extensions.conf diff --git a/workbench-session/.env b/workbench-session/.env new file mode 100644 index 00000000..5138f057 --- /dev/null +++ b/workbench-session/.env @@ -0,0 +1,4 @@ +R_VERSION=4.1.0 +PYTHON_VERSION=3.9.5 +RSW_VERSION=2024.09.1+394.pro7 +DRIVERS_VERSION=2024.03.0-1 diff --git a/workbench-session/.snyk b/workbench-session/.snyk new file mode 100644 index 00000000..909ad99a --- /dev/null +++ b/workbench-session/.snyk @@ -0,0 +1,20 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-GOLANG-GITHUBCOMCREWJAMSAML-5971016: + - '*': + reason: >- + Reported upstream in + https://github.com/rstudio/rstudio-pro/issues/6529 + expires: 2024-08-31T00:00:00.000Z + created: 2024-07-02T20:33:30.847Z + SNYK-GOLANG-GITHUBCOMGOJOSEGOJOSEV3-6070737: + - '*': + reason: >- + Confirmed fixed upstream in + https://github.com/rstudio/rstudio-pro/issues/6635. Patch will be + ingested in Workbench 2024.08.0 (expected within 1 week). + expires: 2024-08-07T00:00:00.000Z + created: 2024-07-31T17:46:24.852Z +patch: {} diff --git a/workbench-session/Dockerfile.ubuntu2204 b/workbench-session/Dockerfile.ubuntu2204 new file mode 100644 index 00000000..95b04386 --- /dev/null +++ b/workbench-session/Dockerfile.ubuntu2204 @@ -0,0 +1,64 @@ +FROM product-base-pro as build + +ARG DEBIAN_FRONTEND=noninteractive +ARG R_VERSION=4.4.0 +ARG R_VERSION_ALT=4.3.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2024.09.1+394.pro7 +ARG RSW_NAME=rstudio-workbench +ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 +ARG SCRIPTS_DIR=/opt/positscripts + +ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + krb5-user \ + libcurl4-gnutls-dev \ + libuser \ + libuser1-dev \ + libpq-dev \ + rrdtool \ + subversion \ + && RSW_VERSION_URL=$(echo -n "${RSW_VERSION}" | sed 's/+/-/g') \ + && curl -fsSL -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION_URL}-amd64.deb" \ + # Pre 7/25/23 packages + && gpg --keyserver keyserver.ubuntu.com --recv-keys 3F32EE77E331692F \ + # Post 7/25 packages + && gpg --keyserver keys.openpgp.org --recv-keys 51C0B5BB19F92D60 \ + && dpkg-sig --verify ./rstudio-workbench.deb \ + && apt-get install -yq --no-install-recommends ./rstudio-workbench.deb \ + && rm ./rstudio-workbench.deb \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/rstudio-server/r-versions + +### Install Quarto to PATH ### +RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto + +### Install TinyTeX using Quarto ### +RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex + +RUN /opt/python/"${PYTHON_VERSION}"/bin/python -m venv /opt/python/jupyter \ + && /opt/python/jupyter/bin/python -m pip install --upgrade pip \ + && /opt/python/jupyter/bin/python -m pip install --upgrade setuptools \ + && /opt/python/jupyter/bin/python -m pip install \ + jupyterlab~=4.2.4 \ + notebook \ + pwb_jupyterlab~=1.0 \ + && ln -s /opt/python/jupyter/bin/jupyter /usr/local/bin/jupyter \ + && /opt/python/${PYTHON_VERSION}/bin/python -m pip install ipykernel \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python -m pip install ipykernel \ + && /opt/python/${PYTHON_VERSION}/bin/python -m ipykernel install --name py${PYTHON_VERSION} --display-name "Python ${PYTHON_VERSION}" \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python -m ipykernel install --name py${PYTHON_VERSION_ALT} --display-name "Python ${PYTHON_VERSION_ALT}" \ + && /opt/python/jupyter/bin/python -m pip cache purge + +ENV PATH="/opt/python/jupyter/bin:${PATH}" + +COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf + +EXPOSE 8788/tcp diff --git a/workbench-session/NEWS.md b/workbench-session/NEWS.md new file mode 100644 index 00000000..77917139 --- /dev/null +++ b/workbench-session/NEWS.md @@ -0,0 +1,92 @@ +# 2023.03.1 +- Update Python VSCode extension to version 2023.6.1 + +# 2023.03.0 + +- Update R versions to 4.1.3 and 4.2.3 +- Update Python versions to 3.8.15 and 3.9.14 + +# 2022.12.0 + +- Upgrade workbench to 2022.12.0 +- Use the new `WORKBENCH_JUPYTER_PATH` env var to configure jupyter location +- Remove `vscode.conf/exe` configuration in favor of the new, internal default `code-server` installation +- Refactor image to build _FROM_ the new `product-base-pro` image + +# 2022.07.0 + +- Add a `/usr/local/bin/jupyter` symlink +- Configure Workbench to preinstall R, Python, and Quarto extensions for users on session startup +- Add workbench_jupyterlab python package for the jupyterlab extension + +# 2021.09.0 + +- **BREAKING**: Rename `RSP_` variables to `RSW_`. Rename RStudio Server Pro to RStudio Workbench +- **BREAKING**: Change `R_VERSION` to 4.1.0 +- **BREAKING**: Change `PYTHON_VERSION` to 3.9.5 +- **BREAKING**: R and Python packages are no longer installed by default in session images. + +We have removed R and Python packages from the session image. This should: +- reduce the image build time and size +- make it easier for data scientists to develop reproducible environments +- eliminate conflicts between system libraries and subsequent user updates + +If the removal of these system libraries introduces some problems for you, please file an issue. + +# 1.4.1717-3 + +## Overview + +R is at `/opt/R/4.0.2/bin/R` +python is at `/opt/python/3.7.7/bin/python` +jupyter is at `/opt/python/3.7.7/bin/jupyter` +code-server is at `/opt/code-server/bin/code-server` + +## Changes + +- Update RStudio Professional Drivers to 1.8.0 +- `BREAKING`: `code-server` no longer supports the `/opt/code-server/code-server` location. + - As a result, you need to set `exe=/opt/code-server/bin/code-server` + - After two changes in a row, we suspect this is the final change for a while +_vscode.conf_ +``` +enabled=1 +exe=/opt/code-server/bin/code-server +``` + +# 1.4.1106-5 + +## Overview + +R is at `/opt/R/4.0.2/bin/R` +python is at `/opt/python/3.7.7/bin/python` +jupyter is at `/opt/python/3.7.7/bin/jupyter` +code-server is at `/opt/code-server/code-server` + +## Changes + +- `BREAKING`: Changed code-server version and insulated against version upgrades in the future. To update: +change: + +_vscode.conf_ +``` +enabled=1 +exe=/opt/code-server/code-server-3.2.0-linux-x86_64/code-server +``` +to: +_vscode.conf_ +``` +enabled=1 +exe=/opt/code-server/code-server +``` + +- Update Drivers to version 1.7.0 + +# 1.4.1103-4 + +## Overview + +R is at `/opt/R/4.0.2/bin/R` +python is at `/opt/python/3.7.7/bin/python` +jupyter is at `/opt/python/3.7.7/bin/jupyter` +code-server is at `/opt/code-server/code-server-3.2.0-linux-x86_64/code-server` diff --git a/workbench-session/README.md b/workbench-session/README.md new file mode 100644 index 00000000..243c17d1 --- /dev/null +++ b/workbench-session/README.md @@ -0,0 +1,78 @@ +# Quick reference + +* Maintained by: [the Posit Docker team](https://github.com/rstudio/rstudio-docker-products) +* Where to get help: [our Github Issues page](https://github.com/rstudio/rstudio-docker-products/issues) +* RStudio Workbench image: [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench) +* RStudio r-session-complete image: [Docker Hub](https://hub.docker.com/r/rstudio/r-session-complete) + +# Supported tags and respective Dockerfile links + +* [`jammy`, `ubuntu2204`, `jammy-2024.09.1`, `ubuntu2204-2024.09.1`](https://github.com/rstudio/rstudio-docker-products/blob/main/r-session-complete/Dockerfile.ubuntu2204) + +# What are the r-session-complete images? + +Images for R and Python sessions and jobs to be used RStudio Workbench, Launcher, and Kubernetes. + +# Notice for support + +1. This image may introduce **BREAKING** changes; as such we recommend: + - Avoid using the `{operating-system}` tags to avoid unexpected version changes, and + - Always read through the [NEWS](./NEWS.md) to understand the changes before updating. +1. Outdated images will be removed periodically from DockerHub as product version updates are made. Please make plans to + update at times or use your own build of the images. +1. These images are meant as a starting point for your needs. Consider creating a fork of this repo, where you can + continue to merge in changes we make while having your own security scanning, base OS in use, or other custom + changes. We + provide [instructions for how to build and use](#how-to-use-these-docker-images) + for these cases. +1. **Security Note:** These images are provided AS IS based on the build environment at the time their product version was released/updated. They should be reviewed and updated before production use. If your organization has a specific set of security requirements related to CVE/Vulnerability severity levels, you should plan to use the [instructions for building](https://github.com/rstudio/rstudio-docker-products#instructions-for-building) to clone this repository, and rebuild these images to your specific internal security standards. + +# How to use these images + +The Docker images built from these Dockerfiles are intended to be used for R and +Jupyter sessions and jobs with RStudio Workbench (RSW), Launcher, and +Kubernetes. + +Note: These Docker images are not equipped or intended to be used to run RStudio +Workbench within a Docker container. Visit the +[rstudio/rstudio-worbench Docker Hub page](https://hub.docker.com/r/rstudio/rstudio-workbench) +for images built for that purpose. + +For more information about RStudio Workbench and Launcher, refer to the +[Launcher Overview](https://solutions.rstudio.com/launcher/overview/) on the +RStudio Solutions website. + +For more information about how to use these images with RStudio Workbench and +Launcher, refer to the RStudio support article on [Using Docker images with +RStudio Workbench, Launcher, and +Kubernetes](https://support.rstudio.com/hc/en-us/articles/360019253393-Using-Docker-images-with-RStudio-Server-Pro-Launcher-and-Kubernetes). + +We provide simple ways to extend and build the Dockerfiles. After you have cloned the repo, you can create your own +containers fairly simply with the provided Justfile. + +## Overview + +Built images are available from the +[rstudio/r-session-complete](https://hub.docker.com/r/rstudio/r-session-complete) +repository on Docker Hub. + +These images include the following layers: + +* Base OS +* RSW session components +* System packages required for R, R packages, and RStudio Professional Drivers +* One version of R +* One version of Python +* Jupyter Notebooks, JupyterLab, and RSW/RSC notebook extensions +* RStudio Professional Drivers + +# Licensing + +The license associated with the RStudio Docker Products repository is located [in LICENSE.md](https://github.com/rstudio/rstudio-docker-products/blob/main/LICENSE.md). + +As is the case with all container images, the images themselves also contain other software which may be under other +licenses (i.e. bash, linux, system libraries, etc., along with any other direct or indirect dependencies of the primary +software being contained). + +It is an image user's responsibility to ensure that use of this image (and any of its dependent layers) complies with +all relevant licenses for the software contained in the image. diff --git a/workbench-session/docker-compose.test.yml b/workbench-session/docker-compose.test.yml new file mode 100644 index 00000000..80d19f5d --- /dev/null +++ b/workbench-session/docker-compose.test.yml @@ -0,0 +1,16 @@ +version: '2.3' +services: + + sut: + image: $IMAGE_NAME + command: /run_tests.sh + entrypoint: [] + environment: + # uses .env by default + - RSW_VERSION + - R_VERSION + - PYTHON_VERSION + volumes: + - "./test/run_tests.sh:/run_tests.sh" + - "./test/goss.yaml:/tmp/goss.yaml" + - "./test/goss_vars.yaml:/tmp/goss_vars.yaml" diff --git a/workbench-session/positron.extensions.conf b/workbench-session/positron.extensions.conf new file mode 100644 index 00000000..42191cc9 --- /dev/null +++ b/workbench-session/positron.extensions.conf @@ -0,0 +1,2 @@ +posit.shiny +posit.publisher diff --git a/workbench-session/test/goss.yaml b/workbench-session/test/goss.yaml new file mode 100644 index 00000000..030ff180 --- /dev/null +++ b/workbench-session/test/goss.yaml @@ -0,0 +1,69 @@ +file: + /usr/lib/rstudio-server: + exists: true + /usr/lib/rstudio-server/bin/rsession: + exists: true + /usr/local/bin/jupyter: + exists: true + {{ $version_split := split "." .Env.RSW_VERSION }} + {{ if or (ge ($version_split._0 | atoi) 2025) (and (ge ($version_split._0 | atoi) 2024) (ge ($version_split._1 | atoi) 7)) }} + /usr/lib/rstudio-server/bin/pwb-code-server/bin/code-server: + exists: true + {{ else }} + /usr/lib/rstudio-server/bin/code-server/bin/code-server: + exists: true + {{ end }} + /opt/rstudio-drivers: + exists: true + filetype: directory + /var/lib/rstudio-server/r-versions: + exists: false + /usr/local/bin/quarto: + exists: true + filetype: symlink + +command: + "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": + title: jupyter_works + timeout: 60000 + exit-status: 0 + +# Ensure correct R version + "/opt/R/{{.Env.R_VERSION}}/bin/R --version": + title: r_version_match + exit-status: 0 + stdout: [ + "{{.Env.R_VERSION}}" + ] + +# Ensure correct python version + "/opt/python/{{.Env.PYTHON_VERSION}}/bin/python3 --version": + title: python_version_matches + exit-status: 0 + stdout: [ + "{{ .Env.PYTHON_VERSION }}" + ] + + "python3 --version": + title: python_in_path_var + exit-status: 0 + stdout: [ + "{{ .Env.PYTHON_VERSION }}" + ] + + "jupyter --version": + title: jupyter_in_path_var + timeout: 60000 + exit-status: 0 + +# Ensure Quarto works + "/usr/local/bin/quarto check --quiet": + title: quarto_check + exit-status: 0 + +# Ensure TinyTeX is installed + "quarto list tools": + title: quarto_tinytex_installed + exit-status: 0 + stderr: + - "/tinytex\\s+Up to date\\s+v\\d{4}\\.\\d{2}(\\.\\d{2})?\\s+v\\d{4}\\.\\d{2}(\\.\\d{2})?/" diff --git a/workbench-session/test/run_tests.sh b/workbench-session/test/run_tests.sh new file mode 100755 index 00000000..8aff0f84 --- /dev/null +++ b/workbench-session/test/run_tests.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} + +if [ -f /etc/debian_version ]; then + OS="ubuntu" +else + echo "OS not supported. Exiting" + exit 1 +fi + +# install goss to tmp location and make executable +curl -fsSL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ + && chmod +x /tmp/goss \ + && GOSS=/tmp/goss + +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/workbench-session/vscode.extensions.conf b/workbench-session/vscode.extensions.conf new file mode 100644 index 00000000..80e04b95 --- /dev/null +++ b/workbench-session/vscode.extensions.conf @@ -0,0 +1,5 @@ +quarto.quarto +REditorSupport.r@2.8.2 +ms-python.python +posit.shiny +ms-toolsai.jupyter From 0ecedaaeb0d7e4828fddad1d8941af63dde70a7e Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Fri, 15 Nov 2024 13:13:19 -0500 Subject: [PATCH 06/23] Remove RSW from Workbench Session --- workbench-session-init/README.md | 4 +- workbench-session/.env | 1 - workbench-session/Dockerfile.ubuntu2204 | 21 +---- workbench-session/NEWS.md | 94 +---------------------- workbench-session/README.md | 18 ++--- workbench-session/docker-compose.test.yml | 1 - 6 files changed, 16 insertions(+), 123 deletions(-) diff --git a/workbench-session-init/README.md b/workbench-session-init/README.md index 662b595f..76c82b74 100644 --- a/workbench-session-init/README.md +++ b/workbench-session-init/README.md @@ -7,8 +7,8 @@ This directory contains a Dockerfile and script that will create an init contain * Maintained by: [the Posit Docker team](https://github.com/rstudio/rstudio-docker-products) * Where to get help: [our Github Issues page](https://github.com/rstudio/rstudio-docker-products/issues) * Posit Workbench image: [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench) -* RStudio r-session-complete image: [Docker Hub](https://hub.docker.com/r/rstudio/r-session-complete) -* Workbench Session Init image: [Docker Hub](https://hub.docker.com/r/rstudio/workbench-session-init) +* Posit Workbench session image: [Docker Hub](https://hub.docker.com/r/rstudio/workbench-session) +* Posit Workbench session init image: [Docker Hub](https://hub.docker.com/r/rstudio/workbench-session-init) ## Supported tags and respective Dockerfile links diff --git a/workbench-session/.env b/workbench-session/.env index 5138f057..1c5333d8 100644 --- a/workbench-session/.env +++ b/workbench-session/.env @@ -1,4 +1,3 @@ R_VERSION=4.1.0 PYTHON_VERSION=3.9.5 -RSW_VERSION=2024.09.1+394.pro7 DRIVERS_VERSION=2024.03.0-1 diff --git a/workbench-session/Dockerfile.ubuntu2204 b/workbench-session/Dockerfile.ubuntu2204 index 95b04386..28743789 100644 --- a/workbench-session/Dockerfile.ubuntu2204 +++ b/workbench-session/Dockerfile.ubuntu2204 @@ -6,9 +6,6 @@ ARG R_VERSION_ALT=4.3.3 ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG JUPYTERLAB_VERSION=3.6.5 -ARG RSW_VERSION=2024.09.1+394.pro7 -ARG RSW_NAME=rstudio-workbench -ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 ARG SCRIPTS_DIR=/opt/positscripts ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter @@ -23,22 +20,9 @@ RUN apt-get update \ libpq-dev \ rrdtool \ subversion \ - && RSW_VERSION_URL=$(echo -n "${RSW_VERSION}" | sed 's/+/-/g') \ - && curl -fsSL -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION_URL}-amd64.deb" \ - # Pre 7/25/23 packages - && gpg --keyserver keyserver.ubuntu.com --recv-keys 3F32EE77E331692F \ - # Post 7/25 packages - && gpg --keyserver keys.openpgp.org --recv-keys 51C0B5BB19F92D60 \ - && dpkg-sig --verify ./rstudio-workbench.deb \ - && apt-get install -yq --no-install-recommends ./rstudio-workbench.deb \ - && rm ./rstudio-workbench.deb \ && apt-get autoremove -y \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /var/lib/rstudio-server/r-versions - -### Install Quarto to PATH ### -RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto + && rm -rf /var/lib/apt/lists/* ### Install TinyTeX using Quarto ### RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex @@ -57,7 +41,8 @@ RUN /opt/python/"${PYTHON_VERSION}"/bin/python -m venv /opt/python/jupyter \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python -m ipykernel install --name py${PYTHON_VERSION_ALT} --display-name "Python ${PYTHON_VERSION_ALT}" \ && /opt/python/jupyter/bin/python -m pip cache purge -ENV PATH="/opt/python/jupyter/bin:${PATH}" +# Add Jupyter, Python, and Quarto to the PATH +ENV PATH="/opt/python/jupyter/bin:/opt/python/bin:/usr/lib/rstudio-server/bin/quarto/bin:${PATH}" COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf diff --git a/workbench-session/NEWS.md b/workbench-session/NEWS.md index 77917139..e76e114b 100644 --- a/workbench-session/NEWS.md +++ b/workbench-session/NEWS.md @@ -1,92 +1,4 @@ -# 2023.03.1 -- Update Python VSCode extension to version 2023.6.1 +# 2024-11-15 -# 2023.03.0 - -- Update R versions to 4.1.3 and 4.2.3 -- Update Python versions to 3.8.15 and 3.9.14 - -# 2022.12.0 - -- Upgrade workbench to 2022.12.0 -- Use the new `WORKBENCH_JUPYTER_PATH` env var to configure jupyter location -- Remove `vscode.conf/exe` configuration in favor of the new, internal default `code-server` installation -- Refactor image to build _FROM_ the new `product-base-pro` image - -# 2022.07.0 - -- Add a `/usr/local/bin/jupyter` symlink -- Configure Workbench to preinstall R, Python, and Quarto extensions for users on session startup -- Add workbench_jupyterlab python package for the jupyterlab extension - -# 2021.09.0 - -- **BREAKING**: Rename `RSP_` variables to `RSW_`. Rename RStudio Server Pro to RStudio Workbench -- **BREAKING**: Change `R_VERSION` to 4.1.0 -- **BREAKING**: Change `PYTHON_VERSION` to 3.9.5 -- **BREAKING**: R and Python packages are no longer installed by default in session images. - -We have removed R and Python packages from the session image. This should: -- reduce the image build time and size -- make it easier for data scientists to develop reproducible environments -- eliminate conflicts between system libraries and subsequent user updates - -If the removal of these system libraries introduces some problems for you, please file an issue. - -# 1.4.1717-3 - -## Overview - -R is at `/opt/R/4.0.2/bin/R` -python is at `/opt/python/3.7.7/bin/python` -jupyter is at `/opt/python/3.7.7/bin/jupyter` -code-server is at `/opt/code-server/bin/code-server` - -## Changes - -- Update RStudio Professional Drivers to 1.8.0 -- `BREAKING`: `code-server` no longer supports the `/opt/code-server/code-server` location. - - As a result, you need to set `exe=/opt/code-server/bin/code-server` - - After two changes in a row, we suspect this is the final change for a while -_vscode.conf_ -``` -enabled=1 -exe=/opt/code-server/bin/code-server -``` - -# 1.4.1106-5 - -## Overview - -R is at `/opt/R/4.0.2/bin/R` -python is at `/opt/python/3.7.7/bin/python` -jupyter is at `/opt/python/3.7.7/bin/jupyter` -code-server is at `/opt/code-server/code-server` - -## Changes - -- `BREAKING`: Changed code-server version and insulated against version upgrades in the future. To update: -change: - -_vscode.conf_ -``` -enabled=1 -exe=/opt/code-server/code-server-3.2.0-linux-x86_64/code-server -``` -to: -_vscode.conf_ -``` -enabled=1 -exe=/opt/code-server/code-server -``` - -- Update Drivers to version 1.7.0 - -# 1.4.1103-4 - -## Overview - -R is at `/opt/R/4.0.2/bin/R` -python is at `/opt/python/3.7.7/bin/python` -jupyter is at `/opt/python/3.7.7/bin/jupyter` -code-server is at `/opt/code-server/code-server-3.2.0-linux-x86_64/code-server` +- Add NEWS.md +- Add daily builds diff --git a/workbench-session/README.md b/workbench-session/README.md index 243c17d1..1c97a7bd 100644 --- a/workbench-session/README.md +++ b/workbench-session/README.md @@ -2,8 +2,9 @@ * Maintained by: [the Posit Docker team](https://github.com/rstudio/rstudio-docker-products) * Where to get help: [our Github Issues page](https://github.com/rstudio/rstudio-docker-products/issues) -* RStudio Workbench image: [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench) -* RStudio r-session-complete image: [Docker Hub](https://hub.docker.com/r/rstudio/r-session-complete) +* Posit Workbench image: [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench) +* Posit Workbench session image: [Docker Hub](https://hub.docker.com/r/rstudio/workbench-session) +* Posit Workbench session init image: [Docker Hub](https://hub.docker.com/r/rstudio/workbench-session-init) # Supported tags and respective Dockerfile links @@ -33,22 +34,20 @@ The Docker images built from these Dockerfiles are intended to be used for R and Jupyter sessions and jobs with RStudio Workbench (RSW), Launcher, and Kubernetes. -Note: These Docker images are not equipped or intended to be used to run RStudio +Note: These Docker images are not equipped or intended to be used to run Posit Workbench within a Docker container. Visit the [rstudio/rstudio-worbench Docker Hub page](https://hub.docker.com/r/rstudio/rstudio-workbench) for images built for that purpose. -For more information about RStudio Workbench and Launcher, refer to the +For more information about Posit Workbench and Launcher, refer to the [Launcher Overview](https://solutions.rstudio.com/launcher/overview/) on the RStudio Solutions website. For more information about how to use these images with RStudio Workbench and Launcher, refer to the RStudio support article on [Using Docker images with -RStudio Workbench, Launcher, and -Kubernetes](https://support.rstudio.com/hc/en-us/articles/360019253393-Using-Docker-images-with-RStudio-Server-Pro-Launcher-and-Kubernetes). +RStudio Workbench, Launcher, and Kubernetes](https://support.rstudio.com/hc/en-us/articles/360019253393-Using-Docker-images-with-RStudio-Server-Pro-Launcher-and-Kubernetes). -We provide simple ways to extend and build the Dockerfiles. After you have cloned the repo, you can create your own -containers fairly simply with the provided Justfile. +We provide simple ways to extend and build the Dockerfiles. After you have cloned the repo, you can create your own containers fairly simply with the provided Justfile. ## Overview @@ -59,8 +58,7 @@ repository on Docker Hub. These images include the following layers: * Base OS -* RSW session components -* System packages required for R, R packages, and RStudio Professional Drivers +* System packages required for R, R packages, RStudio Professional Drivers, and Workbench Session Components * One version of R * One version of Python * Jupyter Notebooks, JupyterLab, and RSW/RSC notebook extensions diff --git a/workbench-session/docker-compose.test.yml b/workbench-session/docker-compose.test.yml index 80d19f5d..ab22c81a 100644 --- a/workbench-session/docker-compose.test.yml +++ b/workbench-session/docker-compose.test.yml @@ -7,7 +7,6 @@ services: entrypoint: [] environment: # uses .env by default - - RSW_VERSION - R_VERSION - PYTHON_VERSION volumes: From 323250f65c43e674a29230fb46fb669d35a6fae5 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Tue, 19 Nov 2024 13:01:48 -0500 Subject: [PATCH 07/23] Add workbench session target/matrix --- .github/workflows/build-bake.yaml | 42 +++++++++++++++++++++++++++++++ docker-bake.hcl | 30 ++++++++++++++++++++++ workbench-session/README.md | 2 +- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index e2d9f539..bb8301ca 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -323,6 +323,48 @@ jobs: snyk-org: ${{ secrets.SNYK_ORG }} snyk-token: '${{ secrets.SNYK_TOKEN }}' + workbench-session: + needs: [setup] + name: Workbench Session + runs-on: ubuntu-latest-8x + + concurrency: + group: bake-workbench-session-${{ github.ref }} + cancel-in-progress: true + + env: + target: workbench-session + GIT_SHA: ${{ needs.setup.outputs.GIT_SHA }} + + steps: + - name: Checkout + if: github.event_name == 'schedule' + uses: actions/checkout@v4 + with: + ref: 'main' + + - name: Checkout + if: github.event_name != 'schedule' + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' + workbench-session-init: needs: [setup, versions] name: Workbench Session Init diff --git a/docker-bake.hcl b/docker-bake.hcl index a46dac02..6068f8c6 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -185,6 +185,10 @@ variable WORKBENCH_BUILD_MATRIX { } } +variable WORKBENCH_SESSION_MATRIX { + default = PRO_BUILD_MATRIX +} + variable WORKBENCH_SESSION_INIT_BUILD_MATRIX { default = { builds = [ @@ -221,6 +225,7 @@ group "default" { "package-manager", "r-session-complete", "workbench", + "workbench-session", "workbench-session-init", ] } @@ -441,6 +446,31 @@ target "r-session-complete" { } } +target "workbench-session" { + inherits = ["base"] + name = "workbench-session-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + + tags = [ + "ghcr.io/rstudio/workbench-session:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/workbench-session:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-session" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = DEFAULT_JUPYTERLAB_VERSION + } +} + target "workbench" { inherits = ["base"] diff --git a/workbench-session/README.md b/workbench-session/README.md index 1c97a7bd..c921d26c 100644 --- a/workbench-session/README.md +++ b/workbench-session/README.md @@ -52,7 +52,7 @@ We provide simple ways to extend and build the Dockerfiles. After you have clone ## Overview Built images are available from the -[rstudio/r-session-complete](https://hub.docker.com/r/rstudio/r-session-complete) +[rstudio/workbench-session](https://hub.docker.com/r/rstudio/workbench-session) repository on Docker Hub. These images include the following layers: From e5090aa27a51135159be23b8e136db86931d1ea6 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Tue, 19 Nov 2024 16:30:07 -0500 Subject: [PATCH 08/23] Put Quarto on path in earlier layer --- workbench-session/Dockerfile.ubuntu2204 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workbench-session/Dockerfile.ubuntu2204 b/workbench-session/Dockerfile.ubuntu2204 index 28743789..06461bf1 100644 --- a/workbench-session/Dockerfile.ubuntu2204 +++ b/workbench-session/Dockerfile.ubuntu2204 @@ -24,6 +24,9 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Add Jupyter, Python, and Quarto to the PATH +ENV PATH="/opt/python/jupyter/bin:/opt/python/bin:/usr/lib/rstudio-server/bin/quarto/bin:${PATH}" + ### Install TinyTeX using Quarto ### RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex @@ -41,9 +44,6 @@ RUN /opt/python/"${PYTHON_VERSION}"/bin/python -m venv /opt/python/jupyter \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python -m ipykernel install --name py${PYTHON_VERSION_ALT} --display-name "Python ${PYTHON_VERSION_ALT}" \ && /opt/python/jupyter/bin/python -m pip cache purge -# Add Jupyter, Python, and Quarto to the PATH -ENV PATH="/opt/python/jupyter/bin:/opt/python/bin:/usr/lib/rstudio-server/bin/quarto/bin:${PATH}" - COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf EXPOSE 8788/tcp From e1ebceaebbaab59eb65c7a7a61fe3d20a1149836 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Tue, 19 Nov 2024 16:49:38 -0500 Subject: [PATCH 09/23] Set QUARTO_VERSION as arg --- workbench-session/Dockerfile.ubuntu2204 | 1 + 1 file changed, 1 insertion(+) diff --git a/workbench-session/Dockerfile.ubuntu2204 b/workbench-session/Dockerfile.ubuntu2204 index 06461bf1..060933b3 100644 --- a/workbench-session/Dockerfile.ubuntu2204 +++ b/workbench-session/Dockerfile.ubuntu2204 @@ -7,6 +7,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG JUPYTERLAB_VERSION=3.6.5 ARG SCRIPTS_DIR=/opt/positscripts +ARG QUARTO_VERSION=1.3.340 ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter From 3657dc679807c72895c6694e5fa03008e240151f Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Wed, 20 Nov 2024 09:28:34 -0500 Subject: [PATCH 10/23] Fix goss tests --- workbench-session/test/goss.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/workbench-session/test/goss.yaml b/workbench-session/test/goss.yaml index 030ff180..6d7ffb0d 100644 --- a/workbench-session/test/goss.yaml +++ b/workbench-session/test/goss.yaml @@ -1,26 +1,11 @@ file: - /usr/lib/rstudio-server: - exists: true - /usr/lib/rstudio-server/bin/rsession: - exists: true /usr/local/bin/jupyter: exists: true - {{ $version_split := split "." .Env.RSW_VERSION }} - {{ if or (ge ($version_split._0 | atoi) 2025) (and (ge ($version_split._0 | atoi) 2024) (ge ($version_split._1 | atoi) 7)) }} - /usr/lib/rstudio-server/bin/pwb-code-server/bin/code-server: - exists: true - {{ else }} - /usr/lib/rstudio-server/bin/code-server/bin/code-server: - exists: true - {{ end }} /opt/rstudio-drivers: exists: true filetype: directory /var/lib/rstudio-server/r-versions: exists: false - /usr/local/bin/quarto: - exists: true - filetype: symlink command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": @@ -57,7 +42,7 @@ command: exit-status: 0 # Ensure Quarto works - "/usr/local/bin/quarto check --quiet": + "quarto check --quiet": title: quarto_check exit-status: 0 From 3a5b990cda04918b2920064800e49ecf68ac69c9 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Wed, 20 Nov 2024 10:08:17 -0500 Subject: [PATCH 11/23] Update quarto on PATH --- workbench-session/Dockerfile.ubuntu2204 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-session/Dockerfile.ubuntu2204 b/workbench-session/Dockerfile.ubuntu2204 index 060933b3..a806e1ea 100644 --- a/workbench-session/Dockerfile.ubuntu2204 +++ b/workbench-session/Dockerfile.ubuntu2204 @@ -26,7 +26,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Add Jupyter, Python, and Quarto to the PATH -ENV PATH="/opt/python/jupyter/bin:/opt/python/bin:/usr/lib/rstudio-server/bin/quarto/bin:${PATH}" +ENV PATH="/opt/python/jupyter/bin:/opt/python/bin:/opt/quarto/${QUARTO_VERSION}/bin:${PATH}" ### Install TinyTeX using Quarto ### RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex From 298aaf34ae0101952c33d3d6071eb58c6eb3a341 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Thu, 21 Nov 2024 08:43:19 -0500 Subject: [PATCH 12/23] Don't install tinytex --- workbench-session/Dockerfile.ubuntu2204 | 6 +----- workbench-session/README.md | 10 ++++++---- workbench-session/test/goss.yaml | 13 ++----------- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/workbench-session/Dockerfile.ubuntu2204 b/workbench-session/Dockerfile.ubuntu2204 index a806e1ea..eb4eb82d 100644 --- a/workbench-session/Dockerfile.ubuntu2204 +++ b/workbench-session/Dockerfile.ubuntu2204 @@ -7,7 +7,6 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG JUPYTERLAB_VERSION=3.6.5 ARG SCRIPTS_DIR=/opt/positscripts -ARG QUARTO_VERSION=1.3.340 ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter @@ -26,10 +25,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Add Jupyter, Python, and Quarto to the PATH -ENV PATH="/opt/python/jupyter/bin:/opt/python/bin:/opt/quarto/${QUARTO_VERSION}/bin:${PATH}" - -### Install TinyTeX using Quarto ### -RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex +ENV PATH="/opt/python/jupyter/bin:/opt/python/bin:/usr/lib/rstudio-server/bin/quarto/bin:${PATH}" RUN /opt/python/"${PYTHON_VERSION}"/bin/python -m venv /opt/python/jupyter \ && /opt/python/jupyter/bin/python -m pip install --upgrade pip \ diff --git a/workbench-session/README.md b/workbench-session/README.md index c921d26c..b7a3bf9c 100644 --- a/workbench-session/README.md +++ b/workbench-session/README.md @@ -8,7 +8,7 @@ # Supported tags and respective Dockerfile links -* [`jammy`, `ubuntu2204`, `jammy-2024.09.1`, `ubuntu2204-2024.09.1`](https://github.com/rstudio/rstudio-docker-products/blob/main/r-session-complete/Dockerfile.ubuntu2204) +* [`ubuntu2204-r4.4.1_4.3.3-py3.12.6_3.11.10`, `ubuntu2204-r4.4.1_4.3.3-py3.11.10_3.10.15`, `ubuntu2204-r4.4.0_4.3.3-py3.12.1_3.11.7`](https://github.com/rstudio/rstudio-docker-products/blob/main/workbench-session/Dockerfile.ubuntu2204) # What are the r-session-complete images? @@ -31,7 +31,7 @@ Images for R and Python sessions and jobs to be used RStudio Workbench, Launcher # How to use these images The Docker images built from these Dockerfiles are intended to be used for R and -Jupyter sessions and jobs with RStudio Workbench (RSW), Launcher, and +Jupyter sessions and jobs with Posit Workbench (PWB), Launcher, and Kubernetes. Note: These Docker images are not equipped or intended to be used to run Posit @@ -39,6 +39,8 @@ Workbench within a Docker container. Visit the [rstudio/rstudio-worbench Docker Hub page](https://hub.docker.com/r/rstudio/rstudio-workbench) for images built for that purpose. +Note: These images do not include the Posit Workbench Session Components. To use these images with Posit Workbench, the [session init container](https://hub.docker.com/r/rstudio/workbench-session-init) must be enabled within the Posit Workbench configuration. For more information, refer to the [Posit Workbench documentation](https://docs.rstudio.com/ide/server-pro/launcher/). + For more information about Posit Workbench and Launcher, refer to the [Launcher Overview](https://solutions.rstudio.com/launcher/overview/) on the RStudio Solutions website. @@ -59,8 +61,8 @@ These images include the following layers: * Base OS * System packages required for R, R packages, RStudio Professional Drivers, and Workbench Session Components -* One version of R -* One version of Python +* Two versions of R +* Two versions of Python * Jupyter Notebooks, JupyterLab, and RSW/RSC notebook extensions * RStudio Professional Drivers diff --git a/workbench-session/test/goss.yaml b/workbench-session/test/goss.yaml index 6d7ffb0d..0464eebc 100644 --- a/workbench-session/test/goss.yaml +++ b/workbench-session/test/goss.yaml @@ -6,6 +6,8 @@ file: filetype: directory /var/lib/rstudio-server/r-versions: exists: false + /usr/local/bin/quarto: + exists: true command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": @@ -41,14 +43,3 @@ command: timeout: 60000 exit-status: 0 -# Ensure Quarto works - "quarto check --quiet": - title: quarto_check - exit-status: 0 - -# Ensure TinyTeX is installed - "quarto list tools": - title: quarto_tinytex_installed - exit-status: 0 - stderr: - - "/tinytex\\s+Up to date\\s+v\\d{4}\\.\\d{2}(\\.\\d{2})?\\s+v\\d{4}\\.\\d{2}(\\.\\d{2})?/" From 7d533e6ea7269a52c10de7b29bede59b252cfe5a Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Thu, 21 Nov 2024 09:45:53 -0500 Subject: [PATCH 13/23] Quarto isn't in /usr/local/bin --- workbench-session/test/goss.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/workbench-session/test/goss.yaml b/workbench-session/test/goss.yaml index 0464eebc..2e1d70be 100644 --- a/workbench-session/test/goss.yaml +++ b/workbench-session/test/goss.yaml @@ -6,8 +6,6 @@ file: filetype: directory /var/lib/rstudio-server/r-versions: exists: false - /usr/local/bin/quarto: - exists: true command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": From 7871f1c60e2574420945232bbe49af5a5adceafe Mon Sep 17 00:00:00 2001 From: Skye Turriff Date: Fri, 15 Nov 2024 14:25:56 -0500 Subject: [PATCH 14/23] Update build-manual.yaml Adds the WORKBENCH_SESSION_INIT variable to the manual build flow --- .github/workflows/build-manual.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index 5900804b..e5980f89 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -141,6 +141,8 @@ jobs: product="CONNECT" elif [[ "$product" == "package-manager" ]]; then product="PACKAGE_MANAGER" + elif [[ "$product" == "workbench-session-init" ]]; then + product="WORKBENCH_SESSION_INIT" else product="WORKBENCH" fi From cd7fe09dacf7a090eb0fbd89bf0ec1328a5dcb08 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 20 Nov 2024 10:47:48 -0700 Subject: [PATCH 15/23] Merge versions job into workbench-session-init job --- .github/workflows/build-bake.yaml | 55 ++++++++++++------------------- Justfile | 3 +- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index bb8301ca..43c1b194 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -37,39 +37,6 @@ jobs: GIT_SHA=$(git rev-parse --short HEAD) echo "GIT_SHA=$GIT_SHA" >> $GITHUB_OUTPUT echo "$GIT_SHA" - versions: - name: Fetch Workbench session init container version - runs-on: ubuntu-latest - - concurrency: - group: fetch-versions-${{ github.ref }} - cancel-in-progress: true - - outputs: - WORKBENCH_SESSION_INIT_VERSION: ${{ steps.get-version.outputs.WORKBENCH_SESSION_INIT_VERSION }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Just - uses: extractions/setup-just@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install Python dependencies - run: | - pip install requests - - - name: Get Version - id: get-version - run: | - WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) - echo "WORKBENCH_SESSION_INIT_VERSION=$WORKBENCH_SESSION_INIT_VERSION" >> $GITHUB_OUTPUT base: needs: [setup] @@ -377,7 +344,6 @@ jobs: env: target: workbench-session-init GIT_SHA: ${{ needs.setup.outputs.GIT_SHA }} - WORKBENCH_SESSION_INIT_VERSION: ${{ needs.versions.outputs.WORKBENCH_SESSION_INIT_VERSION }} steps: - name: Checkout @@ -396,7 +362,28 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Python dependencies + run: | + pip install requests + + - name: Get Version + id: get-version + run: | + WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + echo "WORKBENCH_SESSION_INIT_VERSION=$WORKBENCH_SESSION_INIT_VERSION" >> $GITHUB_OUTPUT + - name: Build, Test, and Push + env: + WORKBENCH_SESSION_INIT_VERSION: ${{ steps.get-version.outputs.WORKBENCH_SESSION_INIT_VERSION }} uses: ./.github/actions/bake-test-push with: target: ${{ env.target }} diff --git a/Justfile b/Justfile index 3b74f8a6..2f858d21 100644 --- a/Justfile +++ b/Justfile @@ -59,8 +59,7 @@ bake target="default": docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} # just preview-bake workbench-images dev -preview-build: - just preview-bake "default" +alias preview-build := preview-bake preview-bake target branch="$(git branch --show-current)": just -f {{justfile()}} create-builder || true WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ From afed1baf9ea814ca01a3a0c95c1fe7ec9849e3ac Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 20 Nov 2024 11:10:05 -0700 Subject: [PATCH 16/23] Remove versions as dependency job --- .github/workflows/build-bake.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 43c1b194..0b916df2 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -333,7 +333,7 @@ jobs: snyk-token: '${{ secrets.SNYK_TOKEN }}' workbench-session-init: - needs: [setup, versions] + needs: [setup] name: Workbench Session Init runs-on: ubuntu-latest-8x From 773522f6d0dcf411099643408bad7c6fe55bf48f Mon Sep 17 00:00:00 2001 From: Skye Turriff Date: Wed, 20 Nov 2024 15:12:19 -0500 Subject: [PATCH 17/23] Add Go-based entrypoint for Workbench session init (#871) * add go program * add multi-stage build dockerfile * add positron * fix path * update err msgs and reuse file info * merge comments * enhance file copying with otiai10/copy package and update Dockerfile * remove uneeded components * add shared_run to common deps * fix component filename * update Dockerfile to use ubuntu:22.04 base image, as the scratch image doesn't work nicely with GOSS * remove test var --------- Co-authored-by: Ian Pittwood --- workbench-session-init/Dockerfile.ubuntu2204 | 39 +++- workbench-session-init/entrypoint/go.mod | 9 + workbench-session-init/entrypoint/go.sum | 6 + workbench-session-init/entrypoint/main.go | 170 ++++++++++++++++++ .../entrypoint/main_test.go | 149 +++++++++++++++ workbench-session-init/run.sh | 17 -- workbench-session-init/test/goss.yaml | 2 +- 7 files changed, 368 insertions(+), 24 deletions(-) create mode 100644 workbench-session-init/entrypoint/go.mod create mode 100644 workbench-session-init/entrypoint/go.sum create mode 100644 workbench-session-init/entrypoint/main.go create mode 100644 workbench-session-init/entrypoint/main_test.go delete mode 100644 workbench-session-init/run.sh diff --git a/workbench-session-init/Dockerfile.ubuntu2204 b/workbench-session-init/Dockerfile.ubuntu2204 index 2d0bbc44..7b8ab4f7 100644 --- a/workbench-session-init/Dockerfile.ubuntu2204 +++ b/workbench-session-init/Dockerfile.ubuntu2204 @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 AS build +FROM ubuntu:22.04 AS builder # Install required tools: # - ca-certificates installs necessary certificates to use cURL with HTTPS websites @@ -9,17 +9,44 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ARG RSW_VERSION=2024.09.1+394.pro7 +ARG GO_VERSION=1.22.2 +# Download the RStudio Workbench session components and install Go SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN mkdir -p /pwb-staging && \ RSW_VERSION_URL=$(echo -n "${RSW_VERSION}" | sed 's/+/-/g') && \ - echo "Downloading https://s3.amazonaws.com/rstudio-ide-build/session/multi/x86_64/rsp-session-multi-linux-${RSW_VERSION_URL}-x86_64.tar.gz" && \ curl -fsSL -o /pwb-staging/rsp-session-multi-linux.tar.gz "https://s3.amazonaws.com/rstudio-ide-build/session/multi/x86_64/rsp-session-multi-linux-${RSW_VERSION_URL}-x86_64.tar.gz" && \ mkdir -p /opt/session-components && \ - tar -C /opt/session-components -xf /pwb-staging/rsp-session-multi-linux.tar.gz && \ - chmod -R 755 /opt/session-components && \ + tar -C /opt/session-components -xpf /pwb-staging/rsp-session-multi-linux.tar.gz && \ + chmod 755 /opt/session-components && \ + curl -fsSL -o /pwb-staging/go.tar.gz "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" && \ + tar -C /usr/local -xf /pwb-staging/go.tar.gz && \ rm -rf /pwb-staging -COPY --chmod=755 run.sh /usr/local/bin/run.sh +# Add Go binary to PATH +ENV PATH="/usr/local/go/bin:$PATH" + +# Set the Go workspace +WORKDIR /workspace + +# Copy the Go source code and download dependencies +COPY entrypoint/go.mod entrypoint/go.sum ./ +RUN go mod download + +# Copy the Go source code and build the binary +COPY entrypoint/main.go . +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags '-s -w' -o entrypoint main.go + +# Create the final image +FROM ubuntu:22.04 AS build + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* + +# Copy the compiled Go binary and session components from the builder stage +COPY --from=builder --chmod=755 /workspace/entrypoint /usr/local/bin/entrypoint +COPY --from=builder --chmod=755 /opt/session-components /opt/session-components -ENTRYPOINT ["/usr/local/bin/run.sh"] +ENTRYPOINT ["/usr/local/bin/entrypoint"] diff --git a/workbench-session-init/entrypoint/go.mod b/workbench-session-init/entrypoint/go.mod new file mode 100644 index 00000000..fff4e5d3 --- /dev/null +++ b/workbench-session-init/entrypoint/go.mod @@ -0,0 +1,9 @@ +module entrypoint + +go 1.22.2 + +require ( + github.com/otiai10/copy v1.14.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect +) diff --git a/workbench-session-init/entrypoint/go.sum b/workbench-session-init/entrypoint/go.sum new file mode 100644 index 00000000..21650d9b --- /dev/null +++ b/workbench-session-init/entrypoint/go.sum @@ -0,0 +1,6 @@ +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/workbench-session-init/entrypoint/main.go b/workbench-session-init/entrypoint/main.go new file mode 100644 index 00000000..5eb6afc0 --- /dev/null +++ b/workbench-session-init/entrypoint/main.go @@ -0,0 +1,170 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "time" + + cp "github.com/otiai10/copy" +) + +const ( + sourceDir = "/opt/session-components" + targetDir = "/mnt/init" +) + +var ( + // Read the PWB_SESSION_TYPE environment variable + sessionType = os.Getenv("PWB_SESSION_TYPE") + + // Set the copy options. + // Preserve permissions, times, and owner. + opt = cp.Options{ + PermissionControl: cp.PerservePermission, + PreserveTimes: true, + PreserveOwner: true, + NumOfWorkers: 20, + } + + // List of dependencies common to all session types + commonDeps = []string{ + "bin/git-credential-pwb", + "bin/focal", + "bin/jammy", + "bin/noble", + "bin/opensuse15", + "bin/postback", + "bin/pwb-supervisor", + "bin/quarto", + "bin/r-ldpath", + "bin/rhel8", + "bin/rhel9", + "bin/shared-run", + "R", + "resources", + "www", + "www-symbolmaps", + } + + // Map of session-specific dependencies + sessionDeps = map[string][]string{ + "jupyter": { + "bin/jupyter-session-run", + "bin/node", + "extras", + }, + "positron": { + "bin/positron-server", + "bin/positron-session-run", + "extras", + }, + "rstudio": { + "bin/node", + "bin/rsession-run", + }, + "vscode": { + "bin/pwb-code-server", + "bin/vscode-session-run", + "extras", + }, + } +) + +func main() { + if sessionType == "" { + fmt.Println("PWB_SESSION_TYPE environment variable is not set") + os.Exit(1) + } + + programStart := time.Now() + defer func() { + elapsed := time.Since(programStart) + fmt.Printf("Program took %s\n", elapsed) + }() + + filesToCopy, err := getFilesToCopy(sessionType) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = validateTargetDir(targetDir) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = copyFiles(sourceDir, targetDir, filesToCopy) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Copy operation completed.") +} + +// getFilesToCopy returns the list of files to copy based on the session type. +func getFilesToCopy(sessionType string) ([]string, error) { + files := commonDeps + if deps, ok := sessionDeps[sessionType]; ok { + files = append(files, deps...) + } else { + return nil, fmt.Errorf("unknown session type: %s", sessionType) + } + return files, nil +} + +// validateTargetDir checks if the target directory exists and is empty. +func validateTargetDir(targetDir string) error { + if _, err := os.Stat(targetDir); os.IsNotExist(err) { + return fmt.Errorf("cannot find the copy target %s", targetDir) + } + + isEmpty, err := isDirEmpty(targetDir) + if err != nil { + return fmt.Errorf("error checking if target directory is empty: %v", err) + } + if !isEmpty { + return fmt.Errorf("target directory %s is not empty", targetDir) + } + + return nil +} + +// isDirEmpty checks if a directory is empty. +func isDirEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// copyFiles copies the files from the source directory to the target directory. +// It uses the otiai10/copy package to copy files, with options to preserve +// permissions, times, and owner. +func copyFiles(src, dst string, filesToCopy []string) error { + fmt.Printf("Copying files from %s to %s\n", src, dst) + start := time.Now() + + for _, file := range filesToCopy { + srcPath := filepath.Join(src, file) + dstPath := filepath.Join(dst, file) + err := cp.Copy(srcPath, dstPath, opt) + if err != nil { + return fmt.Errorf("error copying %s: %v", srcPath, err) + } + } + + elapsed := time.Since(start) + fmt.Printf("Copy operation took %s\n", elapsed) + + return nil +} diff --git a/workbench-session-init/entrypoint/main_test.go b/workbench-session-init/entrypoint/main_test.go new file mode 100644 index 00000000..b16aa736 --- /dev/null +++ b/workbench-session-init/entrypoint/main_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "syscall" + "testing" +) + +func TestGetFilesToCopy(t *testing.T) { + tests := []struct { + sessionType string + expected []string + expectError bool + }{ + { + sessionType: "jupyter", + expected: append(commonDeps, sessionDeps["jupyter"]...), + expectError: false, + }, + { + sessionType: "positron", + expected: append(commonDeps, sessionDeps["positron"]...), + expectError: false, + }, + { + sessionType: "rstudio", + expected: append(commonDeps, sessionDeps["rstudio"]...), + expectError: false, + }, + { + sessionType: "vscode", + expected: append(commonDeps, sessionDeps["vscode"]...), + expectError: false, + }, + { + sessionType: "unknown", + expected: nil, + expectError: true, + }, + } + + for _, test := range tests { + t.Run(test.sessionType, func(t *testing.T) { + files, err := getFilesToCopy(test.sessionType) + if test.expectError { + if err == nil { + t.Errorf("Expected error for session type %s, but got none", test.sessionType) + } + } else { + if err != nil { + t.Errorf("Did not expect error for session type %s, but got: %v", test.sessionType, err) + } + if !reflect.DeepEqual(files, test.expected) { + t.Errorf("Files do not match for session type %s. Expected: %v, Got: %v", test.sessionType, test.expected, files) + } + } + }) + } +} + +func TestCopy(t *testing.T) { + // Create temporary source and destination directories + srcDir, err := os.MkdirTemp("", "src") + if err != nil { + t.Fatalf("Failed to create temporary source directory: %v", err) + } + defer os.RemoveAll(srcDir) + + dstDir, err := os.MkdirTemp("", "dst") + if err != nil { + t.Fatalf("Failed to create temporary destination directory: %v", err) + } + defer os.RemoveAll(dstDir) + + // Create a sample directory structure in the source directory that looks like: + // srcDir + // ├── file1.txt + // └── subdir1 + // ├── file2.txt + // └── subdir2 + // └── file3.txt + // |__ subdir3 + err = os.MkdirAll(filepath.Join(srcDir, "subdir1"), 0755) + if err != nil { + t.Fatalf("Failed to create subdir1: %v", err) + } + err = os.WriteFile(filepath.Join(srcDir, "file1.txt"), []byte("file1 content"), 0644) + if err != nil { + t.Fatalf("Failed to create file1.txt: %v", err) + } + err = os.WriteFile(filepath.Join(srcDir, "subdir1", "file2.txt"), []byte("file2 content"), 0600) + if err != nil { + t.Fatalf("Failed to create file2.txt: %v", err) + } + err = os.MkdirAll(filepath.Join(srcDir, "subdir1", "subdir2"), 0755) + if err != nil { + t.Fatalf("Failed to create subdir2: %v", err) + } + err = os.WriteFile(filepath.Join(srcDir, "subdir1", "subdir2", "file3.txt"), []byte("file3 content"), 0644) + if err != nil { + t.Fatalf("Failed to create file3.txt: %v", err) + } + err = os.MkdirAll(filepath.Join(srcDir, "subdir3"), 0755) + if err != nil { + t.Fatalf("Failed to create subdir3: %v", err) + } + + // Copy the directory structure from source to destination + // exclude subdir3 + filesToCopy := []string{ + "file1.txt", + "subdir1", + } + err = copyFiles(srcDir, dstDir, filesToCopy) + if err != nil { + t.Fatalf("Failed to copy files: %v", err) + } + + // Verify that the directory structure and files are correctly copied + verifyFile(t, filepath.Join(dstDir, "file1.txt"), 0644, os.Getuid(), os.Getgid()) + verifyFile(t, filepath.Join(dstDir, "subdir1", "file2.txt"), 0600, os.Getuid(), os.Getgid()) + verifyFile(t, filepath.Join(dstDir, "subdir1", "subdir2", "file3.txt"), 0644, os.Getuid(), os.Getgid()) + // Verify that subdir3 is not copied + if _, err := os.Stat(filepath.Join(dstDir, "subdir3")); !os.IsNotExist(err) { + t.Errorf("Directory subdir3 should not have been copied") + } +} + +func verifyFile(t *testing.T, path string, mode os.FileMode, uid, gid int) { + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Failed to stat file %s: %v", path, err) + } + + if info.Mode() != mode { + t.Errorf("File %s has incorrect permissions: got %v, want %v", path, info.Mode(), mode) + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("Failed to get file ownership for %s", path) + } + + if int(stat.Uid) != uid || int(stat.Gid) != gid { + t.Errorf("File %s has incorrect ownership: got %d:%d, want %d:%d", path, stat.Uid, stat.Gid, uid, gid) + } +} diff --git a/workbench-session-init/run.sh b/workbench-session-init/run.sh deleted file mode 100644 index 9c29238e..00000000 --- a/workbench-session-init/run.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -set -x - -S=/opt/session-components - -# The target should exist and be an empty directory. -T=/mnt/init - -if [ ! -d "${T}" ] ; then - echo "Cannot find the copy target ${T}" - exit 1 -fi - -echo "Copying files from /session-components to /mnt/init" -time cp -r $S/* $T diff --git a/workbench-session-init/test/goss.yaml b/workbench-session-init/test/goss.yaml index 7c9ab9e7..7b384ffd 100644 --- a/workbench-session-init/test/goss.yaml +++ b/workbench-session-init/test/goss.yaml @@ -7,7 +7,7 @@ file: exists: true mode: "0755" filetype: directory - /usr/local/bin/run.sh: + /usr/local/bin/entrypoint: exists: true filetype: file mode: "0755" From 4720f10d065ac70cf8d6e30c1b15548192bbf7e0 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Fri, 22 Nov 2024 16:48:09 -0500 Subject: [PATCH 18/23] Update workbench-session/README.md Co-authored-by: Skye Turriff --- workbench-session/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-session/README.md b/workbench-session/README.md index b7a3bf9c..35875211 100644 --- a/workbench-session/README.md +++ b/workbench-session/README.md @@ -60,7 +60,7 @@ repository on Docker Hub. These images include the following layers: * Base OS -* System packages required for R, R packages, RStudio Professional Drivers, and Workbench Session Components +* System packages required for R, R packages, and RStudio Professional Drivers * Two versions of R * Two versions of Python * Jupyter Notebooks, JupyterLab, and RSW/RSC notebook extensions From 7179f137852e0555a71db6090030bcb3c0bc6255 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Fri, 22 Nov 2024 16:49:32 -0500 Subject: [PATCH 19/23] Use the correct build matrix --- docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 6068f8c6..d3880456 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -461,7 +461,7 @@ target "workbench-session" { product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" } - matrix = BASE_BUILD_MATRIX + matrix = WORKBENCH_SESSION_MATRIX args = { R_VERSION = builds.r_primary R_VERSION_ALT = builds.r_alternate From da276132f383670f12031fa151c010480bc3c1b4 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Fri, 22 Nov 2024 16:49:41 -0500 Subject: [PATCH 20/23] Add check for pip installation in goss tests --- workbench-session/test/goss.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workbench-session/test/goss.yaml b/workbench-session/test/goss.yaml index 2e1d70be..2d9cd604 100644 --- a/workbench-session/test/goss.yaml +++ b/workbench-session/test/goss.yaml @@ -40,4 +40,7 @@ command: title: jupyter_in_path_var timeout: 60000 exit-status: 0 - + + "pip --version": + title: pip_installed + exit-status: 0 From cb4f0aa8462cab2b2f54040461c370fda476e763 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Tue, 26 Nov 2024 09:07:55 -0500 Subject: [PATCH 21/23] COPY positron.extensions.conf --- workbench-session/Dockerfile.ubuntu2204 | 1 + workbench-session/test/goss.yaml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/workbench-session/Dockerfile.ubuntu2204 b/workbench-session/Dockerfile.ubuntu2204 index eb4eb82d..97eb4a85 100644 --- a/workbench-session/Dockerfile.ubuntu2204 +++ b/workbench-session/Dockerfile.ubuntu2204 @@ -42,5 +42,6 @@ RUN /opt/python/"${PYTHON_VERSION}"/bin/python -m venv /opt/python/jupyter \ && /opt/python/jupyter/bin/python -m pip cache purge COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf +COPY positron.extensions.conf /etc/rstudio/positron.extensions.conf EXPOSE 8788/tcp diff --git a/workbench-session/test/goss.yaml b/workbench-session/test/goss.yaml index 2d9cd604..fbed1f8b 100644 --- a/workbench-session/test/goss.yaml +++ b/workbench-session/test/goss.yaml @@ -6,6 +6,10 @@ file: filetype: directory /var/lib/rstudio-server/r-versions: exists: false + /etc/rstudio/vscode.extensions.conf + exists: true + /etc/rstudio/positron.extensions.conf + exists: true command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": From bd0ae19d0045e10a1e08cf958598b000939583c3 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Tue, 26 Nov 2024 09:08:28 -0500 Subject: [PATCH 22/23] Remove Snyk policy file from workbench-session --- workbench-session/.snyk | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 workbench-session/.snyk diff --git a/workbench-session/.snyk b/workbench-session/.snyk deleted file mode 100644 index 909ad99a..00000000 --- a/workbench-session/.snyk +++ /dev/null @@ -1,20 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.25.0 -# ignores vulnerabilities until expiry date; change duration by modifying expiry date -ignore: - SNYK-GOLANG-GITHUBCOMCREWJAMSAML-5971016: - - '*': - reason: >- - Reported upstream in - https://github.com/rstudio/rstudio-pro/issues/6529 - expires: 2024-08-31T00:00:00.000Z - created: 2024-07-02T20:33:30.847Z - SNYK-GOLANG-GITHUBCOMGOJOSEGOJOSEV3-6070737: - - '*': - reason: >- - Confirmed fixed upstream in - https://github.com/rstudio/rstudio-pro/issues/6635. Patch will be - ingested in Workbench 2024.08.0 (expected within 1 week). - expires: 2024-08-07T00:00:00.000Z - created: 2024-07-31T17:46:24.852Z -patch: {} From f46c966fc86244848e1f803a605e4be24451a925 Mon Sep 17 00:00:00 2001 From: Zach Hannum Date: Tue, 26 Nov 2024 11:48:41 -0500 Subject: [PATCH 23/23] Fix missing colons in goss tests --- workbench-session/test/goss.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench-session/test/goss.yaml b/workbench-session/test/goss.yaml index fbed1f8b..f831c5ba 100644 --- a/workbench-session/test/goss.yaml +++ b/workbench-session/test/goss.yaml @@ -6,9 +6,9 @@ file: filetype: directory /var/lib/rstudio-server/r-versions: exists: false - /etc/rstudio/vscode.extensions.conf + /etc/rstudio/vscode.extensions.conf: exists: true - /etc/rstudio/positron.extensions.conf + /etc/rstudio/positron.extensions.conf: exists: true command: