Merge branch 'dev' into tabular-improvement

.ci/check_test_class_names.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+TEST=$(mktemp)
+TEST_EXTRA_CLASSES=$(mktemp)
+
+pytest --collect-only --ignore=test/functional lib/galaxy_test/ test/ > "$TEST"
+pytest -o python_classes='Test* *Test *TestCase' --collect-only --ignore=test/functional lib/galaxy_test/ test/ > "$TEST_EXTRA_CLASSES"
+
+n_tests=$(grep 'tests collected' "$TEST" | sed -e 's/[^0-9]*\([0-9]*\) tests collected.*/\1/')
+n_tests_extra_classes=$(grep 'tests collected' "$TEST_EXTRA_CLASSES" | sed -e 's/[^0-9]*\([0-9]*\) tests collected.*/\1/')
+
+if [ "$n_tests_extra_classes" -gt "$n_tests" ]; then
+    echo "New test class with name not starting with Test introduced, change it to have tests collected by pytest"
+    diff "$TEST" "$TEST_EXTRA_CLASSES"
+    exit 1
+fi

.ci/eslint_wrapper.sh

@@ -5,4 +5,4 @@ set -e
 # Setting NODE_PATH and config appropriately, using dependencies from
 # client/node_modules, run eslint against args passed to this script.
 # Primary use case here is for a pre-commit check.
-NODE_PATH=src/ node client/node_modules/eslint/bin/eslint.js -c client/.eslintrc.json "$@"
+NODE_PATH=src/ node client/node_modules/eslint/bin/eslint.js -c client/.eslintrc.js "$@"

.ci/flake8_ignorelist.txt

@@ -1,18 +1,16 @@
 .git
 .tox
-.venv
-.venv3
+.venv*
 packages/*/.venv
 packages/*/build
 packages/*/dist
 packages/venv
 node_modules
 database
 doc/build
-doc/source/conf.py
 eggs
-lib/galaxy/util/jstree.py
 lib/galaxy/web/proxy/js/node_modules
+lib/tool_shed/test/test_data/repos
 static/maps
 static/scripts
 test/functional/tools/cwl_tools/v1.?/

.ci/ignore-spelling.txt

@@ -0,0 +1,3 @@
+hda
+implementors
+purgable

.ci/validate_test_tools.sh

@@ -7,5 +7,7 @@ xsd_path="lib/galaxy/tools/xsd/galaxy.xsd"
 xmllint --noout "$xsd_path"
 
 test_tools_path='test/functional/tools'
-tool_files_list=$(ls "$test_tools_path"/*.xml | grep -v '_conf.xml$')
+# test all test tools except upload.xml which uses a non-standard conditional
+# (without param) which does not survive xsd validation
+tool_files_list=$(ls "$test_tools_path"/*.xml | grep -v '_conf.xml$' | grep -v upload.xml)
 sh scripts/validate_tools.sh $tool_files_list

.circleci/config.yml

@@ -5,20 +5,8 @@ variables:
     restore_cache:
       keys:
         - v1-repo-{{ .Environment.CIRCLE_SHA1 }}
-  restore_yarn_cache: &restore_yarn_cache
-    restore_cache:
-      keys:
-        - v1-repo-{{ .Environment.CIRCLE_SHA1 }}
-        - yarn-packages-{{ checksum "client/yarn.lock" }}
-  save_yarn_cache: &save_yarn_cache
-    save_cache:
-      key: yarn-packages-{{ checksum "client/yarn.lock" }}
-      paths:
-        - ~/.cache/yarn
   install_tox: &install_tox
     run: pip install tox
-  install_deb_reqs: &install_deb_reqs
-    run: sudo apt-get update && sudo apt-get install ffmpeg nodejs -y
   set_workdir: &set_workdir
     working_directory: ~/repo
   requires_get_code: &requires_get_code
@@ -27,7 +15,7 @@ variables:
 jobs:
   get_code:
     docker:
-      - image: cimg/python:3.7
+      - image: cimg/python:3.8
     <<: *set_workdir
     steps:
       # Replace standard code checkout with shallow clone to speed things up.
@@ -36,7 +24,7 @@ jobs:
           command: |-
               # Add github.com to known hosts
               mkdir -p ~/.ssh
-              echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+              echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
               ' >> ~/.ssh/known_hosts
 
               # Add the user ssh key and set correct perms
@@ -85,41 +73,18 @@ jobs:
             - ~/repo
   validate_test_tools:
     docker:
-      - image: cimg/python:3.7
+      - image: cimg/python:3.8
     <<: *set_workdir
     steps:
       - *restore_repo_cache
       - run: sudo apt-get update
       - run: sudo apt-get install -y libxml2-utils
       - *install_tox
       - run: tox -e validate_test_tools
-  test_galaxy_packages:
-    docker:
-      - image: cimg/python:3.7
-    <<: *set_workdir
-    steps:
-      - *restore_repo_cache
-      - *install_tox
-      - *install_deb_reqs
-      - run: tox -e test_galaxy_packages
-  js_lint:
-    docker:
-      - image: cimg/node:16.13.2
-    <<: *set_workdir
-    steps:
-      - *restore_yarn_cache
-      - run: cd client && yarn install --frozen-lockfile
-      - *save_yarn_cache
-      - run: cd client && yarn run eslint
-      - run: cd client && yarn run format-check
 workflows:
   version: 2
   get_code_and_test:
     jobs:
       - get_code
-      - test_galaxy_packages:
-          <<: *requires_get_code
       - validate_test_tools:
           <<: *requires_get_code
-      - js_lint:
-          <<: *requires_get_code

.flake8

@@ -0,0 +1,11 @@
+[flake8]
+# These are exceptions allowed by Galaxy style guidelines:
+# B008 Do not perform function calls in argument defaults (for FastAPI Depends and Body)
+# E203 is whitespace before ':'; we follow black's formatting here. See https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated
+# E402 module level import not at top of file # TODO, we would like to improve this.
+# E501 is line length (delegated to black)
+# E701,E704 are multiple statements on one line; we follow black's formatting here. See https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#configuration
+# W503 is line breaks before binary operators, which has been reversed in PEP 8.
+# D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time).
+ignore = B008,E203,E402,E501,E701,E704,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413
+exclude = lib/tool_shed/test/test_data/repos

.github/labeler.yml

@@ -9,6 +9,8 @@ area/auth:
   - lib/galaxy/authnz/**/*
 area/client:
   - client/*
+area/database:
+  - lib/galaxy/model/**/*
 area/datatypes:
   - lib/galaxy/datatypes/**/*
   - lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -61,8 +63,8 @@ area/toolshed:
   - lib/toolshed/**/*
   - templates/webapps/tool_shed/**/*
 area/UI-UX:
-  - client/src/**/*
-  - templates/**/*
+  - all: ["client/src/**/*", "!client/src/api/schema/schema.ts"]
+    any: ["templates/**/*"]
 area/util:
   - lib/galaxy/util/**/*
 area/visualizations:

.github/pull_request_template.md

@@ -10,5 +10,4 @@
   1. [add testing steps and prerequisites here if you didn't write automated tests covering all your changes]
 
 ## License
-- [x] I agree to license these contributions under [Galaxy's current license](https://github.com/galaxyproject/galaxy/blob/dev/LICENSE.txt).
-- [x] I agree to allow the Galaxy committers to license these and all my past contributions to the core galaxy codebase under the [MIT license](https://opensource.org/licenses/MIT). If this condition is an issue, uncheck and just let us know why with an e-mail to galaxy-committers@lists.galaxyproject.org.
+- [x] I agree to license these and all my past contributions to the core galaxy codebase under the [MIT license](https://opensource.org/licenses/MIT).

.github/workflows/api.yaml

@@ -19,15 +19,16 @@ env:
   GALAXY_TEST_RAISE_EXCEPTION_ON_HISTORYLESS_HDA: '1'
   GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
 concurrency:
-  group: api-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ['3.7']
+        python-version: ['3.8']
         chunk: [0, 1]
     services:
       postgres:
@@ -46,13 +47,18 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Get full Python version
         id: full-python-version
         shell: bash
-        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
+        run: echo "version=$(python -c 'import sys; print("-".join(str(v) for v in sys.version_info))')" >> $GITHUB_OUTPUT
       - name: Cache pip dir
         uses: actions/cache@v3
         with:
@@ -66,7 +72,7 @@ jobs:
       - name: Run tests
         run: ./run_tests.sh --coverage --skip_flakey_fails -api lib/galaxy_test/api -- --num-shards=2 --shard-id=${{ matrix.chunk }}
         working-directory: 'galaxy root'
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v3
         with:
           flags: api
           working-directory: 'galaxy root'

.github/workflows/build_client.yaml

@@ -0,0 +1,36 @@
+name: Build client for selenium tests
+on:
+  workflow_call:
+    outputs:
+      commit-id:
+        description: Commit ID
+        value: ${{ jobs.build-client.outputs.commit-id }}
+jobs:
+  build-client:
+    runs-on: ubuntu-latest
+    outputs:
+      commit-id: ${{ steps.client-commit.outputs.commit }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
+      - name: get client commit
+        id: client-commit
+        shell: bash
+        run: echo "commit=$(git rev-parse HEAD 2>/dev/null)" >> $GITHUB_OUTPUT
+        working-directory: 'galaxy root'
+      - name: cache client build
+        uses: actions/cache@v3
+        id: cache
+        with:
+          key: galaxy-static-${{ steps.client-commit.outputs.commit }}
+          path: 'galaxy root/static'
+      - name: Build client
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make client
+        working-directory: 'galaxy root'

.github/workflows/build_container_image.yaml

@@ -2,10 +2,12 @@ name: Build Container Image
 on:
   push:
     branches:
-      - 'release*'
       - dev
+      - release_*
+    tags:
+      - v*
 concurrency:
-  group: docker-build-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
   build:
@@ -17,14 +19,22 @@ jobs:
       # https://stackoverflow.com/questions/59810838/how-to-get-the-short-sha-for-the-github-workflow
       - name: Set outputs
         id: commit
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Set branch name
         id: branch
-        run: echo "::set-output name=name::$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/})"
+        run: |
+          if [[ "$GITHUB_REF" == "refs/tags/"* ]]; then
+            echo "name=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" == "refs/heads/dev" ]]; then
+            echo "name=dev" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" == "refs/heads/release_"* ]]; then
+            echo "name=${GITHUB_REF#refs/heads/release_}-auto" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
       - name: Build container image
-        run: docker build . --build-arg GIT_COMMIT=$(git rev-parse HEAD) --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg IMAGE_TAG=${{ steps.branch.outputs.name }} -t galaxyproject/galaxy-min:${{ steps.branch.outputs.name }} -t quay.io/galaxyproject/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile
+        run: docker build . --build-arg GIT_COMMIT=$(git rev-parse HEAD) --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg IMAGE_TAG=${{ steps.branch.outputs.name }} -t galaxy/galaxy-min:${{ steps.branch.outputs.name }} -t quay.io/galaxyproject/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile
       - name: Create auto-expiring one for per-commit auto repository
-        run: echo "FROM galaxyproject/galaxy-min:${{ steps.branch.outputs.name }}" | docker build --label "quay.expires-after"="90d" -t "quay.io/galaxyproject/galaxy-k8s-auto:${{ steps.commit.outputs.sha_short }}" -
+        run: echo "FROM galaxy/galaxy-min:${{ steps.branch.outputs.name }}" | docker build --label "quay.expires-after"="90d" -t "quay.io/galaxyproject/galaxy-k8s-auto:${{ steps.commit.outputs.sha_short }}" -
       - name: Login to quay.io
         uses: actions-hub/docker/login@master
         env:
@@ -47,4 +57,4 @@ jobs:
       - name: Push to DockerHub with branch name
         uses: actions-hub/docker@master
         with:
-          args: push galaxyproject/galaxy-min:${{ steps.branch.outputs.name }}
+          args: push galaxy/galaxy-min:${{ steps.branch.outputs.name }}

.github/workflows/check_test_class_names.yaml

@@ -0,0 +1,28 @@
+name: Check test class names
+on:
+  pull_request:
+    paths:
+      - '.ci/check_test_class_names.sh'
+      - 'lib/galaxy_test/**'
+      - 'test/**'
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8']
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip dir
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt -r lib/galaxy/dependencies/dev-requirements.txt
+      - name: Run tests
+        run: .ci/check_test_class_names.sh