Use flit & upgrade CI/CD workflows

.github/dependabot.yml

@@ -11,12 +11,12 @@ updates:
   open-pull-requests-limit: 15
   target-branch: ci/dependabot-updates
   labels:
-    - dependencies
+  - dependencies
 - package-ecosystem: github-actions
   directory: /
   schedule:
     interval: daily
     time: "05:00"
   target-branch: ci/dependabot-updates
   labels:
-    - CI
+  - CI

.github/mongo/load_data.py

@@ -8,8 +8,11 @@
 client = MongoClient("mongodb://localhost:27017")
 collection = client["aiida_optimade"]["structures"]
 
-with open(Path(__file__).parent.joinpath("test_structures_mongo.json")) as handle:
-    data = bson.json_util.loads(handle.read())
+data = bson.json_util.loads(
+    Path(__file__)
+    .parent.joinpath("test_structures_mongo.json")
+    .read_text(encoding="utf8")
+)
 
 try:
     print(f"Inserting {len(data)} structures into {collection.full_name}")

.github/workflows/cd_release.yml

@@ -13,79 +13,26 @@ env:
 
 jobs:
   publish:
-    name: Update CHANGELOG
+    name: External
+    uses: SINTEF/ci-cd/.github/workflows/cd_release.yml@v2.0.0
     if: github.repository == 'aiidateam/aiida-optimade' && startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-
-    - name: Install Python dependencies
-      run: |
-        python -m pip install -U pip
-        pip install -U setuptools wheel
-        pip install -U -e .[dev]
-
-    - name: Update changelog
-      uses: CharMixer/auto-changelog-action@v1
-      with:
-        token: ${{ secrets.RELEASE_PAT }}
-        release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }}
-
-    - name: Update version - Commit changes and update tag
-      run: .github/utils/update_version.sh
-
-    - name: Update '${{ env.PUBLISH_UPDATE_BRANCH }}'
-      uses: CasperWA/push-protected@v2
-      with:
-        token: ${{ secrets.RELEASE_PAT }}
-        branch: ${{ env.PUBLISH_UPDATE_BRANCH }}
-        sleep: 15
-        force: true
-        tags: true
-
-    - name: Get tagged versions
-      run: echo "PREVIOUS_VERSION=$(git tag -l --sort -version:refname | sed -n 2p)" >> $GITHUB_ENV
-
-    - name: Create release-specific changelog
-      uses: CharMixer/auto-changelog-action@v1
-      with:
-        token: ${{ secrets.RELEASE_PAT }}
-        release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }}
-        since_tag: ${{ env.PREVIOUS_VERSION }}
-        output: release_changelog.md
-
-    - name: Append changelog to release body
-      run: |
-        gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} --jq '.body' > release_body.md
-        cat release_changelog.md >> release_body.md
-        gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} -X PATCH -F body='@release_body.md'
-      env:
-        GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }}
-
-    - name: Build source distribution
-      run: python ./setup.py sdist
-
-    - name: Publish package to TestPyPI
-      uses: pypa/gh-action-pypi-publish@master
-      with:
-        user: __token__
-        password: ${{ secrets.test_pypi_password }}
-        repository_url: https://test.pypi.org/legacy/
-
-    - name: Publish package to PyPI
-      uses: pypa/gh-action-pypi-publish@master
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
+    with:
+      git_username: ${{ env.GIT_USER_NAME }}
+      git_email: ${{ env.GIT_USER_EMAIL }}
+      release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }}
+      package_dirs: aiida_optimade
+      python_version_build: '3.8'
+      version_update_changes_separator: ;
+      version_update_changes: |
+        {package_dir}/__init__.py;__version__ = .+;__version__ = \"{version}\"
+        {package_dir}/config.json;\"version\": ([^,]+),;\"version\": \"{version}\",
+        tests/static/test_config.json;\"version\": ([^,]+),;\"version\": \"{version}\",
+        tests/static/test_mongo_config.json;\"version\": ([^,]+),;\"version\": \"{version}\",
+        .github/mongo/ci_config.json;\"version\": ([^,]+),;\"version\": \"{version}\",
+      build_cmd: pip install -U flit && flit build
+    secrets:
+      PyPI_token: ${{ secrets.pypi_password }}
+      PAT: ${{ secrets.RELEASE_PAT }}
 
   update-stable:
     name: Update `stable` branch
@@ -109,7 +56,7 @@ jobs:
       run: git merge --ff-only origin/${{ env.PUBLISH_UPDATE_BRANCH }}
 
     - name: Update `stable` branch
-      uses: ad-m/github-push-action@master
+      uses: CasperWA/push-protected@v2
       with:
-        github_token: ${{ secrets.RELEASE_PAT }}
+        token: ${{ secrets.RELEASE_PAT }}
         branch: stable

.github/workflows/ci_automerge_dependabot.yml

@@ -6,17 +6,16 @@ on:
     branches:
     - ci/dependabot-updates
 
-jobs:
+env:
+  GIT_USER_NAME: The AiiDA Team
+  GIT_USER_EMAIL: developers@aiida.net
 
+jobs:
   update-dependabot-branch:
     name: Update permanent dependabot branch
     if: github.repository_owner == 'aiidateam' && startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]'
     runs-on: ubuntu-latest
 
-    env:
-      GIT_USER_NAME: The AiiDA Team
-      GIT_USER_EMAIL: developers@aiida.net
-
     steps:
     - name: Setup git config
       run: |
@@ -33,39 +32,33 @@ jobs:
     - name: Setup Python 3.8
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: '3.8'
 
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -U setuptools wheel
-        pip install requests
-
-        while IFS="" read -r line || [ -n "${line}" ]; do
-          if [[ "${line}" =~ ^invoke.*$ ]]; then
-            invoke="${line}"
-          fi
-        done < requirements_dev.txt
-
-        pip install ${invoke}
+        pip install -U setuptools wheel flit
+        pip install .[dev]
 
     - name: Run tasks for `optimade` and `aiida-core` dependencies
       run: |
+        pat_aiida='^.*"aiida-core ~=(.*)".*$'
+        pat_optimade='^.*"optimade\[.*\] ~=(.*)".*$'
         while IFS="" read -r line || [ -n "${line}" ]; do
-          if [[ "${line}" =~ ^aiida-core~=(.*)$ ]]; then
+          if [[ "${line}" =~ $pat_aiida ]]; then
             aiida_core="${BASH_REMATCH[1]}"
-          elif [[ "${line}" =~ ^optimade\[mongo\]~=(.*)$ ]]; then
+          elif [[ "${line}" =~ $pat_optimade ]]; then
             optimade_mongo="${BASH_REMATCH[1]}"
           fi
-        done < requirements.txt
+        done < pyproject.toml
 
         invoke aiida-req --ver="${aiida_core}"
         invoke optimade-req --ver="${optimade_mongo}"
 
-        git add requirements.txt .ci/aiida-version.json Dockerfile profiles/docker-compose*
-        git add requirements.txt README.md Dockerfile profiles/docker-compose*
+        git add pyproject.toml .ci/aiida-version.json Dockerfile profiles/docker-compose*
+        git add pyproject.toml README.md Dockerfile profiles/docker-compose*
 
-        if [ -n "$(git status --porcelain requirements.txt .ci/aiida-version.json Dockerfile README.md)" ]; then
+        if [ -n "$(git status --porcelain pyproject.toml .ci/aiida-version.json Dockerfile README.md)" ]; then
           # Only commit if there's something to commit (git will return non-zero otherwise)
           echo "Committing update to hard-coded versions (AiiDA/OPTIMADE) !"
           git commit -m "Update AiiDA/OPTIMADE"
@@ -77,9 +70,9 @@ jobs:
 
     - name: Update Dependabot branch
       if: env.PUSH_BACK_TO_BRANCH == 'true'
-      uses: ad-m/github-push-action@master
+      uses: CasperWA/push-protected@v2
       with:
-        github_token: ${{ secrets.RELEASE_PAT }}
+        token: ${{ secrets.RELEASE_PAT }}
         branch: ${{ github.event.pull_request.head.ref }}
 
     - name: Activate auto-merge

.github/workflows/ci_dependabot.yml

@@ -3,100 +3,26 @@ name: CI - Single Dependabot PR
 
 on:
   schedule:
-    # At 8:30 every Wednesday (6:30 UTC)
-    # Dependabot runs once a week (every Monday) (pip)
-    # and every day (GH Actions) at 7:00 (5:00 UTC)
+  # At 8:30 every Wednesday (6:30 UTC)
+  # Dependabot runs once a week (every Monday) (pip)
+  # and every day (GH Actions) at 7:00 (5:00 UTC)
   - cron: 30 6 * * 3
+  workflow_dispatch:
 
 jobs:
   create-collected-pr:
-    name: Single dependabot PR
+    name: External
+    uses: SINTEF/ci-cd/.github/workflows/ci_update_dependencies.yml@v2.0.0
     if: github.repository_owner == 'aiidateam'
-    runs-on: ubuntu-latest
-    env:
-      DEPENDABOT_BRANCH: ci/dependabot-updates
-      GIT_USER_NAME: The AiiDA Team
-      GIT_USER_EMAIL: developers@aiida.net
-      DEFAULT_REPO_BRANCH: develop
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-        ref: ${{ env.DEFAULT_REPO_BRANCH }}
-
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-
-    - name: Install `pre-commit`
-      run: |
-        python -m pip install -U pip
-        pip install -U setuptools wheel
-
-        while IFS="" read -r line || [ -n "${line}" ]; do
-          if [[ "${line}" =~ ^pre-commit.*$ ]]; then
-            pre_commit="${line}"
-          fi
-        done < requirements_dev.txt
-
-        pip install ${pre_commit}
-
-    - name: Set up git user info
-      run: |
-        git config --global user.name "${{ env.GIT_USER_NAME }}"
-        git config --global user.email "${{ env.GIT_USER_EMAIL }}"
-
-    - name: Reset to '${{ env.DEPENDABOT_BRANCH }}'
-      run: |
-        git fetch origin ${{ env.DEPENDABOT_BRANCH }}:${{ env.DEPENDABOT_BRANCH }}
-        git reset --hard ${{ env.DEPENDABOT_BRANCH }}
-
-    - name: Auto-update `pre-commit` hooks
-      run: |
-        pre-commit autoupdate
-
-        if [ -n "$(git status --porcelain .pre-commit-config.yaml)" ]; then
-          # Set environment variable notifying next steps that the hooks changed
-          echo "Pre-commit hooks have been updated !"
-          echo "UPDATED_PRE_COMMIT_HOOKS=true" >> $GITHUB_ENV
-        else
-          echo "No pre-commit hooks have been updated."
-          echo "UPDATED_PRE_COMMIT_HOOKS=false" >> $GITHUB_ENV
-        fi
-
-    - name: Possibly run `pre-commit` with updated hooks
-      if: env.UPDATED_PRE_COMMIT_HOOKS == 'true'
-      continue-on-error: true         # Still create the PR if this step fails
-      run: SKIP=codecov-validator pre-commit run --all-files
-
-    - name: Possibly commit changes and updates
-      if: env.UPDATED_PRE_COMMIT_HOOKS == 'true'
-      run: |
-        git add .
-        git commit -m "Update \`pre-commit\` hooks"
-
-    - name: Fetch PR body
-      id: pr_body
-      uses: chuhlomin/render-template@v1.6
-      with:
-        template: .github/utils/single_dependency_pr_body.txt
-
-    - name: Create PR
-      id: cpr
-      uses: peter-evans/create-pull-request@v4
-      with:
-        token: ${{ secrets.RELEASE_PAT }}
-        commit-message: New @dependabot-fueled updates
-        committer: ${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>
-        author: ${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>
-        branch: ci/update-dependencies
-        delete-branch: true
-        title: Update dependencies
-        body: ${{ steps.pr_body.outputs.result }}
-        labels: CI,dependencies
-
-    - name: Information
-      run: 'echo "${{ steps.cpr.outputs.pull-request-operation }} PR #${{ steps.cpr.outputs.pull-request-number }}: ${{ steps.cpr.outputs.pull-request-url
-        }}"'
+    with:
+      git_username: The AiiDA Team
+      git_email: developers@aiida.net
+      permanent_dependencies_branch: ci/dependabot-updates
+      default_repo_branch: develop
+      pr_label: CI,dependencies
+      update_pre-commit: true
+      python_version: '3.8'
+      install_extras: '[dev]'
+      skip_pre-commit_hooks: codecov-validator
+    secrets:
+      PAT: ${{ secrets.RELEASE_PAT }}