From c7b7c71c11a53c6394f01c70ce17f3f725300603 Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 5 Jun 2024 14:12:39 -0400 Subject: [PATCH 1/7] Update CI workflows --- .github/workflows/codecov.yml | 35 --------- .github/workflows/linter.yml | 26 ------- .github/workflows/pr.yml | 14 ++++ .github/workflows/python-publish.yml | 39 ---------- .github/workflows/release.yml | 104 +++++++++++++++++++++++++++ .github/workflows/test.yml | 90 +++++++++++++++++++++++ .github/workflows/tests.yml | 45 ------------ 7 files changed, 208 insertions(+), 145 deletions(-) delete mode 100755 .github/workflows/codecov.yml delete mode 100755 .github/workflows/linter.yml create mode 100644 .github/workflows/pr.yml delete mode 100644 .github/workflows/python-publish.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100755 index b9b17cc..0000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: CodeCov - -on: - pull_request: - branches: [master, main] - push: - branches: [master, main] - schedule: - - cron: "0 13 * * 1" - -jobs: - run: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Python 3.10 - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - name: setup-conda - uses: s-weigand/setup-conda@v1.1.0 - with: - conda-channels: '' - - name: Install dependencies - run: | - conda env update --file primertrim_env.yml - python -m pip install --upgrade pip - python -m pip install pytest - python -m pip install pytest-cov - python -m pip install -e . - - name: Run tests and collect coverage - run: pytest --cov tests/ - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 \ No newline at end of file diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100755 index 9d72b1e..0000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Super-Linter - -on: - pull_request: - branches: [master, main, dev] - push: - branches: [master, main] - -jobs: - super-linter: - name: Lint Codebase - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Run Super-Linter - uses: github/super-linter@v4 - env: - VALIDATE_ALL_CODEBASE: true - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - VALIDATE_PYTHON_BLACK: true - - FILTER_REGEX_INCLUDE: primertrim/.*|tests/.*|setup.py \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..36fed4b --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,14 @@ +name: Tests + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + + workflow_dispatch: + +jobs: + run-tests: + uses: ./.github/workflows/test.yml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index ec70354..0000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e4d934f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,104 @@ +name: Release Package + +on: + release: + types: [published] + + workflow_dispatch: + +jobs: + check-version: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Library + run: | + python -m pip install . + python -m pip install setuptools + + - id: get_version + uses: battila7/get-version-action@v2 + + - name: Check version + run: | + LIBRARY_VERSION=$(unassign --version) + GITHUB_VERSION=${{ steps.get_version.outputs.version-without-v }} + if [[ $LIBRARY_VERSION == $GITHUB_VERSION ]]; then + echo "Versions match, continuing..." + else + echo "Versions don't match, exiting..." + echo "Library version: $LIBRARY_VERSION" + echo "GitHub version: $GITHUB_VERSION" + exit 1 + fi + + run-tests: + uses: ./.github/workflows/test.yml + secrets: inherit + + build-and-publish-to-pypi: + name: Publish to PyPI + runs-on: ubuntu-latest + needs: [run-tests, check-version] + + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build package + run: python -m build + + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + + build-and-push-to-dockerhub: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + needs: [run-tests, check-version] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ctbushman/unassigner + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..30602ed --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,90 @@ +name: Tests + +on: + workflow_call: + + workflow_dispatch: + +jobs: + tests-with-coverage: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: '3.x' + + - name: setup-conda + uses: s-weigand/setup-conda@v1.1.0 + + - name: Install dependencies + run: | + conda install -c bioconda vsearch + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov + python -m pip install . + + - name: Run tests and collect coverage + run: pytest --cov tests + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + + tests: + name: Run Tests + strategy: + fail-fast: false + matrix: + python-version: ['3.12'] + os: [ubuntu-latest, macos-latest] + include: + - python-version: '3.11' + os: ubuntu-latest + - python-version: '3.10' + os: ubuntu-latest + - python-version: '3.9' + os: ubuntu-latest + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: setup-conda + uses: s-weigand/setup-conda@v1.1.0 + + - name: Install dependencies + run: | + conda install -c bioconda vsearch + python -m pip install --upgrade pip + python -m pip install pytest + python -m pip install . + + - name: Run tests + run: pytest -s -vvvv -l --tb=long tests + + lint: + name: Lint Code Base + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Dependencies + run: pip install black + + - name: Lint Code Base + run: | + black --check . \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 07cd464..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,45 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Tests - -on: - pull_request: - branches: [master, main, dev] - push: - branches: [master, main] - schedule: - - cron: "0 13 * * 1" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: setup-conda - uses: s-weigand/setup-conda@v1.1.0 - with: - conda-channels: '' - - name: Install dependencies - run: | - conda env update --file primertrim_env.yml - python -m pip install --upgrade pip - python -m pip install pytest - python -m pip install -e . - - name: Unit tests - run: | - pytest -vvl tests/ - - name: Other tests - if: matrix.python-version == '3.10' - run: | - echo "Other tests" \ No newline at end of file From 3c318d642f3e7358acb069a92aeecc8746f76897 Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 5 Jun 2024 14:19:19 -0400 Subject: [PATCH 2/7] Don't include mac-latest runner --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 30602ed..74aa779 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: fail-fast: false matrix: python-version: ['3.12'] - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] # setup-conda action currently doesn't work with latest macOS runners include: - python-version: '3.11' os: ubuntu-latest From 761d49e7722e7f52eaf082fb041e5bd164a8fa15 Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 5 Jun 2024 14:28:31 -0400 Subject: [PATCH 3/7] Add Dockerfile --- Dockerfile | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c032c7f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM condaforge/mambaforge:latest + +# Setup +WORKDIR /home + +COPY . . + +# Install environment +RUN mamba create --name primertrim -c conda-forge -c bioconda vsearch + +ENV PATH="/opt/conda/envs/primertrim/bin/:${PATH}" + +# "Activate" the environment +SHELL ["conda", "run", "--no-capture-output", "-n", "primertrim", "/bin/bash", "-c"] + +RUN pip install /home/ + +RUN echo "Python: $(python --version), Conda: $(conda --version), Vsearch: $(vsearch -v)" > installed_packages.txt + +# Run +CMD "bash" \ No newline at end of file From 7b84b1814376bd1fda2ca50b983f4981d6fd302e Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 5 Jun 2024 14:28:38 -0400 Subject: [PATCH 4/7] Update badges in README --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2deba16..5e0dc12 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # Primer trim -[![Tests](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/tests.yml/badge.svg)](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/tests.yml) -[![CodeCov](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/codecov.yml/badge.svg)](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/codecov.yml) -[![Super-Linter](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/linter.yml/badge.svg)](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/linter.yml) +[![Tests](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/pr.yml/badge.svg)](https://github.com/PennChopMicrobiomeProgram/primertrim/actions/workflows/pr.yml) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/d6e2cf6c74bf4888b9f09cd4506cccb7)](https://app.codacy.com/gh/PennChopMicrobiomeProgram/primertrim/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) +[![codecov](https://codecov.io/gh/PennChopMicrobiomeProgram/primertrim/graph/badge.svg?token=CEH4E6QJEF)](https://codecov.io/gh/PennChopMicrobiomeProgram/primertrim) +[![DockerHub](https://img.shields.io/docker/pulls/ctbushman/primertrim)](https://hub.docker.com/repository/docker/ctbushman/primertrim/) Detect short primer sequences in FASTQ reads and trim the reads accordingly. From f1bb5c2c8c6173738d4b9322d02c226fc378c819 Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 5 Jun 2024 14:31:20 -0400 Subject: [PATCH 5/7] Fix static analysis issues --- primertrim/align.py | 1 - primertrim/matcher.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/primertrim/align.py b/primertrim/align.py index 0164ecd..dd52da4 100644 --- a/primertrim/align.py +++ b/primertrim/align.py @@ -1,4 +1,3 @@ -import os.path import subprocess import tempfile diff --git a/primertrim/matcher.py b/primertrim/matcher.py index c7db111..2a6f978 100644 --- a/primertrim/matcher.py +++ b/primertrim/matcher.py @@ -34,7 +34,7 @@ def find_in_seqs(self, seqs): def find_match(self, seq): """Returns a PrimerMatch object or None""" - raise NotImplemented() + raise NotImplementedError class CompleteMatcher(Matcher): From d1e42dbaa72e768367a5c51a940a959839654f66 Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 5 Jun 2024 14:39:14 -0400 Subject: [PATCH 6/7] Fix more static analysis errors --- primertrim/matcher.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/primertrim/matcher.py b/primertrim/matcher.py index 2a6f978..fc09fba 100644 --- a/primertrim/matcher.py +++ b/primertrim/matcher.py @@ -58,8 +58,10 @@ def _mismatched_queries(self, n_mismatch): return list(self._iter_mismatched_queries(n_mismatch)) def _iter_mismatched_queries(self, n_mismatch): - # This algorithm is terrible unless the number of mismatches is very small - assert n_mismatch in [0, 1, 2, 3] + if n_mismatch not in [0, 1, 2, 3]: + raise TooManyMismatchesException( + "This algorithm is terrible unless the number of mismatches is very small (0, 1, 2, or 3)" + ) for query in self.queryset: idx_sets = itertools.combinations(range(len(query)), n_mismatch) for idx_set in idx_sets: @@ -114,8 +116,10 @@ def find_match(self, seq): class AlignmentMatcher(Matcher): def __init__(self, queryset, alignment_dir, align_id, cores=1): self.queryset = queryset - assert os.path.exists(alignment_dir) - assert os.path.isdir(alignment_dir) + if not os.path.exists(alignment_dir): + raise FileNotFoundError("Alignment directory does not exist") + if not os.path.isdir(alignment_dir): + raise NotADirectoryError("Alignment directory is not a directory") self.alignment_dir = alignment_dir self.align_id = align_id self.cores = cores @@ -149,8 +153,13 @@ def find_in_seqs(self, seqs): if hit["qstrand"] == "-": start_idx = hit["qlen"] - hit["qend"] end_idx = hit["qlen"] - hit["qstart"] + 1 - assert start_idx < end_idx + if start_idx >= end_idx: + raise ValueError("Invalid start and end indices ({} is greater than or equal to {})".format(start_idx, end_idx)) seq = seqs[seq_id] primerseq = seq[start_idx:end_idx] matchobj = PrimerMatch("Alignment", start_idx, mismatches, primerseq) yield seq_id, matchobj + + +class TooManyMismatchesException(Exception): + pass \ No newline at end of file From b6bf0fd2c4bcf6899c2bd8951dc723274e2fbf07 Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 5 Jun 2024 14:40:27 -0400 Subject: [PATCH 7/7] Reformat --- primertrim/matcher.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/primertrim/matcher.py b/primertrim/matcher.py index fc09fba..141729b 100644 --- a/primertrim/matcher.py +++ b/primertrim/matcher.py @@ -154,7 +154,11 @@ def find_in_seqs(self, seqs): start_idx = hit["qlen"] - hit["qend"] end_idx = hit["qlen"] - hit["qstart"] + 1 if start_idx >= end_idx: - raise ValueError("Invalid start and end indices ({} is greater than or equal to {})".format(start_idx, end_idx)) + raise ValueError( + "Invalid start and end indices ({} is greater than or equal to {})".format( + start_idx, end_idx + ) + ) seq = seqs[seq_id] primerseq = seq[start_idx:end_idx] matchobj = PrimerMatch("Alignment", start_idx, mismatches, primerseq) @@ -162,4 +166,4 @@ def find_in_seqs(self, seqs): class TooManyMismatchesException(Exception): - pass \ No newline at end of file + pass