diff --git a/.github/workflows/ingestion-ci.yml b/.github/workflows/ingestion-ci.yml index 03df48d9..e9a84236 100644 --- a/.github/workflows/ingestion-ci.yml +++ b/.github/workflows/ingestion-ci.yml @@ -1,4 +1,4 @@ -name: Tutorials CI +name: Run tutorials (ingestion) on: push: @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] # , "macos-latest", "windows-latest" - python-version: ["3.9"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 @@ -26,7 +26,7 @@ jobs: - name: File tree if: env.debug == 'true' run: tree - + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-version: latest @@ -42,7 +42,7 @@ jobs: run: | pip install -e . conda list - + - name: Conda info if: env.debug == 'true' shell: bash -el {0} @@ -61,7 +61,7 @@ jobs: curl -L -o adt.h5ad https://figshare.com/ndownloader/files/41671551 curl -L -o atac.h5ad https://figshare.com/ndownloader/files/41671554 curl -L -o rna.h5ad https://figshare.com/ndownloader/files/41671557 - + # Note: we run the following to test that the commands works # However, the following task will replacing the file anyway - name: Preparing the configuration file @@ -69,22 +69,22 @@ jobs: run: | cd teaseq/ingest panpipes ingest config - + - name: Edit the submission file run: | cd teaseq/ingest curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/9a4fcbc3c0a4fdc8e6578d086a88730f/pipeline.yml - + - name: Preparing the submission file run: | cd teaseq/ingest curl -o sample_file_qc.txt https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/40ed291bf9a7ff73c4f396ec63cff2f7/sample_file_qc.txt - + - name: Preparing the QC gene lists run: | cd teaseq/ingest curl -o qc_genelist_1.0.csv https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/fd38f25644105ea357a26e78a59139bb/qc_genelist_1.0.csv - + - name: Replace template contents in configuration file run: | cd teaseq/ingest @@ -105,11 +105,11 @@ jobs: run: | cd teaseq/ingest panpipes ingest make full --local - + - name: File tree if: env.debug == 'true' run: tree teaseq - + - uses: actions/upload-artifact@v4 with: name: teaseq_unfilt.h5mu diff --git a/.github/workflows/integration-ci.yml b/.github/workflows/integration-ci.yml new file mode 100644 index 00000000..159034ad --- /dev/null +++ b/.github/workflows/integration-ci.yml @@ -0,0 +1,98 @@ +name: Run tutorials (integration) + +on: + push: + branches: + - main + pull_request: + branches: + - main + +env: + debug: 'true' + +jobs: + integration: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] # , "macos-latest", "windows-latest" + python-version: ["3.11"] + + steps: + - uses: actions/checkout@v4 + + - name: File tree + if: env.debug == 'true' + run: tree + + - uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + auto-activate-base: true + auto-update-conda: true + channels: conda-forge + channel-priority: strict + activate-environment: pipeline_env + environment-file: pipeline_env.yaml + + - name: Install Panpipes + shell: bash -el {0} + run: | + pip install -e . + conda list + + - name: Conda info + if: env.debug == 'true' + shell: bash -el {0} + run: conda info + + - name: Conda list + if: env.debug == 'true' + shell: pwsh + run: conda list + + # Note: all three files are renamed during the download to trim the "subsample_" prefix + - name: Preparing the data + run: | + mkdir -p teaseq/integration && cd teaseq/integration + curl -L -o teaseq.h5mu https://figshare.com/ndownloader/files/44471927 + + # Note: we run the following to test that the commands works + # However, the following task will replacing the file anyway + - name: Preparing the configuration file + shell: bash -el {0} + run: | + cd teaseq/integration + panpipes integration config + + - name: Edit the submission file + run: | + cd teaseq/integration + curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/f834fdb0a65979b0e4d11d6d75188a8e/pipeline.yml + + - name: Replace template contents in configuration file + run: | + cd teaseq/integration + sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_env+pipeline_env+g' pipeline.yml + + - name: File tree + if: env.debug == 'true' + run: tree teaseq + + - name: Review pipeline tasks + shell: bash -el {0} + run: | + cd teaseq/integration + panpipes integration show full --local + + - name: Run pipeline tasks + shell: bash -el {0} + run: | + cd teaseq/integration + panpipes integration make full --local + + - name: File tree + if: env.debug == 'true' + run: tree teaseq diff --git a/panpipes/python_scripts/batch_correct_scvi.py b/panpipes/python_scripts/batch_correct_scvi.py index b0c82167..6c6ac24b 100644 --- a/panpipes/python_scripts/batch_correct_scvi.py +++ b/panpipes/python_scripts/batch_correct_scvi.py @@ -159,7 +159,7 @@ vae.history["elbo_train"].plot() plt.savefig(os.path.join(args.figdir, "scvi_elbo_train.png")) -fig, axs = plt.subplots(nrows=3, ncols=4, figsize=(16,8)) +fig, axs = plt.subplots(nrows=4, ncols=4, figsize=(16,10)) axs = axs.ravel() for i, kk in enumerate(vae.history.keys()): vae.history[kk].plot(ax=axs[i]) diff --git a/pipeline_env.yaml b/pipeline_env.yaml index b0e11027..fcea02c4 100644 --- a/pipeline_env.yaml +++ b/pipeline_env.yaml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: - - python=3.9 + - python=3.11 - r-base=4.3.0 - r-clustree - r-ggforce diff --git a/pyproject.toml b/pyproject.toml index e539d40b..409ff4c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,43 +26,43 @@ classifiers = [ "Programming Language :: R" ] -requires-python = ">= 3.9" +requires-python = ">= 3.11" dependencies = [ "anndata>=0.8.0", - "apsw", - "bbknn", - "cgatcore", - "drmaa", + "apsw", + "bbknn", + "cgatcore", + "drmaa", "gevent", "harmonypy", - "jax<=0.4.23", - "jaxlib<=0.4.23", - "leidenalg", + "jax", + "jaxlib", + "leidenalg", "louvain", "matplotlib<=3.7.3", "mofapy2", "mudata>=0.2.1", "muon", - "numpy==1.22.4", + "numpy>=1.22.4", "openpyxl", - "packaging", - "pandas>=1.0", - "paramiko", + "packaging", + "pandas>=1.0", + "paramiko", "pep8", "pysam", "pynndescent", "pytest", "pyyaml", - "ruffus", - "scanorama", - "scanpy>=1.9.1", + "ruffus", + "scanorama", + "scanpy>=1.9.1", "scib", "seaborn<=0.12.2", - "scikit-misc", + "scikit-misc", "scirpy", - "scrublet", - "scvi-tools>=0.16.3", + "scrublet", + "scvi-tools>=1.1.1", "sqlalchemy"] [project.optional-dependencies]