github action for integration pipeline

.github/workflows/ingestion-ci.yml

@@ -1,4 +1,4 @@
-name: Tutorials CI
+name: Run tutorials (ingestion)
 
 on:
   push:
@@ -18,15 +18,15 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
-        python-version: ["3.9"]
+        python-version: ["3.11"]
 
     steps:
       - uses: actions/checkout@v4
 
       - name: File tree
         if: env.debug == 'true'
         run: tree
-      
+
       - uses: conda-incubator/setup-miniconda@v3
         with:
           miniforge-version: latest
@@ -42,7 +42,7 @@ jobs:
         run: |
           pip install -e .
           conda list
-          
+
       - name: Conda info
         if: env.debug == 'true'
         shell: bash -el {0}
@@ -61,30 +61,30 @@ jobs:
           curl -L -o adt.h5ad https://figshare.com/ndownloader/files/41671551
           curl -L -o atac.h5ad https://figshare.com/ndownloader/files/41671554
           curl -L -o rna.h5ad https://figshare.com/ndownloader/files/41671557
-      
+
       # Note: we run the following to test that the commands works
       # However, the following task will replacing the file anyway
       - name: Preparing the configuration file
         shell: bash -el {0}
         run: |
           cd teaseq/ingest
           panpipes ingest config
-      
+
       - name: Edit the submission file
         run: |
           cd teaseq/ingest
           curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/9a4fcbc3c0a4fdc8e6578d086a88730f/pipeline.yml
-      
+
       - name: Preparing the submission file
         run: |
           cd teaseq/ingest
           curl -o sample_file_qc.txt https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/40ed291bf9a7ff73c4f396ec63cff2f7/sample_file_qc.txt
-      
+
       - name: Preparing the QC gene lists
         run: |
           cd teaseq/ingest
           curl -o qc_genelist_1.0.csv https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/fd38f25644105ea357a26e78a59139bb/qc_genelist_1.0.csv
-        
+
       - name: Replace template contents in configuration file
         run: |
           cd teaseq/ingest
@@ -105,11 +105,11 @@ jobs:
         run: |
           cd teaseq/ingest
           panpipes ingest make full --local
-        
+
       - name: File tree
         if: env.debug == 'true'
         run: tree teaseq
-        
+
       - uses: actions/upload-artifact@v4
         with:
           name: teaseq_unfilt.h5mu

.github/workflows/integration-ci.yml

@@ -0,0 +1,98 @@
+name: Run tutorials (integration)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  integration:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install -e .
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+      # Note: all three files are renamed during the download to trim the "subsample_" prefix
+      - name: Preparing the data
+        run: |
+          mkdir -p teaseq/integration && cd  teaseq/integration
+          curl -L -o teaseq.h5mu https://figshare.com/ndownloader/files/44471927
+
+      # Note: we run the following to test that the commands works
+      # However, the following task will replacing the file anyway
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration config
+
+      - name: Edit the submission file
+        run: |
+          cd teaseq/integration
+          curl -o pipeline.yml https://panpipes-tutorials.readthedocs.io/en/latest/_downloads/f834fdb0a65979b0e4d11d6d75188a8e/pipeline.yml
+
+      - name: Replace template contents in configuration file
+        run: |
+          cd teaseq/integration
+          sed -i 's+/Users/fabiola.curion/Documents/devel/miniconda3/envs/pipeline_env+pipeline_env+g' pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree teaseq
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd teaseq/integration
+          panpipes integration make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree teaseq

panpipes/python_scripts/batch_correct_scvi.py

@@ -159,7 +159,7 @@
 vae.history["elbo_train"].plot()
 plt.savefig(os.path.join(args.figdir, "scvi_elbo_train.png"))
 
-fig, axs = plt.subplots(nrows=3, ncols=4, figsize=(16,8))
+fig, axs = plt.subplots(nrows=4, ncols=4, figsize=(16,10))
 axs = axs.ravel()
 for i, kk in enumerate(vae.history.keys()):
     vae.history[kk].plot(ax=axs[i])

pipeline_env.yaml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
+  - python=3.11
   - r-base=4.3.0
   - r-clustree
   - r-ggforce

pyproject.toml

@@ -26,43 +26,43 @@ classifiers = [
     "Programming Language :: R"
     ]
 
-requires-python = ">= 3.9"
+requires-python = ">= 3.11"
 
 dependencies = [
     "anndata>=0.8.0",
-    "apsw", 
-    "bbknn", 
-    "cgatcore", 
-    "drmaa", 
+    "apsw",
+    "bbknn",
+    "cgatcore",
+    "drmaa",
     "gevent",
     "harmonypy",
-    "jax<=0.4.23",
-    "jaxlib<=0.4.23",
-    "leidenalg", 
+    "jax",
+    "jaxlib",
+    "leidenalg",
     "louvain",
     "matplotlib<=3.7.3",
     "mofapy2",
     "mudata>=0.2.1",
     "muon",
-    "numpy==1.22.4", 
+    "numpy>=1.22.4",
     "openpyxl",
-    "packaging", 
-    "pandas>=1.0", 
-    "paramiko", 
+    "packaging",
+    "pandas>=1.0",
+    "paramiko",
     "pep8",
     "pysam",
     "pynndescent",
     "pytest",
     "pyyaml",
-    "ruffus", 
-    "scanorama", 
-    "scanpy>=1.9.1", 
+    "ruffus",
+    "scanorama",
+    "scanpy>=1.9.1",
     "scib",
     "seaborn<=0.12.2",
-    "scikit-misc", 
+    "scikit-misc",
     "scirpy",
-    "scrublet", 
-    "scvi-tools>=0.16.3",
+    "scrublet",
+    "scvi-tools>=1.1.1",
     "sqlalchemy"]
 
 [project.optional-dependencies]