Add extraction of Connectivity Search PathCount table (#4)

* initial problem solving

* linting

* updates for linting and functionality

* Update get_table.ipynb

* Update README.md

.gitignore

@@ -163,3 +163,7 @@ cython_debug/
 
 # data ignores
 src/bioprocess_metapath_to_gene_pval_and_dwpc/data/results
+*.sql.gz
+*.parquet
+*.duckdb
+copy_data*.sql

.pre-commit-config.yaml

@@ -3,43 +3,48 @@
 default_language_version:
   python: python3.11
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
+-   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
-      - id: check-added-large-files
-      - id: detect-private-key
-  - repo: https://github.com/tox-dev/pyproject-fmt
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: detect-private-key
+-   repo: https://github.com/tox-dev/pyproject-fmt
     rev: "v2.5.0"
     hooks:
-      - id: pyproject-fmt
-  - repo: https://github.com/codespell-project/codespell
+    -   id: pyproject-fmt
+-   repo: https://github.com/codespell-project/codespell
     rev: v2.3.0
     hooks:
-      - id: codespell
+    -   id: codespell
         exclude: |
           (?x)^(
               .*\.lock |
               .*\.csv
           )$
-  - repo: https://github.com/executablebooks/mdformat
+-   repo: https://github.com/executablebooks/mdformat
     rev: 0.7.18
     hooks:
-      - id: mdformat
+    -   id: mdformat
         additional_dependencies:
-          - mdformat-gfm
-  - repo: https://github.com/adrienverge/yamllint
+        -   mdformat-gfm
+-   repo: https://github.com/adrienverge/yamllint
     rev: v1.35.1
     hooks:
-      - id: yamllint
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.7.2"
+    -   id: yamllint
+        exclude: pre-commit-config.yaml
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.7.4"
     hooks:
-      - id: ruff-format
-      - id: ruff
-  - repo: https://github.com/rhysd/actionlint
+    -   id: ruff-format
+    -   id: ruff
+-   repo: https://github.com/rhysd/actionlint
     rev: v1.7.4
     hooks:
-      - id: actionlint
+    -   id: actionlint
+-   repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update
+    rev: v0.6.0post1
+    hooks:
+    -   id: pre-commit-update

README.md

@@ -8,3 +8,10 @@ Various data analysis performed using [Hetionet](https://het.io/), a [hetnet](ht
 1. Install package locally (e.g. `uv pip install -e ".[dev]"`).
 1. Run tests (e.g. `uv run poe test`, through [poethepoet](https://poethepoet.natn.io/index.html) task).
 1. Run various tasks (e.g. `uv run poe run_bioproc_gene_metapath_test`)
+
+## Tasks
+
+Poe the poet tasks may be run to help generate results without needing to run individual files or perform additional discovery within this project.
+You can show all available tasks with `uv run poe`.
+
+- Create Connectivity Search PathCount table: `uv run poe run_pathcount_extract`

pyproject.toml

@@ -16,6 +16,7 @@ classifiers = [
 dynamic = [ "version" ]
 dependencies = [
   "black>=24.10",
+  "duckdb>=1.1.3",
   "hetmatpy>=0.1",
   "ipywidgets>=8.1.5",
   "isort>=5.13.2",
@@ -85,7 +86,12 @@ test.shell = """
 uv run pre-commit run -a
 uv run pytest
 """
-#
+# run the gene metapath extraction
 run_bioproc_gene_metapath_test.shell = """
 uv run python src/bioprocess_metapath_to_gene_pval_and_dwpc/gather_subset_data_metapath_BPpGdAdG.py
 """
+# run path count table extraction
+run_pathcount_extract.shell = """
+cd src/connectivity_search_PathCount_table
+uv run python get_table.py
+"""

src/connectivity_search_PathCount_table/create_table.public.dj_hetmech_app_pathcount.sql

@@ -0,0 +1,11 @@
+CREATE TABLE public.dj_hetmech_app_pathcount (
+    id integer NOT NULL,
+    path_count integer NOT NULL,
+    dwpc double precision NOT NULL,
+    p_value double precision,
+    metapath_id character varying(20) NOT NULL,
+    source_id integer NOT NULL,
+    target_id integer NOT NULL,
+    dgp_id integer NOT NULL,
+    CONSTRAINT dj_hetmech_app_pathcount_path_count_check CHECK ((path_count >= 0))
+);