Merge pull request #41 from broadinstitute/updates

Analysis updates
broadinstitute · Apr 19, 2024 · b0922ca · b0922ca
2 parents f59bfd6 + 5ad15c9
commit b0922ca
Show file tree

Hide file tree

Showing 60 changed files with 3,100 additions and 460 deletions.
diff --git a/5.reanalysis/4.analysis.ipynb b/5.reanalysis/4.analysis.ipynb
@@ -77,6 +77,65 @@
     "\n",
     "run_analysis(data_level, feature_cols_pattern=\"Cells_|Cytoplasm_|Nuclei_\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Analyzing category: progen\n",
+      "Analyzing category: neuron\n",
+      "Analyzing category: stem\n",
+      "Analyzing category: astro\n",
+      "Analysis complete. Summary results saved to output/analysis_results/var_mad_int_featselect/control_vs_deletion/summary_results.parquet\n",
+      "WARNING: The following values in Metadata_cell_type were not mapped and will be excluded from the analysis: <StringArray>\n",
+      "['progen', 'astro']\n",
+      "Length: 2, dtype: string\n",
+      "Analyzing category: control\n",
+      "Analyzing category: deletion\n",
+      "Analysis complete. Summary results saved to output/analysis_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_neuron/summary_results.parquet\n",
+      "WARNING: The following values in Metadata_cell_type were not mapped and will be excluded from the analysis: <StringArray>\n",
+      "['neuron', 'astro']\n",
+      "Length: 2, dtype: string\n",
+      "Analyzing category: control\n",
+      "Analyzing category: deletion\n",
+      "Analysis complete. Summary results saved to output/analysis_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_progen/summary_results.parquet\n",
+      "WARNING: The following values in Metadata_cell_type were not mapped and will be excluded from the analysis: <StringArray>\n",
+      "['progen', 'neuron']\n",
+      "Length: 2, dtype: string\n",
+      "Analyzing category: control\n",
+      "Analyzing category: deletion\n",
+      "Analysis complete. Summary results saved to output/analysis_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_astro/summary_results.parquet\n",
+      "WARNING: The following values in Metadata_cell_type were not mapped and will be excluded from the analysis: <StringArray>\n",
+      "['stem', 'astro']\n",
+      "Length: 2, dtype: string\n",
+      "Analyzing category: control\n",
+      "Analyzing category: deletion\n",
+      "Analysis complete. Summary results saved to output/analysis_results/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_progen/summary_results.parquet\n",
+      "WARNING: The following values in Metadata_cell_type were not mapped and will be excluded from the analysis: <StringArray>\n",
+      "['progen', 'stem']\n",
+      "Length: 2, dtype: string\n",
+      "Analyzing category: control\n",
+      "Analyzing category: deletion\n",
+      "Analysis complete. Summary results saved to output/analysis_results/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_astro/summary_results.parquet\n",
+      "WARNING: The following values in Metadata_cell_type were not mapped and will be excluded from the analysis: <StringArray>\n",
+      "['neuron', 'stem']\n",
+      "Length: 2, dtype: string\n",
+      "Analyzing category: control\n",
+      "Analyzing category: deletion\n",
+      "Analysis complete. Summary results saved to output/analysis_results/var_mad_int_featselect/cell_type_a_vs_b/progen_vs_astro/summary_results.parquet\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_level = \"var_mad_int_featselect\"\n",
+    "\n",
+    "run_analysis(data_level, feature_cols_pattern=\"Cells_|Cytoplasm_|Nuclei_\")"
+   ]
   }
  ],
  "metadata": {

diff --git a/5.reanalysis/5.inspect-analysis.ipynb b/5.reanalysis/5.inspect-analysis.ipynb
diff --git a/5.reanalysis/7.replicability_map.ipynb b/5.reanalysis/7.replicability_map.ipynb
diff --git a/5.reanalysis/copy_reprocessed_profiles.py b/5.reanalysis/copy_reprocessed_profiles.py
@@ -0,0 +1,49 @@
+import shutil
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+# Define file mappings
+files = [
+    (
+        Path("~/Documents/GitHub/jump-profiling-recipe/outputs/stem_public/profiles_var_mad_int_featselect.parquet").expanduser(),
+        Path("output/processed/var_mad_int_featselect/combined.parquet").expanduser(),
+    ),
+    (
+        Path("~/Documents/GitHub/jump-profiling-recipe/outputs/stem_public/profiles_var_mad_int.parquet").expanduser(),
+        Path("output/processed/var_mad_int/combined.parquet").expanduser(),
+    ),
+    (
+        Path("~/Documents/GitHub/jump-profiling-recipe/outputs/stem/metrics/profiles_var_mad_int_featselect_ap_nonrep.parquet").expanduser(),
+        Path("output/analysis_results/var_mad_int_featselect/metrics/ap_nonrep.parquet").expanduser(),
+    ),
+    (
+        Path("~/Documents/GitHub/jump-profiling-recipe/outputs/stem/metrics/profiles_var_mad_int_featselect_map_nonrep.parquet").expanduser(),
+        Path("output/analysis_results/var_mad_int_featselect/metrics/map_nonrep.parquet").expanduser(),
+    )
+]
+
+# Copy and rename files
+for source, target in files:
+    if not source.exists():
+        logging.warning(f"Source file {source} does not exist. Skipping.")
+        continue
+
+    if target.exists():
+        logging.warning(f"Target file {target} already exists. Skipping.")
+        continue
+
+    target_dir = target.parent
+    if not target_dir.exists():
+        try:
+            target_dir.mkdir(parents=True, exist_ok=True)
+        except OSError as e:
+            logging.error(f"Error creating directory {target_dir}: {e}")
+            continue
+
+    try:
+        shutil.copy(source, target)
+        logging.info(f"Copied {source} to {target}")
+    except IOError as e:
+        logging.error(f"Error copying file {source} to {target}: {e}")
diff --git a/...ect/cell_type_a_vs_b/neuron_vs_astro/classification_vs_significant_features.png b/...ect/cell_type_a_vs_b/neuron_vs_astro/classification_vs_significant_features.png
diff --git a/...nt_featselect/cell_type_a_vs_b/neuron_vs_astro/significant_features_control.png b/...nt_featselect/cell_type_a_vs_b/neuron_vs_astro/significant_features_control.png
diff --git a/...t_featselect/cell_type_a_vs_b/neuron_vs_astro/significant_features_deletion.png b/...t_featselect/cell_type_a_vs_b/neuron_vs_astro/significant_features_deletion.png
diff --git a/...s_results/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_astro/summary_results.parquet b/...s_results/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_astro/summary_results.parquet
diff --git a/...ults/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_astro/test_results_control.parquet b/...ults/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_astro/test_results_control.parquet
diff --git a/...lts/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_astro/test_results_deletion.parquet b/...lts/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_astro/test_results_deletion.parquet
diff --git a/...ct/cell_type_a_vs_b/neuron_vs_progen/classification_vs_significant_features.png b/...ct/cell_type_a_vs_b/neuron_vs_progen/classification_vs_significant_features.png
diff --git a/...t_featselect/cell_type_a_vs_b/neuron_vs_progen/significant_features_control.png b/...t_featselect/cell_type_a_vs_b/neuron_vs_progen/significant_features_control.png
diff --git a/..._featselect/cell_type_a_vs_b/neuron_vs_progen/significant_features_deletion.png b/..._featselect/cell_type_a_vs_b/neuron_vs_progen/significant_features_deletion.png
diff --git a/..._results/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_progen/summary_results.parquet b/..._results/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_progen/summary_results.parquet
diff --git a/...lts/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_progen/test_results_control.parquet b/...lts/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_progen/test_results_control.parquet
diff --git a/...ts/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_progen/test_results_deletion.parquet b/...ts/var_mad_int_featselect/cell_type_a_vs_b/neuron_vs_progen/test_results_deletion.parquet
diff --git a/...ect/cell_type_a_vs_b/progen_vs_astro/classification_vs_significant_features.png b/...ect/cell_type_a_vs_b/progen_vs_astro/classification_vs_significant_features.png
diff --git a/...nt_featselect/cell_type_a_vs_b/progen_vs_astro/significant_features_control.png b/...nt_featselect/cell_type_a_vs_b/progen_vs_astro/significant_features_control.png
diff --git a/...t_featselect/cell_type_a_vs_b/progen_vs_astro/significant_features_deletion.png b/...t_featselect/cell_type_a_vs_b/progen_vs_astro/significant_features_deletion.png
diff --git a/...s_results/var_mad_int_featselect/cell_type_a_vs_b/progen_vs_astro/summary_results.parquet b/...s_results/var_mad_int_featselect/cell_type_a_vs_b/progen_vs_astro/summary_results.parquet
diff --git a/...ults/var_mad_int_featselect/cell_type_a_vs_b/progen_vs_astro/test_results_control.parquet b/...ults/var_mad_int_featselect/cell_type_a_vs_b/progen_vs_astro/test_results_control.parquet
diff --git a/...lts/var_mad_int_featselect/cell_type_a_vs_b/progen_vs_astro/test_results_deletion.parquet b/...lts/var_mad_int_featselect/cell_type_a_vs_b/progen_vs_astro/test_results_deletion.parquet
diff --git a/...elect/cell_type_a_vs_b/stem_vs_astro/classification_vs_significant_features.png b/...elect/cell_type_a_vs_b/stem_vs_astro/classification_vs_significant_features.png
diff --git a/..._int_featselect/cell_type_a_vs_b/stem_vs_astro/significant_features_control.png b/..._int_featselect/cell_type_a_vs_b/stem_vs_astro/significant_features_control.png
diff --git a/...int_featselect/cell_type_a_vs_b/stem_vs_astro/significant_features_deletion.png b/...int_featselect/cell_type_a_vs_b/stem_vs_astro/significant_features_deletion.png
diff --git a/...sis_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_astro/summary_results.parquet b/...sis_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_astro/summary_results.parquet
diff --git a/...esults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_astro/test_results_control.parquet b/...esults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_astro/test_results_control.parquet
diff --git a/...sults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_astro/test_results_deletion.parquet b/...sults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_astro/test_results_deletion.parquet
diff --git a/...lect/cell_type_a_vs_b/stem_vs_neuron/classification_vs_significant_features.png b/...lect/cell_type_a_vs_b/stem_vs_neuron/classification_vs_significant_features.png
diff --git a/...int_featselect/cell_type_a_vs_b/stem_vs_neuron/significant_features_control.png b/...int_featselect/cell_type_a_vs_b/stem_vs_neuron/significant_features_control.png
diff --git a/...nt_featselect/cell_type_a_vs_b/stem_vs_neuron/significant_features_deletion.png b/...nt_featselect/cell_type_a_vs_b/stem_vs_neuron/significant_features_deletion.png
diff --git a/...is_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_neuron/summary_results.parquet b/...is_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_neuron/summary_results.parquet
diff --git a/...sults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_neuron/test_results_control.parquet b/...sults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_neuron/test_results_control.parquet
diff --git a/...ults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_neuron/test_results_deletion.parquet b/...ults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_neuron/test_results_deletion.parquet
diff --git a/...lect/cell_type_a_vs_b/stem_vs_progen/classification_vs_significant_features.png b/...lect/cell_type_a_vs_b/stem_vs_progen/classification_vs_significant_features.png
diff --git a/...int_featselect/cell_type_a_vs_b/stem_vs_progen/significant_features_control.png b/...int_featselect/cell_type_a_vs_b/stem_vs_progen/significant_features_control.png
diff --git a/...nt_featselect/cell_type_a_vs_b/stem_vs_progen/significant_features_deletion.png b/...nt_featselect/cell_type_a_vs_b/stem_vs_progen/significant_features_deletion.png
diff --git a/...is_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_progen/summary_results.parquet b/...is_results/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_progen/summary_results.parquet
diff --git a/...sults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_progen/test_results_control.parquet b/...sults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_progen/test_results_control.parquet
diff --git a/...ults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_progen/test_results_deletion.parquet b/...ults/var_mad_int_featselect/cell_type_a_vs_b/stem_vs_progen/test_results_deletion.parquet
diff --git a/...d_int_featselect/control_vs_deletion/classification_vs_significant_features.png b/...d_int_featselect/control_vs_deletion/classification_vs_significant_features.png
diff --git a/...sults/var_mad_int_featselect/control_vs_deletion/significant_features_astro.png b/...sults/var_mad_int_featselect/control_vs_deletion/significant_features_astro.png
diff --git a/...ults/var_mad_int_featselect/control_vs_deletion/significant_features_neuron.png b/...ults/var_mad_int_featselect/control_vs_deletion/significant_features_neuron.png
diff --git a/...ults/var_mad_int_featselect/control_vs_deletion/significant_features_progen.png b/...ults/var_mad_int_featselect/control_vs_deletion/significant_features_progen.png
diff --git a/...esults/var_mad_int_featselect/control_vs_deletion/significant_features_stem.png b/...esults/var_mad_int_featselect/control_vs_deletion/significant_features_stem.png
diff --git a/...utput/analysis_results/var_mad_int_featselect/control_vs_deletion/summary_results.parquet b/...utput/analysis_results/var_mad_int_featselect/control_vs_deletion/summary_results.parquet
diff --git a/...ut/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_astro.parquet b/...ut/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_astro.parquet
diff --git a/...t/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_neuron.parquet b/...t/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_neuron.parquet
diff --git a/...t/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_progen.parquet b/...t/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_progen.parquet
diff --git a/...put/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_stem.parquet b/...put/analysis_results/var_mad_int_featselect/control_vs_deletion/test_results_stem.parquet
diff --git a/5.reanalysis/output/analysis_results/var_mad_int_featselect/metrics/ap_nonrep.parquet b/5.reanalysis/output/analysis_results/var_mad_int_featselect/metrics/ap_nonrep.parquet
diff --git a/5.reanalysis/output/analysis_results/var_mad_int_featselect/metrics/map_nonrep.parquet b/5.reanalysis/output/analysis_results/var_mad_int_featselect/metrics/map_nonrep.parquet
diff --git a/5.reanalysis/output/processed/var_mad_int/combined.parquet b/5.reanalysis/output/processed/var_mad_int/combined.parquet
diff --git a/5.reanalysis/output/processed/var_mad_int_featselect/combined.parquet b/5.reanalysis/output/processed/var_mad_int_featselect/combined.parquet
diff --git a/ncp/src/visualization.py b/ncp/src/visualization.py
@@ -89,6 +89,7 @@ def visualize_results(input_file, output_dir):
             capsize=3,  # Adds caps to the error bars
         )
 
+    plt.xlim(0.5, 1)
     plt.title("Logistic Regression Accuracy vs Number of Significant Features")
     plt.xlabel("Logistic Regression Accuracy")
     plt.ylabel("Number of Significant Features")