Merge pull request #324 from DendrouLab/fc_filecheck

added check for duplicated rows in submission file
DendrouLab · Dec 4, 2024 · 207f93c · 207f93c
2 parents d32952e + 39ac80f
commit 207f93c
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/panpipes/panpipes/pipeline_ingest.py b/panpipes/panpipes/pipeline_ingest.py
@@ -95,6 +95,12 @@ def unfilt_file():
 
 def gen_load_filtered_anndata_jobs():
     caf = pd.read_csv(PARAMS['submission_file'], sep='\t')
+    duplicated_rows = caf.duplicated()
+
+    if duplicated_rows.any():
+        print(f"Duplicated rows found and removed: {duplicated_rows.sum()} rows.")
+        caf = caf.drop_duplicates()
+
     return gen_load_anndata_jobs(caf, load_raw=False, mode_dictionary=PARAMS["modalities"], load_prot_from_raw=PARAMS['load_prot_from_raw'])