Use pyarrow for reading csvs

PMBio · Dec 6, 2023 · 93fa488 · 93fa488
1 parent 4ac78cd
commit 93fa488
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/deeprvat/preprocessing/preprocess.py b/deeprvat/preprocessing/preprocess.py
@@ -48,10 +48,10 @@ def process_sparse_gt_file(
     samples: List[str],
     calls_to_exclude: pd.DataFrame = None,
 ) -> Tuple[List[np.ndarray], List[np.ndarray]]:
-    sparse_gt = pd.read_csv(
+    sparse_gt = pd.read_table(
         file,
         names=["chrom", "pos", "ref", "alt", "sample", "gt"],
-        sep="\t",
+        engine="pyarrow",
         index_col=None,
     )
     sparse_gt = sparse_gt[sparse_gt["sample"].isin(samples)]
@@ -183,7 +183,7 @@ def process_sparse_gt(
     logging.info("Reading variants...")
     start_time = time.time()
 
-    variants = pd.read_csv(variant_file, sep="\t")
+    variants = pd.read_table(variant_file, engine="pyarrow")
 
     # Filter all variants based on chromosome
     if chromosomes is not None: