Make sure to account for the fact that corrupted FOVs will need to be…

… re-written no matter what

src/toffy/fov_watcher.py

@@ -263,12 +263,14 @@ def _check_fov_status(self, path: str):
 
             return None, None
 
-    def _generate_callback_data(self, point_name: str):
+    def _generate_callback_data(self, point_name: str, overwrite: bool):
         """Runs the `fov_func` and `inter_func` if applicable for a FOV
 
         Args:
             point_name (str):
                 The name of the FOV to run FOV (and intermediate if applicable) callbacks on
+            overwrite (bool):
+                Forces an overwrite of already existing data, needed if a FOV needs re-extraction
         """
         print(f"Discovered {point_name}, beginning per-fov callbacks...")
         logging.info(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} -- Extracting {point_name}\n')
@@ -277,7 +279,7 @@ def _generate_callback_data(self, point_name: str):
             f"Running {self.fov_func.__name__} on {point_name}\n"
         )
 
-        self.fov_func(self.run_folder, point_name)
+        self.fov_func(self.run_folder, point_name, overwrite)
         self.run_structure.processed(point_name)
 
         if self.inter_func:
@@ -409,20 +411,22 @@ def _check_bin_updates(self):
 
                 # re-extract the .bin file
                 # NOTE: since no more FOVs are being written, last_fov_num_processed is irrelevant
-                self._fov_callback_driver(fov_bin_path)
+                self._fov_callback_driver(fov_bin_path, overwrite=True)
 
-    def _fov_callback_driver(self, file_trigger: str):
+    def _fov_callback_driver(self, file_trigger: str, overwrite: bool = False):
         """The FOV and intermediate-level callback motherbase for a single .bin file
 
         Args:
             file_trigger (str):
                 The file that gets caught by the watcher to throw into the pipeline
+            overwrite (bool):
+                Forces an overwrite of already existing data, needed if a FOV needs re-extraction
         """
         # check if what's created is in the run structure
         fov_ready, point_name = self._check_fov_status(file_trigger)
 
         if fov_ready:
-            self._generate_callback_data(point_name)
+            self._generate_callback_data(point_name, overwrite=overwrite)
 
         # needs to update if .bin file processed OR new moly point detected
         is_moly = point_name in self.run_structure.moly_points

src/toffy/watcher_callbacks.py

@@ -122,6 +122,7 @@ def image_stitching(self, tiff_out_dir, **kwargs):
 class FovCallbacks:
     run_folder: str
     point_name: str
+    overwrite: bool
     __panel: pd.DataFrame = field(default=None, init=False)
     __fov_data: xr.DataArray = field(default=None, init=False)
 
@@ -184,7 +185,7 @@ def extract_tiffs(self, tiff_out_dir: str, panel: pd.DataFrame, **kwargs):
         unextracted_chan_tiffs = []
 
         # in the case all images have been extracted, simply return
-        if os.path.exists(extracted_img_dir):
+        if os.path.exists(extracted_img_dir) and not self.overwrite:
             all_chan_tiffs = [f"{ct}.tiff" for ct in panel["Target"]]
             extracted_chan_tiffs = io_utils.list_files(extracted_img_dir, substrs=".tiff")
             unextracted_chan_tiffs = set(all_chan_tiffs).difference(extracted_chan_tiffs)
@@ -194,7 +195,7 @@ def extract_tiffs(self, tiff_out_dir: str, panel: pd.DataFrame, **kwargs):
                 return
 
         # ensure we don't re-extract channels that have already been extracted
-        if unextracted_chan_tiffs:
+        if unextracted_chan_tiffs and not self.overwrite:
             unextracted_chans = io_utils.remove_file_extensions(unextracted_chan_tiffs)
             panel = panel[panel["Target"].isin(unextracted_chans)]
 
@@ -240,7 +241,7 @@ def generate_qc(self, qc_out_dir: str, panel: pd.DataFrame = None, **kwargs):
             os.path.join(qc_out_dir, f"{self.point_name}_total_intensity_stats.csv"),
             os.path.join(qc_out_dir, f"{self.point_name}_percentile_99_9_stats.csv"),
         ]
-        if all([os.path.exists(qc_file) for qc_file in qc_metric_paths]):
+        if all([os.path.exists(qc_file) for qc_file in qc_metric_paths]) and not self.overwrite:
             warnings.warn(f"All QC metrics already extracted for FOV {self.point_name}")
             return
 
@@ -271,7 +272,7 @@ def generate_mph(self, mph_out_dir, **kwargs):
             os.makedirs(mph_out_dir)
 
         mph_pulse_file = os.path.join(mph_out_dir, f"{self.point_name}-mph_pulse.csv")
-        if os.path.exists(mph_pulse_file):
+        if os.path.exists(mph_pulse_file) and not self.overwrite:
             warnings.warn(f"MPH pulse metrics already extracted for FOV {self.point_name}")
             return
 
@@ -301,7 +302,7 @@ def generate_pulse_heights(self, pulse_out_dir: str, panel: pd.DataFrame = None,
             os.makedirs(pulse_out_dir)
 
         pulse_height_file = os.path.join(pulse_out_dir, f"{self.point_name}-pulse_heights.csv")
-        if os.path.exists(pulse_height_file):
+        if os.path.exists(pulse_height_file) and not self.overwrite:
             warnings.warn(f"Pulse heights per mass already extracted for FOV {self.point_name}")
             return
 
@@ -344,9 +345,9 @@ def build_fov_callback(*args, **kwargs):
         misc_utils.verify_in_list(required_arguments=argnames, passed_arguments=list(kwargs.keys()))
 
     # construct actual callback
-    def fov_callback(run_folder: str, point_name: str):
+    def fov_callback(run_folder: str, point_name: str, overwrite: bool = False):
         # construct FovCallback object for given FoV
-        callback_obj = FovCallbacks(run_folder, point_name)
+        callback_obj = FovCallbacks(run_folder, point_name, overwrite)
 
         # for each member, retrieve the member function and run it
         for arg in args: