From 861385aac1e81d3b650cf8d720f7ba80d545a043 Mon Sep 17 00:00:00 2001 From: Avelino Javer Date: Mon, 17 Dec 2018 13:56:33 +0000 Subject: [PATCH] Development (#56) * Update HOWTO.md * bug - silly bug when timestamp is all nan - add an attribute that could be skipped * Update HOWTO.md join trajectories * correct bug with symbolic paths * add different features param files for the WT2 * add abspath and realpath to avoid problems in Linux This is a very bad bug where in linux the path resolution didn't understood the tempoerary and the destination file where the same. Therefore it sometimes delete the source files. * replace debugging call with warning * small bug due to using float as index There were a couple of files where the timestamp_ind was `float`. I force it to be `int` and it fix the problem. * bug * bug --- recipe/meta.yaml | 1 + .../int_ske_orient/checkFinalOrientation.py | 5 +-- .../stage_aligment/alignStageMotion.py | 3 +- ...e.json => WT2_anticlockwise_OPENWORM.json} | 0 .../WT2_anticlockwise_TIERPSY.json | 18 ++++++++++ ...kwise.json => WT2_clockwise_OPENWORM.json} | 0 .../param_files/WT2_clockwise_TIERPSY.json | 18 ++++++++++ tierpsy/gui/TrackerViewerAux.py | 2 +- tierpsy/processing/AnalysisPoints.py | 11 +++--- tierpsy/processing/ProcessLocal.py | 36 ++++++++++--------- 10 files changed, 69 insertions(+), 25 deletions(-) rename tierpsy/extras/param_files/{WT2_anticlockwise.json => WT2_anticlockwise_OPENWORM.json} (100%) mode change 100755 => 100644 create mode 100755 tierpsy/extras/param_files/WT2_anticlockwise_TIERPSY.json rename tierpsy/extras/param_files/{WT2_clockwise.json => WT2_clockwise_OPENWORM.json} (100%) create mode 100755 tierpsy/extras/param_files/WT2_clockwise_TIERPSY.json diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5be24d10..3161567c 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -34,6 +34,7 @@ requirements: - numba - tqdm - requests + - ffmpeg app: entry: tierpsy_gui diff --git a/tierpsy/analysis/int_ske_orient/checkFinalOrientation.py b/tierpsy/analysis/int_ske_orient/checkFinalOrientation.py index d5b726a2..4cabc957 100755 --- a/tierpsy/analysis/int_ske_orient/checkFinalOrientation.py +++ b/tierpsy/analysis/int_ske_orient/checkFinalOrientation.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd import tables +import warnings from scipy.signal import savgol_filter from tierpsy.analysis.ske_orient.checkHeadOrientation import isWormHTSwitched @@ -95,8 +96,8 @@ def getHeadProbMov( p_mov = p_mov.values[ind_valid] if p_mov.size == 0: - import pdb - pdb.set_trace() + w_ind = trajectories_worm['worm_index_joined'].iloc[0] + warnings.warn('There is something weird with trajectory {} in file {}. No valid head tail movements found.'.format(w_ind, skeletons_file)) #average using only the indexes of valid skeletons p_mov_avg = np.nanmean(p_mov) diff --git a/tierpsy/analysis/stage_aligment/alignStageMotion.py b/tierpsy/analysis/stage_aligment/alignStageMotion.py index e7d54de6..2946fc1c 100755 --- a/tierpsy/analysis/stage_aligment/alignStageMotion.py +++ b/tierpsy/analysis/stage_aligment/alignStageMotion.py @@ -45,7 +45,7 @@ def _h_get_stage_inv(skeletons_file, timestamp): # adjust the stage_vec to match the timestamps in the skeletons good = (timestamp_ind >= first_frame) & (timestamp_ind <= last_frame) - ind_ff = timestamp_ind[good] - first_frame + ind_ff = (timestamp_ind[good] - first_frame).astype(np.int) #make sure it is int to be used as index if timestamp_ind.shape[0] > stage_vec_ori.shape[0]: #there are extra elements in the timestamp_ind, let's pad it with the same value in the stage vector extra_n = timestamp_ind.shape[0] - stage_vec_ori.shape[0] @@ -54,6 +54,7 @@ def _h_get_stage_inv(skeletons_file, timestamp): stage_vec_ori = stage_vec_ori[good] stage_vec = np.full((timestamp.size, 2), np.nan) + stage_vec[ind_ff, :] = stage_vec_ori # the negative symbole is to add the stage vector directly, instead of # substracting it. diff --git a/tierpsy/extras/param_files/WT2_anticlockwise.json b/tierpsy/extras/param_files/WT2_anticlockwise_OPENWORM.json old mode 100755 new mode 100644 similarity index 100% rename from tierpsy/extras/param_files/WT2_anticlockwise.json rename to tierpsy/extras/param_files/WT2_anticlockwise_OPENWORM.json diff --git a/tierpsy/extras/param_files/WT2_anticlockwise_TIERPSY.json b/tierpsy/extras/param_files/WT2_anticlockwise_TIERPSY.json new file mode 100755 index 00000000..f38cb3cd --- /dev/null +++ b/tierpsy/extras/param_files/WT2_anticlockwise_TIERPSY.json @@ -0,0 +1,18 @@ +{ + "thresh_block_size": 61, + "thresh_C": 15, + "dilation_size": 10, + "min_area": 100, + "resampling_N": 49, + "expected_fps": 30.0, + "max_area": 100000, + "is_extract_metadata": true, + "keep_border_data": true, + "compression_buff": 1, + "worm_bw_thresh_factor":1.0, + "strel_size":10, + "int_save_maps": true, + "filt_min_displacement":0, + "analysis_type" : "TIERPSY_WT2", + "ventral_side" : "anticlockwise" +} \ No newline at end of file diff --git a/tierpsy/extras/param_files/WT2_clockwise.json b/tierpsy/extras/param_files/WT2_clockwise_OPENWORM.json similarity index 100% rename from tierpsy/extras/param_files/WT2_clockwise.json rename to tierpsy/extras/param_files/WT2_clockwise_OPENWORM.json diff --git a/tierpsy/extras/param_files/WT2_clockwise_TIERPSY.json b/tierpsy/extras/param_files/WT2_clockwise_TIERPSY.json new file mode 100755 index 00000000..04bb3aa7 --- /dev/null +++ b/tierpsy/extras/param_files/WT2_clockwise_TIERPSY.json @@ -0,0 +1,18 @@ +{ + "thresh_block_size": 61, + "thresh_C": 15, + "dilation_size": 10, + "min_area": 100, + "resampling_N": 49, + "expected_fps": 30.0, + "max_area": 100000, + "is_extract_metadata": true, + "keep_border_data": true, + "compression_buff": 1, + "worm_bw_thresh_factor":1.0, + "strel_size":10, + "int_save_maps": true, + "filt_min_displacement":0, + "analysis_type" : "TIERPSY_WT2", + "ventral_side" : "clockwise" +} \ No newline at end of file diff --git a/tierpsy/gui/TrackerViewerAux.py b/tierpsy/gui/TrackerViewerAux.py index 2f73b7f8..a156c3e6 100755 --- a/tierpsy/gui/TrackerViewerAux.py +++ b/tierpsy/gui/TrackerViewerAux.py @@ -309,7 +309,7 @@ def drawSkel(self, worm_img, worm_qimg, row_data, roi_corner=(0, 0)): dat = ske_file_id.get_node(field)[skel_id] dat /= self.microns_per_pixel - if self.stage_position_pix is not None: + if self.stage_position_pix is not None and self.stage_position_pix.size > 0: #subtract stage motion if necessary dat -= self.stage_position_pix[self.frame_number] diff --git a/tierpsy/processing/AnalysisPoints.py b/tierpsy/processing/AnalysisPoints.py index 4206fc6c..69e7b05f 100644 --- a/tierpsy/processing/AnalysisPoints.py +++ b/tierpsy/processing/AnalysisPoints.py @@ -64,12 +64,13 @@ def __next__(self): class AnalysisPoints(object): def __init__(self, video_file, masks_dir, results_dir, json_file = ''): + + self.video_file = os.path.realpath(os.path.abspath(video_file)) + self.results_dir = os.path.realpath(os.path.abspath(results_dir)) + self.masks_dir = os.path.realpath(os.path.abspath(masks_dir)) + - self.getFileNames(video_file, masks_dir, results_dir) - - self.video_file = video_file - self.masks_dir = masks_dir - self.results_dir = results_dir + self.getFileNames(self.video_file, self.masks_dir, self.results_dir) self.param = TrackerParams(json_file) self.checkpoints = CheckPoints(self.file_names, self.param) diff --git a/tierpsy/processing/ProcessLocal.py b/tierpsy/processing/ProcessLocal.py index 67130d06..c73d5a48 100644 --- a/tierpsy/processing/ProcessLocal.py +++ b/tierpsy/processing/ProcessLocal.py @@ -23,11 +23,11 @@ class ProcessLocal(object): def __init__(self, main_file, masks_dir, results_dir, tmp_mask_dir='', tmp_results_dir='', json_file='', analysis_checkpoints = [], - is_copy_video = False, copy_unfinished=False): + is_copy_video = False, copy_unfinished = False): - self.main_file = os.path.realpath(main_file) - self.results_dir = os.path.realpath(results_dir) - self.masks_dir = os.path.realpath(masks_dir) + self.main_file = os.path.realpath(os.path.abspath(main_file)) + self.results_dir = os.path.realpath(os.path.abspath(results_dir)) + self.masks_dir = os.path.realpath(os.path.abspath(masks_dir)) #check that the files do exists if not os.path.exists(self.main_file): @@ -72,9 +72,12 @@ def __init__(self, main_file, masks_dir, results_dir, tmp_mask_dir='', self.unfinished_points_src = self.ap_src.getUnfinishedPoints(self.analysis_checkpoints) self.unfinished_points_tmp = self.ap_tmp.getUnfinishedPoints(self.analysis_checkpoints) + #TODO, here i should be more strict. If there are more unfinished points in temporary, use only the files in src... + + #get the points to be processed compared with the existing files - self.checkpoints2process = self._getPoints2Process() - + self.checkpoints2process = self._getPoints2Process(self.unfinished_points_src, self.unfinished_points_tmp) + # we need to group steps into start and clean steps for the multiprocess # part def start(self): @@ -82,6 +85,7 @@ def start(self): self.start_time = time.time() #copy tmp files + self._copyFinaltoTmp() args = [self.tmp_main_file] argkws = {'masks_dir':self.tmp_mask_dir, 'results_dir':self.tmp_results_dir, @@ -104,21 +108,22 @@ def clean(self): print_flush(progress_str) - def _getPoints2Process(self): + def _getPoints2Process(self, _unfinished_points_src, _unfinished_points_tmp): def assignAndCheckSubset(small_list, larger_list): assert set(small_list).issubset(set(larger_list)) return small_list - if len(self.unfinished_points_src) < len(self.unfinished_points_tmp): - checkpoints2process = assignAndCheckSubset(self.unfinished_points_src, self.unfinished_points_tmp) + if len(_unfinished_points_src) < len(_unfinished_points_tmp): + checkpoints2process = assignAndCheckSubset(_unfinished_points_src, _unfinished_points_tmp) else: - checkpoints2process = assignAndCheckSubset(self.unfinished_points_tmp, self.unfinished_points_src) + checkpoints2process = assignAndCheckSubset(_unfinished_points_tmp, _unfinished_points_src) return checkpoints2process def _copyFinaltoTmp(self): #files that are required as input inputs_required = self._points2Files(self.checkpoints2process, self.ap_tmp, "input_files") + new_created_files = self._getNewFilesCreated(self.checkpoints2process, self.ap_tmp) #files that are required as input but are not produced later on needed_files = inputs_required - new_created_files @@ -145,7 +150,6 @@ def _copyFinaltoTmp(self): files2copy += self._getAddFilesForTmpSW() - self._copyFilesLocal(files2copy) def _getAddFilesForTmpSW(self): @@ -192,8 +196,6 @@ def _copyTmpToFinalAndClean(self): self._deleteTmpFiles() - - def _deleteTmpFiles(self): def _points2FullFiles(points2check, ap_obj, field_name): data = ap_obj.getField(field_name, points2check) @@ -208,9 +210,11 @@ def _points2FullFiles(points2check, ap_obj, field_name): #CLEAN all_tmp_files = _points2FullFiles(self.analysis_checkpoints, self.ap_tmp, "output_files") | \ _points2FullFiles(self.analysis_checkpoints, self.ap_tmp, "input_files") + all_tmp_files = set(map(os.path.realpath, map(os.path.abspath, all_tmp_files))) all_src_files = _points2FullFiles(self.analysis_checkpoints, self.ap_src, "output_files") | \ _points2FullFiles(self.analysis_checkpoints, self.ap_src, "input_files") + all_src_files = set(map(os.path.realpath, map(os.path.abspath, all_src_files))) #remove all tmp files that are not in the source files2remove = all_tmp_files - all_src_files @@ -263,10 +267,10 @@ def _getFilesSrcDstPairs(self, fnames, f2d_src, f2dir_dst): def _copyFilesLocal(self, files2copy): ''' copy files to the source directory''' for files in files2copy: - file_name, destination = files + file_name, destination = map(os.path.realpath, map(os.path.abspath, files)) assert(os.path.exists(destination)) - - if os.path.abspath(os.path.dirname(file_name)) != os.path.abspath(destination): + + if os.path.dirname(file_name) != destination: print_flush('Copying %s to %s' % (file_name, destination)) shutil.copy(file_name, destination)