From 91100519f961a238fa158fd1a3572b4f007cb242 Mon Sep 17 00:00:00 2001 From: Ravi <115461198+BM32ESRF@users.noreply.github.com> Date: Thu, 24 Nov 2022 15:31:34 +0100 Subject: [PATCH] updates prepping for 3.1.12 --- .../LaueNN_pyScript_1_or_2phase.py | 54 ++++++ lauetoolsnn/lauetoolsneuralnetwork.py | 57 ++++-- lauetoolsnn/utils_lauenn.py | 178 ++++++++++++------ 3 files changed, 209 insertions(+), 80 deletions(-) diff --git a/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py b/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py index 2033969..a1ee70c 100644 --- a/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py +++ b/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py @@ -889,6 +889,60 @@ # results = new_MP_function(valu12[0]) # best = results[-2][0][0][0] + def chunker_list(seq, size): + return (seq[i::size] for i in range(size)) + # # ============================================================================= + # # Below code when RAM is of concern + # # ============================================================================= + # from multiprocessing import Process, Queue + # from lauetoolsnn.utils_lauenn import new_MP_function_v1 + # _inputs_queue = Queue() + # _outputs_queue = Queue() + # _worker_processes = {} + # for i in range(ncpu): + # _worker_processes[i]= Process(target=new_MP_function_v1, args=(_inputs_queue, _outputs_queue, i+1)) + # for i in range(ncpu): + # _worker_processes[i].start() + # chunks = chunker_list(valu12, ncpu) + # chunks_mp = list(chunks) + # meta = {'t1':time.time()} + # for ijk in range(int(ncpu)): + # _inputs_queue.put((chunks_mp[ijk], ncpu, meta)) + # ### Update data from multiprocessing + # pbar = tqdm(total=count_global) + # unique_count = [] + # while True: + # if len(np.unique(unique_count)) == count_global: + # print("All files have been treated") + # break + # if not _outputs_queue.empty(): + # n_range = _outputs_queue.qsize() + # for _ in range(n_range): + # r_message_mpdata = _outputs_queue.get() + # strain_matrix_mpdata, strain_matrixs_mpdata, rotation_matrix_mpdata, col_mpdata,\ + # colx_mpdata, coly_mpdata, match_rate_mpdata, mat_global_mpdata,\ + # cnt_mpdata, meta_mpdata, files_treated_mpdata, spots_len_mpdata, \ + # iR_pixel_mpdata, fR_pixel_mpdata, best_match_mpdata, check_mpdata = r_message_mpdata + # for i_mpdata in files_treated_mpdata: + # files_treated.append(i_mpdata) + # for intmat_mpdata in range(int(ubmat)): + # check[cnt_mpdata,intmat_mpdata] = check_mpdata[cnt_mpdata,intmat_mpdata] + # mat_global[intmat_mpdata][0][cnt_mpdata] = mat_global_mpdata[intmat_mpdata][0][cnt_mpdata] + # strain_matrix[intmat_mpdata][0][cnt_mpdata,:,:] = strain_matrix_mpdata[intmat_mpdata][0][cnt_mpdata,:,:] + # strain_matrixs[intmat_mpdata][0][cnt_mpdata,:,:] = strain_matrixs_mpdata[intmat_mpdata][0][cnt_mpdata,:,:] + # rotation_matrix[intmat_mpdata][0][cnt_mpdata,:,:] = rotation_matrix_mpdata[intmat_mpdata][0][cnt_mpdata,:,:] + # col[intmat_mpdata][0][cnt_mpdata,:] = col_mpdata[intmat_mpdata][0][cnt_mpdata,:] + # colx[intmat_mpdata][0][cnt_mpdata,:] = colx_mpdata[intmat_mpdata][0][cnt_mpdata,:] + # coly[intmat_mpdata][0][cnt_mpdata,:] = coly_mpdata[intmat_mpdata][0][cnt_mpdata,:] + # match_rate[intmat_mpdata][0][cnt_mpdata] = match_rate_mpdata[intmat_mpdata][0][cnt_mpdata] + # spots_len[intmat_mpdata][0][cnt_mpdata] = spots_len_mpdata[intmat_mpdata][0][cnt_mpdata] + # iR_pix[intmat_mpdata][0][cnt_mpdata] = iR_pixel_mpdata[intmat_mpdata][0][cnt_mpdata] + # fR_pix[intmat_mpdata][0][cnt_mpdata] = fR_pixel_mpdata[intmat_mpdata][0][cnt_mpdata] + # best_match[intmat_mpdata][0][cnt_mpdata] = best_match_mpdata[intmat_mpdata][0][cnt_mpdata] + # if cnt_mpdata not in unique_count: + # pbar.update(1) + # unique_count.append(cnt_mpdata) + args = zip(valu12) with multiprocessing.Pool(ncpu) as pool: results = pool.starmap(new_MP_function, tqdm(args, total=len(valu12)), chunksize=1) diff --git a/lauetoolsnn/lauetoolsneuralnetwork.py b/lauetoolsnn/lauetoolsneuralnetwork.py index 419bea4..202463e 100644 --- a/lauetoolsnn/lauetoolsneuralnetwork.py +++ b/lauetoolsnn/lauetoolsneuralnetwork.py @@ -6293,26 +6293,43 @@ def predict_preprocess(self,cnt,rotation_matrix,strain_matrix,strain_matrixs, print("Error writing the cor file", path) elif files.split(".")[-1] == "cor": - seednumber = "Experimental COR file" - allres = IOLT.readfile_cor(files, True) - data_theta, data_chi, peakx, peaky, intensity = allres[1:6] - CCDcalib = allres[-1] - detectorparameters = allres[-2] - # print('detectorparameters from file are: '+ str(detectorparameters)) - pixelsize = CCDcalib['pixelsize'] - CCDLabel = CCDcalib['CCDLabel'] - framedim = dictLT.dict_CCD[CCDLabel][0] - dict_dp={} - dict_dp['kf_direction']=default_detector_geom - dict_dp['detectorparameters']=detectorparameters - dict_dp['detectordistance']=detectorparameters[0] - dict_dp['detectordiameter']=pixelsize*framedim[0]#TODO*2 - dict_dp['pixelsize']=pixelsize - dict_dp['dim']=framedim - dict_dp['peakX']=peakx - dict_dp['peakY']=peaky - dict_dp['intensity']=intensity - + try: + seednumber = "Experimental COR file" + allres = IOLT.readfile_cor(files, True) + data_theta, data_chi, peakx, peaky, intensity = allres[1:6] + CCDcalib = allres[-1] + detectorparameters = allres[-2] + # print('detectorparameters from file are: '+ str(detectorparameters)) + pixelsize = CCDcalib['pixelsize'] + CCDLabel = CCDcalib['CCDLabel'] + framedim = dictLT.dict_CCD[CCDLabel][0] + dict_dp={} + dict_dp['kf_direction']=default_detector_geom + dict_dp['detectorparameters']=detectorparameters + dict_dp['detectordistance']=detectorparameters[0] + dict_dp['detectordiameter']=pixelsize*framedim[0]#TODO*2 + dict_dp['pixelsize']=pixelsize + dict_dp['dim']=framedim + dict_dp['peakX']=peakx + dict_dp['peakY']=peaky + dict_dp['intensity']=intensity + except: + print("No cor file found for "+ files) + for intmat in range(matricies): + rotation_matrix[intmat][0][self.cnt,:,:] = np.zeros((3,3)) + strain_matrix[intmat][0][self.cnt,:,:] = np.zeros((3,3)) + strain_matrixs[intmat][0][self.cnt,:,:] = np.zeros((3,3)) + col[intmat][0][self.cnt,:] = 0,0,0 + colx[intmat][0][self.cnt,:] = 0,0,0 + coly[intmat][0][self.cnt,:] = 0,0,0 + match_rate[intmat][0][self.cnt] = 0 + mat_global[intmat][0][self.cnt] = 0 + + cnt += 1 + self.cnt += 1 + peak_detection_error = True + continue + if peak_detection_error: continue diff --git a/lauetoolsnn/utils_lauenn.py b/lauetoolsnn/utils_lauenn.py index 76b1900..78c1eb5 100644 --- a/lauetoolsnn/utils_lauenn.py +++ b/lauetoolsnn/utils_lauenn.py @@ -2234,24 +2234,43 @@ def predict_preprocessMP(files, cnt, print("Error writing the cor file", path) elif files.split(".")[-1] == "cor": - seednumber = "Experimental COR file" - allres = IOLT.readfile_cor(files, True) - data_theta, data_chi, peakx, peaky, intensity = allres[1:6] - CCDcalib = allres[-1] - detectorparameters = allres[-2] - pixelsize = CCDcalib['pixelsize'] - CCDLabel = CCDcalib['CCDLabel'] - framedim = dictLT.dict_CCD[CCDLabel][0] - dict_dp={} - dict_dp['kf_direction']=default_detector_geom - dict_dp['detectorparameters']=detectorparameters - dict_dp['detectordistance']=detectorparameters[0] - dict_dp['detectordiameter']=pixelsize*framedim[0]#TODO*2 - dict_dp['pixelsize']=pixelsize - dict_dp['dim']=framedim - dict_dp['peakX']=peakx - dict_dp['peakY']=peaky - dict_dp['intensity']=intensity + try: + seednumber = "Experimental COR file" + allres = IOLT.readfile_cor(files, True) + data_theta, data_chi, peakx, peaky, intensity = allres[1:6] + CCDcalib = allres[-1] + detectorparameters = allres[-2] + pixelsize = CCDcalib['pixelsize'] + CCDLabel = CCDcalib['CCDLabel'] + framedim = dictLT.dict_CCD[CCDLabel][0] + dict_dp={} + dict_dp['kf_direction']=default_detector_geom + dict_dp['detectorparameters']=detectorparameters + dict_dp['detectordistance']=detectorparameters[0] + dict_dp['detectordiameter']=pixelsize*framedim[0]#TODO*2 + dict_dp['pixelsize']=pixelsize + dict_dp['dim']=framedim + dict_dp['peakX']=peakx + dict_dp['peakY']=peaky + dict_dp['intensity']=intensity + except: + print("Error in Cor file reading for "+ files) + for intmat in range(matricies): + rotation_matrix[intmat][0][cnt,:,:] = np.zeros((3,3)) + strain_matrix[intmat][0][cnt,:,:] = np.zeros((3,3)) + strain_matrixs[intmat][0][cnt,:,:] = np.zeros((3,3)) + col[intmat][0][cnt,:] = 0,0,0 + colx[intmat][0][cnt,:] = 0,0,0 + coly[intmat][0][cnt,:] = 0,0,0 + match_rate[intmat][0][cnt] = 0 + mat_global[intmat][0][cnt] = 0 + spots_len[intmat][0][cnt] = 0 + iR_pix[intmat][0][cnt] = 0 + fR_pix[intmat][0][cnt] = 0 + check[cnt,intmat] = 0 + files_treated.append(files) + return strain_matrix, strain_matrixs, rotation_matrix, col, colx, coly, \ + match_rate, mat_global, cnt, files_treated,spots_len,iR_pix,fR_pix, check, best_match, None sorted_data = np.transpose(np.array([data_theta, data_chi])) tabledistancerandom = np.transpose(GT.calculdist_from_thetachi(sorted_data, sorted_data)) @@ -8378,27 +8397,47 @@ def predict_preprocessMultiProcess(files, cnt, print("Error writing the cor file", path) elif files.split(".")[-1] == "cor": - # print("Entering Cor file read section") - seednumber = "Experimental COR file" - allres = IOLT.readfile_cor(files, True) - data_theta, data_chi, peakx, peaky, intensity = allres[1:6] - CCDcalib = allres[-1] - detectorparameters = allres[-2] - # print('detectorparameters from file are: '+ str(detectorparameters)) - pixelsize = CCDcalib['pixelsize'] - CCDLabel = CCDcalib['CCDLabel'] - framedim = dictLT.dict_CCD[CCDLabel][0] - dict_dp={} - dict_dp['kf_direction']=default_detector_geom - dict_dp['detectorparameters']=detectorparameters - dict_dp['detectordistance']=detectorparameters[0] - dict_dp['detectordiameter']=pixelsize*framedim[0] - dict_dp['pixelsize']=pixelsize - dict_dp['dim']=framedim - dict_dp['peakX']=peakx - dict_dp['peakY']=peaky - dict_dp['intensity']=intensity - + try: + # print("Entering Cor file read section") + seednumber = "Experimental COR file" + allres = IOLT.readfile_cor(files, True) + data_theta, data_chi, peakx, peaky, intensity = allres[1:6] + CCDcalib = allres[-1] + detectorparameters = allres[-2] + # print('detectorparameters from file are: '+ str(detectorparameters)) + pixelsize = CCDcalib['pixelsize'] + CCDLabel = CCDcalib['CCDLabel'] + framedim = dictLT.dict_CCD[CCDLabel][0] + dict_dp={} + dict_dp['kf_direction']=default_detector_geom + dict_dp['detectorparameters']=detectorparameters + dict_dp['detectordistance']=detectorparameters[0] + dict_dp['detectordiameter']=pixelsize*framedim[0] + dict_dp['pixelsize']=pixelsize + dict_dp['dim']=framedim + dict_dp['peakX']=peakx + dict_dp['peakY']=peaky + dict_dp['intensity']=intensity + except: + print("Error in Cor file reading for "+ files) + for intmat in range(matricies): + rotation_matrix[intmat][0][cnt,:,:] = np.zeros((3,3)) + strain_matrix[intmat][0][cnt,:,:] = np.zeros((3,3)) + strain_matrixs[intmat][0][cnt,:,:] = np.zeros((3,3)) + col[intmat][0][cnt,:] = 0,0,0 + colx[intmat][0][cnt,:] = 0,0,0 + coly[intmat][0][cnt,:] = 0,0,0 + match_rate[intmat][0][cnt] = 0 + mat_global[intmat][0][cnt] = 0 + spots_len[intmat][0][cnt] = 0 + iR_pix[intmat][0][cnt] = 0 + fR_pix[intmat][0][cnt] = 0 + check[cnt,intmat] = 0 + # files_treated.append(files) + return strain_matrix, strain_matrixs, rotation_matrix, col, colx, coly, \ + match_rate, mat_global, cnt, files_treated,spots_len,iR_pix,fR_pix, check, best_match + + sorted_data = np.transpose(np.array([data_theta, data_chi])) tabledistancerandom = np.transpose(GT.calculdist_from_thetachi(sorted_data, sorted_data)) @@ -9686,27 +9725,46 @@ def predict_preprocessMultiMatProcess(files, cnt, print("Error writing the cor file", path) elif files.split(".")[-1] == "cor": - # print("Entering Cor file read section") - seednumber = "Experimental COR file" - allres = IOLT.readfile_cor(files, True) - data_theta, data_chi, peakx, peaky, intensity = allres[1:6] - CCDcalib = allres[-1] - detectorparameters = allres[-2] - # print('detectorparameters from file are: '+ str(detectorparameters)) - pixelsize = CCDcalib['pixelsize'] - CCDLabel = CCDcalib['CCDLabel'] - framedim = dictLT.dict_CCD[CCDLabel][0] - dict_dp={} - dict_dp['kf_direction']=default_detector_geom - dict_dp['detectorparameters']=detectorparameters - dict_dp['detectordistance']=detectorparameters[0] - dict_dp['detectordiameter']=pixelsize*framedim[0] - dict_dp['pixelsize']=pixelsize - dict_dp['dim']=framedim - dict_dp['peakX']=peakx - dict_dp['peakY']=peaky - dict_dp['intensity']=intensity - + try: + # print("Entering Cor file read section") + seednumber = "Experimental COR file" + allres = IOLT.readfile_cor(files, True) + data_theta, data_chi, peakx, peaky, intensity = allres[1:6] + CCDcalib = allres[-1] + detectorparameters = allres[-2] + # print('detectorparameters from file are: '+ str(detectorparameters)) + pixelsize = CCDcalib['pixelsize'] + CCDLabel = CCDcalib['CCDLabel'] + framedim = dictLT.dict_CCD[CCDLabel][0] + dict_dp={} + dict_dp['kf_direction']=default_detector_geom + dict_dp['detectorparameters']=detectorparameters + dict_dp['detectordistance']=detectorparameters[0] + dict_dp['detectordiameter']=pixelsize*framedim[0] + dict_dp['pixelsize']=pixelsize + dict_dp['dim']=framedim + dict_dp['peakX']=peakx + dict_dp['peakY']=peaky + dict_dp['intensity']=intensity + except: + print("Error in Cor file reading for "+ files) + for intmat in range(matricies): + rotation_matrix[intmat][0][cnt,:,:] = np.zeros((3,3)) + strain_matrix[intmat][0][cnt,:,:] = np.zeros((3,3)) + strain_matrixs[intmat][0][cnt,:,:] = np.zeros((3,3)) + col[intmat][0][cnt,:] = 0,0,0 + colx[intmat][0][cnt,:] = 0,0,0 + coly[intmat][0][cnt,:] = 0,0,0 + match_rate[intmat][0][cnt] = 0 + mat_global[intmat][0][cnt] = 0 + spots_len[intmat][0][cnt] = 0 + iR_pix[intmat][0][cnt] = 0 + fR_pix[intmat][0][cnt] = 0 + check[cnt,intmat] = 0 + # files_treated.append(files) + return strain_matrix, strain_matrixs, rotation_matrix, col, colx, coly, \ + match_rate, mat_global, cnt, files_treated,spots_len,iR_pix,fR_pix, check, best_match, None + sorted_data = np.transpose(np.array([data_theta, data_chi])) tabledistancerandom = np.transpose(GT.calculdist_from_thetachi(sorted_data, sorted_data))