From a61b58107dc5a44617e22df37721a1868da2d0ea Mon Sep 17 00:00:00 2001 From: Ryan Staten Date: Thu, 24 Jan 2019 13:47:22 -0500 Subject: [PATCH 1/3] add total counts per fiber metric --- py/desispec/qa/qa_quicklook.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/desispec/qa/qa_quicklook.py b/py/desispec/qa/qa_quicklook.py index e8b407a0e..1fa08c805 100644 --- a/py/desispec/qa/qa_quicklook.py +++ b/py/desispec/qa/qa_quicklook.py @@ -1147,8 +1147,9 @@ def run_qa(self,frame,inputs): threshold=[param['CUTBINS']*ii for ii in rdnoise_fib] #- compare the flux sum to threshold - passfibers=np.where(frame.flux.sum(axis=1)>threshold)[0] - ngoodfibers=passfibers.shape[0] - param["N_KNOWN_BROKEN_FIBERS"] + totcounts=frame.flux.sum(axis=1) + passfibers=np.where(totcounts>threshold)[0] + ngoodfibers=passfibers.shape[0] good_fibers=np.array([0]*frame.nspec) good_fibers[passfibers]=1 #- assign 1 for good fiber @@ -1166,7 +1167,7 @@ def run_qa(self,frame,inputs): retval["BOTTOM_MAX_WAVE_INDEX"]=int(bottommax) retval["TOP_MIN_WAVE_INDEX"]=int(topmin) - retval["METRICS"]={"NGOODFIB": ngoodfibers, "GOOD_FIBERS": good_fibers} + retval["METRICS"]={"NGOODFIB": ngoodfibers, "GOOD_FIBERS": good_fibers, "TOTCOUNT_FIB": totcounts} ############################################################### # This section is for adding QA metrics for plotting purposes # From 13ffbbeb58cc52719c24b3425f5f77b5c432499d Mon Sep 17 00:00:00 2001 From: Ryan Staten Date: Thu, 24 Jan 2019 21:59:15 -0500 Subject: [PATCH 2/3] update arc/flat specifications --- py/desispec/data/quicklook/qlconfig_arc.yaml | 10 +++++----- py/desispec/data/quicklook/qlconfig_flat.yaml | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/py/desispec/data/quicklook/qlconfig_arc.yaml b/py/desispec/data/quicklook/qlconfig_arc.yaml index 02d41e11b..7a9861f9a 100644 --- a/py/desispec/data/quicklook/qlconfig_arc.yaml +++ b/py/desispec/data/quicklook/qlconfig_arc.yaml @@ -24,19 +24,19 @@ Algorithms: Get_RMS: PARAMS: {PERCENTILES: [68.2,95.4,99.7], NOISE_AMP_NORMAL_RANGE: [-1.5, 1.5], NOISE_AMP_WARN_RANGE: [-2.0, 2.0],NOISE_AMP_REF:[2.6,2.6,2.6,2.6]} Count_Pixels: - PARAMS: {CUTPIX: 5, LITFRAC_AMP_NORMAL_RANGE: [-0.15, 0.15], LITFRAC_AMP_WARN_RANGE: [-0.2, 0.2],LITFRAC_AMP_REF:[0.01,0.01,0.01,0.01]} + PARAMS: {CUTPIX: 5, LITFRAC_AMP_NORMAL_RANGE: [-0.12, 0.12], LITFRAC_AMP_WARN_RANGE: [-0.16, 0.16],LITFRAC_AMP_REF:[0.05,0.05,0.05,0.05]} Calc_XWSigma: PARAMS: {B_PEAKS: [4047.7,4078.8,4359.6,4801.2,5087.2,5426.2], R_PEAKS: [6144.7,6404.0,6508.3,6931.4,7034.3,7247.1], Z_PEAKS: [8302.6,8379.9,8497.7,8856.3,9151.1,9668.0], PIXEL_RANGE: 7, MAX_SIGMA: 10, - XWSIGMA_NORMAL_RANGE: [-0.9, 0.9], - XWSIGMA_WARN_RANGE: [-1.2, 1.2],XWSIGMA_REF:[1.2,1.2]} + XWSIGMA_NORMAL_RANGE: [-0.45, 0.45], + XWSIGMA_WARN_RANGE: [-0.6, 0.6],XWSIGMA_REF:[1.1,1.2]} Flexure: QA: Trace_Shifts: - PARAMS: {XYSHIFTS_NORMAL_RANGE: [-0.2,0.2], XYSHIFTS_WARN_RANGE: [-0.3,0.3],XYSHIFTS_REF:[0.0,0.0]} + PARAMS: {XYSHIFTS_NORMAL_RANGE: [-0.21,0.21], XYSHIFTS_WARN_RANGE: [-0.28,0.28],XYSHIFTS_REF:[-0.07,-0.07]} Extract_QP: wavelength: { b: [3570,5730,0.8], @@ -50,4 +50,4 @@ Algorithms: NBINS: 5 QA: Check_Resolution: - PARAMS: {CHECKARC_NORMAL_RANGE: [-30.0,30.0], CHECKARC_WARN_RANGE: [-40.0,40.0],CHECKARC_REF:[500]} + PARAMS: {CHECKARC_NORMAL_RANGE: [-30.0,30.0], CHECKARC_WARN_RANGE: [-40.0,40.0],CHECKARC_REF:[480]} diff --git a/py/desispec/data/quicklook/qlconfig_flat.yaml b/py/desispec/data/quicklook/qlconfig_flat.yaml index 6c8234dd5..f00924044 100644 --- a/py/desispec/data/quicklook/qlconfig_flat.yaml +++ b/py/desispec/data/quicklook/qlconfig_flat.yaml @@ -23,11 +23,11 @@ Algorithms: Get_RMS: PARAMS: {PERCENTILES: [68.2,95.4,99.7], NOISE_AMP_NORMAL_RANGE: [-1.5, 1.5], NOISE_AMP_WARN_RANGE: [-2.0, 2.0],NOISE_AMP_REF:[2.6,2.6,2.6,2.6]} Count_Pixels: - PARAMS: {CUTPIX: 5, LITFRAC_AMP_NORMAL_RANGE: [-0.15, 0.15], LITFRAC_AMP_WARN_RANGE: [-0.2, 0.2],LITFRAC_AMP_REF:[0.8,0.8,0.8,0.8]} + PARAMS: {CUTPIX: 5, LITFRAC_AMP_NORMAL_RANGE: [-0.3, 0.3], LITFRAC_AMP_WARN_RANGE: [-0.4, 0.4],LITFRAC_AMP_REF:[0.78,0.78,0.78,0.78]} Flexure: QA: Trace_Shifts: - PARAMS: {XYSHIFTS_NORMAL_RANGE: [-0.2,0.2], XYSHIFTS_WARN_RANGE: [-0.3,0.3],XYSHIFTS_REF:[0.0,0.0]} + PARAMS: {XYSHIFTS_NORMAL_RANGE: [-0.6,0.6], XYSHIFTS_WARN_RANGE: [-0.8,0.8],XYSHIFTS_REF:[0.0,0.0]} Extract_QP: wavelength: { b: [3570,5730,0.8], @@ -40,4 +40,4 @@ Algorithms: ComputeFiberflat_QP: QA: Check_FiberFlat: - PARAMS: {CHECKFLAT_NORMAL_RANGE:[-0.3,0.3], CHECKFLAT_WARN_RANGE:[-0.4,0.4],CHECKFLAT_REF:[1.0]} + PARAMS: {CHECKFLAT_NORMAL_RANGE:[-0.01,0.01], CHECKFLAT_WARN_RANGE:[-0.02,0.02],CHECKFLAT_REF:[1.0]} From 931fc5fe444ade9c59bc8c1d012e871f0a30d48d Mon Sep 17 00:00:00 2001 From: Ryan Staten Date: Thu, 24 Jan 2019 22:09:52 -0500 Subject: [PATCH 3/3] don't output jsons for each QA --- py/desispec/qa/qa_quicklook.py | 76 +++++++++++++++---------------- py/desispec/quicklook/qlconfig.py | 12 ++--- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/py/desispec/qa/qa_quicklook.py b/py/desispec/qa/qa_quicklook.py index 1fa08c805..96fc3fa77 100644 --- a/py/desispec/qa/qa_quicklook.py +++ b/py/desispec/qa/qa_quicklook.py @@ -350,8 +350,8 @@ def run_qa(self,image,inputs): retval["PARAMS"]=param #get_outputs(qafile,qafig,retval,'plot_traceshifts') - outfile = qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# outfile = qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) return retval def get_default_config(self): @@ -475,9 +475,9 @@ def run_qa(self,image,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_bias_overscan(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -693,9 +693,9 @@ def run_qa(self,image,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_RMS(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -930,9 +930,9 @@ def run_qa(self,image,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_XWSigma(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -1040,9 +1040,9 @@ def run_qa(self,image,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_countpix(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -1175,9 +1175,9 @@ def run_qa(self,frame,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_countspectralbins(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -1282,9 +1282,9 @@ def run_qa(self,frame,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_sky_continuum(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -1428,9 +1428,9 @@ def run_qa(self,frame,inputs): zerorband=0. retval["METRICS"]={"SKYRBAND_FIB":zerospec,"SKYRBAND":zerorband} - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) return retval @@ -1536,9 +1536,9 @@ def run_qa(self,frame,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_sky_peaks(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -1645,9 +1645,9 @@ def run_qa(self,frame,skymodel,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_residuals(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -1872,9 +1872,9 @@ def run_qa(self,frame,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_integral(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -1995,9 +1995,9 @@ def run_qa(self,frame,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_SNR(retval,qafig,objlist,fitsnr,rescut,sigmacut,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) @@ -2117,9 +2117,9 @@ def run_qa(self,file_psf,inputs): ############################################################### - if qafile is not None: - outfile=qa.write_qa_ql(qafile,retval) - log.debug("Output QA data is in {}".format(outfile)) +# if qafile is not None: +# outfile=qa.write_qa_ql(qafile,retval) +# log.debug("Output QA data is in {}".format(outfile)) if qafig is not None: fig.plot_lpolyhist(retval,qafig,plotconf=plotconf,hardplots=hardplots) log.debug("Output QA fig {}".format(qafig)) diff --git a/py/desispec/quicklook/qlconfig.py b/py/desispec/quicklook/qlconfig.py index 25ad8ad5c..ab7f17648 100644 --- a/py/desispec/quicklook/qlconfig.py +++ b/py/desispec/quicklook/qlconfig.py @@ -248,19 +248,19 @@ def dump_qa(self): Name and default locations of files are handled by desispec.io.meta.findfile """ #- QA level outputs - qa_outfile = {} + #qa_outfile = {} qa_outfig = {} for PA in self.palist: for QA in self.qalist[PA]: - qa_outfile[QA] = self.io_qa(QA)[0] + #qa_outfile[QA] = self.io_qa(QA)[0] qa_outfig[QA] = self.io_qa(QA)[1] #- make path if needed - path = os.path.normpath(os.path.dirname(qa_outfile[QA])) + path = os.path.normpath(os.path.dirname(qa_outfig[QA])) if not os.path.exists(path): os.makedirs(path) - return (qa_outfile,qa_outfig) + return (qa_outfig) # return ((qa_outfile,qa_outfig),(qa_pa_outfile,qa_pa_outfig)) @property @@ -273,8 +273,8 @@ def qaargs(self): params=self._qaparams(qa) qaopts[qa]={'night' : self.night, 'expid' : self.expid, 'camera': self.camera, 'paname': PA, 'PSFFile': self.psf_filename, - 'amps': self.amps, 'qafile': self.dump_qa()[0][qa], - 'qafig': self.dump_qa()[1][qa], 'FiberMap': self.fibermap, + 'amps': self.amps, #'qafile': self.dump_qa()[0][qa], + 'qafig': self.dump_qa()[qa], 'FiberMap': self.fibermap, 'param': params, 'refKey':self._qaRefKeys[qa], 'singleqa' : self.singqa, 'plotconf':self.plotconf, 'hardplots': self.hardplots