diff --git a/caom2repo/caom2repo/core.py b/caom2repo/caom2repo/core.py index c7cec5e3..031d8980 100755 --- a/caom2repo/caom2repo/core.py +++ b/caom2repo/caom2repo/core.py @@ -67,7 +67,8 @@ # *********************************************************************** # import argparse -import imp +import importlib.util +import importlib.machinery import logging import multiprocessing from multiprocessing import Pool @@ -440,9 +441,14 @@ def _load_plugin_class(self, filepath): mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) if file_ext.lower() == '.pyc': - py_mod = imp.load_compiled(mod_name, filepath) + loader = importlib.machinery.SourcelessFileLoader(mod_name, filepath) else: - py_mod = imp.load_source(mod_name, filepath) + loader = importlib.machinery.SourceFileLoader(mod_name, filepath) + spec = importlib.util.spec_from_file_location(mod_name, filepath, loader=loader) + py_mod = importlib.util.module_from_spec(spec) + # cache the module + sys.modules[py_mod.__name__] = py_mod + spec.loader.exec_module(py_mod) if hasattr(py_mod, expected_class): self.plugin = getattr(py_mod, expected_class)() diff --git a/caom2repo/setup.cfg b/caom2repo/setup.cfg index bef1d626..c2596500 100644 --- a/caom2repo/setup.cfg +++ b/caom2repo/setup.cfg @@ -32,7 +32,7 @@ url = http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2 edit_on_github = False github_project = opencadc/caom2tools # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -version = 1.6 +version = 1.6.1 [options] install_requires = diff --git a/caom2utils/caom2utils/caom2blueprint.py b/caom2utils/caom2utils/caom2blueprint.py index eba614ac..31afcc99 100755 --- a/caom2utils/caom2utils/caom2blueprint.py +++ b/caom2utils/caom2utils/caom2blueprint.py @@ -2268,43 +2268,34 @@ def augment_artifact(self, artifact, index): 'Chunk.metaProducer', index=0, current=chunk.meta_producer) self._get_chunk_naxis(chunk, index) + + # order by which the blueprint is used to set WCS information: + # 1 - try to construct the information for an axis from WCS information + # 2 - if the WCS information is insufficient, try to construct the information from the blueprint + # 3 - Always try to fill the range metadata from the blueprint. if self.blueprint._pos_axes_configed: self._wcs_parser.augment_position(chunk) - if chunk.position is None: - self._try_position_with_blueprint(chunk, index) - if chunk.position: - chunk.position.resolution = _to_float(self._get_from_list( - 'Chunk.position.resolution', index=index, current=chunk.position.resolution)) + self._try_position_with_blueprint(chunk, index) + if self.blueprint._energy_axis_configed: self._wcs_parser.augment_energy(chunk) - if chunk.energy: - chunk.energy.bandpass_name = self._get_from_list( - 'Chunk.energy.bandpassName', index=index) - chunk.energy.transition = self._get_energy_transition( - chunk.energy.transition) - chunk.energy.resolving_power = _to_float(self._get_from_list( - 'Chunk.energy.resolvingPower', index=index)) - else: - if self.blueprint._energy_axis_configed: - self._try_energy_with_blueprint(chunk, index) + self._try_energy_with_blueprint(chunk, index) + if self.blueprint._time_axis_configed: self._wcs_parser.augment_temporal(chunk) - if chunk.time is None: - self._try_time_with_blueprint(chunk, index) + self._try_time_with_blueprint(chunk, index) + if self.blueprint._polarization_axis_configed: self._wcs_parser.augment_polarization(chunk) - if chunk.polarization is None: - self._try_polarization_with_blueprint(chunk, index) + self._try_polarization_with_blueprint(chunk, index) + if self.blueprint._obs_axis_configed: self._wcs_parser.augment_observable(chunk) - if chunk.observable is None and chunk.observable_axis is None: - self._try_observable_with_blueprint(chunk, index) + self._try_observable_with_blueprint(chunk, index) + if self.blueprint._custom_axis_configed: self._wcs_parser.augment_custom(chunk) - - # try to set smaller bits of the chunk WCS elements from the - # blueprint - self._try_range_with_blueprint(chunk, index) + self._try_custom_with_blueprint(chunk, index) self.logger.debug( f'End content artifact augmentation for {artifact.uri}.') @@ -2587,70 +2578,60 @@ def _get_metrics(self, current): self.logger.debug('End Metrics augmentation.') return metrics - def _get_naxis(self, label, index): + def _get_axis_wcs(self, label, wcs, index): """Helper function to construct a CoordAxis1D instance, with all it's members, from the blueprint. - :param label: axis name - must be one of 'energy', 'time', or - 'polarization', as it's used for the blueprint lookup. + :param label: axis name - must be one of 'custom', 'energy', 'time', or 'polarization', as it's used for the + blueprint lookup. :param index: which blueprint index to find a value in :return an instance of CoordAxis1D """ - self.logger.debug( - f'Begin {label} naxis construction from blueprint.') + self.logger.debug(f'Begin {label} axis construction from blueprint.') - aug_axis_ctype = self._get_from_list( - f'Chunk.{label}.axis.axis.ctype', index) - aug_axis_cunit = self._get_from_list( - f'Chunk.{label}.axis.axis.cunit', index) aug_axis = None - if aug_axis_ctype is not None: - aug_axis = Axis(aug_axis_ctype, aug_axis_cunit) - self.logger.debug( - f'Creating {label} Axis for {self.uri} from blueprint') - - aug_error = self._two_param_constructor( - f'Chunk.{label}.axis.error.syser', - f'Chunk.{label}.axis.error.rnder', - index, _to_float, CoordError) - aug_ref_coord = self._two_param_constructor( - f'Chunk.{label}.axis.function.refCoord.pix', - f'Chunk.{label}.axis.function.refCoord.val', - index, _to_float, RefCoord) - aug_delta = _to_float( - self._get_from_list(f'Chunk.{label}.axis.function.delta', - index)) - aug_length = _to_int( - self._get_from_list(f'Chunk.{label}.axis.function.naxis', - index)) - - aug_function = None - if (aug_length is not None and aug_delta is not None and - aug_ref_coord is not None): - aug_function = \ - CoordFunction1D(aug_length, aug_delta, aug_ref_coord) - self.logger.debug( - f'Creating {label} function for {self.uri} from blueprint') + aug_error = None + if wcs is not None and wcs.axis is not None and wcs.axis.axis is not None: + aug_axis = wcs.axis.axis + aug_error = wcs.axis.error + else: + aug_axis_ctype = self._get_from_list(f'Chunk.{label}.axis.axis.ctype', index) + aug_axis_cunit = self._get_from_list(f'Chunk.{label}.axis.axis.cunit', index) + if aug_axis_ctype is not None: + aug_axis = Axis(aug_axis_ctype, aug_axis_cunit) + self.logger.debug(f'Creating {label} Axis for {self.uri} from blueprint') + + aug_error = self._two_param_constructor( + f'Chunk.{label}.axis.error.syser', + f'Chunk.{label}.axis.error.rnder', + index, _to_float, CoordError) aug_naxis = None - if aug_function is None: - aug_range = self._try_range_return(index, label) - if aug_axis is not None and aug_range is not None: - aug_naxis = CoordAxis1D( - axis=aug_axis, error=aug_error, range=aug_range) - self.logger.debug( - f'Creating range {label} CoordAxis1D for {self.uri} from ' - f'blueprint') - else: - if aug_axis is not None and aug_function is not None: - aug_naxis = CoordAxis1D(aug_axis, aug_error, None, None, - aug_function) - self.logger.debug( - f'Creating function {label} CoordAxis1D for {self.uri} ' - f'from blueprint') - self.logger.debug( - f'End {label} naxis construction from blueprint.') - return aug_naxis + aug_range = self._try_range(index, label) + aug_naxis_index = None + if aug_axis is not None: + if aug_range is None: + if wcs is None or wcs.axis is None or wcs.axis.function is None: + aug_ref_coord = self._two_param_constructor( + f'Chunk.{label}.axis.function.refCoord.pix', + f'Chunk.{label}.axis.function.refCoord.val', + index, _to_float, RefCoord) + aug_delta = _to_float(self._get_from_list(f'Chunk.{label}.axis.function.delta', index)) + aug_length = _to_int(self._get_from_list(f'Chunk.{label}.axis.function.naxis', index)) + aug_function = None + if aug_length is not None and aug_delta is not None and aug_ref_coord is not None: + aug_function = CoordFunction1D(aug_length, aug_delta, aug_ref_coord) + aug_naxis = CoordAxis1D(aug_axis, aug_error, None, None, aug_function) + if aug_function is not None: + # if the WCS is described with a Function, cutouts can be supported, so specify an axis + aug_naxis_index = _to_int(self._get_from_list(f'Chunk.{label}Axis', index)) + self.logger.debug(f'Creating function {label} CoordAxis1D for {self.uri} from blueprint') + else: + aug_naxis = CoordAxis1D(axis=aug_axis, error=aug_error, range=aug_range) + self.logger.debug(f'Creating range {label} CoordAxis1D for {self.uri} from blueprint') + + self.logger.debug(f'End {label} axis construction from blueprint.') + return aug_naxis, aug_naxis_index def _get_observable(self, current): """ @@ -2887,6 +2868,25 @@ def _cast_as_bool(self, from_value): result = True return result + def _try_custom_with_blueprint(self, chunk, index): + """ + A mechanism to augment the Custom WCS completely from the blueprint. Do nothing if the WCS information cannot + be correctly created. + + :param chunk: The chunk to modify with the addition of custom information. + :param index: The index in the blueprint for looking up plan information. + """ + self.logger.debug('Begin augmentation with blueprint for custom.') + aug_naxis, aug_naxis_index = self._get_axis_wcs('custom', chunk.custom, index) + if aug_naxis is None: + self.logger.debug('No blueprint custom information.') + else: + # always create a new CustomWCS instance because there's no setter for 'axis' parameter + chunk.custom = CustomWCS(aug_naxis) + chunk.custom_axis = aug_naxis_index + self.logger.debug(f'Updating CustomWCS for {self.uri}.') + self.logger.debug('End augmentation with blueprint for custom.') + def _try_energy_with_blueprint(self, chunk, index): """ A mechanism to augment the Energy WCS completely from the blueprint. @@ -2898,39 +2898,32 @@ def _try_energy_with_blueprint(self, chunk, index): information. """ self.logger.debug('Begin augmentation with blueprint for energy.') - aug_naxis = self._get_naxis('energy', index) - + aug_axis, aug_naxis_index = self._get_axis_wcs('energy', chunk.energy, index) specsys = _to_str(self._get_from_list('Chunk.energy.specsys', index)) - if aug_naxis is None: + if aug_axis is None: self.logger.debug('No blueprint energy information.') else: - if not chunk.energy: - chunk.energy = SpectralWCS(aug_naxis, specsys) - else: - chunk.energy.naxis = aug_naxis + if chunk.energy: + chunk.energy.axis = aug_axis chunk.energy.specsys = specsys + else: + chunk.energy = SpectralWCS(aug_axis, specsys) + self.logger.debug(f'Creating SpectralWCS for {self.uri} from blueprint') + chunk.energy_axis = aug_naxis_index - if chunk.energy is not None: - chunk.energy.ssysobs = self._get_from_list( - 'Chunk.energy.ssysobs', index) - chunk.energy.restfrq = self._get_from_list( - 'Chunk.energy.restfrq', index) - chunk.energy.restwav = self._get_from_list( - 'Chunk.energy.restwav', index) - chunk.energy.velosys = self._get_from_list( - 'Chunk.energy.velosys', index) - chunk.energy.zsource = self._get_from_list( - 'Chunk.energy.zsource', index) - chunk.energy.ssyssrc = self._get_from_list( - 'Chunk.energy.ssyssrc', index) - chunk.energy.velang = self._get_from_list( - 'Chunk.energy.velang', index) - chunk.energy.bandpass_name = self._get_from_list( - 'Chunk.energy.bandpassName', index) - chunk.energy.transition = self._get_from_list( - 'Chunk.energy.transition', index) - chunk.energy.resolving_power = _to_float(self._get_from_list( - 'Chunk.energy.resolvingPower', index)) + if chunk.energy: + chunk.energy.ssysobs = self._get_from_list('Chunk.energy.ssysobs', index, chunk.energy.ssysobs) + chunk.energy.restfrq = self._get_from_list('Chunk.energy.restfrq', index, chunk.energy.restfrq) + chunk.energy.restwav = self._get_from_list('Chunk.energy.restwav', index, chunk.energy.restwav) + chunk.energy.velosys = self._get_from_list('Chunk.energy.velosys', index, chunk.energy.velosys) + chunk.energy.zsource = self._get_from_list('Chunk.energy.zsource', index, chunk.energy.zsource) + chunk.energy.ssyssrc = self._get_from_list('Chunk.energy.ssyssrc', index, chunk.energy.ssyssrc) + chunk.energy.velang = self._get_from_list('Chunk.energy.velang', index, chunk.energy.velang) + chunk.energy.bandpass_name = self._get_from_list( + 'Chunk.energy.bandpassName', index, chunk.energy.bandpass_name) + chunk.energy.transition = self._get_energy_transition(chunk.energy.transition) + chunk.energy.resolving_power = _to_float( + self._get_from_list('Chunk.energy.resolvingPower', index, chunk.energy.resolving_power)) self.logger.debug('End augmentation with blueprint for energy.') def _try_observable_with_blueprint(self, chunk, index): @@ -2946,8 +2939,6 @@ def _try_observable_with_blueprint(self, chunk, index): """ self.logger.debug('Begin augmentation with blueprint for ' 'observable.') - chunk.observable_axis = _to_int( - self._get_from_list('Chunk.observableAxis', index)) aug_axis = self._two_param_constructor( 'Chunk.observable.dependent.axis.ctype', 'Chunk.observable.dependent.axis.cunit', index, _to_str, Axis) @@ -2955,6 +2946,7 @@ def _try_observable_with_blueprint(self, chunk, index): self._get_from_list('Chunk.observable.dependent.bin', index)) if aug_axis is not None and aug_bin is not None: chunk.observable = ObservableAxis(Slice(aug_axis, aug_bin)) + chunk.observable_axis = _to_int(self._get_from_list('Chunk.observableAxis', index)) self.logger.debug('End augmentation with blueprint for polarization.') def _try_polarization_with_blueprint(self, chunk, index): @@ -2970,45 +2962,43 @@ def _try_polarization_with_blueprint(self, chunk, index): """ self.logger.debug('Begin augmentation with blueprint for ' 'polarization.') - chunk.polarization_axis = _to_int( - self._get_from_list('Chunk.polarizationAxis', index)) - aug_naxis = self._get_naxis('polarization', index) - if aug_naxis is not None: + aug_axis, aug_naxis_index = self._get_axis_wcs('polarization', chunk.polarization, index) + if aug_axis is not None: if chunk.polarization: - chunk.polarization.naxis = aug_naxis + chunk.polarization.axis = aug_axis else: - chunk.polarization = PolarizationWCS(aug_naxis) - self.logger.debug( - f'Creating PolarizationWCS for {self.uri} from blueprint') + chunk.polarization = PolarizationWCS(aug_axis) + self.logger.debug(f'Creating PolarizationWCS for {self.uri} from blueprint') + chunk.polarization_axis = aug_naxis_index self.logger.debug('End augmentation with blueprint for polarization.') - def _try_position_range(self, chunk, index): + def _try_position_range(self, index): self.logger.debug('Try to set the range for position from blueprint, since there is no function') - if (self.blueprint._pos_axes_configed and chunk.position is not None - and chunk.position.axis is not None and chunk.position.axis.function is None): - aug_range_c1_start = self._two_param_constructor( - 'Chunk.position.axis.range.start.coord1.pix', - 'Chunk.position.axis.range.start.coord1.val', - index, _to_float, RefCoord) - aug_range_c1_end = self._two_param_constructor( - 'Chunk.position.axis.range.end.coord1.pix', - 'Chunk.position.axis.range.end.coord1.val', - index, _to_float, RefCoord) - aug_range_c2_start = self._two_param_constructor( - 'Chunk.position.axis.range.start.coord2.pix', - 'Chunk.position.axis.range.start.coord2.val', - index, _to_float, RefCoord) - aug_range_c2_end = self._two_param_constructor( - 'Chunk.position.axis.range.end.coord2.pix', - 'Chunk.position.axis.range.end.coord2.val', - index, _to_float, RefCoord) - if (aug_range_c1_start and aug_range_c1_end and aug_range_c2_start - and aug_range_c2_end): - chunk.position.axis.range = CoordRange2D( - Coord2D(aug_range_c1_start, aug_range_c1_end), - Coord2D(aug_range_c2_start, aug_range_c2_end)) - self.logger.debug('Completed setting range for position') + aug_range = None + aug_range_c1_start = self._two_param_constructor( + 'Chunk.position.axis.range.start.coord1.pix', + 'Chunk.position.axis.range.start.coord1.val', + index, _to_float, RefCoord) + aug_range_c1_end = self._two_param_constructor( + 'Chunk.position.axis.range.end.coord1.pix', + 'Chunk.position.axis.range.end.coord1.val', + index, _to_float, RefCoord) + aug_range_c2_start = self._two_param_constructor( + 'Chunk.position.axis.range.start.coord2.pix', + 'Chunk.position.axis.range.start.coord2.val', + index, _to_float, RefCoord) + aug_range_c2_end = self._two_param_constructor( + 'Chunk.position.axis.range.end.coord2.pix', + 'Chunk.position.axis.range.end.coord2.val', + index, _to_float, RefCoord) + if (aug_range_c1_start and aug_range_c1_end and aug_range_c2_start + and aug_range_c2_end): + aug_range = CoordRange2D( + Coord2D(aug_range_c1_start, aug_range_c1_end), + Coord2D(aug_range_c2_start, aug_range_c2_end)) + self.logger.debug('Completed setting range for position') + return aug_range def _try_position_with_blueprint(self, chunk, index): """ @@ -3021,94 +3011,87 @@ def _try_position_with_blueprint(self, chunk, index): information. """ self.logger.debug('Begin augmentation with blueprint for position.') - - aug_x_axis = self._two_param_constructor( - 'Chunk.position.axis.axis1.ctype', - 'Chunk.position.axis.axis1.cunit', index, _to_str, Axis) - aug_y_axis = self._two_param_constructor( - 'Chunk.position.axis.axis2.ctype', - 'Chunk.position.axis.axis2.cunit', index, _to_str, Axis) - aug_x_error = self._two_param_constructor( - 'Chunk.position.axis.error1.syser', - 'Chunk.position.axis.error1.rnder', index, _to_float, CoordError) - aug_y_error = self._two_param_constructor( - 'Chunk.position.axis.error2.syser', - 'Chunk.position.axis.error2.rnder', index, _to_float, CoordError) - aug_dimension = self._two_param_constructor( - 'Chunk.position.axis.function.dimension.naxis1', - 'Chunk.position.axis.function.dimension.naxis2', - index, _to_int, Dimension2D) - aug_x_ref_coord = self._two_param_constructor( - 'Chunk.position.axis.function.refCoord.coord1.pix', - 'Chunk.position.axis.function.refCoord.coord1.val', - index, _to_float, RefCoord) - aug_y_ref_coord = self._two_param_constructor( - 'Chunk.position.axis.function.refCoord.coord2.pix', - 'Chunk.position.axis.function.refCoord.coord2.val', - index, _to_float, RefCoord) - aug_cd11 = _to_float(self._get_from_list( - 'Chunk.position.axis.function.cd11', index)) - aug_cd12 = _to_float(self._get_from_list( - 'Chunk.position.axis.function.cd12', index)) - aug_cd21 = _to_float(self._get_from_list( - 'Chunk.position.axis.function.cd21', index)) - aug_cd22 = _to_float(self._get_from_list( - 'Chunk.position.axis.function.cd22', index)) - - aug_ref_coord = None - if aug_x_ref_coord is not None and aug_y_ref_coord is not None: - aug_ref_coord = Coord2D(aug_x_ref_coord, aug_y_ref_coord) - self.logger.debug( - f'Creating position Coord2D for {self.uri}') - - aug_function = None - if (aug_dimension is not None and aug_ref_coord is not None and - aug_cd11 is not None and aug_cd12 is not None and - aug_cd21 is not None and aug_cd22 is not None): - aug_function = CoordFunction2D(aug_dimension, aug_ref_coord, - aug_cd11, aug_cd12, aug_cd21, - aug_cd22) - self.logger.debug( - f'Creating position CoordFunction2D for {self.uri}') - aug_axis = None - if (aug_x_axis is not None and aug_y_axis is not None and - aug_function is not None): - aug_axis = CoordAxis2D(aug_x_axis, aug_y_axis, aug_x_error, - aug_y_error, None, None, aug_function) - self.logger.debug( - f'Creating position CoordAxis2D for {self.uri}') + if (chunk.position is not None and chunk.position.axis is not None and chunk.position.axis.axis1 is not None + and chunk.position.axis.axis2 is not None): + # preserve the values obtained from file data + aug_x_axis = chunk.position.axis.axis1 + aug_y_axis = chunk.position.axis.axis2 + aug_x_error = chunk.position.axis.error1 + aug_y_error = chunk.position.axis.error2 + else: + aug_x_axis = self._two_param_constructor( + 'Chunk.position.axis.axis1.ctype', + 'Chunk.position.axis.axis1.cunit', index, _to_str, Axis) + aug_y_axis = self._two_param_constructor( + 'Chunk.position.axis.axis2.ctype', + 'Chunk.position.axis.axis2.cunit', index, _to_str, Axis) + aug_x_error = self._two_param_constructor( + 'Chunk.position.axis.error1.syser', + 'Chunk.position.axis.error1.rnder', index, _to_float, CoordError) + aug_y_error = self._two_param_constructor( + 'Chunk.position.axis.error2.syser', + 'Chunk.position.axis.error2.rnder', index, _to_float, CoordError) + aug_range = self._try_position_range(index) + if aug_range is None: + if chunk.position is None or chunk.position.axis is None or chunk.position.axis.function is None: + aug_dimension = self._two_param_constructor( + 'Chunk.position.axis.function.dimension.naxis1', + 'Chunk.position.axis.function.dimension.naxis2', + index, _to_int, Dimension2D) + aug_x_ref_coord = self._two_param_constructor( + 'Chunk.position.axis.function.refCoord.coord1.pix', + 'Chunk.position.axis.function.refCoord.coord1.val', + index, _to_float, RefCoord) + aug_y_ref_coord = self._two_param_constructor( + 'Chunk.position.axis.function.refCoord.coord2.pix', + 'Chunk.position.axis.function.refCoord.coord2.val', + index, _to_float, RefCoord) + aug_cd11 = _to_float(self._get_from_list('Chunk.position.axis.function.cd11', index)) + aug_cd12 = _to_float(self._get_from_list('Chunk.position.axis.function.cd12', index)) + aug_cd21 = _to_float(self._get_from_list('Chunk.position.axis.function.cd21', index)) + aug_cd22 = _to_float(self._get_from_list('Chunk.position.axis.function.cd22', index)) + + aug_ref_coord = None + if aug_x_ref_coord is not None and aug_y_ref_coord is not None: + aug_ref_coord = Coord2D(aug_x_ref_coord, aug_y_ref_coord) + self.logger.debug(f'Creating position Coord2D for {self.uri}') + + aug_function = None + if (aug_dimension is not None and aug_ref_coord is not None and + aug_cd11 is not None and aug_cd12 is not None and + aug_cd21 is not None and aug_cd22 is not None): + aug_function = CoordFunction2D(aug_dimension, aug_ref_coord, aug_cd11, aug_cd12, aug_cd21, + aug_cd22) + self.logger.debug(f'Creating position CoordFunction2D for {self.uri}') + + if (aug_x_axis is not None and aug_y_axis is not None and + aug_function is not None): + aug_axis = CoordAxis2D(aug_x_axis, aug_y_axis, aug_x_error, + aug_y_error, None, None, aug_function) + self.logger.debug(f'Creating position CoordAxis2D for {self.uri}') + + chunk.position_axis_1 = _to_int(self._get_from_list('Chunk.positionAxis1', index)) + chunk.position_axis_2 = _to_int(self._get_from_list('Chunk.positionAxis2', index)) + else: + aug_axis = CoordAxis2D(aug_x_axis, aug_y_axis, aug_x_error, aug_y_error, range=aug_range) if aug_axis is not None: if chunk.position: chunk.position.axis = aug_axis else: chunk.position = SpatialWCS(aug_axis) + self.logger.debug(f'Creating SpatialWCS for {self.uri} from blueprint') if chunk.position: - chunk.position.coordsys = self._get_from_list( - 'Chunk.position.coordsys', index) + chunk.position.coordsys = self._get_from_list('Chunk.position.coordsys', index, chunk.position.coordsys) chunk.position.equinox = _to_float(self._get_from_list( - 'Chunk.position.equinox', index)) + 'Chunk.position.equinox', index, chunk.position.equinox)) chunk.position.resolution = self._get_from_list( - 'Chunk.position.resolution', index) + 'Chunk.position.resolution', index, chunk.position.resolution) self.logger.debug('End augmentation with blueprint for position.') - def _try_range(self, wcs, index, lookup): - self.logger.debug(f'Try to set the range for {lookup}') - aug_range_start = self._two_param_constructor( - f'Chunk.{lookup}.axis.range.start.pix', - f'Chunk.{lookup}.axis.range.start.val', - index, _to_float, RefCoord) - aug_range_end = self._two_param_constructor( - f'Chunk.{lookup}.axis.range.end.pix', - f'Chunk.{lookup}.axis.range.end.val', - index, _to_float, RefCoord) - if aug_range_start and aug_range_end: - wcs.axis.range = CoordRange1D(aug_range_start, aug_range_end) - self.logger.debug(f'Completed setting range for {lookup}') - - def _try_range_return(self, index, lookup): + def _try_range(self, index, lookup): self.logger.debug(f'Try to set the range for {lookup}') result = None aug_range_start = self._two_param_constructor( @@ -3124,23 +3107,6 @@ def _try_range_return(self, index, lookup): self.logger.debug(f'Completed setting range with return for {lookup}') return result - def _try_range_with_blueprint(self, chunk, index): - """Use the blueprint to set elements and attributes that - are not in the scope of astropy and files content, and therefore are - not covered by the *WcsParser classes. Per PD 19/04/18, bounds and - range are not covered by WCS keywords.""" - - for i in ['energy', 'time', 'polarization']: - axis_configed = getattr(self.blueprint, - f'_{i}_axis_configed') - if axis_configed: - wcs = getattr(chunk, i) - if wcs is not None and wcs.axis is not None: - # only try to set the Range information if the Function doesn't exist - if wcs.axis.range is None and wcs.axis.function is None: - self._try_range(wcs, index, i) - self._try_position_range(chunk, index) - def _try_time_with_blueprint(self, chunk, index): """ A mechanism to augment the Time WCS completely from the blueprint. @@ -3153,25 +3119,22 @@ def _try_time_with_blueprint(self, chunk, index): """ self.logger.debug('Begin augmentation with blueprint for temporal.') - chunk.time_axis = _to_int(self._get_from_list('Chunk.timeAxis', index)) - aug_naxis = self._get_naxis('time', index) - if aug_naxis is not None: + aug_axis, aug_axis_index = self._get_axis_wcs('time', chunk.time, index) + if aug_axis is not None: if chunk.time: - chunk.time.naxis = aug_naxis + chunk.time.axis = aug_axis else: - chunk.time = TemporalWCS(aug_naxis) - self.logger.debug('Creating TemporalWCS for {} from blueprint'. - format(self.uri)) - if chunk.time is not None: - chunk.time.exposure = _to_float( - self._get_from_list('Chunk.time.exposure', index)) + chunk.time = TemporalWCS(aug_axis) + self.logger.debug(f'Creating TemporalWCS for {self.uri} from blueprint') + chunk.time_axis = aug_axis_index + + if chunk.time: + chunk.time.exposure = _to_float(self._get_from_list('Chunk.time.exposure', index, chunk.time.exposure)) chunk.time.resolution = _to_float( - self._get_from_list('Chunk.time.resolution', index)) - chunk.time.timesys = _to_str( - self._get_from_list('Chunk.time.timesys', index)) - chunk.time.trefpos = self._get_from_list('Chunk.time.trefpos', - index) - chunk.time.mjdref = self._get_from_list('Chunk.time.mjdref', index) + self._get_from_list('Chunk.time.resolution', index, chunk.time.resolution)) + chunk.time.timesys = _to_str(self._get_from_list('Chunk.time.timesys', index, chunk.time.timesys)) + chunk.time.trefpos = self._get_from_list('Chunk.time.trefpos', index, chunk.time.trefpos) + chunk.time.mjdref = self._get_from_list('Chunk.time.mjdref', index, chunk.time.mjdref) self.logger.debug('End augmentation with blueprint for temporal.') @@ -3958,7 +3921,9 @@ def augment_custom(self, chunk): delta = self.wcs.cd[custom_axis_index][custom_axis_index] else: delta = self.wcs.cdelt[custom_axis_index] - naxis.function = CoordFunction1D(custom_axis_length, delta, self._get_ref_coord(custom_axis_index)) + ref_coord = self._get_ref_coord(custom_axis_index) + if delta and ref_coord: + naxis.function = CoordFunction1D(custom_axis_length, delta, ref_coord) if not chunk.custom: chunk.custom = CustomWCS(naxis) else: @@ -3995,7 +3960,9 @@ def augment_energy(self, chunk): delta = self.wcs.cd[energy_axis_index][energy_axis_index] else: delta = self.wcs.cdelt[energy_axis_index] - naxis.function = CoordFunction1D(energy_axis_length, delta, self._get_ref_coord(energy_axis_index)) + ref_coord = self._get_ref_coord(energy_axis_index) + if delta and ref_coord: + naxis.function = CoordFunction1D(energy_axis_length, delta, ref_coord) specsys = _to_str(self.wcs.specsys) if not chunk.energy: @@ -4128,7 +4095,9 @@ def augment_polarization(self, chunk): delta = self.wcs.cd[polarization_axis_index][polarization_axis_index] else: delta = self.wcs.cdelt[polarization_axis_index] - naxis.function = CoordFunction1D(axis_length, delta, self._get_ref_coord(polarization_axis_index)) + ref_coord = self._get_ref_coord(polarization_axis_index) + if delta and ref_coord: + naxis.function = CoordFunction1D(axis_length, delta, ref_coord) if not chunk.polarization: chunk.polarization = PolarizationWCS(naxis) else: @@ -4274,8 +4243,11 @@ def _get_spatial_axis(self, xindex, yindex): if aug_dimension is None: return None - aug_ref_coord = Coord2D(self._get_ref_coord(xindex), - self._get_ref_coord(yindex)) + x_ref_coord = self._get_ref_coord(xindex) + y_ref_coord = self._get_ref_coord(yindex) + aug_ref_coord = None + if x_ref_coord and y_ref_coord: + aug_ref_coord = Coord2D(x_ref_coord, y_ref_coord) aug_cd11, aug_cd12, aug_cd21, aug_cd22 = \ self._get_cd(xindex, yindex) @@ -4611,7 +4583,8 @@ def _finish_chunk_position(self, chunk): if chunk.position.resolution is None: try: # JJK 30-01-23 - # In a spatial data chunk the resolution is 2 times the pixel size. We can get the pixel size from the wcs + # In a spatial data chunk the resolution is 2 times the pixel size. We can get the pixel size from + # the wcs temp = utils.proj_plane_pixel_scales(self._wcs) chunk.position.resolution = temp[0] except SingularMatrixError as e: @@ -4678,7 +4651,6 @@ def _finish_time(self): self._wcs.wcs.trefpos = x x = self._blueprint._get('Chunk.time.mjdref', self._extension) if x and not ObsBlueprint.needs_lookup(x): -# logging.error(f'{x} {self._wcs.wcs.mjdref}') self._wcs.wcs.mjdref = [x, x] diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.blueprint b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.blueprint new file mode 100644 index 00000000..2a0c672d --- /dev/null +++ b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.blueprint @@ -0,0 +1,93 @@ +Observation.observationID = ['OBSID'], default = None +Observation.type = object +Observation.intent = science +Observation.metaRelease = 2023-10-31T00:45:40.102340 +Observation.metaProducer = brite2caom2/0.1.1 +Observation.algorithm.name = exposure +Observation.instrument.name = UniBRITE +Observation.instrument.keywords = None +Observation.target.name = HD37043 +Observation.target.standard = False +Observation.telescope.geoLocationX = None +Observation.telescope.geoLocationY = None +Observation.telescope.geoLocationZ = None +Observation.telescope.name = UniBRITE +Observation.proposal.id = None +Observation.environment.ambientTemp = None +Plane.productID = timeseries +Plane.metaRelease = 2023-10-31T00:45:40.102340 +Plane.dataRelease = 2023-10-31T00:45:40.102340 +Plane.dataProductType = timeseries +Plane.calibrationLevel = 2 +Plane.metaProducer = brite2caom2/0.1.1 +Plane.provenance.name = BRITE Specific Aperture Photometry - chopping mode +Plane.provenance.version = APa3s2chop +Plane.provenance.project = BRITE-Constellation Nano-Satellites for Astrophysics +Plane.provenance.producer = Adam Popowicz +Plane.provenance.runID = R4 +Plane.provenance.reference = http://brite-wiki.astro.uni.wroc.pl/bwiki/doku.php?id=start +Plane.provenance.lastExecuted = 2023-12-31T00:45:40 +Artifact.productType = _get_artifact_product_type(uri) +Artifact.releaseType = data +Artifact.metaProducer = brite2caom2/0.1.1 +Chunk = include +Chunk.metaProducer = brite2caom2/0.1.1 +Chunk.position.coordsys = ICRS +Chunk.position.equinox = None +Chunk.position.axis.axis1.ctype = RA---TAN +Chunk.position.axis.axis1.cunit = deg +Chunk.position.axis.axis2.ctype = DEC--TAN +Chunk.position.axis.axis2.cunit = deg +Chunk.position.axis.error1.syser = None +Chunk.position.axis.error1.rnder = None +Chunk.position.axis.error2.syser = None +Chunk.position.axis.error2.rnder = None +Chunk.position.axis.function.cd11 = 0.0375 +Chunk.position.axis.function.cd12 = 0.0 +Chunk.position.axis.function.cd21 = 0.0 +Chunk.position.axis.function.cd22 = 0.0375 +Chunk.position.axis.function.dimension.naxis1 = 1 +Chunk.position.axis.function.dimension.naxis2 = 1 +Chunk.position.axis.function.refCoord.coord1.pix = 1.0 +Chunk.position.axis.function.refCoord.coord1.val = 83.85825794708 +Chunk.position.axis.function.refCoord.coord2.pix = 1.0 +Chunk.position.axis.function.refCoord.coord2.val = -5.9099009825 +Chunk.energy.specsys = TOPOCENT +Chunk.energy.ssysobs = None +Chunk.energy.restfrq = None +Chunk.energy.restwav = None +Chunk.energy.velosys = None +Chunk.energy.zsource = None +Chunk.energy.ssyssrc = None +Chunk.energy.velang = None +Chunk.energy.bandpassName = Red +Chunk.energy.resolvingPower = 3.8750000000000018 +Chunk.energy.axis.axis.ctype = WAVE +Chunk.energy.axis.axis.cunit = m +Chunk.energy.axis.error.syser = None +Chunk.energy.axis.error.rnder = None +Chunk.energy.axis.function.naxis = None +Chunk.energy.axis.function.delta = None +Chunk.energy.axis.function.refCoord.pix = None +Chunk.energy.axis.function.refCoord.val = None +Chunk.energy.axis.range.start.pix = 0.5 +Chunk.energy.axis.range.start.val = 5.4e-07 +Chunk.energy.axis.range.end.pix = 1.5 +Chunk.energy.axis.range.end.val = 7e-07 +Chunk.time.exposure = 1.0 +Chunk.time.resolution = None +Chunk.time.timesys = UTC +Chunk.time.trefpos = None +Chunk.time.mjdref = None +Chunk.time.axis.axis.ctype = TIME +Chunk.time.axis.axis.cunit = d +Chunk.time.axis.error.syser = None +Chunk.time.axis.error.rnder = None +Chunk.time.axis.function.naxis = None +Chunk.time.axis.function.delta = None +Chunk.time.axis.function.refCoord.pix = None +Chunk.time.axis.function.refCoord.val = None +Chunk.time.axis.range.start.pix = 0.5 +Chunk.time.axis.range.start.val = _get_time_axis_range_start_val(uri) +Chunk.time.axis.range.end.pix = 1.5 +Chunk.time.axis.range.end.val = _get_time_axis_range_end_val(uri) \ No newline at end of file diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.module b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.module new file mode 100644 index 00000000..243fcc91 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.module @@ -0,0 +1,13 @@ +from caom2 import ProductType + + +def _get_artifact_product_type(uri): + return ProductType.SCIENCE + + +def _get_time_axis_range_end_val(uri): + return 59640.65652099997 + + +def _get_time_axis_range_start_val(uri): + return 59468.76826100005 diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.orig.header b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.orig.header new file mode 100644 index 00000000..e69de29b diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.py b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.py new file mode 100644 index 00000000..243fcc91 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.py @@ -0,0 +1,13 @@ +from caom2 import ProductType + + +def _get_artifact_product_type(uri): + return ProductType.SCIENCE + + +def _get_time_axis_range_end_val(uri): + return 59640.65652099997 + + +def _get_time_axis_range_start_val(uri): + return 59468.76826100005 diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.xml b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.xml new file mode 100644 index 00000000..0ceef5e9 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.xml @@ -0,0 +1,134 @@ + + + BRITE-Constellation + HD36486_65-Ori-VIII-2021_BAb_1_5_A + 2023-10-31T00:45:40.102 + + exposure + + object + science + + HD37043 + + + UniBRITE + + + UniBRITE + + + + HD36486_65-Ori-VIII-2021_BAb_1_5_A + 2023-10-31T00:45:40.102 + 2023-10-31T00:45:40.102 + timeseries + 2 + + BRITE Specific Aperture Photometry - chopping mode + APa3s2chop + BRITE-Constellation Nano-Satellites for Astrophysics + Adam Popowicz<Adam.Popowicz@polsl.pl> + R4 + http://brite-wiki.astro.uni.wroc.pl/bwiki/doku.php?id=start + 2023-12-31T00:45:40.000 + + + + ad:BRITE-Constellation/HD36486.orig + science + data + text/plain + 1600290 + md5:4d3e7821c9ac7167349f999feec20536 + + + 0 + + + 4 + 1 + 2 + + + + RA---TAN + deg + + + DEC--TAN + deg + + + + 1 + 1 + + + + 1.0 + 83.85825794708 + + + 1.0 + -5.9099009825 + + + 0.0375 + 0.0 + 0.0 + 0.0375 + + + ICRS + + + + + WAVE + m + + + + 0.5 + 5.4e-07 + + + 1.5 + 7e-07 + + + + TOPOCENT + Red + 3.8750000000000018 + + + + + TIME + d + + + + 0.5 + 59468.76826100005 + + + 1.5 + 59640.65652099997 + + + + UTC + 1.0 + + + + + + + + + + diff --git a/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.blueprint b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.blueprint new file mode 100644 index 00000000..01a66474 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.blueprint @@ -0,0 +1,63 @@ +Observation.observationID = ['OBSID'], default = None +Observation.type = OBJECT +Observation.intent = science +Observation.metaRelease = 2003-03-25T06:01:59.700 +Observation.metaProducer = gemini2caom2/0.0.0 +Observation.algorithm.name = exposure +Observation.instrument.name = NIRI +Observation.instrument.keywords = ['INSTMODE'], default = None +Observation.telescope.name = ['TELESCOP'], default = None +Observation.telescope.geoLocationX = -5464284.386715498 +Observation.telescope.geoLocationY = -2493782.309107667 +Observation.telescope.geoLocationZ = 2150786.380689657 +Observation.environment.ambientTemp = ['TEMPERAT'], default = None +Plane.productID = N20030325S0098 +Plane.metaRelease = 2003-03-25T06:01:59.700 +Plane.dataRelease = 2004-09-25T00:00:00.000 +Plane.dataProductType = image +Plane.calibrationLevel = 1 +Plane.metaProducer = gemini2caom2/0.0.0 +Plane.provenance.name = Gemini Observatory Data +Plane.provenance.project = Gemini Archive +Plane.provenance.producer = ['ORIGIN'], default = Gemini Observatory +Plane.provenance.reference = http://archive.gemini.edu/searchform/GN-2003A-Q-51-2-004 +Plane.provenance.lastExecuted = ['DATE-FTS'], default = None +Artifact.productType = science +Artifact.releaseType = data +Artifact.uri = gemini:GEMINI/N20030325S0098.fits +Artifact.metaProducer = gemini2caom2/0.0.0 +Chunk = include +Chunk.metaProducer = gemini2caom2/0.0.0 +Chunk.position.coordsys = ['RADESYS'], default = None +Chunk.position.equinox = ['EPOCH'], default = 2000.0 +Chunk.position.axis.axis1.ctype = ['CTYPE1'], default = None +Chunk.position.axis.axis1.cunit = ['CUNIT1'], default = None +Chunk.position.axis.axis2.ctype = ['CTYPE2'], default = None +Chunk.position.axis.axis2.cunit = ['CUNIT2'], default = None +Chunk.position.axis.error1.syser = ['CSYER1'], default = None +Chunk.position.axis.error1.rnder = ['CRDER1'], default = None +Chunk.position.axis.error2.syser = ['CSYER2'], default = None +Chunk.position.axis.error2.rnder = ['CRDER2'], default = None +Chunk.position.axis.function.cd11 = ['CD1_1'], default = None +Chunk.position.axis.function.cd12 = ['CD1_2'], default = None +Chunk.position.axis.function.cd21 = ['CD2_1'], default = None +Chunk.position.axis.function.cd22 = ['CD2_2'], default = None +Chunk.position.axis.function.dimension.naxis1 = ['NAXIS1'], default = None +Chunk.position.axis.function.dimension.naxis2 = ['NAXIS2'], default = None +Chunk.position.axis.function.refCoord.coord1.pix = ['CRPIX1'], default = None +Chunk.position.axis.function.refCoord.coord1.val = ['CRVAL1'], default = None +Chunk.position.axis.function.refCoord.coord2.pix = ['CRPIX2'], default = None +Chunk.position.axis.function.refCoord.coord2.val = ['CRVAL2'], default = None +Chunk.time.exposure = get_exposure(uri) +Chunk.time.resolution = get_exposure(uri) +Chunk.time.timesys = ['TIMESYS'], default = None +Chunk.time.trefpos = ['TREFPOS'], default = None +Chunk.time.mjdref = ['MJDREF'], default = None +Chunk.time.axis.axis.ctype = TIME +Chunk.time.axis.axis.cunit = d +Chunk.time.axis.error.syser = 1e-07 +Chunk.time.axis.error.rnder = 1e-07 +Chunk.time.axis.function.naxis = 1 +Chunk.time.axis.function.delta = get_time_delta() +Chunk.time.axis.function.refCoord.pix = 0.5 +Chunk.time.axis.function.refCoord.val = 52723.25138541667 diff --git a/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.expected.xml b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.expected.xml new file mode 100644 index 00000000..d952d774 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.expected.xml @@ -0,0 +1,122 @@ + + + GEMINI + GN-2003A-Q-51-2-004 + 2003-03-25T06:01:59.700 + + exposure + + OBJECT + science + + B2 0902+34--NE + + + Gemini-North + -5464284.386715498 + -2493782.3091076654 + 2150786.380689657 + + + NIRI + + + + GN-2003A-Q-51-2-004 + 2003-03-25T06:01:59.700 + 2004-09-25T00:00:00.000 + image + 1 + + Gemini Observatory Data + Gemini Archive + NOAO-IRAF FITS Image Kernel July 1999 + http://archive.gemini.edu/searchform/GN-2003A-Q-51-2-004 + + + + ad:GEMINI/N20030325S0098.fits + science + data + application/fits + 4210560 + md5:e385beb60c74bccf297813c26e35fa9b + + + 0 + + + + 1 + + + 2 + 1 + 2 + 4 + + + + RA---TAN + deg + + + DEC--TAN + deg + + + + 1024 + 1024 + + + + 545.1860865851 + 136.37590805622 + + + 489.04922960385 + 34.130334909061 + + + -3.2524270027565e-55 + 1.2869585606175e-07 + -8.9987106577858e-09 + 3.2335265472292e-05 + + + FK5 + 2000.0 + + + + + TIME + d + + + 1e-07 + 1e-07 + + + 1 + 0.000520868055555555 + + 0.5 + 52723.25138541667 + + + + UTC + 45.003 + 45.003 + + + + + + + + + + diff --git a/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.fits.header b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.fits.header new file mode 100644 index 00000000..351b2fa5 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.fits.header @@ -0,0 +1,176 @@ +Filename: N20030325S0098.fits.bz2 + +AstroData Tags: {'UNPREPARED', 'SIDEREAL', 'RAW', 'NIRI', 'IMAGE', 'GEMINI', 'NORTH'} + + +--- PHU --- +SIMPLE = T / Fits standard +BITPIX = 16 / Bits per pixel +NAXIS = 0 / Number of axes +EXTEND = T / FITS dataset may contain extensions +ORIGIN = 'NOAO-IRAF FITS Image Kernel July 1999' / FITS file originator +DATE = '2003-04-07T20:23:14' / Date FITS file was generated +IRAF-TLM= '13:25:08 (04/04/2003)' / Time of last modification +COMMENT FITS (Flexible Image Transport System) format defined in Astronomy and +COMMENT Astrophysics Supplement Series v44/p363, v44/p371, v73/p359, v73/p365. +COMMENT Contact the NASA Science Office of Standards and Technology for the +COMMENT FITS Definition document #100 and other FITS information. +INSTRUME= 'NIRI ' / Instrument used to acquire data. +OBJECT = 'B2 0902+34--NE' / Object Name +OBSTYPE = 'OBJECT ' / Observation type +GEMPRGID= 'GN-2003A-Q-51' / Gemini Science Program ID +OBSID = 'GN-2003A-Q-51-2' / Observation ID / Data label +DATALAB = 'GN-2003A-Q-51-2-004' / Datalabel +OBSERVER= 'F.Rigaut' / Observer +OBSERVAT= 'Gemini-North' / Observatory +TELESCOP= 'Gemini-North' / Telescope +PARALLAX= 0. / Parallax of Target +RADVEL = 0. / Heliocentric Radial Velocity +EPOCH = 2000. / Epoch for Target coordinates +EQUINOX = 2000. / Equinox for Target coordinates +TRKEQUIN= 2000. / Tracking equinox +SSA = 'B.Walls ' / SSA +RA = 136.37583333 / RA of Target +DEC = 34.12916667 / Declination of Target +ELEVATIO= 67.8446069444445 / Current Elevation +AZIMUTH = 45.988375 / Current Azimuth +CRPA = -144.412040662382 / Current Cass Rotator Position Angle +HA = '-01:16:26.35' / Telescope hour angle +LT = '20:01:59.2' / Local time at start of observation +TRKFRAME= ' ' / Tracking co-ordinate +DECTRACK= 0. / Differential tracking rate Dec +TRKEPOCH= 0. / Differential tracking reference epoch +RATRACK = 0. / Differential tracking rate RA +FRAME = 'FK5 ' / Target coordinate system +PMDEC = 0. / Proper Motion in Declination +PMRA = 0. / Proper Motion in RA +WAVELENG= 21200. / Effective Target Wavelength +RAWIQ = '70-percentile' / Raw Image Quality +RAWCC = '70-percentile' / Raw Cloud Cover +RAWWV = '80-percentile' / Raw Water Vapour/Transparency +RAWBG = '80-percentile' / Raw Background +RAWPIREQ= 'YES ' / PI Requirements Met +RAWGEMQA= 'USABLE ' / Gemini Quality Assessment +CGUIDMOD= 'Basic ' / Driving mode for carousel +UT = '06:01:59.7' / Beginning of Observation (UT) +M2BAFFLE= 'NEAR IR ' / Position of M2 baffle +M2CENBAF= 'CLOSED ' / Position of M2 central hole baffle +ST = '07:49:16.4' / Sidereal time at the start of the exposure +XOFFSET = -3.999945169 / Telescope offset in x in arcsec +YOFFSET = -0.0209438553276 / Telescope offset in y in arcsec +RAOFFSET= -7.65706942296E-14 / Telescope offset in RA in arcsec +DECOFFSE= -4.00000000001 / Telescope offset in DEC in arcsec +PA = 0. / Sky Position Angle at start of exposure +SFRT2 = -1.391 / Science fold rotation angle (degrees) +SFTILT = 45.03 / Science fold tilt angle (degrees) +SFLINEAR= 8. / Science fold linear position (mm) +P2ARA = 136.39191667 / RA of PWFS2 guide star +P2ARV = 0. / PWFS2 Heliocentric Radial Velocity +P2AWAVEL= 5000. / PWFS2 Effective Target Wavelength +P2ADEC = 34.01855556 / Declination of PWFS2 guide star +P2AEPOCH= 2000. / Epoch for PWFS2 guide star coordinates +P2AEQUIN= 2000. / Equinox for PWFS2 guide star coordinates +P2AFRAME= 'FK5 ' / PWFS2 Target co-ordinate system +P2AOBJEC= 'GSC249500968' / Object Name for PWFS 2, Chop A +P2APMDEC= 0. / PWFS2 Proper Motion in Declination +P2APMRA = 0. / PWFS2 Proper Motion in RA +P2APARAL= 0. / PWFS2 Parallax of Target +ARRAYID = '47911 ' / Array identification +ARRAYTYP= 'ALADDIN_II' / Array type +CAMERA = 'f6 ' / NIRI camera (f6|f14|f32) +COADDS = 3 / Number of coadds summed +DATE-OBS= '2003-03-25' / Observation date (UT) +EXPTIME = 15.001 / Exposure time (s) for each frame +FILTER1 = 'Kprime_G0206' / Filter 1 name +FILTER2 = 'open ' / Filter 2 name +FILTER3 = 'pupil38_G5207' / Pupil mask, grism, or filter name +FOCUSNAM= 'INDEF ' / Focus position name +FOCUSPOS= -3.35 / Focus stage position in mm +FPMASK = 'f6-cam_G5208' / Focal plane mask name +BEAMSPLT= 'f6 ' / Beam splitter position name +WINDCOVR= 'open ' / Window cover position name (open|closed) +FRMSPCYC= 2 / Frames per cycle,1=sep,2=stare +HDRTIMIN= 'INDEF ' / Header info updated before, after or both +INPORT = 3 / Number of ISS port where NIRI was located +LNRS = 1 / Number of non-destructive read pairs +MODE = 'STARE ' / Stare or sep +NDAVGS = 16 / Number of digital averages +PVIEW = 'out ' / Pupil viewer position name (in|out) +TDETABS = 33.086 / Science detector temperature (K) +TIME = '06:02:00.2' / Beginning of Observation (UT) +TIME-OBS= '06:02:00.2' / Beginning of Observation (UT) +TMOUNT = 22.978 / Mount temperature (K) +UCODENAM= 'gnAII_1024xSU01_4u' / Array microcode file name +UCODETYP= 'RDD ' / Array microcode version +VDDCL1 = -1.157 / VDDCL hi detector clock voltage +VDDCL2 = -3.496 / VDDCL lo detector clock voltage +VDDUC = -3.491 / VDDUC detector bias voltage +VDET = -2.907 / VDETCOM detector bias voltage +VGGCL1 = -4.871 / VGGCL lo detector clock voltage +VGGCL2 = -2.704 / VGGCL hi detector clock voltage +VSET = -1.871 / Vset detector bias voltage +A_TDETAB= 33.086 / Science detector temperature (K) (post exposure +A_TMOUNT= 22.978 / Mount temperature (K) (post exposure) +A_VDDCL1= -1.157 / VDDCL hi detector clock voltage (post exposure) +A_VDDCL2= -3.496 / VDDCL lo detector clock voltage (post exposure) +A_VDDUC = -3.491 / VDDUC detector bias voltage (post exposure) +A_VDET = -2.907 / VDETCOM detector bias voltage (post exposure) +A_VGGCL1= -4.871 / VGGCL lo detector clock voltage (post exposure) +A_VGGCL2= -2.704 / VGGCL hi detector clock voltage (post exposure) +A_VSET = -1.871 / Vset detector bias voltage (post exposure) +UTEND = '06:02:49.2' / End of Observation (UT) +OBSEPOCH= 2003.22724547 / Epoch at start of exposure +AIRMASS = 1.079 / Mean airmass for the observation +AMSTART = 1.08 / Airmass at start of exposure +AMEND = 1.079 / Airmass at end of exposure +CTYPE1 = 'RA---TAN' / the coordinate type for the first axis +CRPIX1 = 545.186086585101 / x-coordinate of reference pixel +CRVAL1 = 136.375908056217 / first axis value at ref pixel +CTYPE2 = 'DEC--TAN' / the coordinate type for the second axis +CRPIX2 = 489.049229603846 / y-coordinate of reference pixel +CRVAL2 = 34.1303349090605 / second axis value at ref pixel +CD1_1 = -3.25242700275654E-05 / partial of first axis coord w.r.t. x +CD1_2 = 1.28695856061755E-07 / partial of first axis coord w.r.t. y +CD2_1 = -8.99871065778577E-09 / partial of second axis coord w.r.t. x +CD2_2 = 3.23352654722922E-05 / partial of second axis coord w.r.t. y +MJD_OBS = 52723.2526998586 / Mean Julian day of observation +INTEGRIT= 'OK ' / Observation status (OK|STOP|ABORT) +FRAME = 'FK5 ' / Target coordinate system +RELEASE = '2004-09-25' / End of proprietary period YYYY-MM-DD +ORIGNAME= 'N20030325S0098.fits' / Original filename prior to processing +HISTORY Corrected metadata: automated fixes from PyFITS + +--- HDU 0 --- +XTENSION= 'IMAGE ' / Image extension +BITPIX = 32 / Bits per pixel +NAXIS = 2 / Number of axes +NAXIS1 = 1024 / Axis length +NAXIS2 = 1024 / Axis length +PCOUNT = 0 / No 'random' parameters +GCOUNT = 1 / Only one group +EXTVER = 1 / Added by AstroData +ORIGIN = 'NOAO-IRAF FITS Image Kernel July 1999' / FITS file originator +INHERIT = F / Inherits global header +DATE = '2003-04-04T23:25:10' / Date FITS file was generated +IRAF-TLM= '13:25:08 (04/04/2003)' / Time of last modification +OBJECT = 'B2 0902+34--NE' / Name of the object observed +CTYPE1 = 'RA---TAN' / the coordinate type for the first axis +CRPIX1 = 545.1860865851 / x-coordinate of reference pixel +CRVAL1 = 136.37590805622 / first axis value at ref pixel +CTYPE2 = 'DEC--TAN' / the coordinate type for the second axis +CRPIX2 = 489.04922960385 / y-coordinate of reference pixel +CRVAL2 = 34.130334909061 / second axis value at ref pixel +CD1_1 = -3.2524270027565E-55 / partial of first axis coord w.r.t. x +CD1_2 = 1.2869585606175E-7 / partial of first axis coord w.r.t. y +CD2_1 = -8.9987106577858E-9 / partial of second axis coord w.r.t. x +CD2_2 = 3.2335265472292E-5 / partial of second axis coord w.r.t. y +MJD_OBS = 52723.252699859 / Mean Julian day of observation +DROINUM = 0 / Which Region of Interest +LOWROW = 0 / Start row of region of interest +LOWCOL = 0 / Start col of region of interest +HIROW = 1023 / End row of region of interest +HICOL = 1023 / End col of region of interest +FRMNAME = 'N20030325S0099:0' / Frame name +FRAMEID = '0 ' / Frame ID +DATATYP = ' ' / Data type +EXTNAME = 'SCI ' / Added by AstroData diff --git a/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.module b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.module new file mode 100644 index 00000000..1d4382b5 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.module @@ -0,0 +1,10 @@ +def get_exposure(uri): + return 45.003 + + +def get_time_delta(uri): + result = None + exptime = get_exposure(0) + if exptime: + result = exptime / ( 24.0 * 3600.0 ) + return result diff --git a/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.py b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.py new file mode 100644 index 00000000..1d4382b5 --- /dev/null +++ b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.py @@ -0,0 +1,10 @@ +def get_exposure(uri): + return 45.003 + + +def get_time_delta(uri): + result = None + exptime = get_exposure(0) + if exptime: + result = exptime / ( 24.0 * 3600.0 ) + return result diff --git a/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/20220201T200117.xml b/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/20220201T200117.xml index c1cb7bed..895e8b5f 100644 --- a/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/20220201T200117.xml +++ b/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/20220201T200117.xml @@ -40,7 +40,6 @@ 1 2 - 3 @@ -102,7 +101,6 @@ 1 2 - 3 @@ -164,7 +162,6 @@ 1 2 - 3 diff --git a/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/taos.blueprint b/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/taos.blueprint index 98f209a7..ee38e282 100644 --- a/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/taos.blueprint +++ b/caom2utils/caom2utils/tests/data/taos_h5file/20220201T200117/taos.blueprint @@ -29,7 +29,6 @@ Chunk.position.axis.error2.syser = None Chunk.position.axis.error2.rnder = None Chunk.position.coordsys = None -Chunk.timeAxis = 3 Chunk.time.axis.axis.ctype = TIME Chunk.time.axis.axis.cunit = s Chunk.time.axis.range.start.pix = 0 diff --git a/caom2utils/caom2utils/tests/test_collections.py b/caom2utils/caom2utils/tests/test_collections.py index 802c9c06..7555c9b5 100644 --- a/caom2utils/caom2utils/tests/test_collections.py +++ b/caom2utils/caom2utils/tests/test_collections.py @@ -66,6 +66,8 @@ # *********************************************************************** # +from astropy.io import fits + from cadcdata import FileInfo from caom2utils import legacy, caom2blueprint, data_util from caom2 import ObservationReader, ObservationWriter @@ -105,7 +107,7 @@ def test_differences(directory): product_id = f'--productID {prod_id}' collection_id = expected.collection data_files = _get_files( - ['header', 'png', 'gif', 'cat', 'fits', 'h5'], directory) + ['header', 'png', 'gif', 'cat', 'fits', 'h5', 'orig'], directory) assert data_files file_meta = _get_uris(collection_id, data_files, expected) @@ -172,20 +174,30 @@ def _vos_client_meta(subject, uri): file_type='application/fits') def _header(fqn): - # during operation, want to use astropy on FITS files - # but during testing want to use headers and built-in Python file - # operations - from urllib.parse import urlparse - from astropy.io import fits - file_uri = urlparse(fqn) - try: - fits_header = open(file_uri.path).read() - headers = data_util.make_headers_from_string(fits_header) - except UnicodeDecodeError: - hdulist = fits.open(fqn, memmap=True, lazy_load_hdus=True) - hdulist.verify('fix') - hdulist.close() - headers = [h.header for h in hdulist] + if '.fits' in fqn: + # during operation, want to use astropy on FITS files + # but during testing want to use headers and built-in Python file + # operations + file_uri = urlparse(fqn) + try: + fits_header = open(file_uri.path).read() + headers = data_util.make_headers_from_string(fits_header) + except UnicodeDecodeError: + hdulist = fits.open(fqn, memmap=True, lazy_load_hdus=True) + hdulist.verify('fix') + hdulist.close() + headers = [h.header for h in hdulist] + else: + # the BRITE record tests blueprint range declarations over-riding function declarations. Do the minimal + # header construction to get that portion of the test to run. + header = fits.Header() + header['BITPIX'] = 8 + header['NAXIS'] = 4 + header['NAXIS1'] = 1 + header['NAXIS2'] = 1 + header['NAXIS3'] = 1 + header['NAXIS4'] = 1 + headers = [header] return headers swc_si_mock.return_value.cadcinfo.side_effect = info_mock @@ -236,6 +248,10 @@ def _get_cardinality(directory): return '--lineage def/cadc:def/def.h5' else: return '--lineage star04239531/cadc:TAOSII/taos2_20220201T201317Z_star04239531.h5' + elif 'brite' in directory: + return '--lineage HD36486_65-Ori-VIII-2021_BAb_1_5_A/ad:BRITE-Constellation/HD36486.orig' + elif 'gemini' in directory: + return '--lineage GN-2003A-Q-51-2-004/ad:GEMINI/N20030325S0098.fits' else: return '' diff --git a/caom2utils/caom2utils/tests/test_fits2caom2.py b/caom2utils/caom2utils/tests/test_fits2caom2.py index 5a33083f..d494e385 100755 --- a/caom2utils/caom2utils/tests/test_fits2caom2.py +++ b/caom2utils/caom2utils/tests/test_fits2caom2.py @@ -79,11 +79,9 @@ from caom2utils.caom2blueprint import _visit, _load_plugin from caom2utils.caom2blueprint import _get_and_update_artifact_meta -from caom2 import ObservationWriter, SimpleObservation, Algorithm -from caom2 import Artifact, ProductType, ReleaseType, ObservationIntentType -from caom2 import get_differences, obs_reader_writer, ObservationReader, Chunk -from caom2 import SpectralWCS, TemporalWCS, PolarizationWCS, SpatialWCS -from caom2 import Axis, CoordAxis1D, CoordAxis2D, ChecksumURI, DataProductType +from caom2 import ObservationWriter, SimpleObservation, Algorithm, Artifact, ProductType, ReleaseType, DataProductType +from caom2 import get_differences, obs_reader_writer, ObservationReader, Chunk, ObservationIntentType, ChecksumURI +from caom2 import CustomWCS, SpectralWCS, TemporalWCS, PolarizationWCS, SpatialWCS, Axis, CoordAxis1D, CoordAxis2D from caom2 import CalibrationLevel import logging @@ -156,7 +154,7 @@ def test_augment_energy(): ex = _get_from_str_xml(EXPECTED_ENERGY_XML, ObservationReader()._get_spectral_wcs, 'energy') result = get_differences(ex, energy) - assert result is None, repr(energy) + assert result is None, result def test_hdf5_wcs_parser_set_wcs(): @@ -226,7 +224,7 @@ def test_augment_artifact_energy_from_blueprint(): ObservationReader()._get_spectral_wcs, 'energy') result = get_differences(ex, test_chunk.energy) - assert result is None + assert result is None, result EXPECTED_POLARIZATION_XML = \ @@ -1172,6 +1170,28 @@ def test_visit(): visit_local=None, **kwargs) +EXPECTED_CUSTOM_RANGE_BOUNDS_XML = ''' + + + + RM + m / s ** 2 + + + + 145.0 + -60000.0 + + + -824.46002 + 1 + + + + + +''' + EXPECTED_ENERGY_RANGE_BOUNDS_XML = ''' @@ -1190,7 +1210,8 @@ def test_visit(): - TOPOCENT + LSRK + 1420406000.0 ''' @@ -1278,9 +1299,12 @@ def test_visit(): def test_augment_artifact_bounds_range_from_blueprint(): - test_blueprint = ObsBlueprint(energy_axis=1, time_axis=2, - polarization_axis=3, - position_axes=(4, 5)) + test_blueprint = ObsBlueprint( + energy_axis=1, time_axis=2, polarization_axis=3, position_axes=(4, 5), custom_axis=6) + test_blueprint.set('Chunk.custom.axis.range.start.pix', '145.0') + test_blueprint.set('Chunk.custom.axis.range.start.val', '-60000.0') + test_blueprint.set('Chunk.custom.axis.range.end.pix', '-824.46002') + test_blueprint.set('Chunk.custom.axis.range.end.val', '1') test_blueprint.set('Chunk.energy.axis.range.start.pix', '145.0') test_blueprint.set('Chunk.energy.axis.range.start.val', '-60000.0') test_blueprint.set('Chunk.energy.axis.range.end.pix', '-824.46002') @@ -1308,12 +1332,18 @@ def test_augment_artifact_bounds_range_from_blueprint(): test_fitsparser = FitsParser(sample_file_4axes, test_blueprint, uri='ad:TEST/test_blueprint') test_chunk = Chunk() + test_chunk.custom = CustomWCS(CoordAxis1D(Axis('RM', 'm / s ** 2'))) test_chunk.energy = SpectralWCS(CoordAxis1D(Axis('WAVE', 'm')), 'TOPOCENT') test_chunk.time = TemporalWCS(CoordAxis1D(Axis('TIME', 'd'))) test_chunk.polarization = PolarizationWCS(CoordAxis1D(Axis('STOKES'))) test_chunk.position = SpatialWCS(CoordAxis2D(Axis('RA', 'deg'), Axis('DEC', 'deg'))) - test_fitsparser._try_range_with_blueprint(test_chunk, 0) + test_fitsparser._try_position_with_blueprint(test_chunk, 0) + test_fitsparser._try_energy_with_blueprint(test_chunk, 0) + test_fitsparser._try_time_with_blueprint(test_chunk, 0) + test_fitsparser._try_polarization_with_blueprint(test_chunk, 0) + test_fitsparser._try_observable_with_blueprint(test_chunk, 0) + test_fitsparser._try_custom_with_blueprint(test_chunk, 0) assert test_chunk.energy.axis.range is not None, \ 'chunk.energy.axis.range should be declared' @@ -1323,13 +1353,14 @@ def test_augment_artifact_bounds_range_from_blueprint(): 'chunk.polarization.axis.range should be declared' assert test_chunk.position.axis.range is not None, \ 'chunk.position.axis.range should be declared' + assert test_chunk.custom.axis.range is not None, 'chunk.custom.axis.range should be declared' ex = _get_from_str_xml(EXPECTED_ENERGY_RANGE_BOUNDS_XML, ObservationReader()._get_spectral_wcs, 'energy') assert ex is not None, \ 'energy string from expected output should be declared' result = get_differences(ex, test_chunk.energy) - assert result is None + assert result is None, f'energy\n{result}' ex = _get_from_str_xml(EXPECTED_TIME_RANGE_BOUNDS_XML, ObservationReader()._get_temporal_wcs, @@ -1337,7 +1368,7 @@ def test_augment_artifact_bounds_range_from_blueprint(): assert ex is not None, \ 'time string from expected output should be declared' result = get_differences(ex, test_chunk.time) - assert result is None + assert result is None, f'time\n{result}' ex = _get_from_str_xml(EXPECTED_POL_RANGE_BOUNDS_XML, ObservationReader()._get_polarization_wcs, @@ -1345,7 +1376,7 @@ def test_augment_artifact_bounds_range_from_blueprint(): assert ex is not None, \ 'polarization string from expected output should be declared' result = get_differences(ex, test_chunk.polarization) - assert result is None + assert result is None, f'polarization\n{result}' ex = _get_from_str_xml(EXPECTED_POS_RANGE_BOUNDS_XML, ObservationReader()._get_spatial_wcs, @@ -1353,7 +1384,14 @@ def test_augment_artifact_bounds_range_from_blueprint(): assert ex is not None, \ 'position string from expected output should be declared' result = get_differences(ex, test_chunk.position) - assert result is None + assert result is None, f'position\n{result}' + + ex = _get_from_str_xml(EXPECTED_CUSTOM_RANGE_BOUNDS_XML, + ObservationReader()._get_custom_wcs, + 'custom') + assert ex is not None, 'custom string from expected output should be declared' + result = get_differences(ex, test_chunk.custom) + assert result is None, f'custom\n{result}' def test_visit_generic_parser(): @@ -1545,7 +1583,7 @@ def get_time_exposure(self, ext): hdr2['BITPIX'] = -32 hdr2['CTYPE1'] = 'TIME' hdr2['CUNIT1'] = 'd' - hdr2['CRPIX1'] = '1' + hdr2['CRPIX1'] = 1.0 hdr2['CRVAL1'] = 590000.00000 test_blueprint = ObsBlueprint(instantiated_class=test_instantiated) test_blueprint.configure_time_axis(1) diff --git a/caom2utils/setup.cfg b/caom2utils/setup.cfg index 233ccd50..fc6813bb 100644 --- a/caom2utils/setup.cfg +++ b/caom2utils/setup.cfg @@ -33,7 +33,7 @@ url = https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2 edit_on_github = False github_project = opencadc/caom2tools # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -version = 1.7.1 +version = 1.7.2 [options] install_requires =