diff --git a/mirar/catalog/__init__.py b/mirar/catalog/__init__.py index 49fad4ab0..819e9cac2 100644 --- a/mirar/catalog/__init__.py +++ b/mirar/catalog/__init__.py @@ -3,5 +3,5 @@ The basic structure is contained in :class:`mirar.catalog.base_catalog` """ -from mirar.catalog.base_catalog import BaseCatalog -from mirar.catalog.gaia import Gaia2Mass +from mirar.catalog.base.base_catalog import BaseCatalog +from mirar.catalog.multibackend import Gaia2Mass diff --git a/mirar/catalog/base/__init__.py b/mirar/catalog/base/__init__.py new file mode 100644 index 000000000..0d98218ee --- /dev/null +++ b/mirar/catalog/base/__init__.py @@ -0,0 +1,7 @@ +""" +Base classes for the catalog system. +""" + +from mirar.catalog.base.base_catalog import BaseCatalog, BaseMultiBackendCatalog +from mirar.catalog.base.base_xmatch_catalog import BaseXMatchCatalog +from mirar.catalog.base.errors import CatalogError diff --git a/mirar/catalog/base_catalog.py b/mirar/catalog/base/base_catalog.py similarity index 64% rename from mirar/catalog/base_catalog.py rename to mirar/catalog/base/base_catalog.py index 9f3656cd5..d56f17a85 100644 --- a/mirar/catalog/base_catalog.py +++ b/mirar/catalog/base/base_catalog.py @@ -5,30 +5,19 @@ import logging from abc import ABC from pathlib import Path +from typing import Type import astropy.table +from mirar.catalog.base.errors import CatalogCacheError from mirar.data import Image from mirar.data.utils import get_image_center_wcs_coords -from mirar.errors import ProcessorError from mirar.paths import BASE_NAME_KEY, REF_CAT_PATH_KEY -from mirar.utils.ldac_tools import get_table_from_ldac, save_table_as_ldac +from mirar.utils.ldac_tools import save_table_as_ldac logger = logging.getLogger(__name__) -class CatalogError(ProcessorError): - """ - Class for errors in Catalog - """ - - -class CatalogCacheError(CatalogError): - """ - Class for errors in CatalogCache - """ - - class ABCatalog: """ Abstract class for catalog objects @@ -139,88 +128,20 @@ def get_output_path(self, output_dir: Path, base_name: str | Path) -> Path: return output_dir.joinpath(cat_base_name) -class BaseXMatchCatalog(ABCatalog, ABC): +class BaseMultiBackendCatalog(ABC): """ - Base Catalog for crossmatching + Base class for composite catalogs that are made up of multiple catalog backends """ - @property - def catalog_name(self): - """ - Name of catalog - """ - raise NotImplementedError + def __new__(cls, *args, backend: str | None = None, **kwargs): + backend_class = cls.set_backend(backend) + return backend_class(*args, **kwargs) - @property - def projection(self): - """ - projection for kowalski xmatch - """ - raise NotImplementedError - - @property - def column_names(self): + @staticmethod + def set_backend(backend: str | None) -> Type[BaseCatalog]: """ - Name of columns - """ - raise NotImplementedError + Set backend for composite catalog - @property - def column_dtypes(self): - """ - dtype of columns + :param backend: Backend name """ - raise NotImplementedError - - @property - def ra_column_name(self): - """ - Name of RA column - """ - raise NotImplementedError - - @property - def dec_column_name(self): - """ - Name of Dec column - """ - raise NotImplementedError - - def __init__(self, *args, num_sources: int = 1, **kwargs): - super().__init__(*args, **kwargs) - self.search_radius_arcsec = self.search_radius_arcmin * 60.0 - self.num_sources = num_sources - - def query(self, coords: dict) -> dict: - """ - Query coords for result - - :param coords: ra/dec - :return: crossmatch - """ - raise NotImplementedError - - -class CatalogFromFile(BaseCatalog): - """ - Local catalog from file - """ - - abbreviation = "local" - - def __init__(self, catalog_path: str | Path, *args, **kwargs): - super().__init__( - min_mag=0, - max_mag=99, - filter_name="None", - search_radius_arcmin=0, - *args, - **kwargs, - ) - self.catalog_path = catalog_path - if isinstance(self.catalog_path, str): - self.catalog_path = Path(self.catalog_path) - - def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: - catalog = get_table_from_ldac(self.catalog_path) - return catalog + raise NotImplementedError() diff --git a/mirar/catalog/gaia.py b/mirar/catalog/base/base_gaia.py similarity index 66% rename from mirar/catalog/gaia.py rename to mirar/catalog/base/base_gaia.py index d85a20873..8c55ba88f 100644 --- a/mirar/catalog/gaia.py +++ b/mirar/catalog/base/base_gaia.py @@ -3,32 +3,28 @@ """ import logging +from abc import ABC from typing import Optional import astropy.table import astropy.units as u import numpy as np from astropy.coordinates import SkyCoord -from astroquery.gaia import Gaia -from mirar.catalog.base_catalog import BaseCatalog +from mirar.catalog.base.base_catalog import BaseCatalog from mirar.utils.ldac_tools import get_table_from_ldac logger = logging.getLogger(__name__) -# Disable astroquery warnings -logging.getLogger("astroquery").setLevel(logging.WARNING) - - # 2MASS values from https://iopscience.iop.org/article/10.1086/376474 zeromag_2mass = {"j": 1594.0 * u.Jansky, "h": 1024.0 * u.Jansky, "k": 666.8 * u.Jansky} offsets_2mass = {key: zm.to("mag(AB)").value for key, zm in zeromag_2mass.items()} -class Gaia2Mass(BaseCatalog): +class BaseGaia2Mass(BaseCatalog, ABC): """ - Crossmatched Gaia/2Mass catalog + Base Gaia/2Mass catalog """ abbreviation = "tmass" @@ -77,48 +73,27 @@ def __init__( if val is None: self.acceptable_ph_quals[filt] = ["A", "B", "C"] + def convert_to_ab_mag(self, src_list: astropy.table.Table) -> astropy.table.Table: + """ + Convert 2MASS magnitudes to AB magnitudes + + :param src_list: Source list + :return: Source list with AB magnitudes + """ + # Convert to AB magnitudes + for key in ["j", "h", "k"]: + offset = offsets_2mass[self.filter_name.lower()] + src_list[f"{key}_m"] += offset + logger.debug(f"Adding {offset:.2f} to convert from 2MASS to AB magnitudes") + return src_list + def get_catalog( self, ra_deg: float, dec_deg: float, ) -> astropy.table.Table: - logger.debug( - f"Querying 2MASS - Gaia cross-match around RA {ra_deg:.4f}, " - f"Dec {dec_deg:.4f} with a radius of {self.search_radius_arcmin:.4f} arcmin" - ) - - cmd = ( - f"SELECT * FROM gaiadr2.gaia_source AS g, " - f"gaiadr2.tmass_best_neighbour AS tbest, " - f"gaiadr1.tmass_original_valid AS tmass " - f"WHERE g.source_id = tbest.source_id " - f"AND tbest.tmass_oid = tmass.tmass_oid " - f"AND CONTAINS(POINT('ICRS', g.ra, g.dec), " - f"CIRCLE('ICRS', {ra_deg:.4f}, {dec_deg:.4f}, " - f"{self.search_radius_arcmin / 60:.4f}))=1 " - f"AND tmass.{self.filter_name}_m > {self.min_mag:.2f} " - f"AND tmass.{self.filter_name}_m < {self.max_mag:.2f} " - f"AND tbest.number_of_mates=0 " - f"AND tbest.number_of_neighbours=1;" - ) - - job = Gaia.launch_job_async(cmd, dump_to_file=False) - src_list = job.get_results() - src_list["ph_qual"] = src_list["ph_qual"].astype(str) - src_list["ra_errdeg"] = src_list["ra_error"] / 3.6e6 - src_list["dec_errdeg"] = src_list["dec_error"] / 3.6e6 - src_list["FLAGS"] = 0 - src_list["magnitude"] = src_list[f"{self.filter_name.lower()}_m"] - src_list["magnitude_err"] = src_list[f"{self.filter_name.lower()}_msigcom"] - logger.debug(f"Found {len(src_list)} sources in Gaia") - - # Convert to AB magnitudes - - offset = offsets_2mass[self.filter_name.lower()] - logger.debug(f"Adding {offset:.2f} to convert from 2MASS to AB magnitudes") - - src_list["magnitude"] += offset + src_list = self.get_source_table(ra_deg, dec_deg) j_phquals = [x[0] for x in src_list["ph_qual"]] h_phquals = [x[1] for x in src_list["ph_qual"]] @@ -134,11 +109,12 @@ def get_catalog( src_list = src_list[src_list["magnitude_err"] < 1.086 / self.snr_threshold] if self.trim: if self.image_catalog_path is None: - logger.error( - "Gaia catalog trimming requested but " - "no sextractor catalog path specified." + err = ( + "Gaia catalog trimming requested " + "but no sextractor catalog path specified." ) - raise ValueError + logger.error(err) + raise ValueError(err) image_catalog = get_table_from_ldac(self.image_catalog_path) src_list = self.trim_catalog(src_list, image_catalog) @@ -170,3 +146,13 @@ def trim_catalog( match_mask = d2d < 2 * u.arcsec matched_catalog = ref_catalog[idx[match_mask]] return matched_catalog + + def get_source_table(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: + """ + Get the source table for a given position + + :param ra_deg: RA in degrees + :param dec_deg: Dec in degrees + :return: Table + """ + raise NotImplementedError diff --git a/mirar/catalog/base/base_xmatch_catalog.py b/mirar/catalog/base/base_xmatch_catalog.py new file mode 100644 index 000000000..279239b5e --- /dev/null +++ b/mirar/catalog/base/base_xmatch_catalog.py @@ -0,0 +1,69 @@ +""" +Base Catalog for crossmatching +""" + +from abc import ABC + +from mirar.catalog.base.base_catalog import ABCatalog + + +class BaseXMatchCatalog(ABCatalog, ABC): + """ + Base Catalog for crossmatching + """ + + @property + def catalog_name(self): + """ + Name of catalog + """ + raise NotImplementedError + + @property + def projection(self): + """ + projection for kowalski xmatch + """ + raise NotImplementedError + + @property + def column_names(self): + """ + Name of columns + """ + raise NotImplementedError + + @property + def column_dtypes(self): + """ + dtype of columns + """ + raise NotImplementedError + + @property + def ra_column_name(self): + """ + Name of RA column + """ + raise NotImplementedError + + @property + def dec_column_name(self): + """ + Name of Dec column + """ + raise NotImplementedError + + def __init__(self, *args, num_sources: int = 1, **kwargs): + super().__init__(*args, **kwargs) + self.search_radius_arcsec = self.search_radius_arcmin * 60.0 + self.num_sources = num_sources + + def query(self, coords: dict) -> dict: + """ + Query coords for result + + :param coords: ra/dec + :return: crossmatch + """ + raise NotImplementedError diff --git a/mirar/catalog/base/catalog_from_file.py b/mirar/catalog/base/catalog_from_file.py new file mode 100644 index 000000000..20c34b81a --- /dev/null +++ b/mirar/catalog/base/catalog_from_file.py @@ -0,0 +1,35 @@ +""" +Module for Catalog base class +""" + +from pathlib import Path + +import astropy.table + +from mirar.catalog.base.base_catalog import BaseCatalog +from mirar.utils.ldac_tools import get_table_from_ldac + + +class CatalogFromFile(BaseCatalog): + """ + Local catalog from file + """ + + abbreviation = "local" + + def __init__(self, catalog_path: str | Path, *args, **kwargs): + super().__init__( + min_mag=0, + max_mag=99, + filter_name="None", + search_radius_arcmin=0, + *args, + **kwargs, + ) + self.catalog_path = catalog_path + if isinstance(self.catalog_path, str): + self.catalog_path = Path(self.catalog_path) + + def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: + catalog = get_table_from_ldac(self.catalog_path) + return catalog diff --git a/mirar/catalog/base/errors.py b/mirar/catalog/base/errors.py new file mode 100644 index 000000000..7db35a14d --- /dev/null +++ b/mirar/catalog/base/errors.py @@ -0,0 +1,17 @@ +""" +Module for errors in Catalog +""" + +from mirar.errors import ProcessorError + + +class CatalogError(ProcessorError): + """ + Class for errors in Catalog + """ + + +class CatalogCacheError(CatalogError): + """ + Class for errors in CatalogCache + """ diff --git a/mirar/catalog/kowalski/base_kowalski_catalog.py b/mirar/catalog/kowalski/base_kowalski_catalog.py index 0198ff2d8..f10a1119a 100644 --- a/mirar/catalog/kowalski/base_kowalski_catalog.py +++ b/mirar/catalog/kowalski/base_kowalski_catalog.py @@ -9,7 +9,7 @@ from penquins import Kowalski -from mirar.catalog.base_catalog import BaseXMatchCatalog +from mirar.catalog.base.base_xmatch_catalog import BaseXMatchCatalog from mirar.errors import ProcessorError logger = logging.getLogger(__name__) diff --git a/mirar/catalog/kowalski/tmass.py b/mirar/catalog/kowalski/tmass.py index 0919c5ed4..a701b7eaa 100644 --- a/mirar/catalog/kowalski/tmass.py +++ b/mirar/catalog/kowalski/tmass.py @@ -2,7 +2,7 @@ Module for querying 2MASS using Kowalski """ -from mirar.catalog.gaia import offsets_2mass +from mirar.catalog.base.base_gaia import offsets_2mass from mirar.catalog.kowalski.base_kowalski_catalog import BaseKowalskiXMatch diff --git a/mirar/catalog/multibackend/__init__.py b/mirar/catalog/multibackend/__init__.py new file mode 100644 index 000000000..154df1b09 --- /dev/null +++ b/mirar/catalog/multibackend/__init__.py @@ -0,0 +1,5 @@ +""" +Composite Catalogs +""" + +from mirar.catalog.multibackend.gaia2mass import Gaia2Mass diff --git a/mirar/catalog/multibackend/gaia2mass.py b/mirar/catalog/multibackend/gaia2mass.py new file mode 100644 index 000000000..b6e6508b2 --- /dev/null +++ b/mirar/catalog/multibackend/gaia2mass.py @@ -0,0 +1,45 @@ +""" +Composite catalog for Gaia 2Mass +""" + +import logging +from typing import Type + +from mirar.catalog.base.base_catalog import BaseCatalog, BaseMultiBackendCatalog +from mirar.catalog.tap.gaia2mass import Gaia, Gaia2MassTAP +from mirar.catalog.vizier.gaia2mass import Gaia2MassVizier + +logger = logging.getLogger(__name__) + + +class Gaia2Mass(BaseMultiBackendCatalog): + """ + Composite catalog for Gaia 2Mass + """ + + abbreviation = "tmass" + + @staticmethod + def set_backend(backend: str | None) -> Type[BaseCatalog]: + + if backend is None: + # pylint: disable=protected-access,no-member + if Gaia._TapPlus__getconnhandler().get_response_status() != 0: + # Gaia goes down sometimes + # response status 0 means it's down, need to confirm what it is when up + # my guess is 200 but could be 1 + backend = "gaia_tap" + else: + logger.warning("Gaia TAP service is down, cannot use default backend.") + + if backend is None: + backend = "vizier" + + logger.debug(f"Backend for Gaia2Mass: {backend}") + + if backend == "gaia_tap": + return Gaia2MassTAP + if backend == "vizier": + return Gaia2MassVizier + + raise NotImplementedError(f"Backend '{backend}' not implemented for Gaia2Mass") diff --git a/mirar/catalog/tap/__init__.py b/mirar/catalog/tap/__init__.py new file mode 100644 index 000000000..e3e084f5b --- /dev/null +++ b/mirar/catalog/tap/__init__.py @@ -0,0 +1,3 @@ +""" +Catalog module for TAP services +""" diff --git a/mirar/catalog/tap/gaia2mass.py b/mirar/catalog/tap/gaia2mass.py new file mode 100644 index 000000000..4a33a8530 --- /dev/null +++ b/mirar/catalog/tap/gaia2mass.py @@ -0,0 +1,60 @@ +""" +Module for obtaining a Gaia/2Mass catalog +""" + +import logging + +import astropy.table +from astroquery.gaia import Gaia + +from mirar.catalog.base.base_gaia import BaseGaia2Mass + +logger = logging.getLogger(__name__) + +# Disable astroquery warnings +logging.getLogger("astroquery").setLevel(logging.WARNING) + + +class Gaia2MassTAP(BaseGaia2Mass): + """ + Crossmatched Gaia/2Mass catalog + """ + + def get_source_table( + self, + ra_deg: float, + dec_deg: float, + ) -> astropy.table.Table: + logger.debug( + f"Querying 2MASS - Gaia cross-match around RA {ra_deg:.4f}, " + f"Dec {dec_deg:.4f} with a radius of {self.search_radius_arcmin:.4f} arcmin" + ) + + cmd = ( + f"SELECT * FROM gaiadr2.gaia_source AS g, " + f"gaiadr2.tmass_best_neighbour AS tbest, " + f"gaiadr1.tmass_original_valid AS tmass " + f"WHERE g.source_id = tbest.source_id " + f"AND tbest.tmass_oid = tmass.tmass_oid " + f"AND CONTAINS(POINT('ICRS', g.ra, g.dec), " + f"CIRCLE('ICRS', {ra_deg:.4f}, {dec_deg:.4f}, " + f"{self.search_radius_arcmin / 60:.4f}))=1 " + f"AND tmass.{self.filter_name}_m > {self.min_mag:.2f} " + f"AND tmass.{self.filter_name}_m < {self.max_mag:.2f} " + f"AND tbest.number_of_mates=0 " + f"AND tbest.number_of_neighbours=1;" + ) + + job = Gaia.launch_job_async(cmd, dump_to_file=False) + src_list = job.get_results() + + src_list = self.convert_to_ab_mag(src_list) + + src_list["ph_qual"] = src_list["ph_qual"].astype(str) + src_list["ra_errdeg"] = src_list["ra_error"] / 3.6e6 + src_list["dec_errdeg"] = src_list["dec_error"] / 3.6e6 + src_list["FLAGS"] = 0 + src_list["magnitude"] = src_list[f"{self.filter_name.lower()}_m"] + src_list["magnitude_err"] = src_list[f"{self.filter_name.lower()}_msigcom"] + logger.debug(f"Found {len(src_list)} sources in Gaia") + return src_list diff --git a/mirar/catalog/vizier/__init__.py b/mirar/catalog/vizier/__init__.py index 3937d82a1..09bfcd908 100644 --- a/mirar/catalog/vizier/__init__.py +++ b/mirar/catalog/vizier/__init__.py @@ -2,6 +2,7 @@ Module for catalogs using Vizier """ +from mirar.catalog.vizier.gaia2mass import Gaia2MassVizier from mirar.catalog.vizier.ps1 import PS1 from mirar.catalog.vizier.sdss import SDSS, NotInSDSSError, in_sdss from mirar.catalog.vizier.skymapper import SkyMapper diff --git a/mirar/catalog/vizier/base_vizier_catalog.py b/mirar/catalog/vizier/base_vizier_catalog.py index 72fe91dde..4b51f20a5 100644 --- a/mirar/catalog/vizier/base_vizier_catalog.py +++ b/mirar/catalog/vizier/base_vizier_catalog.py @@ -11,7 +11,7 @@ from astropy.table import Table from astroquery.vizier import Vizier -from mirar.catalog.base_catalog import BaseCatalog +from mirar.catalog.base.base_catalog import BaseCatalog from mirar.errors import ProcessorError @@ -30,7 +30,7 @@ class VizierCatalog(BaseCatalog, ABC): """ @property - def catalog_vizier_code(self): + def catalog_vizier_code(self) -> str | list[str]: """Code of catalog in Vizier""" raise NotImplementedError() @@ -75,6 +75,12 @@ def filter_catalog(self, table: astropy.table.Table) -> astropy.table.Table: """ return table + def get_column_filters(self) -> dict: + """ + Returns the column filters to be applied to the query + """ + return {} + def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: logger.debug( f"Querying {self.abbreviation} catalog around RA {ra_deg:.4f}, " @@ -84,8 +90,9 @@ def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: viz_cat = Vizier( columns=["*"], column_filters={ - f"{self.get_mag_key()}": f"< {self.max_mag}", + f"{self.get_mag_key()}": f"{self.min_mag} .. {self.max_mag}", f"{self.get_mag_error_key()}": f"<{1.086 / self.snr_threshold:.3f}", + **self.get_column_filters(), }, row_limit=-1, ) @@ -104,7 +111,8 @@ def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: self.check_coverage(ra_deg, dec_deg) return Table() - table = query[0] + table = self.join_query(query) + logger.debug(f"Table columns are: {table.colnames}") if self.get_mag_key() not in table.colnames: err = ( @@ -124,6 +132,15 @@ def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: table = self.filter_catalog(table) return table + def join_query(self, query: dict) -> astropy.table.Table: + """ + Join the query results into a single table + + :param query: Query results + :return: Table + """ + return query[0] + @staticmethod def check_coverage(ra_deg: float, dec_deg: float): """ diff --git a/mirar/catalog/vizier/gaia2mass.py b/mirar/catalog/vizier/gaia2mass.py new file mode 100644 index 000000000..ae536aa89 --- /dev/null +++ b/mirar/catalog/vizier/gaia2mass.py @@ -0,0 +1,82 @@ +""" +Module for querying Gaia catalog +""" + +import logging + +import astropy.table +from astropy.table import join + +from mirar.catalog.base.base_gaia import BaseGaia2Mass +from mirar.catalog.vizier.base_vizier_catalog import VizierCatalog + +logger = logging.getLogger(__name__) + +# Disable astroquery warnings +logging.getLogger("astroquery").setLevel(logging.WARNING) + + +class Gaia2MassVizier(BaseGaia2Mass, VizierCatalog): + """ + Gaia DR3 catalog + """ + + catalog_vizier_code = ["I/355/gaiadr3", "II/246/out"] + + ra_key = "RA_ICRS" + dec_key = "DE_ICRS" + + def get_mag_key(self) -> str: + """ + Returns the key for mag in table + + :return: Mag key + """ + return f"{self.filter_name.upper()}mag" + + def get_column_filters(self) -> dict: + """ + Get the column filters for the catalog + + :return: Column filters + """ + return {"n2MASS": "= 1", "m2MASS": "= 0"} + + def get_source_table(self, ra_deg: float, dec_deg: float) -> astropy.table.Table: + """ + Query the Gaia catalog for sources around a given position + + :param ra_deg: Right ascension in degrees + :param dec_deg: Declination in degrees + :return: Table of sources + """ + tmass_cat = VizierCatalog.get_catalog(self, ra_deg, dec_deg) + + tmass_cat.rename_column("Jmag", "j_m") + tmass_cat.rename_column("Hmag", "h_m") + tmass_cat.rename_column("Kmag", "k_m") + self.convert_to_ab_mag(tmass_cat) + + # Previously, the magnitude was set in vegamag, but now it is in AB mag` + tmass_cat["magnitude"] = tmass_cat[f"{self.filter_name.lower()}_m"] + + tmass_cat.rename_column("Qflg", "ph_qual") + + tmass_cat["ra_errdeg"] = tmass_cat["e_RA_ICRS"] / 3.6e6 + tmass_cat["dec_errdeg"] = tmass_cat["e_DE_ICRS"] / 3.6e6 + + tmass_cat.rename_column("RPmag", "phot_rp_mean_mag") + tmass_cat.rename_column("BPmag", "phot_bp_mean_mag") + tmass_cat.rename_column("e_Jmag", "j_msigcom") + tmass_cat.rename_column("e_Hmag", "h_msigcom") + tmass_cat.rename_column("e_Kmag", "k_msigcom") + return tmass_cat + + def join_query(self, query: dict) -> astropy.table.Table: + """ + Join the two queries together + + :param query: + :return: + """ + return join(query[0], query[1], keys="_2MASS") diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index 611800d06..d2c5ddabc 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -14,7 +14,7 @@ from astropy.wcs import WCS, NoConvergence from mirar.catalog import Gaia2Mass -from mirar.catalog.base_catalog import CatalogFromFile +from mirar.catalog.base.catalog_from_file import CatalogFromFile from mirar.catalog.vizier import PS1 from mirar.data import Image, ImageBatch, SourceBatch from mirar.data.utils.compress import decode_img diff --git a/mirar/processors/astromatic/scamp/scamp.py b/mirar/processors/astromatic/scamp/scamp.py index bda37febd..1fadeae0d 100644 --- a/mirar/processors/astromatic/scamp/scamp.py +++ b/mirar/processors/astromatic/scamp/scamp.py @@ -11,7 +11,7 @@ import numpy as np from astropy.io import fits -from mirar.catalog.base_catalog import BaseCatalog +from mirar.catalog.base.base_catalog import BaseCatalog from mirar.data import Image, ImageBatch from mirar.paths import ( BASE_NAME_KEY, diff --git a/mirar/processors/astrometry/validate.py b/mirar/processors/astrometry/validate.py index fbd7f8e53..e77aa8008 100644 --- a/mirar/processors/astrometry/validate.py +++ b/mirar/processors/astrometry/validate.py @@ -10,7 +10,7 @@ from astropy.stats import sigma_clipped_stats from astropy.table import Table -from mirar.catalog.base_catalog import BaseCatalog +from mirar.catalog.base.base_catalog import BaseCatalog from mirar.data import Image, ImageBatch from mirar.errors import ProcessorError from mirar.paths import get_output_dir diff --git a/mirar/processors/photcal/photcalibrator.py b/mirar/processors/photcal/photcalibrator.py index 1908a31d1..22cb480bb 100644 --- a/mirar/processors/photcal/photcalibrator.py +++ b/mirar/processors/photcal/photcalibrator.py @@ -12,7 +12,7 @@ from astropy.io.fits.verify import VerifyWarning from astropy.table import Table -from mirar.catalog.base_catalog import BaseCatalog +from mirar.catalog.base.base_catalog import BaseCatalog from mirar.data import Image, ImageBatch from mirar.paths import MAGLIM_KEY, ZP_KEY, ZP_NSTARS_KEY, ZP_STD_KEY, get_output_dir from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map diff --git a/mirar/processors/xmatch.py b/mirar/processors/xmatch.py index 2a0836f7b..3fe3c85ab 100644 --- a/mirar/processors/xmatch.py +++ b/mirar/processors/xmatch.py @@ -8,7 +8,7 @@ import numpy as np from astropy.coordinates import SkyCoord -from mirar.catalog.base_catalog import BaseXMatchCatalog +from mirar.catalog.base.base_xmatch_catalog import BaseXMatchCatalog from mirar.data import SourceBatch from mirar.processors.base_processor import BaseSourceProcessor diff --git a/tests/test_winter_pipeline.py b/tests/test_winter_pipeline.py index b7b4adb35..132938569 100644 --- a/tests/test_winter_pipeline.py +++ b/tests/test_winter_pipeline.py @@ -13,61 +13,61 @@ logger = logging.getLogger(__name__) expected_zp = { - "ZP_2.0": 23.959992146021143, - "ZP_2.0_std": 0.04468593502379257, - "ZP_2.0_nstars": 118, - "ZP_3.0": 24.488647137271013, - "ZP_3.0_std": 0.043678000162985946, - "ZP_3.0_nstars": 118, - "ZP_4.0": 24.67403631422544, - "ZP_4.0_std": 0.040829710238539714, - "ZP_4.0_nstars": 118, - "ZP_5.0": 24.736553404147642, - "ZP_5.0_std": 0.040354725833759444, - "ZP_5.0_nstars": 118, - "ZP_6.0": 24.77188170392595, - "ZP_6.0_std": 0.039765732833753016, - "ZP_6.0_nstars": 118, - "ZP_7.0": 24.782397765144424, - "ZP_7.0_std": 0.04069653155959119, - "ZP_7.0_nstars": 118, - "ZP_8.0": 24.799413106226112, - "ZP_8.0_std": 0.040876147446689284, - "ZP_8.0_nstars": 118, - "ZP_AUTO": 24.787539557156656, - "ZP_AUTO_std": 0.04316430194985392, - "ZP_AUTO_nstars": 118, - "ZP_PSF": 24.680094141593663, - "ZP_PSF_std": 0.03875121055985624, - "ZP_PSF_nstars": 117, - "SCORMEAN": -0.07678255815161257, - "SCORMED": -0.07307016879473982, - "SCORSTD": 1.2836771224604269, + "ZP_2.0": 24.039165434205593, + "ZP_2.0_std": 0.040682124370149605, + "ZP_2.0_nstars": 111, + "ZP_3.0": 24.448507155050052, + "ZP_3.0_std": 0.0471318743148075, + "ZP_3.0_nstars": 111, + "ZP_4.0": 24.635763963930522, + "ZP_4.0_std": 0.04354799201975238, + "ZP_4.0_nstars": 111, + "ZP_5.0": 24.716636555577335, + "ZP_5.0_std": 0.04169164522178617, + "ZP_5.0_nstars": 111, + "ZP_6.0": 24.7431289901993, + "ZP_6.0_std": 0.041704603416498676, + "ZP_6.0_nstars": 111, + "ZP_7.0": 24.76461130331333, + "ZP_7.0_std": 0.04157117729511644, + "ZP_7.0_nstars": 111, + "ZP_8.0": 24.777898334765396, + "ZP_8.0_std": 0.041735663104911024, + "ZP_8.0_nstars": 111, + "ZP_AUTO": 24.783530950220236, + "ZP_AUTO_std": 0.04301252694189325, + "ZP_AUTO_nstars": 111, + "ZP_PSF": 24.662168539775998, + "ZP_PSF_std": 0.04163357501847084, + "ZP_PSF_nstars": 110, + "SCORMEAN": -0.07430693836056637, + "SCORMED": -0.07059966355274964, + "SCORSTD": 1.2762548812602645, } expected_dataframe_values = { "magpsf": [ - 16.39444674612076, - 15.405810813687662, - 17.082173218426902, - 17.082294915192634, - 17.403596735252293, - 17.160933991469214, - 14.66480196304238, - 14.657302634953442, - 16.816141837382137, - 16.25453113498711, + 16.539472228516754, + 14.816334995321805, + 15.486920360136258, + 17.178854595239663, + 16.357246610808602, + 16.813796280840393, + 17.17955668916707, + 17.208511947122744, + 14.795105119473721, + 16.686258217055684, ], "magap": [ - 16.48067858723591, - 15.915934341171843, - 18.53190378870282, - 18.264227223646593, - 16.264918566468864, - 17.122984755361113, - 15.45802381805369, - 15.202372311912047, - 16.160387918403462, - 16.456876102361825, + 16.320732009563713, + 15.413317709017791, + 16.223866004796673, + 18.095085591114476, + 16.959889773064127, + 16.88041688053796, + 19.116076621089814, + 16.984351826663126, + 15.198017344346496, + 16.06515983909224, ], } @@ -146,7 +146,7 @@ def test_pipeline(self): candidates_table = source_table.get_data() - self.assertEqual(len(candidates_table), 1295) + self.assertEqual(len(candidates_table), 1274) for key, value in expected_dataframe_values.items(): if isinstance(value, list): for ind, val in enumerate(value):