Refactor catalogs, introduce backup Gaia via Vizier (#849)

mirar/catalog/__init__.py

@@ -3,5 +3,5 @@
 The basic structure is contained in :class:`mirar.catalog.base_catalog`
 """
 
-from mirar.catalog.base_catalog import BaseCatalog
-from mirar.catalog.gaia import Gaia2Mass
+from mirar.catalog.base.base_catalog import BaseCatalog
+from mirar.catalog.multibackend import Gaia2Mass

mirar/catalog/base/__init__.py

@@ -0,0 +1,7 @@
+"""
+Base classes for the catalog system.
+"""
+
+from mirar.catalog.base.base_catalog import BaseCatalog, BaseMultiBackendCatalog
+from mirar.catalog.base.base_xmatch_catalog import BaseXMatchCatalog
+from mirar.catalog.base.errors import CatalogError

mirar/catalog/base_catalog.py → mirar/catalog/base/base_catalog.py

@@ -5,30 +5,19 @@
 import logging
 from abc import ABC
 from pathlib import Path
+from typing import Type
 
 import astropy.table
 
+from mirar.catalog.base.errors import CatalogCacheError
 from mirar.data import Image
 from mirar.data.utils import get_image_center_wcs_coords
-from mirar.errors import ProcessorError
 from mirar.paths import BASE_NAME_KEY, REF_CAT_PATH_KEY
-from mirar.utils.ldac_tools import get_table_from_ldac, save_table_as_ldac
+from mirar.utils.ldac_tools import save_table_as_ldac
 
 logger = logging.getLogger(__name__)
 
 
-class CatalogError(ProcessorError):
-    """
-    Class for errors in Catalog
-    """
-
-
-class CatalogCacheError(CatalogError):
-    """
-    Class for errors in CatalogCache
-    """
-
-
 class ABCatalog:
     """
     Abstract class for catalog objects
@@ -139,88 +128,20 @@ def get_output_path(self, output_dir: Path, base_name: str | Path) -> Path:
         return output_dir.joinpath(cat_base_name)
 
 
-class BaseXMatchCatalog(ABCatalog, ABC):
+class BaseMultiBackendCatalog(ABC):
     """
-    Base Catalog for crossmatching
+    Base class for composite catalogs that are made up of multiple catalog backends
     """
 
-    @property
-    def catalog_name(self):
-        """
-        Name of catalog
-        """
-        raise NotImplementedError
+    def __new__(cls, *args, backend: str | None = None, **kwargs):
+        backend_class = cls.set_backend(backend)
+        return backend_class(*args, **kwargs)
 
-    @property
-    def projection(self):
-        """
-        projection for kowalski xmatch
-        """
-        raise NotImplementedError
-
-    @property
-    def column_names(self):
+    @staticmethod
+    def set_backend(backend: str | None) -> Type[BaseCatalog]:
         """
-        Name of columns
-        """
-        raise NotImplementedError
+        Set backend for composite catalog
 
-    @property
-    def column_dtypes(self):
-        """
-        dtype of columns
+        :param backend: Backend name
         """
-        raise NotImplementedError
-
-    @property
-    def ra_column_name(self):
-        """
-        Name of RA column
-        """
-        raise NotImplementedError
-
-    @property
-    def dec_column_name(self):
-        """
-        Name of Dec column
-        """
-        raise NotImplementedError
-
-    def __init__(self, *args, num_sources: int = 1, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.search_radius_arcsec = self.search_radius_arcmin * 60.0
-        self.num_sources = num_sources
-
-    def query(self, coords: dict) -> dict:
-        """
-        Query coords for result
-
-        :param coords: ra/dec
-        :return: crossmatch
-        """
-        raise NotImplementedError
-
-
-class CatalogFromFile(BaseCatalog):
-    """
-    Local catalog from file
-    """
-
-    abbreviation = "local"
-
-    def __init__(self, catalog_path: str | Path, *args, **kwargs):
-        super().__init__(
-            min_mag=0,
-            max_mag=99,
-            filter_name="None",
-            search_radius_arcmin=0,
-            *args,
-            **kwargs,
-        )
-        self.catalog_path = catalog_path
-        if isinstance(self.catalog_path, str):
-            self.catalog_path = Path(self.catalog_path)
-
-    def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table:
-        catalog = get_table_from_ldac(self.catalog_path)
-        return catalog
+        raise NotImplementedError()

mirar/catalog/gaia.py → mirar/catalog/base/base_gaia.py

@@ -3,32 +3,28 @@
 """
 
 import logging
+from abc import ABC
 from typing import Optional
 
 import astropy.table
 import astropy.units as u
 import numpy as np
 from astropy.coordinates import SkyCoord
-from astroquery.gaia import Gaia
 
-from mirar.catalog.base_catalog import BaseCatalog
+from mirar.catalog.base.base_catalog import BaseCatalog
 from mirar.utils.ldac_tools import get_table_from_ldac
 
 logger = logging.getLogger(__name__)
 
-# Disable astroquery warnings
-logging.getLogger("astroquery").setLevel(logging.WARNING)
-
-
 # 2MASS values from https://iopscience.iop.org/article/10.1086/376474
 zeromag_2mass = {"j": 1594.0 * u.Jansky, "h": 1024.0 * u.Jansky, "k": 666.8 * u.Jansky}
 
 offsets_2mass = {key: zm.to("mag(AB)").value for key, zm in zeromag_2mass.items()}
 
 
-class Gaia2Mass(BaseCatalog):
+class BaseGaia2Mass(BaseCatalog, ABC):
     """
-    Crossmatched Gaia/2Mass catalog
+    Base Gaia/2Mass catalog
     """
 
     abbreviation = "tmass"
@@ -77,48 +73,27 @@ def __init__(
             if val is None:
                 self.acceptable_ph_quals[filt] = ["A", "B", "C"]
 
+    def convert_to_ab_mag(self, src_list: astropy.table.Table) -> astropy.table.Table:
+        """
+        Convert 2MASS magnitudes to AB magnitudes
+
+        :param src_list: Source list
+        :return: Source list with AB magnitudes
+        """
+        # Convert to AB magnitudes
+        for key in ["j", "h", "k"]:
+            offset = offsets_2mass[self.filter_name.lower()]
+            src_list[f"{key}_m"] += offset
+            logger.debug(f"Adding {offset:.2f} to convert from 2MASS to AB magnitudes")
+        return src_list
+
     def get_catalog(
         self,
         ra_deg: float,
         dec_deg: float,
     ) -> astropy.table.Table:
-        logger.debug(
-            f"Querying 2MASS - Gaia cross-match around RA {ra_deg:.4f}, "
-            f"Dec {dec_deg:.4f} with a radius of {self.search_radius_arcmin:.4f} arcmin"
-        )
-
-        cmd = (
-            f"SELECT * FROM gaiadr2.gaia_source AS g, "
-            f"gaiadr2.tmass_best_neighbour AS tbest, "
-            f"gaiadr1.tmass_original_valid AS tmass "
-            f"WHERE g.source_id = tbest.source_id "
-            f"AND tbest.tmass_oid = tmass.tmass_oid "
-            f"AND CONTAINS(POINT('ICRS', g.ra, g.dec), "
-            f"CIRCLE('ICRS', {ra_deg:.4f}, {dec_deg:.4f}, "
-            f"{self.search_radius_arcmin / 60:.4f}))=1 "
-            f"AND tmass.{self.filter_name}_m > {self.min_mag:.2f} "
-            f"AND tmass.{self.filter_name}_m < {self.max_mag:.2f} "
-            f"AND tbest.number_of_mates=0 "
-            f"AND tbest.number_of_neighbours=1;"
-        )
-
-        job = Gaia.launch_job_async(cmd, dump_to_file=False)
-        src_list = job.get_results()
-        src_list["ph_qual"] = src_list["ph_qual"].astype(str)
-        src_list["ra_errdeg"] = src_list["ra_error"] / 3.6e6
-        src_list["dec_errdeg"] = src_list["dec_error"] / 3.6e6
-        src_list["FLAGS"] = 0
-        src_list["magnitude"] = src_list[f"{self.filter_name.lower()}_m"]
-        src_list["magnitude_err"] = src_list[f"{self.filter_name.lower()}_msigcom"]
-        logger.debug(f"Found {len(src_list)} sources in Gaia")
-
-        # Convert to AB magnitudes
-
-        offset = offsets_2mass[self.filter_name.lower()]
 
-        logger.debug(f"Adding {offset:.2f} to convert from 2MASS to AB magnitudes")
-
-        src_list["magnitude"] += offset
+        src_list = self.get_source_table(ra_deg, dec_deg)
 
         j_phquals = [x[0] for x in src_list["ph_qual"]]
         h_phquals = [x[1] for x in src_list["ph_qual"]]
@@ -134,11 +109,12 @@ def get_catalog(
         src_list = src_list[src_list["magnitude_err"] < 1.086 / self.snr_threshold]
         if self.trim:
             if self.image_catalog_path is None:
-                logger.error(
-                    "Gaia catalog trimming requested but "
-                    "no sextractor catalog path specified."
+                err = (
+                    "Gaia catalog trimming requested "
+                    "but no sextractor catalog path specified."
                 )
-                raise ValueError
+                logger.error(err)
+                raise ValueError(err)
 
             image_catalog = get_table_from_ldac(self.image_catalog_path)
             src_list = self.trim_catalog(src_list, image_catalog)
@@ -170,3 +146,13 @@ def trim_catalog(
         match_mask = d2d < 2 * u.arcsec
         matched_catalog = ref_catalog[idx[match_mask]]
         return matched_catalog
+
+    def get_source_table(self, ra_deg: float, dec_deg: float) -> astropy.table.Table:
+        """
+        Get the source table for a given position
+
+        :param ra_deg: RA in degrees
+        :param dec_deg: Dec in degrees
+        :return: Table
+        """
+        raise NotImplementedError

mirar/catalog/base/base_xmatch_catalog.py

@@ -0,0 +1,69 @@
+"""
+Base Catalog for crossmatching
+"""
+
+from abc import ABC
+
+from mirar.catalog.base.base_catalog import ABCatalog
+
+
+class BaseXMatchCatalog(ABCatalog, ABC):
+    """
+    Base Catalog for crossmatching
+    """
+
+    @property
+    def catalog_name(self):
+        """
+        Name of catalog
+        """
+        raise NotImplementedError
+
+    @property
+    def projection(self):
+        """
+        projection for kowalski xmatch
+        """
+        raise NotImplementedError
+
+    @property
+    def column_names(self):
+        """
+        Name of columns
+        """
+        raise NotImplementedError
+
+    @property
+    def column_dtypes(self):
+        """
+        dtype of columns
+        """
+        raise NotImplementedError
+
+    @property
+    def ra_column_name(self):
+        """
+        Name of RA column
+        """
+        raise NotImplementedError
+
+    @property
+    def dec_column_name(self):
+        """
+        Name of Dec column
+        """
+        raise NotImplementedError
+
+    def __init__(self, *args, num_sources: int = 1, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.search_radius_arcsec = self.search_radius_arcmin * 60.0
+        self.num_sources = num_sources
+
+    def query(self, coords: dict) -> dict:
+        """
+        Query coords for result
+
+        :param coords: ra/dec
+        :return: crossmatch
+        """
+        raise NotImplementedError

mirar/catalog/base/catalog_from_file.py

@@ -0,0 +1,35 @@
+"""
+Module for Catalog base class
+"""
+
+from pathlib import Path
+
+import astropy.table
+
+from mirar.catalog.base.base_catalog import BaseCatalog
+from mirar.utils.ldac_tools import get_table_from_ldac
+
+
+class CatalogFromFile(BaseCatalog):
+    """
+    Local catalog from file
+    """
+
+    abbreviation = "local"
+
+    def __init__(self, catalog_path: str | Path, *args, **kwargs):
+        super().__init__(
+            min_mag=0,
+            max_mag=99,
+            filter_name="None",
+            search_radius_arcmin=0,
+            *args,
+            **kwargs,
+        )
+        self.catalog_path = catalog_path
+        if isinstance(self.catalog_path, str):
+            self.catalog_path = Path(self.catalog_path)
+
+    def get_catalog(self, ra_deg: float, dec_deg: float) -> astropy.table.Table:
+        catalog = get_table_from_ldac(self.catalog_path)
+        return catalog