From 95f666afb2fe797ab324fb3272bc98796f97a53f Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Sun, 17 Nov 2024 19:16:43 +0000 Subject: [PATCH 01/16] Refactored data file access to use get_pkg_data_filename instead of self-written wrappers around download_data. --- specreduce/calibration_data.py | 222 +++++++++------------------------ 1 file changed, 56 insertions(+), 166 deletions(-) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index 51a3e49..b022417 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -5,10 +5,11 @@ import warnings from pathlib import Path from typing import Sequence +from urllib.error import HTTPError from astropy import units as u from astropy.table import Table, vstack, QTable -from astropy.utils.data import download_file +from astropy.utils.data import get_pkg_data_filename, conf from astropy.utils.exceptions import AstropyUserWarning from astropy.coordinates import SpectralCoord @@ -16,8 +17,6 @@ from specutils.utils.wcs_utils import vac_to_air __all__ = [ - 'get_reference_file_path', - 'get_pypeit_data_path', 'get_available_line_catalogs', 'load_pypeit_calibration_lines', 'load_MAST_calspec', @@ -89,6 +88,8 @@ 'ArI' ] +_SPECREDUCE_DATA_URL = "https://raw.githubusercontent.com/astropy/specreduce-data/main/specreduce_data/reference_data" +_PYPEIT_DATA_URL = "https://raw.githubusercontent.com/pypeit/pypeit/release/pypeit/data" def get_available_line_catalogs() -> dict: """ @@ -100,112 +101,6 @@ def get_available_line_catalogs() -> dict: } -def get_reference_file_path( - path: str | Path | None = None, - cache: bool = True, - repo_url: str = "https://raw.githubusercontent.com/astropy/specreduce-data", - repo_branch: str = "main", - repo_data_path: str = "specreduce_data/reference_data", - show_progress: bool = False -) -> Path | None: - """ - Utility to load reference data via GitHub raw user content. By default the ``specreduce_data`` - repository at https://github.com/astropy/specreduce-data is used. - - Parameters - ---------- - path : Path of reference file relative to the reference_data directory within - specified package. - - cache : Set whether file is cached if file is downloaded. - - repo_url : Base repository URL for the reference data. - - repo_branch : Branch of repository from which to fetch the reference data. - - repo_data_path : Path within the repository where the reference data is located. - - show_progress : Set whether download progress bar is shown if file is downloaded. - - Returns - ------- - file_path : Local path to reference data file or None if the path cannot be constructed - or if the file itself is not valid. - - Examples - -------- - >>> from specreduce.calibration_data import get_reference_file_path - >>> kpno_extinction_file = get_reference_file_path("extinction/kpnoextinct.dat") - """ - if path is None: - return None - - remote_url = f"{repo_url}/{repo_branch}/{repo_data_path}/{path}" - try: - file_path = Path( - download_file( - remote_url, - cache=cache, - show_progress=show_progress, - pkgname='specreduce' - ) - ) - except Exception as e: - msg = f"Downloading of {remote_url} failed: {e}" - warnings.warn(msg, AstropyUserWarning) - return None - - # final sanity check to make sure file_path is actually a file. - if file_path.exists() and file_path.is_file(): - return file_path - else: - warnings.warn(f"Able to construct {file_path}, but it is not a file.") - return None - - -def get_pypeit_data_path( - path: str | Path | None = None, - cache: bool = True, - show_progress: bool = False -) -> Path | None: - """ - Convenience utility to facilitate access to ``pypeit`` reference data. The data is accessed - directly from the release branch on GitHub and downloaded/cached - using `~astropy.utils.data.download_file`. - - Parameters - ---------- - path : Filename of reference file relative to the reference_data directory within - ``specreduce_data`` package. - - cache : Set whether file is cached if file is downloaded. - - show_progress : Set whether download progress bar is shown if file is downloaded. - - Returns - ------- - file_path : Path to reference data file or None if the path cannot be - constructed or if the file itself is not valid. - - Examples - -------- - >>> from specreduce.calibration_data import get_pypeit_data_path - >>> pypeit_he_linelist = get_pypeit_data_path("arc_lines/lists/HeI_lines.dat") - """ - repo_url = "https://raw.githubusercontent.com/pypeit/pypeit" - repo_branch = "release" - repo_data_path = "pypeit/data" - - return get_reference_file_path( - path=path, - cache=cache, - repo_url=repo_url, - repo_branch=repo_branch, - repo_data_path=repo_data_path, - show_progress=show_progress - ) - - def load_pypeit_calibration_lines( lamps: Sequence | None = None, wave_air: bool = False, @@ -255,38 +150,37 @@ def load_pypeit_calibration_lines( else: lamps = [lamps] - linelists = [] - for lamp in lamps: - if lamp in PYPEIT_CALIBRATION_LINELISTS: - list_path = f"arc_lines/lists/{lamp}_lines.dat" - lines_file = get_pypeit_data_path( - list_path, - cache=cache, - show_progress=show_progress - ) - lines_tab = Table.read( - lines_file, - format='ascii.fixed_width', - comment='#' - ) - if lines_tab is not None: - linelists.append(lines_tab) + with conf.set_temp("dataurl", _PYPEIT_DATA_URL): + linelists = [] + for lamp in lamps: + if lamp in PYPEIT_CALIBRATION_LINELISTS: + list_path = f"arc_lines/lists/{lamp}_lines.dat" + lines_file = get_pkg_data_filename(list_path, + package="specreduce", + show_progress=show_progress) + lines_tab = Table.read( + lines_file, + format='ascii.fixed_width', + comment='#' + ) + if lines_tab is not None: + linelists.append(lines_tab) + else: + warnings.warn( + f"{lamp} not in the list of supported calibration " + "line lists: {PYPEIT_CALIBRATION_LINELISTS}." + ) + if len(linelists) == 0: + warnings.warn(f"No calibration lines loaded from {lamps}.") + linelist = None else: - warnings.warn( - f"{lamp} not in the list of supported calibration " - "line lists: {PYPEIT_CALIBRATION_LINELISTS}." - ) - if len(linelists) == 0: - warnings.warn(f"No calibration lines loaded from {lamps}.") - linelist = None - else: - linelist = QTable(vstack(linelists)) - linelist.rename_column('wave', 'wavelength') - # pypeit linelists use vacuum wavelengths in angstroms - linelist['wavelength'] *= u.Angstrom - if wave_air: - linelist['wavelength'] = vac_to_air(linelist['wavelength']) - return linelist + linelist = QTable(vstack(linelists)) + linelist.rename_column('wave', 'wavelength') + # pypeit linelists use vacuum wavelengths in angstroms + linelist['wavelength'] *= u.Angstrom + if wave_air: + linelist['wavelength'] = vac_to_air(linelist['wavelength']) + return linelist def load_MAST_calspec( @@ -326,12 +220,7 @@ def load_MAST_calspec( else: url = f"https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/{filename}" try: - file_path = download_file( - url, - cache=cache, - show_progress=show_progress, - pkgname='specreduce' - ) + file_path = get_pkg_data_filename(url, package='specreduce', show_progress=show_progress) except Exception as e: msg = f"Downloading of {url} failed: {e}" warnings.warn(msg, AstropyUserWarning) @@ -385,15 +274,16 @@ def load_onedstds( warnings.warn(msg, AstropyUserWarning) return None - spec_path = get_reference_file_path( - path=Path("onedstds") / Path(dataset) / Path(specfile), - cache=cache, - show_progress=show_progress - ) - if spec_path is None: - msg = f"Can't load {specfile} from {dataset}." - warnings.warn(msg, AstropyUserWarning) - return None + with conf.set_temp("dataurl", _SPECREDUCE_DATA_URL): + try: + spec_path = get_pkg_data_filename( + str(Path("onedstds") / Path(dataset) / Path(specfile)), + show_progress=show_progress + ) + except HTTPError: + msg = f"Can't load {specfile} from {dataset}." + warnings.warn(msg, AstropyUserWarning) + return None t = Table.read(spec_path, format="ascii", names=['wavelength', 'ABmag', 'binsize']) @@ -481,13 +371,12 @@ def __init__( f"of available models: {SUPPORTED_EXTINCTION_MODELS}" ) raise ValueError(msg) - model_file = Path("extinction") / Path(f"{model}extinct.dat") - model_path = get_reference_file_path( - path=model_file, - cache=cache, - show_progress=show_progress - ) - t = Table.read(model_path, format="ascii", names=['wavelength', 'extinction']) + with conf.set_temp("dataurl", _SPECREDUCE_DATA_URL): + model_path = get_pkg_data_filename( + str(Path("extinction") / Path(f"{model}extinct.dat")), + show_progress=show_progress + ) + t = Table.read(model_path, format="ascii", names=['wavelength', 'extinction']) # the specreduce_data models all provide wavelengths in angstroms spectral_axis = t['wavelength'].data * u.angstrom @@ -547,10 +436,11 @@ def __init__( **kwargs: str ) -> None: if data_file is None: - data_path = Path("extinction") / Path("atm_trans_am1.0.dat") - data_file = get_reference_file_path(path=data_path) - - t = Table.read(Path(data_file), format="ascii", names=['wavelength', 'extinction']) + with conf.set_temp("dataurl", _SPECREDUCE_DATA_URL): + data_file = get_pkg_data_filename( + str(Path("extinction") / Path("atm_trans_am1.0.dat")), + ) + t = Table.read(Path(data_file), format="ascii", names=['wavelength', 'extinction']) # spectral axis is given in microns spectral_axis = t['wavelength'].data * wave_unit From ab920e512bd0ab522c8b4c081f88c5899e00e609 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Mon, 18 Nov 2024 18:44:04 +0000 Subject: [PATCH 02/16] Updated data retrieval in calibration_data.py to use get_pkg_data_fileobj and get_pkg_data_filename and translate URLError errors into warnings. --- specreduce/calibration_data.py | 85 +++++++++++++++++----------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index b022417..0ed8c61 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -5,11 +5,11 @@ import warnings from pathlib import Path from typing import Sequence -from urllib.error import HTTPError +from urllib.error import URLError from astropy import units as u from astropy.table import Table, vstack, QTable -from astropy.utils.data import get_pkg_data_filename, conf +from astropy.utils.data import get_pkg_data_filename, conf, get_pkg_data_fileobj from astropy.utils.exceptions import AstropyUserWarning from astropy.coordinates import SpectralCoord @@ -88,8 +88,11 @@ 'ArI' ] -_SPECREDUCE_DATA_URL = "https://raw.githubusercontent.com/astropy/specreduce-data/main/specreduce_data/reference_data" -_PYPEIT_DATA_URL = "https://raw.githubusercontent.com/pypeit/pypeit/release/pypeit/data" +SPECREDUCE_DATA_URL = ("https://raw.githubusercontent.com/astropy/specreduce-data/" + "main/specreduce_data/reference_data/") + +PYPEIT_DATA_URL = ("https://raw.githubusercontent.com/pypeit/" + "pypeit/release/pypeit/data/") def get_available_line_catalogs() -> dict: """ @@ -150,21 +153,19 @@ def load_pypeit_calibration_lines( else: lamps = [lamps] - with conf.set_temp("dataurl", _PYPEIT_DATA_URL): + with conf.set_temp("dataurl", PYPEIT_DATA_URL): linelists = [] for lamp in lamps: if lamp in PYPEIT_CALIBRATION_LINELISTS: - list_path = f"arc_lines/lists/{lamp}_lines.dat" - lines_file = get_pkg_data_filename(list_path, - package="specreduce", - show_progress=show_progress) - lines_tab = Table.read( - lines_file, - format='ascii.fixed_width', - comment='#' - ) - if lines_tab is not None: + try: + data_path = f"arc_lines/lists/{lamp}_lines.dat" + with get_pkg_data_fileobj(data_path, cache=cache) as data_file: + lines_tab = Table.read(data_file.read(), + format='ascii.fixed_width', + comment='#') linelists.append(lines_tab) + except URLError as e: + warnings.warn(f"Downloading of {data_path} failed: {e}", AstropyUserWarning) else: warnings.warn( f"{lamp} not in the list of supported calibration " @@ -218,13 +219,14 @@ def load_MAST_calspec( if filename.exists() and filename.is_file(): file_path = filename else: - url = f"https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/{filename}" - try: - file_path = get_pkg_data_filename(url, package='specreduce', show_progress=show_progress) - except Exception as e: - msg = f"Downloading of {url} failed: {e}" - warnings.warn(msg, AstropyUserWarning) - file_path = None + url = "https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/" + with conf.set_temp("dataurl", url): + try: + file_path = get_pkg_data_filename(str(filename), show_progress=show_progress) + except URLError as e: + msg = f"Downloading of {filename} failed: {e}" + warnings.warn(msg, AstropyUserWarning) + file_path = None if file_path is None: return None @@ -274,19 +276,18 @@ def load_onedstds( warnings.warn(msg, AstropyUserWarning) return None - with conf.set_temp("dataurl", _SPECREDUCE_DATA_URL): + with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): try: - spec_path = get_pkg_data_filename( - str(Path("onedstds") / Path(dataset) / Path(specfile)), - show_progress=show_progress - ) - except HTTPError: - msg = f"Can't load {specfile} from {dataset}." + data_path = str(Path("onedstds") / Path(dataset) / Path(specfile)) + with get_pkg_data_fileobj(data_path, cache=cache) as data_file: + t = Table.read(data_file.read(), + format="ascii", + names=['wavelength', 'ABmag', 'binsize']) + except URLError as e: + msg = f"Can't load {specfile} from {dataset}: {e}." warnings.warn(msg, AstropyUserWarning) return None - t = Table.read(spec_path, format="ascii", names=['wavelength', 'ABmag', 'binsize']) - # the specreduce_data standard star spectra all provide wavelengths in angstroms spectral_axis = t['wavelength'].data * u.angstrom @@ -371,12 +372,12 @@ def __init__( f"of available models: {SUPPORTED_EXTINCTION_MODELS}" ) raise ValueError(msg) - with conf.set_temp("dataurl", _SPECREDUCE_DATA_URL): - model_path = get_pkg_data_filename( - str(Path("extinction") / Path(f"{model}extinct.dat")), - show_progress=show_progress - ) - t = Table.read(model_path, format="ascii", names=['wavelength', 'extinction']) + with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): + data_path = str(Path("extinction") / Path(f"{model}extinct.dat")) + with get_pkg_data_fileobj(data_path, cache=cache) as data_file: + t = Table.read(data_file.read(), + format="ascii", + names=['wavelength', 'extinction']) # the specreduce_data models all provide wavelengths in angstroms spectral_axis = t['wavelength'].data * u.angstrom @@ -436,11 +437,11 @@ def __init__( **kwargs: str ) -> None: if data_file is None: - with conf.set_temp("dataurl", _SPECREDUCE_DATA_URL): - data_file = get_pkg_data_filename( - str(Path("extinction") / Path("atm_trans_am1.0.dat")), - ) - t = Table.read(Path(data_file), format="ascii", names=['wavelength', 'extinction']) + with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): + data_path = str(Path("extinction") / Path("atm_trans_am1.0.dat")) + data_file = get_pkg_data_filename(data_path) + + t = Table.read(data_file, format="ascii", names=['wavelength', 'extinction']) # spectral axis is given in microns spectral_axis = t['wavelength'].data * wave_unit From 714dc6cf76ed4266959b5b357f2e28e4b8c4b834 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Mon, 18 Nov 2024 19:02:55 +0000 Subject: [PATCH 03/16] Added a missing blank line. --- specreduce/calibration_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index 0ed8c61..bb4dafe 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -94,6 +94,7 @@ PYPEIT_DATA_URL = ("https://raw.githubusercontent.com/pypeit/" "pypeit/release/pypeit/data/") + def get_available_line_catalogs() -> dict: """ Returns a dictionary of available line catalogs. Currently only ``pypeit`` From e775f3972aa49433014ebb0cf6aec8a447fd5ebc Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Mon, 18 Nov 2024 19:15:59 +0000 Subject: [PATCH 04/16] Removed test_get_reference_file_path.py. --- .../tests/test_get_reference_file_path.py | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 specreduce/tests/test_get_reference_file_path.py diff --git a/specreduce/tests/test_get_reference_file_path.py b/specreduce/tests/test_get_reference_file_path.py deleted file mode 100644 index d619f2c..0000000 --- a/specreduce/tests/test_get_reference_file_path.py +++ /dev/null @@ -1,23 +0,0 @@ -import pytest - -from specreduce.calibration_data import get_reference_file_path, get_pypeit_data_path - - -@pytest.mark.remote_data -def test_get_reference_file_path(): - """ - Test to make sure a calibration reference file provided by specreduce_data can be accessed. - """ - test_path = "extinction/apoextinct.dat" - p = get_reference_file_path(path=test_path) - assert p is not None - - -@pytest.mark.remote_data -def test_get_pypeit_data_path(): - """ - Test to make sure pypeit reference data can be loaded - """ - test_path = "arc_lines/lists/HeI_lines.dat" - p = get_pypeit_data_path(path=test_path, show_progress=False) - assert p is not None From db5d5ad8b41c0fb6b2d4c9e51d9316228ce9e6df Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Thu, 28 Nov 2024 20:41:04 +0000 Subject: [PATCH 05/16] - Cleaned up the code dealing with downloading data files in calibration_data. - Marked several of the function arguments used by the previous approach as deprecated (pending) using deprecated_renamed_argument. - Improved tests to bring calibration_data coverage to 87%. --- specreduce/calibration_data.py | 25 +++++++++++-------------- specreduce/tests/test_extinction.py | 9 +++++++++ specreduce/tests/test_linelists.py | 9 +++++++++ specreduce/tests/test_specphot_stds.py | 8 ++++++++ 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index bb4dafe..2a7fd57 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -9,6 +9,7 @@ from astropy import units as u from astropy.table import Table, vstack, QTable +from astropy.utils import deprecated_renamed_argument from astropy.utils.data import get_pkg_data_filename, conf, get_pkg_data_fileobj from astropy.utils.exceptions import AstropyUserWarning from astropy.coordinates import SpectralCoord @@ -105,6 +106,7 @@ def get_available_line_catalogs() -> dict: } +@deprecated_renamed_argument('show_progress', None, '1.4.2', pending=True) def load_pypeit_calibration_lines( lamps: Sequence | None = None, wave_air: bool = False, @@ -158,19 +160,16 @@ def load_pypeit_calibration_lines( linelists = [] for lamp in lamps: if lamp in PYPEIT_CALIBRATION_LINELISTS: + data_path = f"arc_lines/lists/{lamp}_lines.dat" try: - data_path = f"arc_lines/lists/{lamp}_lines.dat" with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - lines_tab = Table.read(data_file.read(), - format='ascii.fixed_width', - comment='#') - linelists.append(lines_tab) + linelists.append(Table.read(data_file.read(), format='ascii.fixed_width', comment='#')) except URLError as e: warnings.warn(f"Downloading of {data_path} failed: {e}", AstropyUserWarning) else: warnings.warn( f"{lamp} not in the list of supported calibration " - "line lists: {PYPEIT_CALIBRATION_LINELISTS}." + f"line lists: {PYPEIT_CALIBRATION_LINELISTS}." ) if len(linelists) == 0: warnings.warn(f"No calibration lines loaded from {lamps}.") @@ -185,6 +184,7 @@ def load_pypeit_calibration_lines( return linelist +@deprecated_renamed_argument('cache', None, '1.4.2', pending=True) def load_MAST_calspec( filename: str | Path, cache: bool = True, @@ -220,8 +220,7 @@ def load_MAST_calspec( if filename.exists() and filename.is_file(): file_path = filename else: - url = "https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/" - with conf.set_temp("dataurl", url): + with conf.set_temp("dataurl", "https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/"): try: file_path = get_pkg_data_filename(str(filename), show_progress=show_progress) except URLError as e: @@ -244,6 +243,7 @@ def load_MAST_calspec( return spectrum +@deprecated_renamed_argument('show_progress', None, '1.4.2', pending=True) def load_onedstds( dataset: str = "snfactory", specfile: str = "EG131.dat", @@ -281,9 +281,7 @@ def load_onedstds( try: data_path = str(Path("onedstds") / Path(dataset) / Path(specfile)) with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - t = Table.read(data_file.read(), - format="ascii", - names=['wavelength', 'ABmag', 'binsize']) + t = Table.read(data_file.read(), format="ascii", names=['wavelength', 'ABmag', 'binsize']) except URLError as e: msg = f"Can't load {specfile} from {dataset}: {e}." warnings.warn(msg, AstropyUserWarning) @@ -335,6 +333,7 @@ class AtmosphericExtinction(Spectrum1D): transmission : Extinction expressed as fractional transmission """ + @deprecated_renamed_argument('show_progress', None, '1.4.2', pending=True) def __init__( self, model: str = "kpno", @@ -376,9 +375,7 @@ def __init__( with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): data_path = str(Path("extinction") / Path(f"{model}extinct.dat")) with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - t = Table.read(data_file.read(), - format="ascii", - names=['wavelength', 'extinction']) + t = Table.read(data_file.read(), format="ascii", names=['wavelength', 'extinction']) # the specreduce_data models all provide wavelengths in angstroms spectral_axis = t['wavelength'].data * u.angstrom diff --git a/specreduce/tests/test_extinction.py b/specreduce/tests/test_extinction.py index 53b6b1a..1e91582 100644 --- a/specreduce/tests/test_extinction.py +++ b/specreduce/tests/test_extinction.py @@ -57,6 +57,15 @@ def test_custom_linear_model(): assert len(ext.transmission) > 0 +@pytest.mark.remote_data +def test_unsupported_model(): + """ + Test loading of a nonexistent model + """ + with pytest.raises(ValueError, match='Requested extinction model,'): + ext = AtmosphericExtinction(model='bad_model') + + @pytest.mark.remote_data def test_missing_extinction_unit(): """ diff --git a/specreduce/tests/test_linelists.py b/specreduce/tests/test_linelists.py index ab72562..81af54d 100644 --- a/specreduce/tests/test_linelists.py +++ b/specreduce/tests/test_linelists.py @@ -53,6 +53,15 @@ def test_pypeit_empty(): assert line_tab is None +@pytest.mark.remote_data +def test_pypeit_none(): + """ + Test to make sure None is returned if calibration lamp list is None. + """ + line_tab = load_pypeit_calibration_lines(None, cache=True, show_progress=False) + assert line_tab is None + + @pytest.mark.remote_data def test_pypeit_input_validation(): """ diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index ea15326..485326e 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -1,4 +1,5 @@ import pytest +from astropy.utils.exceptions import AstropyUserWarning from specreduce.calibration_data import load_MAST_calspec, load_onedstds @@ -15,3 +16,10 @@ def test_load_onedstds(): sp = load_onedstds() assert sp is not None assert len(sp.spectral_axis) > 0 + + +@pytest.mark.remote_data +def test_load_onedstds_bad_dataset(): + with pytest.warns(AstropyUserWarning, match="Specfied dataset,"): + sp = load_onedstds("snffactory") + assert sp is None From 673184797d5cfa6d0a935ba293814e8ea79edca8 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Fri, 29 Nov 2024 16:16:25 +0000 Subject: [PATCH 06/16] Improved calibration_data test coverage. --- specreduce/calibration_data.py | 4 ++-- specreduce/tests/test_linelists.py | 9 +++++++++ specreduce/tests/test_specphot_stds.py | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index 2a7fd57..dbf8800 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -353,13 +353,13 @@ def __init__( extinction, u.MagUnit(u.dimensionless_unscaled) ).to(u.dimensionless_unscaled) # Spectrum1D wants this to be linear - if isinstance(extinction, (u.LogUnit, u.Magnitude)) or extinction.unit == u.mag: + elif isinstance(extinction, (u.LogUnit, u.Magnitude)) or extinction.unit == u.mag: # if in log or magnitudes, recast into Magnitude with dimensionless physical units extinction = u.Magnitude( extinction.value, u.MagUnit(u.dimensionless_unscaled) ).to(u.dimensionless_unscaled) - if extinction.unit != u.dimensionless_unscaled: + elif extinction.unit != u.dimensionless_unscaled: # if we're given something linear that's not dimensionless_unscaled, # it's an error msg = "Input extinction must have unscaled dimensionless units." diff --git a/specreduce/tests/test_linelists.py b/specreduce/tests/test_linelists.py index 81af54d..2bfe6ef 100644 --- a/specreduce/tests/test_linelists.py +++ b/specreduce/tests/test_linelists.py @@ -43,6 +43,15 @@ def test_pypeit_comma_list(): assert "NeI" in line_tab['ion'] +@pytest.mark.remote_data +def test_pypeit_nonexisting_lamp(): + """ + Test to make sure a warning is raised if the lamp list includes a bad lamp name. + """ + with pytest.warns(UserWarning, match='NeJ not in the list'): + line_tab = load_pypeit_calibration_lines(["HeI", "NeJ"], cache=True, show_progress=False) + + @pytest.mark.remote_data def test_pypeit_empty(): """ diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index 485326e..58381c6 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -1,3 +1,5 @@ +from urllib.error import URLError + import pytest from astropy.utils.exceptions import AstropyUserWarning @@ -11,6 +13,12 @@ def test_load_MAST(): assert len(sp.spectral_axis) > 0 +@pytest.mark.remote_data +def test_load_MAST_bad_filename(): + with pytest.warns(AstropyUserWarning, match="Downloading of"): + sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) + + @pytest.mark.remote_data def test_load_onedstds(): sp = load_onedstds() @@ -23,3 +31,10 @@ def test_load_onedstds_bad_dataset(): with pytest.warns(AstropyUserWarning, match="Specfied dataset,"): sp = load_onedstds("snffactory") assert sp is None + + +@pytest.mark.remote_data +def test_load_onedstds_bad_specfile(): + with pytest.warns(AstropyUserWarning, match="Can't load"): + sp = load_onedstds(specfile="FG131.dat") + assert sp is None From ffde805bb324698c7692a5568b48a8db014cc8b7 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Fri, 29 Nov 2024 16:25:37 +0000 Subject: [PATCH 07/16] Fixed codestyle errors in calibration_data and its tests. --- specreduce/calibration_data.py | 18 +++++++++++++----- specreduce/tests/test_extinction.py | 3 +-- specreduce/tests/test_linelists.py | 2 +- specreduce/tests/test_specphot_stds.py | 4 +--- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index dbf8800..c534fe4 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -163,9 +163,12 @@ def load_pypeit_calibration_lines( data_path = f"arc_lines/lists/{lamp}_lines.dat" try: with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - linelists.append(Table.read(data_file.read(), format='ascii.fixed_width', comment='#')) + linelists.append(Table.read(data_file.read(), + format='ascii.fixed_width', + comment='#')) except URLError as e: - warnings.warn(f"Downloading of {data_path} failed: {e}", AstropyUserWarning) + warnings.warn(f"Downloading of {data_path} failed: {e}", + AstropyUserWarning) else: warnings.warn( f"{lamp} not in the list of supported calibration " @@ -220,7 +223,8 @@ def load_MAST_calspec( if filename.exists() and filename.is_file(): file_path = filename else: - with conf.set_temp("dataurl", "https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/"): + with conf.set_temp("dataurl", + "https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/"): try: file_path = get_pkg_data_filename(str(filename), show_progress=show_progress) except URLError as e: @@ -281,7 +285,9 @@ def load_onedstds( try: data_path = str(Path("onedstds") / Path(dataset) / Path(specfile)) with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - t = Table.read(data_file.read(), format="ascii", names=['wavelength', 'ABmag', 'binsize']) + t = Table.read(data_file.read(), + format="ascii", + names=['wavelength', 'ABmag', 'binsize']) except URLError as e: msg = f"Can't load {specfile} from {dataset}: {e}." warnings.warn(msg, AstropyUserWarning) @@ -375,7 +381,9 @@ def __init__( with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): data_path = str(Path("extinction") / Path(f"{model}extinct.dat")) with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - t = Table.read(data_file.read(), format="ascii", names=['wavelength', 'extinction']) + t = Table.read(data_file.read(), + format="ascii", + names=['wavelength', 'extinction']) # the specreduce_data models all provide wavelengths in angstroms spectral_axis = t['wavelength'].data * u.angstrom diff --git a/specreduce/tests/test_extinction.py b/specreduce/tests/test_extinction.py index 1e91582..bbe8f88 100644 --- a/specreduce/tests/test_extinction.py +++ b/specreduce/tests/test_extinction.py @@ -63,7 +63,7 @@ def test_unsupported_model(): Test loading of a nonexistent model """ with pytest.raises(ValueError, match='Requested extinction model,'): - ext = AtmosphericExtinction(model='bad_model') + AtmosphericExtinction(model='bad_model') @pytest.mark.remote_data @@ -75,7 +75,6 @@ def test_missing_extinction_unit(): extinction = 1. / wave with pytest.warns(AstropyUserWarning): ext = AtmosphericExtinction(extinction=extinction, spectral_axis=wave * u.um) - assert len(ext.extinction_mag) > 0 assert len(ext.transmission) > 0 diff --git a/specreduce/tests/test_linelists.py b/specreduce/tests/test_linelists.py index 2bfe6ef..247868c 100644 --- a/specreduce/tests/test_linelists.py +++ b/specreduce/tests/test_linelists.py @@ -49,7 +49,7 @@ def test_pypeit_nonexisting_lamp(): Test to make sure a warning is raised if the lamp list includes a bad lamp name. """ with pytest.warns(UserWarning, match='NeJ not in the list'): - line_tab = load_pypeit_calibration_lines(["HeI", "NeJ"], cache=True, show_progress=False) + load_pypeit_calibration_lines(["HeI", "NeJ"], cache=True, show_progress=False) @pytest.mark.remote_data diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index 58381c6..b347684 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -1,5 +1,3 @@ -from urllib.error import URLError - import pytest from astropy.utils.exceptions import AstropyUserWarning @@ -16,7 +14,7 @@ def test_load_MAST(): @pytest.mark.remote_data def test_load_MAST_bad_filename(): with pytest.warns(AstropyUserWarning, match="Downloading of"): - sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) + load_MAST_calspec("j191b2b_005.fits", show_progress=False) @pytest.mark.remote_data From cedb9afeb911397d2bc2c652303985a9241e6076 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Fri, 29 Nov 2024 16:34:27 +0000 Subject: [PATCH 08/16] Trying to figure out why some tests suddenly fail in Python 3.11. --- specreduce/tests/test_linelists.py | 2 +- specreduce/tests/test_specphot_stds.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specreduce/tests/test_linelists.py b/specreduce/tests/test_linelists.py index 247868c..d3e5784 100644 --- a/specreduce/tests/test_linelists.py +++ b/specreduce/tests/test_linelists.py @@ -49,7 +49,7 @@ def test_pypeit_nonexisting_lamp(): Test to make sure a warning is raised if the lamp list includes a bad lamp name. """ with pytest.warns(UserWarning, match='NeJ not in the list'): - load_pypeit_calibration_lines(["HeI", "NeJ"], cache=True, show_progress=False) + line_tab = load_pypeit_calibration_lines(["HeI", "NeJ"], cache=True, show_progress=False) # noqa @pytest.mark.remote_data diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index b347684..b8e34fb 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -14,7 +14,7 @@ def test_load_MAST(): @pytest.mark.remote_data def test_load_MAST_bad_filename(): with pytest.warns(AstropyUserWarning, match="Downloading of"): - load_MAST_calspec("j191b2b_005.fits", show_progress=False) + sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) # noqa @pytest.mark.remote_data From b80551bd6e96421b17dcae63b50a83d55c573a33 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Fri, 29 Nov 2024 16:43:26 +0000 Subject: [PATCH 09/16] Still trying to figure out why some tests suddenly fail in Python 3.11. --- specreduce/tests/test_linelists.py | 2 +- specreduce/tests/test_specphot_stds.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/specreduce/tests/test_linelists.py b/specreduce/tests/test_linelists.py index d3e5784..247868c 100644 --- a/specreduce/tests/test_linelists.py +++ b/specreduce/tests/test_linelists.py @@ -49,7 +49,7 @@ def test_pypeit_nonexisting_lamp(): Test to make sure a warning is raised if the lamp list includes a bad lamp name. """ with pytest.warns(UserWarning, match='NeJ not in the list'): - line_tab = load_pypeit_calibration_lines(["HeI", "NeJ"], cache=True, show_progress=False) # noqa + load_pypeit_calibration_lines(["HeI", "NeJ"], cache=True, show_progress=False) @pytest.mark.remote_data diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index b8e34fb..e67f1c8 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -14,7 +14,8 @@ def test_load_MAST(): @pytest.mark.remote_data def test_load_MAST_bad_filename(): with pytest.warns(AstropyUserWarning, match="Downloading of"): - sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) # noqa + sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) + assert sp is None @pytest.mark.remote_data From 3b87c60fd4dc6b7c8756d41277422bcc6c45b67e Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Fri, 29 Nov 2024 16:48:44 +0000 Subject: [PATCH 10/16] Ignoring 'pytest.PytestUnraisableExceptionWarning' warnings that are raised for "test_make_2d_arc_pass_wcs" and "test_load_onedstds" on Python 3.11. --- specreduce/tests/test_specphot_stds.py | 2 +- specreduce/tests/test_synth_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index e67f1c8..5d66e5a 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -17,7 +17,7 @@ def test_load_MAST_bad_filename(): sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) assert sp is None - +@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") @pytest.mark.remote_data def test_load_onedstds(): sp = load_onedstds() diff --git a/specreduce/tests/test_synth_data.py b/specreduce/tests/test_synth_data.py index b8bcb87..ae06b89 100644 --- a/specreduce/tests/test_synth_data.py +++ b/specreduce/tests/test_synth_data.py @@ -27,9 +27,9 @@ def test_make_2d_arc_image_defaults(): ccdim = make_2d_arc_image() assert isinstance(ccdim, CCDData) - @pytest.mark.remote_data @pytest.mark.filterwarnings("ignore:No observer defined on WCS") +@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") def test_make_2d_arc_pass_wcs(): nx = 3000 ny = 1000 From 1c4ae4aad830c3a5bc76b475b5d230cf633921ed Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Fri, 29 Nov 2024 16:50:14 +0000 Subject: [PATCH 11/16] Fixed code style for the changed tests. --- specreduce/tests/test_specphot_stds.py | 1 + specreduce/tests/test_synth_data.py | 1 + 2 files changed, 2 insertions(+) diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index 5d66e5a..756861a 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -17,6 +17,7 @@ def test_load_MAST_bad_filename(): sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) assert sp is None + @pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") @pytest.mark.remote_data def test_load_onedstds(): diff --git a/specreduce/tests/test_synth_data.py b/specreduce/tests/test_synth_data.py index ae06b89..0e1d910 100644 --- a/specreduce/tests/test_synth_data.py +++ b/specreduce/tests/test_synth_data.py @@ -27,6 +27,7 @@ def test_make_2d_arc_image_defaults(): ccdim = make_2d_arc_image() assert isinstance(ccdim, CCDData) + @pytest.mark.remote_data @pytest.mark.filterwarnings("ignore:No observer defined on WCS") @pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") From 86f3d95e1daa630239dbf8f91ada1c4b237b3e47 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Sun, 1 Dec 2024 12:14:11 +0000 Subject: [PATCH 12/16] Commented out the PytestUnraisableExceptionWarning filters. Better to find the reason why they happen in Python 3.11 rather than just to ignore them. --- specreduce/tests/test_specphot_stds.py | 2 +- specreduce/tests/test_synth_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index 756861a..1c231a3 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -18,7 +18,7 @@ def test_load_MAST_bad_filename(): assert sp is None -@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") +#@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") @pytest.mark.remote_data def test_load_onedstds(): sp = load_onedstds() diff --git a/specreduce/tests/test_synth_data.py b/specreduce/tests/test_synth_data.py index 0e1d910..00ff2f3 100644 --- a/specreduce/tests/test_synth_data.py +++ b/specreduce/tests/test_synth_data.py @@ -30,7 +30,7 @@ def test_make_2d_arc_image_defaults(): @pytest.mark.remote_data @pytest.mark.filterwarnings("ignore:No observer defined on WCS") -@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") +#@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") def test_make_2d_arc_pass_wcs(): nx = 3000 ny = 1000 From 07333192baef30f946716dbed55cd336e03af0dd Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Sun, 1 Dec 2024 15:39:00 +0000 Subject: [PATCH 13/16] - Changed back to use download_file instead of the get_pkg_* functions. This leads to more readable and cleaner code in the end. - Changed the cache type hints to "bool | Literal['update']" --- specreduce/calibration_data.py | 122 +++++++++++-------------- specreduce/tests/test_specphot_stds.py | 3 +- 2 files changed, 55 insertions(+), 70 deletions(-) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index c534fe4..ce02cc2 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -4,13 +4,12 @@ import warnings from pathlib import Path -from typing import Sequence +from typing import Sequence, Literal from urllib.error import URLError from astropy import units as u from astropy.table import Table, vstack, QTable -from astropy.utils import deprecated_renamed_argument -from astropy.utils.data import get_pkg_data_filename, conf, get_pkg_data_fileobj +from astropy.utils.data import download_file from astropy.utils.exceptions import AstropyUserWarning from astropy.coordinates import SpectralCoord @@ -106,11 +105,10 @@ def get_available_line_catalogs() -> dict: } -@deprecated_renamed_argument('show_progress', None, '1.4.2', pending=True) def load_pypeit_calibration_lines( lamps: Sequence | None = None, wave_air: bool = False, - cache: bool = True, + cache: bool | Literal['update'] = True, show_progress: bool = False ) -> QTable | None: """ @@ -156,41 +154,38 @@ def load_pypeit_calibration_lines( else: lamps = [lamps] - with conf.set_temp("dataurl", PYPEIT_DATA_URL): - linelists = [] - for lamp in lamps: - if lamp in PYPEIT_CALIBRATION_LINELISTS: - data_path = f"arc_lines/lists/{lamp}_lines.dat" - try: - with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - linelists.append(Table.read(data_file.read(), - format='ascii.fixed_width', - comment='#')) - except URLError as e: - warnings.warn(f"Downloading of {data_path} failed: {e}", - AstropyUserWarning) - else: - warnings.warn( - f"{lamp} not in the list of supported calibration " - f"line lists: {PYPEIT_CALIBRATION_LINELISTS}." - ) - if len(linelists) == 0: - warnings.warn(f"No calibration lines loaded from {lamps}.") - linelist = None + linelists = [] + for lamp in lamps: + if lamp in PYPEIT_CALIBRATION_LINELISTS: + data_url = f"{PYPEIT_DATA_URL}/arc_lines/lists/{lamp}_lines.dat" + try: + data_path = download_file(data_url, cache=cache, + show_progress=show_progress, + pkgname='specreduce') + linelists.append(Table.read(data_path, format='ascii.fixed_width', comment='#')) + except URLError as e: + warnings.warn(f"Downloading of {data_url} failed: {e}", AstropyUserWarning) else: - linelist = QTable(vstack(linelists)) - linelist.rename_column('wave', 'wavelength') - # pypeit linelists use vacuum wavelengths in angstroms - linelist['wavelength'] *= u.Angstrom - if wave_air: - linelist['wavelength'] = vac_to_air(linelist['wavelength']) - return linelist + warnings.warn( + f"{lamp} not in the list of supported calibration " + f"line lists: {PYPEIT_CALIBRATION_LINELISTS}." + ) + if len(linelists) == 0: + warnings.warn(f"No calibration lines loaded from {lamps}.") + linelist = None + else: + linelist = QTable(vstack(linelists)) + linelist.rename_column('wave', 'wavelength') + # pypeit linelists use vacuum wavelengths in angstroms + linelist['wavelength'] *= u.Angstrom + if wave_air: + linelist['wavelength'] = vac_to_air(linelist['wavelength']) + return linelist -@deprecated_renamed_argument('cache', None, '1.4.2', pending=True) def load_MAST_calspec( filename: str | Path, - cache: bool = True, + cache: bool | Literal['update'] = True, show_progress: bool = False ) -> Spectrum1D | None: """ @@ -223,14 +218,14 @@ def load_MAST_calspec( if filename.exists() and filename.is_file(): file_path = filename else: - with conf.set_temp("dataurl", - "https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/"): - try: - file_path = get_pkg_data_filename(str(filename), show_progress=show_progress) - except URLError as e: - msg = f"Downloading of {filename} failed: {e}" - warnings.warn(msg, AstropyUserWarning) - file_path = None + try: + data_url = f"https://archive.stsci.edu/hlsps/reference-atlases/cdbs/calspec/{filename}" + file_path = download_file(data_url, cache=cache, + show_progress=show_progress, + pkgname='specreduce') + except URLError as e: + warnings.warn(f"Downloading of {filename} failed: {e}", AstropyUserWarning) + file_path = None if file_path is None: return None @@ -247,11 +242,10 @@ def load_MAST_calspec( return spectrum -@deprecated_renamed_argument('show_progress', None, '1.4.2', pending=True) def load_onedstds( dataset: str = "snfactory", specfile: str = "EG131.dat", - cache: bool = True, + cache : bool | Literal['update'] = True, show_progress: bool = False ) -> Spectrum1D | None: """ @@ -281,17 +275,14 @@ def load_onedstds( warnings.warn(msg, AstropyUserWarning) return None - with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): - try: - data_path = str(Path("onedstds") / Path(dataset) / Path(specfile)) - with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - t = Table.read(data_file.read(), - format="ascii", - names=['wavelength', 'ABmag', 'binsize']) - except URLError as e: - msg = f"Can't load {specfile} from {dataset}: {e}." - warnings.warn(msg, AstropyUserWarning) - return None + try: + data_path = download_file(f"{SPECREDUCE_DATA_URL}/onedstds/{dataset}/{specfile}", + cache=cache, show_progress=show_progress, pkgname="specreduce") + t = Table.read(data_path, format="ascii", names=['wavelength', 'ABmag', 'binsize']) + except URLError as e: + msg = f"Can't load {specfile} from {dataset}: {e}." + warnings.warn(msg, AstropyUserWarning) + return None # the specreduce_data standard star spectra all provide wavelengths in angstroms spectral_axis = t['wavelength'].data * u.angstrom @@ -339,13 +330,12 @@ class AtmosphericExtinction(Spectrum1D): transmission : Extinction expressed as fractional transmission """ - @deprecated_renamed_argument('show_progress', None, '1.4.2', pending=True) def __init__( self, model: str = "kpno", extinction: Sequence[float] | u.Quantity | None = None, spectral_axis: SpectralCoord | u.Quantity | None = None, - cache: bool = True, + cache: bool | Literal['update'] = True, show_progress: bool = False, **kwargs: str ) -> None: @@ -378,12 +368,11 @@ def __init__( f"of available models: {SUPPORTED_EXTINCTION_MODELS}" ) raise ValueError(msg) - with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): - data_path = str(Path("extinction") / Path(f"{model}extinct.dat")) - with get_pkg_data_fileobj(data_path, cache=cache) as data_file: - t = Table.read(data_file.read(), - format="ascii", - names=['wavelength', 'extinction']) + + data_file = download_file(f"{SPECREDUCE_DATA_URL}/extinction/{model}extinct.dat", + cache=cache, show_progress=show_progress, + pkgname='specreduce') + t = Table.read(data_file, format="ascii", names=['wavelength', 'extinction']) # the specreduce_data models all provide wavelengths in angstroms spectral_axis = t['wavelength'].data * u.angstrom @@ -443,10 +432,7 @@ def __init__( **kwargs: str ) -> None: if data_file is None: - with conf.set_temp("dataurl", SPECREDUCE_DATA_URL): - data_path = str(Path("extinction") / Path("atm_trans_am1.0.dat")) - data_file = get_pkg_data_filename(data_path) - + data_file = download_file(f"{SPECREDUCE_DATA_URL}/extinction/atm_trans_am1.0.dat") t = Table.read(data_file, format="ascii", names=['wavelength', 'extinction']) # spectral axis is given in microns diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index 1c231a3..9437d8d 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -14,11 +14,10 @@ def test_load_MAST(): @pytest.mark.remote_data def test_load_MAST_bad_filename(): with pytest.warns(AstropyUserWarning, match="Downloading of"): - sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) + sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) assert sp is None -#@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") @pytest.mark.remote_data def test_load_onedstds(): sp = load_onedstds() From 4a5a494c490f1d8f64ab084a1c8f74f620c7e0d3 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Sun, 1 Dec 2024 15:41:01 +0000 Subject: [PATCH 14/16] - Ignoring "ResourceWarning"s in pytest config. These were raised in Python 3.11 for tests that tried to download a nonexistent file even when the URLError was caught correctly. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 41bd7cd..a4c92b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ filterwarnings = [ "ignore:Can\\'t import specreduce_data package", # Python 3.12 warning from dateutil imported by matplotlib "ignore:.*utcfromtimestamp:DeprecationWarning", + "ignore:.*unclosed Date: Sun, 1 Dec 2024 15:46:45 +0000 Subject: [PATCH 15/16] Removed a forgotten warning filter. --- specreduce/tests/test_synth_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/specreduce/tests/test_synth_data.py b/specreduce/tests/test_synth_data.py index 00ff2f3..b8bcb87 100644 --- a/specreduce/tests/test_synth_data.py +++ b/specreduce/tests/test_synth_data.py @@ -30,7 +30,6 @@ def test_make_2d_arc_image_defaults(): @pytest.mark.remote_data @pytest.mark.filterwarnings("ignore:No observer defined on WCS") -#@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") def test_make_2d_arc_pass_wcs(): nx = 3000 ny = 1000 From 9aecda5069ef1b8845e80b59c6ff0385305ffaa4 Mon Sep 17 00:00:00 2001 From: Hannu Parviainen Date: Sun, 1 Dec 2024 15:49:39 +0000 Subject: [PATCH 16/16] Fixed code style errors. --- specreduce/calibration_data.py | 2 +- specreduce/tests/test_specphot_stds.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specreduce/calibration_data.py b/specreduce/calibration_data.py index ce02cc2..84e478f 100644 --- a/specreduce/calibration_data.py +++ b/specreduce/calibration_data.py @@ -245,7 +245,7 @@ def load_MAST_calspec( def load_onedstds( dataset: str = "snfactory", specfile: str = "EG131.dat", - cache : bool | Literal['update'] = True, + cache: bool | Literal['update'] = True, show_progress: bool = False ) -> Spectrum1D | None: """ diff --git a/specreduce/tests/test_specphot_stds.py b/specreduce/tests/test_specphot_stds.py index 9437d8d..e67f1c8 100644 --- a/specreduce/tests/test_specphot_stds.py +++ b/specreduce/tests/test_specphot_stds.py @@ -14,7 +14,7 @@ def test_load_MAST(): @pytest.mark.remote_data def test_load_MAST_bad_filename(): with pytest.warns(AstropyUserWarning, match="Downloading of"): - sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) + sp = load_MAST_calspec("j191b2b_005.fits", show_progress=False) assert sp is None