Skip to content

Commit

Permalink
Merge pull request #652 from garlic-os/issue-638-do-not-overwrite-netrc
Browse files Browse the repository at this point in the history
Update RC files instead of overwriting them
  • Loading branch information
jlmaurer authored Jul 12, 2024
2 parents e4131dc + f72abcc commit 701d788
Show file tree
Hide file tree
Showing 8 changed files with 451 additions and 206 deletions.
9 changes: 4 additions & 5 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,15 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
### Added
* [656](https://github.com/dbekaert/RAiDER/pull/656) - Example run configuration files available through `raider.py --generate_config <example_name>`.
### Changed
* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
* [662](https://github.com/dbekaert/RAiDER/pull/662) - Bumped dem-stitcher to >= v2.5.6, which updates the URL for reading the Geoid EGM 2008.
### Fixed
* [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and added unit tests and bug fixes.
* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
* [652](https://github.com/dbekaert/RAiDER/pull/652) - Changed the behavior of `RAiDER.models.credentials.check_api` to not overwrite the user's API credential files.
* [656](https://github.com/dbekaert/RAiDER/pull/656) - Example run configuration files available through `raider.py --generate_config <example_name>`.
* [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`.
* [658](https://github.com/dbekaert/RAiDER/pull/658) - Fixed opaque error message if a GUNW file is not produced while HyP3 independently against a previous INSAR_ISCE.
* [661](https://github.com/dbekaert/RAiDER/pull/661) - Fixed bug in raiderDownloadGNSS, removed call to scipy.sum, and added unit tests.
* [662](https://github.com/dbekaert/RAiDER/pull/662) - Bumped dem-stitcher to >= v2.5.6, which updates the URL for reading the Geoid EGM 2008.

## [0.5.1]
### Changed
Expand Down
11 changes: 10 additions & 1 deletion test/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import pytest
import subprocess
import shutil
import string
import random
from contextlib import contextmanager
from pathlib import Path

Expand Down Expand Up @@ -82,4 +84,11 @@ def makeLatLonGrid(bbox, reg, out_dir, spacing=0.1):

def make_delay_name(weather_model_name, date, time, kind='ztd'):
assert kind in 'ztd std ray'.split(), 'Incorrect type of delays.'
return f'{weather_model_name}_tropo_{date}T{time.replace(":", "")}_{kind}.nc'
return f'{weather_model_name}_tropo_{date}T{time.replace(":", "")}_{kind}.nc'


def random_string(
length: int = 32,
alphabet: str = string.ascii_letters + string.digits
) -> str:
return ''.join(random.choices(alphabet, k=length))
75 changes: 75 additions & 0 deletions test/credentials/test_createFile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
'''
When update_rc_file is either True or False, the relevant API RC file should be
created if it doesn't exist.
'''
from typing import Tuple

import pytest
import os
from pathlib import Path
from platform import system
from RAiDER.models import credentials
from test import random_string


def get_creds_cds(rc_path: Path) -> Tuple[str, str]:
import cdsapi
cds_credentials = cdsapi.api.read_config(rc_path)
uid, key = cds_credentials['key'].split(':')
return uid, key


def get_creds_ecmwf(rc_path: Path) -> Tuple[str, str]:
import ecmwfapi
# Get current ECMWF API RC file path
old_rc_path = os.getenv("ECMWF_API_RC_FILE", ecmwfapi.api.DEFAULT_RCFILE_PATH)

# Point ecmwfapi to current dir to avoid overwriting ~/.ecmwfapirc
os.environ["ECMWF_API_RC_FILE"] = str(rc_path)
key, _, uid = ecmwfapi.api.get_apikey_values()

# Point ecmwfapi back to previous value and remove local API file
os.environ["ECMWF_API_RC_FILE"] = old_rc_path
return uid, key


def get_creds_netrc(rc_path: Path) -> Tuple[str, str]:
import netrc
host = 'urs.earthdata.nasa.gov'
netrc_credentials = netrc.netrc(rc_path)
uid, _, key = netrc_credentials.authenticators(host)
return uid, key


@pytest.mark.parametrize(
'model_name,get_creds',
(
('ERA5', get_creds_cds),
('ERA5T', get_creds_cds),
('HRES', get_creds_ecmwf),
('GMAO', get_creds_netrc),
('MERRA2', get_creds_netrc)
)
)
def test_createFile(model_name, get_creds):
# Get the rc file's path
hidden_ext = '_' if system() == "Windows" else '.'
rc_filename = credentials.RC_FILENAMES[model_name]
if rc_filename is None:
return
rc_path = Path('./') / (hidden_ext + rc_filename)
rc_path = rc_path.expanduser()
rc_path.unlink(missing_ok=True)

test_uid = random_string()
test_key = random_string()

# Test creation of the rc file
credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False)
assert rc_path.exists(), f'{rc_path} does not exist'

# Check if API is written correctly
uid, key = get_creds(rc_path)
rc_path.unlink()
assert uid == test_uid, f'{rc_path}: UID was not written correctly'
assert key == test_key, f'{rc_path}: KEY was not written correctly'
108 changes: 108 additions & 0 deletions test/credentials/test_envVars.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
'''
Environment variables specific to each model are accepted iff uid and key
arguments are None.
'''

import pytest
from pathlib import Path
from platform import system
from RAiDER.models import credentials
from test import random_string

@pytest.mark.parametrize(
'model_name,template,env_var_name_uid,env_var_name_key',
[
(
'ERA5', (
'url: https://cds.climate.copernicus.eu/api/v2\n'
'key: {uid}:{key}\n'
),
'RAIDER_ECMWF_ERA5_UID',
'RAIDER_ECMWF_ERA5_API_KEY'
),
(
'ERA5T', (
'url: https://cds.climate.copernicus.eu/api/v2\n'
'key: {uid}:{key}\n'
),
'RAIDER_ECMWF_ERA5_UID',
'RAIDER_ECMWF_ERA5_API_KEY'
),
(
'HRES', (
'{{\n'
' "url" : "https://api.ecmwf.int/v1",\n'
' "key" : "{key}",\n'
' "email" : "{uid}"\n'
'}}\n'
),
'RAIDER_HRES_EMAIL',
'RAIDER_HRES_API_KEY'
),
(
# Simulate a .netrc file with multiple sets of credentials.
# The ones not for urs.earthdata.nasa.gov should NOT be touched.
# Indentation is done with TABS, as that is what the netrc package
# generates.
'GMAO', (
'machine example.com\n'
' login johndoe\n'
' password hunter2\n'
'machine urs.earthdata.nasa.gov\n'
' login {uid}\n'
' password {key}\n'
'machine 127.0.0.1\n'
' login bobsmith\n'
' password dolphins\n'
),
'EARTHDATA_USERNAME',
'EARTHDATA_PASSWORD'
),
(
'MERRA2', (
'machine example.com\n'
' login johndoe\n'
' password hunter2\n'
'machine urs.earthdata.nasa.gov\n'
' login {uid}\n'
' password {key}\n'
'machine 127.0.0.1\n'
' login bobsmith\n'
' password dolphins\n'
),
'EARTHDATA_USERNAME',
'EARTHDATA_PASSWORD'
),
]
)
def test_envVars(
monkeypatch,
model_name,
template,
env_var_name_uid,
env_var_name_key
):
hidden_ext = '_' if system() == "Windows" else '.'
rc_filename = credentials.RC_FILENAMES[model_name]
if rc_filename is None:
return
rc_path = Path('./') / (hidden_ext + rc_filename)
rc_path = rc_path.expanduser()
rc_path.unlink(missing_ok=True)

test_uid = random_string()
test_key = random_string()

with monkeypatch.context() as mp:
mp.setenv(env_var_name_uid, test_uid)
mp.setenv(env_var_name_key, test_key)
credentials.check_api(model_name, None, None, './', update_rc_file=False)

expected_content = template.format(uid=test_uid, key=test_key)
actual_content = rc_path.read_text()
rc_path.unlink()

assert (
expected_content == actual_content,
f'{rc_path} was not updated correctly'
)
31 changes: 31 additions & 0 deletions test/credentials/test_updateFalse.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
'''
When update_rc_file is False, the RC file should NOT be modified if it already
exists.
'''
import pytest

from pathlib import Path
from platform import system
from RAiDER.models import credentials


@pytest.mark.parametrize('model_name', 'ERA5 ERA5T HRES GMAO MERRA2'.split())
def test_updateFalse(model_name):
# Get the rc file's path
hidden_ext = '_' if system() == "Windows" else '.'
rc_filename = credentials.RC_FILENAMES[model_name]
if rc_filename is None:
return
rc_path = Path('./') / (hidden_ext + rc_filename)
rc_path = rc_path.expanduser()

# Write some example text to test for
rc_path.write_text('dummy')

# Test creation of this model's RC file in current dir
credentials.check_api(model_name, None, None, './', update_rc_file=False)

# Assert the content was unchanged
content = rc_path.read_text()
rc_path.unlink()
assert content == 'dummy', f'{rc_path} was modified'
91 changes: 91 additions & 0 deletions test/credentials/test_updateTrue.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
'''
When update_rc_file is True, the RC file should be:
- updated if it already exists,
- created if it doesn't,
- and for .netrc files, it should ONLY update the set of credentials related to
the given weather model's API URL.
'''
import pytest

from pathlib import Path
from platform import system
from RAiDER.models import credentials
from test import random_string


@pytest.mark.parametrize(
'model_name,template',
[
(
'ERA5', (
'url: https://cds.climate.copernicus.eu/api/v2\n'
'key: {uid}:{key}\n'
)
),
(
'ERA5T', (
'url: https://cds.climate.copernicus.eu/api/v2\n'
'key: {uid}:{key}\n'
)
),
(
'HRES', (
'{{\n'
' "url" : "https://api.ecmwf.int/v1",\n'
' "key" : "{key}",\n'
' "email" : "{uid}"\n'
'}}\n'
)
),
(
# Simulate a .netrc file with multiple sets of credentials.
# The ones not for urs.earthdata.nasa.gov should NOT be touched.
# Indentation is done with TABS, as that is what the netrc package
# generates.
'GMAO', (
'machine example.com\n'
' login johndoe\n'
' password hunter2\n'
'machine urs.earthdata.nasa.gov\n'
' login {uid}\n'
' password {key}\n'
'machine 127.0.0.1\n'
' login bobsmith\n'
' password dolphins\n'
)
),
(
'MERRA2', (
'machine example.com\n'
' login johndoe\n'
' password hunter2\n'
'machine urs.earthdata.nasa.gov\n'
' login {uid}\n'
' password {key}\n'
'machine 127.0.0.1\n'
' login bobsmith\n'
' password dolphins\n'
)
),
]
)
def test_updateTrue(model_name, template):
# Get the rc file's path
hidden_ext = '_' if system() == "Windows" else '.'
rc_filename = credentials.RC_FILENAMES[model_name]
if rc_filename is None:
return
rc_path = Path('./') / (hidden_ext + rc_filename)

test_uid = random_string()
test_key = random_string()
credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=True)

expected_content = template.format(uid=test_uid, key=test_key)
actual_content = rc_path.read_text()
rc_path.unlink()

assert (
expected_content == actual_content,
f'{rc_path} was not updated correctly'
)
Loading

0 comments on commit 701d788

Please sign in to comment.