From 805fc51ff41e32b5f4692d2a4b441e2a6726ef54 Mon Sep 17 00:00:00 2001 From: micah johnson Date: Thu, 15 Aug 2024 11:45:30 -0600 Subject: [PATCH 1/4] Added more tests to ensure parse none is capturing -9999 --- tests/test_string_management.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_string_management.py b/tests/test_string_management.py index 6e030a6..be1e83f 100644 --- a/tests/test_string_management.py +++ b/tests/test_string_management.py @@ -1,3 +1,4 @@ +import numpy as np import pytest from snowex_db.string_management import * @@ -62,18 +63,24 @@ def test_strip_encapsulated(s, encaps, expected): assert r == expected -def test_parse_none(): +@pytest.mark.parametrize('str_value, expected', [ + # Expected nones + ('NaN', None), + ('NONE', None), + ('-9999', None), + ('-9999.0', None), + (-9999.0, None), + (np.nan, None), + # Shouldn't modify anything + (10.5, 10.5), + ("Comment", "Comment"), +]) +def test_parse_none(str_value, expected): """ Test we can convert nones and nans to None and still pass through everything else. """ - # Assert these are converted to None - for v in ['NaN', '', 'NONE', np.nan]: - assert parse_none(v) is None - - # Assert these are unaffected by function - for v in [10.5, 'Comment']: - assert parse_none(v) == v + assert parse_none(str_value) == expected @pytest.mark.parametrize('args, kwargs, expected', [ From fec931f3f1e3e9ee36411a2bccd3f0f1717c2a42 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 15 Aug 2024 13:02:49 -0600 Subject: [PATCH 2/4] see that -9999 int causes issue --- tests/data/density.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/data/density.csv b/tests/data/density.csv index 72d1adc..09cd7f6 100644 --- a/tests/data/density.csv +++ b/tests/data/density.csv @@ -6,7 +6,7 @@ # Easting,743281 # Northing,4324005 # top (cm),bottom (cm),density A (kg/m3),density B (kg/m3),density C (kg/m3) -35.0,25.0,190.0,245.0,NaN -25.0,15.0,228.0,241.0,NaN -15.0,5.0,217.0,253.0,NaN -12.0,2.0,236.0,NaN,NaN +35.0,25.0,190.0,245.0,-9999 +25.0,15.0,228.0,241.0,-9999 +15.0,5.0,217.0,253.0,-9999 +12.0,2.0,236.0,NaN,-9999 From f1dd9ae2ab15767f56ddc382f6e08909dfbbeb0e Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 15 Aug 2024 13:03:35 -0600 Subject: [PATCH 3/4] Fix -9999 issue with ints --- snowex_db/string_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snowex_db/string_management.py b/snowex_db/string_management.py index bdefa16..2b92cee 100644 --- a/snowex_db/string_management.py +++ b/snowex_db/string_management.py @@ -209,7 +209,7 @@ def parse_none(value): if isinstance(value, str): if value.lower() in ['nan', 'none', '-9999', '-9999.0'] or not value: result = None - elif isinstance(value, float): + elif isinstance(value, float) or isinstance(value, int): if np.isnan(value) or value == -9999: result = None From 477a0eacc4526611a50647f97d7d65c99b516260 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 15 Aug 2024 13:04:29 -0600 Subject: [PATCH 4/4] integer test case --- tests/test_string_management.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_string_management.py b/tests/test_string_management.py index be1e83f..aa0a4fc 100644 --- a/tests/test_string_management.py +++ b/tests/test_string_management.py @@ -67,6 +67,7 @@ def test_strip_encapsulated(s, encaps, expected): # Expected nones ('NaN', None), ('NONE', None), + (-9999, None), # integer case ('-9999', None), ('-9999.0', None), (-9999.0, None),