diff --git a/Makefile b/Makefile index 9746e09..e624f6a 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ clean: # HARMONY-1188 - revert this command to: # pip install -e .[dev] install: + pip install pip --upgrade pip install -r dev-requirements.txt pip install -r requirements.txt diff --git a/harmony/message_utility.py b/harmony/message_utility.py new file mode 100644 index 0000000..655b229 --- /dev/null +++ b/harmony/message_utility.py @@ -0,0 +1,163 @@ +"""Utilities for acting on Harmony Messages. + +These are a collection of useful routines for validation and interrogation of +harmony Messages. +""" + +from typing import Any, List + +from harmony.message import Message + + +def has_self_consistent_grid(message: Message) -> bool: + """ Check the input Harmony message provides enough information to fully + define the target grid. At minimum the message should contain the scale + extents (minimum and maximum values) in the horizontal spatial + dimensions and one of the following two pieces of information: + + * Message.format.scaleSize - defining the x and y pixel size. + * Message.format.height and Message.format.width - the number of pixels + in the x and y dimension. + + If all three pieces of information are supplied, they will be checked + to ensure they are consistent with one another. + + If scaleExtent and scaleSize are defined, along with only one of + height or width, the grid will be considered consistent if the three + values for scaleExtent, scaleSize and specified dimension length, + height or width, are consistent. + + """ + if ( + has_scale_extents(message) and has_scale_sizes(message) + and has_dimensions(message) + ): + consistent_grid = (_has_consistent_dimension(message, 'x') + and _has_consistent_dimension(message, 'y')) + elif ( + has_scale_extents(message) and has_scale_sizes(message) + and rgetattr(message, 'format.height') is not None + ): + consistent_grid = _has_consistent_dimension(message, 'y') + elif ( + has_scale_extents(message) and has_scale_sizes(message) + and rgetattr(message, 'format.width') is not None + ): + consistent_grid = _has_consistent_dimension(message, 'x') + elif ( + has_scale_extents(message) + and (has_scale_sizes(message) or has_dimensions(message)) + ): + consistent_grid = True + else: + consistent_grid = False + + return consistent_grid + + +def has_dimensions(message: Message) -> bool: + """ Ensure the supplied Harmony message contains values for height and + width of the target grid, which define the sizes of the x and y + horizontal spatial dimensions. + + """ + return _has_all_attributes(message, ['format.height', 'format.width']) + + +def has_crs(message: Message) -> bool: + """Returns true if Harmony message contains a crs.""" + target_crs = rgetattr(message, 'format.crs') + return target_crs is not None + + +def has_scale_extents(message: Message) -> bool: + """ Ensure the supplied Harmony message contains values for the minimum and + maximum extents of the target grid in both the x and y dimensions. + + """ + scale_extent_attributes = ['format.scaleExtent.x.min', + 'format.scaleExtent.x.max', + 'format.scaleExtent.y.min', + 'format.scaleExtent.y.max'] + + return _has_all_attributes(message, scale_extent_attributes) + + +def has_scale_sizes(message: Message) -> bool: + """ Ensure the supplied Harmony message contains values for the x and y + horizontal scale sizes for the target grid. + + """ + scale_size_attributes = ['format.scaleSize.x', 'format.scaleSize.y'] + return _has_all_attributes(message, scale_size_attributes) + + +def has_valid_scale_extents(message: Message) -> bool: + """Ensure any input scale_extents are valid.""" + if has_scale_extents(message): + return ( + float(rgetattr(message, 'format.scaleExtent.x.min')) + < float(rgetattr(message, 'format.scaleExtent.x.max')) + ) and ( + float(rgetattr(message, 'format.scaleExtent.y.min')) + < float(rgetattr(message, 'format.scaleExtent.y.max')) + ) + return True + + +def _has_all_attributes(message: Message, attributes: List[str]) -> bool: + """ Ensure that the supplied Harmony message has non-None attribute values + for all the listed attributes. + + """ + return all(rgetattr(message, attribute_name) is not None + for attribute_name in attributes) + + +def _has_consistent_dimension(message: Message, dimension_name: str) -> bool: + """ Ensure a grid dimension has consistent values for the scale extent + (e.g., minimum and maximum values), scale size (resolution) and + dimension length (e.g., width or height). For the grid x dimension, the + calculation is as follows: + + scaleSize.x = (scaleExtent.x.max - scaleExtent.x.min) / (width) + + The message scale sizes is compared to that calculated as above, to + ensure it is within a relative tolerance (1 x 10^-3). + + """ + message_scale_size = getattr(message.format.scaleSize, dimension_name) + scale_extent = getattr(message.format.scaleExtent, dimension_name) + + if dimension_name == 'x': + dimension_elements = message.format.width + else: + dimension_elements = message.format.height + + derived_scale_size = (scale_extent.max - scale_extent.min) / dimension_elements + + return abs(message_scale_size - derived_scale_size) <= 1e-3 + + +def rgetattr(input_object: Any, requested_attribute: str, *args) -> Any: + """ This is a recursive version of the inbuilt `getattr` method, such that + it can be called to retrieve nested attributes. For example: + the Message.subset.shape within the input Harmony message. + + Note, if a default value is specified, this will be returned if any + attribute in the specified chain is absent from the supplied object. + Alternatively, if an absent attribute is specified and no default value + if given in the function call, this function will return `None`. + + """ + if len(args) == 0: + args = (None, ) + + if '.' not in requested_attribute: + result = getattr(input_object, requested_attribute, *args) + else: + attribute_pieces = requested_attribute.split('.') + result = rgetattr(getattr(input_object, attribute_pieces[0], *args), + '.'.join(attribute_pieces[1:]), *args) + + return result diff --git a/tests/test_message_utilities.py b/tests/test_message_utilities.py new file mode 100644 index 0000000..6bdf41d --- /dev/null +++ b/tests/test_message_utilities.py @@ -0,0 +1,477 @@ +from unittest import TestCase + +from harmony.message import Message + +from harmony.message_utility import ( + _has_all_attributes, + _has_consistent_dimension, + has_crs, + has_dimensions, + has_self_consistent_grid, + has_scale_extents, + has_scale_sizes, + has_valid_scale_extents, + rgetattr, +) + + +class TestMessageUtility(TestCase): + """Test Harmony Message utilities.""" + + + def test_self_has_self_consistent_grid(self): + """ Ensure that the function correctly determines if the supplied + Harmony message defines a valid target grid. This should either: + + * Specify scaleExtent and 1 of: + * Height and Width + * Scale sizes (in the x and y horizontal spatial dimensions) + * Specify all three of the above, but the values must be consistent + with one another. + + """ + valid_scale_extents = {'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}} + + valid_scale_sizes = {'x': 0.5, 'y': 1.0} + valid_height = 180 + valid_width = 720 + + with self.subTest('format = None returns False'): + test_message = Message({}) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('All grid parameters = None returns False'): + test_message = Message({'format': {}}) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Only scaleExtent returns False'): + test_message = Message({ + 'format': {'scaleExtents': valid_scale_extents} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Only scaleSize returns False'): + test_message = Message({ + 'format': {'scaleSize': valid_scale_sizes} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Only dimensions (height and width) returns False'): + test_message = Message({ + 'format': {'height': valid_height, 'width': valid_width} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Dimensions and scaleSize returns False'): + # Need the extents to know where to put the pixels! + test_message = Message({ + 'format': {'height': valid_height, + 'scaleSize': valid_scale_sizes, + 'width': valid_width} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Dimensions and scaleExtent returns True'): + test_message = Message({ + 'format': {'height': valid_height, + 'scaleExtent': valid_scale_extents, + 'width': valid_width} + }) + self.assertTrue(has_self_consistent_grid(test_message)) + + with self.subTest('scaleExtent and scaleSize returns True'): + test_message = Message({ + 'format': {'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes} + }) + self.assertTrue(has_self_consistent_grid(test_message)) + + with self.subTest('All three, and consistent returns True'): + test_message = Message({ + 'format': {'height': valid_height, + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'width': valid_width} + }) + self.assertTrue(has_self_consistent_grid(test_message)) + + with self.subTest('All three, but inconsistent returns False'): + test_message = Message({ + 'format': {'height': valid_height + 150, + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'width': valid_width - 150} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + def test_self_has_self_consistent_grid_missing_height_or_width(self): + """ Ensure that the function correctly determines if the supplied + Harmony message defines a valid target grid. These test cases check + when only one of height or width are specified. + + If there is sufficient other grid information, and the one listed + dimension with all three pieces of information is self consistent, + then the message passes validation. + + """ + valid_scale_extents = {'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}} + + valid_scale_sizes = {'x': 0.5, 'y': 1.0} + valid_height = 180 + valid_width = 720 + + with self.subTest('Only height and scaleExtent returns False'): + test_message = Message({ + 'format': {'height': valid_height, + 'scaleExtent': valid_scale_extents} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Only width and scaleExtent returns False'): + test_message = Message({ + 'format': {'scaleExtent': valid_scale_extents, + 'width': valid_width} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Only height and scaleSize returns False'): + test_message = Message({ + 'format': {'height': valid_height, + 'scaleSize': valid_scale_sizes} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Only width and scaleSize returns False'): + test_message = Message({ + 'format': {'scaleSize': valid_scale_sizes, + 'width': valid_width} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Width, scaleSize, scaleExtent inconsistent, False'): + test_message = Message({ + 'format': {'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'width': valid_width - 150} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Height, scaleSize, scaleExtent inconsistent, False'): + test_message = Message({ + 'format': {'height': valid_height + 150, + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes} + }) + self.assertFalse(has_self_consistent_grid(test_message)) + + with self.subTest('Width, scaleSize, scaleExtent consistent, True'): + test_message = Message({ + 'format': {'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'width': valid_width} + }) + self.assertTrue(has_self_consistent_grid(test_message)) + + with self.subTest('Height, scaleSize, scaleExtent consistent, True'): + test_message = Message({ + 'format': {'height': valid_height, + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes} + }) + self.assertTrue(has_self_consistent_grid(test_message)) + + + def test_message_has_crs(self): + message = Message({'format': {'crs': 'EPSG:4326'}}) + self.assertTrue(has_crs(message)) + + def test_message_has_garbage_crs(self): + message = Message({'format': {'crs': 'garbage'}}) + self.assertTrue(has_crs(message)) + + def test_message_has_no_crs(self): + message = Message({}) + self.assertFalse(has_crs(message)) + + def test_has_consistent_dimension(self): + """Ensure, given a scale size (resolution), scale extent (range) and + dimension size, the function can correctly determine if all three + values are consistent with one another. + + """ + valid_scale_extents = { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + } + + valid_scale_sizes = {'x': 0.5, 'y': 1.0} + valid_height = 180 + valid_width = 720 + + with self.subTest('Consistent x dimension returns True'): + test_message = Message( + { + 'format': { + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'width': valid_width, + } + } + ) + self.assertTrue(_has_consistent_dimension(test_message, 'x')) + + with self.subTest('Consistent y dimension returns True'): + test_message = Message( + { + 'format': { + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'height': valid_height, + } + } + ) + self.assertTrue(_has_consistent_dimension(test_message, 'y')) + + with self.subTest('Inconsistent x dimension returns False'): + test_message = Message( + { + 'format': { + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'width': valid_width + 100, + } + } + ) + self.assertFalse(_has_consistent_dimension(test_message, 'x')) + + with self.subTest('Inconsistent y dimension returns False'): + test_message = Message( + { + 'format': { + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'height': valid_height + 100, + } + } + ) + self.assertFalse(_has_consistent_dimension(test_message, 'y')) + + def test_has_all_attributes(self): + """Ensure that the function returns the correct boolean value + indicating if all requested attributes are present in the supplied + object, and have non-None values. + + """ + test_message = Message({'format': {'scaleSize': {'x': 0.5, 'y': 0.5}}}) + + with self.subTest('All attributes present returns True'): + self.assertTrue( + _has_all_attributes( + test_message, ['format.scaleSize.x', 'format.scaleSize.y'] + ) + ) + + with self.subTest('Some attributes present returns False'): + self.assertFalse( + _has_all_attributes( + test_message, ['format.scaleSize.x', 'format.height'] + ) + ) + + with self.subTest('No attributes present returns False'): + self.assertFalse( + _has_all_attributes(test_message, ['format.height', 'format.width']) + ) + + def test_has_scale_sizes(self): + """Ensure the function correctly identifies whether the supplied + Harmony message contains both an x and y scale size. + + """ + with self.subTest('Scale sizes present returns True'): + test_message = Message({'format': {'scaleSize': {'x': 0.5, 'y': 0.5}}}) + self.assertTrue(has_scale_sizes(test_message)) + + with self.subTest('scaleSize.x = None returns False'): + test_message = Message({'format': {'scaleSize': {'y': 0.5}}}) + self.assertFalse(has_scale_sizes(test_message)) + + with self.subTest('scaleSize.y = None returns False'): + test_message = Message({'format': {'scaleSize': {'x': 0.5}}}) + self.assertFalse(has_scale_sizes(test_message)) + + with self.subTest('Both scaleSizes = None returns False'): + test_message = Message({'format': {'scaleSize': {}}}) + self.assertFalse(has_scale_sizes(test_message)) + + with self.subTest('format = None returns False'): + test_message = Message({}) + self.assertFalse(has_scale_sizes(test_message)) + + def test_has_valid_scale_extents(self): + with self.subTest('ScaleExtent present and valid returns True'): + test_message = Message( + { + 'format': { + 'scaleExtent': { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + } + } + } + ) + self.assertTrue(has_valid_scale_extents(test_message)) + + with self.subTest('ScaleExtent missing returns True'): + test_message = Message({'format': {}}) + self.assertTrue(has_valid_scale_extents(test_message)) + + with self.subTest('ScaleExtent present and invalid returns False'): + test_message = Message( + { + 'format': { + 'scaleExtent': { + 'x': {'min': 180, 'max': -180}, + 'y': {'min': -90, 'max': 90}, + } + } + } + ) + self.assertFalse(has_valid_scale_extents(test_message)) + + def test_has_scale_extents(self): + """Ensure the function correctly identifies whether the supplied + Harmony message contains all required elements in the + `format.scaleExtent` attribute. This includes minima and maxima for + both the x and y horizontal spatial dimensions of the target grid. + + """ + x_extents = {'min': -180, 'max': 180} + y_extents = {'min': -90, 'max': 90} + + with self.subTest('Scale extents present returns True'): + test_message = Message( + {'format': {'scaleExtent': {'x': x_extents, 'y': y_extents}}} + ) + self.assertTrue(has_scale_extents(test_message)) + + with self.subTest('scaleExtent.x.min = None returns False'): + test_message = Message( + {'format': {'scaleExtent': {'x': {'max': 180}, 'y': y_extents}}} + ) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('scaleExtent.x.max = None returns False'): + test_message = Message( + {'format': {'scaleExtent': {'x': {'min': -180}, 'y': y_extents}}} + ) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('scaleExtent.x min and max = None returns False'): + test_message = Message( + {'format': {'scaleExtent': {'x': {}, 'y': y_extents}}} + ) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('scaleExtent.y.min = None returns False'): + test_message = Message( + {'format': {'scaleExtent': {'x': x_extents, 'y': {'max': 90}}}} + ) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('scaleExtent.y.max = None returns False'): + test_message = Message( + {'format': {'scaleExtent': {'x': x_extents, 'y': {'min': -90}}}} + ) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('scaleExtent.y min and max = None returns False'): + test_message = Message( + {'format': {'scaleExtent': {'x': x_extents, 'y': {}}}} + ) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('All scaleExtent values = None returns False'): + test_message = Message({'format': {'scaleExtent': {'x': {}, 'y': {}}}}) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('scaleExtent.x and scaleExtent.y = None returns False'): + test_message = Message({'format': {'scaleExtent': {}}}) + self.assertFalse(has_scale_extents(test_message)) + + with self.subTest('format = None returns False'): + test_message = Message({}) + self.assertFalse(has_scale_extents(test_message)) + + def test_has_dimensions(self): + """Ensure the function correctly validates whether the supplied + Harmony message contains both a height an width for the target + grid. + + """ + with self.subTest('height and width present returns True'): + test_message = Message({'format': {'height': 100, 'width': 50}}) + self.assertTrue(has_dimensions(test_message)) + + with self.subTest('height = None returns False'): + test_message = Message({'format': {'width': 50}}) + self.assertFalse(has_dimensions(test_message)) + + with self.subTest('width = None returns False'): + test_message = Message({'format': {'height': 100}}) + self.assertFalse(has_dimensions(test_message)) + + with self.subTest('height = None and width = None returns False'): + test_message = Message({'format': {}}) + self.assertFalse(has_dimensions(test_message)) + + with self.subTest('format = None returns False'): + test_message = Message({}) + self.assertFalse(has_dimensions(test_message)) + + def test_rgetattr(self): + """Ensure that the recursive function can retrieve nested attributes + and uses the default argument when required. + + """ + + class Child: + def __init__(self, name): + self.name = name + + class Parent: + def __init__(self, name, child_name): + self.name = name + self.child = Child(child_name) + + test_parent = Parent('parent_name', 'child_name') + + with self.subTest('Parent level attribute'): + self.assertEqual(rgetattr(test_parent, 'name'), 'parent_name') + + with self.subTest('Nested attribute'): + self.assertEqual(rgetattr(test_parent, 'child.name'), 'child_name') + + with self.subTest('Missing parent with default'): + self.assertEqual(rgetattr(test_parent, 'absent', 'default'), 'default') + + with self.subTest('Missing child attribute with default'): + self.assertEqual( + rgetattr(test_parent, 'child.absent', 'default'), 'default' + ) + + with self.subTest('Child requested, parent missing, default'): + self.assertEqual( + rgetattr(test_parent, 'none.something', 'default'), 'default' + ) + + with self.subTest('Missing parent, with no default'): + self.assertIsNone(rgetattr(test_parent, 'absent')) + + with self.subTest('Missing child, with no default'): + self.assertIsNone(rgetattr(test_parent, 'child.absent')) + + with self.subTest('Child requested, parent missing, no default'): + self.assertIsNone(rgetattr(test_parent, 'absent.something'))