diff --git a/hsmodels/schemas/aggregations.py b/hsmodels/schemas/aggregations.py index d3daa43..eee43d0 100644 --- a/hsmodels/schemas/aggregations.py +++ b/hsmodels/schemas/aggregations.py @@ -1,4 +1,4 @@ -from datetime import date, datetime +from datetime import date from typing import Dict, List, Union from pydantic import AnyUrl, Field, root_validator, validator @@ -38,11 +38,8 @@ ) -class BaseAggregationMetadata(BaseMetadata): +class BaseAggregationMetadataIn(BaseMetadata): - url: AnyUrl = Field( - title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False - ) title: str = Field( title="Aggregation title", description="A string containing a descriptive title for the aggregation" ) @@ -79,7 +76,6 @@ class BaseAggregationMetadata(BaseMetadata): _parse_additional_metadata = root_validator(pre=True, allow_reuse=True)(parse_additional_metadata) _parse_coverages = root_validator(pre=True, allow_reuse=True)(split_coverages) - _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) _subjects_constraint = validator('subjects', allow_reuse=True)(subjects_constraint) _language_constraint = validator('language', allow_reuse=True)(language_constraint) @@ -87,7 +83,7 @@ class BaseAggregationMetadata(BaseMetadata): _normalize_additional_metadata = root_validator(allow_reuse=True, pre=True)(normalize_additional_metadata) -class GeographicRasterMetadata(BaseAggregationMetadata): +class GeographicRasterMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a geographic raster aggregation @@ -101,14 +97,6 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} - type: AggregationType = Field( - const=True, - default=AggregationType.GeographicRasterAggregation, - title="Aggregation type", - description="A string expressing the aggregation type from the list of HydroShare aggregation types", - allow_mutation=False, - ) - band_information: BandInformation = Field( title="Band information", description="An object containing information about the bands contained in the raster dataset", @@ -125,7 +113,26 @@ class Config: _parse_spatial_reference = validator("spatial_reference", pre=True, allow_reuse=True)(parse_spatial_reference) -class GeographicFeatureMetadata(BaseAggregationMetadata): +class GeographicRasterMetadata(GeographicRasterMetadataIn): + + _type = AggregationType.GeographicRasterAggregation + + type: AggregationType = Field( + const=True, + default=_type, + title="Aggregation type", + description="A string expressing the aggregation type from the list of HydroShare aggregation types", + allow_mutation=False, + ) + + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) + + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) + + +class GeographicFeatureMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a geographic feature aggregation @@ -139,14 +146,6 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} - type: AggregationType = Field( - const=True, - default=AggregationType.GeographicFeatureAggregation, - title="Aggregation type", - description="A string expressing the aggregation type from the list of HydroShare aggregation types", - allow_mutation=False, - ) - field_information: List[FieldInformation] = Field( default=[], title="Field information", @@ -165,7 +164,24 @@ class Config: _parse_spatial_reference = validator("spatial_reference", pre=True, allow_reuse=True)(parse_spatial_reference) -class MultidimensionalMetadata(BaseAggregationMetadata): +class GeographicFeatureMetadata(GeographicFeatureMetadataIn): + + type: AggregationType = Field( + const=True, + default=AggregationType.GeographicFeatureAggregation, + title="Aggregation type", + description="A string expressing the aggregation type from the list of HydroShare aggregation types", + allow_mutation=False, + ) + + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) + + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) + + +class MultidimensionalMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a multidimensional space-time aggregation @@ -179,14 +195,6 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} - type: AggregationType = Field( - const=True, - default=AggregationType.MultidimensionalAggregation, - title="Aggregation type", - description="A string expressing the aggregation type from the list of HydroShare aggregation types", - allow_mutation=False, - ) - variables: List[Variable] = Field( default=[], title="Variables", @@ -203,7 +211,24 @@ class Config: ) -class ReferencedTimeSeriesMetadata(BaseAggregationMetadata): +class MultidimensionalMetadata(MultidimensionalMetadataIn): + + type: AggregationType = Field( + const=True, + default=AggregationType.MultidimensionalAggregation, + title="Aggregation type", + description="A string expressing the aggregation type from the list of HydroShare aggregation types", + allow_mutation=False, + ) + + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) + + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) + + +class ReferencedTimeSeriesMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a referenced time series aggregation @@ -217,6 +242,9 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} + +class ReferencedTimeSeriesMetadata(ReferencedTimeSeriesMetadataIn): + type: AggregationType = Field( const=True, default=AggregationType.ReferencedTimeSeriesAggregation, @@ -225,8 +253,14 @@ class Config: allow_mutation=False, ) + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) + + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) + -class FileSetMetadata(BaseAggregationMetadata): +class FileSetMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a file set aggregation @@ -240,6 +274,9 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} + +class FileSetMetadata(FileSetMetadataIn): + type: AggregationType = Field( const=True, default=AggregationType.FileSetAggregation, @@ -248,8 +285,14 @@ class Config: allow_mutation=False, ) + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) + + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) + -class SingleFileMetadata(BaseAggregationMetadata): +class SingleFileMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a single file aggregation @@ -262,6 +305,9 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} + +class SingleFileMetadata(SingleFileMetadataIn): + type: AggregationType = Field( const=True, default=AggregationType.SingleFileAggregation, @@ -270,8 +316,14 @@ class Config: allow_mutation=False, ) + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) -class TimeSeriesMetadata(BaseAggregationMetadata): + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) + + +class TimeSeriesMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a time series aggregation @@ -287,6 +339,19 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} + time_series_results: List[TimeSeriesResult] = Field( + default=[], + title="Time series results", + description="A list of time series results contained within the time series aggregation", + ) + + abstract: str = Field(default=None, title="Abstract", description="A string containing a summary of a aggregation") + + _parse_abstract = root_validator(pre=True, allow_reuse=True)(parse_abstract) + + +class TimeSeriesMetadata(TimeSeriesMetadataIn): + type: AggregationType = Field( const=True, default=AggregationType.TimeSeriesAggregation, @@ -295,18 +360,14 @@ class Config: allow_mutation=False, ) - time_series_results: List[TimeSeriesResult] = Field( - default=[], - title="Time series results", - description="A list of time series results contained within the time series aggregation", + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False ) - abstract: str = Field(default=None, title="Abstract", description="A string containing a summary of a aggregation") - - _parse_abstract = root_validator(pre=True, allow_reuse=True)(parse_abstract) + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) -class ModelProgramMetadata(BaseAggregationMetadata): +class ModelProgramMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a model program aggregation """ @@ -316,14 +377,6 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} - type: AggregationType = Field( - const=True, - default=AggregationType.ModelProgramAggregation, - title="Aggregation type", - description="A string expressing the aggregation type from the list of HydroShare aggregation types", - allow_mutation=False, - ) - version: str = Field( default=None, title="Version", description="The software version or build number of the model", max_length=255 ) @@ -378,7 +431,24 @@ class Config: _parse_file_types = root_validator(pre=True, allow_reuse=True)(parse_file_types) -class ModelInstanceMetadata(BaseAggregationMetadata): +class ModelProgramMetadata(ModelProgramMetadataIn): + + type: AggregationType = Field( + const=True, + default=AggregationType.ModelProgramAggregation, + title="Aggregation type", + description="A string expressing the aggregation type from the list of HydroShare aggregation types", + allow_mutation=False, + ) + + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) + + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) + + +class ModelInstanceMetadataIn(BaseAggregationMetadataIn): """ A class used to represent the metadata associated with a model instance aggregation """ @@ -388,14 +458,6 @@ class Config: schema_config = {'read_only': ['type', 'url'], 'dictionary_field': ['additional_metadata']} - type: AggregationType = Field( - const=True, - default=AggregationType.ModelInstanceAggregation, - title="Aggregation type", - description="A string expressing the aggregation type from the list of HydroShare aggregation types", - allow_mutation=False, - ) - includes_model_output: bool = Field( title="Includes Model Output", description="Indicates whether model output files are included in the aggregation", @@ -418,3 +480,20 @@ class Config: title="JSON metadata schema values URL", description="A URL to a JSON file containing the metadata values conforming to the JSON metadata schema for the related model program", ) + + +class ModelInstanceMetadata(ModelInstanceMetadataIn): + + type: AggregationType = Field( + const=True, + default=AggregationType.ModelInstanceAggregation, + title="Aggregation type", + description="A string expressing the aggregation type from the list of HydroShare aggregation types", + allow_mutation=False, + ) + + url: AnyUrl = Field( + title="Aggregation URL", description="An object containing the URL of the aggregation", allow_mutation=False + ) + + _parse_url = root_validator(pre=True, allow_reuse=True)(parse_url) diff --git a/hsmodels/schemas/base_models.py b/hsmodels/schemas/base_models.py index b598b22..78e01a3 100644 --- a/hsmodels/schemas/base_models.py +++ b/hsmodels/schemas/base_models.py @@ -77,21 +77,6 @@ def json( **dumps_kwargs, ) - @classmethod - def schema(cls, by_alias: bool = True, ref_template: str = default_ref_template) -> 'DictStrAny': - cached = cls.__schema_cache__.get((by_alias, ref_template)) - if cached is not None: - return cached - s = model_schema(cls, by_alias=by_alias, ref_template=ref_template) - - if 'definitions' in s: - # TODO this is a terrible hack that needs to be revisited once we get a form build that embedded properties - from hsmodels.schemas.fields import AdditionalMetadata - - s['definitions']['AdditionalMetadata'] = AdditionalMetadata.schema() - cls.__schema_cache__[(by_alias, ref_template)] = s - return s - class Config: validate_assignment = True @@ -102,14 +87,23 @@ def schema_extra(schema: Dict[str, Any], model) -> None: if "read_only" in schema_config: # set readOnly in json schema for field in schema_config["read_only"]: - schema['properties'][field]['readOnly'] = True + if field in schema['properties']: # ignore unknown properties for inheritance + schema['properties'][field]['readOnly'] = True if "dictionary_field" in schema_config: for field in schema_config["dictionary_field"]: - prop = schema["properties"][field] - prop.pop('default', None) - prop.pop('additionalProperties', None) - prop['type'] = "array" - prop['items'] = {"$ref": "#/definitions/AdditionalMetadata"} + if field in schema['properties']: # ignore unknown properties for inheritance + prop = schema["properties"][field] + prop.pop('default', None) + prop.pop('additionalProperties', None) + prop['type'] = "array" + prop['items'] = { + "type": "object", + "title": "Key-Value", + "description": "A key-value pair", + "default": [], + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + } class BaseCoverage(BaseMetadata): diff --git a/hsmodels/schemas/fields.py b/hsmodels/schemas/fields.py index be9d05b..d7ffea9 100644 --- a/hsmodels/schemas/fields.py +++ b/hsmodels/schemas/fields.py @@ -964,15 +964,3 @@ class Config: url: AnyUrl = Field( title="Model program file url", description="The url of the file used by the model program", default=None ) - - -class AdditionalMetadata(BaseMetadata): - """ - A class used to represent the additional metadata Key/Value pairs - """ - - class Config: - title = "Additional metadata" - - key: str = Field(title="Key", description="A string for the additional metadata key", default="") - value: str = Field(title="Value", description="A string for the additional metadata value", default="") diff --git a/hsmodels/schemas/resource.py b/hsmodels/schemas/resource.py index 4297f66..8d17bfc 100644 --- a/hsmodels/schemas/resource.py +++ b/hsmodels/schemas/resource.py @@ -29,13 +29,14 @@ class ResourceMetadataIn(BaseMetadata): """ - A class used to represent the metadata for a resource that can be modified + A class used to represent the metadata for a resource """ class Config: title = 'Resource Metadata' schema_config = { + 'read_only': ['type', 'identifier', 'created', 'modified', 'published', 'url'], 'dictionary_field': ['additional_metadata'], } @@ -119,17 +120,6 @@ class Config: class ResourceMetadata(ResourceMetadataIn): - """ - A class used to represent the metadata for a resource - """ - - class Config: - title = 'Resource Metadata' - - schema_config = { - 'read_only': ['type', 'identifier', 'created', 'modified', 'published', 'url'], - 'dictionary_field': ['additional_metadata'], - } type: str = Field( const=True, diff --git a/hsmodels/schemas/root_validators.py b/hsmodels/schemas/root_validators.py index ba7b4a8..f1cfa91 100644 --- a/hsmodels/schemas/root_validators.py +++ b/hsmodels/schemas/root_validators.py @@ -90,15 +90,16 @@ def group_user_identifiers(cls, values): def parse_file_types(cls, values): - file_types_list = [] - for file_type in ModelProgramFileType: - if file_type.name in values: - ftypes = values[file_type.name] - if isinstance(ftypes, list): - for ftype in ftypes: - file_types_list.append({"type": file_type, "url": ftype}) - del values[file_type.name] - values['file_types'] = file_types_list + if "file_types" not in values: + file_types_list = [] + for file_type in ModelProgramFileType: + if file_type.name in values: + ftypes = values[file_type.name] + if isinstance(ftypes, list): + for ftype in ftypes: + file_types_list.append({"type": file_type, "url": ftype}) + del values[file_type.name] + values['file_types'] = file_types_list return values diff --git a/setup.py b/setup.py index 8cc3ab8..3356173 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ from setuptools import setup, find_packages -README = 'Refer to the models seciont of https://hydroshare.github.io/hsclient/'# (pathlib.Path(__file__).parent / "README.md").read_text() +README = 'Refer to the models section of https://hydroshare.github.io/hsclient/'# (pathlib.Path(__file__).parent / "README.md").read_text() setup( name='hsmodels', - version='0.3.6', + version='0.4.0', packages=find_packages(include=['hsmodels', 'hsmodels.*', 'hsmodels.schemas.*', 'hsmodels.schemas.rdf.*'], exclude=("tests",)), install_requires=[ diff --git a/tests/data/json/fileset.json b/tests/data/json/fileset.json new file mode 100644 index 0000000..ae18249 --- /dev/null +++ b/tests/data/json/fileset.json @@ -0,0 +1,28 @@ +{ + "title": "asdfa", + "subjects": [ + "key2", + "key1" + ], + "language": "eng", + "additional_metadata": { + "key2": "value2", + "key": "value" + }, + "spatial_coverage": { + "type": "point", + "name": "point coverage", + "east": -97.8454, + "north": 36.5415, + "units": "Decimal degrees", + "projection": "WGS 84 EPSG:4326" + }, + "period_coverage": { + "start": "2020-11-03T00:00:00", + "end": "2020-11-19T00:00:00" + }, + "rights": { + "statement": "my statement", + "url": "http://studio.bakajo.com" + } +} \ No newline at end of file diff --git a/tests/data/json/geographicfeature.json b/tests/data/json/geographicfeature.json new file mode 100644 index 0000000..b632cad --- /dev/null +++ b/tests/data/json/geographicfeature.json @@ -0,0 +1,85 @@ +{ + "title": "watershedsa", + "subjects": [ + "asdf", + "fdas" + ], + "language": "eng", + "additional_metadata": { + "key": "value", + "key2": "value2" + }, + "spatial_coverage": { + "type": "box", + "northlimit": 41.84732649257394, + "eastlimit": -111.40051926333071, + "southlimit": 41.46101782271056, + "westlimit": -111.79812580632701, + "units": "Decimal degrees", + "projection": "WGS 84 EPSG:4326" + }, + "period_coverage": { + "start": "2020-11-05T00:00:00", + "end": "2020-11-26T00:00:00" + }, + "rights": { + "statement": "my statement", + "url": "http://studio.bakajo.com" + }, + "field_information": [ + { + "field_name": "Id", + "field_type": "Integer64", + "field_type_code": "12", + "field_width": 10 + }, + { + "field_name": "Shape_Area", + "field_type": "Real", + "field_type_code": "2", + "field_width": 19, + "field_precision": 11 + }, + { + "field_name": "Area_km2", + "field_type": "Real", + "field_type_code": "2", + "field_width": 19, + "field_precision": 11 + }, + { + "field_name": "OBJECTID", + "field_type": "Integer64", + "field_type_code": "12", + "field_width": 10 + }, + { + "field_name": "gridcode", + "field_type": "Integer64", + "field_type_code": "12", + "field_width": 10 + }, + { + "field_name": "Shape_Leng", + "field_type": "Real", + "field_type_code": "2", + "field_width": 19, + "field_precision": 11 + } + ], + "geometry_information": { + "feature_count": 5, + "geometry_type": "POLYGON" + }, + "spatial_reference": { + "type": "box", + "northlimit": 4633133.1700797, + "eastlimit": 466551.480322555, + "southlimit": 4590013.8756461, + "westlimit": 433743.114844041, + "units": "metre", + "projection_string": "PROJCS[\"NAD83 / UTM zone 12N\",\n GEOGCS[\"NAD83\",\n DATUM[\"North_American_Datum_1983\",\n SPHEROID[\"GRS 1980\",6378137,298.257222101,\n AUTHORITY[\"EPSG\",\"7019\"]],\n TOWGS84[0,0,0,0,0,0,0],\n AUTHORITY[\"EPSG\",\"6269\"]],\n PRIMEM[\"Greenwich\",0,\n AUTHORITY[\"EPSG\",\"8901\"]],\n UNIT[\"degree\",0.0174532925199433,\n AUTHORITY[\"EPSG\",\"9122\"]],\n AUTHORITY[\"EPSG\",\"4269\"]],\n PROJECTION[\"Transverse_Mercator\"],\n PARAMETER[\"latitude_of_origin\",0],\n PARAMETER[\"central_meridian\",-111],\n PARAMETER[\"scale_factor\",0.9996],\n PARAMETER[\"false_easting\",500000],\n PARAMETER[\"false_northing\",0],\n UNIT[\"metre\",1,\n AUTHORITY[\"EPSG\",\"9001\"]],\n AXIS[\"Easting\",EAST],\n AXIS[\"Northing\",NORTH],\n AUTHORITY[\"EPSG\",\"26912\"]]", + "datum": "North_American_Datum_1983", + "projection_name": "NAD83 / UTM zone 12N" + } +} \ No newline at end of file diff --git a/tests/data/json/geographicraster.json b/tests/data/json/geographicraster.json new file mode 100644 index 0000000..9d1d769 --- /dev/null +++ b/tests/data/json/geographicraster.json @@ -0,0 +1,59 @@ +{ + "title": "logan1a", + "subjects": [ + "key2", + "key1" + ], + "language": "eng", + "additional_metadata": { + "key": "value", + "key2": "value2" + }, + "spatial_coverage": { + "type": "box", + "name": "box coverage", + "northlimit": 42.050028785767275, + "eastlimit": -111.5773750264389, + "southlimit": 41.98745777902698, + "westlimit": -111.65768822411239, + "units": "Decimal degrees", + "projection": "WGS 84 EPSG:4326" + }, + "period_coverage": { + "start": "2020-11-05T00:00:00", + "end": "2020-11-26T00:00:00" + }, + "rights": { + "statement": "my statement", + "url": "http://studio.bakajo.com" + }, + "band_information": { + "name": "Band_1", + "variable_name": "variablename", + "variable_unit": "variableunit", + "no_data_value": "-3.4028234663852886E+38", + "maximum_value": "6.9103932008822E-310", + "comment": "comment", + "method": "method", + "minimum_value": "2.67393466E-316" + }, + "spatial_reference": { + "type": "box", + "northlimit": 4655492.446916306, + "eastlimit": 452174.01909127034, + "southlimit": 4648592.446916306, + "westlimit": 445574.01909127034, + "units": "meter", + "projection": "NAD83 / UTM zone 12N", + "projection_string": "PROJCS[\"NAD83 / UTM zone 12N\",GEOGCS[\"NAD83\",DATUM[\"North_American_Datum_1983\",SPHEROID[\"GRS 1980\",6378137,298.2572221010002,AUTHORITY[\"EPSG\",\"7019\"]],AUTHORITY[\"EPSG\",\"6269\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433],AUTHORITY[\"EPSG\",\"4269\"]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",-111],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AUTHORITY[\"EPSG\",\"26912\"]]", + "datum": "North_American_Datum_1983" + }, + "cell_information": { + "name": "logan.vrt", + "rows": 230, + "columns": 220, + "cell_size_x_value": 30.0, + "cell_data_type": "Float32", + "cell_size_y_value": 30.0 + } +} \ No newline at end of file diff --git a/tests/data/json/modelinstance.json b/tests/data/json/modelinstance.json new file mode 100644 index 0000000..942a0e4 --- /dev/null +++ b/tests/data/json/modelinstance.json @@ -0,0 +1,32 @@ +{ + "title": "generic_file", + "subjects": [ + "fdsa", + "asdf" + ], + "language": "eng", + "additional_metadata": { + "1": "2", + "3": "4" + }, + "spatial_coverage": { + "type": "point", + "name": "asdf", + "east": -98.1969, + "north": 41.85, + "units": "Decimal degrees", + "projection": "WGS 84 EPSG:4326" + }, + "period_coverage": { + "start": "2021-10-08T00:00:00", + "end": "2021-10-15T00:00:00" + }, + "rights": { + "statement": "This resource is shared under the Creative Commons Attribution CC BY.", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "includes_model_output": false, + "executed_by": "https://www.hydroshare.org/resource/636729f24c03497e9b844ec3be63e823/data/contents/modelprogram_resmap.xml#aggregation", + "program_schema_json": "https://www.hydroshare.org/resource/636729f24c03497e9b844ec3be63e823/data/contents/modelinstance_schema.json", + "program_schema_json_values": "https://www.hydroshare.org/resource/636729f24c03497e9b844ec3be63e823/data/contents/modelinstance_schema_values.json" +} \ No newline at end of file diff --git a/tests/data/json/modelprogram.json b/tests/data/json/modelprogram.json new file mode 100644 index 0000000..2dcde45 --- /dev/null +++ b/tests/data/json/modelprogram.json @@ -0,0 +1,64 @@ +{ + "title": "setup againas", + "subjects": [ + "asdf", + "fdsa" + ], + "language": "eng", + "additional_metadata": { + "1": "2", + "3": "4" + }, + "rights": { + "statement": "This resource is shared under the Creative Commons Attribution CC BY.", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "version": "1", + "name": "Unknown Model Program", + "programming_languages": [ + "R", + "python" + ], + "operating_systems": [ + "linux", + "windows" + ], + "release_date": "2021-09-22", + "website": "https://www.hydroshare.org", + "code_repository": "https://www.github.com", + "file_types": [ + { + "type": "https://www.hydroshare.org/terms/modelReleaseNotes", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup1.cfg" + }, + { + "type": "https://www.hydroshare.org/terms/modelReleaseNotes", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup.cfg" + }, + { + "type": "https://www.hydroshare.org/terms/modelDocumentation", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup.cfg" + }, + { + "type": "https://www.hydroshare.org/terms/modelDocumentation", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup1.cfg" + }, + { + "type": "https://www.hydroshare.org/terms/modelSoftware", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup1.cfg" + }, + { + "type": "https://www.hydroshare.org/terms/modelSoftware", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup.cfg" + }, + { + "type": "https://www.hydroshare.org/terms/modelEngine", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup.cfg" + }, + { + "type": "https://www.hydroshare.org/terms/modelEngine", + "url": "https://www.hydroshare.org/resource/526e62aac1044285abc2cae4e2f463da/data/contents/setup1.cfg" + } + ], + "program_schema_json": "https://www.hydroshare.org/resource/636729f24c03497e9b844ec3be63e823/data/contents/modelprogram_schema.json" +} \ No newline at end of file diff --git a/tests/data/json/multidimensional.json b/tests/data/json/multidimensional.json new file mode 100644 index 0000000..2283f9c --- /dev/null +++ b/tests/data/json/multidimensional.json @@ -0,0 +1,86 @@ +{ + "title": "changed UEB modael simulation of snow water equivalent in Logan River watershed from 2008 to 2009", + "subjects": [ + "changed", + "Snow water equivalent", + "Logan River", + "UEB" + ], + "language": "eng", + "additional_metadata": { + "key1": "value1", + "key2": "value2" + }, + "spatial_coverage": { + "type": "box", + "name": "named", + "northlimit": 39.4499, + "eastlimit": -90.4626, + "southlimit": 30.5397, + "westlimit": -107.6891, + "units": "Decimal degrees", + "projection": "WGS 84 EPSG:4326" + }, + "period_coverage": { + "start": "2008-10-05T00:00:00", + "end": "2009-06-18T00:00:00" + }, + "rights": { + "statement": "my statement", + "url": "http://studio.bakajo.com" + }, + "variables": [ + { + "name": "x", + "unit": "m", + "type": "Float", + "shape": "x", + "descriptive_name": "x coordinate of projection" + }, + { + "name": "transverse_mercator", + "unit": "Unknown", + "type": "Unknown", + "shape": "Not defined" + }, + { + "name": "time", + "unit": "hours since 2008-10-1 0:0:00 UTC", + "type": "Float", + "shape": "time", + "descriptive_name": "time", + "method": "a comment", + "missing_value": "chnagd" + }, + { + "name": "y", + "unit": "m", + "type": "Float", + "shape": "y", + "descriptive_name": "y coordinate of projection", + "method": "comment", + "missing_value": "changed" + }, + { + "name": "SWE", + "unit": "m", + "type": "Float", + "shape": "time,y,x", + "descriptive_name": "Snow water equivalent", + "method": "comment", + "missing_value": "missing value" + } + ], + "spatial_reference": { + "type": "box", + "northlimit": 47.1121, + "eastlimit": -95.736, + "southlimit": 25.2607, + "westlimit": -120.8727, + "units": "Meter", + "projection_string": "PROJCS[\"NAD83 / UTM zone 12N\",GEOGCS[\"NAD83\",DATUM[\"North_American_Datum_1983\",SPHEROID[\"GRS 1980\",6378137,298.2572221010002,AUTHORITY[\"EPSG\",\"7019\"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY[\"EPSG\",\"6269\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433],AUTHORITY[\"EPSG\",\"4269\"]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",-111],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AUTHORITY[\"EPSG\",\"26912\"]]", + "projection_string_type": "box", + "datum": "North_American_Datum_1983", + "projection_name": "NAD83 / UTM zone 12N" + } +} \ No newline at end of file diff --git a/tests/data/json/referencedtimeseries.refts.json b/tests/data/json/referencedtimeseries.refts.json new file mode 100644 index 0000000..2812df9 --- /dev/null +++ b/tests/data/json/referencedtimeseries.refts.json @@ -0,0 +1,30 @@ +{ + "title": "Sites, Variable", + "subjects": [ + "Time Series", + "CUAHSI" + ], + "language": "eng", + "additional_metadata": { + "key2": "value2", + "one": "two", + "key": "value" + }, + "spatial_coverage": { + "type": "box", + "northlimit": 40.48498, + "eastlimit": -111.46245, + "southlimit": 40.1788719, + "westlimit": -111.639338, + "units": "Decimal degrees", + "projection": "Unknown" + }, + "period_coverage": { + "start": "2016-04-06T00:00:00", + "end": "2017-02-09T00:00:00" + }, + "rights": { + "statement": "my statement", + "url": "http://studio.bakajo.com" + } +} \ No newline at end of file diff --git a/tests/data/json/resource.json b/tests/data/json/resource.json index a6ac2f4..7bdf47b 100644 --- a/tests/data/json/resource.json +++ b/tests/data/json/resource.json @@ -1,7 +1,4 @@ { - "type": "CompositeResource", - "url": "http://www.hydroshare.org/resource/0fdbb27857844644bacc274882601598", - "identifier": "http://www.hydroshare.org/resource/0fdbb27857844644bacc274882601598", "title": "sadfadsgasdf", "abstract": "sadfasdfsadfa", "language": "eng", @@ -87,8 +84,6 @@ "statement": "my statement", "url": "http://studio.bakajo.com" }, - "created": "2020-07-09T19:12:21.354703+00:00", - "modified": "2020-11-13T19:40:57.276064+00:00", "awards": [ { "funding_agency_name": "agency1", diff --git a/tests/data/json/singlefile.json b/tests/data/json/singlefile.json new file mode 100644 index 0000000..3d5a55f --- /dev/null +++ b/tests/data/json/singlefile.json @@ -0,0 +1,28 @@ +{ + "title": "testa", + "subjects": [ + "key2", + "key1" + ], + "language": "eng", + "additional_metadata": { + "key2": "value2", + "key1": "value1" + }, + "spatial_coverage": { + "type": "point", + "name": "point coverage", + "east": -92.3961, + "north": 38.3556, + "units": "Decimal degrees", + "projection": "WGS 84 EPSG:4326" + }, + "period_coverage": { + "start": "2020-11-03T00:00:00", + "end": "2020-11-19T00:00:00" + }, + "rights": { + "statement": "my statement", + "url": "http://studio.bakajo.com" + } +} \ No newline at end of file diff --git a/tests/data/json/timeseries.json b/tests/data/json/timeseries.json new file mode 100644 index 0000000..819cda5 --- /dev/null +++ b/tests/data/json/timeseries.json @@ -0,0 +1,301 @@ +{ + "title": "changed from the Little Bear River, UT", + "subjects": [ + "Temperature" + ], + "language": "eng", + "additional_metadata": {}, + "spatial_coverage": { + "type": "box", + "northlimit": 41.718473, + "eastlimit": -111.799324, + "southlimit": 41.495409, + "westlimit": -111.946402, + "units": "Decimal degrees", + "projection": "WGS 84 EPSG:4326" + }, + "period_coverage": { + "start": "2008-01-01T00:00:00", + "end": "2008-01-30T23:30:00" + }, + "rights": { + "statement": "This resource is shared under the Creative Commons Attribution CC BY.", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "time_series_results": [ + { + "series_id": "182d8fa3-1ebc-11e6-ad49-f45c8999816f", + "unit": { + "type": "Temperature", + "name": "degree celsius", + "abbreviation": "degC" + }, + "status": "Unknown", + "sample_medium": "Surface Water", + "value_count": 1441, + "aggregation_statistic": "Average", + "site": { + "site_code": "USU-LBR-Mendon", + "site_name": "Little Bear River at Mendon Road near Mendon, Utah", + "elevation_m": 1345.0, + "elevation_datum": "NGVD29", + "site_type": "Stream", + "latitude": 41.718473, + "longitude": -111.946402 + }, + "variable": { + "variable_code": "USU36", + "variable_name": "Temperature", + "variable_type": "Water Quality", + "no_data_value": -9999, + "speciation": "Not Applicable" + }, + "method": { + "method_code": "28", + "method_name": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools.", + "method_type": "Instrument deployment", + "method_description": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools." + }, + "processing_level": { + "processing_level_code": "1", + "definition": "Quality controlled data", + "explanation": "Quality controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures." + } + }, + { + "series_id": "4a6f095c-1ebc-11e6-8a10-f45c8999816f", + "unit": { + "type": "Temperature", + "name": "degree celsius", + "abbreviation": "degC" + }, + "status": "Unknown", + "sample_medium": "Surface Water", + "value_count": 1441, + "aggregation_statistic": "Average", + "site": { + "site_code": "USU-LBR-SFLower", + "site_name": "South Fork Little Bear River below Davenport Creek near Avon, Utah", + "elevation_m": 1564.0, + "elevation_datum": "NGVD29", + "site_type": "Stream", + "latitude": 41.506518, + "longitude": -111.815079 + }, + "variable": { + "variable_code": "USU36", + "variable_name": "Temperature", + "variable_type": "Water Quality", + "no_data_value": -9999, + "speciation": "Not Applicable" + }, + "method": { + "method_code": "28", + "method_name": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools.", + "method_type": "Instrument deployment", + "method_description": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools." + }, + "processing_level": { + "processing_level_code": "1", + "definition": "Quality controlled data", + "explanation": "Quality controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures." + } + }, + { + "series_id": "2837b7d9-1ebc-11e6-a16e-f45c8999816f", + "unit": { + "type": "Temperature", + "name": "degree celsius", + "abbreviation": "degC" + }, + "status": "Unknown", + "sample_medium": "changed Surface Water", + "value_count": 1334, + "aggregation_statistic": "Average", + "site": { + "site_code": "USU-LBR-Paradise", + "site_name": "Little Bear River at McMurdy Hollow near Paradise, Utah", + "elevation_m": 1445.0, + "elevation_datum": "NGVD29", + "site_type": "Stream changed", + "latitude": 41.575552, + "longitude": -111.855217 + }, + "variable": { + "variable_code": "USU36", + "variable_name": "Temperature", + "variable_type": "Water Quality", + "no_data_value": -9999, + "speciation": "Not Applicable" + }, + "method": { + "method_code": "28", + "method_name": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools.", + "method_type": "Instrument deployment", + "method_description": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools." + }, + "processing_level": { + "processing_level_code": "2", + "definition": "Quality changed controlled data", + "explanation": "Quality changed controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures." + } + }, + { + "series_id": "33d63705-1ebc-11e6-b8cd-f45c8999816f", + "unit": { + "type": "Temperature", + "name": "degree celsius", + "abbreviation": "degC" + }, + "status": "Unknown", + "sample_medium": "Surface Water", + "value_count": 1441, + "aggregation_statistic": "Average", + "site": { + "site_code": "USU-LBR-Wellsville", + "site_name": "Little Bear River near Wellsville, Utah", + "elevation_m": 1365.0, + "elevation_datum": "NGVD29", + "site_type": "Stream", + "latitude": 41.643457, + "longitude": -111.917649 + }, + "variable": { + "variable_code": "USU36", + "variable_name": "Temperature", + "variable_type": "Water Quality", + "no_data_value": -9999, + "speciation": "Not Applicable" + }, + "method": { + "method_code": "28", + "method_name": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools.", + "method_type": "Instrument deployment", + "method_description": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools." + }, + "processing_level": { + "processing_level_code": "1", + "definition": "Quality controlled data", + "explanation": "Quality controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures." + } + }, + { + "series_id": "51e31687-1ebc-11e6-aa6c-f45c8999816f", + "unit": { + "type": "Temperature", + "name": "degree celsius", + "abbreviation": "degC" + }, + "status": "Unknown", + "sample_medium": "Surface Water", + "value_count": 1441, + "aggregation_statistic": "Average", + "site": { + "site_code": "USU-LBR-SFUpper", + "site_name": "South Fork Little Bear River above Davenport Creek near Avon, Utah", + "elevation_m": 1573.0, + "elevation_datum": "NGVD29", + "site_type": "Stream", + "latitude": 41.495409, + "longitude": -111.817993 + }, + "variable": { + "variable_code": "USU36", + "variable_name": "Temperature", + "variable_type": "Water Quality", + "no_data_value": -9999, + "speciation": "Not Applicable" + }, + "method": { + "method_code": "28", + "method_name": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools.", + "method_type": "Instrument deployment", + "method_description": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools." + }, + "processing_level": { + "processing_level_code": "1", + "definition": "Quality controlled data", + "explanation": "Quality controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures." + } + }, + { + "series_id": "3b9037f8-1ebc-11e6-a304-f45c8999816f", + "unit": { + "type": "Temperature", + "name": "degree celsius", + "abbreviation": "degC" + }, + "status": "Unknown", + "sample_medium": "Surface Water", + "value_count": 1441, + "aggregation_statistic": "Average", + "site": { + "site_code": "USU-LBR-Confluence", + "site_name": "Little Bear River below Confluence of South and East Forks near Avon, Utah", + "elevation_m": 1493.0, + "elevation_datum": "NGVD29", + "site_type": "Stream", + "latitude": 41.536088, + "longitude": -111.830455 + }, + "variable": { + "variable_code": "USU36", + "variable_name": "Temperature", + "variable_type": "Water Quality", + "no_data_value": -9999, + "speciation": "Not Applicable" + }, + "method": { + "method_code": "28", + "method_name": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools.", + "method_type": "Instrument deployment", + "method_description": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools." + }, + "processing_level": { + "processing_level_code": "1", + "definition": "Quality controlled data", + "explanation": "Quality controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures." + } + }, + { + "series_id": "42fbff7a-1ebc-11e6-ae3e-f45c8999816f", + "unit": { + "type": "Temperature", + "name": "degree celsius", + "abbreviation": "degC" + }, + "status": "Unknown", + "sample_medium": "Surface Water", + "value_count": 1441, + "aggregation_statistic": "Average", + "site": { + "site_code": "USU-LBR-EFLower", + "site_name": "East Fork Little Bear River at Paradise Canal Diversion near Avon, Utah", + "elevation_m": 1528.0, + "elevation_datum": "NGVD29", + "site_type": "Stream", + "latitude": 41.529212, + "longitude": -111.799324 + }, + "variable": { + "variable_code": "USU36", + "variable_name": "Temperature", + "variable_type": "Water Quality", + "no_data_value": -9999, + "speciation": "Not Applicable" + }, + "method": { + "method_code": "28", + "method_name": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools.", + "method_type": "Instrument deployment", + "method_description": "Quality Control Level 1 Data Series created from raw QC Level 0 data using ODM Tools." + }, + "processing_level": { + "processing_level_code": "1", + "definition": "Quality controlled data", + "explanation": "Quality controlled data that have passed quality assurance procedures such as routine estimation of timing and sensor calibration or visual inspection and removal of obvious errors. An example is USGS published streamflow records following parsing through USGS quality control procedures." + } + } + ], + "abstract": "This dataset contains time series of observations of water temperature in the Little Bear River, UT. Data were recorded every 30 minutes. The values were recorded using a HydroLab MS5 multi-parameter water quality sonde connected to a Campbell Scientific datalogger." +} \ No newline at end of file diff --git a/tests/test_metadata_json.py b/tests/test_metadata_json.py index 812bb20..e6dab4e 100644 --- a/tests/test_metadata_json.py +++ b/tests/test_metadata_json.py @@ -1,14 +1,35 @@ import json +import os import pytest -from hsmodels.schemas.resource import ResourceMetadata, ResourceMetadataIn +from hsmodels.schemas.aggregations import ( + FileSetMetadataIn, + GeographicFeatureMetadataIn, + GeographicRasterMetadataIn, + ModelInstanceMetadataIn, + ModelProgramMetadataIn, + MultidimensionalMetadataIn, + ReferencedTimeSeriesMetadataIn, + SingleFileMetadataIn, + TimeSeriesMetadataIn, +) +from hsmodels.schemas.resource import ResourceMetadataIn + + +def sorting(item): + if isinstance(item, dict): + return sorted((key, sorting(values)) for key, values in item.items()) + if isinstance(item, list): + return sorted(sorting(x) for x in item) + else: + return item @pytest.fixture() def res_md(): with open("data/json/resource.json", 'r') as f: - return ResourceMetadata(**json.loads(f.read())) + return ResourceMetadataIn(**json.loads(f.read())) def test_resource_additional_metadata_dictionary(res_md): @@ -17,3 +38,30 @@ def test_resource_additional_metadata_dictionary(res_md): assert res_md_in.additional_metadata == {"key1": "value1", "key2": "value2"} assert res_md_in.dict()["additional_metadata"] == {"key1": "value1", "key2": "value2"} + + +metadata_json_input = [ + (ResourceMetadataIn, 'resource.json'), + (GeographicRasterMetadataIn, 'geographicraster.json'), + (GeographicFeatureMetadataIn, 'geographicfeature.json'), + (MultidimensionalMetadataIn, 'multidimensional.json'), + (ReferencedTimeSeriesMetadataIn, 'referencedtimeseries.refts.json'), + (FileSetMetadataIn, 'fileset.json'), + (SingleFileMetadataIn, 'singlefile.json'), + (TimeSeriesMetadataIn, 'timeseries.json'), + (ModelProgramMetadataIn, 'modelprogram.json'), + (ModelInstanceMetadataIn, 'modelinstance.json'), +] + + +@pytest.mark.parametrize("metadata_json_input", metadata_json_input) +def test_metadata_json_serialization(metadata_json_input): + in_schema, metadata_file = metadata_json_input + metadata_file = os.path.join('data', 'json', metadata_file) + with open(metadata_file, 'r') as f: + json_file_str = f.read() + md = in_schema.parse_raw(json_file_str) + from_schema = sorting(json.loads(md.json())) + from_file = sorting(json.loads(json_file_str)) + for i in range(1, len(from_file)): + assert from_file[i] == from_schema[i] diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 9320400..843c32b 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -76,4 +76,11 @@ def test_dictionary_field(additional_metadata_field): for field in fields: assert 'additionalProperties' not in s["properties"][field] assert 'default' not in s["properties"][field] - assert s["properties"][field]['items'] == {'$ref': '#/definitions/AdditionalMetadata'} + assert s["properties"][field]['items'] == { + "type": "object", + "title": "Key-Value", + "description": "A key-value pair", + "default": [], + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + }