From b4c601970e93baefe0ef42da90b9e10a403c90fb Mon Sep 17 00:00:00 2001 From: joelebwf Date: Tue, 4 Jun 2019 13:07:26 -0500 Subject: [PATCH] Fixes issue #171 (#172) Fixed issue #173 --- oblib/data_model.py | 13 +++++++-- oblib/taxonomy.py | 2 ++ oblib/taxonomy_loader.py | 5 ++-- oblib/tests/test_data_model.py | 51 +++++++++++++++++++++++++++++++--- oblib/tests/test_taxonomy.py | 11 +++++++- oblib/validator.py | 4 +-- scripts/cli/cli.py | 13 +++++---- 7 files changed, 80 insertions(+), 19 deletions(-) diff --git a/oblib/data_model.py b/oblib/data_model.py index e26397a..34c0460 100644 --- a/oblib/data_model.py +++ b/oblib/data_model.py @@ -89,6 +89,7 @@ def __init__(self, ob_instance, table_name): # self.contexts stores a list of contexts that have been populated within # this table instance. self.contexts = [] + self.ts = ob_instance.ts relationships = ob_instance.relations # Use the relationships to find the names of my axes: @@ -271,10 +272,12 @@ def _is_valid_context(self, context): raise OBContextError("{} is not a valid Context instance".format(context)) for axis_name in self._axes: - if not axis_name in context.axes: + if self.ts.get_concept_details(axis_name).typed_domain_ref and not axis_name in context.axes: raise OBContextError( "Missing required {} axis for table {}".format( axis_name, self._table_name)) + elif not self.ts.get_concept_details(axis_name).typed_domain_ref and not axis_name in context.axes: + continue # Check that the value is not outside the domain, for domain-based axes: axis = self._axes[axis_name] @@ -1180,12 +1183,16 @@ def _is_valid_unit(self, concept_name, unit_id): Raises: OBUnitError explaining why the unit is not valid. """ - # TODO Refactor to move this logic into the Concept class? + # TODO Refactor to move this logic into the Concept class or place in Parser? + # TODO Examine full definition of valid units and update logic to be completely equitable unitlessTypes = ["xbrli:integerItemType", "xbrli:stringItemType", "xbrli:decimalItemType", "xbrli:booleanItemType", "xbrli:dateItemType", "num:percentItemType", - "xbrli:anyURIItemType"] + "xbrli:anyURIItemType", "dei:legalEntityIdentifierItemType"] + # NOTE: As a quick fix dei:legalEntityIdentifier Type has been added so that the sample programs + # works but this is a case of using hardcoding as opposed to correct logic. + # There is type-checking we can do for these unitless types but we'll handle # it elsewhere diff --git a/oblib/taxonomy.py b/oblib/taxonomy.py index 4c07cfd..ef4fc1e 100644 --- a/oblib/taxonomy.py +++ b/oblib/taxonomy.py @@ -108,6 +108,7 @@ def __init__(self): self.period_independent = None self.substitution_group = None self.type_name = None + self.typed_domain_ref = None self.period_type = None def __repr__(self): @@ -119,6 +120,7 @@ def __repr__(self): "," + str(self.period_independent) + \ "," + str(self.substitution_group) + \ "," + str(self.type_name) + \ + "," + str(self.typed_domain_ref) + \ "," + str(self.period_type) + \ "}" diff --git a/oblib/taxonomy_loader.py b/oblib/taxonomy_loader.py index e96cc68..27fc3cc 100644 --- a/oblib/taxonomy_loader.py +++ b/oblib/taxonomy_loader.py @@ -235,11 +235,10 @@ def startElement(self, name, attrs): element.substitution_group = taxonomy.SubstitutionGroup(item[1]) elif item[0] == "type": element.type_name = item[1] - elif item[0] == "xbrli:periodType": - # element.period_type = item[1] - element.period_type = taxonomy.PeriodType(item[1]) elif item[0] == "xbrldt:typedDomainRef": element.typed_domain_ref = item[1] + elif item[0] == "xbrli:periodType": + element.period_type = taxonomy.PeriodType(item[1]) self._elements[element.id] = element def elements(self): diff --git a/oblib/tests/test_data_model.py b/oblib/tests/test_data_model.py index e6e78cf..1dcbfcd 100644 --- a/oblib/tests/test_data_model.py +++ b/oblib/tests/test_data_model.py @@ -144,11 +144,12 @@ def test_is_valid_context_axes(self): # The context must also provide all of the axes needed to place the # fact within the right table. - # DeviceCost is on the CutSheetDetailsTable so it needs a value - # for ProductIdentifierAxis and TestConditionAxis. + # Context is required with self.assertRaises(ob.OBContextError): doc._is_valid_context("solar:DeviceCost", {}) + # DeviceCost is on the CutSheetDetailsTable so it needs a value + # for the required ProductIdentifierAxis but not for the optional TestConditionAxis. context = data_model.Context(instant = datetime.now(), ProductIdentifierAxis = "placeholder", TestConditionAxis = "solar:StandardTestConditionMember") @@ -161,8 +162,7 @@ def test_is_valid_context_axes(self): badContext = data_model.Context(instant = datetime.now(), ProductIdentifierAxis = "placeholder") - with self.assertRaises(ob.OBContextError): - doc._is_valid_context("solar:DeviceCost", badContext) + doc._is_valid_context("solar:DeviceCost", badContext) # How do we know what are valid values for ProductIdentifierAxis and # TestConditionAxis? (I think they are meant to be UUIDs.) @@ -991,3 +991,46 @@ def test_ct_issue(self): 'us-gaap:SaleLeasebackTransactionDescriptionAxis': 'us-gaap:SaleLeasebackTransactionNameDomain', 'solar:ProjectIdentifierAxis': '1'} doc.set('us-gaap:SaleLeasebackTransactionDescription', 'Sample String', **kwargs) + + def test_optional_required_axis(self): + # Tests that an optional axis can be either present or not in input data. + + # Required With Axis + doc = data_model.OBInstance("System", self.taxonomy) + kwargs = {'duration': 'forever', 'entity': 'PLUTO', + 'solar:TestConditionAxis': 'solar:StandardTestConditionMember', + 'solar:ProductIdentifierAxis': '1', + 'solar:PVSystemIdentifierAxis': '1'} + doc.set('solar:ProductName', 'Sample Product', **kwargs) + + # Required Without Axis + kwargs = {'duration': 'forever', 'entity': 'PLUTO', + 'solar:TestConditionAxis': 'solar:StandardTestConditionMember', + 'solar:ProductIdentifierAxis': '1'} + with self.assertRaises(ob.OBContextError): + doc.set('solar:ProductName', 'Sample Product', **kwargs) + + # Optional With Axis + doc = data_model.OBInstance("IECRECertificate", self.taxonomy) + kwargs = {'duration': 'forever', 'entity': 'PLUTO', + 'solar:PowerPurchaseAgreementContractAxis': '1', + 'solar:EnergyContractYearlyRateAxis': '1', + 'solar:MonthlyPeriodAxis': '1', + 'unit_name': 'USD'} + doc.set('solar:EnergyCharge', '10500.26', **kwargs) + + # Optional Without Axis + kwargs = {'duration': 'forever', 'entity': 'PLUTO', + 'solar:PowerPurchaseAgreementContractAxis': '1', + 'solar:EnergyContractYearlyRateAxis': '1', + 'unit_name': 'USD'} + doc.set('solar:EnergyCharge', '10500.26', **kwargs) + + def test_set_LEI(self): + # Tests that setting a LEI does not require a unit (a bug fix) + + doc = data_model.OBInstance("Utility", self.taxonomy) + kwargs = {'duration': 'forever', 'entity': 'PLUTO', + 'solar:UtilityIdentifierAxis': '1'} + # doc.set('solar:UtilityIdentifier', '1234567890ABCDEFGHIJ', **kwargs) + doc.set('solar:UtilityIdentifier', '12345678901234567890', **kwargs) diff --git a/oblib/tests/test_taxonomy.py b/oblib/tests/test_taxonomy.py index c63782c..8764b35 100644 --- a/oblib/tests/test_taxonomy.py +++ b/oblib/tests/test_taxonomy.py @@ -219,6 +219,13 @@ def test_concept_details(self): self.assertIsInstance(ci.type_name, string_types) self.assertIsInstance(ci.period_type, taxonomy.PeriodType) + ci = tax.semantic.get_concept_details("solar:MonthlyPeriodAxis") + self.assertIsNone(ci.typed_domain_ref) + + ci = tax.semantic.get_concept_details("solar:PVSystemIdentifierAxis") + self.assertIsInstance(ci.typed_domain_ref, string_types) + + # Values checks ci = tax.semantic.get_concept_details("solar:ACDisconnectSwitchMember") self.assertIsNotNone(ci) self.assertTrue(ci.abstract) @@ -230,7 +237,6 @@ def test_concept_details(self): self.assertEqual(ci.type_name, "nonnum:domainItemType") self.assertEqual(ci.period_type, taxonomy.PeriodType.duration) - # Values checks ci = tax.semantic.get_concept_details("solar:AdvisorInvoicesCounterparties") self.assertIsNotNone(ci) self.assertFalse(ci.abstract) @@ -253,6 +259,9 @@ def test_concept_details(self): self.assertEqual(ci.type_name, "dei:legalEntityIdentifierItemType") self.assertEqual(ci.period_type, taxonomy.PeriodType.duration) + ci = tax.semantic.get_concept_details("solar:PVSystemIdentifierAxis") + self.assertEqual(ci.typed_domain_ref, "#solar_PVSystemIdentifierDomain") + with self.assertRaises(KeyError): _ = tax.semantic.get_concept_details("solar:iamnotaconcept") diff --git a/oblib/validator.py b/oblib/validator.py index 11f3615..066f367 100644 --- a/oblib/validator.py +++ b/oblib/validator.py @@ -91,8 +91,8 @@ def validate_concept_value(self, concept_details, value): .format(concept_details.type_name, method_name)) # Check identifiers. This is based upon the name of the field containing - # the word Identifier in it. - if concept_details.id.find("Identifier") != -1: + # the word Identifier in it. Avoid UtilityIdentifier which is a LEI. + if concept_details.id != "solar:UtilityIdentifier" and concept_details.id.find("Identifier") != -1: if not identifier.validate(value): errors += ["'{}' is not valid identifier.".format(concept_details.id)] diff --git a/scripts/cli/cli.py b/scripts/cli/cli.py index e9cab05..255fb08 100644 --- a/scripts/cli/cli.py +++ b/scripts/cli/cli.py @@ -107,6 +107,7 @@ def list_concept_details(args): print("Period Independent:", c.period_independent) print("Substitution Group:", c.substitution_group.value) print("Type: ", c.type_name) + print("Typed Domain Ref: ", c.typed_domain_ref) print("Period Type: ", c.period_type.value) else: print("Not found") @@ -144,22 +145,22 @@ def list_entrypoint_concepts_details(args): "Substitution Group, Type, Period Type") for c in concepts_details: d = concepts_details[c] - print('%s, %s, %s, %s, %s, %s, %s, %s' % + print('%s, %s, %s, %s, %s, %s, %s, %s, %s' % (d.id, d.name, d.abstract, d.nillable, d.period_independent, - d.substitution_group.value, d.type_name, d.period_type.value)) + d.substitution_group.value, d.type_name, d.typed_domain_ref, d.period_type.value)) else: _, concepts_details = taxonomy.semantic.get_entrypoint_concepts(args.entrypoint, details=True) print('%85s %80s %8s %8s %10s %20s %28s %8s' % ("Id", "Name", "Abstract", "Nillable", "Period Ind", "Substitution Group", "Type", "Period Type")) - print('%0.85s %0.80s %0.8s %0.8s %0.10s %0.20s %0.28s %0.8s' % - (DASHES, DASHES, DASHES, DASHES, DASHES, DASHES, DASHES, DASHES)) + print('%0.85s %0.80s %0.8s %0.8s %0.10s %0.20s %0.28s %0.40s %0.8s' % + (DASHES, DASHES, DASHES, DASHES, DASHES, DASHES, DASHES, DASHES, DASHES)) for c in concepts_details: d = concepts_details[c] - print('%85s %80s %8s %8s %10s %20s %28s %8s' % + print('%85s %80s %8s %8s %10s %20s %28s %40s %8s' % (d.id, d.name, d.abstract, d.nillable, d.period_independent, - d.substitution_group.value, d.type_name, d.period_type.value)) + d.substitution_group.value, d.type_name, d.typed_domain_ref, d.period_type.value)) def list_concepts(args):