From 6d48ead18047c5666df45d5f7e5de6a45b92528a Mon Sep 17 00:00:00 2001 From: Douglas Rioux Date: Wed, 29 May 2024 16:39:43 -0400 Subject: [PATCH 1/6] Add `schema_utils` function + test --- dcicutils/schema_utils.py | 13 +++++++++++++ test/test_schema_utils.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/dcicutils/schema_utils.py b/dcicutils/schema_utils.py index 82aa6bb12..6e80e4c8d 100644 --- a/dcicutils/schema_utils.py +++ b/dcicutils/schema_utils.py @@ -1,5 +1,6 @@ import os from typing import Any, Dict, List, Optional, Tuple + from dcicutils.misc_utils import to_camel_case @@ -29,6 +30,7 @@ class EncodedSchemaConstants: LINK_TO = "linkTo" MERGE_REF = "$merge" MIXIN_PROPERTIES = "mixinProperties" + SUBMITTER_REQUIRED = "submitterRequired" UNIQUE_KEY = "uniqueKey" @@ -203,6 +205,17 @@ def get_description(schema: Dict[str, Any]) -> str: return schema.get(SchemaConstants.DESCRIPTION, "") +def is_submitter_required(schema: Dict[str, Any]) -> bool: + """Return True if the schema is marked as required for submitters. + + This is a custom property that can be manually added to a property + to designate explicitly it as required for external submitters. + This is typically validated within the context of a oneOf, anyOf, + or allOf schema, which is tricky to pick up on automatically. + """ + return schema.get(SchemaConstants.SUBMITTER_REQUIRED, False) + + class Schema: def __init__(self, schema: dict, type: Optional[str] = None) -> None: diff --git a/test/test_schema_utils.py b/test/test_schema_utils.py index 8a928ee3b..9fe2b95d5 100644 --- a/test/test_schema_utils.py +++ b/test/test_schema_utils.py @@ -50,6 +50,7 @@ "linkTo": "foo", "enum": ENUM, "description": DESCRIPTION, + "submitterRequired": True, } ARRAY_SCHEMA = {"type": "array", "items": [STRING_SCHEMA]} OBJECT_SCHEMA = {"type": "object", "properties": {"foo": STRING_SCHEMA}} @@ -363,3 +364,15 @@ def test_get_enum(schema: Dict[str, Any], expected: List[str]) -> None: ) def test_get_description(schema: Dict[str, Any], expected: str) -> None: assert schema_utils.get_description(schema) == expected + + +@pytest.mark.parametrize( + "schema,expected", + [ + ({}, False), + (NUMBER_SCHEMA, False), + (STRING_SCHEMA, True), + ], +) +def test_is_submitter_required(schema: Dict[str, Any], expected: bool) -> None: + assert schema_utils.is_submitter_required(schema) == expected From 29c55d7d2aad2002409bb8e3926a8cbfe91004c5 Mon Sep 17 00:00:00 2001 From: Douglas Rioux Date: Thu, 30 May 2024 11:42:38 -0400 Subject: [PATCH 2/6] Add function to retrieve submission comment + test --- dcicutils/schema_utils.py | 10 ++++++++++ test/test_schema_utils.py | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/dcicutils/schema_utils.py b/dcicutils/schema_utils.py index 6e80e4c8d..394c50851 100644 --- a/dcicutils/schema_utils.py +++ b/dcicutils/schema_utils.py @@ -30,6 +30,7 @@ class EncodedSchemaConstants: LINK_TO = "linkTo" MERGE_REF = "$merge" MIXIN_PROPERTIES = "mixinProperties" + SUBMISSION_COMMENT = "submissionComment" SUBMITTER_REQUIRED = "submitterRequired" UNIQUE_KEY = "uniqueKey" @@ -216,6 +217,15 @@ def is_submitter_required(schema: Dict[str, Any]) -> bool: return schema.get(SchemaConstants.SUBMITTER_REQUIRED, False) +def get_submission_comment(schema: Dict[str, Any]) -> str: + """Return the submission comment of a schema. + + Custom property that can be manually added to a schema to provide + additional context for submitters. + """ + return schema.get(SchemaConstants.SUBMISSION_COMMENT, "") + + class Schema: def __init__(self, schema: dict, type: Optional[str] = None) -> None: diff --git a/test/test_schema_utils.py b/test/test_schema_utils.py index 9fe2b95d5..b5dea88d8 100644 --- a/test/test_schema_utils.py +++ b/test/test_schema_utils.py @@ -43,6 +43,7 @@ PATTERN = "some_regex" ENUM = ["foo", "bar"] DESCRIPTION = "foo" +COMMENT = "bar" STRING_SCHEMA = { "type": "string", "format": FORMAT, @@ -51,6 +52,7 @@ "enum": ENUM, "description": DESCRIPTION, "submitterRequired": True, + "submissionComment": COMMENT, } ARRAY_SCHEMA = {"type": "array", "items": [STRING_SCHEMA]} OBJECT_SCHEMA = {"type": "object", "properties": {"foo": STRING_SCHEMA}} @@ -376,3 +378,14 @@ def test_get_description(schema: Dict[str, Any], expected: str) -> None: ) def test_is_submitter_required(schema: Dict[str, Any], expected: bool) -> None: assert schema_utils.is_submitter_required(schema) == expected + + +@pytest.mark.parametrize( + "schema,expected", + [ + ({}, ""), + (STRING_SCHEMA, COMMENT), + ], +) +def test_get_submission_comment(schema: Dict[str, Any], expected: str) -> None: + assert schema_utils.get_submission_comment(schema) == expected From ad86ff1ccca96d41374b0258d0328ef1a24e6327 Mon Sep 17 00:00:00 2001 From: Douglas Rioux Date: Thu, 30 May 2024 12:08:31 -0400 Subject: [PATCH 3/6] Add function to pull example value for property --- dcicutils/schema_utils.py | 19 +++++++++++++++---- test/test_schema_utils.py | 13 +++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/dcicutils/schema_utils.py b/dcicutils/schema_utils.py index 394c50851..0f57f37ba 100644 --- a/dcicutils/schema_utils.py +++ b/dcicutils/schema_utils.py @@ -31,6 +31,7 @@ class EncodedSchemaConstants: MERGE_REF = "$merge" MIXIN_PROPERTIES = "mixinProperties" SUBMISSION_COMMENT = "submissionComment" + SUBMISSION_EXAMPLE = "submissionExample" SUBMITTER_REQUIRED = "submitterRequired" UNIQUE_KEY = "uniqueKey" @@ -209,16 +210,17 @@ def get_description(schema: Dict[str, Any]) -> str: def is_submitter_required(schema: Dict[str, Any]) -> bool: """Return True if the schema is marked as required for submitters. - This is a custom property that can be manually added to a property - to designate explicitly it as required for external submitters. + Specifically, required for external (i.e. non-admin) submitters. + This is typically validated within the context of a oneOf, anyOf, - or allOf schema, which is tricky to pick up on automatically. + or allOf schema on an item type which is used within the team and + by external submitters, and is tricky to pick up on automatically. """ return schema.get(SchemaConstants.SUBMITTER_REQUIRED, False) def get_submission_comment(schema: Dict[str, Any]) -> str: - """Return the submission comment of a schema. + """Return the submission comment for a property. Custom property that can be manually added to a schema to provide additional context for submitters. @@ -226,6 +228,15 @@ def get_submission_comment(schema: Dict[str, Any]) -> str: return schema.get(SchemaConstants.SUBMISSION_COMMENT, "") +def get_submission_example(schema: Dict[str, Any]) -> str: + """Return the submission example for a property. + + Custom property that can be manually added to a schema to provide + an example for submitters. + """ + return schema.get(SchemaConstants.SUBMISSION_EXAMPLE, "") + + class Schema: def __init__(self, schema: dict, type: Optional[str] = None) -> None: diff --git a/test/test_schema_utils.py b/test/test_schema_utils.py index b5dea88d8..223b2a43b 100644 --- a/test/test_schema_utils.py +++ b/test/test_schema_utils.py @@ -44,6 +44,7 @@ ENUM = ["foo", "bar"] DESCRIPTION = "foo" COMMENT = "bar" +EXAMPLE = "baz" STRING_SCHEMA = { "type": "string", "format": FORMAT, @@ -53,6 +54,7 @@ "description": DESCRIPTION, "submitterRequired": True, "submissionComment": COMMENT, + "submissionExample": EXAMPLE, } ARRAY_SCHEMA = {"type": "array", "items": [STRING_SCHEMA]} OBJECT_SCHEMA = {"type": "object", "properties": {"foo": STRING_SCHEMA}} @@ -389,3 +391,14 @@ def test_is_submitter_required(schema: Dict[str, Any], expected: bool) -> None: ) def test_get_submission_comment(schema: Dict[str, Any], expected: str) -> None: assert schema_utils.get_submission_comment(schema) == expected + + +@pytest.mark.parametrize( + "schema,expected", + [ + ({}, ""), + (STRING_SCHEMA, EXAMPLE), + ], +) +def test_get_submission_example(schema: Dict[str, Any], expected: str) -> None: + assert schema_utils.get_submission_example(schema) == expected From 231f43736b014306805f06cc0f6628294a0b66c7 Mon Sep 17 00:00:00 2001 From: Douglas Rioux Date: Thu, 30 May 2024 15:00:41 -0400 Subject: [PATCH 4/6] Add new schema parsing functions + tests --- dcicutils/schema_utils.py | 22 ++++++++++++++++++--- test/test_schema_utils.py | 40 ++++++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/dcicutils/schema_utils.py b/dcicutils/schema_utils.py index 0f57f37ba..b299b9395 100644 --- a/dcicutils/schema_utils.py +++ b/dcicutils/schema_utils.py @@ -9,6 +9,7 @@ class JsonSchemaConstants: ARRAY = "array" BOOLEAN = "boolean" DEFAULT = "default" + DEPENDENT_REQUIRED = "dependentRequired" ENUM = "enum" FORMAT = "format" INTEGER = "integer" @@ -31,8 +32,9 @@ class EncodedSchemaConstants: MERGE_REF = "$merge" MIXIN_PROPERTIES = "mixinProperties" SUBMISSION_COMMENT = "submissionComment" - SUBMISSION_EXAMPLE = "submissionExample" + SUBMISSION_EXAMPLES = "submissionExamples" SUBMITTER_REQUIRED = "submitterRequired" + SUGGESTED_ENUM = "suggested_enum" UNIQUE_KEY = "uniqueKey" @@ -228,13 +230,27 @@ def get_submission_comment(schema: Dict[str, Any]) -> str: return schema.get(SchemaConstants.SUBMISSION_COMMENT, "") -def get_submission_example(schema: Dict[str, Any]) -> str: +def get_submission_examples(schema: Dict[str, Any]) -> List[str]: """Return the submission example for a property. Custom property that can be manually added to a schema to provide an example for submitters. """ - return schema.get(SchemaConstants.SUBMISSION_EXAMPLE, "") + return schema.get(SchemaConstants.SUBMISSION_EXAMPLES, []) + + +def get_suggested_enum(schema: Dict[str, Any]) -> List[str]: + """Return the suggested enum for a property. + + Custom property that can be manually added to a schema to provide + a suggested list of values for submitters. + """ + return schema.get(SchemaConstants.SUGGESTED_ENUM, []) + + +def get_dependent_required(schema: Dict[str, Any]) -> Dict[str, List[str]]: + """Return the dependent required properties of a schema.""" + return schema.get(SchemaConstants.DEPENDENT_REQUIRED, {}) class Schema: diff --git a/test/test_schema_utils.py b/test/test_schema_utils.py index 223b2a43b..151d95651 100644 --- a/test/test_schema_utils.py +++ b/test/test_schema_utils.py @@ -31,6 +31,10 @@ }, "fun": {"type": "array", "items": {"type": "string"}}, } +DEPENDENT_REQUIRED = { + "bar": ["baz"], + "foo": ["fu"], +} SCHEMA = { "required": REQUIRED, "anyOf": ANY_OF, @@ -38,13 +42,14 @@ "identifyingProperties": IDENTIFYING_PROPERTIES, "mixinProperties": MIXIN_PROPERTIES, "properties": PROPERTIES, + "dependentRequired": DEPENDENT_REQUIRED, } FORMAT = "email" PATTERN = "some_regex" ENUM = ["foo", "bar"] DESCRIPTION = "foo" COMMENT = "bar" -EXAMPLE = "baz" +EXAMPLE = ENUM STRING_SCHEMA = { "type": "string", "format": FORMAT, @@ -54,7 +59,8 @@ "description": DESCRIPTION, "submitterRequired": True, "submissionComment": COMMENT, - "submissionExample": EXAMPLE, + "submissionExamples": EXAMPLE, + "suggested_enum": ENUM, } ARRAY_SCHEMA = {"type": "array", "items": [STRING_SCHEMA]} OBJECT_SCHEMA = {"type": "object", "properties": {"foo": STRING_SCHEMA}} @@ -396,9 +402,33 @@ def test_get_submission_comment(schema: Dict[str, Any], expected: str) -> None: @pytest.mark.parametrize( "schema,expected", [ - ({}, ""), + ({}, []), (STRING_SCHEMA, EXAMPLE), ], ) -def test_get_submission_example(schema: Dict[str, Any], expected: str) -> None: - assert schema_utils.get_submission_example(schema) == expected +def test_get_submission_examples(schema: Dict[str, Any], expected: List[str]) -> None: + assert schema_utils.get_submission_examples(schema) == expected + + +@pytest.mark.parametrize( + "schema,expected", + [ + ({}, []), + (STRING_SCHEMA, ENUM), + ], +) +def test_get_suggested_enum(schema: Dict[str, Any], expected: List[str]) -> None: + assert schema_utils.get_suggested_enum(schema) == expected + + +@pytest.mark.parametrize( + "schema,expected", + [ + ({}, {}), + (SCHEMA, DEPENDENT_REQUIRED), + ], +) +def test_get_dependent_required( + schema: Dict[str, Any], expected: Dict[str, List[str]] +) -> None: + assert schema_utils.get_dependent_required(schema) == expected From d5001484e324929b4b6e2f06e4577c15825b40bb Mon Sep 17 00:00:00 2001 From: Douglas Rioux Date: Thu, 30 May 2024 16:36:40 -0400 Subject: [PATCH 5/6] Bump to beta version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c7f564026..43c251511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.9.0" +version = "8.10.0.0b0" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 34e8279c5c61817dc9f9ce83382ead5a5daae261 Mon Sep 17 00:00:00 2001 From: Douglas Rioux Date: Wed, 12 Jun 2024 10:39:07 -0400 Subject: [PATCH 6/6] Bump minor version + changelog --- CHANGELOG.rst | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3a43116e3..dd70792ff 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,13 @@ dcicutils Change Log ---------- +8.11.0 +====== + +* Add more schema parsing functions to `schema_utils`, including for new properties for + generating submission templates + + 8.10.0 ====== diff --git a/pyproject.toml b/pyproject.toml index 4ea9c75b4..315511b2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.10.0" +version = "8.11.0" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT"