From c96c545e1d781b8ff96f2e046721d6cac51a034a Mon Sep 17 00:00:00 2001 From: Nicholas Yager Date: Sat, 1 Jul 2023 12:14:09 -0400 Subject: [PATCH 1/2] feat(grouper): Add class method for removing test resources from a dbt manifest subgraph --- dbt_meshify/storage/yaml_editors.py | 2 +- dbt_meshify/utilities/grouper.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dbt_meshify/storage/yaml_editors.py b/dbt_meshify/storage/yaml_editors.py index bedb7f5..5449762 100644 --- a/dbt_meshify/storage/yaml_editors.py +++ b/dbt_meshify/storage/yaml_editors.py @@ -100,7 +100,7 @@ def add_model_contract_to_yml( catalog_cols = model_catalog.columns or {} if model_catalog else {} catalog_cols = {k.lower(): v for k, v in catalog_cols.items()} - # add the data type to the yml entry for columns that are in yml + # add the data type to the yml entry for columns that are in yml yml_cols = [ {**yml_col, "data_type": catalog_cols[yml_col["name"]].type.lower()} for yml_col in yml_cols diff --git a/dbt_meshify/utilities/grouper.py b/dbt_meshify/utilities/grouper.py index d6957ef..08c025e 100644 --- a/dbt_meshify/utilities/grouper.py +++ b/dbt_meshify/utilities/grouper.py @@ -55,6 +55,12 @@ def classify_resource_access(cls, graph, nodes): } return resources + @classmethod + def clean_subgraph(cls, graph: networkx.DiGraph) -> networkx.DiGraph: + """Generate a subgraph that does not contain test resource types.""" + test_nodes = set(node for node in graph.nodes if node.startswith('test')) + return graph.subgraph(set(graph.nodes) - test_nodes) + def _generate_resource_group( self, name: str, @@ -96,7 +102,8 @@ def _generate_resource_group( f"of the {existing_group} group. Please remove {node} from its group and try again." ) - resources = self.classify_resource_access(self.project.graph.graph, nodes) + cleaned_subgraph = self.clean_subgraph(self.project.graph.graph) + resources = self.classify_resource_access(cleaned_subgraph, nodes) return group, resources From b0cb70e6c93ae79a3e249fad1ce8a4f431200163 Mon Sep 17 00:00:00 2001 From: Nicholas Yager Date: Sat, 1 Jul 2023 12:15:05 -0400 Subject: [PATCH 2/2] test: Add unit test for subgraph cleaning --- .../test_resource_grouper_classification.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_resource_grouper_classification.py b/tests/unit/test_resource_grouper_classification.py index db99141..7683f68 100644 --- a/tests/unit/test_resource_grouper_classification.py +++ b/tests/unit/test_resource_grouper_classification.py @@ -12,6 +12,19 @@ def example_graph(self): graph.add_edges_from([("a", "b"), ("b", "c"), ("b", "d"), ("d", "1")]) return graph + @pytest.fixture + def example_graph_with_tests(self): + graph = networkx.DiGraph() + graph.add_edges_from( + [ + ("source.a", "model.b"), + ("model.b", "test.c"), + ("model.b", "model.d"), + ("model.d", "test.1"), + ] + ) + return graph + def test_resource_grouper_boundary_classification(self, example_graph): nodes = {"a", "b", "c", "d"} resources = ResourceGrouper.classify_resource_access(example_graph, nodes) @@ -22,3 +35,7 @@ def test_resource_grouper_boundary_classification(self, example_graph): "c": AccessType.Public, "d": AccessType.Public, } + + def test_clean_graph_removes_test_nodes(self, example_graph_with_tests): + output_graph = ResourceGrouper.clean_subgraph(example_graph_with_tests) + assert set(output_graph.nodes) == {"source.a", "model.b", "model.d"}