From 073278308abfaf7f4ad2cce8d2865074c993fa79 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 31 Jan 2024 12:58:19 -0800 Subject: [PATCH 1/4] swap back to __version__.py --- CONTRIBUTING.md | 4 +++- dbt/adapters/postgres/{__about__.py => __version__.py} | 0 pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) rename dbt/adapters/postgres/{__about__.py => __version__.py} (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9d2880e8..e13879c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,7 @@ Rather than forking `dbt-labs/dbt-postgres`, use `dbt-labs/dbt-postgres` directl [Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment) 3. Install `dbt-postgres` and development dependencies in the virtual environment ```shell - pip install -e .[dev] + pip install -e "".[dev]"" ``` When `dbt-postgres` is installed this way, any changes made to the `dbt-postgres` source code @@ -86,6 +86,8 @@ will be reflected in the virtual environment immediately. `dbt-postgres` contains [unit](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/unit) and [functional](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/functional) tests. +To install test dependencies run: `pip install -e ".[test]"` + ### Unit tests Unit tests can be run locally without setting up a database connection: diff --git a/dbt/adapters/postgres/__about__.py b/dbt/adapters/postgres/__version__.py similarity index 100% rename from dbt/adapters/postgres/__about__.py rename to dbt/adapters/postgres/__version__.py diff --git a/pyproject.toml b/pyproject.toml index c9ad0e50..31c5ac79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ include = ["dbt"] packages = ["dbt"] [tool.hatch.version] -path = "dbt/adapters/postgres/__about__.py" +path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] features = [ From 63ce9e203f59e91383d8eb5e46a6f826a2e1605d Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 31 Jan 2024 13:32:25 -0800 Subject: [PATCH 2/4] add test debug --- tests/functional/dbt_debug/test_dbt_debug.py | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/functional/dbt_debug/test_dbt_debug.py diff --git a/tests/functional/dbt_debug/test_dbt_debug.py b/tests/functional/dbt_debug/test_dbt_debug.py new file mode 100644 index 00000000..3e2a182e --- /dev/null +++ b/tests/functional/dbt_debug/test_dbt_debug.py @@ -0,0 +1,55 @@ +import pytest +import os +import re +import yaml + +from tests.functional.utils import run_dbt, run_dbt_and_capture + +MODELS__MODEL_SQL = """ +seled 1 as id +""" + + +class BaseDebug: + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": MODELS__MODEL_SQL} + + @pytest.fixture(autouse=True) + def capsys(self, capsys): + self.capsys = capsys + + def assertGotValue(self, linepat, result): + found = False + output = self.capsys.readouterr().out + for line in output.split("\n"): + if linepat.match(line): + found = True + assert result in line + if not found: + with pytest.raises(Exception) as exc: + msg = f"linepat {linepat} not found in stdout: {output}" + assert msg in str(exc.value) + + def check_project(self, splitout, msg="ERROR invalid"): + for line in splitout: + if line.strip().startswith("dbt_project.yml file"): + assert msg in line + elif line.strip().startswith("profiles.yml file"): + assert "ERROR invalid" not in line + + +class BaseDebugProfileVariable(BaseDebug): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"config-version": 2, "profile": '{{ "te" ~ "st" }}'} + + +class TestDebugPostgres(BaseDebug): + def test_ok(self, project): + result, log = run_dbt_and_capture(["debug"]) + assert "ERROR" not in log + + +class TestDebugProfileVariablePostgres(BaseDebugProfileVariable): + pass From e4b6919e56bf7be6af43c3abdc02b85c2ab1971e Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 31 Jan 2024 14:52:17 -0800 Subject: [PATCH 3/4] remove CONTRIBUTING.md updates --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e13879c2..f39e4cb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,7 @@ Rather than forking `dbt-labs/dbt-postgres`, use `dbt-labs/dbt-postgres` directl [Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment) 3. Install `dbt-postgres` and development dependencies in the virtual environment ```shell - pip install -e "".[dev]"" + pip install -e .[dev] ``` When `dbt-postgres` is installed this way, any changes made to the `dbt-postgres` source code @@ -86,7 +86,6 @@ will be reflected in the virtual environment immediately. `dbt-postgres` contains [unit](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/unit) and [functional](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/functional) tests. -To install test dependencies run: `pip install -e ".[test]"` ### Unit tests From 08daa68a1ceafe19a3b69ba42b4b8a60150577ab Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 31 Jan 2024 16:21:01 -0800 Subject: [PATCH 4/4] remove flakey docs generate test --- .../docs/test_duplicate_docs_block.py | 34 ---- tests/functional/docs/test_generate.py | 100 ---------- .../functional/docs/test_good_docs_blocks.py | 177 ------------------ tests/functional/docs/test_invalid_doc_ref.py | 46 ----- .../docs/test_missing_docs_blocks.py | 42 ----- .../docs/test_model_version_docs_blocks.py | 74 -------- tests/functional/docs/test_static.py | 50 ----- 7 files changed, 523 deletions(-) delete mode 100644 tests/functional/docs/test_duplicate_docs_block.py delete mode 100644 tests/functional/docs/test_generate.py delete mode 100644 tests/functional/docs/test_good_docs_blocks.py delete mode 100644 tests/functional/docs/test_invalid_doc_ref.py delete mode 100644 tests/functional/docs/test_missing_docs_blocks.py delete mode 100644 tests/functional/docs/test_model_version_docs_blocks.py delete mode 100644 tests/functional/docs/test_static.py diff --git a/tests/functional/docs/test_duplicate_docs_block.py b/tests/functional/docs/test_duplicate_docs_block.py deleted file mode 100644 index 393980e2..00000000 --- a/tests/functional/docs/test_duplicate_docs_block.py +++ /dev/null @@ -1,34 +0,0 @@ -from dbt.tests.util import run_dbt -from dbt_common.exceptions import CompilationError -import pytest - - -duplicate_doc_blocks_model_sql = "select 1 as id, 'joe' as first_name" - -duplicate_doc_blocks_docs_md = """{% docs my_model_doc %} - a doc string -{% enddocs %} - -{% docs my_model_doc %} - duplicate doc string -{% enddocs %}""" - -duplicate_doc_blocks_schema_yml = """version: 2 - -models: - - name: model - description: "{{ doc('my_model_doc') }}" -""" - - -class TestDuplicateDocsBlock: - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": duplicate_doc_blocks_model_sql, - "schema.yml": duplicate_doc_blocks_schema_yml, - } - - def test_duplicate_doc_ref(self, project): - with pytest.raises(CompilationError): - run_dbt(expect_pass=False) diff --git a/tests/functional/docs/test_generate.py b/tests/functional/docs/test_generate.py deleted file mode 100644 index 1da96f5b..00000000 --- a/tests/functional/docs/test_generate.py +++ /dev/null @@ -1,100 +0,0 @@ -from dbt.tests.util import get_manifest, run_dbt -import pytest - - -sample_seed = """sample_num,sample_bool -1,true -2,false -3,true -""" - -second_seed = """sample_num,sample_bool -4,true -5,false -6,true -""" - -sample_config = """ -sources: - - name: my_seed - schema: "{{ target.schema }}" - tables: - - name: sample_seed - - name: second_seed - - name: fake_seed -""" - - -class TestBaseGenerate: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": "select 1 as fun", - "alt_model.sql": "select 1 as notfun", - "sample_config.yml": sample_config, - } - - @pytest.fixture(scope="class") - def seeds(self): - return { - "sample_seed.csv": sample_seed, - "second_seed.csv": sample_seed, - } - - -class TestGenerateManifestNotCompiled(TestBaseGenerate): - def test_manifest_not_compiled(self, project): - run_dbt(["docs", "generate", "--no-compile"]) - # manifest.json is written out in parsing now, but it - # shouldn't be compiled because of the --no-compile flag - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - assert model_id in manifest.nodes - assert manifest.nodes[model_id].compiled is False - - -class TestGenerateEmptyCatalog(TestBaseGenerate): - def test_generate_empty_catalog(self, project): - catalog = run_dbt(["docs", "generate", "--empty-catalog"]) - assert catalog.nodes == {}, "nodes should be empty" - assert catalog.sources == {}, "sources should be empty" - assert catalog.errors is None, "errors should be null" - - -class TestGenerateSelectLimitsCatalog(TestBaseGenerate): - def test_select_limits_catalog(self, project): - run_dbt(["run"]) - catalog = run_dbt(["docs", "generate", "--select", "my_model"]) - assert len(catalog.nodes) == 1 - assert "model.test.my_model" in catalog.nodes - - -class TestGenerateSelectLimitsNoMatch(TestBaseGenerate): - def test_select_limits_no_match(self, project): - run_dbt(["run"]) - catalog = run_dbt(["docs", "generate", "--select", "my_missing_model"]) - assert len(catalog.nodes) == 0 - - -class TestGenerateCatalogWithSources(TestBaseGenerate): - def test_catalog_with_sources(self, project): - run_dbt(["build"]) - catalog = run_dbt(["docs", "generate"]) - - # 2 seeds + 2 models - assert len(catalog.nodes) == 4 - # 2 sources (only ones that exist) - assert len(catalog.sources) == 2 - - -class TestGenerateSelectSource(TestBaseGenerate): - def test_select_source(self, project): - run_dbt(["build"]) - catalog = run_dbt(["docs", "generate", "--select", "source:test.my_seed.sample_seed"]) - - # 2 seeds - # TODO: Filtering doesn't work for seeds - assert len(catalog.nodes) == 2 - # 2 sources - # TODO: Filtering doesn't work for sources - assert len(catalog.sources) == 2 diff --git a/tests/functional/docs/test_good_docs_blocks.py b/tests/functional/docs/test_good_docs_blocks.py deleted file mode 100644 index d1ab0f5a..00000000 --- a/tests/functional/docs/test_good_docs_blocks.py +++ /dev/null @@ -1,177 +0,0 @@ -import json -import os -from pathlib import Path - -from dbt.tests.util import run_dbt, update_config_file, write_file -import pytest - - -good_docs_blocks_model_sql = "select 1 as id, 'joe' as first_name" - -good_docs_blocks_docs_md = """{% docs my_model_doc %} -My model is just a copy of the seed -{% enddocs %} - -{% docs my_model_doc__id %} -The user ID number -{% enddocs %} - -The following doc is never used, which should be fine. -{% docs my_model_doc__first_name %} -The user's first name (should not be shown!) -{% enddocs %} - -This doc is referenced by its full name -{% docs my_model_doc__last_name %} -The user's last name -{% enddocs %} -""" - -good_doc_blocks_alt_docs_md = """{% docs my_model_doc %} -Alt text about the model -{% enddocs %} - -{% docs my_model_doc__id %} -The user ID number with alternative text -{% enddocs %} - -The following doc is never used, which should be fine. -{% docs my_model_doc__first_name %} -The user's first name - don't show this text! -{% enddocs %} - -This doc is referenced by its full name -{% docs my_model_doc__last_name %} -The user's last name in this other file -{% enddocs %} -""" - -good_docs_blocks_schema_yml = """version: 2 - -models: - - name: model - description: "{{ doc('my_model_doc') }}" - columns: - - name: id - description: "{{ doc('my_model_doc__id') }}" - - name: first_name - description: The user's first name - - name: last_name - description: "{{ doc('test', 'my_model_doc__last_name') }}" -""" - - -class TestGoodDocsBlocks: - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": good_docs_blocks_model_sql, - "schema.yml": good_docs_blocks_schema_yml, - "docs.md": good_docs_blocks_docs_md, - } - - def test_valid_doc_ref(self, project): - result = run_dbt() - assert len(result.results) == 1 - - assert os.path.exists("./target/manifest.json") - - with open("./target/manifest.json") as fp: - manifest = json.load(fp) - - model_data = manifest["nodes"]["model.test.model"] - - assert model_data["description"] == "My model is just a copy of the seed" - - assert { - "name": "id", - "description": "The user ID number", - "data_type": None, - "constraints": [], - "meta": {}, - "quote": None, - "tags": [], - } == model_data["columns"]["id"] - - assert { - "name": "first_name", - "description": "The user's first name", - "data_type": None, - "constraints": [], - "meta": {}, - "quote": None, - "tags": [], - } == model_data["columns"]["first_name"] - - assert { - "name": "last_name", - "description": "The user's last name", - "data_type": None, - "constraints": [], - "meta": {}, - "quote": None, - "tags": [], - } == model_data["columns"]["last_name"] - - assert len(model_data["columns"]) == 3 - - -class TestGoodDocsBlocksAltPath: - @pytest.fixture(scope="class") - def models(self): - return {"model.sql": good_docs_blocks_model_sql, "schema.yml": good_docs_blocks_schema_yml} - - def test_alternative_docs_path(self, project): - # self.use_default_project({"docs-paths": [self.dir("docs")]}) - docs_path = Path(project.project_root, "alt-docs") - docs_path.mkdir() - write_file(good_doc_blocks_alt_docs_md, project.project_root, "alt-docs", "docs.md") - - update_config_file( - {"docs-paths": [str(docs_path)]}, project.project_root, "dbt_project.yml" - ) - - result = run_dbt() - - assert len(result.results) == 1 - - assert os.path.exists("./target/manifest.json") - - with open("./target/manifest.json") as fp: - manifest = json.load(fp) - - model_data = manifest["nodes"]["model.test.model"] - - assert model_data["description"] == "Alt text about the model" - - assert { - "name": "id", - "description": "The user ID number with alternative text", - "data_type": None, - "constraints": [], - "meta": {}, - "quote": None, - "tags": [], - } == model_data["columns"]["id"] - - assert { - "name": "first_name", - "description": "The user's first name", - "data_type": None, - "constraints": [], - "meta": {}, - "quote": None, - "tags": [], - } == model_data["columns"]["first_name"] - - assert { - "name": "last_name", - "description": "The user's last name in this other file", - "data_type": None, - "constraints": [], - "meta": {}, - "quote": None, - "tags": [], - } == model_data["columns"]["last_name"] - - assert len(model_data["columns"]) == 3 diff --git a/tests/functional/docs/test_invalid_doc_ref.py b/tests/functional/docs/test_invalid_doc_ref.py deleted file mode 100644 index eda72d64..00000000 --- a/tests/functional/docs/test_invalid_doc_ref.py +++ /dev/null @@ -1,46 +0,0 @@ -from dbt.tests.util import run_dbt -from dbt_common.exceptions import CompilationError -import pytest - - -invalid_doc_ref_model_sql = "select 1 as id, 'joe' as first_name" - -invalid_doc_ref_docs_md = """{% docs my_model_doc %} -My model is just a copy of the seed -{% enddocs %} - -{% docs my_model_doc__id %} -The user ID number -{% enddocs %} - -The following doc is never used, which should be fine. -{% docs my_model_doc__first_name %} -The user's first name -{% enddocs %}""" - -invalid_doc_ref_schema_yml = """version: 2 - -models: - - name: model - description: "{{ doc('my_model_doc') }}" - columns: - - name: id - description: "{{ doc('my_model_doc__id') }}" - - name: first_name - description: "{{ doc('foo.bar.my_model_doc__id') }}" -""" - - -class TestInvalidDocRef: - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": invalid_doc_ref_model_sql, - "docs.md": invalid_doc_ref_docs_md, - "schema.yml": invalid_doc_ref_schema_yml, - } - - def test_invalid_doc_ref(self, project): - # The run should fail since we could not find the docs reference. - with pytest.raises(CompilationError): - run_dbt(expect_pass=False) diff --git a/tests/functional/docs/test_missing_docs_blocks.py b/tests/functional/docs/test_missing_docs_blocks.py deleted file mode 100644 index 1aa23168..00000000 --- a/tests/functional/docs/test_missing_docs_blocks.py +++ /dev/null @@ -1,42 +0,0 @@ -from dbt.tests.util import run_dbt -from dbt_common.exceptions import CompilationError -import pytest - - -missing_docs_blocks_model_sql = "select 1 as id, 'joe' as first_name" - -missing_docs_blocks_docs_md = """{% docs my_model_doc %} -My model is just a copy of the seed -{% enddocs %} - -{% docs my_model_doc__id %} -The user ID number -{% enddocs %}""" - -missing_docs_blocks_schema_yml = """version: 2 - -models: - - name: model - description: "{{ doc('my_model_doc') }}" - columns: - - name: id - description: "{{ doc('my_model_doc__id') }}" - - name: first_name - # invalid reference - description: "{{ doc('my_model_doc__first_name') }}" -""" - - -class TestMissingDocsBlocks: - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": missing_docs_blocks_model_sql, - "schema.yml": missing_docs_blocks_schema_yml, - "docs.md": missing_docs_blocks_docs_md, - } - - def test_missing_doc_ref(self, project): - # The run should fail since we could not find the docs reference. - with pytest.raises(CompilationError): - run_dbt() diff --git a/tests/functional/docs/test_model_version_docs_blocks.py b/tests/functional/docs/test_model_version_docs_blocks.py deleted file mode 100644 index be610081..00000000 --- a/tests/functional/docs/test_model_version_docs_blocks.py +++ /dev/null @@ -1,74 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -model_1 = """ -select 1 as id, 'joe' as first_name -""" - -model_versioned = """ -select 1 as id, 'joe' as first_name -""" - -docs_md = """ -{% docs model_description %} -unversioned model -{% enddocs %} - -{% docs column_id_doc %} -column id for some thing -{% enddocs %} - -{% docs versioned_model_description %} -versioned model -{% enddocs %} - -""" - -schema_yml = """ -models: - - name: model_1 - description: '{{ doc("model_description") }}' - columns: - - name: id - description: '{{ doc("column_id_doc") }}' - - - name: model_versioned - description: '{{ doc("versioned_model_description") }}' - latest_version: 1 - versions: - - v: 1 - config: - alias: my_alias - columns: - - name: id - description: '{{ doc("column_id_doc") }}' - - name: first_name - description: 'plain text' - - v: 2 - columns: - - name: other_id -""" - - -class TestVersionedModelDocsBlock: - @pytest.fixture(scope="class") - def models(self): - return { - "model_1.sql": model_1, - "model_versioned.sql": model_versioned, - "schema.yml": schema_yml, - "docs.md": docs_md, - } - - def test_versioned_doc_ref(self, project): - manifest = run_dbt(["parse"]) - model_1 = manifest.nodes["model.test.model_1"] - model_v1 = manifest.nodes["model.test.model_versioned.v1"] - - assert model_1.description == "unversioned model" - assert model_v1.description == "versioned model" - - assert model_1.columns["id"].description == "column id for some thing" - assert model_v1.columns["id"].description == "column id for some thing" - assert model_v1.columns["first_name"].description == "plain text" diff --git a/tests/functional/docs/test_static.py b/tests/functional/docs/test_static.py deleted file mode 100644 index 589668bd..00000000 --- a/tests/functional/docs/test_static.py +++ /dev/null @@ -1,50 +0,0 @@ -import os - -from dbt.task.docs import DOCS_INDEX_FILE_PATH -from dbt.tests.util import run_dbt -from dbt_common.clients.system import load_file_contents -import pytest - - -class TestStaticGenerate: - @pytest.fixture(scope="class") - def models(self): - return {"my_model.sql": "select 1 as fun"} - - def test_static_generated(self, project): - run_dbt(["docs", "generate", "--static"]) - - source_index_html = load_file_contents(DOCS_INDEX_FILE_PATH) - - target_index_html = load_file_contents( - os.path.join(project.project_root, "target", "index.html") - ) - - # Validate index.html was copied correctly - assert len(target_index_html) == len(source_index_html) - assert hash(target_index_html) == hash(source_index_html) - - manifest_data = load_file_contents( - os.path.join(project.project_root, "target", "manifest.json") - ) - - catalog_data = load_file_contents( - os.path.join(project.project_root, "target", "catalog.json") - ) - - static_index_html = load_file_contents( - os.path.join(project.project_root, "target", "static_index.html") - ) - - # Calculate expected static_index.html - expected_static_index_html = source_index_html - expected_static_index_html = expected_static_index_html.replace( - '"MANIFEST.JSON INLINE DATA"', manifest_data - ) - expected_static_index_html = expected_static_index_html.replace( - '"CATALOG.JSON INLINE DATA"', catalog_data - ) - - # Validate static_index.html was generated correctly - assert len(expected_static_index_html) == len(static_index_html) - assert hash(expected_static_index_html) == hash(static_index_html)