From 7643530d3dd30ae80334ddfca672ec2fffb6f87c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Oct 2024 17:26:26 -0400 Subject: [PATCH 1/4] drop support for python 3.8 --- .github/workflows/integration-tests.yml | 4 ++-- .github/workflows/unit-tests.yml | 2 +- .pre-commit-config.yaml | 1 - pyproject.toml | 6 ++---- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d7367190..88e601f8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -50,7 +50,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] services: postgres: @@ -107,7 +107,7 @@ jobs: fail-fast: false matrix: platform: [ubuntu-22.04, macos-12] - python-version: ["3.8", "3.11"] + python-version: ["3.9", "3.11"] steps: - name: "Check out repository" uses: actions/checkout@v4 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 91d8c578..f6ad0675 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - name: Check out repository diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c869fe86..5c8f378c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,6 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 diff --git a/pyproject.toml b/pyproject.toml index 2f97d5c5..abf377cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ name = "dbt-postgres" description = "The set of adapter protocols and base functionality that supports integration with dbt-core" readme = "README.md" keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "postgres"] -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] @@ -17,7 +17,6 @@ classifiers = [ "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -56,8 +55,7 @@ path = "dbt/adapters/postgres/__version__.py" dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - 'pre-commit==3.7.0;python_version>="3.9"', - 'pre-commit==3.5.0;python_version=="3.8"', + "pre-commit==3.7.0", ] [tool.hatch.envs.default.scripts] dev = "pre-commit install" From 249c0c53f1844295a91a64cd0142be3219229ffa Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Oct 2024 17:28:52 -0400 Subject: [PATCH 2/4] drop support for python 3.8 --- .changes/unreleased/Breaking Changes-20241017-172848.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Breaking Changes-20241017-172848.yaml diff --git a/.changes/unreleased/Breaking Changes-20241017-172848.yaml b/.changes/unreleased/Breaking Changes-20241017-172848.yaml new file mode 100644 index 00000000..b1f6cb93 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241017-172848.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-17T17:28:48.169961-04:00 +custom: + Author: mikealfare + Issue: "161" From 166ee053b6e91454f206977b7a0aee9736d84a4a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Oct 2024 17:42:32 -0400 Subject: [PATCH 3/4] remove duplicate tests that should exist in dbt-core --- tests/functional/deprecations/fixtures.py | 101 -- .../deprecations/model_deprecations.py | 106 -- .../deprecations/test_config_deprecations.py | 148 -- .../deprecations/test_deprecations.py | 148 -- tests/functional/schema_tests/data/seed.sql | 117 -- .../schema_tests/data/seed_failure.sql | 116 -- tests/functional/schema_tests/fixtures.py | 1275 ----------------- .../schema_tests/test_schema_v2_tests.py | 1110 -------------- 8 files changed, 3121 deletions(-) delete mode 100644 tests/functional/deprecations/fixtures.py delete mode 100644 tests/functional/deprecations/model_deprecations.py delete mode 100644 tests/functional/deprecations/test_config_deprecations.py delete mode 100644 tests/functional/deprecations/test_deprecations.py delete mode 100644 tests/functional/schema_tests/data/seed.sql delete mode 100644 tests/functional/schema_tests/data/seed_failure.sql delete mode 100644 tests/functional/schema_tests/fixtures.py delete mode 100644 tests/functional/schema_tests/test_schema_v2_tests.py diff --git a/tests/functional/deprecations/fixtures.py b/tests/functional/deprecations/fixtures.py deleted file mode 100644 index 0028f206..00000000 --- a/tests/functional/deprecations/fixtures.py +++ /dev/null @@ -1,101 +0,0 @@ -models__already_exists_sql = """ -select 1 as id - -{% if adapter.already_exists(this.schema, this.identifier) and not should_full_refresh() %} - where id > (select max(id) from {{this}}) -{% endif %} -""" - -models_trivial__model_sql = """ -select 1 as id -""" - - -bad_name_yaml = """ -version: 2 - -exposures: - - name: simple exposure spaced!! - type: dashboard - depends_on: - - ref('model') - owner: - email: something@example.com -""" - -# deprecated test config fixtures -data_tests_yaml = """ -models: - - name: model - columns: - - name: id - data_tests: - - not_null -""" - -test_type_mixed_yaml = """ -models: - - name: model - columns: - - name: id - data_tests: - - not_null - tests: - - unique -""" - -old_tests_yaml = """ -models: - - name: model - columns: - - name: id - tests: - - not_null -""" - -sources_old_tests_yaml = """ -sources: - - name: seed_source - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - tests: - - unique -""" - -seed_csv = """id,name -1,Mary -2,Sam -3,John -""" - - -local_dependency__dbt_project_yml = """ - -name: 'local_dep' -version: '1.0' - -seeds: - quote_columns: False - -""" - -local_dependency__schema_yml = """ -sources: - - name: seed_source - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - tests: - - unique -""" - -local_dependency__seed_csv = """id,name -1,Mary -2,Sam -3,John -""" diff --git a/tests/functional/deprecations/model_deprecations.py b/tests/functional/deprecations/model_deprecations.py deleted file mode 100644 index c762e7a6..00000000 --- a/tests/functional/deprecations/model_deprecations.py +++ /dev/null @@ -1,106 +0,0 @@ -from dbt.cli.main import dbtRunner -from dbt.tests.util import run_dbt -from dbt_common.exceptions import EventCompilationError -import pytest - - -deprecated_model__yml = """ -version: 2 - -models: - - name: my_model - description: deprecated - deprecation_date: 1999-01-01 -""" - -deprecating_model__yml = """ -version: 2 - -models: - - name: my_model - description: deprecating in the future - deprecation_date: 2999-01-01 -""" - -model__sql = """ -select 1 as Id -""" - -dependant_model__sql = """ -select * from {{ ref("my_model") }} -""" - - -class TestModelDeprecationWarning: - @pytest.fixture(scope="class") - def models(self): - return {"my_model.sql": model__sql, "my_schema.yml": deprecated_model__yml} - - def test_deprecation_warning(self, project): - events = [] - dbtRunner(callbacks=[events.append]).invoke(["parse"]) - matches = list([e for e in events if e.info.name == "DeprecatedModel"]) - assert len(matches) == 1 - assert matches[0].read_data.model_name == "my_model" - - def test_deprecation_warning_error(self, project): - with pytest.raises(EventCompilationError): - run_dbt(["--warn-error", "parse"]) - - def test_deprecation_warning_error_options(self, project): - with pytest.raises(EventCompilationError): - run_dbt(["--warn-error-options", '{"include": ["DeprecatedModel"]}', "parse"]) - - -class TestUpcomingReferenceDeprecatingWarning: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": model__sql, - "my_dependant_model.sql": dependant_model__sql, - "my_schema.yml": deprecating_model__yml, - } - - def test_deprecation_warning(self, project): - events = [] - dbtRunner(callbacks=[events.append]).invoke(["parse"]) - matches = list([e for e in events if e.info.name == "UpcomingReferenceDeprecation"]) - assert len(matches) == 1 - assert matches[0].read_data.model_name == "my_dependant_model" - assert matches[0].read_data.ref_model_name == "my_model" - - def test_deprecation_warning_error(self, project): - with pytest.raises(EventCompilationError): - run_dbt(["--warn-error", "parse"]) - - def test_deprecation_warning_error_options(self, project): - with pytest.raises(EventCompilationError): - run_dbt( - ["--warn-error-options", '{"include": ["UpcomingReferenceDeprecation"]}', "parse"] - ) - - -class TestDeprecatedReferenceWarning: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": model__sql, - "my_dependant_model.sql": dependant_model__sql, - "my_schema.yml": deprecated_model__yml, - } - - def test_deprecation_warning(self, project): - events = [] - dbtRunner(callbacks=[events.append]).invoke(["parse"]) - matches = list([e for e in events if e.info.name == "DeprecatedReference"]) - assert len(matches) == 1 - assert matches[0].read_data.model_name == "my_dependant_model" - assert matches[0].read_data.ref_model_name == "my_model" - - def test_deprecation_warning_error(self, project): - with pytest.raises(EventCompilationError): - run_dbt(["--warn-error", "parse"]) - - def test_deprecation_warning_error_options(self, project): - with pytest.raises(EventCompilationError): - run_dbt(["--warn-error-options", '{"include": ["DeprecatedReference"]}', "parse"]) diff --git a/tests/functional/deprecations/test_config_deprecations.py b/tests/functional/deprecations/test_config_deprecations.py deleted file mode 100644 index f8623c1a..00000000 --- a/tests/functional/deprecations/test_config_deprecations.py +++ /dev/null @@ -1,148 +0,0 @@ -from dbt.deprecations import active_deprecations, reset_deprecations -from dbt.exceptions import ProjectContractError, YamlParseDictError -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import run_dbt, update_config_file -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.deprecations import fixtures - - -# test deprecation messages -class TestTestsConfigDeprecation: - @pytest.fixture(scope="class") - def models(self): - return {"model.sql": fixtures.models_trivial__model_sql} - - @pytest.fixture(scope="class") - def project_config_update(self, unique_schema): - return {"tests": {"enabled": "true"}} - - def test_tests_config(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["parse"]) - expected = {"project-test-config"} - assert expected == active_deprecations - - def test_tests_config_fail(self, project): - reset_deprecations() - assert active_deprecations == set() - with pytest.raises(CompilationError) as exc: - run_dbt(["--warn-error", "--no-partial-parse", "parse"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "The `tests` config has been renamed to `data_tests`" - assert expected_msg in exc_str - - -class TestSchemaTestDeprecation: - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": fixtures.models_trivial__model_sql, - "schema.yml": fixtures.old_tests_yaml, - } - - def test_tests_config(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["parse"]) - expected = {"project-test-config"} - assert expected == active_deprecations - - def test_schema_tests_fail(self, project): - reset_deprecations() - assert active_deprecations == set() - with pytest.raises(CompilationError) as exc: - run_dbt(["--warn-error", "--no-partial-parse", "parse"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "The `tests` config has been renamed to `data_tests`" - assert expected_msg in exc_str - - -class TestSourceSchemaTestDeprecation: - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": fixtures.sources_old_tests_yaml} - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": fixtures.seed_csv} - - def test_source_tests_config(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["seed"]) - run_dbt(["parse"]) - expected = {"project-test-config"} - assert expected == active_deprecations - - def test_schema_tests(self, project): - run_dbt(["seed"]) - results = run_dbt(["test"]) - assert len(results) == 1 - - -# test for failure with test and data_tests in the same file -class TestBothSchemaTestDeprecation: - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": fixtures.models_trivial__model_sql, - "schema.yml": fixtures.test_type_mixed_yaml, - } - - def test_schema(self, project): - expected_msg = "Invalid test config: cannot have both 'tests' and 'data_tests' defined" - with pytest.raises(YamlParseDictError) as excinfo: - run_dbt(["parse"]) - assert expected_msg in str(excinfo.value) - - -# test for failure with test and data_tests in the same dbt_project.yml -class TestBothProjectTestDeprecation: - @pytest.fixture(scope="class") - def models(self): - return {"model.sql": fixtures.models_trivial__model_sql} - - def test_tests_config(self, project): - config_patch = {"tests": {"+enabled": "true"}, "data_tests": {"+tags": "super"}} - update_config_file(config_patch, project.project_root, "dbt_project.yml") - - expected_msg = "Invalid project config: cannot have both 'tests' and 'data_tests' defined" - with pytest.raises(ProjectContractError) as excinfo: - run_dbt(["parse"]) - assert expected_msg in str(excinfo.value) - - -# test a local dependency can have tests while the rest of the project uses data_tests -class TestTestConfigInDependency: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - local_dependency_files = { - "dbt_project.yml": fixtures.local_dependency__dbt_project_yml, - "models": { - "schema.yml": fixtures.local_dependency__schema_yml, - }, - "seeds": {"seed.csv": fixtures.local_dependency__seed_csv}, - } - write_project_files(project_root, "local_dependency", local_dependency_files) - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": "local_dependency"}]} - - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": fixtures.models_trivial__model_sql, - "schema.yml": fixtures.data_tests_yaml, - } - - def test_test_dep(self, project): - run_dbt(["deps"]) - run_dbt(["seed"]) - run_dbt(["run"]) - results = run_dbt(["test"]) - # 1 data_test in the dep and 1 in the project - assert len(results) == 2 diff --git a/tests/functional/deprecations/test_deprecations.py b/tests/functional/deprecations/test_deprecations.py deleted file mode 100644 index 1f4a31c2..00000000 --- a/tests/functional/deprecations/test_deprecations.py +++ /dev/null @@ -1,148 +0,0 @@ -from dbt.deprecations import active_deprecations, reset_deprecations -from dbt.tests.util import run_dbt, write_file -from dbt_common.exceptions import CompilationError -import pytest -import yaml - -from tests.functional.deprecations import fixtures - - -class TestConfigPathDeprecation: - @pytest.fixture(scope="class") - def models(self): - return {"already_exists.sql": fixtures.models_trivial__model_sql} - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "data-paths": ["data"], - "log-path": "customlogs", - "target-path": "customtarget", - } - - def test_data_path(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["debug"]) - expected = { - "project-config-data-paths", - "project-config-log-path", - "project-config-target-path", - } - assert expected == active_deprecations - - def test_data_path_fail(self, project): - reset_deprecations() - assert active_deprecations == set() - with pytest.raises(CompilationError) as exc: - run_dbt(["--warn-error", "debug"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "The `data-paths` config has been renamed" - assert expected_msg in exc_str - - -class TestPackageInstallPathDeprecation: - @pytest.fixture(scope="class") - def models_trivial(self): - return {"model.sql": fixtures.models_trivial__model_sql} - - @pytest.fixture(scope="class") - def project_config_update(self): - return {"config-version": 2, "clean-targets": ["dbt_modules"]} - - def test_package_path(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["clean"]) - expected = {"install-packages-path"} - assert expected == active_deprecations - - def test_package_path_not_set(self, project): - reset_deprecations() - assert active_deprecations == set() - with pytest.raises(CompilationError) as exc: - run_dbt(["--warn-error", "clean"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "path has changed from `dbt_modules` to `dbt_packages`." - assert expected_msg in exc_str - - -class TestPackageRedirectDeprecation: - @pytest.fixture(scope="class") - def models(self): - return {"already_exists.sql": fixtures.models_trivial__model_sql} - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"package": "fishtown-analytics/dbt_utils", "version": "0.7.0"}]} - - def test_package_redirect(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["deps"]) - expected = {"package-redirect"} - assert expected == active_deprecations - - # if this test comes before test_package_redirect it will raise an exception as expected - def test_package_redirect_fail(self, project): - reset_deprecations() - assert active_deprecations == set() - with pytest.raises(CompilationError) as exc: - run_dbt(["--warn-error", "deps"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "The `fishtown-analytics/dbt_utils` package is deprecated in favor of `dbt-labs/dbt_utils`" - assert expected_msg in exc_str - - -class TestExposureNameDeprecation: - @pytest.fixture(scope="class") - def models(self): - return { - "model.sql": fixtures.models_trivial__model_sql, - "bad_name.yml": fixtures.bad_name_yaml, - } - - def test_exposure_name(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["parse"]) - expected = {"exposure-name"} - assert expected == active_deprecations - - def test_exposure_name_fail(self, project): - reset_deprecations() - assert active_deprecations == set() - with pytest.raises(CompilationError) as exc: - run_dbt(["--warn-error", "--no-partial-parse", "parse"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "Starting in v1.3, the 'name' of an exposure should contain only letters, numbers, and underscores." - assert expected_msg in exc_str - - -class TestPrjectFlagsMovedDeprecation: - @pytest.fixture(scope="class") - def profiles_config_update(self): - return { - "config": {"send_anonymous_usage_stats": False}, - } - - @pytest.fixture(scope="class") - def dbt_project_yml(self, project_root, project_config_update): - project_config = { - "name": "test", - "profile": "test", - } - write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml") - return project_config - - @pytest.fixture(scope="class") - def models(self): - return {"my_model.sql": "select 1 as fun"} - - def test_profile_config_deprecation(self, project): - reset_deprecations() - assert active_deprecations == set() - run_dbt(["parse"]) - expected = {"project-flags-moved"} - assert expected == active_deprecations diff --git a/tests/functional/schema_tests/data/seed.sql b/tests/functional/schema_tests/data/seed.sql deleted file mode 100644 index 8f180150..00000000 --- a/tests/functional/schema_tests/data/seed.sql +++ /dev/null @@ -1,117 +0,0 @@ -create table {schema}.seed ( - favorite_color VARCHAR(10), - id INTEGER, - first_name VARCHAR(11), - email VARCHAR(31), - - net_worth NUMERIC(12, 2) DEFAULT '100.00', - fav_number NUMERIC DEFAULT '3.14159265', - - ip_address VARCHAR(15), - updated_at TIMESTAMP WITHOUT TIME ZONE -); - - -INSERT INTO {schema}.seed - ("favorite_color", "id","first_name","email","ip_address","updated_at") -VALUES - ('blue', 1,'Larry',null,'69.135.206.194','2008-09-12 19:08:31'), - ('blue', 2,'Larry',null,'64.210.133.162','1978-05-09 04:15:14'), - ('blue', 3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), - ('blue', 4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), - ('blue', 5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), - ('blue', 6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), - ('blue', 7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), - ('blue', 8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), - ('blue', 9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), - ('blue', 10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), - ('blue', 11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), - ('blue', 12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), - ('blue', 13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), - ('blue', 14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), - ('blue', 15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), - ('blue', 16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), - ('blue', 17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), - ('blue', 18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), - ('blue', 19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), - ('blue', 20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), - ('blue', 21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), - ('blue', 22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), - ('blue', 23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), - ('blue', 24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), - ('blue', 25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), - ('blue', 26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), - ('blue', 27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), - ('blue', 28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), - ('blue', 29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), - ('blue', 30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), - ('blue', 31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), - ('blue', 32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), - ('blue', 33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), - ('blue', 34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), - ('blue', 35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), - ('blue', 36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), - ('blue', 37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), - ('blue', 38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), - ('blue', 39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), - ('blue', 40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), - ('blue', 41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), - ('blue', 42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), - ('blue', 43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), - ('blue', 44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), - ('blue', 45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), - ('blue', 46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), - ('blue', 47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), - ('blue', 48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), - ('blue', 49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), - ('blue', 50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), - ('green', 51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), - ('green', 52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), - ('green', 53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), - ('green', 54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), - ('green', 55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), - ('green', 56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), - ('green', 57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), - ('green', 58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), - ('green', 59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), - ('green', 60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), - ('green', 61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), - ('green', 62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), - ('green', 63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), - ('green', 64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), - ('green', 65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), - ('green', 66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), - ('green', 67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), - ('green', 68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), - ('green', 69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), - ('green', 70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), - ('green', 71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), - ('green', 72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), - ('green', 73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), - ('green', 74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), - ('green', 75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), - ('green', 76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), - ('green', 77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), - ('green', 78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), - ('green', 79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), - ('green', 80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), - ('green', 81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), - ('green', 82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), - ('green', 83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), - ('green', 84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), - ('green', 85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), - ('green', 86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), - ('green', 87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), - ('green', 88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), - ('green', 89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), - ('green', 90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), - ('green', 91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), - ('green', 92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), - ('green', 93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), - ('green', 94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), - ('green', 95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), - ('green', 96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), - ('green', 97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), - ('green', 98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), - ('green', 99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), - ('green', 100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'); diff --git a/tests/functional/schema_tests/data/seed_failure.sql b/tests/functional/schema_tests/data/seed_failure.sql deleted file mode 100644 index f68c4591..00000000 --- a/tests/functional/schema_tests/data/seed_failure.sql +++ /dev/null @@ -1,116 +0,0 @@ -create table {schema}.seed_failure ( - favorite_color VARCHAR(10), - id INTEGER, - first_name VARCHAR(11), - email VARCHAR(31), - ip_address VARCHAR(15), - updated_at TIMESTAMP WITHOUT TIME ZONE -); - - -INSERT INTO {schema}.seed_failure - ("favorite_color", "id","first_name","email","ip_address","updated_at") -VALUES - -- unaccepted 'red' favorite_color - ('red', 1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), - -- dupicate unique field (id=1) - ('blue', 1,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), - -- null not_null field (id) - ('blue', null,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), - ('blue', 4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), - ('blue', 5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), - ('blue', 6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), - ('blue', 7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), - ('blue', 8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), - ('blue', 9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), - ('blue', 10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), - ('blue', 11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), - ('blue', 12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), - ('blue', 13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), - ('blue', 14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), - ('blue', 15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), - ('blue', 16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), - ('blue', 17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), - ('blue', 18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), - ('blue', 19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), - ('blue', 20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), - ('blue', 21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), - ('blue', 22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), - ('blue', 23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), - ('blue', 24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), - ('blue', 25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), - ('blue', 26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), - ('blue', 27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), - ('blue', 28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), - ('blue', 29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), - ('blue', 30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), - ('blue', 31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), - ('blue', 32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), - ('blue', 33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), - ('blue', 34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), - ('blue', 35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), - ('blue', 36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), - ('blue', 37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), - ('blue', 38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), - ('blue', 39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), - ('blue', 40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), - ('blue', 41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), - ('blue', 42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), - ('blue', 43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), - ('blue', 44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), - ('blue', 45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), - ('blue', 46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), - ('blue', 47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), - ('blue', 48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), - ('blue', 49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), - ('blue', 50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), - ('green', 51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), - ('green', 52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), - ('green', 53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), - ('green', 54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), - ('green', 55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), - ('green', 56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), - ('green', 57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), - ('green', 58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), - ('green', 59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), - ('green', 60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), - ('green', 61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), - ('green', 62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), - ('green', 63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), - ('green', 64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), - ('green', 65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), - ('green', 66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), - ('green', 67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), - ('green', 68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), - ('green', 69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), - ('green', 70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), - ('green', 71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), - ('green', 72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), - ('green', 73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), - ('green', 74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), - ('green', 75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), - ('green', 76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), - ('green', 77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), - ('green', 78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), - ('green', 79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), - ('green', 80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), - ('green', 81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), - ('green', 82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), - ('green', 83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), - ('green', 84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), - ('green', 85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), - ('green', 86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), - ('green', 87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), - ('green', 88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), - ('green', 89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), - ('green', 90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), - ('green', 91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), - ('green', 92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), - ('green', 93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), - ('green', 94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), - ('green', 95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), - ('green', 96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), - ('green', 97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), - ('green', 98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), - ('green', 99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), - ('green', 100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'); diff --git a/tests/functional/schema_tests/fixtures.py b/tests/functional/schema_tests/fixtures.py deleted file mode 100644 index 51ae067b..00000000 --- a/tests/functional/schema_tests/fixtures.py +++ /dev/null @@ -1,1275 +0,0 @@ -wrong_specification_block__schema_yml = """ -version: 2 -models: - - name: some_seed - description: "This is my seed under a model" -""" - -test_context_where_subq_models__schema_yml = """ -version: 2 - -models: - - name: model_a - data_tests: - - self_referential - -""" - -test_context_where_subq_models__model_a_sql = """ -select 1 as fun - -""" - -test_utils__dbt_project_yml = """ -name: 'test_utils' -version: '1.0' -config-version: 2 - -profile: 'default' - -macro-paths: ["macros"] - - -""" - -test_utils__macros__current_timestamp_sql = """ -{% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp', 'test_utils')()) }} -{%- endmacro %} - -{% macro default__current_timestamp() -%} - now() -{%- endmacro %} - -""" - -test_utils__macros__custom_test_sql = """ -{% macro test_dispatch(model) -%} - {{ return(adapter.dispatch('test_dispatch', macro_namespace = 'test_utils')()) }} -{%- endmacro %} - -{% macro default__test_dispatch(model) %} - select {{ adapter.dispatch('current_timestamp', macro_namespace = 'test_utils')() }} -{% endmacro %} - -""" - -local_dependency__dbt_project_yml = """ -name: 'local_dep' -version: '1.0' -config-version: 2 - -profile: 'default' - -macro-paths: ["macros"] - -""" - -local_dependency__macros__equality_sql = """ -{#-- taken from dbt-utils --#} -{% test equality(model, compare_model, compare_columns=None) %} - {{ return(adapter.dispatch('test_equality')(model, compare_model, compare_columns)) }} -{% endtest %} - -{% macro default__test_equality(model, compare_model, compare_columns=None) %} - -{% set set_diff %} - count(*) + abs( - sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - - sum(case when which_diff = 'b_minus_a' then 1 else 0 end) - ) -{% endset %} - -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = set_diff) }} - -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} --- setup -{%- do dbt_utils._is_relation(model, 'test_equality') -%} -{#- -If the compare_cols arg is provided, we can run this test without querying the -information schema — this allows the model to be an ephemeral model --#} - -{%- if not compare_columns -%} - {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} -{%- endif -%} - -{% set compare_cols_csv = compare_columns | join(', ') %} - -with a as ( - select * from {{ model }} -), -b as ( - select * from {{ compare_model }} -), -a_minus_b as ( - select {{compare_cols_csv}} from a - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from b -), -b_minus_a as ( - select {{compare_cols_csv}} from b - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from a -), - -unioned as ( - - select 'a_minus_b' as which_diff, * from a_minus_b - union all - select 'b_minus_a' as which_diff, * from b_minus_a - -) - -select * from unioned - -{% endmacro %} - -""" - -case_sensitive_models__schema_yml = """ -version: 2 - -models: - - name: lowercase - columns: - - name: id - quote: true - data_tests: - - unique - - name: uppercase - columns: - - name: id - quote: true - data_tests: - - unique - -""" - -case_sensitive_models__uppercase_SQL = """ -select 1 as id - -""" - -case_sensitive_models__lowercase_sql = """ -select 1 as id - -""" - -test_context_macros__my_test_sql = """ -{% macro test_call_pkg_macro(model) %} - select {{ adapter.dispatch('current_timestamp', macro_namespace = 'local_utils')() }} -{% endmacro %} - -""" - -test_context_macros__test_my_datediff_sql = """ -{% macro test_my_datediff(model) %} - select {{ local_utils.datediff() }} -{% endmacro %} - -""" - -test_context_macros__custom_schema_tests_sql = """ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "WARN") }} - - select * from {{ model }} - -{% endtest %} - -""" - -test_context_models_namespaced__schema_yml = """ - -version: 2 - -models: - - name: model_a - data_tests: - - type_one - - type_two - - name: model_c - data_tests: - - call_pkg_macro - - test_utils.dispatch - -""" - -test_context_models_namespaced__model_c_sql = """ -select 1 as fun - -""" - -test_context_models_namespaced__model_b_sql = """ -select 1 as notfun - -""" - -test_context_models_namespaced__model_a_sql = """ -select 1 as fun - -""" - -macros_v2__override_get_test_macros_fail__get_test_sql_sql = """ -{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} - select - {{ fail_calc }} as failures, - case when {{ fail_calc }} {{ warn_if }} then 'x' else 'y' end as should_warn, - case when {{ fail_calc }} {{ error_if }} then 'x' else 'y' end as should_error - from ( - {{ main_sql }} - {{ "limit " ~ limit if limit != none }} - ) dbt_internal_test -{% endmacro %} -""" - -macros_v2__macros__tests_sql = """ -{% test every_value_is_blue(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} != 'blue' - -{% endtest %} - - -{% test rejected_values(model, column_name, values) %} - - select * - from {{ model }} - where {{ column_name }} in ( - {% for value in values %} - '{{ value }}' {% if not loop.last %} , {% endif %} - {% endfor %} - ) - -{% endtest %} - - -{% test equivalent(model, value) %} - {% set expected = 'foo-bar' %} - {% set eq = 1 if value == expected else 0 %} - {% set validation_message -%} - 'got "{{ value }}", expected "{{ expected }}"' - {%- endset %} - {% if eq == 0 and execute %} - {{ log(validation_message, info=True) }} - {% endif %} - - select {{ validation_message }} as validation_error - where {{ eq }} = 0 -{% endtest %} - - -""" - -macros_v2__override_get_test_macros__get_test_sql_sql = """ -{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} - select - {{ fail_calc }} as failures, - case when {{ fail_calc }} {{ warn_if }} then 1 else 0 end as should_warn, - case when {{ fail_calc }} {{ error_if }} then 1 else 0 end as should_error - from ( - {{ main_sql }} - {{ "limit " ~ limit if limit != none }} - ) dbt_internal_test -{%- endmacro %} -""" - -macros_v2__custom_configs__test_sql = """ -{% test where(model, column_name) %} - {{ config(where = "1 = 0") }} - select * from {{ model }} -{% endtest %} - -{% test error_if(model, column_name) %} - {{ config(error_if = "<= 0", warn_if = "<= 0") }} - select * from {{ model }} -{% endtest %} - - -{% test warn_if(model, column_name) %} - {{ config(warn_if = "<= 0", severity = "WARN") }} - select * from {{ model }} -{% endtest %} - -{% test limit(model, column_name) %} - {{ config(limit = 0) }} - select * from {{ model }} -{% endtest %} - -{% test fail_calc(model, column_name) %} - {{ config(fail_calc = "count(*) - count(*)") }} - select * from {{ model }} -{% endtest %} - -""" - -test_context_macros_namespaced__my_test_sql = """ -{% macro test_call_pkg_macro(model) %} - select {{ test_utils.current_timestamp() }} -{% endmacro %} - -""" - -test_context_macros_namespaced__custom_schema_tests_sql = """ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "WARN") }} - - select * from {{ model }} - -{% endtest %} - -""" - -seeds__some_seed_csv = """ -col_int,col_str -1,hello -2,goodbye -""" - -test_context_models__schema_yml = """ - -version: 2 - -models: - - name: model_a - data_tests: - - type_one - - type_two - - name: model_c - data_tests: - - call_pkg_macro - - local_utils.dispatch - - my_datediff - -""" - -test_context_models__model_c_sql = """ -select 1 as fun - -""" - -test_context_models__model_b_sql = """ -select 1 as notfun - -""" - -test_context_models__model_a_sql = """ -select 1 as fun - -""" - -name_collision__schema_yml = """ -version: 2 -models: -- name: base - columns: - - name: extension_id - data_tests: - - not_null -- name: base_extension - columns: - - name: id - data_tests: - - not_null - -""" - -name_collision__base_sql = """ -SELECT 'hello_world' AS extension_id -""" - -name_collision__base_extension_sql = """ -SELECT 'NOT_NULL' AS id -""" - - -dupe_generic_tests_collide__schema_yml = """ -version: 2 -models: -- name: model_a - columns: - - name: id - data_tests: - - not_null: - config: - where: "1=1" - - not_null: - config: - where: "1=2" - -""" - -dupe_generic_tests_collide__model_a = """ -SELECT 'NOT_NULL' AS id -""" - - -custom_generic_test_config_custom_macro__schema_yml = """ -version: 2 -models: -- name: model_a - columns: - - name: id - data_tests: - - not_null: - config: - where: "id = (select id from {{ ref('model_a') }} limit 1)" - -""" - -custom_generic_test_config_custom_macro__model_a = """ -SELECT 1 AS id -""" - - -custom_generic_test_names__schema_yml = """ -version: 2 -models: -- name: model_a - columns: - - name: id - data_tests: - - not_null: - name: not_null_where_1_equals_1 - config: - where: "1=1" - - not_null: - name: not_null_where_1_equals_2 - config: - where: "1=2" - -""" - -custom_generic_test_names__model_a = """ -SELECT 'NOT_NULL' AS id -""" - -custom_generic_test_names_alt_format__schema_yml = """ -version: 2 -models: -- name: model_a - columns: - - name: id - data_tests: - - name: not_null_where_1_equals_1 - test_name: not_null - config: - where: "1=1" - - name: not_null_where_1_equals_2 - test_name: not_null - config: - where: "1=2" - -""" - -custom_generic_test_names_alt_format__model_a = """ -SELECT 'NOT_NULL' AS id -""" - - -test_context_where_subq_macros__custom_generic_test_sql = """ -/*{# This test will fail if get_where_subquery() is missing from TestContext + TestMacroNamespace #}*/ - -{% test self_referential(model) %} - - {%- set relation = api.Relation.create(schema=model.schema, identifier=model.table) -%} - {%- set columns = adapter.get_columns_in_relation(relation) -%} - {%- set columns_csv = columns | map(attribute='name') | list | join(', ') -%} - - select {{ columns_csv }} from {{ model }} - limit 0 - -{% endtest %} - -""" - -invalid_schema_models__schema_yml = """ -version: 2 - -models: - name: model - columns: - - name: Id - quote: true - data_tests: - - unique - - not_null - -""" - -invalid_schema_models__model_sql = """ -select 1 as "Id" - -""" - -all_quotes_schema__schema_yml = """# models/schema.yml -# only comments here, which should be okay! -# https://github.com/dbt-labs/dbt-core/issues/3568""" - -models_v2__render_test_cli_arg_models__schema_yml = """ -version: 2 - -models: - - name: model - data_tests: - - equivalent: - value: "{{ var('myvar', 'baz') }}-bar" - -""" - -models_v2__render_test_cli_arg_models__model_sql = """ -select 1 as id - -""" - -models_v2__override_get_test_models__schema_yml = """ -version: 2 - -models: - - name: my_model_pass - description: "The table has 1 null values, and we're okay with that, until it's more than 1." - columns: - - name: id - description: "The number of responses for this favorite color - purple will be null" - data_tests: - - not_null: - error_if: '>1' - warn_if: '>1' - - - name: my_model_warning - description: "The table has 1 null values, and we're okay with that, but let us know" - columns: - - name: id - description: "The number of responses for this favorite color - purple will be null" - data_tests: - - not_null: - error_if: '>1' - - - name: my_model_failure - description: "The table has 2 null values, and we're not okay with that" - columns: - - name: id - description: "The number of responses for this favorite color - purple will be null" - data_tests: - - not_null: - error_if: '>1' - - -""" - -models_v2__override_get_test_models__my_model_warning_sql = """ -select * from {{ ref('my_model_pass') }} -""" - -models_v2__override_get_test_models__my_model_pass_sql = """ -select 1 as id -UNION ALL -select null as id -""" - -models_v2__override_get_test_models__my_model_failure_sql = """ -select * from {{ ref('my_model_pass') }} -UNION ALL -select null as id -""" - -models_v2__models__schema_yml = """ -version: 2 - -models: - - name: table_copy - description: "A copy of the table" - columns: - - name: id - description: "The ID" - data_tests: - - not_null - - unique - tags: - - table_id - - name: first_name - description: "The user's first name" - data_tests: - - not_null - tags: - - table_first_name - - name: ip_address - description: "The user's IP address" - data_tests: - - not_null - - name: updated_at - description: "The update time of the user" - data_tests: - - not_null - - name: email - description: "The user's email address" - data_tests: - - unique - - name: favorite_color - description: "The user's favorite color" - data_tests: - - accepted_values: { - values: ['blue', 'green'], - quote: true, - tags: table_copy_favorite_color # tags can be a single string - } - tags: - - table_favorite_color - - name: fav_number - description: "The user's favorite number" - data_tests: - - accepted_values: - values: [3.14159265] - quote: false - tags: # tags can be a list of strings - - favorite_number_is_pi - - - - name: table_summary - description: "The summary table" - columns: - - name: favorite_color_copy - description: "The favorite color" - data_tests: - - not_null - - unique - - accepted_values: { values: ['blue', 'green'] } - - relationships: { field: favorite_color, to: ref('table_copy') } - tags: - - table_favorite_color - - name: count - description: "The number of responses for this favorite color" - data_tests: - - not_null - -# all of these constraints will fail - - name: table_failure_copy - description: "The table copy that does not comply with the schema" - columns: - - name: id - description: "The user ID" - data_tests: - - not_null - - unique - tags: - - xfail - - name: favorite_color - description: "The user's favorite color" - data_tests: - - accepted_values: { values: ['blue', 'green'] } - tags: - - xfail - -# all of these constraints will fail - - name: table_failure_summary - description: "The table summary that does not comply with the schema" - columns: - - name: favorite_color - description: "The favorite color" - data_tests: - - accepted_values: { values: ['red'] } - - relationships: { field: favorite_color, to: ref('table_copy') } - tags: - - xfail - -# this table is disabled so these tests should be ignored - - name: table_disabled - description: "A disabled table" - columns: - - name: favorite_color - description: "The favorite color" - data_tests: - - accepted_values: { values: ['red'] } - - relationships: { field: favorite_color, to: ref('table_copy') } - -# all of these constraints will fail - - name: table_failure_null_relation - description: "A table with a null value where it should be a foreign key" - columns: - - name: id - description: "The user ID" - data_tests: - - relationships: { field: id, to: ref('table_failure_copy') } - tags: - - xfail - -""" - -models_v2__models__table_summary_sql = """ -{{ - config( - materialized='table' - ) -}} - -select favorite_color as favorite_color_copy, count(*) as count -from {{ ref('table_copy') }} -group by 1 - -""" - -models_v2__models__table_failure_summary_sql = """ -{{ - config( - materialized='table' - ) -}} - --- force a foreign key constraint failure here -select 'purple' as favorite_color, count(*) as count -from {{ ref('table_failure_copy') }} -group by 1 - -""" - -models_v2__models__table_disabled_sql = """ -{{ - config( - enabled=False - ) -}} - --- force a foreign key constraint failure here -select 'purple' as favorite_color, count(*) as count -from {{ ref('table_failure_copy') }} -group by 1 - -""" - -models_v2__models__table_failure_null_relation_sql = """ -{{ - config( - materialized='table' - ) -}} - --- force a foreign key constraint failure here -select 105 as id, count(*) as count -from {{ ref('table_failure_copy') }} -group by 1 - -""" - -models_v2__models__table_failure_copy_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed_failure - -""" - -models_v2__models__table_copy_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed - -""" - -models_v2__malformed__schema_yml = """ -version: 2 - -models: - # this whole model should fail and not run - - name: table_copy - description: "A copy of the table" - columns: - - name: id - description: "The ID" - data_tests: - - not_null - - unique - - name: favorite_color - data_tests: - # this is missing a "-" and is malformed - accepted_values: { values: ['blue', 'green'] } - - # this whole model should pass and run - - name: table_summary - description: "The summary table" - columns: - - name: favorite_color - description: "The favorite color" - data_tests: - - not_null - - unique - - accepted_values: { values: ['blue', 'green'] } - - relationships: { field: favorite_color, to: ref('table_copy') } - - name: count - description: "The number of responses for this favorite color" - data_tests: - - not_null - -""" - -models_v2__malformed__table_summary_sql = """ -{{ - config( - materialized='table' - ) -}} - -select favorite_color, count(*) as count -from {{ ref('table_copy') }} -group by 1 - -""" - -models_v2__malformed__table_copy_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed - -""" - -models_v2__override_get_test_models_fail__schema_yml = """ -version: 2 - -models: - - name: my_model - description: "The table has 1 null values, and we're not okay with that." - columns: - - name: id - description: "The number of responses for this favorite color - purple will be null" - data_tests: - - not_null - - - -""" - -models_v2__override_get_test_models_fail__my_model_sql = """ -select 1 as id -UNION ALL -select null as id -""" - -models_v2__custom_configs__schema_yml = """ -version: 2 - -models: - - name: table_copy - description: "A copy of the table" - # passes - data_tests: - - where - - error_if - - warn_if - - limit - - fail_calc - columns: - - name: id - data_tests: - # relationships with where - - relationships: - to: ref('table_copy') # itself - field: id - where: 1=1 - - name: table_copy_another_one - data_tests: - - where: # test override + weird quoting - config: - where: "\\"favorite_color\\" = 'red'" - - name: "table.copy.with.dots" - description: "A copy of the table with a gross name" - # passes, see https://github.com/dbt-labs/dbt-core/issues/3857 - data_tests: - - where - -""" - -models_v2__custom_configs__table_copy_another_one_sql = """ -select * from {{ ref('table_copy') }} - -""" - -models_v2__custom_configs__table_copy_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed - -""" - -models_v2__custom_configs__table_copy_with_dots_sql = """ -select * from {{ ref('table_copy') }} - -""" - -models_v2__render_test_configured_arg_models__schema_yml = """ -version: 2 - -models: - - name: model - data_tests: - - equivalent: - value: "{{ var('myvar', 'baz') }}-bar" - -""" - -models_v2__render_test_configured_arg_models__model_sql = """ -select 1 as id - -""" - -models_v2__custom__schema_yml = """ -version: 2 - -models: - - name: table_copy - description: "A copy of the table" - columns: - - name: email - data_tests: - - not_null - - name: id - description: "The ID" - data_tests: - - unique - - name: favorite_color - data_tests: - - every_value_is_blue - - rejected_values: { values: ['orange', 'purple'] } - # passes - data_tests: - - local_dep.equality: { compare_model: ref('table_copy') } - -""" - -models_v2__custom__table_copy_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select * from {{ this.schema }}.seed - -""" - -models_v2__limit_null__schema_yml = """ -version: 2 - -models: - - name: table_limit_null - description: "The table has 1 null values, and we're okay with that, until it's more than 1." - columns: - - name: favorite_color_full_list - description: "The favorite color" - - name: count - description: "The number of responses for this favorite color - purple will be null" - data_tests: - - not_null: - error_if: '>1' - warn_if: '>1' - - - name: table_warning_limit_null - description: "The table has 1 null value, and we're okay with 1, but want to know of any." - columns: - - name: favorite_color_full_list - description: "The favorite color" - - name: count - description: "The number of responses for this favorite color - purple will be null" - data_tests: - - not_null: - error_if: '>1' - - - name: table_failure_limit_null - description: "The table has some 2 null values, and that's not ok. Warn and error." - columns: - - name: favorite_color_full_list - description: "The favorite color" - - name: count - description: "The number of responses for this favorite color - purple will be null" - data_tests: - - not_null: - error_if: '>1' - -""" - -models_v2__limit_null__table_warning_limit_null_sql = """ -{{ - config( - materialized='table' - ) -}} - -select * from {{ref('table_limit_null')}} -""" - -models_v2__limit_null__table_limit_null_sql = """ -{{ - config( - materialized='table' - ) -}} - -select favorite_color as favorite_color_full_list, count(*) as count -from {{ this.schema }}.seed -group by 1 - -UNION ALL - -select 'purple' as favorite_color_full_list, null as count -""" - -models_v2__limit_null__table_failure_limit_null_sql = """ -{{ - config( - materialized='table' - ) -}} - -select * from {{ref('table_limit_null')}} - -UNION ALL - -select 'magenta' as favorite_color_full_list, null as count -""" - -local_utils__dbt_project_yml = """ -name: 'local_utils' -version: '1.0' -config-version: 2 - -profile: 'default' - -macro-paths: ["macros"] - - -""" - -local_utils__macros__datediff_sql = """ -{% macro datediff(first_date, second_date, datepart) %} - {{ return(adapter.dispatch('datediff', 'local_utils')(first_date, second_date, datepart)) }} -{% endmacro %} - - -{% macro default__datediff(first_date, second_date, datepart) %} - - datediff( - {{ datepart }}, - {{ first_date }}, - {{ second_date }} - ) - -{% endmacro %} - - -{% macro postgres__datediff(first_date, second_date, datepart) %} - - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ adapter.dispatch('datediff', 'local_utils')(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% else %} - ( 1000 ) - {% endif %} - -{% endmacro %} - - -""" - -local_utils__macros__current_timestamp_sql = """ -{% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp')) }} -{%- endmacro %} - -{% macro default__current_timestamp() -%} - now() -{%- endmacro %} - -""" - -local_utils__macros__custom_test_sql = """ -{% macro test_dispatch(model) -%} - {{ return(adapter.dispatch('test_dispatch', macro_namespace = 'local_utils')()) }} -{%- endmacro %} - -{% macro default__test_dispatch(model) %} - select {{ adapter.dispatch('current_timestamp', macro_namespace = 'local_utils')() }} -{% endmacro %} - -""" - -ephemeral__schema_yml = """ - -version: 2 -models: - - name: ephemeral - columns: - - name: id - data_tests: - - unique - -""" - -ephemeral__ephemeral_sql = """ - -{{ config(materialized='ephemeral') }} - -select 1 as id - -""" - -quote_required_models__schema_yml = """ -version: 2 - -models: - - name: model - columns: - - name: Id - quote: true - data_tests: - - unique - - not_null - - name: model_again - quote_columns: true - columns: - - name: Id - data_tests: - - unique - - not_null - - name: model_noquote - quote_columns: true - columns: - - name: Id - quote: false - data_tests: - - unique - - not_null - -sources: - # this should result in column quoting = true - - name: my_source - schema: "{{ target.schema }}" - quoting: - column: true - tables: - - name: model - quoting: - column: false - columns: - - name: Id - quote: true - data_tests: - - unique - - name: my_source_2 - schema: "{{ target.schema }}" - quoting: - column: false - tables: - # this should result in column quoting = true - - name: model - quoting: - column: true - columns: - - name: Id - data_tests: - - unique - # this should result in column quoting = false - - name: model_noquote - columns: - - name: Id - data_tests: - - unique - - -""" - -quote_required_models__model_again_sql = """ -select 1 as "Id" - -""" - -quote_required_models__model_noquote_sql = """ -select 1 as id - -""" - -quote_required_models__model_sql = """ -select 1 as "Id" - -""" - -alt_local_utils__macros__type_timestamp_sql = """ -{%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', 'local_utils')()) }} -{%- endmacro -%} - -{% macro default__type_timestamp() %} - {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }} -{% endmacro %} -""" - -macro_resolution_order_macros__my_custom_test_sql = """ -{% test my_custom_test(model) %} - select cast(current_timestamp as {{ dbt.type_timestamp() }}) - limit 0 -{% endtest %} -""" - -macro_resolution_order_models__my_model_sql = """ -select 1 as id -""" - -macro_resolution_order_models__config_yml = """ -version: 2 -models: - - name: my_model - data_tests: - - my_custom_test -""" diff --git a/tests/functional/schema_tests/test_schema_v2_tests.py b/tests/functional/schema_tests/test_schema_v2_tests.py deleted file mode 100644 index aae164a1..00000000 --- a/tests/functional/schema_tests/test_schema_v2_tests.py +++ /dev/null @@ -1,1110 +0,0 @@ -import os -import re - -from dbt.contracts.results import TestStatus -from dbt.exceptions import ParsingError -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import run_dbt, write_file -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.schema_tests.fixtures import ( - alt_local_utils__macros__type_timestamp_sql, - all_quotes_schema__schema_yml, - case_sensitive_models__lowercase_sql, - case_sensitive_models__schema_yml, - case_sensitive_models__uppercase_SQL, - custom_generic_test_config_custom_macro__model_a, - custom_generic_test_config_custom_macro__schema_yml, - custom_generic_test_names__model_a, - custom_generic_test_names__schema_yml, - custom_generic_test_names_alt_format__model_a, - custom_generic_test_names_alt_format__schema_yml, - ephemeral__ephemeral_sql, - ephemeral__schema_yml, - invalid_schema_models__model_sql, - invalid_schema_models__schema_yml, - local_dependency__dbt_project_yml, - local_dependency__macros__equality_sql, - local_utils__dbt_project_yml, - local_utils__macros__current_timestamp_sql, - local_utils__macros__custom_test_sql, - local_utils__macros__datediff_sql, - macro_resolution_order_models__config_yml, - macro_resolution_order_macros__my_custom_test_sql, - macro_resolution_order_models__my_model_sql, - macros_v2__custom_configs__test_sql, - macros_v2__macros__tests_sql, - macros_v2__override_get_test_macros__get_test_sql_sql, - macros_v2__override_get_test_macros_fail__get_test_sql_sql, - models_v2__custom__schema_yml, - models_v2__custom__table_copy_sql, - models_v2__custom_configs__schema_yml, - models_v2__custom_configs__table_copy_another_one_sql, - models_v2__custom_configs__table_copy_sql, - models_v2__custom_configs__table_copy_with_dots_sql, - models_v2__limit_null__schema_yml, - models_v2__limit_null__table_failure_limit_null_sql, - models_v2__limit_null__table_limit_null_sql, - models_v2__limit_null__table_warning_limit_null_sql, - models_v2__malformed__schema_yml, - models_v2__malformed__table_copy_sql, - models_v2__malformed__table_summary_sql, - models_v2__models__schema_yml, - models_v2__models__table_copy_sql, - models_v2__models__table_disabled_sql, - models_v2__models__table_failure_copy_sql, - models_v2__models__table_failure_null_relation_sql, - models_v2__models__table_failure_summary_sql, - models_v2__models__table_summary_sql, - models_v2__override_get_test_models__my_model_failure_sql, - models_v2__override_get_test_models__my_model_pass_sql, - models_v2__override_get_test_models__my_model_warning_sql, - models_v2__override_get_test_models__schema_yml, - models_v2__override_get_test_models_fail__my_model_sql, - models_v2__override_get_test_models_fail__schema_yml, - models_v2__render_test_cli_arg_models__model_sql, - models_v2__render_test_cli_arg_models__schema_yml, - models_v2__render_test_configured_arg_models__model_sql, - models_v2__render_test_configured_arg_models__schema_yml, - name_collision__base_sql, - name_collision__base_extension_sql, - name_collision__schema_yml, - quote_required_models__model_again_sql, - quote_required_models__model_noquote_sql, - quote_required_models__model_sql, - quote_required_models__schema_yml, - seeds__some_seed_csv, - test_context_where_subq_models__model_a_sql, - test_context_where_subq_models__schema_yml, - test_context_macros__custom_schema_tests_sql, - test_context_macros__my_test_sql, - test_context_macros__test_my_datediff_sql, - test_context_models__model_a_sql, - test_context_models__model_b_sql, - test_context_models__model_c_sql, - test_context_models__schema_yml, - test_context_macros_namespaced__custom_schema_tests_sql, - test_context_models_namespaced__model_a_sql, - test_context_models_namespaced__model_b_sql, - test_context_models_namespaced__model_c_sql, - test_context_macros_namespaced__my_test_sql, - test_context_models_namespaced__schema_yml, - test_context_where_subq_macros__custom_generic_test_sql, - test_utils__dbt_project_yml, - test_utils__macros__current_timestamp_sql, - test_utils__macros__custom_test_sql, - wrong_specification_block__schema_yml, -) - - -class TestSchemaTests: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) - project.run_sql_file(os.path.join(project.test_data_dir, "seed_failure.sql")) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__models__schema_yml, - "table_summary.sql": models_v2__models__table_summary_sql, - "table_failure_summary.sql": models_v2__models__table_failure_summary_sql, - "table_disabled.sql": models_v2__models__table_disabled_sql, - "table_failure_null_relation.sql": models_v2__models__table_failure_null_relation_sql, - "table_failure_copy.sql": models_v2__models__table_failure_copy_sql, - "table_copy.sql": models_v2__models__table_copy_sql, - } - - def assertTestFailed(self, result): - assert result.status == "fail" - assert not result.skipped - assert result.failures > 0, "test {} did not fail".format(result.node.name) - - def assertTestPassed(self, result): - assert result.status == "pass" - assert not result.skipped - assert result.failures == 0, "test {} failed".format(result.node.name) - - def test_schema_tests( - self, - project, - ): - results = run_dbt() - assert len(results) == 5 - test_results = run_dbt(["test"], expect_pass=False) - # If the disabled model's tests ran, there would be 20 of these. - assert len(test_results) == 19 - - for result in test_results: - # assert that all deliberately failing tests actually fail - if "failure" in result.node.name: - self.assertTestFailed(result) - # assert that actual tests pass - else: - self.assertTestPassed(result) - assert sum(x.failures for x in test_results) == 6 - - def test_schema_test_selection( - self, - project, - ): - results = run_dbt() - assert len(results) == 5 - test_results = run_dbt(["test", "--models", "tag:table_favorite_color"]) - # 1 in table_copy, 4 in table_summary - assert len(test_results) == 5 - for result in test_results: - self.assertTestPassed(result) - - test_results = run_dbt(["test", "--models", "tag:favorite_number_is_pi"]) - assert len(test_results) == 1 - self.assertTestPassed(test_results[0]) - - test_results = run_dbt(["test", "--models", "tag:table_copy_favorite_color"]) - assert len(test_results) == 1 - self.assertTestPassed(test_results[0]) - - def test_schema_test_exclude_failures( - self, - project, - ): - results = run_dbt() - assert len(results) == 5 - test_results = run_dbt(["test", "--exclude", "tag:xfail"]) - # If the failed + disabled model's tests ran, there would be 20 of these. - assert len(test_results) == 13 - for result in test_results: - self.assertTestPassed(result) - test_results = run_dbt(["test", "--models", "tag:xfail"], expect_pass=False) - assert len(test_results) == 6 - for result in test_results: - self.assertTestFailed(result) - - -class TestLimitedSchemaTests: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__limit_null__schema_yml, - "table_warning_limit_null.sql": models_v2__limit_null__table_warning_limit_null_sql, - "table_limit_null.sql": models_v2__limit_null__table_limit_null_sql, - "table_failure_limit_null.sql": models_v2__limit_null__table_failure_limit_null_sql, - } - - def assertTestFailed(self, result): - assert result.status == "fail" - assert not result.skipped - assert result.failures > 0, "test {} did not fail".format(result.node.name) - - def assertTestWarn(self, result): - assert result.status == "warn" - assert not result.skipped - assert result.failures > 0, "test {} passed without expected warning".format( - result.node.name - ) - - def assertTestPassed(self, result): - assert result.status == "pass" - assert not result.skipped - assert result.failures == 0, "test {} failed".format(result.node.name) - - def test_limit_schema_tests( - self, - project, - ): - results = run_dbt() - assert len(results) == 3 - test_results = run_dbt(["test"], expect_pass=False) - assert len(test_results) == 3 - - for result in test_results: - # assert that all deliberately failing tests actually fail - if "failure" in result.node.name: - self.assertTestFailed(result) - # assert that tests with warnings have them - elif "warning" in result.node.name: - self.assertTestWarn(result) - # assert that actual tests pass - else: - self.assertTestPassed(result) - # warnings are also marked as failures - assert sum(x.failures for x in test_results) == 3 - - -class TestDefaultBoolType: - # test with default True/False in get_test_sql macro - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__override_get_test_models__schema_yml, - "my_model_warning.sql": models_v2__override_get_test_models__my_model_warning_sql, - "my_model_pass.sql": models_v2__override_get_test_models__my_model_pass_sql, - "my_model_failure.sql": models_v2__override_get_test_models__my_model_failure_sql, - } - - def assertTestFailed(self, result): - assert result.status == "fail" - assert not result.skipped - assert result.failures > 0, "test {} did not fail".format(result.node.name) - - def assertTestWarn(self, result): - assert result.status == "warn" - assert not result.skipped - assert result.failures > 0, "test {} passed without expected warning".format( - result.node.name - ) - - def assertTestPassed(self, result): - assert result.status == "pass" - assert not result.skipped - assert result.failures == 0, "test {} failed".format(result.node.name) - - def test_limit_schema_tests( - self, - project, - ): - results = run_dbt() - assert len(results) == 3 - test_results = run_dbt(["test"], expect_pass=False) - assert len(test_results) == 3 - - for result in test_results: - # assert that all deliberately failing tests actually fail - if "failure" in result.node.name: - self.assertTestFailed(result) - # assert that tests with warnings have them - elif "warning" in result.node.name: - self.assertTestWarn(result) - # assert that actual tests pass - else: - self.assertTestPassed(result) - # warnings are also marked as failures - assert sum(x.failures for x in test_results) == 3 - - -class TestOtherBoolType: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - macros_v2_file = { - "override_get_test_macros": { - "get_test_sql.sql": macros_v2__override_get_test_macros__get_test_sql_sql - }, - } - write_project_files(project_root, "macros-v2", macros_v2_file) - - # test with expected 0/1 in custom get_test_sql macro - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__override_get_test_models__schema_yml, - "my_model_warning.sql": models_v2__override_get_test_models__my_model_warning_sql, - "my_model_pass.sql": models_v2__override_get_test_models__my_model_pass_sql, - "my_model_failure.sql": models_v2__override_get_test_models__my_model_failure_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["macros-v2/override_get_test_macros"], - } - - def assertTestFailed(self, result): - assert result.status == "fail" - assert not result.skipped - assert result.failures > 0, "test {} did not fail".format(result.node.name) - - def assertTestWarn(self, result): - assert result.status == "warn" - assert not result.skipped - assert result.failures > 0, "test {} passed without expected warning".format( - result.node.name - ) - - def assertTestPassed(self, result): - assert result.status == "pass" - assert not result.skipped - assert result.failures == 0, "test {} failed".format(result.node.name) - - def test_limit_schema_tests( - self, - project, - ): - results = run_dbt() - assert len(results) == 3 - test_results = run_dbt(["test"], expect_pass=False) - assert len(test_results) == 3 - - for result in test_results: - # assert that all deliberately failing tests actually fail - if "failure" in result.node.name: - self.assertTestFailed(result) - # assert that tests with warnings have them - elif "warning" in result.node.name: - self.assertTestWarn(result) - # assert that actual tests pass - else: - self.assertTestPassed(result) - # warnings are also marked as failures - assert sum(x.failures for x in test_results) == 3 - - -class TestNonBoolType: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - macros_v2_file = { - "override_get_test_macros_fail": { - "get_test_sql.sql": macros_v2__override_get_test_macros_fail__get_test_sql_sql - }, - } - write_project_files(project_root, "macros-v2", macros_v2_file) - - # test with invalid 'x'/'y' in custom get_test_sql macro - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__override_get_test_models_fail__schema_yml, - "my_model.sql": models_v2__override_get_test_models_fail__my_model_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["macros-v2/override_get_test_macros_fail"], - } - - def test_limit_schema_tests( - self, - project, - ): - results = run_dbt() - assert len(results) == 1 - run_result = run_dbt(["test"], expect_pass=False) - results = run_result.results - assert len(results) == 1 - assert results[0].status == TestStatus.Error - assert re.search(r"'get_test_sql' returns 'x'", results[0].message) - - -class TestMalformedSchemaTests: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__malformed__schema_yml, - "table_summary.sql": models_v2__malformed__table_summary_sql, - "table_copy.sql": models_v2__malformed__table_copy_sql, - } - - def test_malformed_schema_will_break_run( - self, - project, - ): - with pytest.raises(ParsingError): - run_dbt() - - -class TestCustomConfigSchemaTests: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project, project_root): - project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) - - macros_v2_file = {"custom-configs": {"test.sql": macros_v2__custom_configs__test_sql}} - write_project_files(project_root, "macros-v2", macros_v2_file) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__custom_configs__schema_yml, - "table_copy_another_one.sql": models_v2__custom_configs__table_copy_another_one_sql, - "table_copy.sql": models_v2__custom_configs__table_copy_sql, - "table.copy.with.dots.sql": models_v2__custom_configs__table_copy_with_dots_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["macros-v2/custom-configs"], - } - - def test_config( - self, - project, - ): - """Test that tests use configs properly. All tests for - this project will fail, configs are set to make test pass.""" - results = run_dbt(["test"], expect_pass=False) - - assert len(results) == 8 - for result in results: - assert not result.skipped - - -class TestHooksInTests: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": ephemeral__schema_yml, - "ephemeral.sql": ephemeral__ephemeral_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "on-run-start": ["{{ log('hooks called in tests -- good!') if execute }}"], - "on-run-end": ["{{ log('hooks called in tests -- good!') if execute }}"], - } - - def test_hooks_do_run_for_tests( - self, - project, - ): - # This passes now that hooks run, a behavior we changed in v1.0 - results = run_dbt(["test", "--model", "ephemeral"]) - assert len(results) == 1 - for result in results: - assert result.status == "pass" - assert not result.skipped - assert result.failures == 0, "test {} failed".format(result.node.name) - - -class TestHooksForWhich: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": ephemeral__schema_yml, - "ephemeral.sql": ephemeral__ephemeral_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "on-run-start": [ - "{{exceptions.raise_compiler_error('hooks called in tests -- error') if (execute and flags.WHICH != 'test') }}" - ], - "on-run-end": [ - "{{exceptions.raise_compiler_error('hooks called in tests -- error') if (execute and flags.WHICH != 'test') }}" - ], - } - - def test_these_hooks_dont_run_for_tests( - self, - project, - ): - # This would fail if the hooks ran - results = run_dbt(["test", "--model", "ephemeral"]) - assert len(results) == 1 - for result in results: - assert result.status == "pass" - assert not result.skipped - assert result.failures == 0, "test {} failed".format(result.node.name) - - -class TestCustomSchemaTests: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project, project_root, dbt_integration_project): # noqa: F811 - write_project_files(project_root, "dbt_integration_project", dbt_integration_project) - project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) - - local_dependency_files = { - "dbt_project.yml": local_dependency__dbt_project_yml, - "macros": {"equality.sql": local_dependency__macros__equality_sql}, - } - write_project_files(project_root, "local_dependency", local_dependency_files) - - macros_v2_file = { - "macros": {"tests.sql": macros_v2__macros__tests_sql}, - } - write_project_files(project_root, "macros-v2", macros_v2_file) - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "local": "./local_dependency", - }, - { - "local": "./dbt_integration_project", - }, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - # dbt-utils contains a schema test (equality) - # dbt-integration-project contains a schema.yml file - # both should work! - return { - "config-version": 2, - "macro-paths": ["macros-v2/macros"], - } - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__custom__schema_yml, - "table_copy.sql": models_v2__custom__table_copy_sql, - } - - def test_schema_tests( - self, - project, - ): - run_dbt(["deps"]) - results = run_dbt() - assert len(results) == 4 - - test_results = run_dbt(["test"], expect_pass=False) - assert len(test_results) == 6 - - expected_failures = [ - "not_null_table_copy_email", - "every_value_is_blue_table_copy_favorite_color", - ] - - for result in test_results: - if result.status == "fail": - assert result.node.name in expected_failures - - -class TestQuotedSchemaTestColumns: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": quote_required_models__schema_yml, - "model_again.sql": quote_required_models__model_again_sql, - "model_noquote.sql": quote_required_models__model_noquote_sql, - "model.sql": quote_required_models__model_sql, - } - - def test_quote_required_column( - self, - project, - ): - results = run_dbt() - assert len(results) == 3 - results = run_dbt(["test", "-m", "model"]) - assert len(results) == 2 - results = run_dbt(["test", "-m", "model_again"]) - assert len(results) == 2 - results = run_dbt(["test", "-m", "model_noquote"]) - assert len(results) == 2 - results = run_dbt(["test", "-m", "source:my_source"]) - assert len(results) == 1 - results = run_dbt(["test", "-m", "source:my_source_2"]) - assert len(results) == 2 - - -class TestCliVarsSchemaTests: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - macros_v2_file = { - "macros": {"tests.sql": macros_v2__macros__tests_sql}, - } - write_project_files(project_root, "macros-v2", macros_v2_file) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__render_test_cli_arg_models__schema_yml, - "model.sql": models_v2__render_test_cli_arg_models__model_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["macros-v2/macros"], - } - - def test_argument_rendering( - self, - project, - ): - results = run_dbt() - assert len(results) == 1 - results = run_dbt(["test", "--vars", "{myvar: foo}"]) - assert len(results) == 1 - run_dbt(["test"], expect_pass=False) - - -class TestConfiguredVarsSchemaTests: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - macros_v2_file = { - "macros": {"tests.sql": macros_v2__macros__tests_sql}, - } - write_project_files(project_root, "macros-v2", macros_v2_file) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_v2__render_test_configured_arg_models__schema_yml, - "model.sql": models_v2__render_test_configured_arg_models__model_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["macros-v2/macros"], - "vars": {"myvar": "foo"}, - } - - def test_argument_rendering( - self, - project, - ): - results = run_dbt() - assert len(results) == 1 - results = run_dbt(["test"]) - assert len(results) == 1 - - -class TestSchemaCaseInsensitive: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": case_sensitive_models__schema_yml, - "lowercase.sql": case_sensitive_models__lowercase_sql, - } - - @pytest.fixture(scope="class", autouse=True) - def setUP(self, project): - # Create the uppercase SQL file - model_dir = os.path.join(project.project_root, "models") - write_file(case_sensitive_models__uppercase_SQL, model_dir, "uppercase.SQL") - - def test_schema_lowercase_sql( - self, - project, - ): - results = run_dbt() - assert len(results) == 2 - results = run_dbt(["test", "-m", "lowercase"]) - assert len(results) == 1 - - def test_schema_uppercase_sql( - self, - project, - ): - results = run_dbt() - assert len(results) == 2 - results = run_dbt(["test", "-m", "uppercase"]) - assert len(results) == 1 - - -class TestSchemaTestContext: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - local_utils_files = { - "dbt_project.yml": local_utils__dbt_project_yml, - "macros": { - "datediff.sql": local_utils__macros__datediff_sql, - "current_timestamp.sql": local_utils__macros__current_timestamp_sql, - "custom_test.sql": local_utils__macros__custom_test_sql, - }, - } - write_project_files(project_root, "local_utils", local_utils_files) - - test_context_macros_files = { - "my_test.sql": test_context_macros__my_test_sql, - "test_my_datediff.sql": test_context_macros__test_my_datediff_sql, - "custom_schema_tests.sql": test_context_macros__custom_schema_tests_sql, - } - write_project_files(project_root, "test-context-macros", test_context_macros_files) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": test_context_models__schema_yml, - "model_c.sql": test_context_models__model_c_sql, - "model_b.sql": test_context_models__model_b_sql, - "model_a.sql": test_context_models__model_a_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["test-context-macros"], - "vars": {"local_utils_dispatch_list": ["local_utils"]}, - } - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": "local_utils"}]} - - def test_test_context_tests(self, project): - # This test tests the the TestContext and TestMacroNamespace - # are working correctly - run_dbt(["deps"]) - results = run_dbt() - assert len(results) == 3 - - run_result = run_dbt(["test"], expect_pass=False) - results = run_result.results - results = sorted(results, key=lambda r: r.node.name) - assert len(results) == 5 - # call_pkg_macro_model_c_ - assert results[0].status == TestStatus.Fail - # dispatch_model_c_ - assert results[1].status == TestStatus.Fail - # my_datediff - assert re.search(r"1000", results[2].node.compiled_code) - # type_one_model_a_ - assert results[3].status == TestStatus.Fail - assert re.search(r"union all", results[3].node.compiled_code) - # type_two_model_a_ - assert results[4].status == TestStatus.Warn - assert results[4].node.config.severity == "WARN" - - -class TestSchemaTestContextWithMacroNamespace: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - test_utils_files = { - "dbt_project.yml": test_utils__dbt_project_yml, - "macros": { - "current_timestamp.sql": test_utils__macros__current_timestamp_sql, - "custom_test.sql": test_utils__macros__custom_test_sql, - }, - } - write_project_files(project_root, "test_utils", test_utils_files) - - local_utils_files = { - "dbt_project.yml": local_utils__dbt_project_yml, - "macros": { - "datediff.sql": local_utils__macros__datediff_sql, - "current_timestamp.sql": local_utils__macros__current_timestamp_sql, - "custom_test.sql": local_utils__macros__custom_test_sql, - }, - } - write_project_files(project_root, "local_utils", local_utils_files) - - test_context_macros_namespaced_file = { - "my_test.sql": test_context_macros_namespaced__my_test_sql, - "custom_schema_tests.sql": test_context_macros_namespaced__custom_schema_tests_sql, - } - write_project_files( - project_root, "test-context-macros-namespaced", test_context_macros_namespaced_file - ) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": test_context_models_namespaced__schema_yml, - "model_c.sql": test_context_models_namespaced__model_c_sql, - "model_b.sql": test_context_models_namespaced__model_b_sql, - "model_a.sql": test_context_models_namespaced__model_a_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["test-context-macros-namespaced"], - "dispatch": [ - { - "macro_namespace": "test_utils", - "search_order": ["local_utils", "test_utils"], - } - ], - } - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - {"local": "test_utils"}, - {"local": "local_utils"}, - ] - } - - def test_test_context_with_macro_namespace( - self, - project, - ): - # This test tests the the TestContext and TestMacroNamespace - # are working correctly - run_dbt(["deps"]) - results = run_dbt() - assert len(results) == 3 - - run_result = run_dbt(["test"], expect_pass=False) - results = run_result.results - results = sorted(results, key=lambda r: r.node.name) - assert len(results) == 4 - # call_pkg_macro_model_c_ - assert results[0].status == TestStatus.Fail - # dispatch_model_c_ - assert results[1].status == TestStatus.Fail - # type_one_model_a_ - assert results[2].status == TestStatus.Fail - assert re.search(r"union all", results[2].node.compiled_code) - # type_two_model_a_ - assert results[3].status == TestStatus.Warn - assert results[3].node.config.severity == "WARN" - - -class TestSchemaTestNameCollision: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": name_collision__schema_yml, - "base.sql": name_collision__base_sql, - "base_extension.sql": name_collision__base_extension_sql, - } - - def test_collision_test_names_get_hash( - self, - project, - ): - """The models should produce unique IDs with a has appended""" - results = run_dbt() - test_results = run_dbt(["test"]) - - # both models and both tests run - assert len(results) == 2 - assert len(test_results) == 2 - - # both tests have the same unique id except for the hash - expected_unique_ids = [ - "test.test.not_null_base_extension_id.922d83a56c", - "test.test.not_null_base_extension_id.c8d18fe069", - ] - assert test_results[0].node.unique_id in expected_unique_ids - assert test_results[1].node.unique_id in expected_unique_ids - - -class TestGenericTestsConfigCustomMacros: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": custom_generic_test_config_custom_macro__schema_yml, - "model_a.sql": custom_generic_test_config_custom_macro__model_a, - } - - def test_generic_test_config_custom_macros( - self, - project, - ): - """This test has a reference to a custom macro its configs""" - with pytest.raises(CompilationError) as exc: - run_dbt() - assert "Invalid generic test configuration" in str(exc) - - -class TestGenericTestsCustomNames: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": custom_generic_test_names__schema_yml, - "model_a.sql": custom_generic_test_names__model_a, - } - - # users can define custom names for specific instances of generic tests - def test_generic_tests_with_custom_names( - self, - project, - ): - """These tests don't collide, since they have user-provided custom names""" - results = run_dbt() - test_results = run_dbt(["test"]) - - # model + both tests run - assert len(results) == 1 - assert len(test_results) == 2 - - # custom names propagate to the unique_id - expected_unique_ids = [ - "test.test.not_null_where_1_equals_1.7b96089006", - "test.test.not_null_where_1_equals_2.8ae586e17f", - ] - assert test_results[0].node.unique_id in expected_unique_ids - assert test_results[1].node.unique_id in expected_unique_ids - - -class TestGenericTestsCustomNamesAltFormat(TestGenericTestsCustomNames): - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": custom_generic_test_names_alt_format__schema_yml, - "model_a.sql": custom_generic_test_names_alt_format__model_a, - } - - # exactly as above, just alternative format for yaml definition - def test_collision_test_names_get_hash( - self, - project, - ): - """These tests don't collide, since they have user-provided custom names, - defined using an alternative format""" - super().test_generic_tests_with_custom_names(project) - - -class TestInvalidSchema: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": invalid_schema_models__schema_yml, - "model.sql": invalid_schema_models__model_sql, - } - - def test_invalid_schema_file( - self, - project, - ): - with pytest.raises(ParsingError) as exc: - run_dbt() - assert re.search(r"'models' is not a list", str(exc)) - - -class TestCommentedSchema: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": all_quotes_schema__schema_yml, - "model.sql": invalid_schema_models__model_sql, - } - - def test_quoted_schema_file(self, project): - try: - # A schema file consisting entirely of quotes should not be a problem - run_dbt(["parse"]) - except TypeError: - assert ( - False - ), "`dbt parse` failed with a yaml file that is all comments with the same exception as 3568" - except Exception: - assert False, "`dbt parse` failed with a yaml file that is all comments" - - -class TestWrongSpecificationBlock: - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": wrong_specification_block__schema_yml} - - @pytest.fixture(scope="class") - def seeds(self): - return {"some_seed.csv": seeds__some_seed_csv} - - def test_wrong_specification_block( - self, - project, - ): - with pytest.warns(Warning): - results = run_dbt( - [ - "ls", - "-s", - "some_seed", - "--output", - "json", - "--output-keys", - "name", - "description", - ] - ) - - assert len(results) == 1 - assert results[0] == '{"name": "some_seed", "description": ""}' - - -class TestSchemaTestContextWhereSubq: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - test_context_where_subq_macros_file = { - "custom_generic_test.sql": test_context_where_subq_macros__custom_generic_test_sql - } - write_project_files( - project_root, "test-context-where-subq-macros", test_context_where_subq_macros_file - ) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": test_context_where_subq_models__schema_yml, - "model_a.sql": test_context_where_subq_models__model_a_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["test-context-where-subq-macros"], - } - - def test_test_context_tests( - self, - project, - ): - # This test tests that get_where_subquery() is included in TestContext + TestMacroNamespace, - # otherwise api.Relation.create() will return an error - results = run_dbt() - assert len(results) == 1 - - results = run_dbt(["test"]) - assert len(results) == 1 - - -class TestCustomSchemaTestMacroResolutionOrder: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - alt_local_utils_file = { - "dbt_project.yml": local_utils__dbt_project_yml, - "macros": { - "datediff.sql": alt_local_utils__macros__type_timestamp_sql, - }, - } - write_project_files(project_root, "alt_local_utils", alt_local_utils_file) - - macros_resolution_order_file = { - "my_custom_test.sql": macro_resolution_order_macros__my_custom_test_sql, - } - write_project_files( - project_root, "macro_resolution_order_macros", macros_resolution_order_file - ) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": macro_resolution_order_models__config_yml, - "my_model.sql": macro_resolution_order_models__my_model_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "macro-paths": ["macro_resolution_order_macros"], - } - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": "alt_local_utils"}]} - - def test_macro_resolution_test_namespace( - self, - project, - ): - # https://github.com/dbt-labs/dbt-core/issues/5720 - # Previously, macros called as 'dbt.some_macro' would not correctly - # resolve to 'some_macro' from the 'dbt' namespace during static analysis, - # if 'some_macro' also existed in an installed package, - # leading to the macro being missing in the TestNamespace - run_dbt(["deps"]) - run_dbt(["parse"]) From 9e4cbe01ed98d238b2fc05bd0530d4164857ee64 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 17 Oct 2024 17:53:15 -0400 Subject: [PATCH 4/4] remove duplicate tests that should exist in dbt-core --- tests/functional/logging/test_logging.py | 98 ------------------- tests/functional/logging/test_meta_logging.py | 46 --------- 2 files changed, 144 deletions(-) delete mode 100644 tests/functional/logging/test_logging.py delete mode 100644 tests/functional/logging/test_meta_logging.py diff --git a/tests/functional/logging/test_logging.py b/tests/functional/logging/test_logging.py deleted file mode 100644 index a7e226eb..00000000 --- a/tests/functional/logging/test_logging.py +++ /dev/null @@ -1,98 +0,0 @@ -import json -import os - -from dbt.events.types import InvalidOptionYAML -from dbt.tests.util import get_manifest, run_dbt, read_file -from dbt_common.events.functions import fire_event -import pytest - - -@pytest.fixture(scope="class") -def models(): - return {"my_model.sql": "select 1 as fun"} - - -# This test checks that various events contain node_info, -# which is supplied by the log_contextvars context manager -def test_basic(project, logs_dir): - results = run_dbt(["--log-format=json", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - assert "model.test.my_model" in manifest.nodes - - # get log file - log_file = read_file(logs_dir, "dbt.log") - assert log_file - node_start = False - node_finished = False - connection_reused_data = [] - for log_line in log_file.split("\n"): - # skip empty lines - if len(log_line) == 0: - continue - # The adapter logging also shows up, so skip non-json lines - if "[debug]" in log_line: - continue - log_dct = json.loads(log_line) - log_data = log_dct["data"] - log_event = log_dct["info"]["name"] - if log_event == "ConnectionReused": - connection_reused_data.append(log_data) - if log_event == "NodeStart": - node_start = True - if log_event == "NodeFinished": - node_finished = True - assert log_data["run_result"]["adapter_response"] - if node_start and not node_finished: - if log_event == "NodeExecuting": - assert "node_info" in log_data - if log_event == "JinjaLogDebug": - assert "node_info" in log_data - if log_event == "SQLQuery": - assert "node_info" in log_data - if log_event == "TimingInfoCollected": - assert "node_info" in log_data - assert "timing_info" in log_data - - # windows doesn't have the same thread/connection flow so the ConnectionReused - # events don't show up - if os.name != "nt": - # Verify the ConnectionReused event occurs and has the right data - assert connection_reused_data - for data in connection_reused_data: - assert "conn_name" in data and data["conn_name"] - assert "orig_conn_name" in data and data["orig_conn_name"] - - -def test_formatted_logs(project, logs_dir): - # a basic run of dbt with a single model should have 5 `Formatting` events in the json logs - results = run_dbt(["--log-format=json", "run"]) - assert len(results) == 1 - - # get log file - json_log_file = read_file(logs_dir, "dbt.log") - formatted_json_lines = 0 - for log_line in json_log_file.split("\n"): - # skip the empty line at the end - if len(log_line) == 0: - continue - log_dct = json.loads(log_line) - log_event = log_dct["info"]["name"] - if log_event == "Formatting": - formatted_json_lines += 1 - - assert formatted_json_lines == 5 - - -def test_invalid_event_value(project, logs_dir): - results = run_dbt(["--log-format=json", "run"]) - assert len(results) == 1 - with pytest.raises(Exception): - # This should raise because positional arguments are provided to the event - fire_event(InvalidOptionYAML("testing")) - - # Provide invalid type to "option_name" - with pytest.raises(Exception) as excinfo: - fire_event(InvalidOptionYAML(option_name=1)) - - assert str(excinfo.value) == "[InvalidOptionYAML]: Unable to parse dict {'option_name': 1}" diff --git a/tests/functional/logging/test_meta_logging.py b/tests/functional/logging/test_meta_logging.py deleted file mode 100644 index 7c535bce..00000000 --- a/tests/functional/logging/test_meta_logging.py +++ /dev/null @@ -1,46 +0,0 @@ -import json - -from dbt.tests.util import read_file, run_dbt -import pytest - - -model1 = "select 1 as fun" -model2 = '{{ config(meta={"owners": ["team1", "team2"]})}} select 1 as fun' -model3 = '{{ config(meta={"key": 1})}} select 1 as fun' - - -@pytest.fixture(scope="class") # noqa -def models(): - return {"model1.sql": model1, "model2.sql": model2, "model3.sql": model3} - - -# This test checks that various events contain node_info, -# which is supplied by the log_contextvars context manager -def test_meta(project, logs_dir): - run_dbt(["--log-format=json", "run"]) - - # get log file - log_file = read_file(logs_dir, "dbt.log") - assert log_file - - for log_line in log_file.split("\n"): - # skip empty lines - if len(log_line) == 0: - continue - # The adapter logging also shows up, so skip non-json lines - if "[debug]" in log_line: - continue - - log_dct = json.loads(log_line) - if "node_info" not in log_dct["data"]: - continue - - print(f"--- log_dct: {log_dct}") - node_info = log_dct["data"]["node_info"] - node_path = node_info["node_path"] - if node_path == "model1.sql": - assert node_info["meta"] == {} - elif node_path == "model2.sql": - assert node_info["meta"] == {"owners": ["team1", "team2"]} - elif node_path == "model3.sql": - assert node_info["meta"] == {"key": 1}