Unit tests for KedroDataCatalog (#4171)

* Added KedroDataCatlog tests template Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added test save/load unregistered dataset Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added test_feed_dict Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added exists tests Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added tests for list() Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added test_eq Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added test init/add datasets Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Updated test_adding_datasets_not_allowed Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added shallow copy tests Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added TestKedroDataCatalogFromConfig Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> * Added missing tests Signed-off-by: Elena Khaustova <ymax70rus@gmail.com> --------- Signed-off-by: Elena Khaustova <ymax70rus@gmail.com>
kedro-org · Sep 17, 2024 · 8f870a8 · 8f870a8
1 parent 4b8d90c
commit 8f870a8
Show file tree

Hide file tree

Showing 3 changed files with 755 additions and 70 deletions.
diff --git a/tests/io/conftest.py b/tests/io/conftest.py
@@ -1,6 +1,7 @@
 import numpy as np
 import pandas as pd
 import pytest
+from kedro_datasets.pandas import CSVDataset
 
 
 @pytest.fixture
@@ -21,3 +22,68 @@ def input_data(request):
 @pytest.fixture
 def new_data():
     return pd.DataFrame({"col1": ["a", "b"], "col2": ["c", "d"], "col3": ["e", "f"]})
+
+
+@pytest.fixture
+def filepath(tmp_path):
+    return (tmp_path / "some" / "dir" / "test.csv").as_posix()
+
+
+@pytest.fixture
+def dataset(filepath):
+    return CSVDataset(filepath=filepath, save_args={"index": False})
+
+
+@pytest.fixture
+def sane_config(filepath):
+    return {
+        "catalog": {
+            "boats": {"type": "pandas.CSVDataset", "filepath": filepath},
+            "cars": {
+                "type": "pandas.CSVDataset",
+                "filepath": "s3://test_bucket/test_file.csv",
+                "credentials": "s3_credentials",
+            },
+        },
+        "credentials": {
+            "s3_credentials": {"key": "FAKE_ACCESS_KEY", "secret": "FAKE_SECRET_KEY"}
+        },
+    }
+
+
+@pytest.fixture
+def sane_config_with_nested_creds(sane_config):
+    sane_config["catalog"]["cars"]["credentials"] = {
+        "client_kwargs": {"credentials": "other_credentials"},
+        "key": "secret",
+    }
+    sane_config["credentials"]["other_credentials"] = {
+        "client_kwargs": {
+            "aws_access_key_id": "OTHER_FAKE_ACCESS_KEY",
+            "aws_secret_access_key": "OTHER_FAKE_SECRET_KEY",
+        }
+    }
+    return sane_config
+
+
+@pytest.fixture
+def bad_config(filepath):
+    return {
+        "bad": {"type": "tests.io.test_data_catalog.BadDataset", "filepath": filepath}
+    }
+
+
+@pytest.fixture
+def sane_config_with_tracking_ds(tmp_path):
+    boat_path = (tmp_path / "some" / "dir" / "test.csv").as_posix()
+    plane_path = (tmp_path / "some" / "dir" / "metrics.json").as_posix()
+    return {
+        "catalog": {
+            "boats": {
+                "type": "pandas.CSVDataset",
+                "filepath": boat_path,
+                "versioned": True,
+            },
+            "planes": {"type": "tracking.MetricsDataset", "filepath": plane_path},
+        },
+    }
diff --git a/tests/io/test_data_catalog.py b/tests/io/test_data_catalog.py
@@ -29,64 +29,6 @@
 )
 
 
-@pytest.fixture
-def filepath(tmp_path):
-    return (tmp_path / "some" / "dir" / "test.csv").as_posix()
-
-
-@pytest.fixture
-def dummy_dataframe():
-    return pd.DataFrame({"col1": [1, 2], "col2": [4, 5], "col3": [5, 6]})
-
-
-@pytest.fixture
-def sane_config(filepath):
-    return {
-        "catalog": {
-            "boats": {"type": "pandas.CSVDataset", "filepath": filepath},
-            "cars": {
-                "type": "pandas.CSVDataset",
-                "filepath": "s3://test_bucket/test_file.csv",
-                "credentials": "s3_credentials",
-            },
-        },
-        "credentials": {
-            "s3_credentials": {"key": "FAKE_ACCESS_KEY", "secret": "FAKE_SECRET_KEY"}
-        },
-    }
-
-
-@pytest.fixture
-def sane_config_with_nested_creds(sane_config):
-    sane_config["catalog"]["cars"]["credentials"] = {
-        "client_kwargs": {"credentials": "other_credentials"},
-        "key": "secret",
-    }
-    sane_config["credentials"]["other_credentials"] = {
-        "client_kwargs": {
-            "aws_access_key_id": "OTHER_FAKE_ACCESS_KEY",
-            "aws_secret_access_key": "OTHER_FAKE_SECRET_KEY",
-        }
-    }
-    return sane_config
-
-
-@pytest.fixture
-def sane_config_with_tracking_ds(tmp_path):
-    boat_path = (tmp_path / "some" / "dir" / "test.csv").as_posix()
-    plane_path = (tmp_path / "some" / "dir" / "metrics.json").as_posix()
-    return {
-        "catalog": {
-            "boats": {
-                "type": "pandas.CSVDataset",
-                "filepath": boat_path,
-                "versioned": True,
-            },
-            "planes": {"type": "tracking.MetricsDataset", "filepath": plane_path},
-        },
-    }
-
-
 @pytest.fixture
 def config_with_dataset_factories():
     return {
@@ -180,11 +122,6 @@ def config_with_dataset_factories_only_patterns_no_default(
     return config_with_dataset_factories_only_patterns
 
 
-@pytest.fixture
-def dataset(filepath):
-    return CSVDataset(filepath=filepath, save_args={"index": False})
-
-
 @pytest.fixture
 def multi_catalog():
     csv = CSVDataset(filepath="abc.csv")
@@ -220,13 +157,6 @@ def _describe(self):
         return {}
 
 
-@pytest.fixture
-def bad_config(filepath):
-    return {
-        "bad": {"type": "tests.io.test_data_catalog.BadDataset", "filepath": filepath}
-    }
-
-
 @pytest.fixture
 def data_catalog(dataset):
     return DataCatalog(datasets={"test": dataset})