added option for setting the significance level

Digital-Dermatology · Aug 27, 2024 · 4f55e4e · 4f55e4e
1 parent fd7c60e
commit 4f55e4e
Show file tree

Hide file tree

Showing 6 changed files with 141 additions and 126 deletions.
diff --git a/examples/Investigate_Imagenette.ipynb b/examples/Investigate_Imagenette.ipynb
diff --git a/examples/Investigate_OxfordIIITPet.ipynb b/examples/Investigate_OxfordIIITPet.ipynb
diff --git a/setup.py b/setup.py
@@ -31,7 +31,7 @@ def parse_requirements(filename):
     name=PACKAGE_NAME,
     packages=proj_packages,
     package_dir={PACKAGE_NAME: SOURCE_DIRECTORY},
-    version="0.0.25",
+    version="0.0.26",
     author="Fabian Groeger",
     author_email="fabian.groeger@unibas.ch",
     description="A holistic self-supervised data cleaning strategy to detect irrelevant samples, near duplicates and label errors.",

diff --git a/src/cleaner/auto_cleaning_mixin.py b/src/cleaner/auto_cleaning_mixin.py
@@ -22,6 +22,7 @@ def __init__(
         irrelevant_cut_off: float = 0.01,
         near_duplicate_cut_off: float = 0.01,
         label_error_cut_off: float = 0.01,
+        significance_level: float = 0.05,
         cleaner_kwargs: dict = {},
         **kwargs,
     ):
@@ -30,6 +31,7 @@ def __init__(
         self.irrelevant_cut_off = irrelevant_cut_off
         self.near_duplicate_cut_off = near_duplicate_cut_off
         self.label_error_cut_off = label_error_cut_off
+        self.significance_level = significance_level
         self.cleaner_kwargs = cleaner_kwargs
 
     def perform_auto_cleaning(
@@ -39,6 +41,9 @@ def perform_auto_cleaning(
         output_path: Optional[Union[str, Path]] = None,
     ):
         if self.auto_cleaning:
+            # make sure the significance level is correctly set
+            self.cleaner_kwargs["q"] = self.significance_level
+
             # Near Duplicates
             near_duplicate_issues = issue_manger["near_duplicates"]
             if near_duplicate_issues is not None:

diff --git a/tests/unittests/cleaner/test_auto_cleaning.py b/tests/unittests/cleaner/test_auto_cleaning.py
@@ -73,6 +73,7 @@ def test_predict_auto_cleaning_diff_cut_off(self):
             irrelevant_cut_off=0.01,
             near_duplicate_cut_off=0.01,
             label_error_cut_off=0.01,
+            significance_level=0.01,
         )
         cleaner.fit(emb_space=self.emb_space, labels=self.labels)
         out_dict = cleaner.predict()

diff --git a/tests/unittests/cleaner/test_selfclean_cleaner.py b/tests/unittests/cleaner/test_selfclean_cleaner.py
@@ -13,7 +13,7 @@ def setUp(self):
         self.emb_space = np.random.rand(50, 198)
         self.labels = np.random.randint(5, size=50)
         self.class_labels = [f"test_{x}" for x in np.unique(self.labels)]
-        self.memory_profiling = True
+        self.memory_profiling = False
 
     def test_fit(self):
         cleaner = SelfCleanCleaner(memmap=False)