From 206e90773115d3ee00ada21493c13177235a9ddd Mon Sep 17 00:00:00 2001 From: swu2 Date: Tue, 15 Oct 2024 12:52:19 -0400 Subject: [PATCH] update test cases --- tests/test_binned_dud.py | 5 +- tests/test_extrinsic_generator.py | 89 +++++++++++++++++++++++++++++++ tests/test_multinomial_dud.py | 5 +- tests/test_neyman.py | 3 +- 4 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 tests/test_extrinsic_generator.py diff --git a/tests/test_binned_dud.py b/tests/test_binned_dud.py index 2e6cf58..2737644 100644 --- a/tests/test_binned_dud.py +++ b/tests/test_binned_dud.py @@ -20,8 +20,9 @@ def test_binned_dud_density(self): self.assertAlmostEqual(dud.get_density(label), density, places=3, - msg=f'Test failed: density of label "{label}" does not match expected result') + msg=f'Test failed: density of label "{label}" does not match expected result' + ) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_extrinsic_generator.py b/tests/test_extrinsic_generator.py new file mode 100644 index 0000000..d43868d --- /dev/null +++ b/tests/test_extrinsic_generator.py @@ -0,0 +1,89 @@ +import unittest +import numpy as np +from scipy.stats import beta + +from pyquantifier.distributions import MixtureCUD +from pyquantifier.calibration_curve import PlattScaling +from pyquantifier.data import Item, Dataset + +test_case_list = [ + (MixtureCUD(components=[beta(10, 2), beta(2, 5)], weights=[2, 8]), # classifer_score_density_rv + (21.92, -14.61), # (w, b) + 1000000, # num_base_size + 1000000, # num_sample + 10), # num_bin + (MixtureCUD(components=[beta(10, 2), beta(2, 5)], weights=[2, 8]), + (10, -5), + 1000000, + 1000000, + 10) + ] + + +class TestExtrinsicGenerator(unittest.TestCase): + def setUp(self): + pass + + def test_extrinsic_generator(self): + for case_num, test_case in enumerate(test_case_list): + classifer_score_density_rv, (w, b), num_base_size, num_sample, num_bin = test_case + + # 1. Generate a dataset with `num_sample` items, + # which pos scores follow the `classifer_score_density` distribution + all_labels = ['pos', 'neg'] + simulated_pos_scores = classifer_score_density_rv.generate_data(num_base_size) + + calibration_curve = PlattScaling() + calibration_curve.set_params(w, b) + + items = [] + gt_label_dict = {} + for idx, pos_score in enumerate(simulated_pos_scores): + item = Item(uid=idx+1, all_labels=all_labels, all_probs=[pos_score, 1-pos_score]) + items.append(item) + + calibrated_pos_score = calibration_curve.get_calibrated_prob(pos_score)[0] + gt_label_dict[item.uid] = np.random.choice(all_labels, p=[calibrated_pos_score, 1-calibrated_pos_score]) + # print(gt_label_dict) + + dataset = Dataset(items=items) + + # 2. Select a subset of the dataset and annotate the labels + selected_dataset, selection_weights = dataset.select_sample_for_annotation( + n=num_sample, + strategy='random', + bins=num_bin + ) + annotated_labels = [gt_label_dict[uid] for uid in selected_dataset.df['uid'].values.astype(int)] + selected_dataset.annotate_sample(annotated_labels) + + # 3. Model the sampled dataset and retrieve the estimated w and b + selected_dataset.update_dataset_model(num_bin=num_bin) + selected_dataset.update_calibration_curve(method='platt scaling') + estimated_w, estimated_b = selected_dataset.calibration_curve.get_params() + estimated_w = estimated_w.item() + estimated_b = estimated_b.item() + + # 4. Check if the estimated w and b are close to the ground truth synthetic w and b + with self.subTest(msg=f'Checking case number {case_num+1}'): + print(f'\nestimated w: {estimated_w:.4f}') + print(f'estimated b: {estimated_b:.4f}') + self.assertAlmostEqual(estimated_w, + w, + delta=0.2, + msg=f'Test failed: estimated w does not match expected result' + ) + self.assertAlmostEqual(estimated_b, + b, + delta=0.2, + msg=f'Test failed: estimated b does not match expected result' + ) + + def test_extrinsic_generator2(self): + with self.subTest(msg=f'Checking case 2'): + pass + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_multinomial_dud.py b/tests/test_multinomial_dud.py index 1b68c5f..0d53d79 100644 --- a/tests/test_multinomial_dud.py +++ b/tests/test_multinomial_dud.py @@ -20,8 +20,9 @@ def test_multinomial_dud_density(self): for label, density in expected_result.items(): self.assertEqual(dud.get_density(label), density, - msg=f'Test failed: density of label "{label}" does not match expected result') + msg=f'Test failed: density of label "{label}" does not match expected result' + ) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_neyman.py b/tests/test_neyman.py index becd5c1..43ee1a1 100644 --- a/tests/test_neyman.py +++ b/tests/test_neyman.py @@ -25,7 +25,8 @@ def test_get_neyman_allocation(self): with self.subTest(msg=f'Checking case number {case_num+1}'): self.assertDictEqual(result, expected_dist, - msg='Test failed: output of Neyman allocation does not match expected result') + msg='Test failed: output of Neyman allocation does not match expected result' + ) if __name__ == "__main__":