-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Move testing related function to a dedicated lib + move unit tests (#…
- Loading branch information
1 parent
460a46b
commit b2159bf
Showing
5 changed files
with
163 additions
and
121 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| def get_tests_family_if_failing_tests(test_name_list, failing_tests: set): | ||
| """Get the parent tests of a list of tests only if the marked test is failing | ||
| For example with the test ["TestEKSSuite/TestCPU/TestCPUUtilization", "TestKindSuite/TestCPU"] | ||
| this method should return the set{"TestEKSSuite/TestCPU/TestCPUUtilization", "TestEKSSuite/TestCPU", "TestEKSSuite", "TestKindSuite/TestCPU", "TestKindSuite"} | ||
| if TestKindSuite/TestCPU and TestEKSSuite/TestCPU/TestCPUUtilization are failing | ||
| Another example, with the test ["TestEKSSuite/TestCPU/TestCPUUtilization", "TestKindSuite/TestCPU"] | ||
| if only TestKindSuite/TestCPU is failing, the method should return the set{"TestKindSuite/TestCPU", "TestKindSuite"} | ||
| Args: | ||
| test_name_list (list): List of test names to get the parent tests from | ||
| failing_tests (set): Set of tests that are failing | ||
| """ | ||
| test_name_set = set(test_name_list) | ||
| marked_tests_failing = failing_tests.intersection(test_name_set) | ||
| return get_tests_family(list(marked_tests_failing)) | ||
|
|
||
|
|
||
| def get_tests_family(test_name_list): | ||
| """Get the parent tests of a list of tests | ||
| Get the parent tests of a list of tests | ||
| For example with the test ["TestEKSSuite/TestCPU/TestCPUUtilization", "TestKindSuite/TestCPU"] | ||
| this method should return the set{"TestEKSSuite/TestCPU/TestCPUUtilization", "TestEKSSuite/TestCPU", "TestEKSSuite", "TestKindSuite/TestCPU", "TestKindSuite"} | ||
| Args: | ||
| test_name_list (list): List of test names to get the parent tests from | ||
| """ | ||
| test_family = set(test_name_list) | ||
| for test_name in test_name_list: | ||
| while test_name.count('/') > 0: | ||
| test_name = test_name.rsplit('/', 1)[0] | ||
| test_family.add(test_name) | ||
| return test_family | ||
|
|
||
|
|
||
| def is_known_flaky_test(failing_test, known_flaky_tests, known_flaky_tests_parents): | ||
| """Check if a test is known to be flaky | ||
| If a test is a parent of a test that is known to be flaky, the test should be considered flaky | ||
| For example: | ||
| - if TestEKSSuite/TestCPU is known to be flaky, TestEKSSuite/TestCPU/TestCPUUtilization should be considered flaky | ||
| - if TestEKSSuite/TestCPU is known to be flaky, TestEKSSuite should be considered flaky unless TestEKSSuite/TestCPU is not failing | ||
| - if TestEKSSuite/TestCPU is known to be flaky, TestEKSSuite/TestMemory should not be considered flaky | ||
| Args: | ||
| failing_test (str): The test that is failing | ||
| known_flaky_tests (set): Set of tests that are known to be flaky | ||
| known_flaky_tests_parents (set): Set of tests that are parent of a test that is known to be flaky | ||
| """ | ||
|
|
||
| failing_test_parents = get_tests_family([failing_test]) | ||
|
|
||
| if any(parent in known_flaky_tests for parent in failing_test_parents): | ||
| return True | ||
|
|
||
| return failing_test in known_flaky_tests_parents |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| import unittest | ||
|
|
||
| from tasks.libs.testing.flakes import get_tests_family, get_tests_family_if_failing_tests, is_known_flaky_test | ||
|
|
||
|
|
||
| class TestGetTestParents(unittest.TestCase): | ||
| def test_get_tests_parents(self): | ||
| parents = get_tests_family(["TestEKSSuite/TestCPU/TestCPUUtilization", "TestKindSuite/TestKind"]) | ||
| self.assertEqual( | ||
| parents, | ||
| { | ||
| "TestEKSSuite", | ||
| "TestEKSSuite/TestCPU", | ||
| "TestEKSSuite/TestCPU/TestCPUUtilization", | ||
| "TestKindSuite", | ||
| "TestKindSuite/TestKind", | ||
| }, | ||
| ) | ||
|
|
||
| def test_get_test_parents_empty(self): | ||
| parents = get_tests_family([]) | ||
| self.assertEqual( | ||
| parents, | ||
| set(), | ||
| ) | ||
|
|
||
| def test_get_test_parents_failing_no_failing_tests(self): | ||
| parents = get_tests_family_if_failing_tests(["TestEKSSuite/TestCPU/TestCPUUtilization"], set()) | ||
| self.assertEqual( | ||
| parents, | ||
| set(), | ||
| ) | ||
|
|
||
| def test_get_test_parents_failing_all_failing_tests(self): | ||
| parents = get_tests_family_if_failing_tests( | ||
| ["TestEKSSuite/TestCPU/TestCPUUtilization", "TestKindSuite/TestCPU"], | ||
| {"TestKindSuite/TestCPU", "TestEKSSuite/TestCPU/TestCPUUtilization"}, | ||
| ) | ||
| self.assertEqual( | ||
| parents, | ||
| { | ||
| "TestEKSSuite", | ||
| "TestEKSSuite/TestCPU", | ||
| "TestEKSSuite/TestCPU/TestCPUUtilization", | ||
| "TestKindSuite", | ||
| "TestKindSuite/TestCPU", | ||
| }, | ||
| ) | ||
|
|
||
| def test_get_test_parents_failing_some_failing_tests(self): | ||
| parents = get_tests_family_if_failing_tests( | ||
| ["TestEKSSuite/TestCPU/TestCPUUtilization", "TestKindSuite/TestCPU"], {"TestKindSuite/TestCPU"} | ||
| ) | ||
| self.assertEqual( | ||
| parents, | ||
| { | ||
| "TestKindSuite", | ||
| "TestKindSuite/TestCPU", | ||
| }, | ||
| ) | ||
|
|
||
|
|
||
| class TestIsKnownFlake(unittest.TestCase): | ||
| def test_known_flake(self): | ||
| is_known_flaky = is_known_flaky_test( | ||
| "TestEKSSuite/mario", {"TestEKSSuite/mario"}, {"TestEKSSuite", "TestEKSSuite/mario"} | ||
| ) | ||
| self.assertTrue(is_known_flaky) | ||
|
|
||
| def test_known_flake_parent_failing(self): | ||
| is_known_flaky = is_known_flaky_test( | ||
| "TestEKSSuite", {"TestEKSSuite/mario"}, {"TestEKSSuite", "TestEKSSuite/mario"} | ||
| ) | ||
| self.assertTrue(is_known_flaky) | ||
|
|
||
| def test_known_flake_parent_failing_2(self): | ||
| is_known_flaky = is_known_flaky_test( | ||
| "TestEKSSuite/mario", | ||
| {"TestEKSSuite/mario/luigi"}, | ||
| {"TestEKSSuite", "TestEKSSuite/mario", "TestEKSSuite/mario/luigi"}, | ||
| ) | ||
| self.assertTrue(is_known_flaky) | ||
|
|
||
| def test_not_known_flake(self): | ||
| is_known_flaky = is_known_flaky_test( | ||
| "TestEKSSuite/luigi", {"TestEKSSuite/mario"}, {"TestEKSSuite", "TestEKSSuite/mario"} | ||
| ) | ||
| self.assertFalse(is_known_flaky) | ||
|
|
||
| def test_not_known_flake_ambiguous_start(self): | ||
| is_known_flaky = is_known_flaky_test("TestEKSSuiteVM/mario", {"TestEKSSuite/mario"}, {"TestEKSSuite"}) | ||
| self.assertFalse(is_known_flaky) | ||
|
|
||
| def test_not_known_flake_ambiguous_start_2(self): | ||
| is_known_flaky = is_known_flaky_test("TestEKSSuite/mario", {"TestEKSSuiteVM/mario"}, {"TestEKSSuiteVM"}) | ||
| self.assertFalse(is_known_flaky) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters