add test for steps 5/6 in overload matching

conformance/results/mypy/version.toml

@@ -1,2 +1,2 @@
 version = "mypy 1.14.1"
-test_duration = 1.8
+test_duration = 1.7

conformance/results/pyre/overloads_evaluation.toml

@@ -5,6 +5,7 @@ Does not expand enum arguments to literal variants.
 Does not expand type[A | B] to type[A] and type[B].
 Does not expand tuple arguments to possible combinations.
 Does not prefer variadic match to indeterminate-length unpacked argument.
+Does not treat multiple matches due to gradual types as ambiguous.
 """
 conformance_automated = "Fail"
 errors_diff = """
@@ -19,6 +20,7 @@ Line 145: Unexpected errors ['overloads_evaluation.py:145:4 Assert type [70]: Ex
 Line 166: Unexpected errors ['overloads_evaluation.py:166:24 Incompatible parameter type [6]: In call `expand_tuple`, for 1st positional argument, expected `Tuple[int, int]` but got `Tuple[int, Union[int, str]]`.']
 Line 167: Unexpected errors ['overloads_evaluation.py:167:4 Assert type [70]: Expected `Union[int, str]` but got `int`.']
 Line 193: Unexpected errors ['overloads_evaluation.py:193:4 Assert type [70]: Expected `int` but got `typing_extensions.Literal[0]`.']
+Line 221: Unexpected errors ['overloads_evaluation.py:221:4 Assert type [70]: Expected `typing.Any` but got `int`.']
 """
 output = """
 overloads_evaluation.py:27:0 Missing argument [20]: Call `example1` expects argument `x`.
@@ -37,4 +39,5 @@ overloads_evaluation.py:145:4 Assert type [70]: Expected `Union[int, str]` but g
 overloads_evaluation.py:166:24 Incompatible parameter type [6]: In call `expand_tuple`, for 1st positional argument, expected `Tuple[int, int]` but got `Tuple[int, Union[int, str]]`.
 overloads_evaluation.py:167:4 Assert type [70]: Expected `Union[int, str]` but got `int`.
 overloads_evaluation.py:193:4 Assert type [70]: Expected `int` but got `typing_extensions.Literal[0]`.
+overloads_evaluation.py:221:4 Assert type [70]: Expected `typing.Any` but got `int`.
 """

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 6.0
+test_duration = 6.1

conformance/results/pytype/overloads_evaluation.toml

@@ -2,6 +2,7 @@ conformant = "Partial"
 notes = """
 Does not pick a winning overload based on arity, prior to considering argument types.
 Does not perform argument expansion (on any types) when matching overloads.
+Does not treat multiple matches due to gradual types as ambiguous.
 """
 conformance_automated = "Fail"
 errors_diff = """
@@ -22,6 +23,8 @@ Line 145: Unexpected errors ['overloads_evaluation.py:145:5: \\x1b[1m\\x1b[31mer
 Line 163: Unexpected errors ['overloads_evaluation.py:163:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in expand_tuple: bad return type [bad-return-type]']
 Line 167: Unexpected errors ['overloads_evaluation.py:167:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in check_expand_tuple: int [assert-type]']
 Line 185: Unexpected errors ['overloads_evaluation.py:185:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in variadic: bad return type [bad-return-type]']
+Line 214: Unexpected errors ['overloads_evaluation.py:214:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in example4: bad return type [bad-return-type]']
+Line 221: Unexpected errors ['overloads_evaluation.py:221:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in check_example4: int [assert-type]']
 """
 output = """
 overloads_evaluation.py:23:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in example1: bad return type [bad-return-type]
@@ -126,4 +129,14 @@ overloads_evaluation.py:185:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in
     return 1
            \u001b[1m\u001b[31m~\u001b[39m\u001b[0m
 
+overloads_evaluation.py:214:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in example4: bad return type [bad-return-type]
+
+    return 1
+           \u001b[1m\u001b[31m~\u001b[39m\u001b[0m
+
+overloads_evaluation.py:221:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in check_example4: int [assert-type]
+
+    assert_type(ret2, Any)
+    \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
 """

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.10.11"
-test_duration = 31.2
+test_duration = 31.1

conformance/results/results.html

@@ -159,16 +159,16 @@ <h3>Python Type System Conformance Test Results</h3>
         <div class="table_container"><table><tbody>
 <tr><th class="col1">&nbsp;</th>
 <th class='tc-header'><div class='tc-name'>mypy 1.14.1</div>
-<div class='tc-time'>1.8sec</div>
+<div class='tc-time'>1.7sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyright 1.1.391</div>
 <div class='tc-time'>1.6sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.23</div>
-<div class='tc-time'>6.0sec</div>
+<div class='tc-time'>6.1sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pytype 2024.10.11</div>
-<div class='tc-time'>31.2sec</div>
+<div class='tc-time'>31.1sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">
@@ -693,8 +693,8 @@ <h3>Python Type System Conformance Test Results</h3>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_evaluation</th>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p><p>Does not expand boolean arguments to Literal[True] and Literal[False].</p><p>Does not expand enum arguments to literal variants.</p><p>Does not expand tuple arguments to possible combinations.</p></span></div></th>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not expand boolean arguments to Literal[True] and Literal[False].</p><p>Does not expand enum arguments to literal variants.</p><p>Does not expand tuple arguments to possible combinations.</p></span></div></th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not expand boolean arguments to Literal[True] and Literal[False].</p><p>Does not expand enum arguments to literal variants.</p><p>Does not expand type[A | B] to type[A] and type[B].</p><p>Does not expand tuple arguments to possible combinations.</p><p>Does not prefer variadic match to indeterminate-length unpacked argument.</p></span></div></th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p><p>Does not perform argument expansion (on any types) when matching overloads.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not expand boolean arguments to Literal[True] and Literal[False].</p><p>Does not expand enum arguments to literal variants.</p><p>Does not expand type[A | B] to type[A] and type[B].</p><p>Does not expand tuple arguments to possible combinations.</p><p>Does not prefer variadic match to indeterminate-length unpacked argument.</p><p>Does not treat multiple matches due to gradual types as ambiguous.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p><p>Does not perform argument expansion (on any types) when matching overloads.</p><p>Does not treat multiple matches due to gradual types as ambiguous.</p></span></div></th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_overlap</th>
 <th class="column col2 conformant">Pass</th>

conformance/tests/overloads_evaluation.py

@@ -3,7 +3,7 @@
 """
 
 from enum import Enum
-from typing import assert_type, Literal, overload
+from typing import Any, assert_type, Literal, overload
 
 
 # > Step 1: Examine the argument list to determine the number of
@@ -191,3 +191,31 @@ def variadic(*args: int) -> int | str:
 def check_variadic(v: list[int]) -> None:
     ret1 = variadic(*v)
     assert_type(ret1, int)
+
+
+# > Step 5: For each argument, determine whether all possible
+# > :term:`materializations <materialize>` of the argument's type are assignable to
+# > the corresponding parameter type for each of the remaining overloads. If so,
+# > eliminate all of the subsequent remaining overloads.
+
+@overload
+def example4(x: list[int], y: int) -> int:
+   ...
+
+@overload
+def example4(x: list[str], y: str) -> int:
+    ...
+
+@overload
+def example4(x: int, y: int) -> list[int]:
+    ...
+
+def example4(x: list[int] | list[str] | int, y: int | str) -> int | list[int]:
+    return 1
+
+def check_example4(v1: list[Any], v2: Any):
+    ret1 = example4(v1, v2)
+    assert_type(ret1, int)
+
+    ret2 = example4(v2, 1)
+    assert_type(ret2, Any)