From 7d1abe9502641a3602e9773aebc29ee56d8f40ae Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 12 Mar 2024 20:11:58 -0400 Subject: [PATCH 01/18] gh-116682: stdout may be empty in test_cancel_futures_wait_false (#116683) If the `shutdown()` call happens before the worker thread starts executing the task, then nothing will be printed to stdout. --- Lib/test/test_concurrent_futures/test_shutdown.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 45dab7a75fdd50..7a4065afd46fc8 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -247,7 +247,9 @@ def test_cancel_futures_wait_false(self): # Errors in atexit hooks don't change the process exit code, check # stderr manually. self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") + # gh-116682: stdout may be empty if shutdown happens before task + # starts executing. + self.assertIn(out.strip(), [b"apple", b""]) class ProcessPoolShutdownTest(ExecutorShutdownTest): From bf121d6a694bea4fe9864a19879fe0c70c4e0656 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Tue, 12 Mar 2024 19:59:42 -0500 Subject: [PATCH 02/18] GH-116554: Relax list.sort()'s notion of "descending" runs (#116578) * GH-116554: Relax list.sort()'s notion of "descending" run Rewrote `count_run()` so that sub-runs of equal elements no longer end a descending run. Both ascending and descending runs can have arbitrarily many sub-runs of arbitrarily many equal elements now. This is tricky, because we only use ``<`` comparisons, so checking for equality doesn't come "for free". Surprisingly, it turned out there's a very cheap (one comparison) way to determine whether an ascending run consisted of all-equal elements. That sealed the deal. In addition, after a descending run is reversed in-place, we now go on to see whether it can be extended by an ascending run that just happens to be adjacent. This succeeds in finding at least one additional element to append about half the time, and so appears to more than repay its cost (the savings come from getting to skip a binary search, when a short run is artificially forced to length MIINRUN later, for each new element `count_run()` can add to the initial run). While these have been in the back of my mind for years, a question on StackOverflow pushed it to action: https://stackoverflow.com/questions/78108792/ They were wondering why it took about 4x longer to sort a list like: [999_999, 999_999, ..., 2, 2, 1, 1, 0, 0] than "similar" lists. Of course that runs very much faster after this patch. Co-authored-by: Alex Waygood Co-authored-by: Pieter Eendebak --- Lib/test/test_sort.py | 21 +++ ...-03-11-00-45-39.gh-issue-116554.gYumG5.rst | 1 + Objects/listobject.c | 157 ++++++++++++------ Objects/listsort.txt | 43 +++-- 4 files changed, 156 insertions(+), 66 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index 3b6ad4d17b0416..2a7cfb7affaa21 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -128,6 +128,27 @@ def bad_key(x): x = [e for e, i in augmented] # a stable sort of s check("stability", x, s) + def test_small_stability(self): + from itertools import product + from operator import itemgetter + + # Exhaustively test stability across all lists of small lengths + # and only a few distinct elements. + # This can provoke edge cases that randomization is unlikely to find. + # But it can grow very expensive quickly, so don't overdo it. + NELTS = 3 + MAXSIZE = 9 + + pick0 = itemgetter(0) + for length in range(MAXSIZE + 1): + # There are NELTS ** length distinct lists. + for t in product(range(NELTS), repeat=length): + xs = list(zip(t, range(length))) + # Stability forced by index in each element. + forced = sorted(xs) + # Use key= to hide the index from compares. + native = sorted(xs, key=pick0) + self.assertEqual(forced, native) #============================================================================== class TestBugs(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst new file mode 100644 index 00000000000000..82f92789de0a39 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst @@ -0,0 +1 @@ +``list.sort()`` now exploits more cases of partial ordering, particularly those with long descending runs with sub-runs of equal values. Those are recognized as single runs now (previously, each block of repeated values caused a new run to be created). diff --git a/Objects/listobject.c b/Objects/listobject.c index 759902c06b4ef3..7179d5929e2148 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1618,10 +1618,11 @@ struct s_MergeState { /* binarysort is the best method for sorting small arrays: it does few compares, but can do data movement quadratic in the number of elements. - [lo, hi) is a contiguous slice of a list, and is sorted via + [lo.keys, hi) is a contiguous slice of a list of keys, and is sorted via binary insertion. This sort is stable. - On entry, must have lo <= start <= hi, and that [lo, start) is already - sorted (pass start == lo if you don't know!). + On entry, must have lo.keys <= start <= hi, and that + [lo.keys, start) is already sorted (pass start == lo.keys if you don't + know!). If islt() complains return -1, else 0. Even in case of error, the output slice will be some permutation of the input (nothing is lost or duplicated). @@ -1634,7 +1635,7 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) PyObject *pivot; assert(lo.keys <= start && start <= hi); - /* assert [lo, start) is sorted */ + /* assert [lo.keys, start) is sorted */ if (lo.keys == start) ++start; for (; start < hi; ++start) { @@ -1643,9 +1644,9 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) r = start; pivot = *r; /* Invariants: - * pivot >= all in [lo, l). + * pivot >= all in [lo.keys, l). * pivot < all in [r, start). - * The second is vacuously true at the start. + * These are vacuously true at the start. */ assert(l < r); do { @@ -1656,7 +1657,7 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) l = p+1; } while (l < r); assert(l == r); - /* The invariants still hold, so pivot >= all in [lo, l) and + /* The invariants still hold, so pivot >= all in [lo.keys, l) and pivot < all in [l, start), so pivot belongs at l. Note that if there are elements equal to pivot, l points to the first slot after them -- that's why this sort is stable. @@ -1671,7 +1672,7 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) p = start + offset; pivot = *p; l += offset; - for (p = start + offset; p > l; --p) + for ( ; p > l; --p) *p = *(p-1); *l = pivot; } @@ -1682,56 +1683,115 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) return -1; } -/* -Return the length of the run beginning at lo, in the slice [lo, hi). lo < hi -is required on entry. "A run" is the longest ascending sequence, with - - lo[0] <= lo[1] <= lo[2] <= ... - -or the longest descending sequence, with - - lo[0] > lo[1] > lo[2] > ... +static void +sortslice_reverse(sortslice *s, Py_ssize_t n) +{ + reverse_slice(s->keys, &s->keys[n]); + if (s->values != NULL) + reverse_slice(s->values, &s->values[n]); +} -Boolean *descending is set to 0 in the former case, or to 1 in the latter. -For its intended use in a stable mergesort, the strictness of the defn of -"descending" is needed so that the caller can safely reverse a descending -sequence without violating stability (strict > ensures there are no equal -elements to get out of order). +/* +Return the length of the run beginning at slo->keys, spanning no more than +nremaining elements. The run beginning there may be ascending or descending, +but the function permutes it in place, if needed, so that it's always ascending +upon return. Returns -1 in case of error. */ static Py_ssize_t -count_run(MergeState *ms, PyObject **lo, PyObject **hi, int *descending) +count_run(MergeState *ms, sortslice *slo, Py_ssize_t nremaining) { - Py_ssize_t k; + Py_ssize_t k; /* used by IFLT macro expansion */ Py_ssize_t n; + PyObject ** const lo = slo->keys; - assert(lo < hi); - *descending = 0; - ++lo; - if (lo == hi) - return 1; - - n = 2; - IFLT(*lo, *(lo-1)) { - *descending = 1; - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) - ; - else - break; - } + /* In general, as things go on we've established that the slice starts + with a monotone run of n elements, starting at lo. */ + + /* We're n elements into the slice, and the most recent neq+1 elments are + * all equal. This reverses them in-place, and resets neq for reuse. + */ +#define REVERSE_LAST_NEQ \ + if (neq) { \ + sortslice slice = *slo; \ + ++neq; \ + sortslice_advance(&slice, n - neq); \ + sortslice_reverse(&slice, neq); \ + neq = 0; \ + } + + /* Sticking to only __lt__ compares is confusing and error-prone. But in + * this routine, almost all uses of IFLT can be captured by tiny macros + * giving mnemonic names to the intent. Note that inline functions don't + * work for this (IFLT expands to code including `goto fail`). + */ +#define IF_NEXT_LARGER IFLT(lo[n-1], lo[n]) +#define IF_NEXT_SMALLER IFLT(lo[n], lo[n-1]) + + assert(nremaining); + /* try ascending run first */ + for (n = 1; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; } - else { - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) + if (n == nremaining) + return n; + /* lo[n] is strictly less */ + /* If n is 1 now, then the first compare established it's a descending + * run, so fall through to the descending case. But if n > 1, there are + * n elements in an ascending run terminated by the strictly less lo[n]. + * If the first key < lo[n-1], *somewhere* along the way the sequence + * increased, so we're done (there is no descending run). + * Else first key >= lo[n-1], which implies that the entire ascending run + * consists of equal elements. In that case, this is a descending run, + * and we reverse the all-equal prefix in-place. + */ + if (n > 1) { + IFLT(lo[0], lo[n-1]) + return n; + sortslice_reverse(slo, n); + } + ++n; /* in all cases it's been established that lo[n] has been resolved */ + + /* Finish descending run. All-squal subruns are reversed in-place on the + * fly. Their original order will be restored at the end by the whole-slice + * reversal. + */ + Py_ssize_t neq = 0; + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER { + /* This ends the most recent run of equal elments, but still in + * the "descending" direction. + */ + REVERSE_LAST_NEQ + } + else { + IF_NEXT_LARGER /* descending run is over */ break; + else /* not x < y and not y < x implies x == y */ + ++neq; } } + REVERSE_LAST_NEQ + sortslice_reverse(slo, n); /* transform to ascending run */ + + /* And after reversing, it's possible this can be extended by a + * naturally increasing suffix; e.g., [3, 2, 3, 4, 1] makes an + * ascending run from the first 4 elements. + */ + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; + } return n; fail: return -1; + +#undef REVERSE_LAST_NEQ +#undef IF_NEXT_SMALLER +#undef IF_NEXT_LARGER } /* @@ -2449,14 +2509,6 @@ merge_compute_minrun(Py_ssize_t n) return n + r; } -static void -reverse_sortslice(sortslice *s, Py_ssize_t n) -{ - reverse_slice(s->keys, &s->keys[n]); - if (s->values != NULL) - reverse_slice(s->values, &s->values[n]); -} - /* Here we define custom comparison functions to optimize for the cases one commonly * encounters in practice: homogeneous lists, often of one of the basic types. */ @@ -2824,15 +2876,12 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) */ minrun = merge_compute_minrun(nremaining); do { - int descending; Py_ssize_t n; /* Identify next run. */ - n = count_run(&ms, lo.keys, lo.keys + nremaining, &descending); + n = count_run(&ms, &lo, nremaining); if (n < 0) goto fail; - if (descending) - reverse_sortslice(&lo, n); /* If short, extend to min(minrun, nremaining). */ if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 32a59e510f0754..4f84e2c87da7f1 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -212,24 +212,43 @@ A detailed description of timsort follows. Runs ---- -count_run() returns the # of elements in the next run. A run is either -"ascending", which means non-decreasing: +count_run() returns the # of elements in the next run, and, if it's a +descending run, reverses it in-place. A run is either "ascending", which +means non-decreasing: a0 <= a1 <= a2 <= ... -or "descending", which means strictly decreasing: +or "descending", which means non-increasing: - a0 > a1 > a2 > ... + a0 >= a1 >= a2 >= ... Note that a run is always at least 2 long, unless we start at the array's -last element. - -The definition of descending is strict, because the main routine reverses -a descending run in-place, transforming a descending run into an ascending -run. Reversal is done via the obvious fast "swap elements starting at each -end, and converge at the middle" method, and that can violate stability if -the slice contains any equal elements. Using a strict definition of -descending ensures that a descending run contains distinct elements. +last element. If all elements in the array are equal, it can be viewed as +both ascending and descending. Upon return, the run count_run() identifies +is always ascending. + +Reversal is done via the obvious fast "swap elements starting at each +end, and converge at the middle" method. That can violate stability if +the slice contains any equal elements. For that reason, for a long time +the code used strict inequality (">" rather than ">=") in its definition +of descending. + +Removing that restriction required some complication: when processing a +descending run, all-equal sub-runs of elements are reversed in-place, on the +fly. Their original relative order is restored "by magic" via the final +"reverse the entire run" step. + +This makes processing descending runs a little more costly. We only use +`__lt__` comparisons, so that `x == y` has to be deduced from +`not x < y and not y < x`. But so long as a run remains strictly decreasing, +only one of those compares needs to be done per loop iteration. So the primsry +extra cost is paid only when there are equal elements, and they get some +compensating benefit by not needing to end the descending run. + +There's one more trick added since the original: after reversing a descending +run, it's possible that it can be extended by an adjacent ascending run. For +example, given [3, 2, 1, 3, 4, 5, 0], the 3-element descending prefix is +reversed in-place, and then extended by [3, 4, 5]. If an array is random, it's very unlikely we'll see long runs. If a natural run contains less than minrun elements (see next section), the main loop From 93a687a3731caf9b27be0cd021b0337010441366 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 12 Mar 2024 21:33:42 -0500 Subject: [PATCH 03/18] Minor clarity improvement for the iter_index() recipe. Also add value subsequence tests. (gh-116696) --- Doc/library/itertools.rst | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index debb4138888e99..36e555d37d45de 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -902,18 +902,19 @@ which incur interpreter overhead. # iter_index('AABCADEAF', 'A') --> 0 1 4 7 seq_index = getattr(iterable, 'index', None) if seq_index is None: - # Slow path for general iterables + # Path for general iterables it = islice(iterable, start, stop) for i, element in enumerate(it, start): if element is value or element == value: yield i else: - # Fast path for sequences + # Path for sequences with an index() method stop = len(iterable) if stop is None else stop - i = start - 1 + i = start try: while True: - yield (i := seq_index(value, i+1, stop)) + yield (i := seq_index(value, i, stop)) + i += 1 except ValueError: pass @@ -1412,6 +1413,22 @@ The following recipes have a more mathematical flavor: >>> ''.join(input_iterator) 'DEAF' + >>> # Verify that the target value can be a sequence. + >>> seq = [[10, 20], [30, 40], 30, 40, [30, 40], 50] + >>> target = [30, 40] + >>> list(iter_index(seq, target)) + [1, 4] + + >>> # Verify faithfulness to type specific index() method behaviors. + >>> # For example, bytes and str perform subsequence searches + >>> # that do not match the general behavior specified + >>> # in collections.abc.Sequence.index(). + >>> seq = 'abracadabra' + >>> target = 'ab' + >>> list(iter_index(seq, target)) + [0, 7] + + >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] From 3f1b6efee95c06f8912bcea4031afacdbc0d5684 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 13 Mar 2024 05:19:33 +0100 Subject: [PATCH 04/18] Docs: fix broken links (#116651) --- Doc/faq/extending.rst | 2 +- Doc/library/math.rst | 2 +- Doc/library/pathlib.rst | 2 +- Doc/library/sqlite3.rst | 2 +- Doc/library/venv.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2a8b976925d042..1cff2c4091df06 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -50,7 +50,7 @@ to learn Python's C API. If you need to interface to some C or C++ library for which no Python extension currently exists, you can try wrapping the library's data types and functions with a tool such as `SWIG `_. `SIP -`__, `CXX +`__, `CXX `_ `Boost `_, or `Weave `_ are also diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 3c850317f60858..93755be717e2ef 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -592,7 +592,7 @@ Special functions The :func:`erf` function can be used to compute traditional statistical functions such as the `cumulative standard normal distribution - `_:: + `_:: def phi(x): 'Cumulative distribution function for the standard normal distribution' diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4b461a5d4a2949..9041f37bd668fa 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -7,7 +7,7 @@ .. versionadded:: 3.4 -**Source code:** :source:`Lib/pathlib.py` +**Source code:** :source:`Lib/pathlib/` .. index:: single: path; operations diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 87d5ef1e42ca3a..e76dc91bf2d875 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1135,7 +1135,7 @@ Connection objects .. versionchanged:: 3.12 Added the *entrypoint* parameter. - .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ + .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension .. method:: iterdump(*, filter=None) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 2e7ff345a06234..a4273f97b7a8db 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -54,7 +54,7 @@ See :pep:`405` for more background on Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - `__ + `__ .. include:: ../includes/wasm-notavail.rst From 43986f55671ba2f7b08f8c5cea69aa136a093697 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 13 Mar 2024 01:30:39 -0400 Subject: [PATCH 05/18] gh-111307: Update design FAQ 'switch' entry (#115899) --- Doc/faq/design.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 300e1b6cc40a58..c8beb64e39bc1a 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -259,9 +259,11 @@ is evaluated in all cases. Why isn't there a switch or case statement in Python? ----------------------------------------------------- -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. +In general, structured switch statements execute one block of code +when an expression has a particular value or set of values. +Since Python 3.10 one can easily match literal values, or constants +within a namespace, with a ``match ... case`` statement. +An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For @@ -290,6 +292,9 @@ It's suggested that you use a prefix for the method names, such as ``visit_`` in this example. Without such a prefix, if values are coming from an untrusted source, an attacker would be able to call any method on your object. +Imitating switch with fallthrough, as with C's switch-case-default, +is possible, much harder, and less needed. + Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? -------------------------------------------------------------------------------------------------------- From 27df81d5643f32be6ae84a00c5cf84b58e849b21 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 13 Mar 2024 09:41:37 +0300 Subject: [PATCH 06/18] gh-115264: Fix `test_functools` with `-00` mode (#115276) --- Lib/test/test_functools.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 4eb322644fc541..1a6d8afe6ed6fe 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2696,7 +2696,10 @@ def static_func(arg: int) -> str: A().static_func ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual(A.func.__name__, 'func') @@ -2785,7 +2788,10 @@ def decorated_classmethod(cls, arg: int) -> str: WithSingleDispatch().decorated_classmethod ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual( @@ -3128,7 +3134,10 @@ def test_access_from_class(self): self.assertIsInstance(CachedCostItem.cost, py_functools.cached_property) def test_doc(self): - self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + self.assertEqual(CachedCostItem.cost.__doc__, + ("The cost of the item." + if support.HAVE_DOCSTRINGS + else None)) def test_module(self): self.assertEqual(CachedCostItem.cost.__module__, CachedCostItem.__module__) From ee0dbbc04504e0e0f1455e2bab8801ce0a682afd Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 13 Mar 2024 09:46:48 +0300 Subject: [PATCH 07/18] gh-116491: Improve `test_win32_ver` (#116506) --- Lib/test/test_platform.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index bbd0c06efed2c9..9f8aeeea257311 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -326,8 +326,36 @@ def test_java_ver(self): res = platform.java_ver() self.assertEqual(len(res), 4) + @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): - res = platform.win32_ver() + release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' + res = platform.win32_ver(release1, version1, csd1, ptype1) + self.assertEqual(len(res), 4) + release, version, csd, ptype = res + if release: + # Currently, release names always come from internal dicts, + # but this could change over time. For now, we just check that + # release is something different from what we have passed. + self.assertNotEqual(release, release1) + if version: + # It is rather hard to test explicit version without + # going deep into the details. + self.assertIn('.', version) + for v in version.split('.'): + int(v) # should not fail + if csd: + self.assertTrue(csd.startswith('SP'), msg=csd) + if ptype: + if os.cpu_count() > 1: + self.assertIn('Multiprocessor', ptype) + else: + self.assertIn('Uniprocessor', ptype) + + @unittest.skipIf(support.MS_WINDOWS, 'This test only makes sense on non Windows') + def test_win32_ver_on_non_windows(self): + release, version, csd, ptype = 'a', '1.0', 'c', 'd' + res = platform.win32_ver(release, version, csd, ptype) + self.assertSequenceEqual(res, (release, version, csd, ptype), seq_type=tuple) def test_mac_ver(self): res = platform.mac_ver() From e82f6dfae54170559fe46000fe6db9fee2a21a96 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 13 Mar 2024 02:12:30 -0500 Subject: [PATCH 08/18] Modernize roundrobin() recipe and improve variable names (gh-116710) --- Doc/library/itertools.rst | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 36e555d37d45de..26d5cdaff8c14f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -932,31 +932,25 @@ which incur interpreter overhead. # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF - args = [iter(iterable)] * n + iterators = [iter(iterable)] * n match incomplete: case 'fill': - return zip_longest(*args, fillvalue=fillvalue) + return zip_longest(*iterators, fillvalue=fillvalue) case 'strict': - return zip(*args, strict=True) + return zip(*iterators, strict=True) case 'ignore': - return zip(*args) + return zip(*iterators) case _: raise ValueError('Expected fill, strict, or ignore') def roundrobin(*iterables): "Visit input iterables in a cycle until each is exhausted." # roundrobin('ABC', 'D', 'EF') --> A D E B F C - # Recipe credited to George Sakkis - num_active = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while num_active: - try: - for next in nexts: - yield next() - except StopIteration: - # Remove the iterator we just exhausted from the cycle. - num_active -= 1 - nexts = cycle(islice(nexts, num_active)) + # Algorithm credited to George Sakkis + iterators = map(iter, iterables) + for num_active in range(len(iterables), 0, -1): + iterators = cycle(islice(iterators, num_active)) + yield from map(next, iterators) def partition(predicate, iterable): """Partition entries into false entries and true entries. @@ -997,10 +991,10 @@ The following recipes have a more mathematical flavor: s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - def sum_of_squares(it): + def sum_of_squares(iterable): "Add up the squares of the input values." # sum_of_squares([10, 20, 30]) --> 1400 - return math.sumprod(*tee(it)) + return math.sumprod(*tee(iterable)) def reshape(matrix, cols): "Reshape a 2-D matrix to have a given number of columns." @@ -1570,6 +1564,9 @@ The following recipes have a more mathematical flavor: >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] + >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] + >>> collections.Counter(roundrobin(ranges)) == collections.Counter(ranges) + True >>> def is_odd(x): ... return x % 2 == 1 From ba82a241ac7ddee7cad18e9994f8dd560c68df02 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Wed, 13 Mar 2024 17:21:30 +1000 Subject: [PATCH 09/18] gh-96471: Add ShutDown to queue.py '__all__' (#116699) --- Lib/queue.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/queue.py b/Lib/queue.py index 18fad8df08e469..387ce5425879a4 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -10,7 +10,15 @@ except ImportError: SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] +__all__ = [ + 'Empty', + 'Full', + 'ShutDown', + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'SimpleQueue', +] try: From 8332e85b2f079e8b9334666084d1f8495cff25c1 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 13 Mar 2024 01:28:01 -0700 Subject: [PATCH 10/18] gh-116626: Emit `CALL` events for all `INSTRUMENTED_CALL_FUNCTION_EX` (GH-116627) --- Lib/test/test_monitoring.py | 15 ++++++++++ ...-03-11-22-05-56.gh-issue-116626.GsyczB.rst | 1 + Python/bytecodes.c | 29 ++++++++++--------- Python/generated_cases.c.h | 28 +++++++++--------- 4 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1e77eb6a2eea4c..11fb2d87f6c995 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1807,6 +1807,21 @@ def test_gh108976(self): sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) sys.monitoring.set_events(0, 0) + def test_call_function_ex(self): + def f(a, b): + return a + b + args = (1, 2) + + call_data = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) + sys.monitoring.set_events(0, E.CALL) + f(*args) + sys.monitoring.set_events(0, 0) + self.assertEqual(call_data[0], (f, 1)) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst new file mode 100644 index 00000000000000..5b18d04cca64b5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst @@ -0,0 +1 @@ +Ensure ``INSTRUMENTED_CALL_FUNCTION_EX`` always emits :monitoring-event:`CALL` diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 03e5f4e330bdd8..ec05e40bd23fcb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3729,9 +3729,7 @@ dummy_func( } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( @@ -3739,17 +3737,20 @@ dummy_func( frame, this_instr, func, arg); if (err) GOTO_ERROR(error); result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); + + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } } } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 53c0211be2fe6c..72892725fb25d8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1205,9 +1205,7 @@ } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( @@ -1215,17 +1213,19 @@ frame, this_instr, func, arg); if (err) GOTO_ERROR(error); result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } } } } From aa7bcf284f006434b07839d82f325618f7a5c06c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Mar 2024 11:40:28 +0200 Subject: [PATCH 11/18] gh-116401: Fix blocking os.fwalk() and shutil.rmtree() on opening a named pipe (GH-116421) --- Lib/os.py | 4 +- Lib/shutil.py | 4 +- Lib/test/test_glob.py | 12 +++ Lib/test/test_os.py | 82 ++++++++++++++++++- Lib/test/test_shutil.py | 17 ++++ ...-03-06-18-30-37.gh-issue-116401.3Wcda2.rst | 2 + 6 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst diff --git a/Lib/os.py b/Lib/os.py index 7f38e14e7bdd96..7661ce68ca3be2 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -475,7 +475,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= # lstat()/open()/fstat() trick. if not follow_symlinks: orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY, dir_fd=dir_fd) + topfd = open(top, O_RDONLY | O_NONBLOCK, dir_fd=dir_fd) try: if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and path.samestat(orig_st, stat(topfd)))): @@ -524,7 +524,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): assert entries is not None name, entry = name orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY, dir_fd=topfd) + dirfd = open(name, O_RDONLY | O_NONBLOCK, dir_fd=topfd) except OSError as err: if onerror is not None: onerror(err) diff --git a/Lib/shutil.py b/Lib/shutil.py index f8be82d97616d3..94b09509008b0b 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -681,7 +681,7 @@ def _rmtree_safe_fd(topfd, path, onexc): continue if is_dir: try: - dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) + dirfd = os.open(entry.name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=topfd) dirfd_closed = False except FileNotFoundError: continue @@ -786,7 +786,7 @@ def onexc(*args): onexc(os.lstat, path, err) return try: - fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd) + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dir_fd) fd_closed = False except OSError as err: onexc(os.open, path, err) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 8b2ea8f89f5daf..1fdf2817e236f6 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -344,6 +344,18 @@ def test_glob_non_directory(self): eq(self.rglob('nonexistent', '*'), []) eq(self.rglob('nonexistent', '**'), []) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_glob_named_pipe(self): + path = os.path.join(self.tempdir, 'mypipe') + os.mkfifo(path) + self.assertEqual(self.rglob('mypipe'), [path]) + self.assertEqual(self.rglob('mypipe*'), [path]) + self.assertEqual(self.rglob('mypipe', ''), []) + self.assertEqual(self.rglob('mypipe', 'sub'), []) + self.assertEqual(self.rglob('mypipe', '*'), []) + def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index fc886f967c11bf..4c157842d95523 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1298,6 +1298,7 @@ def test_ror_operator(self): class WalkTests(unittest.TestCase): """Tests for os.walk().""" + is_fwalk = False # Wrapper to hide minor differences between os.walk and os.fwalk # to tests both functions with the same code base @@ -1332,14 +1333,14 @@ def setUp(self): self.sub11_path = join(self.sub1_path, "SUB11") sub2_path = join(self.walk_path, "SUB2") sub21_path = join(sub2_path, "SUB21") - tmp1_path = join(self.walk_path, "tmp1") + self.tmp1_path = join(self.walk_path, "tmp1") tmp2_path = join(self.sub1_path, "tmp2") tmp3_path = join(sub2_path, "tmp3") tmp5_path = join(sub21_path, "tmp3") self.link_path = join(sub2_path, "link") t2_path = join(os_helper.TESTFN, "TEST2") tmp4_path = join(os_helper.TESTFN, "TEST2", "tmp4") - broken_link_path = join(sub2_path, "broken_link") + self.broken_link_path = join(sub2_path, "broken_link") broken_link2_path = join(sub2_path, "broken_link2") broken_link3_path = join(sub2_path, "broken_link3") @@ -1349,13 +1350,13 @@ def setUp(self): os.makedirs(sub21_path) os.makedirs(t2_path) - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: + for path in self.tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: with open(path, "x", encoding='utf-8') as f: f.write("I'm " + path + " and proud of it. Blame test_os.\n") if os_helper.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) + os.symlink('broken', self.broken_link_path, True) os.symlink(join('tmp3', 'broken'), broken_link2_path, True) os.symlink(join('SUB21', 'tmp5'), broken_link3_path, True) self.sub2_tree = (sub2_path, ["SUB21", "link"], @@ -1451,6 +1452,11 @@ def test_walk_symlink(self): else: self.fail("Didn't follow symlink with followlinks=True") + walk_it = self.walk(self.broken_link_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1472,6 +1478,73 @@ def test_walk_bad_dir(self): finally: os.rename(path1new, path1) + def test_walk_bad_dir2(self): + walk_it = self.walk('nonexisting') + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk('nonexisting', follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe(self): + path = os_helper.TESTFN + '-pipe' + os.mkfifo(path) + self.addCleanup(os.unlink, path) + + walk_it = self.walk(path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe2(self): + path = os_helper.TESTFN + '-dir' + os.mkdir(path) + self.addCleanup(shutil.rmtree, path) + os.mkfifo(os.path.join(path, 'mypipe')) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + next(walk_it) + self.assertRaises(StopIteration, next, walk_it) + self.assertEqual(errors, []) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(root, path) + self.assertEqual(dirs, []) + self.assertEqual(files, ['mypipe']) + dirs.extend(files) + files.clear() + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + if self.is_fwalk: + self.assertEqual(errors, []) + else: + self.assertEqual(len(errors), 1, errors) + self.assertIsInstance(errors[0], NotADirectoryError) + def test_walk_many_open_files(self): depth = 30 base = os.path.join(os_helper.TESTFN, 'deep') @@ -1537,6 +1610,7 @@ def test_walk_above_recursion_limit(self): @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): """Tests for os.fwalk().""" + is_fwalk = True def walk(self, top, **kwargs): for root, dirs, files, root_fd in self.fwalk(top, **kwargs): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d96dad4eb9475d..60e88d57b2b23d 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -667,6 +667,23 @@ def test_rmtree_on_junction(self): finally: shutil.rmtree(TESTFN, ignore_errors=True) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + os.mkfifo(TESTFN) + try: + with self.assertRaises(NotADirectoryError): + shutil.rmtree(TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + finally: + os.unlink(TESTFN) + + os.mkdir(TESTFN) + os.mkfifo(os.path.join(TESTFN, 'mypipe')) + shutil.rmtree(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override diff --git a/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst b/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst new file mode 100644 index 00000000000000..121f0065ecca95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst @@ -0,0 +1,2 @@ +Fix blocking :func:`os.fwalk` and :func:`shutil.rmtree` on opening named +pipe. From fcd49b4f47f1edd9a2717f6619da7e7af8ea73cf Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 13 Mar 2024 15:38:03 +0300 Subject: [PATCH 12/18] gh-116714: Handle errors correctly in `PyFloat_GetInfo` (#116715) Co-authored-by: Serhiy Storchaka --- Objects/floatobject.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 37d2d312a6a0b7..96227f2cf7d76f 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -98,10 +98,18 @@ PyFloat_GetInfo(void) return NULL; } -#define SetIntFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyLong_FromLong(flag)) -#define SetDblFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyFloat_FromDouble(flag)) +#define SetFlag(CALL) \ + do { \ + PyObject *flag = (CALL); \ + if (flag == NULL) { \ + Py_CLEAR(floatinfo); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(floatinfo, pos++, flag); \ + } while (0) + +#define SetIntFlag(FLAG) SetFlag(PyLong_FromLong((FLAG))) +#define SetDblFlag(FLAG) SetFlag(PyFloat_FromDouble((FLAG))) SetDblFlag(DBL_MAX); SetIntFlag(DBL_MAX_EXP); @@ -116,11 +124,8 @@ PyFloat_GetInfo(void) SetIntFlag(FLT_ROUNDS); #undef SetIntFlag #undef SetDblFlag +#undef SetFlag - if (PyErr_Occurred()) { - Py_CLEAR(floatinfo); - return NULL; - } return floatinfo; } From 617aca9e745b3dfb5e6bc8cda07632d2f716426d Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 13 Mar 2024 20:57:48 +0800 Subject: [PATCH 13/18] gh-115419: Change default sym to not_null (GH-116562) --- Lib/test/test_generated_cases.py | 4 +- Python/optimizer_bytecodes.c | 9 +- Python/optimizer_cases.c.h | 181 ++++++++++--------- Tools/cases_generator/optimizer_generator.py | 4 +- 4 files changed, 104 insertions(+), 94 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 32c2c2fca05c4e..7b9dd36f85454f 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -908,7 +908,7 @@ def test_overridden_abstract_args(self): case OP2: { _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); + out = sym_new_not_null(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; @@ -933,7 +933,7 @@ def test_no_overridden_case(self): output = """ case OP: { _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); + out = sym_new_not_null(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e3f7c9822103e6..54abbcd74d7934 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -449,6 +449,14 @@ dummy_func(void) { } } + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { + (void)owner; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + if (oparg & 1) { + OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); + } + } + op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); @@ -513,7 +521,6 @@ dummy_func(void) { OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); } - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { if (!sym_set_type(callable, &PyFunction_Type)) { goto hit_bottom; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fed5730d2e50c1..7e4214cc9acf39 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -80,7 +80,7 @@ case _END_SEND: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-2] = value; stack_pointer += -1; @@ -89,7 +89,7 @@ case _UNARY_NEGATIVE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -97,7 +97,7 @@ case _UNARY_NOT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -205,7 +205,7 @@ case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -213,7 +213,7 @@ case _UNARY_INVERT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -482,7 +482,7 @@ case _BINARY_SUBSCR: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -491,7 +491,7 @@ case _BINARY_SLICE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -505,7 +505,7 @@ case _BINARY_SUBSCR_LIST_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -514,7 +514,7 @@ case _BINARY_SUBSCR_STR_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -523,7 +523,7 @@ case _BINARY_SUBSCR_TUPLE_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -532,7 +532,7 @@ case _BINARY_SUBSCR_DICT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -573,7 +573,7 @@ case _CALL_INTRINSIC_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -581,7 +581,7 @@ case _CALL_INTRINSIC_2: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -608,7 +608,7 @@ case _GET_AITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -616,7 +616,7 @@ case _GET_ANEXT: { _Py_UopsSymbol *awaitable; - awaitable = sym_new_unknown(ctx); + awaitable = sym_new_not_null(ctx); if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; stack_pointer += 1; @@ -625,7 +625,7 @@ case _GET_AWAITABLE: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -644,7 +644,7 @@ case _LOAD_ASSERTION_ERROR: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -653,7 +653,7 @@ case _LOAD_BUILD_CLASS: { _Py_UopsSymbol *bc; - bc = sym_new_unknown(ctx); + bc = sym_new_not_null(ctx); if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; stack_pointer += 1; @@ -686,9 +686,9 @@ case _UNPACK_SEQUENCE_TWO_TUPLE: { _Py_UopsSymbol *val1; _Py_UopsSymbol *val0; - val1 = sym_new_unknown(ctx); + val1 = sym_new_not_null(ctx); if (val1 == NULL) goto out_of_space; - val0 = sym_new_unknown(ctx); + val0 = sym_new_not_null(ctx); if (val0 == NULL) goto out_of_space; stack_pointer[-1] = val1; stack_pointer[0] = val0; @@ -700,7 +700,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = sym_new_not_null(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -711,7 +711,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = sym_new_not_null(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -754,7 +754,7 @@ case _LOAD_LOCALS: { _Py_UopsSymbol *locals; - locals = sym_new_unknown(ctx); + locals = sym_new_not_null(ctx); if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; stack_pointer += 1; @@ -763,7 +763,7 @@ case _LOAD_FROM_DICT_OR_GLOBALS: { _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); + v = sym_new_not_null(ctx); if (v == NULL) goto out_of_space; stack_pointer[-1] = v; break; @@ -771,7 +771,7 @@ case _LOAD_NAME: { _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); + v = sym_new_not_null(ctx); if (v == NULL) goto out_of_space; stack_pointer[0] = v; stack_pointer += 1; @@ -781,7 +781,7 @@ case _LOAD_GLOBAL: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -802,7 +802,7 @@ case _LOAD_GLOBAL_MODULE: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -815,7 +815,7 @@ case _LOAD_GLOBAL_BUILTINS: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -839,7 +839,7 @@ case _LOAD_FROM_DICT_OR_DEREF: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; @@ -847,7 +847,7 @@ case _LOAD_DEREF: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -865,7 +865,7 @@ case _BUILD_STRING: { _Py_UopsSymbol *str; - str = sym_new_unknown(ctx); + str = sym_new_not_null(ctx); if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -874,7 +874,7 @@ case _BUILD_TUPLE: { _Py_UopsSymbol *tup; - tup = sym_new_unknown(ctx); + tup = sym_new_not_null(ctx); if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -883,7 +883,7 @@ case _BUILD_LIST: { _Py_UopsSymbol *list; - list = sym_new_unknown(ctx); + list = sym_new_not_null(ctx); if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -902,7 +902,7 @@ case _BUILD_SET: { _Py_UopsSymbol *set; - set = sym_new_unknown(ctx); + set = sym_new_not_null(ctx); if (set == NULL) goto out_of_space; stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -911,7 +911,7 @@ case _BUILD_MAP: { _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); + map = sym_new_not_null(ctx); if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -924,7 +924,7 @@ case _BUILD_CONST_KEY_MAP: { _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); + map = sym_new_not_null(ctx); if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -950,7 +950,7 @@ case _LOAD_SUPER_ATTR_ATTR: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer += -2; @@ -960,9 +960,9 @@ case _LOAD_SUPER_ATTR_METHOD: { _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); + self_or_null = sym_new_not_null(ctx); if (self_or_null == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -971,12 +971,15 @@ } case _LOAD_ATTR: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null = NULL; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); - if (self_or_null == NULL) goto out_of_space; + owner = stack_pointer[-1]; + (void)owner; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + if (oparg & 1) { + OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -1222,7 +1225,7 @@ case _CONTAINS_OP_SET: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1231,7 +1234,7 @@ case _CONTAINS_OP_DICT: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1241,9 +1244,9 @@ case _CHECK_EG_MATCH: { _Py_UopsSymbol *rest; _Py_UopsSymbol *match; - rest = sym_new_unknown(ctx); + rest = sym_new_not_null(ctx); if (rest == NULL) goto out_of_space; - match = sym_new_unknown(ctx); + match = sym_new_not_null(ctx); if (match == NULL) goto out_of_space; stack_pointer[-2] = rest; stack_pointer[-1] = match; @@ -1252,7 +1255,7 @@ case _CHECK_EXC_MATCH: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1264,7 +1267,7 @@ case _IS_NONE: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1272,7 +1275,7 @@ case _GET_LEN: { _Py_UopsSymbol *len_o; - len_o = sym_new_unknown(ctx); + len_o = sym_new_not_null(ctx); if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; stack_pointer += 1; @@ -1281,7 +1284,7 @@ case _MATCH_CLASS: { _Py_UopsSymbol *attrs; - attrs = sym_new_unknown(ctx); + attrs = sym_new_not_null(ctx); if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1290,7 +1293,7 @@ case _MATCH_MAPPING: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1299,7 +1302,7 @@ case _MATCH_SEQUENCE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1308,7 +1311,7 @@ case _MATCH_KEYS: { _Py_UopsSymbol *values_or_none; - values_or_none = sym_new_unknown(ctx); + values_or_none = sym_new_not_null(ctx); if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1317,7 +1320,7 @@ case _GET_ITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1325,7 +1328,7 @@ case _GET_YIELD_FROM_ITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1335,7 +1338,7 @@ case _FOR_ITER_TIER_TWO: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1356,7 +1359,7 @@ case _ITER_NEXT_LIST: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1375,7 +1378,7 @@ case _ITER_NEXT_TUPLE: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1408,9 +1411,9 @@ case _BEFORE_ASYNC_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); + exit = sym_new_not_null(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1421,9 +1424,9 @@ case _BEFORE_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); + exit = sym_new_not_null(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1433,7 +1436,7 @@ case _WITH_EXCEPT_START: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1443,9 +1446,9 @@ case _PUSH_EXC_INFO: { _Py_UopsSymbol *prev_exc; _Py_UopsSymbol *new_exc; - prev_exc = sym_new_unknown(ctx); + prev_exc = sym_new_not_null(ctx); if (prev_exc == NULL) goto out_of_space; - new_exc = sym_new_unknown(ctx); + new_exc = sym_new_not_null(ctx); if (new_exc == NULL) goto out_of_space; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; @@ -1493,7 +1496,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1501,7 +1504,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1632,7 +1635,7 @@ case _CALL_TYPE_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -1641,7 +1644,7 @@ case _CALL_STR_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -1650,7 +1653,7 @@ case _CALL_TUPLE_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -1666,7 +1669,7 @@ case _CALL_BUILTIN_CLASS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1675,7 +1678,7 @@ case _CALL_BUILTIN_O: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1684,7 +1687,7 @@ case _CALL_BUILTIN_FAST: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1693,7 +1696,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1702,7 +1705,7 @@ case _CALL_LEN: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1711,7 +1714,7 @@ case _CALL_ISINSTANCE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1720,7 +1723,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1729,7 +1732,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1738,7 +1741,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1747,7 +1750,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1764,7 +1767,7 @@ case _MAKE_FUNCTION: { _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); + func = sym_new_not_null(ctx); if (func == NULL) goto out_of_space; stack_pointer[-1] = func; break; @@ -1772,7 +1775,7 @@ case _SET_FUNCTION_ATTRIBUTE: { _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); + func = sym_new_not_null(ctx); if (func == NULL) goto out_of_space; stack_pointer[-2] = func; stack_pointer += -1; @@ -1781,7 +1784,7 @@ case _BUILD_SLICE: { _Py_UopsSymbol *slice; - slice = sym_new_unknown(ctx); + slice = sym_new_not_null(ctx); if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -1790,7 +1793,7 @@ case _CONVERT_VALUE: { _Py_UopsSymbol *result; - result = sym_new_unknown(ctx); + result = sym_new_not_null(ctx); if (result == NULL) goto out_of_space; stack_pointer[-1] = result; break; @@ -1798,7 +1801,7 @@ case _FORMAT_SIMPLE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -1806,7 +1809,7 @@ case _FORMAT_WITH_SPEC: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1826,7 +1829,7 @@ case _BINARY_OP: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1953,7 +1956,7 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index a0a2f10aa760b7..1c6b708e82321a 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -83,14 +83,14 @@ def emit_default(out: CWriter, uop: Uop) -> None: if var.name != "unused" and not var.peek: if var.is_array(): out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_unknown(ctx);\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") out.emit(f"if ({var.name}[_i] == NULL) goto out_of_space;\n") out.emit("}\n") elif var.name == "null": out.emit(f"{var.name} = sym_new_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") else: - out.emit(f"{var.name} = sym_new_unknown(ctx);\n") + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") From 33662d4e01d73cd4f29a25efc2ef09288129023f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Mar 2024 15:03:13 +0200 Subject: [PATCH 14/18] gh-90300: Fix cmdline.rst (GH-116721) * Fix the description of the "-b" option. * Add references to environment variables for "-s" and "-X dev" options. --- Doc/using/cmdline.rst | 13 ++++++++----- Python/initconfig.c | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 36cddffb9eae34..565d86cb1a0dd3 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -242,12 +242,13 @@ Miscellaneous options .. option:: -b - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the - option is given twice (:option:`!-bb`). + Issue a warning when converting :class:`bytes` or :class:`bytearray` to + :class:`str` without specifying encoding or comparing :class:`!bytes` or + :class:`!bytearray` with :class:`!str` or :class:`!bytes` with :class:`int`. + Issue an error when the option is given twice (:option:`!-bb`). .. versionchanged:: 3.5 - Affects comparisons of :class:`bytes` with :class:`int`. + Affects also comparisons of :class:`bytes` with :class:`int`. .. option:: -B @@ -386,6 +387,8 @@ Miscellaneous options Don't add the :data:`user site-packages directory ` to :data:`sys.path`. + See also :envvar:`PYTHONNOUSERSITE`. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -517,7 +520,7 @@ Miscellaneous options asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. * ``-X dev``: enable :ref:`Python Development Mode `, introducing additional runtime checks that are too expensive to be enabled by - default. + default. See also :envvar:`PYTHONDEVMODE`. * ``-X utf8`` enables the :ref:`Python UTF-8 Mode `. ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode ` (even when it would otherwise activate automatically). diff --git a/Python/initconfig.c b/Python/initconfig.c index e3a62e53334163..a01a9f56f37832 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -142,8 +142,8 @@ static const char usage_line[] = /* Lines sorted by option name; keep in sync with usage_envvars* below */ static const char usage_help[] = "\ Options (and corresponding environment variables):\n\ --b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ - and comparing bytes/bytearray with str. (-bb: issue errors)\n\ +-b : issue warnings about converting bytes/bytearray to str and comparing\n\ + bytes/bytearray with str or bytes with int. (-bb: issue errors)\n\ -B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : turn on parser debugging output (for experts only, only works on\n\ From 612f1ec988314bc0bc42a1b908751950331e2ede Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 13 Mar 2024 14:20:33 +0100 Subject: [PATCH 15/18] gh-110918: Fix side effects of regrtest test_match_tests() (#116718) test_match_tests now saves and restores patterns. Add get_match_tests() function to libregrtest.filter. Previously, running test_regrtest multiple times in a row only ran tests once: "./python -m test test_regrtest -R 3:3. --- Lib/test/libregrtest/filter.py | 5 +++++ Lib/test/test_regrtest.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py index 817624d79e9263..41372e427ffd03 100644 --- a/Lib/test/libregrtest/filter.py +++ b/Lib/test/libregrtest/filter.py @@ -27,6 +27,11 @@ def _is_full_match_test(pattern): return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) +def get_match_tests(): + global _test_patterns + return _test_patterns + + def set_match_tests(patterns): global _test_matchers, _test_patterns diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 7e1eaa7d6a515e..903ad50ba088e8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -27,7 +27,7 @@ from test.libregrtest import main from test.libregrtest import setup from test.libregrtest import utils -from test.libregrtest.filter import set_match_tests, match_test +from test.libregrtest.filter import get_match_tests, set_match_tests, match_test from test.libregrtest.result import TestStats from test.libregrtest.utils import normalize_test_name @@ -2298,6 +2298,10 @@ def __init__(self, test_id): def id(self): return self.test_id + # Restore patterns once the test completes + patterns = get_match_tests() + self.addCleanup(set_match_tests, patterns) + test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') test_copy = Test('test.test_shutil.TestCopy.test_copy') From 186af3cf21705badec086ec16f231ac390747d3b Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Wed, 13 Mar 2024 13:22:47 +0000 Subject: [PATCH 16/18] [doc]: Update cookbook recipe for Qt6. (GH-116719) --- Doc/howto/logging-cookbook.rst | 49 +++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index f7d885ec88483d..ad3e34d0b33bd2 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -3418,9 +3418,10 @@ The worker thread is implemented using Qt's ``QThread`` class rather than the :mod:`threading` module, as there are circumstances where one has to use ``QThread``, which offers better integration with other ``Qt`` components. -The code should work with recent releases of either ``PySide2`` or ``PyQt5``. -You should be able to adapt the approach to earlier versions of Qt. Please -refer to the comments in the code snippet for more detailed information. +The code should work with recent releases of either ``PySide6``, ``PyQt6``, +``PySide2`` or ``PyQt5``. You should be able to adapt the approach to earlier +versions of Qt. Please refer to the comments in the code snippet for more +detailed information. .. code-block:: python3 @@ -3430,16 +3431,25 @@ refer to the comments in the code snippet for more detailed information. import sys import time - # Deal with minor differences between PySide2 and PyQt5 + # Deal with minor differences between different Qt packages try: - from PySide2 import QtCore, QtGui, QtWidgets + from PySide6 import QtCore, QtGui, QtWidgets Signal = QtCore.Signal Slot = QtCore.Slot except ImportError: - from PyQt5 import QtCore, QtGui, QtWidgets - Signal = QtCore.pyqtSignal - Slot = QtCore.pyqtSlot - + try: + from PyQt6 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + except ImportError: + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot logger = logging.getLogger(__name__) @@ -3511,8 +3521,14 @@ refer to the comments in the code snippet for more detailed information. while not QtCore.QThread.currentThread().isInterruptionRequested(): delay = 0.5 + random.random() * 2 time.sleep(delay) - level = random.choice(LEVELS) - logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + try: + if random.random() < 0.1: + raise ValueError('Exception raised: %d' % i) + else: + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + except ValueError as e: + logger.exception('Failed: %s', e, extra=extra) i += 1 # @@ -3539,7 +3555,10 @@ refer to the comments in the code snippet for more detailed information. self.textedit = te = QtWidgets.QPlainTextEdit(self) # Set whatever the default monospace font is for the platform f = QtGui.QFont('nosuchfont') - f.setStyleHint(f.Monospace) + if hasattr(f, 'Monospace'): + f.setStyleHint(f.Monospace) + else: + f.setStyleHint(f.StyleHint.Monospace) # for Qt6 te.setFont(f) te.setReadOnly(True) PB = QtWidgets.QPushButton @@ -3626,7 +3645,11 @@ refer to the comments in the code snippet for more detailed information. app = QtWidgets.QApplication(sys.argv) example = Window(app) example.show() - sys.exit(app.exec_()) + if hasattr(app, 'exec'): + rc = app.exec() + else: + rc = app.exec_() + sys.exit(rc) if __name__=='__main__': main() From 8e2aab7ad5e1c8b3360c1e1b80ddadc0845eaa3e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 13 Mar 2024 09:27:36 -0400 Subject: [PATCH 17/18] gh-116604: Fix test_gc on free-threaded build (#116662) The free-threaded GC only does full collections, so it uses a threshold that is a maximum of a fixed value (default 2000) and proportional to the number of live objects. If there were many live objects after the previous collection, then the threshold may be larger than 10,000 causing `test_indirect_calls_with_gc_disabled` to fail. This manually sets the threshold to `(1000, 0, 0)` for the test. The `0` disables the proportional scaling. --- Lib/test/support/__init__.py | 10 ++++++++++ Lib/test/test_gc.py | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index af43446c26120e..ce693e51aab31c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -797,6 +797,16 @@ def disable_gc(): if have_gc: gc.enable() +@contextlib.contextmanager +def gc_threshold(*args): + import gc + old_threshold = gc.get_threshold() + gc.set_threshold(*args) + try: + yield + finally: + gc.set_threshold(*old_threshold) + def python_is_optimized(): """Find if Python was built with optimizations.""" diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 2aea025fcc140a..f1a7afac0bcd19 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -5,7 +5,7 @@ from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script -from test.support import threading_helper +from test.support import threading_helper, gc_threshold import gc import sys @@ -1330,6 +1330,7 @@ def callback(ignored): # with an empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1397,6 +1398,7 @@ def __del__(self): # empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) def test_indirect_calls_with_gc_disabled(self): junk = [] i = 0 From 7f418fb111dec325b5c9fe6f6e96076049322f02 Mon Sep 17 00:00:00 2001 From: Nir Friedman Date: Wed, 13 Mar 2024 11:58:30 -0400 Subject: [PATCH 18/18] gh-98731: Improvements to the logging documentation (GH-101618) Co-authored-by: Vinay Sajip --- Doc/howto/logging.rst | 118 +++++++++++-------------------- Doc/library/logging.rst | 152 ++++++++++++++++++---------------------- 2 files changed, 108 insertions(+), 162 deletions(-) diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 347330e98dd00c..ab758a885b3556 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -25,10 +25,12 @@ or *severity*. When to use logging ^^^^^^^^^^^^^^^^^^^ -Logging provides a set of convenience functions for simple logging usage. These -are :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and -:func:`critical`. To determine when to use logging, see the table below, which -states, for each of a set of common tasks, the best tool to use for it. +You can access logging functionality by creating a logger via ``logger = +getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +:meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and +:meth:`~Logger.critical` methods. To determine when to use logging, and to see +which logger methods to use when, see the table below. It states, for each of a +set of common tasks, the best tool to use for that task. +-------------------------------------+--------------------------------------+ | Task you want to perform | The best tool for the task | @@ -37,8 +39,8 @@ states, for each of a set of common tasks, the best tool to use for it. | usage of a command line script or | | | program | | +-------------------------------------+--------------------------------------+ -| Report events that occur during | :func:`logging.info` (or | -| normal operation of a program (e.g. | :func:`logging.debug` for very | +| Report events that occur during | A logger's :meth:`~Logger.info` (or | +| normal operation of a program (e.g. | :meth:`~Logger.debug` method for very| | for status monitoring or fault | detailed output for diagnostic | | investigation) | purposes) | +-------------------------------------+--------------------------------------+ @@ -47,22 +49,23 @@ states, for each of a set of common tasks, the best tool to use for it. | | the client application should be | | | modified to eliminate the warning | | | | -| | :func:`logging.warning` if there is | -| | nothing the client application can do| -| | about the situation, but the event | -| | should still be noted | +| | A logger's :meth:`~Logger.warning` | +| | method if there is nothing the client| +| | application can do about the | +| | situation, but the event should still| +| | be noted | +-------------------------------------+--------------------------------------+ | Report an error regarding a | Raise an exception | | particular runtime event | | +-------------------------------------+--------------------------------------+ -| Report suppression of an error | :func:`logging.error`, | -| without raising an exception (e.g. | :func:`logging.exception` or | -| error handler in a long-running | :func:`logging.critical` as | +| Report suppression of an error | A logger's :meth:`~Logger.error`, | +| without raising an exception (e.g. | :meth:`~Logger.exception` or | +| error handler in a long-running | :meth:`~Logger.critical` method as | | server process) | appropriate for the specific error | | | and application domain | +-------------------------------------+--------------------------------------+ -The logging functions are named after the level or severity of the events +The logger methods are named after the level or severity of the events they are used to track. The standard levels and their applicability are described below (in increasing order of severity): @@ -115,12 +118,18 @@ If you type these lines into a script and run it, you'll see: WARNING:root:Watch out! printed out on the console. The ``INFO`` message doesn't appear because the -default level is ``WARNING``. The printed message includes the indication of -the level and the description of the event provided in the logging call, i.e. -'Watch out!'. Don't worry about the 'root' part for now: it will be explained -later. The actual output can be formatted quite flexibly if you need that; -formatting options will also be explained later. - +default level is ``WARNING``. The printed message includes the indication of the +level and the description of the event provided in the logging call, i.e. +'Watch out!'. The actual output can be formatted quite flexibly if you need +that; formatting options will also be explained later. + +Notice that in this example, we use functions directly on the ``logging`` +module, like ``logging.debug``, rather than creating a logger and calling +functions on it. These functions operation on the root logger, but can be useful +as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in +this example. In larger programs you'll usually want to control the logging +configuration explicitly however - so for that reason as well as others, it's +better to create loggers and call their methods. Logging to a file ^^^^^^^^^^^^^^^^^ @@ -130,11 +139,12 @@ look at that next. Be sure to try the following in a newly started Python interpreter, and don't just continue from the session described above:: import logging + logger = logging.getLogger(__name__) logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) - logging.debug('This message should go to the log file') - logging.info('So should this') - logging.warning('And this, too') - logging.error('And non-ASCII stuff, too, like Øresund and Malmö') + logger.debug('This message should go to the log file') + logger.info('So should this') + logger.warning('And this, too') + logger.error('And non-ASCII stuff, too, like Øresund and Malmö') .. versionchanged:: 3.9 The *encoding* argument was added. In earlier Python versions, or if not @@ -148,10 +158,10 @@ messages: .. code-block:: none - DEBUG:root:This message should go to the log file - INFO:root:So should this - WARNING:root:And this, too - ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö + DEBUG:__main__:This message should go to the log file + INFO:__main__:So should this + WARNING:__main__:And this, too + ERROR:__main__:And non-ASCII stuff, too, like Øresund and Malmö This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to @@ -180,11 +190,9 @@ following example:: raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, ...) -The call to :func:`basicConfig` should come *before* any calls to -:func:`debug`, :func:`info`, etc. Otherwise, those functions will call -:func:`basicConfig` for you with the default options. As it's intended as a -one-off simple configuration facility, only the first call will actually do -anything: subsequent calls are effectively no-ops. +The call to :func:`basicConfig` should come *before* any calls to a logger's +methods such as :meth:`~Logger.debug`, :meth:`~Logger.info`, etc. Otherwise, +that logging event may not be handled in the desired manner. If you run the above script several times, the messages from successive runs are appended to the file *example.log*. If you want each run to start afresh, @@ -197,50 +205,6 @@ The output will be the same as before, but the log file is no longer appended to, so the messages from earlier runs are lost. -Logging from multiple modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program consists of multiple modules, here's an example of how you -could organize logging in it:: - - # myapp.py - import logging - import mylib - - def main(): - logging.basicConfig(filename='myapp.log', level=logging.INFO) - logging.info('Started') - mylib.do_something() - logging.info('Finished') - - if __name__ == '__main__': - main() - -:: - - # mylib.py - import logging - - def do_something(): - logging.info('Doing something') - -If you run *myapp.py*, you should see this in *myapp.log*: - -.. code-block:: none - - INFO:root:Started - INFO:root:Doing something - INFO:root:Finished - -which is hopefully what you were expecting to see. You can generalize this to -multiple modules, using the pattern in *mylib.py*. Note that for this simple -usage pattern, you won't know, by looking in the log file, *where* in your -application your messages came from, apart from looking at the event -description. If you want to track the location of your messages, you'll need -to refer to the documentation beyond the tutorial level -- see -:ref:`logging-advanced-tutorial`. - - Logging variable data ^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index e38b7435498507..4e7d18bda8bf7d 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -30,13 +30,53 @@ is that all Python modules can participate in logging, so your application log can include your own messages integrated with messages from third-party modules. -The simplest example: +Here's a simple example of idiomatic usage: :: + + # myapp.py + import logging + import mylib + logger = logging.getLogger(__name__) + + def main(): + logging.basicConfig(filename='myapp.log', level=logging.INFO) + logger.info('Started') + mylib.do_something() + logger.info('Finished') + + if __name__ == '__main__': + main() + +:: + + # mylib.py + import logging + logger = logging.getLogger(__name__) + + def do_something(): + logger.info('Doing something') + +If you run *myapp.py*, you should see this in *myapp.log*: .. code-block:: none - >>> import logging - >>> logging.warning('Watch out!') - WARNING:root:Watch out! + INFO:__main__:Started + INFO:mylib:Doing something + INFO:__main__:Finished + +The key features of this idiomatic usage is that the majority of code is simply +creating a module level logger with ``getLogger(__name__)``, and using that +logger to do any needed logging. This is concise while allowing downstream code +fine grained control if needed. Logged messages to the module-level logger get +forwarded up to handlers of loggers in higher-level modules, all the way up to +the root logger; for this reason this approach is known as hierarchical logging. + +For logging to be useful, it needs to be configured: setting the levels and +destinations for each logger, potentially changing how specific modules log, +often based on command-line arguments or application configuration. In most +cases, like the one above, only the root logger needs to be so configured, since +all the lower level loggers at module level eventually forward their messages to +its handlers. :func:`~logging.basicConfig` provides a quick way to configure +the root logger that handles many use cases. The module provides a lot of functionality and flexibility. If you are unfamiliar with logging, the best way to get to grips with it is to view the @@ -1151,89 +1191,31 @@ functions. .. function:: debug(msg, *args, **kwargs) - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none + This is a convenience function that calls :meth:`Logger.debug`, on the root + logger. The handling of the arguments is in every way identical + to what is described in that method. - Stack (most recent call last): + The only difference is that if the root logger has no handlers, then + :func:`basicConfig` is called, prior to calling ``debug`` on the root logger. - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. + For very short scripts or quick demonstrations of ``logging`` facilities, + ``debug`` and the other module-level functions may be convenient. However, + most programs will want to carefully and explicitly control the logging + configuration, and should therefore prefer creating a module-level logger and + calling :meth:`Logger.debug` (or other level-specific methods) on it, as + described at the beginnning of this documentation. - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - This function (as well as :func:`info`, :func:`warning`, :func:`error` and - :func:`critical`) will call :func:`basicConfig` if the root logger doesn't - have any handler attached. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. .. function:: info(msg, *args, **kwargs) - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`INFO` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: warning(msg, *args, **kwargs) - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`WARNING` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. note:: There is an obsolete function ``warn`` which is functionally identical to ``warning``. As ``warn`` is deprecated, please do not use @@ -1246,26 +1228,26 @@ functions. .. function:: error(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: critical(msg, *args, **kwargs) - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`CRITICAL` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: exception(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. Exception info is added to the logging message. This function should only be called from an exception handler. .. function:: log(level, msg, *args, **kwargs) - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. + Logs a message with level *level* on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: disable(level=CRITICAL)