Merge branch 'main' into merge-copy-copytree

.azure-pipelines/ci.yml

@@ -1,4 +1,4 @@
-trigger: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7']
+trigger: ['main', '3.13', '3.12', '3.11', '3.10', '3.9', '3.8']
 
 jobs:
 - job: Prebuild

.github/workflows/reusable-macos.yml

@@ -48,8 +48,10 @@ jobs:
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix openssl@3.0)"
     - name: Build CPython
-      run: make -j8
+      run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
     - name: Display build info
       run: make pythoninfo
+    - name: Check compiler warnings
+      run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
     - name: Tests
       run: make test

.github/workflows/reusable-ubuntu.yml

@@ -80,7 +80,7 @@ jobs:
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+      run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
     - name: Remount sources writable for tests
       # some tests write to srcdir, lack of pyc files slows down testing
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw

Doc/Makefile

@@ -6,6 +6,7 @@
 # You can set these variables from the command line.
 PYTHON       = python3
 VENVDIR      = ./venv
+UV           = uv
 SPHINXBUILD  = PATH=$(VENVDIR)/bin:$$PATH sphinx-build
 BLURB        = PATH=$(VENVDIR)/bin:$$PATH blurb
 JOBS         = auto
@@ -150,14 +151,10 @@ gettext: build
 htmlview: html
 	$(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))"
 
-.PHONY: ensure-sphinx-autobuild
-ensure-sphinx-autobuild: venv
-	$(call ensure_package,sphinx-autobuild)
-
 .PHONY: htmllive
 htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild
 htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0
-htmllive: ensure-sphinx-autobuild html
+htmllive: _ensure-sphinx-autobuild html
 
 .PHONY: clean
 clean: clean-venv
@@ -174,15 +171,15 @@ venv:
 		echo "To recreate it, remove it first with \`make clean-venv'."; \
 	else \
 		echo "Creating venv in $(VENVDIR)"; \
-		if uv --version > /dev/null; then \
-			uv venv $(VENVDIR); \
-			VIRTUAL_ENV=$(VENVDIR) uv pip install -r $(REQUIREMENTS); \
+		if $(UV) --version >/dev/null 2>&1; then \
+			$(UV) venv $(VENVDIR); \
+			VIRTUAL_ENV=$(VENVDIR) $(UV) pip install -r $(REQUIREMENTS); \
 		else \
 			$(PYTHON) -m venv $(VENVDIR); \
 			$(VENVDIR)/bin/python3 -m pip install --upgrade pip; \
 			$(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \
-			echo "The venv has been created in the $(VENVDIR) directory"; \
 		fi; \
+		echo "The venv has been created in the $(VENVDIR) directory"; \
 	fi
 
 .PHONY: dist
@@ -240,17 +237,24 @@ dist:
 	rm -r dist/python-$(DISTVERSION)-docs-texinfo
 	rm dist/python-$(DISTVERSION)-docs-texinfo.tar
 
-define ensure_package
-	if uv --version > /dev/null; then \
-		$(VENVDIR)/bin/python3 -m $(1) --version > /dev/null || VIRTUAL_ENV=$(VENVDIR) uv pip install $(1); \
+.PHONY: _ensure-package
+_ensure-package: venv
+	if $(UV) --version >/dev/null 2>&1; then \
+		VIRTUAL_ENV=$(VENVDIR) $(UV) pip install $(PACKAGE); \
 	else \
-		$(VENVDIR)/bin/python3 -m $(1) --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install $(1); \
+		$(VENVDIR)/bin/python3 -m pip install $(PACKAGE); \
 	fi
-endef
+
+.PHONY: _ensure-pre-commit
+_ensure-pre-commit:
+	make _ensure-package PACKAGE=pre-commit
+
+.PHONY: _ensure-sphinx-autobuild
+_ensure-sphinx-autobuild:
+	make _ensure-package PACKAGE=sphinx-autobuild
 
 .PHONY: check
-check: venv
-	$(call ensure_package,pre_commit)
+check: _ensure-pre-commit
 	$(VENVDIR)/bin/python3 -m pre_commit run --all-files
 
 .PHONY: serve

Doc/library/dis.rst

@@ -1081,11 +1081,15 @@ iterations of the loop.
 .. opcode:: BUILD_TUPLE (count)
 
    Creates a tuple consuming *count* items from the stack, and pushes the
-   resulting tuple onto the stack.::
+   resulting tuple onto the stack::
 
-      assert count > 0
-      STACK, values = STACK[:-count], STACK[-count:]
-      STACK.append(tuple(values))
+      if count == 0:
+          value = ()
+      else:
+          STACK = STACK[:-count]
+          value = tuple(STACK[-count:])
+
+      STACK.append(value)
 
 
 .. opcode:: BUILD_LIST (count)

Doc/library/inspect.rst

@@ -153,6 +153,19 @@ attributes (see :ref:`import-mod-attrs` for module attributes):
 |                 | f_trace           | tracing function for this |
 |                 |                   | frame, or ``None``        |
 +-----------------+-------------------+---------------------------+
+|                 | f_trace_lines     | indicate whether a        |
+|                 |                   | tracing event is          |
+|                 |                   | triggered for each source |
+|                 |                   | source line               |
++-----------------+-------------------+---------------------------+
+|                 | f_trace_opcodes   | indicate whether          |
+|                 |                   | per-opcode events are     |
+|                 |                   | requested                 |
++-----------------+-------------------+---------------------------+
+|                 | clear()           | used to clear all         |
+|                 |                   | references to local       |
+|                 |                   | variables                 |
++-----------------+-------------------+---------------------------+
 | code            | co_argcount       | number of arguments (not  |
 |                 |                   | including keyword only    |
 |                 |                   | arguments, \* or \*\*     |
@@ -214,6 +227,18 @@ attributes (see :ref:`import-mod-attrs` for module attributes):
 |                 |                   | arguments and local       |
 |                 |                   | variables                 |
 +-----------------+-------------------+---------------------------+
+|                 | co_lines()        | returns an iterator that  |
+|                 |                   | yields successive         |
+|                 |                   | bytecode ranges           |
++-----------------+-------------------+---------------------------+
+|                 | co_positions()    | returns an iterator of    |
+|                 |                   | source code positions for |
+|                 |                   | each bytecode instruction |
++-----------------+-------------------+---------------------------+
+|                 | replace()         | returns a copy of the     |
+|                 |                   | code object with new      |
+|                 |                   | values                    |
++-----------------+-------------------+---------------------------+
 | generator       | __name__          | name                      |
 +-----------------+-------------------+---------------------------+
 |                 | __qualname__      | qualified name            |

Include/internal/pycore_parser.h

@@ -21,15 +21,20 @@ extern "C" {
 struct _parser_runtime_state {
 #ifdef Py_DEBUG
     long memo_statistics[_PYPEGEN_NSTATISTICS];
+#ifdef Py_GIL_DISABLED
+    PyMutex mutex;
+#endif
 #else
     int _not_used;
 #endif
     struct _expr dummy_name;
 };
 
 _Py_DECLARE_STR(empty, "")
+#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
 #define _parser_runtime_state_INIT \
     { \
+        .mutex = {0}, \
         .dummy_name = { \
             .kind = Name_kind, \
             .v.Name.id = &_Py_STR(empty), \
@@ -40,6 +45,20 @@ _Py_DECLARE_STR(empty, "")
             .end_col_offset = 0, \
         }, \
     }
+#else
+#define _parser_runtime_state_INIT \
+    { \
+        .dummy_name = { \
+            .kind = Name_kind, \
+            .v.Name.id = &_Py_STR(empty), \
+            .v.Name.ctx = Load, \
+            .lineno = 1, \
+            .col_offset = 0, \
+            .end_lineno = 1, \
+            .end_col_offset = 0, \
+        }, \
+    }
+#endif
 
 extern struct _mod* _PyParser_ASTFromString(
     const char *str,

Lib/_android_support.py

@@ -1,19 +1,20 @@
 import io
 import sys
-
+from threading import RLock
+from time import sleep, time
 
 # The maximum length of a log message in bytes, including the level marker and
-# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in
-# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages
-# longer than this will be be truncated by logcat. This limit has already been
-# reduced at least once in the history of Android (from 4076 to 4068 between API
-# level 23 and 26), so leave some headroom.
+# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71.
+# Messages longer than this will be be truncated by logcat. This limit has already
+# been reduced at least once in the history of Android (from 4076 to 4068 between
+# API level 23 and 26), so leave some headroom.
 MAX_BYTES_PER_WRITE = 4000
 
 # UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
-# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE.
+# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE.
 # However, if the actual number of bytes per character is smaller than that,
-# then TextIOWrapper may still join multiple consecutive text writes into binary
+# then we may still join multiple consecutive text writes into binary
 # writes containing a larger number of characters.
 MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
 
@@ -26,18 +27,22 @@ def init_streams(android_log_write, stdout_prio, stderr_prio):
     if sys.executable:
         return  # Not embedded in an app.
 
+    global logcat
+    logcat = Logcat(android_log_write)
+
     sys.stdout = TextLogStream(
-        android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors)
+        stdout_prio, "python.stdout", errors=sys.stdout.errors)
     sys.stderr = TextLogStream(
-        android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors)
+        stderr_prio, "python.stderr", errors=sys.stderr.errors)
 
 
 class TextLogStream(io.TextIOWrapper):
-    def __init__(self, android_log_write, prio, tag, **kwargs):
+    def __init__(self, prio, tag, **kwargs):
         kwargs.setdefault("encoding", "UTF-8")
-        kwargs.setdefault("line_buffering", True)
-        super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs)
-        self._CHUNK_SIZE = MAX_BYTES_PER_WRITE
+        super().__init__(BinaryLogStream(prio, tag), **kwargs)
+        self._lock = RLock()
+        self._pending_bytes = []
+        self._pending_bytes_count = 0
 
     def __repr__(self):
         return f"<TextLogStream {self.buffer.tag!r}>"
@@ -52,19 +57,48 @@ def write(self, s):
         s = str.__str__(s)
 
         # We want to emit one log message per line wherever possible, so split
-        # the string before sending it to the superclass. Note that
-        # "".splitlines() == [], so nothing will be logged for an empty string.
-        for line in s.splitlines(keepends=True):
-            while line:
-                super().write(line[:MAX_CHARS_PER_WRITE])
-                line = line[MAX_CHARS_PER_WRITE:]
+        # the string into lines first. Note that "".splitlines() == [], so
+        # nothing will be logged for an empty string.
+        with self._lock:
+            for line in s.splitlines(keepends=True):
+                while line:
+                    chunk = line[:MAX_CHARS_PER_WRITE]
+                    line = line[MAX_CHARS_PER_WRITE:]
+                    self._write_chunk(chunk)
 
         return len(s)
 
+    # The size and behavior of TextIOWrapper's buffer is not part of its public
+    # API, so we handle buffering ourselves to avoid truncation.
+    def _write_chunk(self, s):
+        b = s.encode(self.encoding, self.errors)
+        if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE:
+            self.flush()
+
+        self._pending_bytes.append(b)
+        self._pending_bytes_count += len(b)
+        if (
+            self.write_through
+            or b.endswith(b"\n")
+            or self._pending_bytes_count > MAX_BYTES_PER_WRITE
+        ):
+            self.flush()
+
+    def flush(self):
+        with self._lock:
+            self.buffer.write(b"".join(self._pending_bytes))
+            self._pending_bytes.clear()
+            self._pending_bytes_count = 0
+
+    # Since this is a line-based logging system, line buffering cannot be turned
+    # off, i.e. a newline always causes a flush.
+    @property
+    def line_buffering(self):
+        return True
+
 
 class BinaryLogStream(io.RawIOBase):
-    def __init__(self, android_log_write, prio, tag):
-        self.android_log_write = android_log_write
+    def __init__(self, prio, tag):
         self.prio = prio
         self.tag = tag
 
@@ -85,10 +119,48 @@ def write(self, b):
 
         # Writing an empty string to the stream should have no effect.
         if b:
-            # Encode null bytes using "modified UTF-8" to avoid truncating the
-            # message. This should not affect the return value, as the caller
-            # may be expecting it to match the length of the input.
-            self.android_log_write(self.prio, self.tag,
-                                   b.replace(b"\x00", b"\xc0\x80"))
-
+            logcat.write(self.prio, self.tag, b)
         return len(b)
+
+
+# When a large volume of data is written to logcat at once, e.g. when a test
+# module fails in --verbose3 mode, there's a risk of overflowing logcat's own
+# buffer and losing messages. We avoid this by imposing a rate limit using the
+# token bucket algorithm, based on a conservative estimate of how fast `adb
+# logcat` can consume data.
+MAX_BYTES_PER_SECOND = 1024 * 1024
+
+# The logcat buffer size of a device can be determined by running `logcat -g`.
+# We set the token bucket size to half of the buffer size of our current minimum
+# API level, because other things on the system will be producing messages as
+# well.
+BUCKET_SIZE = 128 * 1024
+
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
+PER_MESSAGE_OVERHEAD = 28
+
+
+class Logcat:
+    def __init__(self, android_log_write):
+        self.android_log_write = android_log_write
+        self._lock = RLock()
+        self._bucket_level = 0
+        self._prev_write_time = time()
+
+    def write(self, prio, tag, message):
+        # Encode null bytes using "modified UTF-8" to avoid them truncating the
+        # message.
+        message = message.replace(b"\x00", b"\xc0\x80")
+
+        with self._lock:
+            now = time()
+            self._bucket_level += (
+                (now - self._prev_write_time) * MAX_BYTES_PER_SECOND)
+            self._bucket_level = min(self._bucket_level, BUCKET_SIZE)
+            self._prev_write_time = now
+
+            self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message)
+            if self._bucket_level < 0:
+                sleep(-self._bucket_level / MAX_BYTES_PER_SECOND)
+
+            self.android_log_write(prio, tag, message)

Lib/idlelib/configdialog.py

@@ -111,7 +111,7 @@ def create_widgets(self):
             load_configs: Load pages except for extensions.
             activate_config_changes: Tell editors to reload.
         """
-        self.frame = frame = Frame(self, padding="5px")
+        self.frame = frame = Frame(self, padding=5)
         self.frame.grid(sticky="nwes")
         self.note = note = Notebook(frame)
         self.extpage = ExtPage(note)

Lib/idlelib/searchbase.py

@@ -86,7 +86,7 @@ def create_widgets(self):
         top.wm_iconname(self.icon)
         _setup_dialog(top)
         self.top = top
-        self.frame = Frame(top, padding="5px")
+        self.frame = Frame(top, padding=5)
         self.frame.grid(sticky="nwes")
         top.grid_columnconfigure(0, weight=100)
         top.grid_rowconfigure(0, weight=100)