From a135a6d2c6d503b186695f01efa7eed65611b04e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 11 Dec 2023 11:44:22 +0000 Subject: [PATCH 01/95] gh-112943: Correctly compute end offsets for multiline tokens in the tokenize module (#112949) --- Lib/test/test_tokenize.py | 10 ++++++++++ ...023-12-11-00-50-00.gh-issue-112943.RHNZie.rst | 2 ++ Parser/pegen.c | 16 +++++++++++----- Parser/pegen.h | 1 + Python/Python-tokenize.c | 2 +- 5 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 290f4608c5e739..21e8637a7ca905 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -615,6 +615,16 @@ def test_string(self): OP '}' (3, 0) (3, 1) FSTRING_MIDDLE '__' (3, 1) (3, 3) FSTRING_END "'" (3, 3) (3, 4) + """) + + self.check_tokenize("""\ + '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli + aktualni pracownicy, obecni pracownicy''' +""", """\ + INDENT ' ' (1, 0) (1, 4) + STRING "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\\n aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45) + NEWLINE '\\n' (2, 45) (2, 46) + DEDENT '' (3, 0) (3, 0) """) def test_function(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst new file mode 100644 index 00000000000000..4bc2fe7c26d904 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst @@ -0,0 +1,2 @@ +Correctly compute end column offsets for multiline tokens in the +:mod:`tokenize` module. Patch by Pablo Galindo diff --git a/Parser/pegen.c b/Parser/pegen.c index 0c60394e4f199b..7766253a76066f 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -19,12 +19,8 @@ _PyPegen_interactive_exit(Parser *p) } Py_ssize_t -_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) +_PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset) { - const char *str = PyUnicode_AsUTF8(line); - if (!str) { - return -1; - } Py_ssize_t len = strlen(str); if (col_offset > len + 1) { col_offset = len + 1; @@ -39,6 +35,16 @@ _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) return size; } +Py_ssize_t +_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) +{ + const char *str = PyUnicode_AsUTF8(line); + if (!str) { + return -1; + } + return _PyPegen_byte_offset_to_character_offset_raw(str, col_offset); +} + // Here, mark is the start of the node, while p->mark is the end. // If node==NULL, they should be the same. int diff --git a/Parser/pegen.h b/Parser/pegen.h index 424f80acd7be3b..57b45a54c36c57 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -149,6 +149,7 @@ expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset); +Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset); // Error handling functions and APIs typedef enum { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 364fe55d0a05e4..a7891709b3b44a 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -225,7 +225,7 @@ tokenizeriter_next(tokenizeriterobject *it) col_offset = _PyPegen_byte_offset_to_character_offset(line, token.start - line_start); } if (token.end != NULL && token.end >= it->tok->line_start) { - end_col_offset = _PyPegen_byte_offset_to_character_offset(line, token.end - it->tok->line_start); + end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, token.end - it->tok->line_start); } if (it->tok->tok_extra_tokens) { From 97cd45bfdbb6525457ba9d6824386f1e0eea6657 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 11 Dec 2023 06:48:52 -0500 Subject: [PATCH 02/95] Fix SyntaxWarning in test_syntax.py (GH-112944) --- Lib/test/test_syntax.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 99433df73387d0..ece1366076798c 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2336,7 +2336,7 @@ def test_error_parenthesis(self): # Examples with dencodings s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' - self._check_error(s, "'\(' was never closed") + self._check_error(s, r"'\(' was never closed") def test_error_string_literal(self): From c27e9d5d17abf468ea10081e177ba673316b1b98 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 11 Dec 2023 14:14:36 +0000 Subject: [PATCH 03/95] GH-111485: Factor out generation of uop IDs from cases generator. (GH-112877) --- .gitattributes | 1 + Include/internal/pycore_opcode_metadata.h | 97 +------- Include/internal/pycore_uop_ids.h | 268 ++++++++++++++++++++++ Makefile.pre.in | 3 + Tools/cases_generator/analyzer.py | 2 + Tools/cases_generator/generate_cases.py | 2 +- Tools/cases_generator/tier1_generator.py | 2 +- Tools/cases_generator/uop_id_generator.py | 91 ++++++++ 8 files changed, 368 insertions(+), 98 deletions(-) create mode 100644 Include/internal/pycore_uop_ids.h create mode 100644 Tools/cases_generator/uop_id_generator.py diff --git a/.gitattributes b/.gitattributes index acfd62411542f1..22afffb05abb20 100644 --- a/.gitattributes +++ b/.gitattributes @@ -76,6 +76,7 @@ Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated +Include/internal/pycore_uop_ids.h generated Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 774c0f99379ed6..1f460640a1e398 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -25,102 +25,7 @@ ((OP) == POP_BLOCK) || \ 0) -#define _EXIT_TRACE 300 -#define _SET_IP 301 -#define _SPECIALIZE_TO_BOOL 302 -#define _TO_BOOL 303 -#define _GUARD_BOTH_INT 304 -#define _BINARY_OP_MULTIPLY_INT 305 -#define _BINARY_OP_ADD_INT 306 -#define _BINARY_OP_SUBTRACT_INT 307 -#define _GUARD_BOTH_FLOAT 308 -#define _BINARY_OP_MULTIPLY_FLOAT 309 -#define _BINARY_OP_ADD_FLOAT 310 -#define _BINARY_OP_SUBTRACT_FLOAT 311 -#define _GUARD_BOTH_UNICODE 312 -#define _BINARY_OP_ADD_UNICODE 313 -#define _BINARY_OP_INPLACE_ADD_UNICODE 314 -#define _SPECIALIZE_BINARY_SUBSCR 315 -#define _BINARY_SUBSCR 316 -#define _SPECIALIZE_STORE_SUBSCR 317 -#define _STORE_SUBSCR 318 -#define _POP_FRAME 319 -#define _SPECIALIZE_SEND 320 -#define _SEND 321 -#define _SPECIALIZE_UNPACK_SEQUENCE 322 -#define _UNPACK_SEQUENCE 323 -#define _SPECIALIZE_STORE_ATTR 324 -#define _STORE_ATTR 325 -#define _SPECIALIZE_LOAD_GLOBAL 326 -#define _LOAD_GLOBAL 327 -#define _GUARD_GLOBALS_VERSION 328 -#define _GUARD_BUILTINS_VERSION 329 -#define _LOAD_GLOBAL_MODULE 330 -#define _LOAD_GLOBAL_BUILTINS 331 -#define _SPECIALIZE_LOAD_SUPER_ATTR 332 -#define _LOAD_SUPER_ATTR 333 -#define _SPECIALIZE_LOAD_ATTR 334 -#define _LOAD_ATTR 335 -#define _GUARD_TYPE_VERSION 336 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 -#define _LOAD_ATTR_INSTANCE_VALUE 338 -#define _CHECK_ATTR_MODULE 339 -#define _LOAD_ATTR_MODULE 340 -#define _CHECK_ATTR_WITH_HINT 341 -#define _LOAD_ATTR_WITH_HINT 342 -#define _LOAD_ATTR_SLOT 343 -#define _CHECK_ATTR_CLASS 344 -#define _LOAD_ATTR_CLASS 345 -#define _GUARD_DORV_VALUES 346 -#define _STORE_ATTR_INSTANCE_VALUE 347 -#define _STORE_ATTR_SLOT 348 -#define _SPECIALIZE_COMPARE_OP 349 -#define _COMPARE_OP 350 -#define _POP_JUMP_IF_FALSE 351 -#define _POP_JUMP_IF_TRUE 352 -#define _IS_NONE 353 -#define _SPECIALIZE_FOR_ITER 354 -#define _FOR_ITER 355 -#define _FOR_ITER_TIER_TWO 356 -#define _ITER_CHECK_LIST 357 -#define _ITER_JUMP_LIST 358 -#define _GUARD_NOT_EXHAUSTED_LIST 359 -#define _ITER_NEXT_LIST 360 -#define _ITER_CHECK_TUPLE 361 -#define _ITER_JUMP_TUPLE 362 -#define _GUARD_NOT_EXHAUSTED_TUPLE 363 -#define _ITER_NEXT_TUPLE 364 -#define _ITER_CHECK_RANGE 365 -#define _ITER_JUMP_RANGE 366 -#define _GUARD_NOT_EXHAUSTED_RANGE 367 -#define _ITER_NEXT_RANGE 368 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 369 -#define _GUARD_KEYS_VERSION 370 -#define _LOAD_ATTR_METHOD_WITH_VALUES 371 -#define _LOAD_ATTR_METHOD_NO_DICT 372 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 373 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 374 -#define _CHECK_ATTR_METHOD_LAZY_DICT 375 -#define _LOAD_ATTR_METHOD_LAZY_DICT 376 -#define _SPECIALIZE_CALL 377 -#define _CALL 378 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 379 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 380 -#define _CHECK_PEP_523 381 -#define _CHECK_FUNCTION_EXACT_ARGS 382 -#define _CHECK_STACK_SPACE 383 -#define _INIT_CALL_PY_EXACT_ARGS 384 -#define _PUSH_FRAME 385 -#define _SPECIALIZE_BINARY_OP 386 -#define _BINARY_OP 387 -#define _GUARD_IS_TRUE_POP 388 -#define _GUARD_IS_FALSE_POP 389 -#define _GUARD_IS_NONE_POP 390 -#define _GUARD_IS_NOT_NONE_POP 391 -#define _JUMP_TO_TOP 392 -#define _SAVE_RETURN_OFFSET 393 -#define _INSERT 394 -#define _CHECK_VALIDITY 395 +#include "pycore_uop_ids.h" extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h new file mode 100644 index 00000000000000..03e089953698c9 --- /dev/null +++ b/Include/internal/pycore_uop_ids.h @@ -0,0 +1,268 @@ +// This file is generated by Tools/cases_generator/uop_id_generator.py +// from: +// ['./Python/bytecodes.c'] +// Do not edit! +#ifndef Py_CORE_UOP_IDS_H +#define Py_CORE_UOP_IDS_H +#ifdef __cplusplus +extern "C" { +#endif + +#define _EXIT_TRACE 300 +#define _SET_IP 301 +#define _NOP NOP +#define _RESUME RESUME +#define _RESUME_CHECK RESUME_CHECK +#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME +#define _LOAD_FAST_CHECK LOAD_FAST_CHECK +#define _LOAD_FAST LOAD_FAST +#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR +#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _LOAD_CONST LOAD_CONST +#define _STORE_FAST STORE_FAST +#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST +#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _POP_TOP POP_TOP +#define _PUSH_NULL PUSH_NULL +#define _INSTRUMENTED_END_FOR INSTRUMENTED_END_FOR +#define _END_SEND END_SEND +#define _INSTRUMENTED_END_SEND INSTRUMENTED_END_SEND +#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _UNARY_NOT UNARY_NOT +#define _SPECIALIZE_TO_BOOL 302 +#define _TO_BOOL 303 +#define _TO_BOOL_BOOL TO_BOOL_BOOL +#define _TO_BOOL_INT TO_BOOL_INT +#define _TO_BOOL_LIST TO_BOOL_LIST +#define _TO_BOOL_NONE TO_BOOL_NONE +#define _TO_BOOL_STR TO_BOOL_STR +#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE +#define _UNARY_INVERT UNARY_INVERT +#define _GUARD_BOTH_INT 304 +#define _BINARY_OP_MULTIPLY_INT 305 +#define _BINARY_OP_ADD_INT 306 +#define _BINARY_OP_SUBTRACT_INT 307 +#define _GUARD_BOTH_FLOAT 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_ADD_FLOAT 310 +#define _BINARY_OP_SUBTRACT_FLOAT 311 +#define _GUARD_BOTH_UNICODE 312 +#define _BINARY_OP_ADD_UNICODE 313 +#define _BINARY_OP_INPLACE_ADD_UNICODE 314 +#define _SPECIALIZE_BINARY_SUBSCR 315 +#define _BINARY_SUBSCR 316 +#define _BINARY_SLICE BINARY_SLICE +#define _STORE_SLICE STORE_SLICE +#define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT +#define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT +#define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT +#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT +#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM +#define _LIST_APPEND LIST_APPEND +#define _SET_ADD SET_ADD +#define _SPECIALIZE_STORE_SUBSCR 317 +#define _STORE_SUBSCR 318 +#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT +#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT +#define _DELETE_SUBSCR DELETE_SUBSCR +#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 +#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 +#define _RAISE_VARARGS RAISE_VARARGS +#define _INTERPRETER_EXIT INTERPRETER_EXIT +#define _POP_FRAME 319 +#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE +#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _GET_AITER GET_AITER +#define _GET_ANEXT GET_ANEXT +#define _GET_AWAITABLE GET_AWAITABLE +#define _SPECIALIZE_SEND 320 +#define _SEND 321 +#define _SEND_GEN SEND_GEN +#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE +#define _YIELD_VALUE YIELD_VALUE +#define _POP_EXCEPT POP_EXCEPT +#define _RERAISE RERAISE +#define _END_ASYNC_FOR END_ASYNC_FOR +#define _CLEANUP_THROW CLEANUP_THROW +#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR +#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS +#define _STORE_NAME STORE_NAME +#define _DELETE_NAME DELETE_NAME +#define _SPECIALIZE_UNPACK_SEQUENCE 322 +#define _UNPACK_SEQUENCE 323 +#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE +#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE +#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNPACK_EX UNPACK_EX +#define _SPECIALIZE_STORE_ATTR 324 +#define _STORE_ATTR 325 +#define _DELETE_ATTR DELETE_ATTR +#define _STORE_GLOBAL STORE_GLOBAL +#define _DELETE_GLOBAL DELETE_GLOBAL +#define _LOAD_LOCALS LOAD_LOCALS +#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS +#define _LOAD_NAME LOAD_NAME +#define _SPECIALIZE_LOAD_GLOBAL 326 +#define _LOAD_GLOBAL 327 +#define _GUARD_GLOBALS_VERSION 328 +#define _GUARD_BUILTINS_VERSION 329 +#define _LOAD_GLOBAL_MODULE 330 +#define _LOAD_GLOBAL_BUILTINS 331 +#define _DELETE_FAST DELETE_FAST +#define _MAKE_CELL MAKE_CELL +#define _DELETE_DEREF DELETE_DEREF +#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF +#define _LOAD_DEREF LOAD_DEREF +#define _STORE_DEREF STORE_DEREF +#define _COPY_FREE_VARS COPY_FREE_VARS +#define _BUILD_STRING BUILD_STRING +#define _BUILD_TUPLE BUILD_TUPLE +#define _BUILD_LIST BUILD_LIST +#define _LIST_EXTEND LIST_EXTEND +#define _SET_UPDATE SET_UPDATE +#define _BUILD_SET BUILD_SET +#define _BUILD_MAP BUILD_MAP +#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS +#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP +#define _DICT_UPDATE DICT_UPDATE +#define _DICT_MERGE DICT_MERGE +#define _MAP_ADD MAP_ADD +#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _SPECIALIZE_LOAD_SUPER_ATTR 332 +#define _LOAD_SUPER_ATTR 333 +#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR +#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD +#define _SPECIALIZE_LOAD_ATTR 334 +#define _LOAD_ATTR 335 +#define _GUARD_TYPE_VERSION 336 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 +#define _LOAD_ATTR_INSTANCE_VALUE 338 +#define _CHECK_ATTR_MODULE 339 +#define _LOAD_ATTR_MODULE 340 +#define _CHECK_ATTR_WITH_HINT 341 +#define _LOAD_ATTR_WITH_HINT 342 +#define _LOAD_ATTR_SLOT 343 +#define _CHECK_ATTR_CLASS 344 +#define _LOAD_ATTR_CLASS 345 +#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN +#define _GUARD_DORV_VALUES 346 +#define _STORE_ATTR_INSTANCE_VALUE 347 +#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT +#define _STORE_ATTR_SLOT 348 +#define _SPECIALIZE_COMPARE_OP 349 +#define _COMPARE_OP 350 +#define _COMPARE_OP_FLOAT COMPARE_OP_FLOAT +#define _COMPARE_OP_INT COMPARE_OP_INT +#define _COMPARE_OP_STR COMPARE_OP_STR +#define _IS_OP IS_OP +#define _CONTAINS_OP CONTAINS_OP +#define _CHECK_EG_MATCH CHECK_EG_MATCH +#define _CHECK_EXC_MATCH CHECK_EXC_MATCH +#define _IMPORT_NAME IMPORT_NAME +#define _IMPORT_FROM IMPORT_FROM +#define _JUMP_FORWARD JUMP_FORWARD +#define _JUMP_BACKWARD JUMP_BACKWARD +#define _ENTER_EXECUTOR ENTER_EXECUTOR +#define _POP_JUMP_IF_FALSE 351 +#define _POP_JUMP_IF_TRUE 352 +#define _IS_NONE 353 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _GET_LEN GET_LEN +#define _MATCH_CLASS MATCH_CLASS +#define _MATCH_MAPPING MATCH_MAPPING +#define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _MATCH_KEYS MATCH_KEYS +#define _GET_ITER GET_ITER +#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER +#define _SPECIALIZE_FOR_ITER 354 +#define _FOR_ITER 355 +#define _FOR_ITER_TIER_TWO 356 +#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER +#define _ITER_CHECK_LIST 357 +#define _ITER_JUMP_LIST 358 +#define _GUARD_NOT_EXHAUSTED_LIST 359 +#define _ITER_NEXT_LIST 360 +#define _ITER_CHECK_TUPLE 361 +#define _ITER_JUMP_TUPLE 362 +#define _GUARD_NOT_EXHAUSTED_TUPLE 363 +#define _ITER_NEXT_TUPLE 364 +#define _ITER_CHECK_RANGE 365 +#define _ITER_JUMP_RANGE 366 +#define _GUARD_NOT_EXHAUSTED_RANGE 367 +#define _ITER_NEXT_RANGE 368 +#define _FOR_ITER_GEN FOR_ITER_GEN +#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH +#define _BEFORE_WITH BEFORE_WITH +#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _PUSH_EXC_INFO PUSH_EXC_INFO +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 369 +#define _GUARD_KEYS_VERSION 370 +#define _LOAD_ATTR_METHOD_WITH_VALUES 371 +#define _LOAD_ATTR_METHOD_NO_DICT 372 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 373 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 374 +#define _CHECK_ATTR_METHOD_LAZY_DICT 375 +#define _LOAD_ATTR_METHOD_LAZY_DICT 376 +#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _SPECIALIZE_CALL 377 +#define _CALL 378 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 379 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 380 +#define _CHECK_PEP_523 381 +#define _CHECK_FUNCTION_EXACT_ARGS 382 +#define _CHECK_STACK_SPACE 383 +#define _INIT_CALL_PY_EXACT_ARGS 384 +#define _PUSH_FRAME 385 +#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS +#define _CALL_TYPE_1 CALL_TYPE_1 +#define _CALL_STR_1 CALL_STR_1 +#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT +#define _EXIT_INIT_CHECK EXIT_INIT_CHECK +#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS +#define _CALL_BUILTIN_O CALL_BUILTIN_O +#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS +#define _CALL_LEN CALL_LEN +#define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_LIST_APPEND CALL_LIST_APPEND +#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS +#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST +#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW +#define _CALL_KW CALL_KW +#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX +#define _CALL_FUNCTION_EX CALL_FUNCTION_EX +#define _MAKE_FUNCTION MAKE_FUNCTION +#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE +#define _RETURN_GENERATOR RETURN_GENERATOR +#define _BUILD_SLICE BUILD_SLICE +#define _CONVERT_VALUE CONVERT_VALUE +#define _FORMAT_SIMPLE FORMAT_SIMPLE +#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC +#define _COPY COPY +#define _SPECIALIZE_BINARY_OP 386 +#define _BINARY_OP 387 +#define _SWAP SWAP +#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION +#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD +#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE +#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE +#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE +#define _GUARD_IS_TRUE_POP 388 +#define _GUARD_IS_FALSE_POP 389 +#define _GUARD_IS_NONE_POP 390 +#define _GUARD_IS_NOT_NONE_POP 391 +#define _JUMP_TO_TOP 392 +#define _SAVE_RETURN_OFFSET 393 +#define _INSERT 394 +#define _CHECK_VALIDITY 395 + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_IDS_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 42a7545be7e666..4317c947aeb919 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1595,10 +1595,13 @@ regen-cases: $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) \ $(srcdir)/Tools/cases_generator/opcode_id_generator.py -o $(srcdir)/Include/opcode_ids.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) \ + $(srcdir)/Tools/cases_generator/uop_id_generator.py -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) \ $(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 027f9861a1c0eb..945de63d209935 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -109,6 +109,7 @@ class Uop: body: list[lexer.Token] properties: Properties _size: int = -1 + implicitly_created: bool = False def dump(self, indent: str) -> None: print( @@ -328,6 +329,7 @@ def desugar_inst( assert inst.kind == "inst" name = inst.name uop = make_uop("_" + inst.name, inst) + uop.implicitly_created = True uops[inst.name] = uop add_instruction(name, [uop], instructions) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index d0fdc4a0aeb7b0..f7c362131a7a9f 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -381,7 +381,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_pseudo_instrs() self.out.emit("") - self.write_uop_items(lambda name, counter: f"#define {name} {counter}") + self.out.emit('#include "pycore_uop_ids.h"') self.write_stack_effect_functions() diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 9787403b3bbc47..56ae660a686822 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -404,7 +404,7 @@ def generate_tier1( if __name__ == "__main__": args = arg_parser.parse_args() if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) + args.input.append(DEFAULT_INPUT.as_posix()) data = analyze_files(args.input) with open(args.output, "w") as outfile: generate_tier1(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py new file mode 100644 index 00000000000000..4a96dbc171ee22 --- /dev/null +++ b/Tools/cases_generator/uop_id_generator.py @@ -0,0 +1,91 @@ +"""Generate the list of uop IDs. +Reads the instruction definitions from bytecodes.c. +Writes the IDs to pycore_uop_ids.h by default. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, +) +from cwriter import CWriter +from typing import TextIO + + +DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_ids.h" + + +def generate_uop_ids( + filenames: str, analysis: Analysis, outfile: TextIO, distinct_namespace: bool +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + out.emit( + """#ifndef Py_CORE_UOP_IDS_H +#define Py_CORE_UOP_IDS_H +#ifdef __cplusplus +extern "C" { +#endif + +""" + ) + + next_id = 1 if distinct_namespace else 300 + # These two are first by convention + out.emit(f"#define _EXIT_TRACE {next_id}\n") + next_id += 1 + out.emit(f"#define _SET_IP {next_id}\n") + next_id += 1 + PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP", "_CACHE", "_RESERVED", "_EXTENDED_ARG"} + + for uop in analysis.uops.values(): + if uop.name in PRE_DEFINED: + continue + if uop.implicitly_created and not distinct_namespace: + out.emit(f"#define {uop.name} {uop.name[1:]}\n") + else: + out.emit(f"#define {uop.name} {next_id}\n") + next_id += 1 + + out.emit("\n") + out.emit("#ifdef __cplusplus\n") + out.emit("}\n") + out.emit("#endif\n") + out.emit("#endif /* !Py_OPCODE_IDS_H */\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with all uop IDs.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) +arg_parser.add_argument( + "-n", + "--namespace", + help="Give uops a distinct namespace", + action="store_true", +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT.as_posix()) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_uop_ids(args.input, data, outfile, args.namespace) From 3251ba8f1af535bf28e31a6832511ba19e96b262 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 11 Dec 2023 15:22:36 +0100 Subject: [PATCH 04/95] gh-112898: warn about unsaved files when quitting IDLE on macOS (#112939) * gh-112898: warn about unsaved files when quitting IDLE on macOS Implement the TK function ``::tk::mac::Quit`` on macOS to ensure that IDLE asks about saving unsaved files when quitting IDLE. Co-authored-by: Christopher Chavez chrischavez@gmx.us --- Lib/idlelib/macosx.py | 2 +- .../next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index 2ea02ec04d661a..332952f4572cbd 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -221,7 +221,7 @@ def help_dialog(event=None): # The binding above doesn't reliably work on all versions of Tk # on macOS. Adding command definition below does seem to do the # right thing for now. - root.createcommand('exit', flist.close_all_callback) + root.createcommand('::tk::mac::Quit', flist.close_all_callback) if isCarbonTk(): # for Carbon AquaTk, replace the default Tk apple menu diff --git a/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst b/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst new file mode 100644 index 00000000000000..1c20e46b1e5f7b --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst @@ -0,0 +1 @@ +Fix processing unsaved files when quitting IDLE on macOS. From 27a5fd8cb8c88537216d7a498eba9d9177951d76 Mon Sep 17 00:00:00 2001 From: Sidney Markowitz Date: Tue, 12 Dec 2023 05:21:18 +1300 Subject: [PATCH 05/95] gh-94606: Fix error when message with Unicode surrogate not surrogateescaped string (GH-94641) Co-authored-by: Serhiy Storchaka --- Lib/email/message.py | 29 ++++++++++--------- Lib/email/utils.py | 4 +-- Lib/test/test_email/test_message.py | 29 +++++++++++++++++++ ...2-07-07-05-37-53.gh-issue-94606.hojJ54.rst | 3 ++ 4 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst diff --git a/Lib/email/message.py b/Lib/email/message.py index 411118c74dabb4..fe769580fed5d0 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -289,25 +289,26 @@ def get_payload(self, i=None, decode=False): # cte might be a Header, so for now stringify it. cte = str(self.get('content-transfer-encoding', '')).lower() # payload may be bytes here. - if isinstance(payload, str): - if utils._has_surrogates(payload): - bpayload = payload.encode('ascii', 'surrogateescape') - if not decode: + if not decode: + if isinstance(payload, str) and utils._has_surrogates(payload): + try: + bpayload = payload.encode('ascii', 'surrogateescape') try: payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') - elif decode: - try: - bpayload = payload.encode('ascii') - except UnicodeError: - # This won't happen for RFC compliant messages (messages - # containing only ASCII code points in the unicode input). - # If it does happen, turn the string into bytes in a way - # guaranteed not to fail. - bpayload = payload.encode('raw-unicode-escape') - if not decode: + except UnicodeEncodeError: + pass return payload + if isinstance(payload, str): + try: + bpayload = payload.encode('ascii', 'surrogateescape') + except UnicodeEncodeError: + # This won't happen for RFC compliant messages (messages + # containing only ASCII code points in the unicode input). + # If it does happen, turn the string into bytes in a way + # guaranteed not to fail. + bpayload = payload.encode('raw-unicode-escape') if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': diff --git a/Lib/email/utils.py b/Lib/email/utils.py index a49a8fa986ce0c..9175f2fdb6e69e 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -44,10 +44,10 @@ escapesre = re.compile(r'[\\"]') def _has_surrogates(s): - """Return True if s contains surrogate-escaped binary data.""" + """Return True if s may contain surrogate-escaped binary data.""" # This check is based on the fact that unless there are surrogates, utf8 # (Python's default encoding) can encode any string. This is the fastest - # way to check for surrogates, see issue 11454 for timings. + # way to check for surrogates, see bpo-11454 (moved to gh-55663) for timings. try: s.encode() return False diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index d3f396f02e7a72..034f7626c1fc7c 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -748,6 +748,35 @@ def test_iter_attachments_mutation(self): self.assertEqual(len(list(m.iter_attachments())), 2) self.assertEqual(m.get_payload(), orig) + get_payload_surrogate_params = { + + 'good_surrogateescape': ( + "String that can be encod\udcc3\udcabd with surrogateescape", + b'String that can be encod\xc3\xabd with surrogateescape' + ), + + 'string_with_utf8': ( + "String with utf-8 charactër", + b'String with utf-8 charact\xebr' + ), + + 'surrogate_and_utf8': ( + "String that cannot be ëncod\udcc3\udcabd with surrogateescape", + b'String that cannot be \xebncod\\udcc3\\udcabd with surrogateescape' + ), + + 'out_of_range_surrogate': ( + "String with \udfff cannot be encoded with surrogateescape", + b'String with \\udfff cannot be encoded with surrogateescape' + ), + } + + def get_payload_surrogate_as_gh_94606(self, msg, expected): + """test for GH issue 94606""" + m = self._str_msg(msg) + payload = m.get_payload(decode=True) + self.assertEqual(expected, payload) + class TestEmailMessage(TestEmailMessageBase, TestEmailBase): message = EmailMessage diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst new file mode 100644 index 00000000000000..5201ab7d842088 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst @@ -0,0 +1,3 @@ +Fix UnicodeEncodeError when :func:`email.message.get_payload` reads a message +with a Unicode surrogate character and the message content is not well-formed for +surrogateescape encoding. Patch by Sidney Markowitz. From f4fe65e2dd7eda33c098c8af3a1974c5f7f11ab7 Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Mon, 11 Dec 2023 10:43:07 -0600 Subject: [PATCH 06/95] gh-111178: Avoid calling functions from incompatible pointer types in memoryobject.c (GH-112863) * Make memory_clear() compatible with inquiry * Make memory_traverse() compatible with traverseproc * Make memory_dealloc() compatible with destructor * Make memory_repr() compatible with reprfunc * Make memory_hash() compatible with hashfunc * Make memoryiter_next() compatible with iternextfunc * Make memoryiter_traverse() compatible with traverseproc * Make memoryiter_dealloc() compatible with destructor * Make several functions compatible with getter * Make a few functions compatible with getter * Make memory_item() compatible with ssizeargfunc * Make memory_subscript() compatible with binaryfunc * Make memory_length() compatible with lenfunc * Make memory_ass_sub() compatible with objobjargproc * Make memory_releasebuf() compatible with releasebufferproc * Make memory_getbuf() compatible with getbufferproc * Make mbuf_clear() compatible with inquiry * Make mbuf_traverse() compatible with traverseproc * Make mbuf_dealloc() compatible with destructor --- Objects/memoryobject.c | 151 ++++++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 61 deletions(-) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index bcdd2ff0ceafe6..6a38952fdc1f3b 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -119,8 +119,9 @@ mbuf_release(_PyManagedBufferObject *self) } static void -mbuf_dealloc(_PyManagedBufferObject *self) +mbuf_dealloc(PyObject *_self) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; assert(self->exports == 0); mbuf_release(self); if (self->flags&_Py_MANAGED_BUFFER_FREE_FORMAT) @@ -129,15 +130,17 @@ mbuf_dealloc(_PyManagedBufferObject *self) } static int -mbuf_traverse(_PyManagedBufferObject *self, visitproc visit, void *arg) +mbuf_traverse(PyObject *_self, visitproc visit, void *arg) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; Py_VISIT(self->master.obj); return 0; } static int -mbuf_clear(_PyManagedBufferObject *self) +mbuf_clear(PyObject *_self) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; assert(self->exports >= 0); mbuf_release(self); return 0; @@ -148,7 +151,7 @@ PyTypeObject _PyManagedBuffer_Type = { "managedbuffer", sizeof(_PyManagedBufferObject), 0, - (destructor)mbuf_dealloc, /* tp_dealloc */ + mbuf_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -165,8 +168,8 @@ PyTypeObject _PyManagedBuffer_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)mbuf_traverse, /* tp_traverse */ - (inquiry)mbuf_clear /* tp_clear */ + mbuf_traverse, /* tp_traverse */ + mbuf_clear /* tp_clear */ }; @@ -1137,8 +1140,9 @@ memoryview_release_impl(PyMemoryViewObject *self) } static void -memory_dealloc(PyMemoryViewObject *self) +memory_dealloc(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; assert(self->exports == 0); _PyObject_GC_UNTRACK(self); (void)_memory_release(self); @@ -1149,15 +1153,17 @@ memory_dealloc(PyMemoryViewObject *self) } static int -memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) +memory_traverse(PyObject *_self, visitproc visit, void *arg) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_VISIT(self->mbuf); return 0; } static int -memory_clear(PyMemoryViewObject *self) +memory_clear(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; (void)_memory_release(self); Py_CLEAR(self->mbuf); return 0; @@ -1510,8 +1516,9 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) /**************************************************************************/ static int -memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) +memory_getbuf(PyObject *_self, Py_buffer *view, int flags) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *base = &self->view; int baseflags = self->flags; @@ -1589,8 +1596,9 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) } static void -memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) +memory_releasebuf(PyObject *_self, Py_buffer *view) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; self->exports--; return; /* PyBuffer_Release() decrements view->obj after this function returns. */ @@ -1598,8 +1606,8 @@ memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) /* Buffer methods */ static PyBufferProcs memory_as_buffer = { - (getbufferproc)memory_getbuf, /* bf_getbuffer */ - (releasebufferproc)memory_releasebuf, /* bf_releasebuffer */ + memory_getbuf, /* bf_getbuffer */ + memory_releasebuf, /* bf_releasebuffer */ }; @@ -2344,8 +2352,9 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, } static PyObject * -memory_repr(PyMemoryViewObject *self) +memory_repr(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; if (self->flags & _Py_MEMORYVIEW_RELEASED) return PyUnicode_FromFormat("", self); else @@ -2421,8 +2430,9 @@ ptr_from_tuple(const Py_buffer *view, PyObject *tup) with the type specified by view->format. Otherwise, the item is a sub-view. The function is used in memory_subscript() and memory_as_sequence. */ static PyObject * -memory_item(PyMemoryViewObject *self, Py_ssize_t index) +memory_item(PyObject *_self, Py_ssize_t index) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); const char *fmt; @@ -2546,8 +2556,9 @@ is_multiindex(PyObject *key) 0-d memoryview objects can be referenced using mv[...] or mv[()] but not with anything else. */ static PyObject * -memory_subscript(PyMemoryViewObject *self, PyObject *key) +memory_subscript(PyObject *_self, PyObject *key) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view; view = &(self->view); @@ -2575,7 +2586,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) index = PyNumber_AsSsize_t(key, PyExc_IndexError); if (index == -1 && PyErr_Occurred()) return NULL; - return memory_item(self, index); + return memory_item((PyObject *)self, index); } else if (PySlice_Check(key)) { CHECK_RESTRICTED(self); @@ -2608,8 +2619,9 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) } static int -memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) +memory_ass_sub(PyObject *_self, PyObject *key, PyObject *value) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); Py_buffer src; const char *fmt; @@ -2710,8 +2722,9 @@ memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) } static Py_ssize_t -memory_length(PyMemoryViewObject *self) +memory_length(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED_INT(self); if (self->view.ndim == 0) { PyErr_SetString(PyExc_TypeError, "0-dim memory has no length"); @@ -2722,17 +2735,17 @@ memory_length(PyMemoryViewObject *self) /* As mapping */ static PyMappingMethods memory_as_mapping = { - (lenfunc)memory_length, /* mp_length */ - (binaryfunc)memory_subscript, /* mp_subscript */ - (objobjargproc)memory_ass_sub, /* mp_ass_subscript */ + memory_length, /* mp_length */ + memory_subscript, /* mp_subscript */ + memory_ass_sub, /* mp_ass_subscript */ }; /* As sequence */ static PySequenceMethods memory_as_sequence = { - (lenfunc)memory_length, /* sq_length */ + memory_length, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ - (ssizeargfunc)memory_item, /* sq_item */ + memory_item, /* sq_item */ }; @@ -3034,8 +3047,9 @@ memory_richcompare(PyObject *v, PyObject *w, int op) /**************************************************************************/ static Py_hash_t -memory_hash(PyMemoryViewObject *self) +memory_hash(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; if (self->hash == -1) { Py_buffer *view = &self->view; char *mem = view->buf; @@ -3112,8 +3126,9 @@ _IntTupleFromSsizet(int len, Py_ssize_t *vals) } static PyObject * -memory_obj_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_obj_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &self->view; CHECK_RELEASED(self); @@ -3124,78 +3139,89 @@ memory_obj_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) } static PyObject * -memory_nbytes_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_nbytes_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromSsize_t(self->view.len); } static PyObject * -memory_format_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_format_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyUnicode_FromString(self->view.format); } static PyObject * -memory_itemsize_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_itemsize_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromSsize_t(self->view.itemsize); } static PyObject * -memory_shape_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_shape_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.shape); } static PyObject * -memory_strides_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_strides_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.strides); } static PyObject * -memory_suboffsets_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_suboffsets_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets); } static PyObject * -memory_readonly_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_readonly_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(self->view.readonly); } static PyObject * -memory_ndim_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_ndim_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromLong(self->view.ndim); } static PyObject * -memory_c_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_c_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_C_CONTIGUOUS(self->flags)); } static PyObject * -memory_f_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_f_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_F_CONTIGUOUS(self->flags)); } static PyObject * -memory_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_ANY_CONTIGUOUS(self->flags)); } @@ -3232,18 +3258,18 @@ PyDoc_STRVAR(memory_contiguous_doc, static PyGetSetDef memory_getsetlist[] = { - {"obj", (getter)memory_obj_get, NULL, memory_obj_doc}, - {"nbytes", (getter)memory_nbytes_get, NULL, memory_nbytes_doc}, - {"readonly", (getter)memory_readonly_get, NULL, memory_readonly_doc}, - {"itemsize", (getter)memory_itemsize_get, NULL, memory_itemsize_doc}, - {"format", (getter)memory_format_get, NULL, memory_format_doc}, - {"ndim", (getter)memory_ndim_get, NULL, memory_ndim_doc}, - {"shape", (getter)memory_shape_get, NULL, memory_shape_doc}, - {"strides", (getter)memory_strides_get, NULL, memory_strides_doc}, - {"suboffsets", (getter)memory_suboffsets_get, NULL, memory_suboffsets_doc}, - {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, - {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, - {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, + {"obj", memory_obj_get, NULL, memory_obj_doc}, + {"nbytes", memory_nbytes_get, NULL, memory_nbytes_doc}, + {"readonly", memory_readonly_get, NULL, memory_readonly_doc}, + {"itemsize", memory_itemsize_get, NULL, memory_itemsize_doc}, + {"format", memory_format_get, NULL, memory_format_doc}, + {"ndim", memory_ndim_get, NULL, memory_ndim_doc}, + {"shape", memory_shape_get, NULL, memory_shape_doc}, + {"strides", memory_strides_get, NULL, memory_strides_doc}, + {"suboffsets", memory_suboffsets_get, NULL, memory_suboffsets_doc}, + {"c_contiguous", memory_c_contiguous, NULL, memory_c_contiguous_doc}, + {"f_contiguous", memory_f_contiguous, NULL, memory_f_contiguous_doc}, + {"contiguous", memory_contiguous, NULL, memory_contiguous_doc}, {NULL, NULL, NULL, NULL}, }; @@ -3276,23 +3302,26 @@ typedef struct { } memoryiterobject; static void -memoryiter_dealloc(memoryiterobject *it) +memoryiter_dealloc(PyObject *self) { + memoryiterobject *it = (memoryiterobject *)self; _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -memoryiter_traverse(memoryiterobject *it, visitproc visit, void *arg) +memoryiter_traverse(PyObject *self, visitproc visit, void *arg) { + memoryiterobject *it = (memoryiterobject *)self; Py_VISIT(it->it_seq); return 0; } static PyObject * -memoryiter_next(memoryiterobject *it) +memoryiter_next(PyObject *self) { + memoryiterobject *it = (memoryiterobject *)self; PyMemoryViewObject *seq; seq = it->it_seq; if (seq == NULL) { @@ -3347,7 +3376,7 @@ memory_iter(PyObject *seq) return NULL; } it->it_fmt = fmt; - it->it_length = memory_length(obj); + it->it_length = memory_length((PyObject *)obj); it->it_index = 0; it->it_seq = (PyMemoryViewObject*)Py_NewRef(obj); _PyObject_GC_TRACK(it); @@ -3359,12 +3388,12 @@ PyTypeObject _PyMemoryIter_Type = { .tp_name = "memory_iterator", .tp_basicsize = sizeof(memoryiterobject), // methods - .tp_dealloc = (destructor)memoryiter_dealloc, + .tp_dealloc = memoryiter_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)memoryiter_traverse, + .tp_traverse = memoryiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)memoryiter_next, + .tp_iternext = memoryiter_next, }; PyTypeObject PyMemoryView_Type = { @@ -3372,16 +3401,16 @@ PyTypeObject PyMemoryView_Type = { "memoryview", /* tp_name */ offsetof(PyMemoryViewObject, ob_array), /* tp_basicsize */ sizeof(Py_ssize_t), /* tp_itemsize */ - (destructor)memory_dealloc, /* tp_dealloc */ + memory_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)memory_repr, /* tp_repr */ + memory_repr, /* tp_repr */ 0, /* tp_as_number */ &memory_as_sequence, /* tp_as_sequence */ &memory_as_mapping, /* tp_as_mapping */ - (hashfunc)memory_hash, /* tp_hash */ + memory_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -3390,8 +3419,8 @@ PyTypeObject PyMemoryView_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_SEQUENCE, /* tp_flags */ memoryview__doc__, /* tp_doc */ - (traverseproc)memory_traverse, /* tp_traverse */ - (inquiry)memory_clear, /* tp_clear */ + memory_traverse, /* tp_traverse */ + memory_clear, /* tp_clear */ memory_richcompare, /* tp_richcompare */ offsetof(PyMemoryViewObject, weakreflist),/* tp_weaklistoffset */ memory_iter, /* tp_iter */ From d7b5f102319bb0389c5248e9ecf533eae4163424 Mon Sep 17 00:00:00 2001 From: James Morris <6653392+J-M0@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:27:15 -0500 Subject: [PATCH 07/95] gh-112507: Detect Cygwin and MSYS with `uname` instead of `$OSTYPE` (GH-112508) Detect Cygwin and MSYS with `uname` instead of `$OSTYPE` `$OSTYPE` is not defined by POSIX and may not be present in other shells. `uname` is always available in any shell. --- Lib/venv/scripts/common/activate | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 6fdf423a1c516a..a4e0609045a9d5 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -36,14 +36,18 @@ deactivate () { deactivate nondestructive # on Windows, a path can contain colons and backslashes and has to be converted: -if [ "${OSTYPE:-}" = "cygwin" ] || [ "${OSTYPE:-}" = "msys" ] ; then - # transform D:\path\to\venv to /d/path/to/venv on MSYS - # and to /cygdrive/d/path/to/venv on Cygwin - export VIRTUAL_ENV=$(cygpath "__VENV_DIR__") -else - # use the path as-is - export VIRTUAL_ENV="__VENV_DIR__" -fi +case "$(uname)" in + CYGWIN*|MSYS*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS + # and to /cygdrive/d/path/to/venv on Cygwin + VIRTUAL_ENV=$(cygpath "__VENV_DIR__") + export VIRTUAL_ENV + ;; + *) + # use the path as-is + export VIRTUAL_ENV="__VENV_DIR__" + ;; +esac _OLD_VIRTUAL_PATH="$PATH" PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" From 0738b9a338fd27ff2d4456dd9c15801a8858ffd9 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Dec 2023 21:29:43 +0300 Subject: [PATCH 08/95] gh-108303: Move `double_const` to `test_import` where it belongs (#112108) --- Lib/test/test_import/__init__.py | 9 ++++++--- Lib/test/{ => test_import/data}/double_const.py | 0 2 files changed, 6 insertions(+), 3 deletions(-) rename Lib/test/{ => test_import/data}/double_const.py (100%) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 1ecac4f37fe1c1..bbfbb57b1d8299 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -409,9 +409,12 @@ def test_case_sensitivity(self): import RAnDoM def test_double_const(self): - # Another brief digression to test the accuracy of manifest float - # constants. - from test import double_const # don't blink -- that *was* the test + # Importing double_const checks that float constants + # serialiazed by marshal as PYC files don't lose precision + # (SF bug 422177). + from test.test_import.data import double_const + unload('test.test_import.data.double_const') + from test.test_import.data import double_const def test_import(self): def test_with_extension(ext): diff --git a/Lib/test/double_const.py b/Lib/test/test_import/data/double_const.py similarity index 100% rename from Lib/test/double_const.py rename to Lib/test/test_import/data/double_const.py From d70e27f25886e3ac1aa9fcc2d44dd38b4001d8bb Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 11 Dec 2023 13:33:21 -0500 Subject: [PATCH 09/95] gh-112529: Use atomic operations for `gcstate->collecting` (#112533) * gh-112529: Use atomic operations for `gcstate->collecting` The `collecting` field in `GCState` is used to prevent overlapping garbage collections within the same interpreter. This is updated to use atomic operations in order to be thread-safe in `--disable-gil` builds. The GC code is refactored a bit to support this. More of the logic is pushed down to `gc_collect_main()` so that we can safely order the logic setting `collecting`, the selection of the generation, and the invocation of callbacks with respect to the atomic operations and the (future) stop-the-world pauses. The change uses atomic operations for both `--disable-gil` and the default build (with the GIL) to avoid extra `#ifdef` guards and ease the maintenance burden. --- Modules/gcmodule.c | 352 ++++++++++++++++++++++----------------------- 1 file changed, 168 insertions(+), 184 deletions(-) diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 8233fc56159b60..2d1f381e622226 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -74,6 +74,20 @@ module gc #define AS_GC(op) _Py_AS_GC(op) #define FROM_GC(gc) _Py_FROM_GC(gc) +// Automatically choose the generation that needs collecting. +#define GENERATION_AUTO (-1) + +typedef enum { + // GC was triggered by heap allocation + _Py_GC_REASON_HEAP, + + // GC was called during shutdown + _Py_GC_REASON_SHUTDOWN, + + // GC was called by gc.collect() or PyGC_Collect() + _Py_GC_REASON_MANUAL +} _PyGC_Reason; + static inline int gc_is_collecting(PyGC_Head *g) @@ -1194,19 +1208,122 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, gc_list_merge(resurrected, old_generation); } + +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, Py_ssize_t collected, + Py_ssize_t uncollectable) +{ + assert(!_PyErr_Occurred(tstate)); + + /* we may get called very early */ + GCState *gcstate = &tstate->interp->gc; + if (gcstate->callbacks == NULL) { + return; + } + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", collected, + "uncollectable", uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); +} + + +/* Find the oldest generation (highest numbered) where the count + * exceeds the threshold. Objects in the that generation and + * generations younger than it will be collected. */ +static int +gc_select_generation(GCState *gcstate) +{ + for (int i = NUM_GENERATIONS-1; i >= 0; i--) { + if (gcstate->generations[i].count > gcstate->generations[i].threshold) { + /* Avoid quadratic performance degradation in number + of tracked objects (see also issue #4074): + + To limit the cost of garbage collection, there are two strategies; + - make each collection faster, e.g. by scanning fewer objects + - do less collections + This heuristic is about the latter strategy. + + In addition to the various configurable thresholds, we only trigger a + full collection if the ratio + + long_lived_pending / long_lived_total + + is above a given value (hardwired to 25%). + + The reason is that, while "non-full" collections (i.e., collections of + the young and middle generations) will always examine roughly the same + number of objects -- determined by the aforementioned thresholds --, + the cost of a full collection is proportional to the total number of + long-lived objects, which is virtually unbounded. + + Indeed, it has been remarked that doing a full collection every + of object creations entails a dramatic performance + degradation in workloads which consist in creating and storing lots of + long-lived objects (e.g. building a large list of GC-tracked objects would + show quadratic performance, instead of linear as expected: see issue #4074). + + Using the above ratio, instead, yields amortized linear performance in + the total number of objects (the effect of which can be summarized + thusly: "each full garbage collection is more and more costly as the + number of objects grows, but we do fewer and fewer of them"). + + This heuristic was suggested by Martin von Löwis on python-dev in + June 2008. His original analysis and proposal can be found at: + http://mail.python.org/pipermail/python-dev/2008-June/080579.html + */ + if (i == NUM_GENERATIONS - 1 + && gcstate->long_lived_pending < gcstate->long_lived_total / 4) + continue; + return i; + } + } + return -1; +} + + /* This is the main function. Read this to understand how the * collection process works. */ static Py_ssize_t -gc_collect_main(PyThreadState *tstate, int generation, - Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, - int nofail) +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - GC_STAT_ADD(generation, collections, 1); -#ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; - } -#endif int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ @@ -1223,6 +1340,36 @@ gc_collect_main(PyThreadState *tstate, int generation, assert(gcstate->garbage != NULL); assert(!_PyErr_Occurred(tstate)); + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + + if (generation == GENERATION_AUTO) { + // Select the oldest generation that needs collecting. We will collect + // objects from that generation and all generations younger than it. + generation = gc_select_generation(gcstate); + if (generation < 0) { + // No generation needs to be collected. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } + } + + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; + } +#endif + GC_STAT_ADD(generation, collections, 1); + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, 0, 0); + } + if (gcstate->debug & DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); @@ -1342,7 +1489,7 @@ gc_collect_main(PyThreadState *tstate, int generation, } if (_PyErr_Occurred(tstate)) { - if (nofail) { + if (reason == _Py_GC_REASON_SHUTDOWN) { _PyErr_Clear(tstate); } else { @@ -1351,13 +1498,6 @@ gc_collect_main(PyThreadState *tstate, int generation, } /* Update stats */ - if (n_collected) { - *n_collected = m; - } - if (n_uncollectable) { - *n_uncollectable = n; - } - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; stats->collections++; stats->collected += m; @@ -1376,134 +1516,13 @@ gc_collect_main(PyThreadState *tstate, int generation, PyDTrace_GC_DONE(n + m); } - assert(!_PyErr_Occurred(tstate)); - return n + m; -} - -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping - */ -static void -invoke_gc_callback(PyThreadState *tstate, const char *phase, - int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) -{ - assert(!_PyErr_Occurred(tstate)); - - /* we may get called very early */ - GCState *gcstate = &tstate->interp->gc; - if (gcstate->callbacks == NULL) { - return; - } - - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", - "generation", generation, - "collected", collected, - "uncollectable", uncollectable); - if (info == NULL) { - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; - } + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, m, n); } - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; - } - - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; icallbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_WriteUnraisable(cb); - } - else { - Py_DECREF(r); - } - Py_DECREF(cb); - } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!_PyErr_Occurred(tstate)); -} - -/* Perform garbage collection of a generation and invoke - * progress callbacks. - */ -static Py_ssize_t -gc_collect_with_callback(PyThreadState *tstate, int generation) -{ assert(!_PyErr_Occurred(tstate)); - Py_ssize_t result, collected, uncollectable; - invoke_gc_callback(tstate, "start", generation, 0, 0); - result = gc_collect_main(tstate, generation, &collected, &uncollectable, 0); - invoke_gc_callback(tstate, "stop", generation, collected, uncollectable); - assert(!_PyErr_Occurred(tstate)); - return result; -} - -static Py_ssize_t -gc_collect_generations(PyThreadState *tstate) -{ - GCState *gcstate = &tstate->interp->gc; - /* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ - Py_ssize_t n = 0; - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - continue; - n = gc_collect_with_callback(tstate, i); - break; - } - } - return n; + _Py_atomic_store_int(&gcstate->collecting, 0); + return n + m; } #include "clinic/gcmodule.c.h" @@ -1574,18 +1593,7 @@ gc_collect_impl(PyObject *module, int generation) return -1; } - GCState *gcstate = &tstate->interp->gc; - Py_ssize_t n; - if (gcstate->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - gcstate->collecting = 1; - n = gc_collect_with_callback(tstate, generation); - gcstate->collecting = 0; - } - return n; + return gc_collect_main(tstate, generation, _Py_GC_REASON_MANUAL); } /*[clinic input] @@ -2124,17 +2132,9 @@ PyGC_Collect(void) } Py_ssize_t n; - if (gcstate->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - gcstate->collecting = 1; - PyObject *exc = _PyErr_GetRaisedException(tstate); - n = gc_collect_with_callback(tstate, NUM_GENERATIONS - 1); - _PyErr_SetRaisedException(tstate, exc); - gcstate->collecting = 0; - } + PyObject *exc = _PyErr_GetRaisedException(tstate); + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + _PyErr_SetRaisedException(tstate, exc); return n; } @@ -2148,16 +2148,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - GCState *gcstate = &tstate->interp->gc; - if (gcstate->collecting) { - return 0; - } - - Py_ssize_t n; - gcstate->collecting = 1; - n = gc_collect_main(tstate, NUM_GENERATIONS - 1, NULL, NULL, 1); - gcstate->collecting = 0; - return n; + return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); } void @@ -2275,10 +2266,6 @@ PyObject_IS_GC(PyObject *obj) void _Py_ScheduleGC(PyInterpreterState *interp) { - GCState *gcstate = &interp->gc; - if (gcstate->collecting == 1) { - return; - } _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); } @@ -2296,7 +2283,7 @@ _PyObject_GC_Link(PyObject *op) if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled && gcstate->generations[0].threshold && - !gcstate->collecting && + !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { _Py_ScheduleGC(tstate->interp); @@ -2306,10 +2293,7 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { - GCState *gcstate = &tstate->interp->gc; - gcstate->collecting = 1; - gc_collect_generations(tstate); - gcstate->collecting = 0; + gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); } static PyObject * From 0066ab5bc58a036b3f448cd6f9bbdd92120e39ba Mon Sep 17 00:00:00 2001 From: colorfulappl Date: Tue, 12 Dec 2023 03:27:06 +0800 Subject: [PATCH 10/95] gh-90350: Optimize builtin functions min() and max() (GH-30286) Builtin functions min() and max() now use METH_FASTCALL --- ...3-12-11-19-53-32.gh-issue-90350.-FQy3E.rst | 1 + Python/bltinmodule.c | 81 ++++++++++--------- 2 files changed, 46 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst new file mode 100644 index 00000000000000..6b7881bbd19f59 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst @@ -0,0 +1 @@ +Optimize builtin functions :func:`min` and :func:`max`. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 960bca01990c83..e54d5cbacdc96f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1766,35 +1766,27 @@ builtin_locals_impl(PyObject *module) static PyObject * -min_max(PyObject *args, PyObject *kwds, int op) +min_max(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, int op) { - PyObject *v, *it, *item, *val, *maxitem, *maxval, *keyfunc=NULL; - PyObject *emptytuple, *defaultval = NULL; - static char *kwlist[] = {"key", "default", NULL}; - const char *name = op == Py_LT ? "min" : "max"; - const int positional = PyTuple_Size(args) > 1; - int ret; + PyObject *it = NULL, *item, *val, *maxitem, *maxval, *keyfunc=NULL; + PyObject *defaultval = NULL; + static const char * const keywords[] = {"key", "default", NULL}; + static _PyArg_Parser _parser_min = {"|$OO:min", keywords, 0}; + static _PyArg_Parser _parser_max = {"|$OO:max", keywords, 0}; + const char *name = (op == Py_LT) ? "min" : "max"; + _PyArg_Parser *_parser = (op == Py_LT) ? &_parser_min : &_parser_max; - if (positional) { - v = args; - } - else if (!PyArg_UnpackTuple(args, name, 1, 1, &v)) { - if (PyExceptionClass_Check(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, "%s expected at least 1 argument, got 0", name); - } + if (nargs == 0) { + PyErr_Format(PyExc_TypeError, "%s expected at least 1 argument, got 0", name); return NULL; } - emptytuple = PyTuple_New(0); - if (emptytuple == NULL) - return NULL; - ret = PyArg_ParseTupleAndKeywords(emptytuple, kwds, - (op == Py_LT) ? "|$OO:min" : "|$OO:max", - kwlist, &keyfunc, &defaultval); - Py_DECREF(emptytuple); - if (!ret) + if (kwnames != NULL && !_PyArg_ParseStackAndKeywords(args + nargs, 0, kwnames, _parser, + &keyfunc, &defaultval)) { return NULL; + } + const int positional = nargs > 1; // False iff nargs == 1 if (positional && defaultval != NULL) { PyErr_Format(PyExc_TypeError, "Cannot specify a default for %s() with multiple " @@ -1802,9 +1794,11 @@ min_max(PyObject *args, PyObject *kwds, int op) return NULL; } - it = PyObject_GetIter(v); - if (it == NULL) { - return NULL; + if (!positional) { + it = PyObject_GetIter(args[0]); + if (it == NULL) { + return NULL; + } } if (keyfunc == Py_None) { @@ -1813,7 +1807,24 @@ min_max(PyObject *args, PyObject *kwds, int op) maxitem = NULL; /* the result */ maxval = NULL; /* the value associated with the result */ - while (( item = PyIter_Next(it) )) { + while (1) { + if (it == NULL) { + if (nargs-- <= 0) { + break; + } + item = *args++; + Py_INCREF(item); + } + else { + item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + goto Fail_it; + } + break; + } + } + /* get the value from the key function */ if (keyfunc != NULL) { val = PyObject_CallOneArg(keyfunc, item); @@ -1847,8 +1858,6 @@ min_max(PyObject *args, PyObject *kwds, int op) } } } - if (PyErr_Occurred()) - goto Fail_it; if (maxval == NULL) { assert(maxitem == NULL); if (defaultval != NULL) { @@ -1860,7 +1869,7 @@ min_max(PyObject *args, PyObject *kwds, int op) } else Py_DECREF(maxval); - Py_DECREF(it); + Py_XDECREF(it); return maxitem; Fail_it_item_and_val: @@ -1870,15 +1879,15 @@ min_max(PyObject *args, PyObject *kwds, int op) Fail_it: Py_XDECREF(maxval); Py_XDECREF(maxitem); - Py_DECREF(it); + Py_XDECREF(it); return NULL; } /* AC: cannot convert yet, waiting for *args support */ static PyObject * -builtin_min(PyObject *self, PyObject *args, PyObject *kwds) +builtin_min(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return min_max(args, kwds, Py_LT); + return min_max(args, nargs, kwnames, Py_LT); } PyDoc_STRVAR(min_doc, @@ -1893,9 +1902,9 @@ With two or more positional arguments, return the smallest argument."); /* AC: cannot convert yet, waiting for *args support */ static PyObject * -builtin_max(PyObject *self, PyObject *args, PyObject *kwds) +builtin_max(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return min_max(args, kwds, Py_GT); + return min_max(args, nargs, kwnames, Py_GT); } PyDoc_STRVAR(max_doc, @@ -3054,8 +3063,8 @@ static PyMethodDef builtin_methods[] = { BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF - {"max", _PyCFunction_CAST(builtin_max), METH_VARARGS | METH_KEYWORDS, max_doc}, - {"min", _PyCFunction_CAST(builtin_min), METH_VARARGS | METH_KEYWORDS, min_doc}, + {"max", _PyCFunction_CAST(builtin_max), METH_FASTCALL | METH_KEYWORDS, max_doc}, + {"min", _PyCFunction_CAST(builtin_min), METH_FASTCALL | METH_KEYWORDS, min_doc}, {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF From a01022af23b27a9bfb4fadbcdb60b1ddf24a7220 Mon Sep 17 00:00:00 2001 From: achhina Date: Mon, 11 Dec 2023 15:45:08 -0500 Subject: [PATCH 11/95] GH-83162: Rename re.error for better clarity. (#101677) Renamed re.error for clarity, and kept re.error for backward compatibility. Updated idlelib files at TJR's request. --------- Co-authored-by: Matthias Bussonnier Co-authored-by: Hugo van Kemenade Co-authored-by: Terry Jan Reedy --- Doc/library/re.rst | 8 +++- Doc/whatsnew/3.13.rst | 5 ++ Lib/idlelib/replace.py | 2 +- Lib/idlelib/searchengine.py | 2 +- Lib/pstats.py | 2 +- Lib/re/__init__.py | 7 +-- Lib/re/_compiler.py | 6 +-- Lib/re/_constants.py | 5 +- Lib/test/test_re.py | 47 ++++++++++--------- ...3-02-08-00-43-29.gh-issue-83162.ufdI9F.rst | 3 ++ 10 files changed, 53 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 251ec8ca0021a6..302f7224de4a7a 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1093,12 +1093,12 @@ Functions Exceptions ^^^^^^^^^^ -.. exception:: error(msg, pattern=None, pos=None) +.. exception:: PatternError(msg, pattern=None, pos=None) Exception raised when a string passed to one of the functions here is not a valid regular expression (for example, it might contain unmatched parentheses) or when some other error occurs during compilation or matching. It is never an - error if a string contains no match for a pattern. The error instance has + error if a string contains no match for a pattern. The ``PatternError`` instance has the following additional attributes: .. attribute:: msg @@ -1124,6 +1124,10 @@ Exceptions .. versionchanged:: 3.5 Added additional attributes. + .. versionchanged:: 3.13 + ``PatternError`` was originally named ``error``; the latter is kept as an alias for + backward compatibility. + .. _re-objects: Regular Expression Objects diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 00f396846e29bd..d599ba9ae6fac8 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -298,6 +298,11 @@ pdb command line option or :envvar:`PYTHONSAFEPATH` environment variable). (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) +re +-- +* Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. + :exc:`!re.error` is kept for backward compatibility. + sqlite3 ------- diff --git a/Lib/idlelib/replace.py b/Lib/idlelib/replace.py index a29ca591427491..7997f24f1b0fa6 100644 --- a/Lib/idlelib/replace.py +++ b/Lib/idlelib/replace.py @@ -120,7 +120,7 @@ def _replace_expand(self, m, repl): if self.engine.isre(): try: new = m.expand(repl) - except re.error: + except re.PatternError: self.engine.report_error(repl, 'Invalid Replace Expression') new = None else: diff --git a/Lib/idlelib/searchengine.py b/Lib/idlelib/searchengine.py index 0684142f43644a..ceb38cfaef900b 100644 --- a/Lib/idlelib/searchengine.py +++ b/Lib/idlelib/searchengine.py @@ -84,7 +84,7 @@ def getprog(self): flags = flags | re.IGNORECASE try: prog = re.compile(pat, flags) - except re.error as e: + except re.PatternError as e: self.report_error(pat, e.msg, e.pos) return None return prog diff --git a/Lib/pstats.py b/Lib/pstats.py index 51bcca84188740..2f054bb4011e7f 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -329,7 +329,7 @@ def eval_print_amount(self, sel, list, msg): if isinstance(sel, str): try: rex = re.compile(sel) - except re.error: + except re.PatternError: msg += " \n" % sel return new_list, msg new_list = [] diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index 428d1b0d5fbd87..7e8abbf6ffe155 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -117,7 +117,8 @@ U UNICODE For compatibility only. Ignored for string patterns (it is the default), and forbidden for bytes patterns. -This module also defines an exception 'error'. +This module also defines exception 'PatternError', aliased to 'error' for +backward compatibility. """ @@ -133,7 +134,7 @@ "findall", "finditer", "compile", "purge", "escape", "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", - "UNICODE", "NOFLAG", "RegexFlag", + "UNICODE", "NOFLAG", "RegexFlag", "PatternError" ] __version__ = "2.2.1" @@ -155,7 +156,7 @@ class RegexFlag: _numeric_repr_ = hex # sre exception -error = _compiler.error +PatternError = error = _compiler.PatternError # -------------------------------------------------------------------- # public interface diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index f87712d6d6f9f8..7b888f877eb3dc 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -150,7 +150,7 @@ def _compile(code, pattern, flags): if lo > MAXCODE: raise error("looks too much behind") if lo != hi: - raise error("look-behind requires fixed-width pattern") + raise PatternError("look-behind requires fixed-width pattern") emit(lo) # look behind _compile(code, av[1], flags) emit(SUCCESS) @@ -209,7 +209,7 @@ def _compile(code, pattern, flags): else: code[skipyes] = _len(code) - skipyes + 1 else: - raise error("internal: unsupported operand type %r" % (op,)) + raise PatternError(f"internal: unsupported operand type {op!r}") def _compile_charset(charset, flags, code): # compile charset subprogram @@ -235,7 +235,7 @@ def _compile_charset(charset, flags, code): else: emit(av) else: - raise error("internal: unsupported set operator %r" % (op,)) + raise PatternError(f"internal: unsupported set operator {op!r}") emit(FAILURE) def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index d8e483ac4f23b4..9c3c294ba448b4 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -20,7 +20,7 @@ # SRE standard exception (access as sre.error) # should this really be here? -class error(Exception): +class PatternError(Exception): """Exception raised for invalid regular expressions. Attributes: @@ -53,6 +53,9 @@ def __init__(self, msg, pattern=None, pos=None): super().__init__(msg) +# Backward compatibility after renaming in 3.13 +error = PatternError + class _NamedIntConstant(int): def __new__(cls, value, name): self = super(_NamedIntConstant, cls).__new__(cls, value) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 1eca22f45378df..993a7d6e264a1f 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -47,7 +47,7 @@ def recurse(actual, expect): recurse(actual, expect) def checkPatternError(self, pattern, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(pattern) with self.subTest(pattern=pattern): err = cm.exception @@ -56,7 +56,7 @@ def checkPatternError(self, pattern, errmsg, pos=None): self.assertEqual(err.pos, pos) def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.sub(pattern, repl, string) with self.subTest(pattern=pattern, repl=repl): err = cm.exception @@ -64,6 +64,9 @@ def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): if pos is not None: self.assertEqual(err.pos, pos) + def test_error_is_PatternError_alias(self): + assert re.error is re.PatternError + def test_keep_buffer(self): # See bug 14212 b = bytearray(b'x') @@ -154,7 +157,7 @@ def test_basic_re_sub(self): (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)+chr(8))) for c in 'cdehijklmopqsuwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ': with self.subTest(c): - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): self.assertEqual(re.sub('a', '\\' + c, 'a'), '\\' + c) self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest') @@ -836,10 +839,10 @@ def test_other_escapes(self): re.purge() # for warnings for c in 'ceghijklmopqyzCEFGHIJKLMNOPQRTVXY': with self.subTest(c): - self.assertRaises(re.error, re.compile, '\\%c' % c) + self.assertRaises(re.PatternError, re.compile, '\\%c' % c) for c in 'ceghijklmopqyzABCEFGHIJKLMNOPQRTVXYZ': with self.subTest(c): - self.assertRaises(re.error, re.compile, '[\\%c]' % c) + self.assertRaises(re.PatternError, re.compile, '[\\%c]' % c) def test_named_unicode_escapes(self): # test individual Unicode named escapes @@ -970,14 +973,14 @@ def test_lookbehind(self): self.assertIsNone(re.match(r'(?:(a)|(x))b(?<=(?(1)c|x))c', 'abc')) self.assertTrue(re.match(r'(?:(a)|(x))b(?<=(?(1)b|x))c', 'abc')) # Group used before defined. - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?(2)b|x))(c)') self.assertIsNone(re.match(r'(a)b(?<=(?(1)c|x))(c)', 'abc')) self.assertTrue(re.match(r'(a)b(?<=(?(1)b|x))(c)', 'abc')) # Group defined in the same lookbehind pattern - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)\2)(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?P.)(?P=a))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)\2)(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?P.)(?P=a))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') def test_ignore_case(self): self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") @@ -1318,8 +1321,8 @@ def test_sre_byte_literals(self): self.assertTrue(re.match((r"\x%02x" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0")) self.assertTrue(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z")) - self.assertRaises(re.error, re.compile, br"\u1234") - self.assertRaises(re.error, re.compile, br"\U00012345") + self.assertRaises(re.PatternError, re.compile, br"\u1234") + self.assertRaises(re.PatternError, re.compile, br"\U00012345") self.assertTrue(re.match(br"\0", b"\000")) self.assertTrue(re.match(br"\08", b"\0008")) self.assertTrue(re.match(br"\01", b"\001")) @@ -1341,8 +1344,8 @@ def test_sre_byte_class_literals(self): self.assertTrue(re.match((r"[\x%02x]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02x0]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02xz]" % i).encode(), bytes([i]))) - self.assertRaises(re.error, re.compile, br"[\u1234]") - self.assertRaises(re.error, re.compile, br"[\U00012345]") + self.assertRaises(re.PatternError, re.compile, br"[\u1234]") + self.assertRaises(re.PatternError, re.compile, br"[\U00012345]") self.checkPatternError(br"[\567]", r'octal escape value \567 outside of ' r'range 0-0o377', 1) @@ -1675,11 +1678,11 @@ def test_ascii_and_unicode_flag(self): self.assertIsNone(pat.match(b'\xe0')) # Incompatibilities self.assertRaises(ValueError, re.compile, br'\w', re.UNICODE) - self.assertRaises(re.error, re.compile, br'(?u)\w') + self.assertRaises(re.PatternError, re.compile, br'(?u)\w') self.assertRaises(ValueError, re.compile, r'\w', re.UNICODE | re.ASCII) self.assertRaises(ValueError, re.compile, r'(?u)\w', re.ASCII) self.assertRaises(ValueError, re.compile, r'(?a)\w', re.UNICODE) - self.assertRaises(re.error, re.compile, r'(?au)\w') + self.assertRaises(re.PatternError, re.compile, r'(?au)\w') def test_locale_flag(self): enc = locale.getpreferredencoding() @@ -1720,11 +1723,11 @@ def test_locale_flag(self): self.assertIsNone(pat.match(bletter)) # Incompatibilities self.assertRaises(ValueError, re.compile, '', re.LOCALE) - self.assertRaises(re.error, re.compile, '(?L)') + self.assertRaises(re.PatternError, re.compile, '(?L)') self.assertRaises(ValueError, re.compile, b'', re.LOCALE | re.ASCII) self.assertRaises(ValueError, re.compile, b'(?L)', re.ASCII) self.assertRaises(ValueError, re.compile, b'(?a)', re.LOCALE) - self.assertRaises(re.error, re.compile, b'(?aL)') + self.assertRaises(re.PatternError, re.compile, b'(?aL)') def test_scoped_flags(self): self.assertTrue(re.match(r'(?i:a)b', 'Ab')) @@ -2060,7 +2063,7 @@ def test_locale_compiled(self): self.assertIsNone(p4.match(b'\xc5\xc5')) def test_error(self): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile('(\u20ac))') err = cm.exception self.assertIsInstance(err.pattern, str) @@ -2072,14 +2075,14 @@ def test_error(self): self.assertIn(' at position 3', str(err)) self.assertNotIn(' at position 3', err.msg) # Bytes pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(b'(\xa4))') err = cm.exception self.assertIsInstance(err.pattern, bytes) self.assertEqual(err.pattern, b'(\xa4))') self.assertEqual(err.pos, 3) # Multiline pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(""" ( abc @@ -2820,7 +2823,7 @@ def test_re_tests(self): with self.subTest(pattern=pattern, string=s): if outcome == SYNTAX_ERROR: # Expected a syntax error - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): re.compile(pattern) continue diff --git a/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst b/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst new file mode 100644 index 00000000000000..6074dd7f101a6d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst @@ -0,0 +1,3 @@ +Renamed :exc:`!re.error` to :exc:`PatternError` for clarity, and kept +:exc:`!re.error` for backward compatibility. Patch by Matthias Bussonnier and +Adam Chhina. From 1c5fc02fd0576be125638a5261be12eb3224be81 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 11 Dec 2023 21:54:17 +0000 Subject: [PATCH 12/95] gh-71383: Update Tcl/Tk version in Windows to our patched build containing a targeted upstream fix (GH-112973) --- .../Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst | 2 ++ PCbuild/get_externals.bat | 6 +++--- PCbuild/tcltk.props | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst diff --git a/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst b/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst new file mode 100644 index 00000000000000..cf2883357a962a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst @@ -0,0 +1,2 @@ +Update Tcl/Tk in Windows installer to 8.6.13 with a patch to suppress +incorrect ThemeChanged warnings. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 94437f054d788c..6151990096e0be 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -55,8 +55,8 @@ set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 set libraries=%libraries% sqlite-3.43.1.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 set libraries=%libraries% xz-5.2.5 set libraries=%libraries% zlib-1.2.13 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.11 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 96dd289face6a5..8ddf01d5dd1dca 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.13.0 + 8.6.13.1 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) From fed294c6453527addd1644633849e2d8492058c5 Mon Sep 17 00:00:00 2001 From: Yan Yanchii <46005801+WolframAlph@users.noreply.github.com> Date: Tue, 12 Dec 2023 00:23:41 +0100 Subject: [PATCH 13/95] gh-112978: Remove redundant condition inside `take_gil` (gh-112979) --- Python/ceval_gil.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 636e4db898f2d9..7581daa55b5e46 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -307,10 +307,6 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->mutex); - if (!_Py_atomic_load_int_relaxed(&gil->locked)) { - goto _ready; - } - int drop_requested = 0; while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; @@ -345,7 +341,6 @@ take_gil(PyThreadState *tstate) } } -_ready: #ifdef FORCE_SWITCHING /* This mutex must be taken before modifying gil->last_holder: see drop_gil(). */ From fdee7b7b3e15931d58f07e5449de2e55b4d48b05 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 11 Dec 2023 19:04:48 -0500 Subject: [PATCH 14/95] gh-112532: Require mimalloc in `--disable-gil` builds (gh-112883) --- Include/internal/pycore_pymem_init.h | 14 +++++++++++++- Lib/test/support/__init__.py | 2 +- Lib/test/test_capi/test_mem.py | 7 +++++++ Lib/test/test_cmd_line.py | 15 ++++++++++++--- Lib/test/test_embed.py | 14 ++++++++++---- Lib/test/test_os.py | 5 ++++- Objects/obmalloc.c | 15 +++++++++++++-- Programs/_testembed.c | 12 ++++++++++++ configure | 2 ++ configure.ac | 2 ++ 10 files changed, 76 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 11fbe16fa6f1d5..360fb9218a9cda 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -18,7 +18,19 @@ extern void * _PyMem_RawRealloc(void *, void *, size_t); extern void _PyMem_RawFree(void *, void *); #define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} -#if defined(WITH_PYMALLOC) +#ifdef Py_GIL_DISABLED +// Py_GIL_DISABLED requires mimalloc +extern void* _PyObject_MiMalloc(void *, size_t); +extern void* _PyObject_MiCalloc(void *, size_t, size_t); +extern void _PyObject_MiFree(void *, void *); +extern void* _PyObject_MiRealloc(void *, void *, size_t); +# define PYOBJ_ALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree} +extern void* _PyMem_MiMalloc(void *, size_t); +extern void* _PyMem_MiCalloc(void *, size_t, size_t); +extern void _PyMem_MiFree(void *, void *); +extern void* _PyMem_MiRealloc(void *, void *, size_t); +# define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree} +#elif defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); extern void _PyObject_Free(void *, void *); diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c22d73c231b46e..b605951320dc8b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1844,7 +1844,7 @@ def restore(self): def with_pymalloc(): import _testcapi - return _testcapi.WITH_PYMALLOC + return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED def with_mimalloc(): diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 72f23b1a34080e..04f17a9ec9e72a 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -152,6 +152,8 @@ class C(): pass self.assertGreaterEqual(count, i*5-2) +# Py_GIL_DISABLED requires mimalloc (not malloc) +@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc') class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' @@ -161,6 +163,11 @@ class PyMemPymallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'pymalloc_debug' +@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc') +class PyMemMimallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'mimalloc_debug' + + @unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') class PyMemDefaultTests(PyMemDebugTests): # test default allocator of Python compiled in debug mode diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 7a27952c345b9c..1fe3b2fe53c0b6 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -738,6 +738,8 @@ def test_xdev(self): out = self.run_xdev("-c", code, check_exitcode=False) if support.with_pymalloc(): alloc_name = "pymalloc_debug" + elif support.Py_GIL_DISABLED: + alloc_name = "mimalloc_debug" else: alloc_name = "malloc_debug" self.assertEqual(out, alloc_name) @@ -814,9 +816,13 @@ def check_pythonmalloc(self, env_var, name): @support.cpython_only def test_pythonmalloc(self): # Test the PYTHONMALLOC environment variable + malloc = not support.Py_GIL_DISABLED pymalloc = support.with_pymalloc() mimalloc = support.with_mimalloc() - if pymalloc: + if support.Py_GIL_DISABLED: + default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc' + default_name_debug = 'mimalloc_debug' + elif pymalloc: default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc' default_name_debug = 'pymalloc_debug' else: @@ -826,9 +832,12 @@ def test_pythonmalloc(self): tests = [ (None, default_name), ('debug', default_name_debug), - ('malloc', 'malloc'), - ('malloc_debug', 'malloc_debug'), ] + if malloc: + tests.extend([ + ('malloc', 'malloc'), + ('malloc_debug', 'malloc_debug'), + ]) if pymalloc: tests.extend(( ('pymalloc', 'pymalloc'), diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d2d6c1b61e46f0..6c60854bbd76cc 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -23,6 +23,12 @@ PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 PYMEM_ALLOCATOR_MALLOC = 3 +PYMEM_ALLOCATOR_MIMALLOC = 7 +if support.Py_GIL_DISABLED: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MIMALLOC +else: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MALLOC + Py_STATS = hasattr(sys, '_stats_on') # _PyCoreConfig_InitCompatConfig() @@ -841,7 +847,7 @@ def test_init_global_config(self): def test_init_from_config(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, 'utf8_mode': 1, } config = { @@ -908,7 +914,7 @@ def test_init_from_config(self): def test_init_compat_env(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, } config = { 'use_hash_seed': 1, @@ -942,7 +948,7 @@ def test_init_compat_env(self): def test_init_python_env(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, 'utf8_mode': 1, } config = { @@ -984,7 +990,7 @@ def test_init_env_dev_mode(self): api=API_COMPAT) def test_init_env_dev_mode_alloc(self): - preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC) + preconfig = dict(allocator=ALLOCATOR_FOR_CONFIG) config = dict(dev_mode=1, faulthandler=1, warnoptions=['default']) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c31c9684051196..d4680ef0f0e03f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -5080,7 +5080,10 @@ def test_fork(self): support.wait_process(pid, exitcode=0) """ assert_python_ok("-c", code) - assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + if support.Py_GIL_DISABLED: + assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug") + else: + assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") @unittest.skipUnless(sys.platform in ("linux", "darwin"), "Only Linux and macOS detect this today.") diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index b737c030957564..99c95d90658b08 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -16,6 +16,10 @@ # include "mimalloc/internal.h" // for stats #endif +#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC) +# error "Py_GIL_DISABLED requires WITH_MIMALLOC" +#endif + #undef uint #define uint pymem_uint @@ -153,7 +157,12 @@ void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); # define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} #endif // WITH_PYMALLOC -#if defined(WITH_PYMALLOC) +#if defined(Py_GIL_DISABLED) +// Py_GIL_DISABLED requires using mimalloc for "mem" and "obj" domains. +# define PYRAW_ALLOC MALLOC_ALLOC +# define PYMEM_ALLOC MIMALLOC_ALLOC +# define PYOBJ_ALLOC MIMALLOC_OBJALLOC +#elif defined(WITH_PYMALLOC) # define PYRAW_ALLOC MALLOC_ALLOC # define PYMEM_ALLOC PYMALLOC_ALLOC # define PYOBJ_ALLOC PYMALLOC_ALLOC @@ -350,7 +359,7 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) else if (strcmp(name, "debug") == 0) { *allocator = PYMEM_ALLOCATOR_DEBUG; } -#ifdef WITH_PYMALLOC +#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED) else if (strcmp(name, "pymalloc") == 0) { *allocator = PYMEM_ALLOCATOR_PYMALLOC; } @@ -366,12 +375,14 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) *allocator = PYMEM_ALLOCATOR_MIMALLOC_DEBUG; } #endif +#ifndef Py_GIL_DISABLED else if (strcmp(name, "malloc") == 0) { *allocator = PYMEM_ALLOCATOR_MALLOC; } else if (strcmp(name, "malloc_debug") == 0) { *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; } +#endif else { /* unknown allocator */ return -1; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 1f9aa4b3d449a1..30998bf80f9ce4 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -576,7 +576,11 @@ static int test_init_from_config(void) _PyPreConfig_InitCompatConfig(&preconfig); putenv("PYTHONMALLOC=malloc_debug"); +#ifndef Py_GIL_DISABLED preconfig.allocator = PYMEM_ALLOCATOR_MALLOC; +#else + preconfig.allocator = PYMEM_ALLOCATOR_MIMALLOC; +#endif putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; @@ -765,7 +769,11 @@ static int test_init_dont_parse_argv(void) static void set_most_env_vars(void) { putenv("PYTHONHASHSEED=42"); +#ifndef Py_GIL_DISABLED putenv("PYTHONMALLOC=malloc"); +#else + putenv("PYTHONMALLOC=mimalloc"); +#endif putenv("PYTHONTRACEMALLOC=2"); putenv("PYTHONPROFILEIMPORTTIME=1"); putenv("PYTHONNODEBUGRANGES=1"); @@ -851,7 +859,11 @@ static int test_init_env_dev_mode_alloc(void) /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); +#ifndef Py_GIL_DISABLED putenv("PYTHONMALLOC=malloc"); +#else + putenv("PYTHONMALLOC=mimalloc"); +#endif _testembed_Py_InitializeFromConfig(); dump_config(); Py_Finalize(); diff --git a/configure b/configure index 5f880d6d8edd96..c4486441041a70 100755 --- a/configure +++ b/configure @@ -16891,6 +16891,8 @@ printf "%s\n" "#define WITH_MIMALLOC 1" >>confdefs.h MIMALLOC_HEADERS='$(MIMALLOC_HEADERS)' +elif test "$disable_gil" = "yes"; then + as_fn_error $? "--disable-gil requires mimalloc memory allocator (--with-mimalloc)." "$LINENO" 5 fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_mimalloc" >&5 diff --git a/configure.ac b/configure.ac index c07d7ce6bdc918..a725309424a49e 100644 --- a/configure.ac +++ b/configure.ac @@ -4558,6 +4558,8 @@ if test "$with_mimalloc" != no; then with_mimalloc=yes AC_DEFINE([WITH_MIMALLOC], [1], [Define if you want to compile in mimalloc memory allocator.]) AC_SUBST([MIMALLOC_HEADERS], ['$(MIMALLOC_HEADERS)']) +elif test "$disable_gil" = "yes"; then + AC_MSG_ERROR([--disable-gil requires mimalloc memory allocator (--with-mimalloc).]) fi AC_MSG_RESULT([$with_mimalloc]) From 5b8664433829ea967c150363cf49a5c4c1380fe8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 11 Dec 2023 16:42:30 -0800 Subject: [PATCH 15/95] A smattering of cleanups in uop debug output and lltrace (#112980) * Include destination T1 opcode in Error debug message * Include destination T1 opcode in DEOPT debug message * Remove obsolete comment from remove_unneeded_uops * Change lltrace_instruction() to print caller's opcode/oparg --- Python/ceval.c | 18 ++++++++++-------- Python/ceval_macros.h | 2 +- Python/optimizer_analysis.c | 1 - 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index f8fa50eb46c75e..d92ab926f84963 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -135,14 +135,14 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) static void lltrace_instruction(_PyInterpreterFrame *frame, PyObject **stack_pointer, - _Py_CODEUNIT *next_instr) + _Py_CODEUNIT *next_instr, + int opcode, + int oparg) { if (frame->owner == FRAME_OWNED_BY_CSTACK) { return; } dump_stack(frame, stack_pointer); - int oparg = next_instr->op.arg; - int opcode = next_instr->op.code; const char *opname = _PyOpcode_OpName[opcode]; assert(opname != NULL); int offset = (int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame))); @@ -1051,9 +1051,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - DPRINTF(2, "Error: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", + DPRINTF(2, "Error: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1)); + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1064,14 +1065,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int deoptimize: // On DEOPT_IF we just repeat the last instruction. // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", + frame->instr_ptr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); + DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1)); + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); frame->return_offset = 0; // Dispatch to frame->instr_ptr _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); Py_DECREF(current_executor); // Fall through // Jump here from ENTER_EXECUTOR diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b0cb7c8926338c..f298c602b1042b 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -81,7 +81,7 @@ /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE #define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \ - lltrace_instruction(frame, stack_pointer, next_instr); } + lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); } #else #define PRE_DISPATCH_GOTO() ((void)0) #endif diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0f9bc085f22f1c..8b471d70a10d7d 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -15,7 +15,6 @@ static void remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { - // Note that we don't enter stubs, those SET_IPs are needed. int last_set_ip = -1; bool maybe_invalid = false; for (int pc = 0; pc < buffer_size; pc++) { From 616622cab7200bac476bf753d6cc98f4ee48808c Mon Sep 17 00:00:00 2001 From: wim glenn Date: Mon, 11 Dec 2023 19:47:12 -0600 Subject: [PATCH 16/95] gh-112983: Add the known magic value of 3495 for Python 3.11 bytecode (#112985) Add the known magic value of 3495 for Python 3.11 bytecode --- Lib/importlib/_bootstrap_external.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 4e96a916324824..d537598ac16433 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -413,6 +413,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) # Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) # Python 3.11a7 3494 (New location info table) +# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) # Python 3.12a1 3500 (Remove PRECALL opcode) # Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) # Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) From e0fb7004ede71389c9dd462cd03352cc3c3a4d8c Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 12 Dec 2023 01:00:51 -0500 Subject: [PATCH 17/95] gh-112953: Rename idlelib/NEWS.txt to News3.txt and update (#112988) --- Lib/idlelib/{NEWS.txt => News3.txt} | 14 +++++++++++++- Lib/idlelib/help_about.py | 6 +++--- Lib/idlelib/idle_test/test_help_about.py | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) rename Lib/idlelib/{NEWS.txt => News3.txt} (99%) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/News3.txt similarity index 99% rename from Lib/idlelib/NEWS.txt rename to Lib/idlelib/News3.txt index f258797c6e0fb3..4fba4165fddab5 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/News3.txt @@ -1,9 +1,21 @@ +What's New in IDLE 3.13.0 +(since 3.12.0) +Released on 2024-10-xx +========================= + + +gh-112939: Fix processing unsaved files when quitting IDLE on macOS. +Patch by Ronald Oussoren and Christopher Chavez. + +gh-79871: Add docstrings to debugger.py. Fix two bugs in +test_debugger and expand coverage by 47%. Patch by Anthony Shaw. + + What's New in IDLE 3.12.0 (since 3.11.0) Released on 2023-10-02 ========================= - gh-104719: Remove IDLE's modification of tokenize.tabsize and test other uses of tokenize data and methods. diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index cfa4ca781f087d..aa1c352897f9e7 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -129,11 +129,11 @@ def create_widgets(self): idle.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_buttons = Frame(frame_background, bg=self.bg) idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) - self.readme = Button(idle_buttons, text='README', width=8, + self.readme = Button(idle_buttons, text='Readme', width=8, highlightbackground=self.bg, command=self.show_readme) self.readme.pack(side=LEFT, padx=10, pady=10) - self.idle_news = Button(idle_buttons, text='NEWS', width=8, + self.idle_news = Button(idle_buttons, text='News', width=8, highlightbackground=self.bg, command=self.show_idle_news) self.idle_news.pack(side=LEFT, padx=10, pady=10) @@ -167,7 +167,7 @@ def show_readme(self): def show_idle_news(self): "Handle News button event." - self.display_file_text('About - NEWS', 'NEWS.txt', 'utf-8') + self.display_file_text('About - News', 'News3.txt', 'utf-8') def display_printer_text(self, title, printer): """Create textview for built-in constants. diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 8b79487b15d4cd..7e16abdb7c9f96 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -71,7 +71,7 @@ def test_file_buttons(self): """Test buttons that display files.""" dialog = self.dialog button_sources = [(self.dialog.readme, 'README.txt', 'readme'), - (self.dialog.idle_news, 'NEWS.txt', 'news'), + (self.dialog.idle_news, 'News3.txt', 'news'), (self.dialog.idle_credits, 'CREDITS.txt', 'credits')] for button, filename, name in button_sources: From 0d2fe6bab01541301abe98a23ee15a16f493fe74 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 12 Dec 2023 10:25:51 +0100 Subject: [PATCH 18/95] gh-87286: Add a number of LOG_* constants to syslog (#24432) * bpo-43120: Add a number of LOG_* constants to syslog This adds a number of syslog facilities to the syslogmodule.c. These values are available on macOS. * Switch contant documentation to the data directive This fixes a CI warning and matches the pattern used in the documentation for ``os``. * Update Doc/library/syslog.rst Co-authored-by: Hugo van Kemenade Co-authored-by: Alex Waygood --- Doc/library/syslog.rst | 74 ++++++++++++++----- Doc/tools/.nitignore | 1 - ...3-12-07-16-55-41.gh-issue-87286.MILC9_.rst | 3 + Modules/syslogmodule.c | 24 ++++++ 4 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index b5ab446e0096ed..7b27fc7e85b62d 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -15,7 +15,7 @@ facility. This module wraps the system ``syslog`` family of routines. A pure Python library that can speak to a syslog server is available in the -:mod:`logging.handlers` module as :class:`SysLogHandler`. +:mod:`logging.handlers` module as :class:`~logging.handlers.SysLogHandler`. The module defines the following functions: @@ -107,22 +107,62 @@ The module defines the following functions: The module defines the following constants: -Priority levels (high to low): - :const:`LOG_EMERG`, :const:`LOG_ALERT`, :const:`LOG_CRIT`, :const:`LOG_ERR`, - :const:`LOG_WARNING`, :const:`LOG_NOTICE`, :const:`LOG_INFO`, - :const:`LOG_DEBUG`. - -Facilities: - :const:`LOG_KERN`, :const:`LOG_USER`, :const:`LOG_MAIL`, :const:`LOG_DAEMON`, - :const:`LOG_AUTH`, :const:`LOG_LPR`, :const:`LOG_NEWS`, :const:`LOG_UUCP`, - :const:`LOG_CRON`, :const:`LOG_SYSLOG`, :const:`LOG_LOCAL0` to - :const:`LOG_LOCAL7`, and, if defined in ````, - :const:`LOG_AUTHPRIV`. - -Log options: - :const:`LOG_PID`, :const:`LOG_CONS`, :const:`LOG_NDELAY`, and, if defined - in ````, :const:`LOG_ODELAY`, :const:`LOG_NOWAIT`, and - :const:`LOG_PERROR`. + +.. data:: LOG_EMERG + LOG_ALERT + LOG_CRIT + LOG_ERR + LOG_WARNING + LOG_NOTICE + LOG_INFO + LOG_DEBUG + + Priority levels (high to low). + + +.. data:: LOG_AUTH + LOG_AUTHPRIV + LOG_CRON + LOG_DAEMON + LOG_FTP + LOG_INSTALL + LOG_KERN + LOG_LAUNCHD + LOG_LPR + LOG_MAIL + LOG_NETINFO + LOG_NEWS + LOG_RAS + LOG_REMOTEAUTH + LOG_SYSLOG + LOG_USER + LOG_UUCP + LOG_LOCAL0 + LOG_LOCAL1 + LOG_LOCAL2 + LOG_LOCAL3 + LOG_LOCAL4 + LOG_LOCAL5 + LOG_LOCAL6 + LOG_LOCAL7 + + Facilities, depending on availability in ```` for :const:`LOG_AUTHPRIV`, + :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, + :const:`LOG_INSTALL` and :const:`LOG_RAS`. + + .. versionchanged:: 3.13 + Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, + :const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD`. + +.. data:: LOG_PID + LOG_CONS + LOG_NDELAY + LOG_ODELAY + LOG_NOWAIT + LOG_PERROR + + Log options, depending on availability in ```` for + :const:`LOG_ODELAY`, :const:`LOG_NOWAIT` and :const:`LOG_PERROR`. Examples diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 511648ab6991c6..75d50fee33a064 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -91,7 +91,6 @@ Doc/library/ssl.rst Doc/library/stdtypes.rst Doc/library/string.rst Doc/library/subprocess.rst -Doc/library/syslog.rst Doc/library/tarfile.rst Doc/library/termios.rst Doc/library/test.rst diff --git a/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst b/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst new file mode 100644 index 00000000000000..bfeec3c95207cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst @@ -0,0 +1,3 @@ +Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, +:const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD` tot the +:mod:`syslog` module, all of them constants on used on macOS. diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 6a44850e291448..62c7816f891ee2 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -406,6 +406,30 @@ syslog_exec(PyObject *module) ADD_INT_MACRO(module, LOG_AUTHPRIV); #endif +#ifdef LOG_FTP + ADD_INT_MACRO(module, LOG_FTP); +#endif + +#ifdef LOG_NETINFO + ADD_INT_MACRO(module, LOG_NETINFO); +#endif + +#ifdef LOG_REMOTEAUTH + ADD_INT_MACRO(module, LOG_REMOTEAUTH); +#endif + +#ifdef LOG_INSTALL + ADD_INT_MACRO(module, LOG_INSTALL); +#endif + +#ifdef LOG_RAS + ADD_INT_MACRO(module, LOG_RAS); +#endif + +#ifdef LOG_LAUNCHD + ADD_INT_MACRO(module, LOG_LAUNCHD); +#endif + return 0; } From c454e934d36193709aadba8e8e28739790086b95 Mon Sep 17 00:00:00 2001 From: Sam James Date: Tue, 12 Dec 2023 10:25:27 +0000 Subject: [PATCH 19/95] gh-112970: Detect and use closefrom() when available (#112969) glibc-2.34 implements closefrom(3) using the same semantics as on BSD. Check for closefrom() in configure and use the check result in fileutils.c, rather than hardcoding a FreeBSD check. Some implementations of closefrom() return an int. Explicitly discard the return value by casting it to void, to avoid future compiler warnings. Signed-off-by: Sam James --- .../Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst | 1 + Python/fileutils.c | 6 +++--- configure | 6 ++++++ configure.ac | 2 +- pyconfig.h.in | 3 +++ 5 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst diff --git a/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst b/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst new file mode 100644 index 00000000000000..58ca26af511383 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst @@ -0,0 +1 @@ +Use :c:func:`!closefrom` on Linux where available (e.g. glibc-2.34), rather than only FreeBSD. diff --git a/Python/fileutils.c b/Python/fileutils.c index 9d12bc89c95436..882d3299575cf3 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2878,9 +2878,9 @@ _Py_GetLocaleconvNumeric(struct lconv *lc, * non-opened fd in the middle. * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop. */ -#ifdef __FreeBSD__ +#ifdef HAVE_CLOSEFROM # define USE_CLOSEFROM -#endif /* __FreeBSD__ */ +#endif /* HAVE_CLOSEFROM */ #ifdef HAVE_FDWALK # define USE_FDWALK @@ -2922,7 +2922,7 @@ _Py_closerange(int first, int last) #ifdef USE_CLOSEFROM if (last >= sysconf(_SC_OPEN_MAX)) { /* Any errors encountered while closing file descriptors are ignored */ - closefrom(first); + (void)closefrom(first); } else #endif /* USE_CLOSEFROM */ diff --git a/configure b/configure index c4486441041a70..cad3bce0c7de87 100755 --- a/configure +++ b/configure @@ -17225,6 +17225,12 @@ if test "x$ac_cv_func_clock" = xyes then : printf "%s\n" "#define HAVE_CLOCK 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "closefrom" "ac_cv_func_closefrom" +if test "x$ac_cv_func_closefrom" = xyes +then : + printf "%s\n" "#define HAVE_CLOSEFROM 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "close_range" "ac_cv_func_close_range" if test "x$ac_cv_func_close_range" = xyes diff --git a/configure.ac b/configure.ac index a725309424a49e..7dda0b3fff95be 100644 --- a/configure.ac +++ b/configure.ac @@ -4745,7 +4745,7 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ - accept4 alarm bind_textdomain_codeset chmod chown clock close_range confstr \ + accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 2978fa2c17301f..9c429c03722383 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -157,6 +157,9 @@ /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME +/* Define to 1 if you have the `closefrom' function. */ +#undef HAVE_CLOSEFROM + /* Define to 1 if you have the `close_range' function. */ #undef HAVE_CLOSE_RANGE From 0c55f270604f8541bcf43526e5cf6c6eddfff451 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 12 Dec 2023 12:12:17 +0000 Subject: [PATCH 20/95] GH-111485: Factor out tier 2 code generation from the rest of the interpreter code generator (GH-112968) --- Include/internal/pycore_uop_ids.h | 2 +- Include/opcode_ids.h | 2 +- Makefile.pre.in | 3 +- Python/bytecodes.c | 3 + Python/executor_cases.c.h | 1616 ++++++++++-------- Python/generated_cases.c.h | 5 +- Tools/cases_generator/analyzer.py | 19 + Tools/cases_generator/generate_cases.py | 8 - Tools/cases_generator/generators_common.py | 175 +- Tools/cases_generator/opcode_id_generator.py | 2 +- Tools/cases_generator/stack.py | 87 + Tools/cases_generator/tier1_generator.py | 228 +-- Tools/cases_generator/tier2_generator.py | 202 +++ Tools/cases_generator/uop_id_generator.py | 13 +- 14 files changed, 1391 insertions(+), 974 deletions(-) create mode 100644 Tools/cases_generator/tier2_generator.py diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 03e089953698c9..c96ea51ae1acb6 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -1,6 +1,6 @@ // This file is generated by Tools/cases_generator/uop_id_generator.py // from: -// ['./Python/bytecodes.c'] +// Python/bytecodes.c // Do not edit! #ifndef Py_CORE_UOP_IDS_H #define Py_CORE_UOP_IDS_H diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 47f809e345f61c..e2e27ca00fd47b 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -1,6 +1,6 @@ // This file is generated by Tools/cases_generator/opcode_id_generator.py // from: -// ['./Python/bytecodes.c'] +// Python/bytecodes.c // Do not edit! #ifndef Py_OPCODE_IDS_H diff --git a/Makefile.pre.in b/Makefile.pre.in index 4317c947aeb919..5fb6ffc4e8f0cf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1589,7 +1589,6 @@ regen-cases: $(CASESFLAG) \ -t $(srcdir)/Python/opcode_targets.h.new \ -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ - -e $(srcdir)/Python/executor_cases.c.h.new \ -p $(srcdir)/Lib/_opcode_metadata.py.new \ -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ $(srcdir)/Python/bytecodes.c @@ -1599,6 +1598,8 @@ regen-cases: $(srcdir)/Tools/cases_generator/uop_id_generator.py -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) \ $(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) \ + $(srcdir)/Tools/cases_generator/tier2_generator.py -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bcad8dcf0e7dab..e0f373536ce67c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3967,6 +3967,7 @@ dummy_func( } inst(EXTENDED_ARG, ( -- )) { + TIER_ONE_ONLY assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -3975,11 +3976,13 @@ dummy_func( } inst(CACHE, (--)) { + TIER_ONE_ONLY assert(0 && "Executing a cache."); Py_UNREACHABLE(); } inst(RESERVED, (--)) { + TIER_ONE_ONLY assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 974e3f28a411b8..9dda3c9a743258 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier2_generator.py // from: // Python/bytecodes.c // Do not edit! @@ -8,102 +8,104 @@ #endif #define TIER_TWO 2 - case NOP: { + case _NOP: { break; } - case RESUME_CHECK: { -#if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + case _RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) goto deoptimize; _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif + #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + if (eval_breaker != version) goto deoptimize; break; } - case LOAD_FAST_CHECK: { - oparg = CURRENT_OPARG(); + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + + case _LOAD_FAST_CHECK: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error_tier_two; Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_FAST: { - oparg = CURRENT_OPARG(); + case _LOAD_FAST: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_FAST_AND_CLEAR: { - oparg = CURRENT_OPARG(); + case _LOAD_FAST_AND_CLEAR: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_CONST: { - oparg = CURRENT_OPARG(); + case _LOAD_CONST: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case STORE_FAST: { - oparg = CURRENT_OPARG(); + case _STORE_FAST: { PyObject *value; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; SETLOCAL(oparg, value); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case POP_TOP: { + case _POP_TOP: { PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case PUSH_NULL: { + case _PUSH_NULL: { PyObject *res; res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case END_SEND: { + case _END_SEND: { PyObject *value; PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; break; } - case UNARY_NEGATIVE: { + case _UNARY_NEGATIVE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -114,7 +116,7 @@ break; } - case UNARY_NOT: { + case _UNARY_NOT: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -136,19 +138,19 @@ break; } - case TO_BOOL_BOOL: { + case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + if (!PyBool_Check(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); break; } - case TO_BOOL_INT: { + case _TO_BOOL_INT: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + if (!PyLong_CheckExact(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -162,11 +164,11 @@ break; } - case TO_BOOL_LIST: { + case _TO_BOOL_LIST: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + if (!PyList_CheckExact(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -174,23 +176,23 @@ break; } - case TO_BOOL_NONE: { + case _TO_BOOL_NONE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + if (!Py_IsNone(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; break; } - case TO_BOOL_STR: { + case _TO_BOOL_STR: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + if (!PyUnicode_CheckExact(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -205,14 +207,14 @@ break; } - case TO_BOOL_ALWAYS_TRUE: { + case _TO_BOOL_ALWAYS_TRUE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; uint32_t version = (uint32_t)CURRENT_OPERAND(); // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + if (Py_TYPE(value)->tp_version_tag != version) goto deoptimize; STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; @@ -220,7 +222,7 @@ break; } - case UNARY_INVERT: { + case _UNARY_INVERT: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -236,8 +238,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); - DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); + if (!PyLong_CheckExact(left)) goto deoptimize; + if (!PyLong_CheckExact(right)) goto deoptimize; break; } @@ -252,8 +254,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -268,8 +270,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -284,8 +286,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -294,8 +296,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); - DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); + if (!PyFloat_CheckExact(left)) goto deoptimize; + if (!PyFloat_CheckExact(right)) goto deoptimize; break; } @@ -307,11 +309,11 @@ left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -323,11 +325,11 @@ left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -339,11 +341,11 @@ left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -352,8 +354,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); - DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); + if (!PyUnicode_CheckExact(left)) goto deoptimize; + if (!PyUnicode_CheckExact(right)) goto deoptimize; break; } @@ -368,8 +370,40 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_OP_INPLACE_ADD_UNICODE: { + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + if (*target_local != left) goto deoptimize; + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error_tier_two; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + stack_pointer += -2; break; } @@ -383,12 +417,12 @@ Py_DECREF(container); Py_DECREF(sub); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SLICE: { + case _BINARY_SLICE: { PyObject *stop; PyObject *start; PyObject *container; @@ -408,12 +442,12 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error_tier_two; - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; break; } - case STORE_SLICE: { + case _STORE_SLICE: { PyObject *stop; PyObject *start; PyObject *container; @@ -434,88 +468,86 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error_tier_two; - STACK_SHRINK(4); + stack_pointer += -4; break; } - case BINARY_SUBSCR_LIST_INT: { + case _BINARY_SUBSCR_LIST_INT: { PyObject *sub; PyObject *list; PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyList_CheckExact(list)) goto deoptimize; // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + if (index >= PyList_GET_SIZE(list)) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SUBSCR_STR_INT: { + case _BINARY_SUBSCR_STR_INT: { PyObject *sub; PyObject *str; PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyUnicode_CheckExact(str)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + if (PyUnicode_GET_LENGTH(str) <= index) goto deoptimize; // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SUBSCR_TUPLE_INT: { + case _BINARY_SUBSCR_TUPLE_INT: { PyObject *sub; PyObject *tuple; PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyTuple_CheckExact(tuple)) goto deoptimize; // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + if (index >= PyTuple_GET_SIZE(tuple)) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SUBSCR_DICT: { + case _BINARY_SUBSCR_DICT: { PyObject *sub; PyObject *dict; PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + if (!PyDict_CheckExact(dict)) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -524,32 +556,35 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case LIST_APPEND: { - oparg = CURRENT_OPARG(); + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + + case _LIST_APPEND: { PyObject *v; PyObject *list; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case SET_ADD: { - oparg = CURRENT_OPARG(); + case _SET_ADD: { PyObject *v; PyObject *set; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } @@ -566,54 +601,52 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_3_error_tier_two; - STACK_SHRINK(3); + stack_pointer += -3; break; } - case STORE_SUBSCR_LIST_INT: { + case _STORE_SUBSCR_LIST_INT: { PyObject *sub; PyObject *list; PyObject *value; sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyList_CheckExact(list)) goto deoptimize; // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + if (index >= PyList_GET_SIZE(list)) goto deoptimize; STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(3); + stack_pointer += -3; break; } - case STORE_SUBSCR_DICT: { + case _STORE_SUBSCR_DICT: { PyObject *sub; PyObject *dict; PyObject *value; sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + if (!PyDict_CheckExact(dict)) goto deoptimize; STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error_tier_two; - STACK_SHRINK(3); + stack_pointer += -3; break; } - case DELETE_SUBSCR: { + case _DELETE_SUBSCR: { PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -623,14 +656,14 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_2_error_tier_two; - STACK_SHRINK(2); + stack_pointer += -2; break; } - case CALL_INTRINSIC_1: { - oparg = CURRENT_OPARG(); + case _CALL_INTRINSIC_1: { PyObject *value; PyObject *res; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); @@ -640,11 +673,11 @@ break; } - case CALL_INTRINSIC_2: { - oparg = CURRENT_OPARG(); + case _CALL_INTRINSIC_2: { PyObject *value1; PyObject *value2; PyObject *res; + oparg = CURRENT_OPARG(); value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); @@ -652,19 +685,31 @@ Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } + case _INTERPRETER_EXIT: { + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; + } + case _POP_FRAME: { PyObject *retval; retval = stack_pointer[-1]; - STACK_SHRINK(1); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: @@ -674,26 +719,28 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif break; } - case GET_AITER: { + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ + + case _GET_AITER: { PyObject *obj; PyObject *iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " @@ -702,14 +749,11 @@ Py_DECREF(obj); if (true) goto pop_1_error_tier_two; } - iter = (*getter)(obj); Py_DECREF(obj); if (iter == NULL) goto pop_1_error_tier_two; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", @@ -721,14 +765,13 @@ break; } - case GET_ANEXT: { + case _GET_ANEXT: { PyObject *aiter; PyObject *awaitable; aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { @@ -738,7 +781,6 @@ if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { @@ -752,7 +794,6 @@ type->tp_name); GOTO_ERROR(error); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( @@ -760,31 +801,27 @@ "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); GOTO_ERROR(error); } else { Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[0] = awaitable; + stack_pointer += 1; break; } - case GET_AWAITABLE: { - oparg = CURRENT_OPARG(); + case _GET_AWAITABLE: { PyObject *iterable; PyObject *iter; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { @@ -798,30 +835,62 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = iter; break; } - case POP_EXCEPT: { + /* _SEND is not a viable micro-op for tier 2 */ + + /* _SEND_GEN is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + + case _YIELD_VALUE: { + PyObject *retval; + oparg = CURRENT_OPARG(); + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + + case _POP_EXCEPT: { PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case LOAD_ASSERTION_ERROR: { + case _LOAD_ASSERTION_ERROR: { PyObject *value; value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_BUILD_CLASS: { + case _LOAD_BUILD_CLASS: { PyObject *bc; if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two; if (bc == NULL) { @@ -829,14 +898,14 @@ "__build_class__ not found"); if (true) goto error_tier_two; } - STACK_GROW(1); - stack_pointer[-1] = bc; + stack_pointer[0] = bc; + stack_pointer += 1; break; } - case STORE_NAME: { - oparg = CURRENT_OPARG(); + case _STORE_NAME: { PyObject *v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -848,16 +917,16 @@ if (true) goto pop_1_error_tier_two; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, v); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, v); Py_DECREF(v); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case DELETE_NAME: { + case _DELETE_NAME: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -871,99 +940,95 @@ // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } break; } case _UNPACK_SEQUENCE: { - oparg = CURRENT_OPARG(); PyObject *seq; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); if (res == 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_SEQUENCE_TWO_TUPLE: { - oparg = CURRENT_OPARG(); + case _UNPACK_SEQUENCE_TWO_TUPLE: { PyObject *seq; PyObject **values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + if (!PyTuple_CheckExact(seq)) goto deoptimize; + if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize; assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_SEQUENCE_TUPLE: { - oparg = CURRENT_OPARG(); + case _UNPACK_SEQUENCE_TUPLE: { PyObject *seq; PyObject **values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + if (!PyTuple_CheckExact(seq)) goto deoptimize; + if (PyTuple_GET_SIZE(seq) != oparg) goto deoptimize; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_SEQUENCE_LIST: { - oparg = CURRENT_OPARG(); + case _UNPACK_SEQUENCE_LIST: { PyObject *seq; PyObject **values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + if (!PyList_CheckExact(seq)) goto deoptimize; + if (PyList_GET_SIZE(seq) != oparg) goto deoptimize; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_EX: { - oparg = CURRENT_OPARG(); + case _UNPACK_EX: { PyObject *seq; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error_tier_two; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; } case _STORE_ATTR: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *v; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -971,35 +1036,35 @@ Py_DECREF(v); Py_DECREF(owner); if (err) goto pop_2_error_tier_two; - STACK_SHRINK(2); + stack_pointer += -2; break; } - case DELETE_ATTR: { - oparg = CURRENT_OPARG(); + case _DELETE_ATTR: { PyObject *owner; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case STORE_GLOBAL: { - oparg = CURRENT_OPARG(); + case _STORE_GLOBAL: { PyObject *v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case DELETE_GLOBAL: { + case _DELETE_GLOBAL: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; @@ -1008,14 +1073,14 @@ if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } GOTO_ERROR(error); } break; } - case LOAD_LOCALS: { + case _LOAD_LOCALS: { PyObject *locals; locals = LOCALS(); if (locals == NULL) { @@ -1024,15 +1089,15 @@ if (true) goto error_tier_two; } Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + stack_pointer[0] = locals; + stack_pointer += 1; break; } - case LOAD_FROM_DICT_OR_GLOBALS: { - oparg = CURRENT_OPARG(); + case _LOAD_FROM_DICT_OR_GLOBALS: { PyObject *mod_or_class_dict; PyObject *v; + oparg = CURRENT_OPARG(); mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { @@ -1048,8 +1113,8 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } @@ -1059,9 +1124,9 @@ break; } - case LOAD_NAME: { - oparg = CURRENT_OPARG(); + case _LOAD_NAME: { PyObject *v; + oparg = CURRENT_OPARG(); PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1082,34 +1147,34 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } } - STACK_GROW(1); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; break; } case _LOAD_GLOBAL: { - oparg = CURRENT_OPARG(); PyObject *res; PyObject *null = NULL; + oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } if (true) goto error_tier_two; } @@ -1124,25 +1189,24 @@ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two; if (res == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); if (true) goto error_tier_two; } } } null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); break; } case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); + if (!PyDict_CheckExact(dict)) goto deoptimize; + if (dict->ma_keys->dk_version != version) goto deoptimize; assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1150,51 +1214,49 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); + if (!PyDict_CheckExact(dict)) goto deoptimize; + if (dict->ma_keys->dk_version != version) goto deoptimize; assert(DK_IS_UNICODE(dict->ma_keys)); break; } case _LOAD_GLOBAL_MODULE: { - oparg = CURRENT_OPARG(); PyObject *res; PyObject *null = NULL; + oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); + if (res == NULL) goto deoptimize; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); break; } case _LOAD_GLOBAL_BUILTINS: { - oparg = CURRENT_OPARG(); PyObject *res; PyObject *null = NULL; + oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); + if (res == NULL) goto deoptimize; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); break; } - case DELETE_FAST: { + case _DELETE_FAST: { oparg = CURRENT_OPARG(); PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error_tier_two; @@ -1202,7 +1264,7 @@ break; } - case MAKE_CELL: { + case _MAKE_CELL: { oparg = CURRENT_OPARG(); // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). @@ -1215,7 +1277,7 @@ break; } - case DELETE_DEREF: { + case _DELETE_DEREF: { oparg = CURRENT_OPARG(); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); @@ -1230,10 +1292,10 @@ break; } - case LOAD_FROM_DICT_OR_DEREF: { - oparg = CURRENT_OPARG(); + case _LOAD_FROM_DICT_OR_DEREF: { PyObject *class_dict; PyObject *value; + oparg = CURRENT_OPARG(); class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); @@ -1256,9 +1318,9 @@ break; } - case LOAD_DEREF: { - oparg = CURRENT_OPARG(); + case _LOAD_DEREF: { PyObject *value; + oparg = CURRENT_OPARG(); PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1266,24 +1328,24 @@ if (true) goto error_tier_two; } Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case STORE_DEREF: { - oparg = CURRENT_OPARG(); + case _STORE_DEREF: { PyObject *v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case COPY_FREE_VARS: { + case _COPY_FREE_VARS: { oparg = CURRENT_OPARG(); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); @@ -1298,131 +1360,126 @@ break; } - case BUILD_STRING: { - oparg = CURRENT_OPARG(); + case _BUILD_STRING: { PyObject **pieces; PyObject *str; - pieces = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + if (str == NULL) { stack_pointer += -oparg; goto error_tier_two; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; break; } - case BUILD_TUPLE: { - oparg = CURRENT_OPARG(); + case _BUILD_TUPLE: { PyObject **values; PyObject *tup; - values = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + if (tup == NULL) { stack_pointer += -oparg; goto error_tier_two; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; break; } - case BUILD_LIST: { - oparg = CURRENT_OPARG(); + case _BUILD_LIST: { PyObject **values; PyObject *list; - values = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + if (list == NULL) { stack_pointer += -oparg; goto error_tier_two; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; break; } - case LIST_EXTEND: { - oparg = CURRENT_OPARG(); + case _LIST_EXTEND: { PyObject *iterable; PyObject *list; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); if (true) goto pop_1_error_tier_two; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case SET_UPDATE: { - oparg = CURRENT_OPARG(); + case _SET_UPDATE: { PyObject *iterable; PyObject *set; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case BUILD_SET: { - oparg = CURRENT_OPARG(); + case _BUILD_SET: { PyObject **values; PyObject *set; - values = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; if (err == 0) - err = PySet_Add(set, item); + err = PySet_Add(set, item); Py_DECREF(item); } if (err != 0) { Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error_tier_two; } + if (true) { stack_pointer += -oparg; goto error_tier_two; } } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; break; } - case BUILD_MAP: { - oparg = CURRENT_OPARG(); + case _BUILD_MAP: { PyObject **values; PyObject *map; - values = stack_pointer - oparg*2; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg*2]; map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + values, 2, + values+1, 2, + oparg); for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error_tier_two; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -oparg*2; goto error_tier_two; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; break; } - case SETUP_ANNOTATIONS: { + case _SETUP_ANNOTATIONS: { int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -1446,13 +1503,13 @@ break; } - case BUILD_CONST_KEY_MAP: { - oparg = CURRENT_OPARG(); + case _BUILD_CONST_KEY_MAP: { PyObject *keys; PyObject **values; PyObject *map; + oparg = CURRENT_OPARG(); keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1460,43 +1517,43 @@ GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error_tier_two; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -1 - oparg; goto error_tier_two; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; break; } - case DICT_UPDATE: { - oparg = CURRENT_OPARG(); + case _DICT_UPDATE: { PyObject *update; PyObject *dict; + oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); } Py_DECREF(update); if (true) goto pop_1_error_tier_two; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case DICT_MERGE: { - oparg = CURRENT_OPARG(); + case _DICT_MERGE: { PyObject *update; PyObject *dict; PyObject *callable; + oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; @@ -1506,15 +1563,15 @@ if (true) goto pop_1_error_tier_two; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case MAP_ADD: { - oparg = CURRENT_OPARG(); + case _MAP_ADD: { PyObject *value; PyObject *key; PyObject *dict; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; key = stack_pointer[-2]; dict = stack_pointer[-3 - (oparg - 1)]; @@ -1522,22 +1579,24 @@ /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two; - STACK_SHRINK(2); + stack_pointer += -2; break; } - case LOAD_SUPER_ATTR_ATTR: { - oparg = CURRENT_OPARG(); + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + case _LOAD_SUPER_ATTR_ATTR: { PyObject *self; PyObject *class; PyObject *global_super; PyObject *attr; + oparg = CURRENT_OPARG(); self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; + if (!PyType_Check(class)) goto deoptimize; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1545,24 +1604,24 @@ Py_DECREF(class); Py_DECREF(self); if (attr == NULL) goto pop_3_error_tier_two; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); break; } - case LOAD_SUPER_ATTR_METHOD: { - oparg = CURRENT_OPARG(); + case _LOAD_SUPER_ATTR_METHOD: { PyObject *self; PyObject *class; PyObject *global_super; PyObject *attr; PyObject *self_or_null; + oparg = CURRENT_OPARG(); self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; + if (!PyType_Check(class)) goto deoptimize; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1581,17 +1640,17 @@ Py_DECREF(self); self_or_null = NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; break; } case _LOAD_ATTR: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -1611,7 +1670,7 @@ the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN - */ + */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error_tier_two; self_or_null = NULL; @@ -1623,9 +1682,9 @@ Py_DECREF(owner); if (attr == NULL) goto pop_1_error_tier_two; } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); break; } @@ -1635,7 +1694,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); + if (tp->tp_version_tag != type_version) goto deoptimize; break; } @@ -1645,27 +1704,27 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; break; } case _LOAD_ATTR_INSTANCE_VALUE: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); break; } @@ -1673,18 +1732,18 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); + if (!PyModule_CheckExact(owner)) goto deoptimize; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, _CHECK_ATTR_MODULE); + if (dict->ma_keys->dk_version != type_version) goto deoptimize; break; } case _LOAD_ATTR_MODULE: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; @@ -1692,14 +1751,14 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - DEOPT_IF(attr == NULL, _LOAD_ATTR_MODULE); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); break; } @@ -1708,62 +1767,62 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), _CHECK_ATTR_WITH_HINT); + if (_PyDictOrValues_IsValues(dorv)) goto deoptimize; PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, _CHECK_ATTR_WITH_HINT); + if (dict == NULL) goto deoptimize; assert(PyDict_CheckExact((PyObject *)dict)); break; } case _LOAD_ATTR_WITH_HINT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); + if (hint >= (size_t)dict->ma_keys->dk_nentries) goto deoptimize; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + if (ep->me_key != name) goto deoptimize; attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + if (ep->me_key != name) goto deoptimize; attr = ep->me_value; } - DEOPT_IF(attr == NULL, _LOAD_ATTR_WITH_HINT); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); break; } case _LOAD_ATTR_SLOT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); break; } @@ -1771,36 +1830,40 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); + if (!PyType_Check(owner)) goto deoptimize; assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); + if (((PyTypeObject *)owner)->tp_version_tag != type_version) goto deoptimize; break; } case _LOAD_ATTR_CLASS: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject * descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); break; } + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + case _GUARD_DORV_VALUES: { PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); + if (!_PyDictOrValues_IsValues(dorv)) goto deoptimize; break; } @@ -1822,10 +1885,12 @@ Py_DECREF(old_value); } Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; break; } + /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + case _STORE_ATTR_SLOT: { PyObject *owner; PyObject *value; @@ -1838,15 +1903,15 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; break; } case _COMPARE_OP: { - oparg = CURRENT_OPARG(); PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; assert((oparg >> 5) <= Py_GE); @@ -1860,20 +1925,20 @@ if (res_bool < 0) goto pop_2_error_tier_two; res = res_bool ? Py_True : Py_False; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COMPARE_OP_FLOAT: { - oparg = CURRENT_OPARG(); + case _COMPARE_OP_FLOAT: { PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + if (!PyFloat_CheckExact(left)) goto deoptimize; + if (!PyFloat_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -1883,22 +1948,22 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COMPARE_OP_INT: { - oparg = CURRENT_OPARG(); + case _COMPARE_OP_INT: { PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + if (!PyLong_CheckExact(left)) goto deoptimize; + if (!PyLong_CheckExact(right)) goto deoptimize; + if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize; + if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -1910,20 +1975,20 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COMPARE_OP_STR: { - oparg = CURRENT_OPARG(); + case _COMPARE_OP_STR: { PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + if (!PyUnicode_CheckExact(left)) goto deoptimize; + if (!PyUnicode_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -1934,32 +1999,32 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case IS_OP: { - oparg = CURRENT_OPARG(); + case _IS_OP: { PyObject *right; PyObject *left; PyObject *b; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; break; } - case CONTAINS_OP: { - oparg = CURRENT_OPARG(); + case _CONTAINS_OP: { PyObject *right; PyObject *left; PyObject *b; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; int res = PySequence_Contains(right, left); @@ -1967,12 +2032,12 @@ Py_DECREF(right); if (res < 0) goto pop_2_error_tier_two; b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; break; } - case CHECK_EG_MATCH: { + case _CHECK_EG_MATCH: { PyObject *match_type; PyObject *exc_value; PyObject *rest; @@ -1984,18 +2049,15 @@ Py_DECREF(match_type); if (true) goto pop_2_error_tier_two; } - match = NULL; rest = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error_tier_two; - assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error_tier_two; - if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -2004,7 +2066,7 @@ break; } - case CHECK_EXC_MATCH: { + case _CHECK_EXC_MATCH: { PyObject *right; PyObject *left; PyObject *b; @@ -2012,10 +2074,9 @@ left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error_tier_two; + Py_DECREF(right); + if (true) goto pop_1_error_tier_two; } - int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); b = res ? Py_True : Py_False; @@ -2023,6 +2084,18 @@ break; } + case _JUMP_FORWARD: { + oparg = CURRENT_OPARG(); + JUMPBY(oparg); + break; + } + + /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + case _IS_NONE: { PyObject *value; PyObject *b; @@ -2038,7 +2111,18 @@ break; } - case GET_LEN: { + case _JUMP_BACKWARD_NO_INTERRUPT: { + oparg = CURRENT_OPARG(); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); + break; + } + + case _GET_LEN: { PyObject *obj; PyObject *len_o; obj = stack_pointer[-1]; @@ -2047,17 +2131,17 @@ if (len_i < 0) goto error_tier_two; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = len_o; + stack_pointer[0] = len_o; + stack_pointer += 1; break; } - case MATCH_CLASS: { - oparg = CURRENT_OPARG(); + case _MATCH_CLASS: { PyObject *names; PyObject *type; PyObject *subject; PyObject *attrs; + oparg = CURRENT_OPARG(); names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; @@ -2073,36 +2157,37 @@ } else { if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two; + // Error! attrs = Py_None; // Failure! } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + stack_pointer[-3] = attrs; + stack_pointer += -2; break; } - case MATCH_MAPPING: { + case _MATCH_MAPPING: { PyObject *subject; PyObject *res; subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case MATCH_SEQUENCE: { + case _MATCH_SEQUENCE: { PyObject *subject; PyObject *res; subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case MATCH_KEYS: { + case _MATCH_KEYS: { PyObject *keys; PyObject *subject; PyObject *values_or_none; @@ -2111,12 +2196,12 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + stack_pointer[0] = values_or_none; + stack_pointer += 1; break; } - case GET_ITER: { + case _GET_ITER: { PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -2128,7 +2213,7 @@ break; } - case GET_YIELD_FROM_ITER: { + case _GET_YIELD_FROM_ITER: { PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -2160,6 +2245,8 @@ break; } + /* _FOR_ITER is not a viable micro-op for tier 2 */ + case _FOR_ITER_TIER_TWO: { PyObject *iter; PyObject *next; @@ -2177,29 +2264,33 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - DEOPT_IF(true, _FOR_ITER_TIER_TWO); + if (true) goto deoptimize; } // Common case: no jump, leave it to the code generator - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); + if (Py_TYPE(iter) != &PyListIter_Type) goto deoptimize; break; } + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + case _GUARD_NOT_EXHAUSTED_LIST: { PyObject *iter; iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_LIST); - DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_LIST); + if (seq == NULL) goto deoptimize; + if (it->it_index >= PyList_GET_SIZE(seq)) goto deoptimize; break; } @@ -2213,26 +2304,28 @@ assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); + if (Py_TYPE(iter) != &PyTupleIter_Type) goto deoptimize; break; } + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + case _GUARD_NOT_EXHAUSTED_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_TUPLE); - DEOPT_IF(it->it_index >= PyTuple_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_TUPLE); + if (seq == NULL) goto deoptimize; + if (it->it_index >= PyTuple_GET_SIZE(seq)) goto deoptimize; break; } @@ -2246,8 +2339,8 @@ assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } @@ -2255,16 +2348,18 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); + if (Py_TYPE(r) != &PyRangeIter_Type) goto deoptimize; break; } + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + case _GUARD_NOT_EXHAUSTED_RANGE: { PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - DEOPT_IF(r->len <= 0, _GUARD_NOT_EXHAUSTED_RANGE); + if (r->len <= 0) goto deoptimize; break; } @@ -2280,12 +2375,14 @@ r->len--; next = PyLong_FromLong(value); if (next == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } - case BEFORE_ASYNC_WITH: { + /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ + + case _BEFORE_ASYNC_WITH: { PyObject *mgr; PyObject *exit; PyObject *res; @@ -2319,13 +2416,13 @@ Py_DECREF(exit); if (true) goto pop_1_error_tier_two; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case BEFORE_WITH: { + case _BEFORE_WITH: { PyObject *mgr; PyObject *exit; PyObject *res; @@ -2362,13 +2459,13 @@ Py_DECREF(exit); if (true) goto pop_1_error_tier_two; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case WITH_EXCEPT_START: { + case _WITH_EXCEPT_START: { PyObject *val; PyObject *lasti; PyObject *exit_func; @@ -2383,9 +2480,8 @@ - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ + */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -2399,14 +2495,14 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case PUSH_EXC_INFO: { + case _PUSH_EXC_INFO: { PyObject *new_exc; PyObject *prev_exc; new_exc = stack_pointer[-1]; @@ -2419,9 +2515,9 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; break; } @@ -2430,7 +2526,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; break; } @@ -2440,17 +2536,17 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) goto deoptimize; break; } case _LOAD_ATTR_METHOD_WITH_VALUES: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject * descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -2458,19 +2554,19 @@ attr = Py_NewRef(descr); assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); break; } case _LOAD_ATTR_METHOD_NO_DICT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject * descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2478,33 +2574,34 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = Py_NewRef(descr); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject * descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject * descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2512,6 +2609,7 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); break; } @@ -2522,45 +2620,49 @@ assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, _CHECK_ATTR_METHOD_LAZY_DICT); + if (dict != NULL) goto deoptimize; break; } case _LOAD_ATTR_METHOD_LAZY_DICT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject * descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = Py_NewRef(descr); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); break; } + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + + /* _CALL is not a viable micro-op for tier 2 */ + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject *null; PyObject *callable; + oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + if (null != NULL) goto deoptimize; + if (Py_TYPE(callable) != &PyMethod_Type) goto deoptimize; break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject *callable; PyObject *func; PyObject *self; + oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; STAT_INC(CALL, hit); self = Py_NewRef(((PyMethodObject *)callable)->im_self); @@ -2574,43 +2676,43 @@ } case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); + if (tstate->interp->eval_frame) goto deoptimize; break; } case _CHECK_FUNCTION_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject *self_or_null; PyObject *callable; + oparg = CURRENT_OPARG(); self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); + if (!PyFunction_Check(callable)) goto deoptimize; PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); + if (func->func_version != func_version) goto deoptimize; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); + if (code->co_argcount != oparg + (self_or_null != NULL)) goto deoptimize; break; } case _CHECK_STACK_SPACE: { - oparg = CURRENT_OPARG(); PyObject *callable; + oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); - DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) goto deoptimize; + if (tstate->py_recursion_remaining <= 1) goto deoptimize; break; } case _INIT_CALL_PY_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject **args; PyObject *self_or_null; PyObject *callable; _PyInterpreterFrame *new_frame; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int argcount = oparg; @@ -2624,129 +2726,130 @@ for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = args[i]; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = (PyObject *)new_frame; + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; break; } case _PUSH_FRAME: { _PyInterpreterFrame *new_frame; new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; - STACK_SHRINK(1); // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif + stack_pointer += (((0) ? 1 : 0)); break; } - case CALL_TYPE_1: { - oparg = CURRENT_OPARG(); + /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ + + case _CALL_TYPE_1: { PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); + if (null != NULL) goto deoptimize; PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + if (callable != (PyObject *)&PyType_Type) goto deoptimize; STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case CALL_STR_1: { - oparg = CURRENT_OPARG(); + case _CALL_STR_1: { PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + if (null != NULL) goto deoptimize; + if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize; STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_TUPLE_1: { - oparg = CURRENT_OPARG(); + case _CALL_TUPLE_1: { PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + if (null != NULL) goto deoptimize; + if (callable != (PyObject *)&PyTuple_Type) goto deoptimize; STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case EXIT_INIT_CHECK: { + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + + case _EXIT_INIT_CHECK: { PyObject *should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } - STACK_SHRINK(1); + stack_pointer += -1; break; } - case CALL_BUILTIN_CLASS: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_CLASS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -2754,9 +2857,9 @@ args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); + if (!PyType_Check(callable)) goto deoptimize; PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + if (tp->tp_vectorcall == NULL) goto deoptimize; STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -2764,21 +2867,20 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_BUILTIN_O: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_O: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ @@ -2787,9 +2889,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + if (total_args != 1) goto deoptimize; + if (!PyCFunction_CheckExact(callable)) goto deoptimize; + if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -2801,24 +2903,22 @@ res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_BUILTIN_FAST: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_FAST: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ @@ -2827,8 +2927,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + if (!PyCFunction_CheckExact(callable)) goto deoptimize; + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -2837,32 +2937,30 @@ args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_BUILTIN_FAST_WITH_KEYWORDS: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -2871,36 +2969,34 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + if (!PyCFunction_CheckExact(callable)) goto deoptimize; + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) goto deoptimize; STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_LEN: { - oparg = CURRENT_OPARG(); + case _CALL_LEN: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ @@ -2909,9 +3005,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) goto deoptimize; PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + if (callable != interp->callable_cache.len) goto deoptimize; STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -2920,23 +3016,21 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case CALL_ISINSTANCE: { - oparg = CURRENT_OPARG(); + case _CALL_ISINSTANCE: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ @@ -2945,9 +3039,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + if (total_args != 2) goto deoptimize; PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + if (callable != interp->callable_cache.isinstance) goto deoptimize; STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -2957,24 +3051,48 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case CALL_METHOD_DESCRIPTOR_O: { + case _CALL_LIST_APPEND: { + PyObject **args; + PyObject *self; + PyObject *callable; oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + if (callable != interp->callable_cache.list_append) goto deoptimize; + assert(self != NULL); + if (!PyList_Check(self)) goto deoptimize; + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + DISPATCH(); + } + + case _CALL_METHOD_DESCRIPTOR_O: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -2983,13 +3101,13 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (total_args != 2) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + if (meth->ml_flags != METH_O) goto deoptimize; PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3003,21 +3121,20 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - oparg = CURRENT_OPARG(); + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -3026,39 +3143,37 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) goto deoptimize; PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + if (!Py_IS_TYPE(self, d_type)) goto deoptimize; STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_METHOD_DESCRIPTOR_NOARGS: { - oparg = CURRENT_OPARG(); + case _CALL_METHOD_DESCRIPTOR_NOARGS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); @@ -3067,13 +3182,13 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) goto deoptimize; PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (meth->ml_flags != METH_NOARGS) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3086,21 +3201,20 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_METHOD_DESCRIPTOR_FAST: { - oparg = CURRENT_OPARG(); + case _CALL_METHOD_DESCRIPTOR_FAST: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -3110,14 +3224,14 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + if (meth->ml_flags != METH_FASTCALL) goto deoptimize; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; STAT_INC(CALL, hit); _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3126,93 +3240,121 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case MAKE_FUNCTION: { + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + + /* _CALL_KW is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + case _MAKE_FUNCTION: { PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - + PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { GOTO_ERROR(error); } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; break; } - case SET_FUNCTION_ATTRIBUTE: { - oparg = CURRENT_OPARG(); + case _SET_FUNCTION_ATTRIBUTE: { PyObject *func; PyObject *attr; + oparg = CURRENT_OPARG(); func = stack_pointer[-1]; attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; default: - Py_UNREACHABLE(); + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = func; + stack_pointer[-2] = func; + stack_pointer += -1; break; } - case BUILD_SLICE: { - oparg = CURRENT_OPARG(); + case _RETURN_GENERATOR: { + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + GOTO_ERROR(error); + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + case _BUILD_SLICE: { PyObject *step = NULL; PyObject *stop; PyObject *start; PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; + oparg = CURRENT_OPARG(); + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error_tier_two; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error_tier_two; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); break; } - case CONVERT_VALUE: { - oparg = CURRENT_OPARG(); + case _CONVERT_VALUE: { PyObject *value; PyObject *result; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); @@ -3224,7 +3366,7 @@ break; } - case FORMAT_SIMPLE: { + case _FORMAT_SIMPLE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -3242,7 +3384,7 @@ break; } - case FORMAT_WITH_SPEC: { + case _FORMAT_WITH_SPEC: { PyObject *fmt_spec; PyObject *value; PyObject *res; @@ -3252,28 +3394,28 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COPY: { - oparg = CURRENT_OPARG(); + case _COPY: { PyObject *bottom; PyObject *top; + oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + stack_pointer[0] = top; + stack_pointer += 1; break; } case _BINARY_OP: { - oparg = CURRENT_OPARG(); PyObject *rhs; PyObject *lhs; PyObject *res; + oparg = CURRENT_OPARG(); rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; assert(_PyEval_BinaryOps[oparg]); @@ -3281,15 +3423,15 @@ Py_DECREF(lhs); Py_DECREF(rhs); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case SWAP: { - oparg = CURRENT_OPARG(); + case _SWAP: { PyObject *top; PyObject *bottom; + oparg = CURRENT_OPARG(); top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); @@ -3298,38 +3440,52 @@ break; } + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + case _GUARD_IS_TRUE_POP: { PyObject *flag; flag = stack_pointer[-1]; - DEOPT_IF(Py_IsFalse(flag), _GUARD_IS_TRUE_POP); + if (Py_IsFalse(flag)) goto deoptimize; assert(Py_IsTrue(flag)); - STACK_SHRINK(1); + stack_pointer += -1; break; } case _GUARD_IS_FALSE_POP: { PyObject *flag; flag = stack_pointer[-1]; - DEOPT_IF(Py_IsTrue(flag), _GUARD_IS_FALSE_POP); + if (Py_IsTrue(flag)) goto deoptimize; assert(Py_IsFalse(flag)); - STACK_SHRINK(1); + stack_pointer += -1; break; } case _GUARD_IS_NONE_POP: { PyObject *val; val = stack_pointer[-1]; - DEOPT_IF(!Py_IsNone(val), _GUARD_IS_NONE_POP); - STACK_SHRINK(1); + if (!Py_IsNone(val)) goto deoptimize; + stack_pointer += -1; break; } case _GUARD_IS_NOT_NONE_POP: { PyObject *val; val = stack_pointer[-1]; - DEOPT_IF(Py_IsNone(val), _GUARD_IS_NOT_NONE_POP); + if (Py_IsNone(val)) goto deoptimize; Py_DECREF(val); - STACK_SHRINK(1); + stack_pointer += -1; break; } @@ -3365,8 +3521,8 @@ } case _INSERT: { - oparg = CURRENT_OPARG(); PyObject *top; + oparg = CURRENT_OPARG(); top = stack_pointer[-1]; // Inserts TOS at position specified by oparg; memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); @@ -3376,7 +3532,7 @@ case _CHECK_VALIDITY: { TIER_TWO_ONLY - DEOPT_IF(!current_executor->base.vm_data.valid, _CHECK_VALIDITY); + if (!current_executor->base.vm_data.valid) goto deoptimize; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 24243ecfb5b8df..8f68bc6cb5ab40 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,6 +1,6 @@ // This file is generated by Tools/cases_generator/tier1_generator.py // from: -// ['./Python/bytecodes.c'] +// Python/bytecodes.c // Do not edit! #ifdef TIER_TWO @@ -725,6 +725,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); + TIER_ONE_ONLY assert(0 && "Executing a cache."); Py_UNREACHABLE(); } @@ -2364,6 +2365,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); + TIER_ONE_ONLY assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -4704,6 +4706,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); + TIER_ONE_ONLY assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 945de63d209935..bcc13538e51d9b 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -15,6 +15,7 @@ class Properties: needs_this: bool always_exits: bool stores_sp: bool + tier_one_only: bool def dump(self, indent: str) -> None: print(indent, end="") @@ -33,6 +34,7 @@ def from_list(properties: list["Properties"]) -> "Properties": needs_this=any(p.needs_this for p in properties), always_exits=any(p.always_exits for p in properties), stores_sp=any(p.stores_sp for p in properties), + tier_one_only=any(p.tier_one_only for p in properties), ) @@ -46,6 +48,7 @@ def from_list(properties: list["Properties"]) -> "Properties": needs_this=False, always_exits=False, stores_sp=False, + tier_one_only=False, ) @@ -124,6 +127,21 @@ def size(self) -> int: self._size = sum(c.size for c in self.caches) return self._size + def is_viable(self) -> bool: + if self.name == "_SAVE_RETURN_OFFSET": + return True # Adjusts next_instr, but only in tier 1 code + if self.properties.needs_this: + return False + if "INSTRUMENTED" in self.name: + return False + if "replaced" in self.annotations: + return False + if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): + return False + if len([c for c in self.caches if c.name != "unused"]) > 1: + return False + return True + Part = Uop | Skip @@ -292,6 +310,7 @@ def compute_properties(op: parser.InstDef) -> Properties: needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), stores_sp=variable_used(op, "STORE_SP"), + tier_one_only=variable_used(op, "TIER_ONE_ONLY"), ) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f7c362131a7a9f..c6ed5911b846bf 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -128,13 +128,6 @@ arg_parser.add_argument( "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" ) -arg_parser.add_argument( - "-e", - "--executor-cases", - type=str, - help="Write executor cases to this file", - default=DEFAULT_EXECUTOR_OUTPUT, -) arg_parser.add_argument( "-a", "--abstract-interpreter-cases", @@ -846,7 +839,6 @@ def main() -> None: a.assign_opcode_ids() a.write_opcode_targets(args.opcode_targets_h) a.write_metadata(args.metadata, args.pymetadata) - a.write_executor_instructions(args.executor_cases, args.emit_line_directives) a.write_abstract_interpreter_instructions( args.abstract_interpreter_cases, args.emit_line_directives ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 76900d1efffd5d..e0674a7343498d 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -1,19 +1,186 @@ from pathlib import Path from typing import TextIO +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + StackItem, + analysis_error, +) +from cwriter import CWriter +from typing import Callable, Mapping, TextIO, Iterator +from lexer import Token +from stack import StackOffset, Stack + + ROOT = Path(__file__).parent.parent.parent -DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute() +DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute().as_posix() def root_relative_path(filename: str) -> str: - return Path(filename).relative_to(ROOT).as_posix() + return Path(filename).absolute().relative_to(ROOT).as_posix() -def write_header(generator: str, source: str, outfile: TextIO) -> None: +def write_header(generator: str, sources: list[str], outfile: TextIO) -> None: outfile.write( f"""// This file is generated by {root_relative_path(generator)} // from: -// {source} +// {", ".join(root_relative_path(src) for src in sources)} // Do not edit! """ ) + + +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + out.emit(tkn) + + +def replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("DEOPT_IF", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(", ") + assert inst is not None + assert inst.family is not None + out.emit(inst.family.name) + out.emit(");\n") + + +def replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label) + out.emit(close) + + +def replace_decrefs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size != "1": + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit("}\n") + elif var.condition: + out.emit(f"Py_XDECREF({var.name});\n") + else: + out.emit(f"Py_DECREF({var.name});\n") + + +def replace_store_sp( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + stack.flush(out) + out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + + +def replace_check_eval_breaker( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + if not uop.properties.ends_with_eval_breaker: + out.emit_at("CHECK_EVAL_BREAKER();", tkn) + + +REPLACEMENT_FUNCTIONS = { + "DEOPT_IF": replace_deopt, + "ERROR_IF": replace_error, + "DECREF_INPUTS": replace_decrefs, + "CHECK_EVAL_BREAKER": replace_check_eval_breaker, + "STORE_SP": replace_store_sp, +} + +ReplacementFunctionType = Callable[ + [CWriter, Token, Iterator[Token], Uop, Stack, Instruction | None], None +] + + +def emit_tokens( + out: CWriter, + uop: Uop, + stack: Stack, + inst: Instruction | None, + replacement_functions: Mapping[ + str, ReplacementFunctionType + ] = REPLACEMENT_FUNCTIONS, +) -> None: + tkns = uop.body[1:-1] + if not tkns: + return + tkn_iter = iter(tkns) + out.start_line() + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in replacement_functions: + replacement_functions[tkn.text](out, tkn, tkn_iter, uop, stack, inst) + else: + out.emit(tkn) diff --git a/Tools/cases_generator/opcode_id_generator.py b/Tools/cases_generator/opcode_id_generator.py index a1f6f62156ebd3..ddbb409bbced39 100644 --- a/Tools/cases_generator/opcode_id_generator.py +++ b/Tools/cases_generator/opcode_id_generator.py @@ -24,7 +24,7 @@ DEFAULT_OUTPUT = ROOT / "Include/opcode_ids.h" -def generate_opcode_header(filenames: str, analysis: Analysis, outfile: TextIO) -> None: +def generate_opcode_header(filenames: list[str], analysis: Analysis, outfile: TextIO) -> None: write_header(__file__, filenames, outfile) out = CWriter(outfile, 0, False) out.emit("\n") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 9cd8e1a3b2edf8..c36a56ebf2d381 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -2,6 +2,7 @@ from analyzer import StackItem from dataclasses import dataclass from formatting import maybe_parenthesize +from cwriter import CWriter def var_size(var: StackItem) -> str: @@ -79,3 +80,89 @@ def to_c(self) -> str: def clear(self) -> None: self.popped = [] self.pushed = [] + + +class SizeMismatch(Exception): + pass + + +class Stack: + def __init__(self) -> None: + self.top_offset = StackOffset() + self.base_offset = StackOffset() + self.peek_offset = StackOffset() + self.variables: list[StackItem] = [] + self.defined: set[str] = set() + + def pop(self, var: StackItem) -> str: + self.top_offset.pop(var) + if not var.peek: + self.peek_offset.pop(var) + indirect = "&" if var.is_array() else "" + if self.variables: + popped = self.variables.pop() + if popped.size != var.size: + raise SizeMismatch( + f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " + f"Expected {var.size} got {popped.size}" + ) + if popped.name == var.name: + return "" + elif popped.name == "unused": + self.defined.add(var.name) + return ( + f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + ) + elif var.name == "unused": + return "" + else: + self.defined.add(var.name) + return f"{var.name} = {popped.name};\n" + self.base_offset.pop(var) + if var.name == "unused": + return "" + else: + self.defined.add(var.name) + cast = f"({var.type})" if (not indirect and var.type) else "" + assign = ( + f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];" + ) + if var.condition: + return f"if ({var.condition}) {{ {assign} }}\n" + return f"{assign}\n" + + def push(self, var: StackItem) -> str: + self.variables.append(var) + if var.is_array() and var.name not in self.defined and var.name != "unused": + c_offset = self.top_offset.to_c() + self.top_offset.push(var) + self.defined.add(var.name) + return f"{var.name} = &stack_pointer[{c_offset}];\n" + else: + self.top_offset.push(var) + return "" + + def flush(self, out: CWriter) -> None: + for var in self.variables: + if not var.peek: + cast = "(PyObject *)" if var.type else "" + if var.name != "unused" and not var.is_array(): + if var.condition: + out.emit(f" if ({var.condition}) ") + out.emit( + f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n" + ) + self.base_offset.push(var) + if self.base_offset.to_c() != self.top_offset.to_c(): + print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) + assert False + number = self.base_offset.to_c() + if number != "0": + out.emit(f"stack_pointer += {number};\n") + self.variables = [] + self.base_offset.clear() + self.top_offset.clear() + self.peek_offset.clear() + + def as_comment(self) -> str: + return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 56ae660a686822..11885dca6fe1a2 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -21,11 +21,12 @@ DEFAULT_INPUT, ROOT, write_header, + emit_tokens, ) from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import StackOffset +from stack import StackOffset, Stack, SizeMismatch DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" @@ -34,88 +35,6 @@ FOOTER = "#undef TIER_ONE\n" -class SizeMismatch(Exception): - pass - - -class Stack: - def __init__(self) -> None: - self.top_offset = StackOffset() - self.base_offset = StackOffset() - self.peek_offset = StackOffset() - self.variables: list[StackItem] = [] - self.defined: set[str] = set() - - def pop(self, var: StackItem) -> str: - self.top_offset.pop(var) - if not var.peek: - self.peek_offset.pop(var) - indirect = "&" if var.is_array() else "" - if self.variables: - popped = self.variables.pop() - if popped.size != var.size: - raise SizeMismatch( - f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " - f"Expected {var.size} got {popped.size}" - ) - if popped.name == var.name: - return "" - elif popped.name == "unused": - self.defined.add(var.name) - return ( - f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" - ) - elif var.name == "unused": - return "" - else: - self.defined.add(var.name) - return f"{var.name} = {popped.name};\n" - self.base_offset.pop(var) - if var.name == "unused": - return "" - else: - self.defined.add(var.name) - assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];" - if var.condition: - return f"if ({var.condition}) {{ {assign} }}\n" - return f"{assign}\n" - - def push(self, var: StackItem) -> str: - self.variables.append(var) - if var.is_array() and var.name not in self.defined and var.name != "unused": - c_offset = self.top_offset.to_c() - self.top_offset.push(var) - self.defined.add(var.name) - return f"{var.name} = &stack_pointer[{c_offset}];\n" - else: - self.top_offset.push(var) - return "" - - def flush(self, out: CWriter) -> None: - for var in self.variables: - if not var.peek: - if var.name != "unused" and not var.is_array(): - if var.condition: - out.emit(f" if ({var.condition}) ") - out.emit( - f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n" - ) - self.base_offset.push(var) - if self.base_offset.to_c() != self.top_offset.to_c(): - print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) - assert False - number = self.base_offset.to_c() - if number != "0": - out.emit(f"stack_pointer += {number};\n") - self.variables = [] - self.base_offset.clear() - self.top_offset.clear() - self.peek_offset.clear() - - def as_comment(self) -> str: - return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" - - def declare_variables(inst: Instruction, out: CWriter) -> None: variables = {"unused"} for uop in inst.parts: @@ -138,145 +57,6 @@ def declare_variables(inst: Instruction, out: CWriter) -> None: out.emit(f"{type}{var.name};\n") -def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: - parens = 0 - for tkn in tkn_iter: - if tkn.kind == end and parens == 0: - return - if tkn.kind == "LPAREN": - parens += 1 - if tkn.kind == "RPAREN": - parens -= 1 - out.emit(tkn) - - -def replace_deopt( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - unused: Stack, - inst: Instruction, -) -> None: - out.emit_at("DEOPT_IF", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - out.emit(", ") - assert inst.family is not None - out.emit(inst.family.name) - out.emit(");\n") - - -def replace_error( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "COMMA") - label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit(") ") - c_offset = stack.peek_offset.to_c() - try: - offset = -int(c_offset) - close = ";\n" - except ValueError: - offset = None - out.emit(f"{{ stack_pointer += {c_offset}; ") - close = "; }\n" - out.emit("goto ") - if offset: - out.emit(f"pop_{offset}_") - out.emit(label) - out.emit(close) - - -def replace_decrefs( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - out.emit_at("", tkn) - for var in uop.stack.inputs: - if var.name == "unused" or var.name == "null" or var.peek: - continue - if var.size != "1": - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"Py_DECREF({var.name}[_i]);\n") - out.emit("}\n") - elif var.condition: - out.emit(f"Py_XDECREF({var.name});\n") - else: - out.emit(f"Py_DECREF({var.name});\n") - - -def replace_store_sp( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - out.emit_at("", tkn) - stack.flush(out) - out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") - - -def replace_check_eval_breaker( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - if not uop.properties.ends_with_eval_breaker: - out.emit_at("CHECK_EVAL_BREAKER();", tkn) - - -REPLACEMENT_FUNCTIONS = { - "DEOPT_IF": replace_deopt, - "ERROR_IF": replace_error, - "DECREF_INPUTS": replace_decrefs, - "CHECK_EVAL_BREAKER": replace_check_eval_breaker, - "STORE_SP": replace_store_sp, -} - - -# Move this to formatter -def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None: - tkns = uop.body[1:-1] - if not tkns: - return - tkn_iter = iter(tkns) - out.start_line() - for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: - REPLACEMENT_FUNCTIONS[tkn.text](out, tkn, tkn_iter, uop, stack, inst) - else: - out.emit(tkn) - - def write_uop( uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool ) -> int: @@ -334,7 +114,7 @@ def uses_this(inst: Instruction) -> bool: def generate_tier1( - filenames: str, analysis: Analysis, outfile: TextIO, lines: bool + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: write_header(__file__, filenames, outfile) outfile.write( @@ -404,7 +184,7 @@ def generate_tier1( if __name__ == "__main__": args = arg_parser.parse_args() if len(args.input) == 0: - args.input.append(DEFAULT_INPUT.as_posix()) + args.input.append(DEFAULT_INPUT) data = analyze_files(args.input) with open(args.output, "w") as outfile: generate_tier1(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py new file mode 100644 index 00000000000000..d2b503961aa7b9 --- /dev/null +++ b/Tools/cases_generator/tier2_generator.py @@ -0,0 +1,202 @@ +"""Generate the cases for the tier 2 interpreter. +Reads the instruction definitions from bytecodes.c. +Writes the cases to executor_cases.c.h, which is #included in ceval.c. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + emit_tokens, + emit_to, + REPLACEMENT_FUNCTIONS, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import StackOffset, Stack, SizeMismatch + +DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" + + +def declare_variables(uop: Uop, out: CWriter) -> None: + variables = {"unused"} + for var in reversed(uop.stack.inputs): + if var.name not in variables: + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + for var in uop.stack.outputs: + if var.name not in variables: + variables.add(var.name) + type = var.type if var.type else "PyObject *" + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + + +def tier2_replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label + "_tier_two") + out.emit(close) + + +def tier2_replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(") goto deoptimize;\n") + + +TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() +TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error +TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt + + +def is_super(uop: Uop) -> bool: + for tkn in uop.body: + if tkn.kind == "IDENTIFIER" and tkn.text == "oparg1": + return True + return False + + +def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: + try: + out.start_line() + if uop.properties.oparg: + out.emit("oparg = CURRENT_OPARG();\n") + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = "PyObject *" + else: + type = f"uint{cache.size*16}_t" + out.emit(f"{type} {cache.name} = ({type})CURRENT_OPERAND();\n") + emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_tier2( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + outfile.write( + """ +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 +""" + ) + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, uop in analysis.uops.items(): + if uop.properties.tier_one_only: + continue + if is_super(uop): + continue + if not uop.is_viable(): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + declare_variables(uop, out) + stack = Stack() + write_uop(uop, out, stack) + out.start_line() + if not uop.properties.always_exits: + stack.flush(out) + if uop.properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("break;\n") + out.start_line() + out.emit("}") + out.emit("\n\n") + outfile.write("#undef TIER_TWO\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier2(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py index 4a96dbc171ee22..277da25835f6fb 100644 --- a/Tools/cases_generator/uop_id_generator.py +++ b/Tools/cases_generator/uop_id_generator.py @@ -24,8 +24,11 @@ DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_ids.h" +OMIT = {"_CACHE", "_RESERVED", "_EXTENDED_ARG"} + + def generate_uop_ids( - filenames: str, analysis: Analysis, outfile: TextIO, distinct_namespace: bool + filenames: list[str], analysis: Analysis, outfile: TextIO, distinct_namespace: bool ) -> None: write_header(__file__, filenames, outfile) out = CWriter(outfile, 0, False) @@ -45,11 +48,15 @@ def generate_uop_ids( next_id += 1 out.emit(f"#define _SET_IP {next_id}\n") next_id += 1 - PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP", "_CACHE", "_RESERVED", "_EXTENDED_ARG"} + PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"} for uop in analysis.uops.values(): if uop.name in PRE_DEFINED: continue + # TODO: We should omit all tier-1 only uops, but + # generate_cases.py still generates code for those. + if uop.name in OMIT: + continue if uop.implicitly_created and not distinct_namespace: out.emit(f"#define {uop.name} {uop.name[1:]}\n") else: @@ -85,7 +92,7 @@ def generate_uop_ids( if __name__ == "__main__": args = arg_parser.parse_args() if len(args.input) == 0: - args.input.append(DEFAULT_INPUT.as_posix()) + args.input.append(DEFAULT_INPUT) data = analyze_files(args.input) with open(args.output, "w") as outfile: generate_uop_ids(args.input, data, outfile, args.namespace) From f26bfe4b25f7e5a4f68fcac26207b7175abad208 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 12 Dec 2023 13:57:45 +0100 Subject: [PATCH 21/95] gh-51944: fix type and missing addition in gh-112823 (#112996) Fix the defition of VDSUSP and add CCTS_OFLOW (which was mentioned in NEWS, but not actually addded) --- Modules/termios.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/termios.c b/Modules/termios.c index 1d97a3a2757966..c4f0fd9d50044a 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -840,6 +840,9 @@ static struct constant { #ifdef CDSR_OFLOW {"CDSR_OFLOW", CDSR_OFLOW}, #endif +#ifdef CCTS_OFLOW + {"CCTS_OFLOW", CCTS_OFLOW}, +#endif #ifdef CCAR_OFLOW {"CCAR_OFLOW", CCAR_OFLOW}, #endif @@ -912,7 +915,7 @@ static struct constant { {"VSTOP", VSTOP}, {"VSUSP", VSUSP}, #ifdef VDSUSP - {"VDSUSP", VREPRINT}, + {"VDSUSP", VDSUSP}, #endif {"VEOL", VEOL}, #ifdef VREPRINT From 86a77f4e1a5ceaff1036b0072521e12752b5df47 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 12 Dec 2023 08:24:31 -0700 Subject: [PATCH 22/95] gh-76785: Fixes for test.support.interpreters (gh-112982) This involves a number of changes for PEP 734. --- .github/CODEOWNERS | 15 +- Include/internal/pycore_crossinterp.h | 10 + Include/internal/pycore_interp.h | 6 +- Lib/test/support/interpreters/__init__.py | 160 +++ .../channels.py} | 124 +- Lib/test/support/interpreters/queues.py | 156 +++ Lib/test/test__xxinterpchannels.py | 3 +- Lib/test/test__xxsubinterpreters.py | 82 +- Lib/test/test_capi/test_misc.py | 132 ++ Lib/test/test_import/__init__.py | 11 +- Lib/test/test_importlib/test_util.py | 16 +- Lib/test/test_interpreters.py | 1136 ----------------- Lib/test/test_interpreters/__init__.py | 5 + Lib/test/test_interpreters/__main__.py | 4 + Lib/test/test_interpreters/test_api.py | 642 ++++++++++ Lib/test/test_interpreters/test_channels.py | 328 +++++ Lib/test/test_interpreters/test_lifecycle.py | 189 +++ Lib/test/test_interpreters/test_queues.py | 233 ++++ Lib/test/test_interpreters/test_stress.py | 38 + Lib/test/test_interpreters/utils.py | 73 ++ Lib/test/test_sys.py | 4 +- Lib/test/test_threading.py | 2 +- Modules/_testcapimodule.c | 34 + Modules/_testinternalcapi.c | 12 + Modules/_xxinterpchannelsmodule.c | 157 --- Modules/_xxsubinterpretersmodule.c | 367 +++++- Python/crossinterp.c | 62 + Python/pylifecycle.c | 6 + Python/pystate.c | 2 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 + 30 files changed, 2506 insertions(+), 1507 deletions(-) create mode 100644 Lib/test/support/interpreters/__init__.py rename Lib/test/support/{interpreters.py => interpreters/channels.py} (56%) create mode 100644 Lib/test/support/interpreters/queues.py delete mode 100644 Lib/test/test_interpreters.py create mode 100644 Lib/test/test_interpreters/__init__.py create mode 100644 Lib/test/test_interpreters/__main__.py create mode 100644 Lib/test/test_interpreters/test_api.py create mode 100644 Lib/test/test_interpreters/test_channels.py create mode 100644 Lib/test/test_interpreters/test_lifecycle.py create mode 100644 Lib/test/test_interpreters/test_queues.py create mode 100644 Lib/test/test_interpreters/test_stress.py create mode 100644 Lib/test/test_interpreters/utils.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aa6d937d9cbc31..f8291d8689dd95 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -78,6 +78,11 @@ Python/traceback.c @iritkatriel **/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw +# Subinterpreters +Lib/test/support/interpreters/** @ericsnowcurrently +Modules/_xx*interp*module.c @ericsnowcurrently +Lib/test/test_interpreters/** @ericsnowcurrently + # Dates and times **/*datetime* @pganssle @abalkin **/*str*time* @pganssle @abalkin @@ -148,7 +153,15 @@ Doc/c-api/stable.rst @encukou **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger -**/*queue* @rhettinger +Doc/**/*queue* @rhettinger +PCbuild/**/*queue* @rhettinger +Modules/_queuemodule.c @rhettinger +Lib/*queue*.py @rhettinger +Lib/asyncio/*queue*.py @rhettinger +Lib/multiprocessing/*queue*.py @rhettinger +Lib/test/*queue*.py @rhettinger +Lib/test_asyncio/*queue*.py @rhettinger +Lib/test_multiprocessing/*queue*.py @rhettinger **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 2e6d09a49f95d3..ce95979f8d343b 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -11,6 +11,13 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_pyerrors.h" +/**************/ +/* exceptions */ +/**************/ + +PyAPI_DATA(PyObject *) PyExc_InterpreterError; +PyAPI_DATA(PyObject *) PyExc_InterpreterNotFoundError; + /***************************/ /* cross-interpreter calls */ @@ -160,6 +167,9 @@ struct _xi_state { extern PyStatus _PyXI_Init(PyInterpreterState *interp); extern void _PyXI_Fini(PyInterpreterState *interp); +extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); +extern void _PyXI_FiniTypes(PyInterpreterState *interp); + /***************************/ /* short-term data sharing */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 2a683196eeced3..04d7a6a615e370 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -250,9 +250,9 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst // Export for the _xxinterpchannels module. PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); -extern int _PyInterpreterState_IDInitref(PyInterpreterState *); -extern int _PyInterpreterState_IDIncref(PyInterpreterState *); -extern void _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py new file mode 100644 index 00000000000000..2d6376deb5907e --- /dev/null +++ b/Lib/test/support/interpreters/__init__.py @@ -0,0 +1,160 @@ +"""Subinterpreters High Level Module.""" + +import threading +import weakref +import _xxsubinterpreters as _interpreters + +# aliases: +from _xxsubinterpreters import ( + InterpreterError, InterpreterNotFoundError, + is_shareable, +) + + +__all__ = [ + 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', + 'Interpreter', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', +] + + +_queuemod = None + +def __getattr__(name): + if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): + global create_queue, Queue, QueueEmpty, QueueFull + ns = globals() + from .queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, + ) + return ns[name] + else: + raise AttributeError(name) + + +class ExecFailure(RuntimeError): + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and snapshot.msg: + msg = f'{snapshot.type.__name__}: {snapshot.msg}' + else: + msg = snapshot.type.__name__ or snapshot.msg + super().__init__(msg) + self.snapshot = excinfo + + +def create(): + """Return a new (idle) Python interpreter.""" + id = _interpreters.create(isolated=True) + return Interpreter(id) + + +def list_all(): + """Return all existing interpreters.""" + return [Interpreter(id) for id in _interpreters.list_all()] + + +def get_current(): + """Return the currently running interpreter.""" + id = _interpreters.get_current() + return Interpreter(id) + + +def get_main(): + """Return the main interpreter.""" + id = _interpreters.get_main() + return Interpreter(id) + + +_known = weakref.WeakValueDictionary() + +class Interpreter: + """A single Python interpreter.""" + + def __new__(cls, id, /): + # There is only one instance for any given ID. + if not isinstance(id, int): + raise TypeError(f'id must be an int, got {id!r}') + id = int(id) + try: + self = _known[id] + assert hasattr(self, '_ownsref') + except KeyError: + # This may raise InterpreterNotFoundError: + _interpreters._incref(id) + try: + self = super().__new__(cls) + self._id = id + self._ownsref = True + except BaseException: + _interpreters._deccref(id) + raise + _known[id] = self + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + def __del__(self): + self._decref() + + def _decref(self): + if not self._ownsref: + return + self._ownsref = False + try: + _interpreters._decref(self.id) + except InterpreterNotFoundError: + pass + + @property + def id(self): + return self._id + + def is_running(self): + """Return whether or not the identified interpreter is running.""" + return _interpreters.is_running(self._id) + + def close(self): + """Finalize and destroy the interpreter. + + Attempting to destroy the current interpreter results + in a RuntimeError. + """ + return _interpreters.destroy(self._id) + + def exec_sync(self, code, /, channels=None): + """Run the given source code in the interpreter. + + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then an ExecFailure + is raised, which summarizes the unhandled exception. The actual + exception is discarded because objects cannot be shared between + interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. + """ + excinfo = _interpreters.exec(self._id, code, channels) + if excinfo is not None: + raise ExecFailure(excinfo) + + def run(self, code, /, channels=None): + def task(): + self.exec_sync(code, channels=channels) + t = threading.Thread(target=task) + t.start() + return t diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters/channels.py similarity index 56% rename from Lib/test/support/interpreters.py rename to Lib/test/support/interpreters/channels.py index 089fe7ef56df78..75a5a60f54f926 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters/channels.py @@ -1,11 +1,9 @@ -"""Subinterpreters High Level Module.""" +"""Cross-interpreter Channels High Level Module.""" import time -import _xxsubinterpreters as _interpreters import _xxinterpchannels as _channels # aliases: -from _xxsubinterpreters import is_shareable from _xxinterpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, @@ -13,123 +11,13 @@ __all__ = [ - 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', - 'RunFailedError', + 'create', 'list_all', 'SendChannel', 'RecvChannel', - 'create_channel', 'list_all_channels', 'is_shareable', - 'ChannelError', 'ChannelNotFoundError', - 'ChannelEmptyError', - ] + 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', +] -class RunFailedError(RuntimeError): - - def __init__(self, excinfo): - msg = excinfo.formatted - if not msg: - if excinfo.type and snapshot.msg: - msg = f'{snapshot.type.__name__}: {snapshot.msg}' - else: - msg = snapshot.type.__name__ or snapshot.msg - super().__init__(msg) - self.snapshot = excinfo - - -def create(*, isolated=True): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=isolated) - return Interpreter(id, isolated=isolated) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] - - -def get_current(): - """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) - - -def get_main(): - """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) - - -class Interpreter: - """A single Python interpreter.""" - - def __init__(self, id, *, isolated=None): - if not isinstance(id, (int, _interpreters.InterpreterID)): - raise TypeError(f'id must be an int, got {id!r}') - self._id = id - self._isolated = isolated - - def __repr__(self): - data = dict(id=int(self._id), isolated=self._isolated) - kwargs = (f'{k}={v!r}' for k, v in data.items()) - return f'{type(self).__name__}({", ".join(kwargs)})' - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - if not isinstance(other, Interpreter): - return NotImplemented - else: - return other._id == self._id - - @property - def id(self): - return self._id - - @property - def isolated(self): - if self._isolated is None: - # XXX The low-level function has not been added yet. - # See bpo-.... - self._isolated = _interpreters.is_isolated(self._id) - return self._isolated - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in a RuntimeError. - """ - return _interpreters.destroy(self._id) - - # XXX Rename "run" to "exec"? - def run(self, src_str, /, channels=None): - """Run the given source code in the interpreter. - - This is essentially the same as calling the builtin "exec" - with this interpreter, using the __dict__ of its __main__ - module as both globals and locals. - - There is no return value. - - If the code raises an unhandled exception then a RunFailedError - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. - - This blocks the current Python thread until done. During - that time, the previous interpreter is allowed to run - in other threads. - """ - excinfo = _interpreters.exec(self._id, src_str, channels) - if excinfo is not None: - raise RunFailedError(excinfo) - - -def create_channel(): +def create(): """Return (recv, send) for a new cross-interpreter channel. The channel may be used to pass data safely between interpreters. @@ -139,7 +27,7 @@ def create_channel(): return recv, send -def list_all_channels(): +def list_all(): """Return a list of (recv, send) for all open channels.""" return [(RecvChannel(cid), SendChannel(cid)) for cid in _channels.list_all()] diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py new file mode 100644 index 00000000000000..ed6b0d551dd890 --- /dev/null +++ b/Lib/test/support/interpreters/queues.py @@ -0,0 +1,156 @@ +"""Cross-interpreter Queues High Level Module.""" + +import queue +import time +import weakref +import _xxinterpchannels as _channels +import _xxinterpchannels as _queues + +# aliases: +from _xxinterpchannels import ( + ChannelError as QueueError, + ChannelNotFoundError as QueueNotFoundError, +) + +__all__ = [ + 'create', 'list_all', + 'Queue', + 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', +] + + +def create(maxsize=0): + """Return a new cross-interpreter queue. + + The queue may be used to pass data safely between interpreters. + """ + # XXX honor maxsize + qid = _queues.create() + return Queue._with_maxsize(qid, maxsize) + + +def list_all(): + """Return a list of all open queues.""" + return [Queue(qid) + for qid in _queues.list_all()] + + +class QueueEmpty(queue.Empty): + """Raised from get_nowait() when the queue is empty. + + It is also raised from get() if it times out. + """ + + +class QueueFull(queue.Full): + """Raised from put_nowait() when the queue is full. + + It is also raised from put() if it times out. + """ + + +_known_queues = weakref.WeakValueDictionary() + +class Queue: + """A cross-interpreter queue.""" + + @classmethod + def _with_maxsize(cls, id, maxsize): + if not isinstance(maxsize, int): + raise TypeError(f'maxsize must be an int, got {maxsize!r}') + elif maxsize < 0: + maxsize = 0 + else: + maxsize = int(maxsize) + self = cls(id) + self._maxsize = maxsize + return self + + def __new__(cls, id, /): + # There is only one instance for any given ID. + if isinstance(id, int): + id = _channels._channel_id(id, force=False) + elif not isinstance(id, _channels.ChannelID): + raise TypeError(f'id must be an int, got {id!r}') + key = int(id) + try: + self = _known_queues[key] + except KeyError: + self = super().__new__(cls) + self._id = id + self._maxsize = 0 + _known_queues[key] = self + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + @property + def id(self): + return int(self._id) + + @property + def maxsize(self): + return self._maxsize + + @property + def _info(self): + return _channels.get_info(self._id) + + def empty(self): + return self._info.count == 0 + + def full(self): + if self._maxsize <= 0: + return False + return self._info.count >= self._maxsize + + def qsize(self): + return self._info.count + + def put(self, obj, timeout=None): + # XXX block if full + _channels.send(self._id, obj, blocking=False) + + def put_nowait(self, obj): + # XXX raise QueueFull if full + return _channels.send(self._id, obj, blocking=False) + + def get(self, timeout=None, *, + _sentinel=object(), + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the queue. + + This blocks while the queue is empty. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + obj = _channels.recv(self._id, _sentinel) + while obj is _sentinel: + time.sleep(_delay) + if timeout is not None and time.time() >= end: + raise QueueEmpty + obj = _channels.recv(self._id, _sentinel) + return obj + + def get_nowait(self, *, _sentinel=object()): + """Return the next object from the channel. + + If the queue is empty then raise QueueEmpty. Otherwise this + is the same as get(). + """ + obj = _channels.recv(self._id, _sentinel) + if obj is _sentinel: + raise QueueEmpty + return obj + + +# XXX add this: +#_channels._register_queue_type(Queue) diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 2b75e2f1916c82..13c8a10296e502 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -79,8 +79,7 @@ def __new__(cls, name=None, id=None): name = 'interp' elif name == 'main': raise ValueError('name mismatch (unexpected "main")') - if not isinstance(id, interpreters.InterpreterID): - id = interpreters.InterpreterID(id) + assert isinstance(id, int), repr(id) elif not name or name == 'main': name = 'main' id = main diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 64a9db95e5eaf5..260ab64b07cb2d 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -15,6 +15,7 @@ interpreters = import_helper.import_module('_xxsubinterpreters') +from _xxsubinterpreters import InterpreterNotFoundError ################################## @@ -266,7 +267,7 @@ def test_main(self): main = interpreters.get_main() cur = interpreters.get_current() self.assertEqual(cur, main) - self.assertIsInstance(cur, interpreters.InterpreterID) + self.assertIsInstance(cur, int) def test_subinterpreter(self): main = interpreters.get_main() @@ -275,7 +276,7 @@ def test_subinterpreter(self): import _xxsubinterpreters as _interpreters cur = _interpreters.get_current() print(cur) - assert isinstance(cur, _interpreters.InterpreterID) + assert isinstance(cur, int) """)) cur = int(out.strip()) _, expected = interpreters.list_all() @@ -289,7 +290,7 @@ def test_from_main(self): [expected] = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, expected) - self.assertIsInstance(main, interpreters.InterpreterID) + self.assertIsInstance(main, int) def test_from_subinterpreter(self): [expected] = interpreters.list_all() @@ -298,7 +299,7 @@ def test_from_subinterpreter(self): import _xxsubinterpreters as _interpreters main = _interpreters.get_main() print(main) - assert isinstance(main, _interpreters.InterpreterID) + assert isinstance(main, int) """)) main = int(out.strip()) self.assertEqual(main, expected) @@ -333,11 +334,11 @@ def test_from_subinterpreter(self): def test_already_destroyed(self): interp = interpreters.create() interpreters.destroy(interp) - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.is_running(interp) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.is_running(1_000_000) def test_bad_id(self): @@ -345,70 +346,11 @@ def test_bad_id(self): interpreters.is_running(-1) -class InterpreterIDTests(TestBase): - - def test_with_int(self): - id = interpreters.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) - - def test_coerce_id(self): - class Int(str): - def __index__(self): - return 10 - - id = interpreters.InterpreterID(Int(), force=True) - self.assertEqual(int(id), 10) - - def test_bad_id(self): - self.assertRaises(TypeError, interpreters.InterpreterID, object()) - self.assertRaises(TypeError, interpreters.InterpreterID, 10.0) - self.assertRaises(TypeError, interpreters.InterpreterID, '10') - self.assertRaises(TypeError, interpreters.InterpreterID, b'10') - self.assertRaises(ValueError, interpreters.InterpreterID, -1) - self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64) - - def test_does_not_exist(self): - id = interpreters.create() - with self.assertRaises(RuntimeError): - interpreters.InterpreterID(int(id) + 1) # unforced - - def test_str(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = interpreters.create() - id2 = interpreters.InterpreterID(int(id1)) - id3 = interpreters.create() - - self.assertTrue(id1 == id1) - self.assertTrue(id1 == id2) - self.assertTrue(id1 == int(id1)) - self.assertTrue(int(id1) == id1) - self.assertTrue(id1 == float(int(id1))) - self.assertTrue(float(int(id1)) == id1) - self.assertFalse(id1 == float(int(id1)) + 0.1) - self.assertFalse(id1 == str(int(id1))) - self.assertFalse(id1 == 2**1000) - self.assertFalse(id1 == float('inf')) - self.assertFalse(id1 == 'spam') - self.assertFalse(id1 == id3) - - self.assertFalse(id1 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id1 != id3) - - class CreateTests(TestBase): def test_in_main(self): id = interpreters.create() - self.assertIsInstance(id, interpreters.InterpreterID) + self.assertIsInstance(id, int) self.assertIn(id, interpreters.list_all()) @@ -444,7 +386,7 @@ def test_in_subinterpreter(self): import _xxsubinterpreters as _interpreters id = _interpreters.create() print(id) - assert isinstance(id, _interpreters.InterpreterID) + assert isinstance(id, int) """)) id2 = int(out.strip()) @@ -536,11 +478,11 @@ def f(): def test_already_destroyed(self): id = interpreters.create() interpreters.destroy(id) - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.destroy(id) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.destroy(1_000_000) def test_bad_id(self): @@ -741,7 +683,7 @@ def test_does_not_exist(self): id = 0 while id in interpreters.list_all(): id += 1 - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.run_string(id, 'print("spam")') def test_error_id(self): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3d86ae37190475..e6b532e858c8f9 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1527,6 +1527,7 @@ def test_isolated_subinterpreter(self): maxtext = 250 main_interpid = 0 interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) _interpreters.run_string(interpid, f"""if True: import json import os @@ -2020,6 +2021,137 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +@requires_subinterpreters +class InterpreterIDTests(unittest.TestCase): + + InterpreterID = _testcapi.get_interpreterid_type() + + def new_interpreter(self): + def ensure_destroyed(interpid): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + id = _interpreters.create() + self.addCleanup(lambda: ensure_destroyed(id)) + return id + + def test_with_int(self): + id = self.InterpreterID(10, force=True) + + self.assertEqual(int(id), 10) + + def test_coerce_id(self): + class Int(str): + def __index__(self): + return 10 + + id = self.InterpreterID(Int(), force=True) + self.assertEqual(int(id), 10) + + def test_bad_id(self): + for badid in [ + object(), + 10.0, + '10', + b'10', + ]: + with self.subTest(badid): + with self.assertRaises(TypeError): + self.InterpreterID(badid) + + badid = -1 + with self.subTest(badid): + with self.assertRaises(ValueError): + self.InterpreterID(badid) + + badid = 2**64 + with self.subTest(badid): + with self.assertRaises(OverflowError): + self.InterpreterID(badid) + + def test_exists(self): + id = self.new_interpreter() + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(int(id) + 1) # unforced + + def test_does_not_exist(self): + id = self.new_interpreter() + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(int(id) + 1) # unforced + + def test_destroyed(self): + id = _interpreters.create() + _interpreters.destroy(id) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(id) # unforced + + def test_str(self): + id = self.InterpreterID(10, force=True) + self.assertEqual(str(id), '10') + + def test_repr(self): + id = self.InterpreterID(10, force=True) + self.assertEqual(repr(id), 'InterpreterID(10)') + + def test_equality(self): + id1 = self.new_interpreter() + id2 = self.InterpreterID(id1) + id3 = self.InterpreterID( + self.new_interpreter()) + + self.assertTrue(id2 == id2) # identity + self.assertTrue(id2 == id1) # int-equivalent + self.assertTrue(id1 == id2) # reversed + self.assertTrue(id2 == int(id2)) + self.assertTrue(id2 == float(int(id2))) + self.assertTrue(float(int(id2)) == id2) + self.assertFalse(id2 == float(int(id2)) + 0.1) + self.assertFalse(id2 == str(int(id2))) + self.assertFalse(id2 == 2**1000) + self.assertFalse(id2 == float('inf')) + self.assertFalse(id2 == 'spam') + self.assertFalse(id2 == id3) + + self.assertFalse(id2 != id2) + self.assertFalse(id2 != id1) + self.assertFalse(id1 != id2) + self.assertTrue(id2 != id3) + + def test_linked_lifecycle(self): + id1 = _interpreters.create() + _testcapi.unlink_interpreter_refcount(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + id2 = self.InterpreterID(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 1) + + # The interpreter isn't linked to ID objects, so it isn't destroyed. + del id2 + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + _testcapi.link_interpreter_refcount(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + id3 = self.InterpreterID(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 1) + + # The interpreter is linked now so is destroyed. + del id3 + with self.assertRaises(_interpreters.InterpreterNotFoundError): + _testinternalcapi.get_interpreter_refcount(id1) + + class BuiltinStaticTypesTests(unittest.TestCase): TYPES = [ diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index bbfbb57b1d8299..48c0a43f29e27f 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1971,6 +1971,7 @@ def test_disallowed_reimport(self): print(_testsinglephase) ''') interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) excsnap = _interpreters.run_string(interpid, script) self.assertIsNot(excsnap, None) @@ -2105,12 +2106,18 @@ def re_load(self, name, mod): def add_subinterpreter(self): interpid = _interpreters.create(isolated=False) - _interpreters.run_string(interpid, textwrap.dedent(''' + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + _interpreters.exec(interpid, textwrap.dedent(''' import sys import _testinternalcapi ''')) def clean_up(): - _interpreters.run_string(interpid, textwrap.dedent(f''' + _interpreters.exec(interpid, textwrap.dedent(f''' name = {self.NAME!r} if name in sys.modules: sys.modules.pop(name)._clear_globals() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 914176559806f4..fe5e7b31d9c32b 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -657,14 +657,26 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): interpid = _interpreters.create(isolated=True) - excsnap = _interpreters.run_string(interpid, script) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) if excsnap is not None: if excsnap.type.__name__ == 'ImportError': raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): interpid = _interpreters.create(isolated=False) - excsnap = _interpreters.run_string(interpid, script) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) if excsnap is not None: if excsnap.type.__name__ == 'ImportError': raise ImportError(excsnap.msg) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py deleted file mode 100644 index 5663706c0ccfb7..00000000000000 --- a/Lib/test/test_interpreters.py +++ /dev/null @@ -1,1136 +0,0 @@ -import contextlib -import json -import os -import os.path -import sys -import threading -from textwrap import dedent -import unittest -import time - -from test import support -from test.support import import_helper -from test.support import threading_helper -from test.support import os_helper -_interpreters = import_helper.import_module('_xxsubinterpreters') -_channels = import_helper.import_module('_xxinterpchannels') -from test.support import interpreters - - -def _captured_script(script): - r, w = os.pipe() - indented = script.replace('\n', '\n ') - wrapped = dedent(f""" - import contextlib - with open({w}, 'w', encoding='utf-8') as spipe: - with contextlib.redirect_stdout(spipe): - {indented} - """) - return wrapped, open(r, encoding='utf-8') - - -def clean_up_interpreters(): - for interp in interpreters.list_all(): - if interp.id == 0: # main - continue - try: - interp.close() - except RuntimeError: - pass # already destroyed - - -def _run_output(interp, request, channels=None): - script, rpipe = _captured_script(request) - with rpipe: - interp.run(script, channels=channels) - return rpipe.read() - - -@contextlib.contextmanager -def _running(interp): - r, w = os.pipe() - def run(): - interp.run(dedent(f""" - # wait for "signal" - with open({r}) as rpipe: - rpipe.read() - """)) - - t = threading.Thread(target=run) - t.start() - - yield - - with open(w, 'w') as spipe: - spipe.write('done') - t.join() - - -class TestBase(unittest.TestCase): - - def pipe(self): - def ensure_closed(fd): - try: - os.close(fd) - except OSError: - pass - r, w = os.pipe() - self.addCleanup(lambda: ensure_closed(r)) - self.addCleanup(lambda: ensure_closed(w)) - return r, w - - def tearDown(self): - clean_up_interpreters() - - -class CreateTests(TestBase): - - def test_in_main(self): - interp = interpreters.create() - self.assertIsInstance(interp, interpreters.Interpreter) - self.assertIn(interp, interpreters.list_all()) - - def test_in_thread(self): - lock = threading.Lock() - interp = None - def f(): - nonlocal interp - interp = interpreters.create() - lock.acquire() - lock.release() - t = threading.Thread(target=f) - with lock: - t.start() - t.join() - self.assertIn(interp, interpreters.list_all()) - - def test_in_subinterpreter(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - interp = interpreters.create() - print(interp.id) - """)) - interp2 = interpreters.Interpreter(int(out)) - self.assertEqual(interpreters.list_all(), [main, interp, interp2]) - - def test_after_destroy_all(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp_lst = [] - for _ in range(3): - interps = interpreters.create() - interp_lst.append(interps) - # Now destroy them. - for interp in interp_lst: - interp.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp}) - - def test_after_destroy_some(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - # Now destroy 2 of them. - interp1.close() - interp2.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) - - -class GetCurrentTests(TestBase): - - def test_main(self): - main = interpreters.get_main() - current = interpreters.get_current() - self.assertEqual(current, main) - - def test_subinterpreter(self): - main = _interpreters.get_main() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - cur = interpreters.get_current() - print(cur.id) - """)) - current = interpreters.Interpreter(int(out)) - self.assertNotEqual(current, main) - - -class ListAllTests(TestBase): - - def test_initial(self): - interps = interpreters.list_all() - self.assertEqual(1, len(interps)) - - def test_after_creating(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, first.id, second.id]) - - def test_after_destroying(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - first.close() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, second.id]) - - -class TestInterpreterAttrs(TestBase): - - def test_id_type(self): - main = interpreters.get_main() - current = interpreters.get_current() - interp = interpreters.create() - self.assertIsInstance(main.id, _interpreters.InterpreterID) - self.assertIsInstance(current.id, _interpreters.InterpreterID) - self.assertIsInstance(interp.id, _interpreters.InterpreterID) - - def test_main_id(self): - main = interpreters.get_main() - self.assertEqual(main.id, 0) - - def test_custom_id(self): - interp = interpreters.Interpreter(1) - self.assertEqual(interp.id, 1) - - with self.assertRaises(TypeError): - interpreters.Interpreter('1') - - def test_id_readonly(self): - interp = interpreters.Interpreter(1) - with self.assertRaises(AttributeError): - interp.id = 2 - - @unittest.skip('not ready yet (see bpo-32604)') - def test_main_isolated(self): - main = interpreters.get_main() - self.assertFalse(main.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_subinterpreter_isolated_default(self): - interp = interpreters.create() - self.assertFalse(interp.isolated) - - def test_subinterpreter_isolated_explicit(self): - interp1 = interpreters.create(isolated=True) - interp2 = interpreters.create(isolated=False) - self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_custom_isolated_default(self): - interp = interpreters.Interpreter(1) - self.assertFalse(interp.isolated) - - def test_custom_isolated_explicit(self): - interp1 = interpreters.Interpreter(1, isolated=True) - interp2 = interpreters.Interpreter(1, isolated=False) - self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) - - def test_isolated_readonly(self): - interp = interpreters.Interpreter(1) - with self.assertRaises(AttributeError): - interp.isolated = True - - def test_equality(self): - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(interp1, interp1) - self.assertNotEqual(interp1, interp2) - - -class TestInterpreterIsRunning(TestBase): - - def test_main(self): - main = interpreters.get_main() - self.assertTrue(main.is_running()) - - @unittest.skip('Fails on FreeBSD') - def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interp.is_running()) - - with _running(interp): - self.assertTrue(interp.is_running()) - self.assertFalse(interp.is_running()) - - def test_finished(self): - r, w = self.pipe() - interp = interpreters.create() - interp.run(f"""if True: - import os - os.write({w}, b'x') - """) - self.assertFalse(interp.is_running()) - self.assertEqual(os.read(r, 1), b'x') - - def test_from_subinterpreter(self): - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - if _interpreters.is_running({interp.id}): - print(True) - else: - print(False) - """)) - self.assertEqual(out.strip(), 'True') - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(RuntimeError): - interp.is_running() - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.is_running() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.is_running() - - def test_with_only_background_threads(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - self.assertFalse(interp.is_running()) - - os.write(w_thread, DONE) - interp.run('t.join()') - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterClose(TestBase): - - def test_basic(self): - main = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2, interp3}) - interp2.close() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp3}) - - def test_all(self): - before = set(interpreters.list_all()) - interps = set() - for _ in range(3): - interp = interpreters.create() - interps.add(interp) - self.assertEqual(set(interpreters.list_all()), before | interps) - for interp in interps: - interp.close() - self.assertEqual(set(interpreters.list_all()), before) - - def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - main.close() - - def f(): - with self.assertRaises(RuntimeError): - main.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(RuntimeError): - interp.close() - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.close() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.close() - - def test_from_current(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - from test.support import interpreters - interp = interpreters.Interpreter({int(interp.id)}) - try: - interp.close() - except RuntimeError: - print('failed') - """)) - self.assertEqual(out.strip(), 'failed') - self.assertEqual(set(interpreters.list_all()), {main, interp}) - - def test_from_sibling(self): - main, = interpreters.list_all() - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2}) - interp1.run(dedent(f""" - from test.support import interpreters - interp2 = interpreters.Interpreter(int({interp2.id})) - interp2.close() - interp3 = interpreters.create() - interp3.close() - """)) - self.assertEqual(set(interpreters.list_all()), {main, interp1}) - - def test_from_other_thread(self): - interp = interpreters.create() - def f(): - interp.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - @unittest.skip('Fails on FreeBSD') - def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.close() - self.assertTrue(interp.is_running()) - - def test_subthreads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - import time - - done = False - - def notify_fini(): - global done - done = True - t.join() - threading._register_atexit(notify_fini) - - def task(): - while not done: - time.sleep(0.1) - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - interp.close() - - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterRun(TestBase): - - def test_success(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - interp.run(script) - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_failure(self): - interp = interpreters.create() - with self.assertRaises(interpreters.RunFailedError): - interp.run('raise Exception') - - def test_in_thread(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - def f(): - interp.run(script) - - t = threading.Thread(target=f) - t.start() - t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - @support.requires_fork() - def test_fork(self): - interp = interpreters.create() - import tempfile - with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: - file.write('') - file.flush() - - expected = 'spam spam spam spam spam' - script = dedent(f""" - import os - try: - os.fork() - except RuntimeError: - with open('{file.name}', 'w', encoding='utf-8') as out: - out.write('{expected}') - """) - interp.run(script) - - file.seek(0) - content = file.read() - self.assertEqual(content, expected) - - @unittest.skip('Fails on FreeBSD') - def test_already_running(self): - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.run('print("spam")') - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.run('print("spam")') - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.run('print("spam")') - - def test_bad_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.run(10) - - def test_bytes_for_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.run(b'print("spam")') - - def test_with_background_threads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - RAN = b'R' - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - os.write({w_interp}, {RAN!r}) - """) - interp.run(f"""if True: - os.write({w_interp}, {RAN!r}) - """) - - os.write(w_thread, DONE) - interp.run('t.join()') - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), FINISHED) - - # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. - - -class StressTests(TestBase): - - # In these tests we generally want a lot of interpreters, - # but not so many that any test takes too long. - - @support.requires_resource('cpu') - def test_create_many_sequential(self): - alive = [] - for _ in range(100): - interp = interpreters.create() - alive.append(interp) - - @support.requires_resource('cpu') - def test_create_many_threaded(self): - alive = [] - def task(): - interp = interpreters.create() - alive.append(interp) - threads = (threading.Thread(target=task) for _ in range(200)) - with threading_helper.start_threads(threads): - pass - - -class StartupTests(TestBase): - - # We want to ensure the initial state of subinterpreters - # matches expectations. - - _subtest_count = 0 - - @contextlib.contextmanager - def subTest(self, *args): - with super().subTest(*args) as ctx: - self._subtest_count += 1 - try: - yield ctx - finally: - if self._debugged_in_subtest: - if self._subtest_count == 1: - # The first subtest adds a leading newline, so we - # compensate here by not printing a trailing newline. - print('### end subtest debug ###', end='') - else: - print('### end subtest debug ###') - self._debugged_in_subtest = False - - def debug(self, msg, *, header=None): - if header: - self._debug(f'--- {header} ---') - if msg: - if msg.endswith(os.linesep): - self._debug(msg[:-len(os.linesep)]) - else: - self._debug(msg) - self._debug('') - self._debug('------') - else: - self._debug(msg) - - _debugged = False - _debugged_in_subtest = False - def _debug(self, msg): - if not self._debugged: - print() - self._debugged = True - if self._subtest is not None: - if True: - if not self._debugged_in_subtest: - self._debugged_in_subtest = True - print('### start subtest debug ###') - print(msg) - else: - print(msg) - - def create_temp_dir(self): - import tempfile - tmp = tempfile.mkdtemp(prefix='test_interpreters_') - tmp = os.path.realpath(tmp) - self.addCleanup(os_helper.rmtree, tmp) - return tmp - - def write_script(self, *path, text): - filename = os.path.join(*path) - dirname = os.path.dirname(filename) - if dirname: - os.makedirs(dirname, exist_ok=True) - with open(filename, 'w', encoding='utf-8') as outfile: - outfile.write(dedent(text)) - return filename - - @support.requires_subprocess() - def run_python(self, argv, *, cwd=None): - # This method is inspired by - # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. - import shlex - import subprocess - if isinstance(argv, str): - argv = shlex.split(argv) - argv = [sys.executable, *argv] - try: - proc = subprocess.run( - argv, - cwd=cwd, - capture_output=True, - text=True, - ) - except Exception as exc: - self.debug(f'# cmd: {shlex.join(argv)}') - if isinstance(exc, FileNotFoundError) and not exc.filename: - if os.path.exists(argv[0]): - exists = 'exists' - else: - exists = 'does not exist' - self.debug(f'{argv[0]} {exists}') - raise # re-raise - assert proc.stderr == '' or proc.returncode != 0, proc.stderr - if proc.returncode != 0 and support.verbose: - self.debug(f'# python3 {shlex.join(argv[1:])} failed:') - self.debug(proc.stdout, header='stdout') - self.debug(proc.stderr, header='stderr') - self.assertEqual(proc.returncode, 0) - self.assertEqual(proc.stderr, '') - return proc.stdout - - def test_sys_path_0(self): - # The main interpreter's sys.path[0] should be used by subinterpreters. - script = ''' - import sys - from test.support import interpreters - - orig = sys.path[0] - - interp = interpreters.create() - interp.run(f"""if True: - import json - import sys - print(json.dumps({{ - 'main': {orig!r}, - 'sub': sys.path[0], - }}, indent=4), flush=True) - """) - ''' - # / - # pkg/ - # __init__.py - # __main__.py - # script.py - # script.py - cwd = self.create_temp_dir() - self.write_script(cwd, 'pkg', '__init__.py', text='') - self.write_script(cwd, 'pkg', '__main__.py', text=script) - self.write_script(cwd, 'pkg', 'script.py', text=script) - self.write_script(cwd, 'script.py', text=script) - - cases = [ - ('script.py', cwd), - ('-m script', cwd), - ('-m pkg', cwd), - ('-m pkg.script', cwd), - ('-c "import script"', ''), - ] - for argv, expected in cases: - with self.subTest(f'python3 {argv}'): - out = self.run_python(argv, cwd=cwd) - data = json.loads(out) - sp0_main, sp0_sub = data['main'], data['sub'] - self.assertEqual(sp0_sub, sp0_main) - self.assertEqual(sp0_sub, expected) - # XXX Also check them all with the -P cmdline flag? - - -class FinalizationTests(TestBase): - - def test_gh_109793(self): - import subprocess - argv = [sys.executable, '-c', '''if True: - import _xxsubinterpreters as _interpreters - interpid = _interpreters.create() - raise Exception - '''] - proc = subprocess.run(argv, capture_output=True, text=True) - self.assertIn('Traceback', proc.stderr) - if proc.returncode == 0 and support.verbose: - print() - print("--- cmd unexpected succeeded ---") - print(f"stdout:\n{proc.stdout}") - print(f"stderr:\n{proc.stderr}") - print("------") - self.assertEqual(proc.returncode, 1) - - -class TestIsShareable(TestBase): - - def test_default_shareables(self): - shareables = [ - # singletons - None, - # builtin objects - b'spam', - 'spam', - 10, - -10, - True, - False, - 100.0, - (), - (1, ('spam', 'eggs'), True), - ] - for obj in shareables: - with self.subTest(obj): - shareable = interpreters.is_shareable(obj) - self.assertTrue(shareable) - - def test_not_shareable(self): - class Cheese: - def __init__(self, name): - self.name = name - def __str__(self): - return self.name - - class SubBytes(bytes): - """A subclass of a shareable type.""" - - not_shareables = [ - # singletons - NotImplemented, - ..., - # builtin types and objects - type, - object, - object(), - Exception(), - # user-defined types and objects - Cheese, - Cheese('Wensleydale'), - SubBytes(b'spam'), - ] - for obj in not_shareables: - with self.subTest(repr(obj)): - self.assertFalse( - interpreters.is_shareable(obj)) - - -class TestChannels(TestBase): - - def test_create(self): - r, s = interpreters.create_channel() - self.assertIsInstance(r, interpreters.RecvChannel) - self.assertIsInstance(s, interpreters.SendChannel) - - def test_list_all(self): - self.assertEqual(interpreters.list_all_channels(), []) - created = set() - for _ in range(3): - ch = interpreters.create_channel() - created.add(ch) - after = set(interpreters.list_all_channels()) - self.assertEqual(after, created) - - def test_shareable(self): - rch, sch = interpreters.create_channel() - - self.assertTrue( - interpreters.is_shareable(rch)) - self.assertTrue( - interpreters.is_shareable(sch)) - - sch.send_nowait(rch) - sch.send_nowait(sch) - rch2 = rch.recv() - sch2 = rch.recv() - - self.assertEqual(rch2, rch) - self.assertEqual(sch2, sch) - - def test_is_closed(self): - rch, sch = interpreters.create_channel() - rbefore = rch.is_closed - sbefore = sch.is_closed - rch.close() - rafter = rch.is_closed - safter = sch.is_closed - - self.assertFalse(rbefore) - self.assertFalse(sbefore) - self.assertTrue(rafter) - self.assertTrue(safter) - - -class TestRecvChannelAttrs(TestBase): - - def test_id_type(self): - rch, _ = interpreters.create_channel() - self.assertIsInstance(rch.id, _channels.ChannelID) - - def test_custom_id(self): - rch = interpreters.RecvChannel(1) - self.assertEqual(rch.id, 1) - - with self.assertRaises(TypeError): - interpreters.RecvChannel('1') - - def test_id_readonly(self): - rch = interpreters.RecvChannel(1) - with self.assertRaises(AttributeError): - rch.id = 2 - - def test_equality(self): - ch1, _ = interpreters.create_channel() - ch2, _ = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendChannelAttrs(TestBase): - - def test_id_type(self): - _, sch = interpreters.create_channel() - self.assertIsInstance(sch.id, _channels.ChannelID) - - def test_custom_id(self): - sch = interpreters.SendChannel(1) - self.assertEqual(sch.id, 1) - - with self.assertRaises(TypeError): - interpreters.SendChannel('1') - - def test_id_readonly(self): - sch = interpreters.SendChannel(1) - with self.assertRaises(AttributeError): - sch.id = 2 - - def test_equality(self): - _, ch1 = interpreters.create_channel() - _, ch2 = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendRecv(TestBase): - - def test_send_recv_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_same_interpreter(self): - interp = interpreters.create() - interp.run(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - assert obj == orig, 'expected: obj == orig' - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_send_recv_different_threads(self): - r, s = interpreters.create_channel() - - def f(): - while True: - try: - obj = r.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - s.send(obj) - t = threading.Thread(target=f) - t.start() - - orig = b'spam' - s.send(orig) - obj = r.recv() - t.join() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main_with_default(self): - r, _ = interpreters.create_channel() - obj = r.recv_nowait(None) - - self.assertIsNone(obj) - - def test_send_recv_nowait_same_interpreter(self): - interp = interpreters.create() - interp.run(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - assert obj == orig, 'expected: obj == orig' - # When going back to the same interpreter we get the same object. - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_nowait_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv_nowait() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv_nowait() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_recv_timeout(self): - r, _ = interpreters.create_channel() - with self.assertRaises(TimeoutError): - r.recv(timeout=1) - - def test_recv_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv() - - def test_send_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send(b'spam') - - def test_recv_nowait_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv_nowait() - - def test_send_nowait_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send_nowait(b'spam') - - def test_recv_nowait_empty(self): - ch, _ = interpreters.create_channel() - with self.assertRaises(interpreters.ChannelEmptyError): - ch.recv_nowait() - - def test_recv_nowait_default(self): - default = object() - rch, sch = interpreters.create_channel() - obj1 = rch.recv_nowait(default) - sch.send_nowait(None) - sch.send_nowait(1) - sch.send_nowait(b'spam') - sch.send_nowait(b'eggs') - obj2 = rch.recv_nowait(default) - obj3 = rch.recv_nowait(default) - obj4 = rch.recv_nowait() - obj5 = rch.recv_nowait(default) - obj6 = rch.recv_nowait(default) - - self.assertIs(obj1, default) - self.assertIs(obj2, None) - self.assertEqual(obj3, 1) - self.assertEqual(obj4, b'spam') - self.assertEqual(obj5, b'eggs') - self.assertIs(obj6, default) - - def test_send_buffer(self): - buf = bytearray(b'spamspamspam') - obj = None - rch, sch = interpreters.create_channel() - - def f(): - nonlocal obj - while True: - try: - obj = rch.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - t = threading.Thread(target=f) - t.start() - - sch.send_buffer(buf) - t.join() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) - - def test_send_buffer_nowait(self): - buf = bytearray(b'spamspamspam') - rch, sch = interpreters.create_channel() - sch.send_buffer_nowait(buf) - obj = rch.recv() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_interpreters/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_interpreters/__main__.py b/Lib/test/test_interpreters/__main__.py new file mode 100644 index 00000000000000..8641229877b2be --- /dev/null +++ b/Lib/test/test_interpreters/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +nittest.main() diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py new file mode 100644 index 00000000000000..e4ae9d005b5282 --- /dev/null +++ b/Lib/test/test_interpreters/test_api.py @@ -0,0 +1,642 @@ +import os +import threading +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from test.support import interpreters +from test.support.interpreters import InterpreterNotFoundError +from .utils import _captured_script, _run_output, _running, TestBase + + +class ModuleTests(TestBase): + + def test_queue_aliases(self): + first = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + second = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + self.assertEqual(second, first) + + +class CreateTests(TestBase): + + def test_in_main(self): + interp = interpreters.create() + self.assertIsInstance(interp, interpreters.Interpreter) + self.assertIn(interp, interpreters.list_all()) + + def test_in_thread(self): + lock = threading.Lock() + interp = None + def f(): + nonlocal interp + interp = interpreters.create() + lock.acquire() + lock.release() + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(interp, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + interp = interpreters.create() + print(interp.id) + """)) + interp2 = interpreters.Interpreter(int(out)) + self.assertEqual(interpreters.list_all(), [main, interp, interp2]) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp_lst = [] + for _ in range(3): + interps = interpreters.create() + interp_lst.append(interps) + # Now destroy them. + for interp in interp_lst: + interp.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + # Now destroy 2 of them. + interp1.close() + interp2.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) + + +class GetMainTests(TestBase): + + def test_id(self): + main = interpreters.get_main() + self.assertEqual(main.id, 0) + + def test_current(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertIs(main, current) + + def test_idempotent(self): + main1 = interpreters.get_main() + main2 = interpreters.get_main() + self.assertIs(main1, main2) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertEqual(current, main) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(cur.id) + """)) + current = interpreters.Interpreter(int(out)) + self.assertEqual(current, interp) + self.assertNotEqual(current, main) + + def test_idempotent(self): + with self.subTest('main'): + cur1 = interpreters.get_current() + cur2 = interpreters.get_current() + self.assertIs(cur1, cur2) + + with self.subTest('subinterpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + cur = interpreters.get_current() + print(id(cur)) + """)) + objid1, objid2 = (int(v) for v in out.splitlines()) + self.assertEqual(objid1, objid2) + + with self.subTest('per-interpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + """)) + id1 = int(out) + id2 = id(interp) + self.assertNotEqual(id1, id2) + + +class ListAllTests(TestBase): + + def test_initial(self): + interps = interpreters.list_all() + self.assertEqual(1, len(interps)) + + def test_after_creating(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, first.id, second.id]) + + def test_after_destroying(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + first.close() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, second.id]) + + def test_idempotent(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + expected = [main, first, second] + + actual = interpreters.list_all() + + self.assertEqual(actual, expected) + for interp1, interp2 in zip(actual, expected): + self.assertIs(interp1, interp2) + + +class InterpreterObjectTests(TestBase): + + def test_init_int(self): + interpid = interpreters.get_current().id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp.id, interpid) + + def test_init_interpreter_id(self): + interpid = interpreters.get_current()._id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp._id, interpid) + + def test_init_unsupported(self): + actualid = interpreters.get_current().id + for interpid in [ + str(actualid), + float(actualid), + object(), + None, + '', + ]: + with self.subTest(repr(interpid)): + with self.assertRaises(TypeError): + interpreters.Interpreter(interpid) + + def test_idempotent(self): + main = interpreters.get_main() + interp = interpreters.Interpreter(main.id) + self.assertIs(interp, main) + + def test_init_does_not_exist(self): + with self.assertRaises(InterpreterNotFoundError): + interpreters.Interpreter(1_000_000) + + def test_init_bad_id(self): + with self.assertRaises(ValueError): + interpreters.Interpreter(-1) + + def test_id_type(self): + main = interpreters.get_main() + current = interpreters.get_current() + interp = interpreters.create() + self.assertIsInstance(main.id, int) + self.assertIsInstance(current.id, int) + self.assertIsInstance(interp.id, int) + + def test_id_readonly(self): + interp = interpreters.create() + with self.assertRaises(AttributeError): + interp.id = 1_000_000 + + def test_hashable(self): + interp = interpreters.create() + expected = hash(interp.id) + actual = hash(interp) + self.assertEqual(actual, expected) + + def test_equality(self): + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(interp1, interp1) + self.assertNotEqual(interp1, interp2) + + +class TestInterpreterIsRunning(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(main.is_running()) + + @unittest.skip('Fails on FreeBSD') + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interp.is_running()) + + with _running(interp): + self.assertTrue(interp.is_running()) + self.assertFalse(interp.is_running()) + + def test_finished(self): + r, w = self.pipe() + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + os.write({w}, b'x') + """) + self.assertFalse(interp.is_running()) + self.assertEqual(os.read(r, 1), b'x') + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + if _interpreters.is_running({interp.id}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.is_running() + + def test_with_only_background_threads(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + self.assertFalse(interp.is_running()) + + os.write(w_thread, DONE) + interp.exec_sync('t.join()') + self.assertEqual(os.read(r_interp, 1), FINISHED) + + +class TestInterpreterClose(TestBase): + + def test_basic(self): + main = interpreters.get_main() + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2, interp3}) + interp2.close() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp3}) + + def test_all(self): + before = set(interpreters.list_all()) + interps = set() + for _ in range(3): + interp = interpreters.create() + interps.add(interp) + self.assertEqual(set(interpreters.list_all()), before | interps) + for interp in interps: + interp.close() + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + main.close() + + def f(): + with self.assertRaises(RuntimeError): + main.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.close() + + def test_from_current(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + from test.support import interpreters + interp = interpreters.Interpreter({interp.id}) + try: + interp.close() + except RuntimeError: + print('failed') + """)) + self.assertEqual(out.strip(), 'failed') + self.assertEqual(set(interpreters.list_all()), {main, interp}) + + def test_from_sibling(self): + main, = interpreters.list_all() + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2}) + interp1.exec_sync(dedent(f""" + from test.support import interpreters + interp2 = interpreters.Interpreter({interp2.id}) + interp2.close() + interp3 = interpreters.create() + interp3.close() + """)) + self.assertEqual(set(interpreters.list_all()), {main, interp1}) + + def test_from_other_thread(self): + interp = interpreters.create() + def f(): + interp.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + @unittest.skip('Fails on FreeBSD') + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.close() + self.assertTrue(interp.is_running()) + + def test_subthreads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + +class TestInterpreterExecSync(TestBase): + + def test_success(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + interp.exec_sync(script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + interp = interpreters.create() + with self.assertRaises(interpreters.ExecFailure): + interp.exec_sync('raise Exception') + + def test_in_thread(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interp.exec_sync(script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + @support.requires_fork() + def test_fork(self): + interp = interpreters.create() + import tempfile + with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w', encoding='utf-8') as out: + out.write('{expected}') + """) + interp.exec_sync(script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + @unittest.skip('Fails on FreeBSD') + def test_already_running(self): + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.exec_sync('print("spam")') + + def test_bad_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec_sync(10) + + def test_bytes_for_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec_sync(b'print("spam")') + + def test_with_background_threads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + RAN = b'R' + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + os.write({w_interp}, {RAN!r}) + """) + interp.exec_sync(f"""if True: + os.write({w_interp}, {RAN!r}) + """) + + os.write(w_thread, DONE) + interp.exec_sync('t.join()') + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), FINISHED) + + # test_xxsubinterpreters covers the remaining + # Interpreter.exec_sync() behavior. + + +class TestInterpreterRun(TestBase): + + def test_success(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + t = interp.run(script) + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + caught = False + def excepthook(args): + nonlocal caught + caught = True + threading.excepthook = excepthook + try: + interp = interpreters.create() + t = interp.run('raise Exception') + t.join() + + self.assertTrue(caught) + except BaseException: + threading.excepthook = threading.__excepthook__ + + +class TestIsShareable(TestBase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + True, + False, + 100.0, + (), + (1, ('spam', 'eggs'), True), + ] + for obj in shareables: + with self.subTest(obj): + shareable = interpreters.is_shareable(obj) + self.assertTrue(shareable) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py new file mode 100644 index 00000000000000..3c3e18832d4168 --- /dev/null +++ b/Lib/test/test_interpreters/test_channels.py @@ -0,0 +1,328 @@ +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +_channels = import_helper.import_module('_xxinterpchannels') +from test.support import interpreters +from test.support.interpreters import channels +from .utils import _run_output, TestBase + + +class TestChannels(TestBase): + + def test_create(self): + r, s = channels.create() + self.assertIsInstance(r, channels.RecvChannel) + self.assertIsInstance(s, channels.SendChannel) + + def test_list_all(self): + self.assertEqual(channels.list_all(), []) + created = set() + for _ in range(3): + ch = channels.create() + created.add(ch) + after = set(channels.list_all()) + self.assertEqual(after, created) + + def test_shareable(self): + rch, sch = channels.create() + + self.assertTrue( + interpreters.is_shareable(rch)) + self.assertTrue( + interpreters.is_shareable(sch)) + + sch.send_nowait(rch) + sch.send_nowait(sch) + rch2 = rch.recv() + sch2 = rch.recv() + + self.assertEqual(rch2, rch) + self.assertEqual(sch2, sch) + + def test_is_closed(self): + rch, sch = channels.create() + rbefore = rch.is_closed + sbefore = sch.is_closed + rch.close() + rafter = rch.is_closed + safter = sch.is_closed + + self.assertFalse(rbefore) + self.assertFalse(sbefore) + self.assertTrue(rafter) + self.assertTrue(safter) + + +class TestRecvChannelAttrs(TestBase): + + def test_id_type(self): + rch, _ = channels.create() + self.assertIsInstance(rch.id, _channels.ChannelID) + + def test_custom_id(self): + rch = channels.RecvChannel(1) + self.assertEqual(rch.id, 1) + + with self.assertRaises(TypeError): + channels.RecvChannel('1') + + def test_id_readonly(self): + rch = channels.RecvChannel(1) + with self.assertRaises(AttributeError): + rch.id = 2 + + def test_equality(self): + ch1, _ = channels.create() + ch2, _ = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + +class TestSendChannelAttrs(TestBase): + + def test_id_type(self): + _, sch = channels.create() + self.assertIsInstance(sch.id, _channels.ChannelID) + + def test_custom_id(self): + sch = channels.SendChannel(1) + self.assertEqual(sch.id, 1) + + with self.assertRaises(TypeError): + channels.SendChannel('1') + + def test_id_readonly(self): + sch = channels.SendChannel(1) + with self.assertRaises(AttributeError): + sch.id = 2 + + def test_equality(self): + _, ch1 = channels.create() + _, ch2 = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + +class TestSendRecv(TestBase): + + def test_send_recv_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + from test.support.interpreters import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_send_recv_different_threads(self): + r, s = channels.create() + + def f(): + while True: + try: + obj = r.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + s.send(orig) + obj = r.recv() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main_with_default(self): + r, _ = channels.create() + obj = r.recv_nowait(None) + + self.assertIsNone(obj) + + def test_send_recv_nowait_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + from test.support.interpreters import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + assert obj == orig, 'expected: obj == orig' + # When going back to the same interpreter we get the same object. + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_nowait_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv_nowait() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv_nowait() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_recv_timeout(self): + r, _ = channels.create() + with self.assertRaises(TimeoutError): + r.recv(timeout=1) + + def test_recv_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv() + + def test_send_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send(b'spam') + + def test_recv_nowait_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv_nowait() + + def test_send_nowait_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send_nowait(b'spam') + + def test_recv_nowait_empty(self): + ch, _ = channels.create() + with self.assertRaises(channels.ChannelEmptyError): + ch.recv_nowait() + + def test_recv_nowait_default(self): + default = object() + rch, sch = channels.create() + obj1 = rch.recv_nowait(default) + sch.send_nowait(None) + sch.send_nowait(1) + sch.send_nowait(b'spam') + sch.send_nowait(b'eggs') + obj2 = rch.recv_nowait(default) + obj3 = rch.recv_nowait(default) + obj4 = rch.recv_nowait() + obj5 = rch.recv_nowait(default) + obj6 = rch.recv_nowait(default) + + self.assertIs(obj1, default) + self.assertIs(obj2, None) + self.assertEqual(obj3, 1) + self.assertEqual(obj4, b'spam') + self.assertEqual(obj5, b'eggs') + self.assertIs(obj6, default) + + def test_send_buffer(self): + buf = bytearray(b'spamspamspam') + obj = None + rch, sch = channels.create() + + def f(): + nonlocal obj + while True: + try: + obj = rch.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + t = threading.Thread(target=f) + t.start() + + sch.send_buffer(buf) + t.join() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + def test_send_buffer_nowait(self): + buf = bytearray(b'spamspamspam') + rch, sch = channels.create() + sch.send_buffer_nowait(buf) + obj = rch.recv() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py new file mode 100644 index 00000000000000..c2917d839904f9 --- /dev/null +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -0,0 +1,189 @@ +import contextlib +import json +import os +import os.path +import sys +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +from test.support import os_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from .utils import TestBase + + +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + +class FinalizationTests(TestBase): + + def test_gh_109793(self): + # Make sure finalization finishes and the correct error code + # is reported, even when subinterpreters get cleaned up at the end. + import subprocess + argv = [sys.executable, '-c', '''if True: + from test.support import interpreters + interp = interpreters.create() + raise Exception + '''] + proc = subprocess.run(argv, capture_output=True, text=True) + self.assertIn('Traceback', proc.stderr) + if proc.returncode == 0 and support.verbose: + print() + print("--- cmd unexpected succeeded ---") + print(f"stdout:\n{proc.stdout}") + print(f"stderr:\n{proc.stderr}") + print("------") + self.assertEqual(proc.returncode, 1) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py new file mode 100644 index 00000000000000..2af90b14d3e3c4 --- /dev/null +++ b/Lib/test/test_interpreters/test_queues.py @@ -0,0 +1,233 @@ +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxinterpchannels') +#import_helper.import_module('_xxinterpqueues') +from test.support import interpreters +from test.support.interpreters import queues +from .utils import _run_output, TestBase + + +class QueueTests(TestBase): + + def test_create(self): + with self.subTest('vanilla'): + queue = queues.create() + self.assertEqual(queue.maxsize, 0) + + with self.subTest('small maxsize'): + queue = queues.create(3) + self.assertEqual(queue.maxsize, 3) + + with self.subTest('big maxsize'): + queue = queues.create(100) + self.assertEqual(queue.maxsize, 100) + + with self.subTest('no maxsize'): + queue = queues.create(0) + self.assertEqual(queue.maxsize, 0) + + with self.subTest('negative maxsize'): + queue = queues.create(-1) + self.assertEqual(queue.maxsize, 0) + + with self.subTest('bad maxsize'): + with self.assertRaises(TypeError): + queues.create('1') + + @unittest.expectedFailure + def test_shareable(self): + queue1 = queues.create() + queue2 = queues.create() + queue1.put(queue2) + queue3 = queue1.get() + self.assertIs(queue3, queue1) + + def test_id_type(self): + queue = queues.create() + self.assertIsInstance(queue.id, int) + + def test_custom_id(self): + with self.assertRaises(queues.QueueNotFoundError): + queues.Queue(1_000_000) + + def test_id_readonly(self): + queue = queues.create() + with self.assertRaises(AttributeError): + queue.id = 1_000_000 + + def test_maxsize_readonly(self): + queue = queues.create(10) + with self.assertRaises(AttributeError): + queue.maxsize = 1_000_000 + + def test_hashable(self): + queue = queues.create() + expected = hash(queue.id) + actual = hash(queue) + self.assertEqual(actual, expected) + + def test_equality(self): + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(queue1, queue1) + self.assertNotEqual(queue1, queue2) + + +class TestQueueOps(TestBase): + + def test_empty(self): + queue = queues.create() + before = queue.empty() + queue.put(None) + during = queue.empty() + queue.get() + after = queue.empty() + + self.assertIs(before, True) + self.assertIs(during, False) + self.assertIs(after, True) + + def test_full(self): + expected = [False, False, False, True, False, False, False] + actual = [] + queue = queues.create(3) + for _ in range(3): + actual.append(queue.full()) + queue.put(None) + actual.append(queue.full()) + for _ in range(3): + queue.get() + actual.append(queue.full()) + + self.assertEqual(actual, expected) + + def test_qsize(self): + expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] + actual = [] + queue = queues.create() + for _ in range(3): + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + for _ in range(3): + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + + self.assertEqual(actual, expected) + + def test_put_get_main(self): + expected = list(range(20)) + queue = queues.create() + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + @unittest.expectedFailure + def test_put_timeout(self): + queue = queues.create(2) + queue.put(None) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1) + queue.get() + queue.put(None) + + @unittest.expectedFailure + def test_put_nowait(self): + queue = queues.create(2) + queue.put_nowait(None) + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + queue.get() + queue.put_nowait(None) + + def test_get_timeout(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get(timeout=0.1) + + def test_get_nowait(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_get_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + from test.support.interpreters import queues + queue = queues.create() + orig = b'spam' + queue.put(orig) + obj = queue.get() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.expectedFailure + def test_put_get_different_interpreters(self): + queue1 = queues.create() + queue2 = queues.create() + obj1 = b'spam' + queue1.put(obj1) + out = _run_output( + interpreters.create(), + dedent(f""" + import test.support.interpreters.queue as queues + queue1 = queues.Queue({queue1.id}) + queue2 = queues.Queue({queue2.id}) + obj = queue1.get() + assert obj == b'spam', 'expected: obj == obj1' + # When going to another interpreter we get a copy. + assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + obj2 = b'eggs' + print(id(obj2)) + queue2.put(obj2) + """)) + obj2 = queue2.get() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_put_get_different_threads(self): + queue1 = queues.create() + queue2 = queues.create() + + def f(): + while True: + try: + obj = queue1.get(timeout=0.1) + break + except queues.QueueEmpty: + continue + queue2.put(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + queue1.put(orig) + obj = queue2.get() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py new file mode 100644 index 00000000000000..3cc570b3bf7128 --- /dev/null +++ b/Lib/test/test_interpreters/test_stress.py @@ -0,0 +1,38 @@ +import threading +import unittest + +from test import support +from test.support import import_helper +from test.support import threading_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from test.support import interpreters +from .utils import TestBase + + +class StressTests(TestBase): + + # In these tests we generally want a lot of interpreters, + # but not so many that any test takes too long. + + @support.requires_resource('cpu') + def test_create_many_sequential(self): + alive = [] + for _ in range(100): + interp = interpreters.create() + alive.append(interp) + + @support.requires_resource('cpu') + def test_create_many_threaded(self): + alive = [] + def task(): + interp = interpreters.create() + alive.append(interp) + threads = (threading.Thread(target=task) for _ in range(200)) + with threading_helper.start_threads(threads): + pass + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py new file mode 100644 index 00000000000000..623c8737b79831 --- /dev/null +++ b/Lib/test/test_interpreters/utils.py @@ -0,0 +1,73 @@ +import contextlib +import os +import threading +from textwrap import dedent +import unittest + +from test.support import interpreters + + +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w', encoding='utf-8') as spipe: + with contextlib.redirect_stdout(spipe): + {indented} + """) + return wrapped, open(r, encoding='utf-8') + + +def clean_up_interpreters(): + for interp in interpreters.list_all(): + if interp.id == 0: # main + continue + try: + interp.close() + except RuntimeError: + pass # already destroyed + + +def _run_output(interp, request, channels=None): + script, rpipe = _captured_script(request) + with rpipe: + interp.exec_sync(script, channels=channels) + return rpipe.read() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.exec_sync(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + +class TestBase(unittest.TestCase): + + def pipe(self): + def ensure_closed(fd): + try: + os.close(fd) + except OSError: + pass + r, w = os.pipe() + self.addCleanup(lambda: ensure_closed(r)) + self.addCleanup(lambda: ensure_closed(w)) + return r, w + + def tearDown(self): + clean_up_interpreters() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index db5ba16c4d9739..6c87dfabad9f0f 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -729,7 +729,7 @@ def test_subinterp_intern_dynamically_allocated(self): self.assertIs(t, s) interp = interpreters.create() - interp.run(textwrap.dedent(f''' + interp.exec_sync(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) != {id(s)}, (id(t), {id(s)}) @@ -744,7 +744,7 @@ def test_subinterp_intern_statically_allocated(self): t = sys.intern(s) interp = interpreters.create() - interp.run(textwrap.dedent(f''' + interp.exec_sync(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) == {id(t)}, (id(t), {id(t)}) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 146e2dbc0fc396..a5744a4037ecea 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1365,7 +1365,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import os import threading import time diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9fdd67093338e4..b3ddfae58e6fc0 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -13,6 +13,7 @@ #include "_testcapi/parts.h" #include "frameobject.h" // PyFrame_New() +#include "interpreteridobject.h" // PyInterpreterID_Type #include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX @@ -1451,6 +1452,36 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } +static PyObject * +get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return Py_NewRef(&PyInterpreterID_Type); +} + +static PyObject * +link_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 1); + Py_RETURN_NONE; +} + +static PyObject * +unlink_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 0); + Py_RETURN_NONE; +} + static PyMethodDef ml; static PyObject * @@ -3237,6 +3268,9 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, + {"get_interpreterid_type", get_interpreterid_type, METH_NOARGS}, + {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, + {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ba7653f2d9c7aa..7d277df164d3ec 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1475,6 +1475,17 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } +static PyObject * +get_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + return NULL; + } + return PyLong_FromLongLong(interp->id_refcount); +} + + static void _xid_capsule_destructor(PyObject *capsule) { @@ -1693,6 +1704,7 @@ static PyMethodDef module_functions[] = { {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, + {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 1c9ae3b87adf7c..97729ec269cb62 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -8,7 +8,6 @@ #include "Python.h" #include "interpreteridobject.h" #include "pycore_crossinterp.h" // struct _xid -#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_interp.h" // _PyInterpreterState_LookUpID() #ifdef MS_WINDOWS @@ -263,136 +262,6 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) } -/* Cross-interpreter Buffer Views *******************************************/ - -// XXX Release when the original interpreter is destroyed. - -typedef struct { - PyObject_HEAD - Py_buffer *view; - int64_t interpid; -} XIBufferViewObject; - -static PyObject * -xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) -{ - assert(data->data != NULL); - assert(data->obj == NULL); - assert(data->interpid >= 0); - XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); - if (self == NULL) { - return NULL; - } - PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)data->data; - self->interpid = data->interpid; - return (PyObject *)self; -} - -static void -xibufferview_dealloc(XIBufferViewObject *self) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid); - /* If the interpreter is no longer alive then we have problems, - since other objects may be using the buffer still. */ - assert(interp != NULL); - - if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { - // XXX Emit a warning? - PyErr_Clear(); - } - - PyTypeObject *tp = Py_TYPE(self); - tp->tp_free(self); - /* "Instances of heap-allocated types hold a reference to their type." - * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol - * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse - */ - // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, - // like we do for _abc._abc_data? - Py_DECREF(tp); -} - -static int -xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) -{ - /* Only PyMemoryView_FromObject() should ever call this, - via _memoryview_from_xid() below. */ - *view = *self->view; - view->obj = (PyObject *)self; - // XXX Should we leave it alone? - view->internal = NULL; - return 0; -} - -static PyType_Slot XIBufferViewType_slots[] = { - {Py_tp_dealloc, (destructor)xibufferview_dealloc}, - {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, - // We don't bother with Py_bf_releasebuffer since we don't need it. - {0, NULL}, -}; - -static PyType_Spec XIBufferViewType_spec = { - .name = MODULE_NAME ".CrossInterpreterBufferView", - .basicsize = sizeof(XIBufferViewObject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), - .slots = XIBufferViewType_slots, -}; - - -/* extra XID types **********************************************************/ - -static PyTypeObject * _get_current_xibufferview_type(void); - -static PyObject * -_memoryview_from_xid(_PyCrossInterpreterData *data) -{ - PyTypeObject *cls = _get_current_xibufferview_type(); - if (cls == NULL) { - return NULL; - } - PyObject *obj = xibufferview_from_xid(cls, data); - if (obj == NULL) { - return NULL; - } - return PyMemoryView_FromObject(obj); -} - -static int -_memoryview_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); - if (view == NULL) { - return -1; - } - if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { - PyMem_RawFree(view); - return -1; - } - _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, - _memoryview_from_xid); - return 0; -} - -static int -register_builtin_xid_types(struct xid_class_registry *classes) -{ - PyTypeObject *cls; - crossinterpdatafunc func; - - // builtin memoryview - cls = &PyMemoryView_Type; - func = _memoryview_shared; - if (register_xid_class(cls, func, classes)) { - return -1; - } - - return 0; -} - - /* module state *************************************************************/ typedef struct { @@ -405,7 +274,6 @@ typedef struct { /* heap types */ PyTypeObject *ChannelInfoType; PyTypeObject *ChannelIDType; - PyTypeObject *XIBufferViewType; /* exceptions */ PyObject *ChannelError; @@ -449,7 +317,6 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) /* heap types */ Py_VISIT(state->ChannelInfoType); Py_VISIT(state->ChannelIDType); - Py_VISIT(state->XIBufferViewType); /* exceptions */ Py_VISIT(state->ChannelError); @@ -474,7 +341,6 @@ clear_module_state(module_state *state) (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); } Py_CLEAR(state->ChannelIDType); - Py_CLEAR(state->XIBufferViewType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -487,17 +353,6 @@ clear_module_state(module_state *state) } -static PyTypeObject * -_get_current_xibufferview_type(void) -{ - module_state *state = _get_current_module_state(); - if (state == NULL) { - return NULL; - } - return state->XIBufferViewType; -} - - /* channel-specific code ****************************************************/ #define CHANNEL_SEND 1 @@ -3463,18 +3318,6 @@ module_exec(PyObject *mod) goto error; } - // XIBufferView - state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL, - xid_classes); - if (state->XIBufferViewType == NULL) { - goto error; - } - - // Register external types. - if (register_builtin_xid_types(xid_classes) < 0) { - goto error; - } - /* Make sure chnnels drop objects owned by this interpreter. */ PyInterpreterState *interp = _get_current_interp(); PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 02c2abed27ddfa..37959e953ee4f5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -6,11 +6,14 @@ #endif #include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_crossinterp.h" // struct _xid -#include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() +#include "pycore_pyerrors.h" // _Py_excinfo #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "interpreteridobject.h" @@ -28,11 +31,260 @@ _get_current_interp(void) return PyInterpreterState_Get(); } +static int64_t +pylong_to_interpid(PyObject *idobj) +{ + assert(PyLong_CheckExact(idobj)); + + if (_PyLong_IsNegative((PyLongObject *)idobj)) { + PyErr_Format(PyExc_ValueError, + "interpreter ID must be a non-negative int, got %R", + idobj); + return -1; + } + + int overflow; + long long id = PyLong_AsLongLongAndOverflow(idobj, &overflow); + if (id == -1) { + if (!overflow) { + assert(PyErr_Occurred()); + return -1; + } + assert(!PyErr_Occurred()); + // For now, we don't worry about if LLONG_MAX < INT64_MAX. + goto bad_id; + } +#if LLONG_MAX > INT64_MAX + if (id > INT64_MAX) { + goto bad_id; + } +#endif + return (int64_t)id; + +bad_id: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %O", idobj); + return -1; +} + +static int64_t +convert_interpid_obj(PyObject *arg) +{ + int64_t id = -1; + if (_PyIndex_Check(arg)) { + PyObject *idobj = PyNumber_Long(arg); + if (idobj == NULL) { + return -1; + } + id = pylong_to_interpid(idobj); + Py_DECREF(idobj); + if (id < 0) { + return -1; + } + } + else { + PyErr_Format(PyExc_TypeError, + "interpreter ID must be an int, got %.100s", + Py_TYPE(arg)->tp_name); + return -1; + } + return id; +} + +static PyInterpreterState * +look_up_interp(PyObject *arg) +{ + int64_t id = convert_interpid_obj(arg); + if (id < 0) { + return NULL; + } + return _PyInterpreterState_LookUpID(id); +} + + +static PyObject * +interpid_to_pylong(int64_t id) +{ + assert(id < LLONG_MAX); + return PyLong_FromLongLong(id); +} + +static PyObject * +get_interpid_obj(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IDInitref(interp) != 0) { + return NULL; + }; + int64_t id = PyInterpreterState_GetID(interp); + if (id < 0) { + return NULL; + } + return interpid_to_pylong(id); +} + +static PyObject * +_get_current_module(void) +{ + PyObject *name = PyUnicode_FromString(MODULE_NAME); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + + +/* Cross-interpreter Buffer Views *******************************************/ + +// XXX Release when the original interpreter is destroyed. + +typedef struct { + PyObject_HEAD + Py_buffer *view; + int64_t interpid; +} XIBufferViewObject; + +static PyObject * +xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) +{ + assert(data->data != NULL); + assert(data->obj == NULL); + assert(data->interpid >= 0); + XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); + if (self == NULL) { + return NULL; + } + PyObject_Init((PyObject *)self, cls); + self->view = (Py_buffer *)data->data; + self->interpid = data->interpid; + return (PyObject *)self; +} + +static void +xibufferview_dealloc(XIBufferViewObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid); + /* If the interpreter is no longer alive then we have problems, + since other objects may be using the buffer still. */ + assert(interp != NULL); + + if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { + // XXX Emit a warning? + PyErr_Clear(); + } + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + /* "Instances of heap-allocated types hold a reference to their type." + * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol + * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse + */ + // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, + // like we do for _abc._abc_data? + Py_DECREF(tp); +} + +static int +xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) +{ + /* Only PyMemoryView_FromObject() should ever call this, + via _memoryview_from_xid() below. */ + *view = *self->view; + view->obj = (PyObject *)self; + // XXX Should we leave it alone? + view->internal = NULL; + return 0; +} + +static PyType_Slot XIBufferViewType_slots[] = { + {Py_tp_dealloc, (destructor)xibufferview_dealloc}, + {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, + // We don't bother with Py_bf_releasebuffer since we don't need it. + {0, NULL}, +}; + +static PyType_Spec XIBufferViewType_spec = { + .name = MODULE_NAME ".CrossInterpreterBufferView", + .basicsize = sizeof(XIBufferViewObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .slots = XIBufferViewType_slots, +}; + + +static PyTypeObject * _get_current_xibufferview_type(void); + +static PyObject * +_memoryview_from_xid(_PyCrossInterpreterData *data) +{ + PyTypeObject *cls = _get_current_xibufferview_type(); + if (cls == NULL) { + return NULL; + } + PyObject *obj = xibufferview_from_xid(cls, data); + if (obj == NULL) { + return NULL; + } + return PyMemoryView_FromObject(obj); +} + +static int +_memoryview_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); + if (view == NULL) { + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { + PyMem_RawFree(view); + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, + _memoryview_from_xid); + return 0; +} + +static int +register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) +{ + // XIBufferView + assert(*p_state == NULL); + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &XIBufferViewType_spec, NULL); + if (cls == NULL) { + return -1; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return -1; + } + *p_state = cls; + + // Register XID for the builtin memoryview type. + if (_PyCrossInterpreterData_RegisterClass( + &PyMemoryView_Type, _memoryview_shared) < 0) { + return -1; + } + // We don't ever bother un-registering memoryview. + + return 0; +} + + /* module state *************************************************************/ typedef struct { int _notused; + + /* heap types */ + PyTypeObject *XIBufferViewType; } module_state; static inline module_state * @@ -44,19 +296,51 @@ get_module_state(PyObject *mod) return state; } +static module_state * +_get_current_module_state(void) +{ + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME " module not imported yet"); + return NULL; + } + module_state *state = get_module_state(mod); + Py_DECREF(mod); + return state; +} + static int traverse_module_state(module_state *state, visitproc visit, void *arg) { + /* heap types */ + Py_VISIT(state->XIBufferViewType); + return 0; } static int clear_module_state(module_state *state) { + /* heap types */ + Py_CLEAR(state->XIBufferViewType); + return 0; } +static PyTypeObject * +_get_current_xibufferview_type(void) +{ + module_state *state = _get_current_module_state(); + if (state == NULL) { + return NULL; + } + return state->XIBufferViewType; +} + + /* Python code **************************************************************/ static const char * @@ -254,7 +538,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) assert(tstate != NULL); PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = PyInterpreterState_GetIDObject(interp); + PyObject *idobj = get_interpid_obj(interp); if (idobj == NULL) { // XXX Possible GILState issues? save_tstate = PyThreadState_Swap(tstate); @@ -273,7 +557,9 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) PyDoc_STRVAR(create_doc, "create() -> ID\n\ \n\ -Create a new interpreter and return a unique generated ID."); +Create a new interpreter and return a unique generated ID.\n\ +\n\ +The caller is responsible for destroying the interpreter before exiting."); static PyObject * @@ -288,7 +574,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; } @@ -345,7 +631,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) interp = PyInterpreterState_Head(); while (interp != NULL) { - id = PyInterpreterState_GetIDObject(interp); + id = get_interpid_obj(interp); if (id == NULL) { Py_DECREF(ids); return NULL; @@ -377,7 +663,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return PyInterpreterState_GetIDObject(interp); + return get_interpid_obj(interp); } PyDoc_STRVAR(get_current_doc, @@ -391,7 +677,7 @@ interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { // Currently, 0 is always the main interpreter. int64_t id = 0; - return PyInterpreterID_New(id); + return PyLong_FromLongLong(id); } PyDoc_STRVAR(get_main_doc, @@ -481,7 +767,7 @@ _interp_exec(PyObject *self, PyObject **p_excinfo) { // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id_arg); + PyInterpreterState *interp = look_up_interp(id_arg); if (interp == NULL) { return -1; } @@ -667,7 +953,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; } @@ -683,6 +969,49 @@ PyDoc_STRVAR(is_running_doc, Return whether or not the identified interpreter is running."); +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_incref", kwlist, &id)) { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + if (_PyInterpreterState_IDInitref(interp) < 0) { + return NULL; + } + _PyInterpreterState_IDIncref(interp); + + Py_RETURN_NONE; +} + + +static PyObject * +interp_decref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_incref", kwlist, &id)) { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDDecref(interp); + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, @@ -707,6 +1036,11 @@ static PyMethodDef module_functions[] = { {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + {"_incref", _PyCFunction_CAST(interp_incref), + METH_VARARGS | METH_KEYWORDS, NULL}, + {"_decref", _PyCFunction_CAST(interp_decref), + METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} /* sentinel */ }; @@ -720,8 +1054,17 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { - // PyInterpreterID - if (PyModule_AddType(mod, &PyInterpreterID_Type) < 0) { + module_state *state = get_module_state(mod); + + // exceptions + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { + goto error; + } + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { + goto error; + } + + if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { goto error; } diff --git a/Python/crossinterp.c b/Python/crossinterp.c index f74fee38648266..a31b5ef4613dbd 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -12,6 +12,53 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() +/**************/ +/* exceptions */ +/**************/ + +/* InterpreterError extends Exception */ + +static PyTypeObject _PyExc_InterpreterError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "InterpreterError", + .tp_doc = PyDoc_STR("An interpreter was not found."), + //.tp_base = (PyTypeObject *)PyExc_BaseException, +}; +PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; + +/* InterpreterNotFoundError extends InterpreterError */ + +static PyTypeObject _PyExc_InterpreterNotFoundError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "InterpreterNotFoundError", + .tp_doc = PyDoc_STR("An interpreter was not found."), + .tp_base = &_PyExc_InterpreterError, +}; +PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; + +/* lifecycle */ + +static int +init_exceptions(PyInterpreterState *interp) +{ + _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { + return -1; + } + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { + return -1; + } + return 0; +} + +static void +fini_exceptions(PyInterpreterState *interp) +{ + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError); + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError); +} + + /***************************/ /* cross-interpreter calls */ /***************************/ @@ -2099,3 +2146,18 @@ _PyXI_Fini(PyInterpreterState *interp) _xidregistry_fini(_get_global_xidregistry(interp->runtime)); } } + +PyStatus +_PyXI_InitTypes(PyInterpreterState *interp) +{ + if (init_exceptions(interp) < 0) { + return _PyStatus_ERR("failed to initialize an exception type"); + } + return _PyStatus_OK(); +} + +void +_PyXI_FiniTypes(PyInterpreterState *interp) +{ + fini_exceptions(interp); +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 45a119fcca7f2c..b5c7dc5da596de 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -734,6 +734,11 @@ pycore_init_types(PyInterpreterState *interp) return status; } + status = _PyXI_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -1742,6 +1747,7 @@ finalize_interp_types(PyInterpreterState *interp) { _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); + _PyXI_FiniTypes(interp); _PyExc_Fini(interp); _PyAsyncGen_Fini(interp); _PyContext_Fini(interp); diff --git a/Python/pystate.c b/Python/pystate.c index 1a7c0c968504d1..f0c5259967d907 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1216,7 +1216,7 @@ _PyInterpreterState_LookUpID(int64_t requested_id) HEAD_UNLOCK(runtime); } if (interp == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_InterpreterNotFoundError, "unrecognized interpreter ID %lld", requested_id); } return interp; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index aa8ce49ae86376..e3a1b5d532bda2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -290,6 +290,10 @@ Objects/exceptions.c - PyExc_UnicodeWarning - Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - +Python/crossinterp.c - _PyExc_InterpreterError - +Python/crossinterp.c - _PyExc_InterpreterNotFoundError - +Python/crossinterp.c - PyExc_InterpreterError - +Python/crossinterp.c - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons From eafc2381a0b891383098b08300ae766868a19ba6 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 12 Dec 2023 18:29:08 +0200 Subject: [PATCH 23/95] Update pre-commit to fix Sphinx Lint (#113015) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 35d9c64a8c5c15..9bd9c59a1ddc74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.2 + rev: v0.1.7 hooks: - id: ruff name: Run Ruff on Lib/test/ @@ -24,7 +24,7 @@ repos: types_or: [c, inc, python, rst] - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.8.1 + rev: v0.9.1 hooks: - id: sphinx-lint args: [--enable=default-role] From fe9991bb672dd95fb9cd38b5a03180719ac4e722 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 12 Dec 2023 18:31:04 +0200 Subject: [PATCH 24/95] gh-112999: Replace the outdated "deprecated" directives with "versionchanged" (GH-113000) --- Doc/library/hmac.rst | 17 +++++++---------- Doc/library/random.rst | 8 ++++---- Doc/using/cmdline.rst | 4 +--- Lib/hmac.py | 2 +- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index b2ca0455d3745c..43012e03c580e8 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -14,7 +14,7 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. -.. function:: new(key, msg=None, digestmod='') +.. function:: new(key, msg=None, digestmod) Return a new hmac object. *key* is a bytes or bytearray object giving the secret key. If *msg* is present, the method call ``update(msg)`` is made. @@ -27,10 +27,9 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. Parameter *msg* can be of any type supported by :mod:`hashlib`. Parameter *digestmod* can be the name of a hash algorithm. - .. deprecated-removed:: 3.4 3.8 - MD5 as implicit default digest for *digestmod* is deprecated. - The digestmod parameter is now required. Pass it as a keyword - argument to avoid awkwardness when you do not have an initial msg. + .. versionchanged:: 3.8 + The *digestmod* argument is now required. Pass it as a keyword + argument to avoid awkwardness when you do not have an initial *msg*. .. function:: digest(key, msg, digest) @@ -114,11 +113,9 @@ A hash object has the following attributes: .. versionadded:: 3.4 -.. deprecated:: 3.9 - - The undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, and - ``HMAC.outer`` are internal implementation details and will be removed in - Python 3.10. +.. versionchanged:: 3.10 + Removed the undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, + and ``HMAC.outer``. This module also provides the following helper function: diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 76ae97a8be7e63..feaf260caf3568 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -220,8 +220,8 @@ Functions for sequences generated. For example, a sequence of length 2080 is the largest that can fit within the period of the Mersenne Twister random number generator. - .. deprecated-removed:: 3.9 3.11 - The optional parameter *random*. + .. versionchanged:: 3.11 + Removed the optional parameter *random*. .. function:: sample(population, k, *, counts=None) @@ -407,9 +407,9 @@ Alternative Generator Class that implements the default pseudo-random number generator used by the :mod:`random` module. - .. deprecated-removed:: 3.9 3.11 + .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: - :class:`NoneType`, :class:`int`, :class:`float`, :class:`str`, + ``None``, :class:`int`, :class:`float`, :class:`str`, :class:`bytes`, or :class:`bytearray`. .. class:: SystemRandom([seed]) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 56235bf4c28c7c..dac4956b551dd3 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -590,9 +590,7 @@ Miscellaneous options .. versionadded:: 3.10 The ``-X warn_default_encoding`` option. - - .. deprecated-removed:: 3.9 3.10 - The ``-X oldparser`` option. + Removed the ``-X oldparser`` option. .. versionadded:: 3.11 The ``-X no_debug_ranges`` option. diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4f920db954ca..8b4eb2fe741e60 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -53,7 +53,7 @@ def __init__(self, key, msg=None, digestmod=''): raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) if not digestmod: - raise TypeError("Missing required parameter 'digestmod'.") + raise TypeError("Missing required argument 'digestmod'.") if _hashopenssl and isinstance(digestmod, (str, _functype)): try: From cde141717578f22947553db776980aa3e8801353 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 12 Dec 2023 09:54:39 -0700 Subject: [PATCH 25/95] gh-76785: Fix test_threading on Non-Subinterpreter Builds (gh-113014) gh-112982 broke test_threading on one of the s390 buildbots (Fedora Clang Installed). Apparently ImportError is raised (rather than ModuleNotFoundError) for the name part of "from" imports. This fixes that by catching ImportError in test_threading.py. --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index a5744a4037ecea..3060af44fd7e3d 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -28,7 +28,7 @@ try: from test.support import interpreters -except ModuleNotFoundError: +except ImportError: interpreters = None threading_helper.requires_working_threading(module=True) From a49b427b0265c415d9089da0be39f4b5ccd1f15f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 12 Dec 2023 10:43:30 -0700 Subject: [PATCH 26/95] gh-76785: More Fixes for test.support.interpreters (gh-113012) This brings the module (along with the associated extension modules) mostly in sync with PEP 734. There are only a few small things to wrap up. --- Lib/test/support/interpreters/queues.py | 158 +- Lib/test/test_interpreters/test_queues.py | 96 +- Modules/Setup | 1 + Modules/Setup.stdlib.in | 3 + Modules/_xxinterpchannelsmodule.c | 5 +- Modules/_xxinterpqueuesmodule.c | 1685 +++++++++++++++++++ PC/config.c | 2 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Tools/build/generate_stdlib_module_names.py | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + configure | 29 + configure.ac | 2 + 13 files changed, 1899 insertions(+), 88 deletions(-) create mode 100644 Modules/_xxinterpqueuesmodule.c diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index ed6b0d551dd890..aead0c40ca9667 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -3,13 +3,11 @@ import queue import time import weakref -import _xxinterpchannels as _channels -import _xxinterpchannels as _queues +import _xxinterpqueues as _queues # aliases: -from _xxinterpchannels import ( - ChannelError as QueueError, - ChannelNotFoundError as QueueNotFoundError, +from _xxinterpqueues import ( + QueueError, QueueNotFoundError, ) __all__ = [ @@ -19,14 +17,27 @@ ] +class QueueEmpty(_queues.QueueEmpty, queue.Empty): + """Raised from get_nowait() when the queue is empty. + + It is also raised from get() if it times out. + """ + + +class QueueFull(_queues.QueueFull, queue.Full): + """Raised from put_nowait() when the queue is full. + + It is also raised from put() if it times out. + """ + + def create(maxsize=0): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. """ - # XXX honor maxsize - qid = _queues.create() - return Queue._with_maxsize(qid, maxsize) + qid = _queues.create(maxsize) + return Queue(qid) def list_all(): @@ -35,53 +46,37 @@ def list_all(): for qid in _queues.list_all()] -class QueueEmpty(queue.Empty): - """Raised from get_nowait() when the queue is empty. - - It is also raised from get() if it times out. - """ - - -class QueueFull(queue.Full): - """Raised from put_nowait() when the queue is full. - - It is also raised from put() if it times out. - """ - _known_queues = weakref.WeakValueDictionary() class Queue: """A cross-interpreter queue.""" - @classmethod - def _with_maxsize(cls, id, maxsize): - if not isinstance(maxsize, int): - raise TypeError(f'maxsize must be an int, got {maxsize!r}') - elif maxsize < 0: - maxsize = 0 - else: - maxsize = int(maxsize) - self = cls(id) - self._maxsize = maxsize - return self - def __new__(cls, id, /): # There is only one instance for any given ID. if isinstance(id, int): - id = _channels._channel_id(id, force=False) - elif not isinstance(id, _channels.ChannelID): + id = int(id) + else: raise TypeError(f'id must be an int, got {id!r}') - key = int(id) try: - self = _known_queues[key] + self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id - self._maxsize = 0 - _known_queues[key] = self + _known_queues[id] = self + _queues.bind(id) return self + def __del__(self): + try: + _queues.release(self._id) + except QueueNotFoundError: + pass + try: + del _known_queues[self._id] + except KeyError: + pass + def __repr__(self): return f'{type(self).__name__}({self.id})' @@ -90,39 +85,58 @@ def __hash__(self): @property def id(self): - return int(self._id) + return self._id @property def maxsize(self): - return self._maxsize - - @property - def _info(self): - return _channels.get_info(self._id) + try: + return self._maxsize + except AttributeError: + self._maxsize = _queues.get_maxsize(self._id) + return self._maxsize def empty(self): - return self._info.count == 0 + return self.qsize() == 0 def full(self): - if self._maxsize <= 0: - return False - return self._info.count >= self._maxsize + return _queues.is_full(self._id) def qsize(self): - return self._info.count + return _queues.get_count(self._id) - def put(self, obj, timeout=None): - # XXX block if full - _channels.send(self._id, obj, blocking=False) + def put(self, obj, timeout=None, *, + _delay=10 / 1000, # 10 milliseconds + ): + """Add the object to the queue. + + This blocks while the queue is full. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + while True: + try: + _queues.put(self._id, obj) + except _queues.QueueFull as exc: + if timeout is not None and time.time() >= end: + exc.__class__ = QueueFull + raise # re-raise + time.sleep(_delay) + else: + break def put_nowait(self, obj): - # XXX raise QueueFull if full - return _channels.send(self._id, obj, blocking=False) + try: + return _queues.put(self._id, obj) + except _queues.QueueFull as exc: + exc.__class__ = QueueFull + raise # re-raise def get(self, timeout=None, *, - _sentinel=object(), - _delay=10 / 1000, # 10 milliseconds - ): + _delay=10 / 1000, # 10 milliseconds + ): """Return the next object from the queue. This blocks while the queue is empty. @@ -132,25 +146,27 @@ def get(self, timeout=None, *, if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout - obj = _channels.recv(self._id, _sentinel) - while obj is _sentinel: - time.sleep(_delay) - if timeout is not None and time.time() >= end: - raise QueueEmpty - obj = _channels.recv(self._id, _sentinel) + while True: + try: + return _queues.get(self._id) + except _queues.QueueEmpty as exc: + if timeout is not None and time.time() >= end: + exc.__class__ = QueueEmpty + raise # re-raise + time.sleep(_delay) return obj - def get_nowait(self, *, _sentinel=object()): + def get_nowait(self): """Return the next object from the channel. If the queue is empty then raise QueueEmpty. Otherwise this is the same as get(). """ - obj = _channels.recv(self._id, _sentinel) - if obj is _sentinel: - raise QueueEmpty - return obj + try: + return _queues.get(self._id) + except _queues.QueueEmpty as exc: + exc.__class__ = QueueEmpty + raise # re-raise -# XXX add this: -#_channels._register_queue_type(Queue) +_queues._register_queue_type(Queue) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 2af90b14d3e3c4..2a8ca99c1f6e3f 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -5,13 +5,21 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxinterpchannels') -#import_helper.import_module('_xxinterpqueues') +_queues = import_helper.import_module('_xxinterpqueues') from test.support import interpreters from test.support.interpreters import queues from .utils import _run_output, TestBase +class TestBase(TestBase): + def tearDown(self): + for qid in _queues.list_all(): + try: + _queues.destroy(qid) + except Exception: + pass + + class QueueTests(TestBase): def test_create(self): @@ -32,20 +40,47 @@ def test_create(self): self.assertEqual(queue.maxsize, 0) with self.subTest('negative maxsize'): - queue = queues.create(-1) - self.assertEqual(queue.maxsize, 0) + queue = queues.create(-10) + self.assertEqual(queue.maxsize, -10) with self.subTest('bad maxsize'): with self.assertRaises(TypeError): queues.create('1') - @unittest.expectedFailure def test_shareable(self): queue1 = queues.create() - queue2 = queues.create() - queue1.put(queue2) - queue3 = queue1.get() - self.assertIs(queue3, queue1) + + interp = interpreters.create() + interp.exec_sync(dedent(f""" + from test.support.interpreters import queues + queue1 = queues.Queue({queue1.id}) + """)); + + with self.subTest('same interpreter'): + queue2 = queues.create() + queue1.put(queue2) + queue3 = queue1.get() + self.assertIs(queue3, queue2) + + with self.subTest('from current interpreter'): + queue4 = queues.create() + queue1.put(queue4) + out = _run_output(interp, dedent(""" + queue4 = queue1.get() + print(queue4.id) + """)) + qid = int(out) + self.assertEqual(qid, queue4.id) + + with self.subTest('from subinterpreter'): + out = _run_output(interp, dedent(""" + queue5 = queues.create() + queue1.put(queue5) + print(queue5.id) + """)) + qid = int(out) + queue5 = queue1.get() + self.assertEqual(queue5.id, qid) def test_id_type(self): queue = queues.create() @@ -137,7 +172,6 @@ def test_put_get_main(self): self.assertEqual(actual, expected) - @unittest.expectedFailure def test_put_timeout(self): queue = queues.create(2) queue.put(None) @@ -147,7 +181,6 @@ def test_put_timeout(self): queue.get() queue.put(None) - @unittest.expectedFailure def test_put_nowait(self): queue = queues.create(2) queue.put_nowait(None) @@ -179,31 +212,64 @@ def test_put_get_same_interpreter(self): assert obj is not orig, 'expected: obj is not orig' """)) - @unittest.expectedFailure def test_put_get_different_interpreters(self): + interp = interpreters.create() queue1 = queues.create() queue2 = queues.create() + self.assertEqual(len(queues.list_all()), 2) + obj1 = b'spam' queue1.put(obj1) + out = _run_output( - interpreters.create(), + interp, dedent(f""" - import test.support.interpreters.queue as queues + from test.support.interpreters import queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) + assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' obj = queue1.get() + assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' assert obj == b'spam', 'expected: obj == obj1' # When going to another interpreter we get a copy. assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' obj2 = b'eggs' print(id(obj2)) + assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' queue2.put(obj2) + assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' """)) - obj2 = queue2.get() + self.assertEqual(len(queues.list_all()), 2) + self.assertEqual(queue1.qsize(), 0) + self.assertEqual(queue2.qsize(), 1) + obj2 = queue2.get() self.assertEqual(obj2, b'eggs') self.assertNotEqual(id(obj2), int(out)) + def test_put_cleared_with_subinterpreter(self): + interp = interpreters.create() + queue = queues.create() + + out = _run_output( + interp, + dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj1 = b'spam' + obj2 = b'eggs' + queue.put(obj1) + queue.put(obj2) + """)) + self.assertEqual(queue.qsize(), 2) + + obj1 = queue.get() + self.assertEqual(obj1, b'spam') + self.assertEqual(queue.qsize(), 1) + + del interp + self.assertEqual(queue.qsize(), 0) + def test_put_get_different_threads(self): queue1 = queues.create() queue2 = queues.create() diff --git a/Modules/Setup b/Modules/Setup index 1367f0ef4fa54a..8ad9a5aebbfcaa 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_xxsubinterpreters _xxsubinterpretersmodule.c #_xxinterpchannels _xxinterpchannelsmodule.c +#_xxinterpqueues _xxinterpqueuesmodule.c #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c #_testbuffer _testbuffer.c #_testinternalcapi _testinternalcapi.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 54650ea9c1d4ac..8a65a9cffb1b9d 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -41,8 +41,11 @@ @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c @MODULE__STRUCT_TRUE@_struct _struct.c + +# build supports subinterpreters @MODULE__XXSUBINTERPRETERS_TRUE@_xxsubinterpreters _xxsubinterpretersmodule.c @MODULE__XXINTERPCHANNELS_TRUE@_xxinterpchannels _xxinterpchannelsmodule.c +@MODULE__XXINTERPQUEUES_TRUE@_xxinterpqueues _xxinterpqueuesmodule.c @MODULE__ZONEINFO_TRUE@_zoneinfo _zoneinfo.c # needs libm diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 97729ec269cb62..4e9b8a82a3f630 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2629,10 +2629,11 @@ _get_current_channelend_type(int end) cls = state->recv_channel_type; } if (cls == NULL) { - PyObject *highlevel = PyImport_ImportModule("interpreters"); + // Force the module to be loaded, to register the type. + PyObject *highlevel = PyImport_ImportModule("interpreters.channel"); if (highlevel == NULL) { PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters"); + highlevel = PyImport_ImportModule("test.support.interpreters.channel"); if (highlevel == NULL) { return NULL; } diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c new file mode 100644 index 00000000000000..2cc3a2ac5dc85f --- /dev/null +++ b/Modules/_xxinterpqueuesmodule.c @@ -0,0 +1,1685 @@ +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_crossinterp.h" // struct _xid + + +#define MODULE_NAME "_xxinterpqueues" + + +#define GLOBAL_MALLOC(TYPE) \ + PyMem_RawMalloc(sizeof(TYPE)) +#define GLOBAL_FREE(VAR) \ + PyMem_RawFree(VAR) + + +#define XID_IGNORE_EXC 1 +#define XID_FREE 2 + +static int +_release_xid_data(_PyCrossInterpreterData *data, int flags) +{ + int ignoreexc = flags & XID_IGNORE_EXC; + PyObject *exc; + if (ignoreexc) { + exc = PyErr_GetRaisedException(); + } + int res; + if (flags & XID_FREE) { + res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + } + else { + res = _PyCrossInterpreterData_Release(data); + } + if (res < 0) { + /* The owning interpreter is already destroyed. */ + if (ignoreexc) { + // XXX Emit a warning? + PyErr_Clear(); + } + } + if (flags & XID_FREE) { + /* Either way, we free the data. */ + } + if (ignoreexc) { + PyErr_SetRaisedException(exc); + } + return res; +} + + +static PyInterpreterState * +_get_current_interp(void) +{ + // PyInterpreterState_Get() aborts if lookup fails, so don't need + // to check the result for NULL. + return PyInterpreterState_Get(); +} + +static PyObject * +_get_current_module(void) +{ + PyObject *name = PyUnicode_FromString(MODULE_NAME); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + + +struct idarg_int64_converter_data { + // input: + const char *label; + // output: + int64_t id; +}; + +static int +idarg_int64_converter(PyObject *arg, void *ptr) +{ + int64_t id; + struct idarg_int64_converter_data *data = ptr; + + const char *label = data->label; + if (label == NULL) { + label = "ID"; + } + + if (PyIndex_Check(arg)) { + int overflow = 0; + id = PyLong_AsLongLongAndOverflow(arg, &overflow); + if (id == -1 && PyErr_Occurred()) { + return 0; + } + else if (id == -1 && overflow == 1) { + PyErr_Format(PyExc_OverflowError, + "max %s is %lld, got %R", label, INT64_MAX, arg); + return 0; + } + else if (id < 0) { + PyErr_Format(PyExc_ValueError, + "%s must be a non-negative int, got %R", label, arg); + return 0; + } + } + else { + PyErr_Format(PyExc_TypeError, + "%s must be an int, got %.100s", + label, Py_TYPE(arg)->tp_name); + return 0; + } + data->id = id; + return 1; +} + + +/* module state *************************************************************/ + +typedef struct { + /* external types (added at runtime by interpreters module) */ + PyTypeObject *queue_type; + + /* QueueError (and its subclasses) */ + PyObject *QueueError; + PyObject *QueueNotFoundError; + PyObject *QueueEmpty; + PyObject *QueueFull; +} module_state; + +static inline module_state * +get_module_state(PyObject *mod) +{ + assert(mod != NULL); + module_state *state = PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static int +traverse_module_state(module_state *state, visitproc visit, void *arg) +{ + /* external types */ + Py_VISIT(state->queue_type); + + /* QueueError */ + Py_VISIT(state->QueueError); + Py_VISIT(state->QueueNotFoundError); + Py_VISIT(state->QueueEmpty); + Py_VISIT(state->QueueFull); + + return 0; +} + +static int +clear_module_state(module_state *state) +{ + /* external types */ + Py_CLEAR(state->queue_type); + + /* QueueError */ + Py_CLEAR(state->QueueError); + Py_CLEAR(state->QueueNotFoundError); + Py_CLEAR(state->QueueEmpty); + Py_CLEAR(state->QueueFull); + + return 0; +} + + +/* error codes **************************************************************/ + +#define ERR_EXCEPTION_RAISED (-1) +// multi-queue errors +#define ERR_QUEUES_ALLOC (-11) +#define ERR_QUEUE_ALLOC (-12) +#define ERR_NO_NEXT_QUEUE_ID (-13) +#define ERR_QUEUE_NOT_FOUND (-14) +// single-queue errors +#define ERR_QUEUE_EMPTY (-21) +#define ERR_QUEUE_FULL (-22) + +static int +resolve_module_errcode(module_state *state, int errcode, int64_t qid, + PyObject **p_exctype, PyObject **p_msgobj) +{ + PyObject *exctype = NULL; + PyObject *msg = NULL; + switch (errcode) { + case ERR_NO_NEXT_QUEUE_ID: + exctype = state->QueueError; + msg = PyUnicode_FromString("ran out of queue IDs"); + break; + case ERR_QUEUE_NOT_FOUND: + exctype = state->QueueNotFoundError; + msg = PyUnicode_FromFormat("queue %" PRId64 " not found", qid); + break; + case ERR_QUEUE_EMPTY: + exctype = state->QueueEmpty; + msg = PyUnicode_FromFormat("queue %" PRId64 " is empty", qid); + break; + case ERR_QUEUE_FULL: + exctype = state->QueueFull; + msg = PyUnicode_FromFormat("queue %" PRId64 " is full", qid); + break; + default: + PyErr_Format(PyExc_ValueError, + "unsupported error code %d", errcode); + return -1; + } + + if (msg == NULL) { + assert(PyErr_Occurred()); + return -1; + } + *p_exctype = exctype; + *p_msgobj = msg; + return 0; +} + + +/* QueueError ***************************************************************/ + +static int +add_exctype(PyObject *mod, PyObject **p_state_field, + const char *qualname, const char *doc, PyObject *base) +{ + const char *dot = strrchr(qualname, '.'); + assert(dot != NULL); + const char *name = dot+1; + assert(*p_state_field == NULL); + assert(!PyObject_HasAttrStringWithError(mod, name)); + PyObject *exctype = PyErr_NewExceptionWithDoc(qualname, doc, base, NULL); + if (exctype == NULL) { + return -1; + } + if (PyModule_AddType(mod, (PyTypeObject *)exctype) < 0) { + Py_DECREF(exctype); + return -1; + } + *p_state_field = exctype; + return 0; +} + +static int +add_QueueError(PyObject *mod) +{ + module_state *state = get_module_state(mod); + +#define PREFIX "test.support.interpreters." +#define ADD_EXCTYPE(NAME, BASE, DOC) \ + if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ + return -1; \ + } + ADD_EXCTYPE(QueueError, PyExc_RuntimeError, + "Indicates that a queue-related error happened.") + ADD_EXCTYPE(QueueNotFoundError, state->QueueError, NULL) + ADD_EXCTYPE(QueueEmpty, state->QueueError, NULL) + ADD_EXCTYPE(QueueFull, state->QueueError, NULL) +#undef ADD_EXCTYPE +#undef PREFIX + + return 0; +} + +static int +handle_queue_error(int err, PyObject *mod, int64_t qid) +{ + if (err == 0) { + assert(!PyErr_Occurred()); + return 0; + } + assert(err < 0); + assert((err == -1) == (PyErr_Occurred() != NULL)); + + module_state *state; + switch (err) { + case ERR_QUEUE_ALLOC: // fall through + case ERR_QUEUES_ALLOC: + PyErr_NoMemory(); + break; + default: + state = get_module_state(mod); + assert(state->QueueError != NULL); + PyObject *exctype = NULL; + PyObject *msg = NULL; + if (resolve_module_errcode(state, err, qid, &exctype, &msg) < 0) { + return -1; + } + PyObject *exc = PyObject_CallOneArg(exctype, msg); + Py_DECREF(msg); + if (exc == NULL) { + return -1; + } + PyErr_SetObject(exctype, exc); + Py_DECREF(exc); + } + return 1; +} + + +/* the basic queue **********************************************************/ + +struct _queueitem; + +typedef struct _queueitem { + _PyCrossInterpreterData *data; + struct _queueitem *next; +} _queueitem; + +static void +_queueitem_init(_queueitem *item, _PyCrossInterpreterData *data) +{ + *item = (_queueitem){ + .data = data, + }; +} + +static void +_queueitem_clear(_queueitem *item) +{ + item->next = NULL; + + if (item->data != NULL) { + // It was allocated in queue_put(). + (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + item->data = NULL; + } +} + +static _queueitem * +_queueitem_new(_PyCrossInterpreterData *data) +{ + _queueitem *item = GLOBAL_MALLOC(_queueitem); + if (item == NULL) { + PyErr_NoMemory(); + return NULL; + } + _queueitem_init(item, data); + return item; +} + +static void +_queueitem_free(_queueitem *item) +{ + _queueitem_clear(item); + GLOBAL_FREE(item); +} + +static void +_queueitem_free_all(_queueitem *item) +{ + while (item != NULL) { + _queueitem *last = item; + item = item->next; + _queueitem_free(last); + } +} + +static void +_queueitem_popped(_queueitem *item, _PyCrossInterpreterData **p_data) +{ + *p_data = item->data; + // We clear them here, so they won't be released in _queueitem_clear(). + item->data = NULL; + _queueitem_free(item); +} + + +/* the queue */ +typedef struct _queue { + Py_ssize_t num_waiters; // protected by global lock + PyThread_type_lock mutex; + int alive; + struct _queueitems { + Py_ssize_t maxsize; + Py_ssize_t count; + _queueitem *first; + _queueitem *last; + } items; +} _queue; + +static int +_queue_init(_queue *queue, Py_ssize_t maxsize) +{ + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_QUEUE_ALLOC; + } + *queue = (_queue){ + .mutex = mutex, + .alive = 1, + .items = { + .maxsize = maxsize, + }, + }; + return 0; +} + +static void +_queue_clear(_queue *queue) +{ + assert(!queue->alive); + assert(queue->num_waiters == 0); + _queueitem_free_all(queue->items.first); + assert(queue->mutex != NULL); + PyThread_free_lock(queue->mutex); + *queue = (_queue){0}; +} + +static void +_queue_kill_and_wait(_queue *queue) +{ + // Mark it as dead. + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + assert(queue->alive); + queue->alive = 0; + PyThread_release_lock(queue->mutex); + + // Wait for all waiters to fail. + while (queue->num_waiters > 0) { + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + PyThread_release_lock(queue->mutex); + }; +} + +static void +_queue_mark_waiter(_queue *queue, PyThread_type_lock parent_mutex) +{ + if (parent_mutex != NULL) { + PyThread_acquire_lock(parent_mutex, WAIT_LOCK); + queue->num_waiters += 1; + PyThread_release_lock(parent_mutex); + } + else { + // The caller must be holding the parent lock already. + queue->num_waiters += 1; + } +} + +static void +_queue_unmark_waiter(_queue *queue, PyThread_type_lock parent_mutex) +{ + if (parent_mutex != NULL) { + PyThread_acquire_lock(parent_mutex, WAIT_LOCK); + queue->num_waiters -= 1; + PyThread_release_lock(parent_mutex); + } + else { + // The caller must be holding the parent lock already. + queue->num_waiters -= 1; + } +} + +static int +_queue_lock(_queue *queue) +{ + // The queue must be marked as a waiter already. + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + if (!queue->alive) { + PyThread_release_lock(queue->mutex); + return ERR_QUEUE_NOT_FOUND; + } + return 0; +} + +static void +_queue_unlock(_queue *queue) +{ + PyThread_release_lock(queue->mutex); +} + +static int +_queue_add(_queue *queue, _PyCrossInterpreterData *data) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + Py_ssize_t maxsize = queue->items.maxsize; + if (maxsize <= 0) { + maxsize = PY_SSIZE_T_MAX; + } + if (queue->items.count >= maxsize) { + _queue_unlock(queue); + return ERR_QUEUE_FULL; + } + + _queueitem *item = _queueitem_new(data); + if (item == NULL) { + _queue_unlock(queue); + return -1; + } + + queue->items.count += 1; + if (queue->items.first == NULL) { + queue->items.first = item; + } + else { + queue->items.last->next = item; + } + queue->items.last = item; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_next(_queue *queue, _PyCrossInterpreterData **p_data) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + assert(queue->items.count >= 0); + _queueitem *item = queue->items.first; + if (item == NULL) { + _queue_unlock(queue); + return ERR_QUEUE_EMPTY; + } + queue->items.first = item->next; + if (queue->items.last == item) { + queue->items.last = NULL; + } + queue->items.count -= 1; + + _queueitem_popped(item, p_data); + + _queue_unlock(queue); + return 0; +} + +static int +_queue_get_maxsize(_queue *queue, Py_ssize_t *p_maxsize) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + *p_maxsize = queue->items.maxsize; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_is_full(_queue *queue, int *p_is_full) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + assert(queue->items.count <= queue->items.maxsize); + *p_is_full = queue->items.count == queue->items.maxsize; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_get_count(_queue *queue, Py_ssize_t *p_count) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + *p_count = queue->items.count; + + _queue_unlock(queue); + return 0; +} + +static void +_queue_clear_interpreter(_queue *queue, int64_t interpid) +{ + int err = _queue_lock(queue); + if (err == ERR_QUEUE_NOT_FOUND) { + // The queue is already destroyed, so there's nothing to clear. + assert(!PyErr_Occurred()); + return; + } + assert(err == 0); // There should be no other errors. + + _queueitem *prev = NULL; + _queueitem *next = queue->items.first; + while (next != NULL) { + _queueitem *item = next; + next = item->next; + if (item->data->interpid == interpid) { + if (prev == NULL) { + queue->items.first = item->next; + } + else { + prev->next = item->next; + } + _queueitem_free(item); + queue->items.count -= 1; + } + else { + prev = item; + } + } + + _queue_unlock(queue); +} + + +/* external queue references ************************************************/ + +struct _queueref; + +typedef struct _queueref { + struct _queueref *next; + int64_t qid; + Py_ssize_t refcount; + _queue *queue; +} _queueref; + +static _queueref * +_queuerefs_find(_queueref *first, int64_t qid, _queueref **pprev) +{ + _queueref *prev = NULL; + _queueref *ref = first; + while (ref != NULL) { + if (ref->qid == qid) { + break; + } + prev = ref; + ref = ref->next; + } + if (pprev != NULL) { + *pprev = prev; + } + return ref; +} + + +/* a collection of queues ***************************************************/ + +typedef struct _queues { + PyThread_type_lock mutex; + _queueref *head; + int64_t count; + int64_t next_id; +} _queues; + +static void +_queues_init(_queues *queues, PyThread_type_lock mutex) +{ + queues->mutex = mutex; + queues->head = NULL; + queues->count = 0; + queues->next_id = 1; +} + +static void +_queues_fini(_queues *queues) +{ + assert(queues->count == 0); + assert(queues->head == NULL); + if (queues->mutex != NULL) { + PyThread_free_lock(queues->mutex); + queues->mutex = NULL; + } +} + +static int64_t +_queues_next_id(_queues *queues) // needs lock +{ + int64_t qid = queues->next_id; + if (qid < 0) { + /* overflow */ + return ERR_NO_NEXT_QUEUE_ID; + } + queues->next_id += 1; + return qid; +} + +static int +_queues_lookup(_queues *queues, int64_t qid, _queue **res) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = _queuerefs_find(queues->head, qid, NULL); + if (ref == NULL) { + PyThread_release_lock(queues->mutex); + return ERR_QUEUE_NOT_FOUND; + } + assert(ref->queue != NULL); + _queue *queue = ref->queue; + _queue_mark_waiter(queue, NULL); + // The caller must unmark it. + + PyThread_release_lock(queues->mutex); + + *res = queue; + return 0; +} + +static int64_t +_queues_add(_queues *queues, _queue *queue) +{ + int64_t qid = -1; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + // Create a new ref. + int64_t _qid = _queues_next_id(queues); + if (_qid < 0) { + goto done; + } + _queueref *ref = GLOBAL_MALLOC(_queueref); + if (ref == NULL) { + qid = ERR_QUEUE_ALLOC; + goto done; + } + *ref = (_queueref){ + .qid = _qid, + .queue = queue, + }; + + // Add it to the list. + // We assume that the queue is a new one (not already in the list). + ref->next = queues->head; + queues->head = ref; + queues->count += 1; + + qid = _qid; +done: + PyThread_release_lock(queues->mutex); + return qid; +} + +static void +_queues_remove_ref(_queues *queues, _queueref *ref, _queueref *prev, + _queue **p_queue) +{ + assert(ref->queue != NULL); + + if (ref == queues->head) { + queues->head = ref->next; + } + else { + prev->next = ref->next; + } + ref->next = NULL; + queues->count -= 1; + + *p_queue = ref->queue; + ref->queue = NULL; + GLOBAL_FREE(ref); +} + +static int +_queues_remove(_queues *queues, int64_t qid, _queue **p_queue) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *prev = NULL; + _queueref *ref = _queuerefs_find(queues->head, qid, &prev); + if (ref == NULL) { + PyThread_release_lock(queues->mutex); + return ERR_QUEUE_NOT_FOUND; + } + + _queues_remove_ref(queues, ref, prev, p_queue); + PyThread_release_lock(queues->mutex); + + return 0; +} + +static int +_queues_incref(_queues *queues, int64_t qid) +{ + // XXX Track interpreter IDs? + int res = -1; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = _queuerefs_find(queues->head, qid, NULL); + if (ref == NULL) { + assert(!PyErr_Occurred()); + res = ERR_QUEUE_NOT_FOUND; + goto done; + } + ref->refcount += 1; + + res = 0; +done: + PyThread_release_lock(queues->mutex); + return res; +} + +static void _queue_free(_queue *); + +static void +_queues_decref(_queues *queues, int64_t qid) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *prev = NULL; + _queueref *ref = _queuerefs_find(queues->head, qid, &prev); + if (ref == NULL) { + assert(!PyErr_Occurred()); + // Already destroyed. + // XXX Warn? + goto finally; + } + assert(ref->refcount > 0); + ref->refcount -= 1; + + // Destroy if no longer used. + assert(ref->queue != NULL); + if (ref->refcount == 0) { + _queue *queue = NULL; + _queues_remove_ref(queues, ref, prev, &queue); + PyThread_release_lock(queues->mutex); + + _queue_kill_and_wait(queue); + _queue_free(queue); + return; + } + +finally: + PyThread_release_lock(queues->mutex); +} + +static int64_t * +_queues_list_all(_queues *queues, int64_t *count) +{ + int64_t *qids = NULL; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(queues->count)); + if (ids == NULL) { + goto done; + } + _queueref *ref = queues->head; + for (int64_t i=0; ref != NULL; ref = ref->next, i++) { + ids[i] = ref->qid; + } + *count = queues->count; + + qids = ids; +done: + PyThread_release_lock(queues->mutex); + return qids; +} + +static void +_queues_clear_interpreter(_queues *queues, int64_t interpid) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = queues->head; + for (; ref != NULL; ref = ref->next) { + assert(ref->queue != NULL); + _queue_clear_interpreter(ref->queue, interpid); + } + + PyThread_release_lock(queues->mutex); +} + + +/* "high"-level queue-related functions *************************************/ + +static void +_queue_free(_queue *queue) +{ + _queue_clear(queue); + GLOBAL_FREE(queue); +} + +// Create a new queue. +static int64_t +queue_create(_queues *queues, Py_ssize_t maxsize) +{ + _queue *queue = GLOBAL_MALLOC(_queue); + if (queue == NULL) { + return ERR_QUEUE_ALLOC; + } + int err = _queue_init(queue, maxsize); + if (err < 0) { + GLOBAL_FREE(queue); + return (int64_t)err; + } + int64_t qid = _queues_add(queues, queue); + if (qid < 0) { + _queue_clear(queue); + GLOBAL_FREE(queue); + } + return qid; +} + +// Completely destroy the queue. +static int +queue_destroy(_queues *queues, int64_t qid) +{ + _queue *queue = NULL; + int err = _queues_remove(queues, qid, &queue); + if (err < 0) { + return err; + } + _queue_kill_and_wait(queue); + _queue_free(queue); + return 0; +} + +// Push an object onto the queue. +static int +queue_put(_queues *queues, int64_t qid, PyObject *obj) +{ + // Look up the queue. + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + assert(queue != NULL); + + // Convert the object to cross-interpreter data. + _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); + if (data == NULL) { + _queue_unmark_waiter(queue, queues->mutex); + return -1; + } + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + _queue_unmark_waiter(queue, queues->mutex); + GLOBAL_FREE(data); + return -1; + } + + // Add the data to the queue. + int res = _queue_add(queue, data); + _queue_unmark_waiter(queue, queues->mutex); + if (res != 0) { + // We may chain an exception here: + (void)_release_xid_data(data, 0); + GLOBAL_FREE(data); + return res; + } + + return 0; +} + +// Pop the next object off the queue. Fail if empty. +// XXX Support a "wait" mutex? +static int +queue_get(_queues *queues, int64_t qid, PyObject **res) +{ + int err; + *res = NULL; + + // Look up the queue. + _queue *queue = NULL; + err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + // Past this point we are responsible for releasing the mutex. + assert(queue != NULL); + + // Pop off the next item from the queue. + _PyCrossInterpreterData *data = NULL; + err = _queue_next(queue, &data); + _queue_unmark_waiter(queue, queues->mutex); + if (err != 0) { + return err; + } + else if (data == NULL) { + assert(!PyErr_Occurred()); + return 0; + } + + // Convert the data back to an object. + PyObject *obj = _PyCrossInterpreterData_NewObject(data); + if (obj == NULL) { + assert(PyErr_Occurred()); + // It was allocated in queue_put(), so we free it. + (void)_release_xid_data(data, XID_IGNORE_EXC | XID_FREE); + return -1; + } + // It was allocated in queue_put(), so we free it. + int release_res = _release_xid_data(data, XID_FREE); + if (release_res < 0) { + // The source interpreter has been destroyed already. + assert(PyErr_Occurred()); + Py_DECREF(obj); + return -1; + } + + *res = obj; + return 0; +} + +static int +queue_get_maxsize(_queues *queues, int64_t qid, Py_ssize_t *p_maxsize) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_get_maxsize(queue, p_maxsize); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + +static int +queue_is_full(_queues *queues, int64_t qid, int *p_is_full) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_is_full(queue, p_is_full); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + +static int +queue_get_count(_queues *queues, int64_t qid, Py_ssize_t *p_count) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_get_count(queue, p_count); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + + +/* external Queue objects ***************************************************/ + +static int _queueobj_shared(PyThreadState *, + PyObject *, _PyCrossInterpreterData *); + +static int +set_external_queue_type(PyObject *module, PyTypeObject *queue_type) +{ + module_state *state = get_module_state(module); + + if (state->queue_type != NULL) { + PyErr_SetString(PyExc_TypeError, "already registered"); + return -1; + } + state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); + + if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) { + return -1; + } + + return 0; +} + +static PyTypeObject * +get_external_queue_type(PyObject *module) +{ + module_state *state = get_module_state(module); + + PyTypeObject *cls = state->queue_type; + if (cls == NULL) { + // Force the module to be loaded, to register the type. + PyObject *highlevel = PyImport_ImportModule("interpreters.queue"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.interpreters.queue"); + if (highlevel == NULL) { + return NULL; + } + } + Py_DECREF(highlevel); + cls = state->queue_type; + assert(cls != NULL); + } + return cls; +} + + +// XXX Use a new __xid__ protocol instead? + +struct _queueid_xid { + int64_t qid; +}; + +static _queues * _get_global_queues(void); + +static void * +_queueid_xid_new(int64_t qid) +{ + _queues *queues = _get_global_queues(); + if (_queues_incref(queues, qid) < 0) { + return NULL; + } + + struct _queueid_xid *data = PyMem_RawMalloc(sizeof(struct _queueid_xid)); + if (data == NULL) { + _queues_incref(queues, qid); + return NULL; + } + data->qid = qid; + return (void *)data; +} + +static void +_queueid_xid_free(void *data) +{ + int64_t qid = ((struct _queueid_xid *)data)->qid; + PyMem_RawFree(data); + _queues *queues = _get_global_queues(); + _queues_decref(queues, qid); +} + +static PyObject * +_queueobj_from_xid(_PyCrossInterpreterData *data) +{ + int64_t qid = *(int64_t *)data->data; + PyObject *qidobj = PyLong_FromLongLong(qid); + if (qidobj == NULL) { + return NULL; + } + + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME " module not imported yet"); + return NULL; + } + + PyTypeObject *cls = get_external_queue_type(mod); + Py_DECREF(mod); + if (cls == NULL) { + Py_DECREF(qidobj); + return NULL; + } + PyObject *obj = PyObject_CallOneArg((PyObject *)cls, (PyObject *)qidobj); + Py_DECREF(qidobj); + return obj; +} + +static int +_queueobj_shared(PyThreadState *tstate, PyObject *queueobj, + _PyCrossInterpreterData *data) +{ + PyObject *qidobj = PyObject_GetAttrString(queueobj, "_id"); + if (qidobj == NULL) { + return -1; + } + struct idarg_int64_converter_data converted = { + .label = "queue ID", + }; + int res = idarg_int64_converter(qidobj, &converted); + Py_DECREF(qidobj); + if (!res) { + assert(PyErr_Occurred()); + return -1; + } + + void *raw = _queueid_xid_new(converted.id); + if (raw == NULL) { + Py_DECREF(qidobj); + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, + _queueobj_from_xid); + Py_DECREF(qidobj); + data->free = _queueid_xid_free; + return 0; +} + + +/* module level code ********************************************************/ + +/* globals is the process-global state for the module. It holds all + the data that we need to share between interpreters, so it cannot + hold PyObject values. */ +static struct globals { + int module_count; + _queues queues; +} _globals = {0}; + +static int +_globals_init(void) +{ + // XXX This isn't thread-safe. + _globals.module_count++; + if (_globals.module_count > 1) { + // Already initialized. + return 0; + } + + assert(_globals.queues.mutex == NULL); + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_QUEUES_ALLOC; + } + _queues_init(&_globals.queues, mutex); + return 0; +} + +static void +_globals_fini(void) +{ + // XXX This isn't thread-safe. + _globals.module_count--; + if (_globals.module_count > 0) { + return; + } + + _queues_fini(&_globals.queues); +} + +static _queues * +_get_global_queues(void) +{ + return &_globals.queues; +} + + +static void +clear_interpreter(void *data) +{ + if (_globals.module_count == 0) { + return; + } + PyInterpreterState *interp = (PyInterpreterState *)data; + assert(interp == _get_current_interp()); + int64_t interpid = PyInterpreterState_GetID(interp); + _queues_clear_interpreter(&_globals.queues, interpid); +} + + +typedef struct idarg_int64_converter_data qidarg_converter_data; + +static int +qidarg_converter(PyObject *arg, void *ptr) +{ + qidarg_converter_data *data = ptr; + if (data->label == NULL) { + data->label = "queue ID"; + } + return idarg_int64_converter(arg, ptr); +} + + +static PyObject * +queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"maxsize", NULL}; + Py_ssize_t maxsize = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n:create", kwlist, + &maxsize)) { + return NULL; + } + + int64_t qid = queue_create(&_globals.queues, maxsize); + if (qid < 0) { + (void)handle_queue_error((int)qid, self, qid); + return NULL; + } + + PyObject *qidobj = PyLong_FromLongLong(qid); + if (qidobj == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + int err = queue_destroy(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + // XXX issue a warning? + PyErr_Clear(); + } + PyErr_SetRaisedException(exc); + return NULL; + } + + return qidobj; +} + +PyDoc_STRVAR(queuesmod_create_doc, +"create() -> qid\n\ +\n\ +Create a new cross-interpreter queue and return its unique generated ID.\n\ +It is a new reference as though bind() had been called on the queue."); + +static PyObject * +queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:destroy", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + int err = queue_destroy(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_destroy_doc, +"destroy(qid)\n\ +\n\ +Clear and destroy the queue. Afterward attempts to use the queue\n\ +will behave as though it never existed."); + +static PyObject * +queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t count = 0; + int64_t *qids = _queues_list_all(&_globals.queues, &count); + if (qids == NULL) { + if (count == 0) { + return PyList_New(0); + } + return NULL; + } + PyObject *ids = PyList_New((Py_ssize_t)count); + if (ids == NULL) { + goto finally; + } + int64_t *cur = qids; + for (int64_t i=0; i < count; cur++, i++) { + PyObject *qidobj = PyLong_FromLongLong(*cur); + if (qidobj == NULL) { + Py_SETREF(ids, NULL); + break; + } + PyList_SET_ITEM(ids, (Py_ssize_t)i, qidobj); + } + +finally: + PyMem_Free(qids); + return ids; +} + +PyDoc_STRVAR(queuesmod_list_all_doc, +"list_all() -> [qid]\n\ +\n\ +Return the list of IDs for all queues."); + +static PyObject * +queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", "obj", NULL}; + qidarg_converter_data qidarg; + PyObject *obj; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:put", kwlist, + qidarg_converter, &qidarg, &obj)) { + return NULL; + } + int64_t qid = qidarg.id; + + /* Queue up the object. */ + int err = queue_put(&_globals.queues, qid, obj); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_put_doc, +"put(qid, obj)\n\ +\n\ +Add the object's data to the queue."); + +static PyObject * +queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", "default", NULL}; + qidarg_converter_data qidarg; + PyObject *dflt = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O:get", kwlist, + qidarg_converter, &qidarg, &dflt)) { + return NULL; + } + int64_t qid = qidarg.id; + + PyObject *obj = NULL; + int err = queue_get(&_globals.queues, qid, &obj); + if (err == ERR_QUEUE_EMPTY && dflt != NULL) { + assert(obj == NULL); + obj = Py_NewRef(dflt); + } + else if (handle_queue_error(err, self, qid)) { + return NULL; + } + return obj; +} + +PyDoc_STRVAR(queuesmod_get_doc, +"get(qid, [default]) -> obj\n\ +\n\ +Return a new object from the data at the front of the queue.\n\ +\n\ +If there is nothing to receive then raise QueueEmpty, unless\n\ +a default value is provided. In that case return it."); + +static PyObject * +queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:bind", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + // XXX Check module state if bound already. + + int err = _queues_incref(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + // XXX Update module state. + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_bind_doc, +"bind(qid)\n\ +\n\ +Take a reference to the identified queue.\n\ +The queue is not destroyed until there are no references left."); + +static PyObject * +queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) +{ + // Note that only the current interpreter is affected. + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:release", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + // XXX Check module state if bound already. + // XXX Update module state. + + _queues_decref(&_globals.queues, qid); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_release_doc, +"release(qid)\n\ +\n\ +Release a reference to the queue.\n\ +The queue is destroyed once there are no references left."); + +static PyObject * +queuesmod_get_maxsize(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_maxsize", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + Py_ssize_t maxsize = -1; + int err = queue_get_maxsize(&_globals.queues, qid, &maxsize); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + return PyLong_FromLongLong(maxsize); +} + +PyDoc_STRVAR(queuesmod_get_maxsize_doc, +"get_maxsize(qid)\n\ +\n\ +Return the maximum number of items in the queue."); + +static PyObject * +queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:is_full", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + int is_full; + int err = queue_is_full(&_globals.queues, qid, &is_full); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + if (is_full) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(queuesmod_is_full_doc, +"is_full(qid)\n\ +\n\ +Return true if the queue has a maxsize and has reached it."); + +static PyObject * +queuesmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_count", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + Py_ssize_t count = -1; + int err = queue_get_count(&_globals.queues, qid, &count); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + assert(count >= 0); + return PyLong_FromSsize_t(count); +} + +PyDoc_STRVAR(queuesmod_get_count_doc, +"get_count(qid)\n\ +\n\ +Return the number of items in the queue."); + +static PyObject * +queuesmod__register_queue_type(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"queuetype", NULL}; + PyObject *queuetype; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_register_queue_type", kwlist, + &queuetype)) { + return NULL; + } + if (!PyType_Check(queuetype)) { + PyErr_SetString(PyExc_TypeError, "expected a type for 'queuetype'"); + return NULL; + } + PyTypeObject *cls_queue = (PyTypeObject *)queuetype; + + if (set_external_queue_type(self, cls_queue) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyMethodDef module_functions[] = { + {"create", _PyCFunction_CAST(queuesmod_create), + METH_VARARGS | METH_KEYWORDS, queuesmod_create_doc}, + {"destroy", _PyCFunction_CAST(queuesmod_destroy), + METH_VARARGS | METH_KEYWORDS, queuesmod_destroy_doc}, + {"list_all", queuesmod_list_all, + METH_NOARGS, queuesmod_list_all_doc}, + {"put", _PyCFunction_CAST(queuesmod_put), + METH_VARARGS | METH_KEYWORDS, queuesmod_put_doc}, + {"get", _PyCFunction_CAST(queuesmod_get), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_doc}, + {"bind", _PyCFunction_CAST(queuesmod_bind), + METH_VARARGS | METH_KEYWORDS, queuesmod_bind_doc}, + {"release", _PyCFunction_CAST(queuesmod_release), + METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, + {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, + {"is_full", _PyCFunction_CAST(queuesmod_is_full), + METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, + {"get_count", _PyCFunction_CAST(queuesmod_get_count), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_count_doc}, + {"_register_queue_type", _PyCFunction_CAST(queuesmod__register_queue_type), + METH_VARARGS | METH_KEYWORDS, NULL}, + + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static int +module_exec(PyObject *mod) +{ + if (_globals_init() != 0) { + return -1; + } + + /* Add exception types */ + if (add_QueueError(mod) < 0) { + goto error; + } + + /* Make sure queues drop objects owned by this interpreter. */ + PyInterpreterState *interp = _get_current_interp(); + PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); + + return 0; + +error: + _globals_fini(); + return -1; +} + +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + module_state *state = get_module_state(mod); + traverse_module_state(state, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + module_state *state = get_module_state(mod); + + if (state->queue_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); + } + + // Now we clear the module state. + clear_module_state(state); + return 0; +} + +static void +module_free(void *mod) +{ + module_state *state = get_module_state(mod); + + // Now we clear the module state. + clear_module_state(state); + + _globals_fini(); +} + +static struct PyModuleDef moduledef = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = MODULE_NAME, + .m_doc = module_doc, + .m_size = sizeof(module_state), + .m_methods = module_functions, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, +}; + +PyMODINIT_FUNC +PyInit__xxinterpqueues(void) +{ + return PyModuleDef_Init(&moduledef); +} diff --git a/PC/config.c b/PC/config.c index da2bde640961e0..f754ce6d3b057b 100644 --- a/PC/config.c +++ b/PC/config.c @@ -37,6 +37,7 @@ extern PyObject* PyInit__weakref(void); extern PyObject* PyInit_xxsubtype(void); extern PyObject* PyInit__xxsubinterpreters(void); extern PyObject* PyInit__xxinterpchannels(void); +extern PyObject* PyInit__xxinterpqueues(void); extern PyObject* PyInit__random(void); extern PyObject* PyInit_itertools(void); extern PyObject* PyInit__collections(void); @@ -142,6 +143,7 @@ struct _inittab _PyImport_Inittab[] = { {"xxsubtype", PyInit_xxsubtype}, {"_xxsubinterpreters", PyInit__xxsubinterpreters}, {"_xxinterpchannels", PyInit__xxinterpchannels}, + {"_xxinterpqueues", PyInit__xxinterpqueues}, #ifdef _Py_HAVE_ZLIB {"zlib", PyInit_zlib}, #endif diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 278f1f5622543c..778fc834c0db9c 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -458,6 +458,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index c9b34c64fbf75f..a96ca24cf08b66 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1505,6 +1505,9 @@ Modules + + Modules + Parser diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 766a85d3d6f39e..5dce4e042d1eb4 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -36,6 +36,7 @@ '_testsinglephase', '_xxsubinterpreters', '_xxinterpchannels', + '_xxinterpqueues', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index ff6e1ef4f993ba..2f9e80d6ab6737 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -165,6 +165,7 @@ Python/pylifecycle.c fatal_error reentrant - # explicitly protected, internal-only Modules/_xxinterpchannelsmodule.c - _globals - +Modules/_xxinterpqueuesmodule.c - _globals - # set once during module init Modules/_decimal/_decimal.c - minalloc_is_set - diff --git a/configure b/configure index cad3bce0c7de87..668a0efd77db0e 100755 --- a/configure +++ b/configure @@ -769,6 +769,8 @@ MODULE__MULTIPROCESSING_FALSE MODULE__MULTIPROCESSING_TRUE MODULE__ZONEINFO_FALSE MODULE__ZONEINFO_TRUE +MODULE__XXINTERPQUEUES_FALSE +MODULE__XXINTERPQUEUES_TRUE MODULE__XXINTERPCHANNELS_FALSE MODULE__XXINTERPCHANNELS_TRUE MODULE__XXSUBINTERPRETERS_FALSE @@ -28025,6 +28027,7 @@ case $ac_sys_system in #( py_cv_module__tkinter=n/a py_cv_module__xxsubinterpreters=n/a py_cv_module__xxinterpchannels=n/a + py_cv_module__xxinterpqueues=n/a py_cv_module_grp=n/a py_cv_module_pwd=n/a py_cv_module_resource=n/a @@ -28524,6 +28527,28 @@ then : +fi + + + if test "$py_cv_module__xxinterpqueues" != "n/a" +then : + py_cv_module__xxinterpqueues=yes +fi + if test "$py_cv_module__xxinterpqueues" = yes; then + MODULE__XXINTERPQUEUES_TRUE= + MODULE__XXINTERPQUEUES_FALSE='#' +else + MODULE__XXINTERPQUEUES_TRUE='#' + MODULE__XXINTERPQUEUES_FALSE= +fi + + as_fn_append MODULE_BLOCK "MODULE__XXINTERPQUEUES_STATE=$py_cv_module__xxinterpqueues$as_nl" + if test "x$py_cv_module__xxinterpqueues" = xyes +then : + + + + fi @@ -30760,6 +30785,10 @@ if test -z "${MODULE__XXINTERPCHANNELS_TRUE}" && test -z "${MODULE__XXINTERPCHAN as_fn_error $? "conditional \"MODULE__XXINTERPCHANNELS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__XXINTERPQUEUES_TRUE}" && test -z "${MODULE__XXINTERPQUEUES_FALSE}"; then + as_fn_error $? "conditional \"MODULE__XXINTERPQUEUES\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__ZONEINFO_TRUE}" && test -z "${MODULE__ZONEINFO_FALSE}"; then as_fn_error $? "conditional \"MODULE__ZONEINFO\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 7dda0b3fff95be..020553abd71b4f 100644 --- a/configure.ac +++ b/configure.ac @@ -7120,6 +7120,7 @@ AS_CASE([$ac_sys_system], [_tkinter], [_xxsubinterpreters], [_xxinterpchannels], + [_xxinterpqueues], [grp], [pwd], [resource], @@ -7236,6 +7237,7 @@ PY_STDLIB_MOD_SIMPLE([_struct]) PY_STDLIB_MOD_SIMPLE([_typing]) PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters]) PY_STDLIB_MOD_SIMPLE([_xxinterpchannels]) +PY_STDLIB_MOD_SIMPLE([_xxinterpqueues]) PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules From 9898e6104171dcdd88b32776e69ca2cddf515e63 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 12 Dec 2023 11:06:06 -0700 Subject: [PATCH 27/95] gh-76785: Add Interpreter.prepare_main() (gh-113021) This is one of the last pieces to get test.support.interpreters in sync with PEP 734. --- Lib/test/support/interpreters/__init__.py | 16 +++++-- Lib/test/test__xxinterpchannels.py | 4 +- Lib/test/test__xxsubinterpreters.py | 24 ++++++---- Lib/test/test_interpreters/test_api.py | 57 +++++++++++++++++++++++ Lib/test/test_interpreters/utils.py | 6 ++- Modules/_xxsubinterpretersmodule.c | 56 ++++++++++++++++++++++ 6 files changed, 146 insertions(+), 17 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 2d6376deb5907e..9cd1c3de0274d2 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -130,7 +130,15 @@ def close(self): """ return _interpreters.destroy(self._id) - def exec_sync(self, code, /, channels=None): + def prepare_main(self, ns=None, /, **kwargs): + """Bind the given values into the interpreter's __main__. + + The values must be shareable. + """ + ns = dict(ns, **kwargs) if ns is not None else kwargs + _interpreters.set___main___attrs(self._id, ns) + + def exec_sync(self, code, /): """Run the given source code in the interpreter. This is essentially the same as calling the builtin "exec" @@ -148,13 +156,13 @@ def exec_sync(self, code, /, channels=None): that time, the previous interpreter is allowed to run in other threads. """ - excinfo = _interpreters.exec(self._id, code, channels) + excinfo = _interpreters.exec(self._id, code) if excinfo is not None: raise ExecFailure(excinfo) - def run(self, code, /, channels=None): + def run(self, code, /): def task(): - self.exec_sync(code, channels=channels) + self.exec_sync(code) t = threading.Thread(target=task) t.start() return t diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 13c8a10296e502..cc2ed7849b0c0f 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -586,12 +586,12 @@ def test_run_string_arg_unresolved(self): cid = channels.create() interp = interpreters.create() + interpreters.set___main___attrs(interp, dict(cid=cid.send)) out = _run_output(interp, dedent(""" import _xxinterpchannels as _channels print(cid.end) _channels.send(cid, b'spam', blocking=False) - """), - dict(cid=cid.send)) + """)) obj = channels.recv(cid) self.assertEqual(obj, b'spam') diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 260ab64b07cb2d..a76e4d0ade5b8a 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -33,10 +33,10 @@ def _captured_script(script): return wrapped, open(r, encoding="utf-8") -def _run_output(interp, request, shared=None): +def _run_output(interp, request): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script, shared) + interpreters.run_string(interp, script) return rpipe.read() @@ -630,10 +630,10 @@ def test_shareable_types(self): ] for obj in objects: with self.subTest(obj): + interpreters.set___main___attrs(interp, dict(obj=obj)) interpreters.run_string( interp, f'assert(obj == {obj!r})', - shared=dict(obj=obj), ) def test_os_exec(self): @@ -721,7 +721,8 @@ def test_with_shared(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script, shared) + interpreters.set___main___attrs(self.id, shared) + interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -742,7 +743,8 @@ def test_shared_overwrites(self): ns2 = dict(vars()) del ns2['__builtins__'] """) - interpreters.run_string(self.id, script, shared) + interpreters.set___main___attrs(self.id, shared) + interpreters.run_string(self.id, script) r, w = os.pipe() script = dedent(f""" @@ -773,7 +775,8 @@ def test_shared_overwrites_default_vars(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script, shared) + interpreters.set___main___attrs(self.id, shared) + interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -1036,7 +1039,8 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - interpreters.run_func(self.id, script, shared=dict(w=w)) + interpreters.set___main___attrs(self.id, dict(w=w)) + interpreters.run_func(self.id, script) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1052,7 +1056,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') def f(): - interpreters.run_func(self.id, script, shared=dict(w=w)) + interpreters.set___main___attrs(self.id, dict(w=w)) + interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() @@ -1072,7 +1077,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') code = script.__code__ - interpreters.run_func(self.id, code, shared=dict(w=w)) + interpreters.set___main___attrs(self.id, dict(w=w)) + interpreters.run_func(self.id, code) with open(r, encoding="utf-8") as outfile: out = outfile.read() diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index e4ae9d005b5282..b702338c3de1ad 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -452,6 +452,63 @@ def task(): self.assertEqual(os.read(r_interp, 1), FINISHED) +class TestInterpreterPrepareMain(TestBase): + + def test_empty(self): + interp = interpreters.create() + with self.assertRaises(ValueError): + interp.prepare_main() + + def test_dict(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_tuple(self): + values = {'spam': 42, 'eggs': 'ham'} + values = tuple(values.items()) + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(**values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_dict_and_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values, foo='bar') + out = _run_output(interp, dedent(""" + print(spam, eggs, foo) + """)) + self.assertEqual(out.strip(), '42 ham bar') + + def test_not_shareable(self): + interp = interpreters.create() + # XXX TypeError? + with self.assertRaises(ValueError): + interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) + + # Make sure neither was actually bound. + with self.assertRaises(interpreters.ExecFailure): + interp.exec_sync('print(foo)') + with self.assertRaises(interpreters.ExecFailure): + interp.exec_sync('print(spam)') + + class TestInterpreterExecSync(TestBase): def test_success(self): diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 623c8737b79831..11b6f126dff0f4 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -29,10 +29,12 @@ def clean_up_interpreters(): pass # already destroyed -def _run_output(interp, request, channels=None): +def _run_output(interp, request, init=None): script, rpipe = _captured_script(request) with rpipe: - interp.exec_sync(script, channels=channels) + if init: + interp.prepare_main(init) + interp.exec_sync(script) return rpipe.read() diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 37959e953ee4f5..4bb54c93b0a61b 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -685,6 +685,60 @@ PyDoc_STRVAR(get_main_doc, \n\ Return the ID of main interpreter."); +static PyObject * +interp_set___main___attrs(PyObject *self, PyObject *args) +{ + PyObject *id, *updates; + if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME ".set___main___attrs", + &id, &updates)) + { + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = PyInterpreterID_LookUp(id); + if (interp == NULL) { + return NULL; + } + + // Check the updates. + if (updates != Py_None) { + Py_ssize_t size = PyObject_Size(updates); + if (size < 0) { + return NULL; + } + if (size == 0) { + PyErr_SetString(PyExc_ValueError, + "arg 2 must be a non-empty mapping"); + return NULL; + } + } + + _PyXI_session session = {0}; + + // Prep and switch interpreters, including apply the updates. + if (_PyXI_Enter(&session, interp, updates) < 0) { + if (!PyErr_Occurred()) { + _PyXI_ApplyCapturedException(&session); + assert(PyErr_Occurred()); + } + else { + assert(!_PyXI_HasCapturedException(&session)); + } + return NULL; + } + + // Clean up and switch back. + _PyXI_Exit(&session); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(set___main___attrs_doc, +"set___main___attrs(id, ns)\n\ +\n\ +Bind the given attributes in the interpreter's __main__ module."); + static PyUnicodeObject * convert_script_arg(PyObject *arg, const char *fname, const char *displayname, const char *expected) @@ -1033,6 +1087,8 @@ static PyMethodDef module_functions[] = { {"run_func", _PyCFunction_CAST(interp_run_func), METH_VARARGS | METH_KEYWORDS, run_func_doc}, + {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), + METH_VARARGS, set___main___attrs_doc}, {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, From 956023826a393b5704d3414dcd01f1bcbeaeda15 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 12 Dec 2023 19:02:24 +0000 Subject: [PATCH 28/95] GH-108866: Guarantee forward progress in executors. (GH-113006) --- Include/cpython/optimizer.h | 2 +- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uops.h | 2 +- .../2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst | 3 +++ Python/bytecodes.c | 11 ++++------- Python/generated_cases.c.h | 12 +++++------- Python/optimizer.c | 8 ++++---- 7 files changed, 19 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index adc2c1fc442280..d521eac79d1b97 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -32,7 +32,7 @@ typedef struct { typedef struct _PyExecutorObject { PyObject_VAR_HEAD /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ - struct _PyInterpreterFrame *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); + _Py_CODEUNIT *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ /* Data needed by the executor goes here, but is opaque to the VM */ } _PyExecutorObject; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 1f460640a1e398..2c512d97c421c9 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1568,7 +1568,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [_IS_NONE] = { true, INSTR_FMT_IX, 0 }, diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index ea8f90bf8c1d8f..153884f4bd2902 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -24,7 +24,7 @@ typedef struct { _PyUOpInstruction trace[1]; } _PyUOpExecutorObject; -_PyInterpreterFrame *_PyUOpExecute( +_Py_CODEUNIT *_PyUOpExecute( _PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer); diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst new file mode 100644 index 00000000000000..96606924d4a3ec --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst @@ -0,0 +1,3 @@ +Change the API and contract of ``_PyExecutorObject`` to return the +next_instr pointer, instead of the frame, and to always execute at least one +instruction. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e0f373536ce67c..1ae83422730f8f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2352,20 +2352,17 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->instr_ptr = next_instr; Py_INCREF(executor); if (executor->execute == _PyUOpExecute) { current_executor = (_PyUOpExecutorObject *)executor; GOTO_TIER_TWO(); } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; + next_instr = executor->execute(executor, frame, stack_pointer); + frame = tstate->current_frame; + if (next_instr == NULL) { goto resume_with_error; } - goto enter_tier_one; + stack_pointer = _PyFrame_GetStackPointer(frame); } replaced op(_POP_JUMP_IF_FALSE, (unused/1, cond -- )) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8f68bc6cb5ab40..65e6f11f68b38c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2328,20 +2328,18 @@ CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->instr_ptr = next_instr; Py_INCREF(executor); if (executor->execute == _PyUOpExecute) { current_executor = (_PyUOpExecutorObject *)executor; GOTO_TIER_TWO(); } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; + next_instr = executor->execute(executor, frame, stack_pointer); + frame = tstate->current_frame; + if (next_instr == NULL) { goto resume_with_error; } - goto enter_tier_one; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); } TARGET(EXIT_INIT_CHECK) { diff --git a/Python/optimizer.c b/Python/optimizer.c index dd24fbebbfd2a9..7c46bd69157170 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -167,6 +167,7 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI } _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; + /* Start optimizing at the destination to guarantee forward progress */ int err = opt->optimize(opt, code, dest, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { assert(executor == NULL); @@ -247,14 +248,13 @@ PyTypeObject _PyCounterExecutor_Type = { .tp_methods = executor_methods, }; -static _PyInterpreterFrame * +static _Py_CODEUNIT * counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject **stack_pointer) { ((_PyCounterExecutorObject *)self)->optimizer->count++; _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = ((_PyCounterExecutorObject *)self)->next_instr; Py_DECREF(self); - return frame; + return ((_PyCounterExecutorObject *)self)->next_instr; } static int @@ -891,7 +891,7 @@ uop_optimize( /* Dummy execute() function for UOp Executor. * The actual implementation is inlined in ceval.c, * in _PyEval_EvalFrameDefault(). */ -_PyInterpreterFrame * +_Py_CODEUNIT * _PyUOpExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) { Py_FatalError("Tier 2 is now inlined into Tier 1"); From 81a15ea74e2607728fceb822dfcc1aabff00478a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 12 Dec 2023 20:21:12 +0000 Subject: [PATCH 29/95] gh-101100: Further improve docs on function attributes (#113001) --- Doc/c-api/function.rst | 2 +- Doc/reference/datamodel.rst | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 75a05b4197c460..e7fb5090c09933 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -35,7 +35,7 @@ There are a few functions specific to Python functions. must be a dictionary with the global variables accessible to the function. The function's docstring and name are retrieved from the code object. - :func:`~function.__module__` + :attr:`~function.__module__` is retrieved from *globals*. The argument defaults, annotations and closure are set to ``NULL``. :attr:`~function.__qualname__` is set to the same value as the code object's :attr:`~codeobject.co_qualname` field. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index de79d72a05c68a..16ee3e300c2b8b 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -601,7 +601,7 @@ Most of these attributes check the type of the assigned value: or ``None`` if unavailable. * - .. attribute:: function.__defaults__ - - A :class:`tuple` containing default parameter values + - A :class:`tuple` containing default :term:`parameter` values for those parameters that have defaults, or ``None`` if no parameters have a default value. @@ -614,19 +614,22 @@ Most of these attributes check the type of the assigned value: See also: :attr:`__dict__ attributes `. * - .. attribute:: function.__annotations__ - - A :class:`dictionary ` containing annotations of parameters. + - A :class:`dictionary ` containing annotations of + :term:`parameters `. The keys of the dictionary are the parameter names, and ``'return'`` for the return annotation, if provided. See also: :ref:`annotations-howto`. * - .. attribute:: function.__kwdefaults__ - A :class:`dictionary ` containing defaults for keyword-only - parameters. + :term:`parameters `. * - .. attribute:: function.__type_params__ - A :class:`tuple` containing the :ref:`type parameters ` of a :ref:`generic function `. + .. versionadded:: 3.12 + Function objects also support getting and setting arbitrary attributes, which can be used, for example, to attach metadata to functions. Regular attribute dot-notation is used to get and set such attributes. From dfaa9e060bf6d69cb862a2ac140b8fccbebf3000 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 12 Dec 2023 16:17:08 -0500 Subject: [PATCH 30/95] gh-113010: Don't decrement deferred in pystats (#113032) This fixes a recently introduced bug where the deferred count is being unnecessarily decremented to counteract an increment elsewhere that is no longer happening. This caused the values to flip around to "very large" 64-bit numbers. --- Python/ceval_macros.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f298c602b1042b..ac44aecae046d8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -258,10 +258,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ - else { \ - /* This is about to be (incorrectly) incremented: */ \ - STAT_DEC((INSTNAME), deferred); \ - } \ } while (0) #else #define UPDATE_MISS_STATS(INSTNAME) ((void)0) From 7316dfb0ebc46aedf484c1f15f03a0a309d12a42 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 12 Dec 2023 13:43:08 -0800 Subject: [PATCH 31/95] gh-112320: Implement on-trace confidence tracking for branches (#112321) We track the confidence as a scaled int. --- Include/cpython/pystats.h | 1 + Lib/test/test_capi/test_misc.py | 31 +++++++++++++++++++ ...-11-22-13-17-54.gh-issue-112320.EddM51.rst | 4 +++ Python/optimizer.c | 20 ++++++++++-- Python/specialize.c | 1 + Tools/scripts/summarize_stats.py | 2 ++ 6 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 294bf1505f0115..ba67eefef3e37a 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -114,6 +114,7 @@ typedef struct _optimization_stats { uint64_t trace_too_short; uint64_t inner_loop; uint64_t recursive_call; + uint64_t low_confidence; UOpStats opcode[512]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index e6b532e858c8f9..776ee913a02216 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2985,6 +2985,37 @@ def testfunc(n, m): uops = {opname for opname, _, _ in ex} self.assertIn("_FOR_ITER_TIER_TWO", uops) + def test_confidence_score(self): + def testfunc(n): + bits = 0 + for i in range(n): + if i & 0x01: + bits += 1 + if i & 0x02: + bits += 1 + if i&0x04: + bits += 1 + if i&0x08: + bits += 1 + if i&0x10: + bits += 1 + if i&0x20: + bits += 1 + return bits + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + x = testfunc(20) + + self.assertEqual(x, 40) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + ops = [opname for opname, _, _ in ex] + count = ops.count("_GUARD_IS_TRUE_POP") + # Because Each 'if' halves the score, the second branch is + # too much already. + self.assertEqual(count, 1) + @unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') class TestPyThreadId(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst new file mode 100644 index 00000000000000..0da2fd33b0ea52 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst @@ -0,0 +1,4 @@ +The Tier 2 translator now tracks the confidence level for staying "on trace" +(i.e. not exiting back to the Tier 1 interpreter) for branch instructions +based on the number of bits set in the branch "counter". Trace translation +ends when the confidence drops below 1/3rd. diff --git a/Python/optimizer.c b/Python/optimizer.c index 7c46bd69157170..d44e733bc346fa 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -409,6 +409,9 @@ BRANCH_TO_GUARD[4][2] = { #define TRACE_STACK_SIZE 5 +#define CONFIDENCE_RANGE 1000 +#define CONFIDENCE_CUTOFF 333 + /* Returns 1 on success, * 0 if it failed to produce a worthwhile trace, * and -1 on an error. @@ -431,6 +434,7 @@ translate_bytecode_to_trace( _Py_CODEUNIT *instr; } trace_stack[TRACE_STACK_SIZE]; int trace_stack_depth = 0; + int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -513,7 +517,6 @@ translate_bytecode_to_trace( uint32_t oparg = instr->op.arg; uint32_t extras = 0; - if (opcode == EXTENDED_ARG) { instr++; extras += 1; @@ -543,11 +546,22 @@ translate_bytecode_to_trace( int counter = instr[1].cache; int bitcount = _Py_popcount32(counter); int jump_likely = bitcount > 8; + if (jump_likely) { + confidence = confidence * bitcount / 16; + } + else { + confidence = confidence * (16 - bitcount) / 16; + } + if (confidence < CONFIDENCE_CUTOFF) { + DPRINTF(2, "Confidence too low (%d)\n", confidence); + OPT_STAT_INC(low_confidence); + goto done; + } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, uopcode=%s\n", + DPRINTF(2, "%s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", _PyUOpName(opcode), oparg, - counter, bitcount, jump_likely, _PyUOpName(uopcode)); + counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); ADD_TO_TRACE(uopcode, max_length, 0, target); if (jump_likely) { _Py_CODEUNIT *target_instr = next_instr + oparg; diff --git a/Python/specialize.c b/Python/specialize.c index ba704cbbb464d7..7c2a4a42b1dcc3 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -233,6 +233,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); + fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 360b7c720bd1f0..80a1280c025aca 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -386,6 +386,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: trace_too_short = self._data["Optimization trace too short"] inner_loop = self._data["Optimization inner loop"] recursive_call = self._data["Optimization recursive call"] + low_confidence = self._data["Optimization low confidence"] return { "Optimization attempts": (attempts, None), @@ -396,6 +397,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: "Trace too short": (trace_too_short, attempts), "Inner loop found": (inner_loop, attempts), "Recursive call": (recursive_call, attempts), + "Low confidence": (low_confidence, attempts), "Traces executed": (executed, None), "Uops executed": (uops, executed), } From 8a4c1f3ff1e3d7ed2e00e77b94056f9bb7f9ae3b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 12 Dec 2023 17:00:54 -0700 Subject: [PATCH 32/95] gh-76785: Show the Traceback for Uncaught Subinterpreter Exceptions (gh-113034) When an exception is uncaught in Interpreter.exec_sync(), it helps to show that exception's error display if uncaught in the calling interpreter. We do so here by generating a TracebackException in the subinterpreter and passing it between interpreters using pickle. --- Include/internal/pycore_crossinterp.h | 2 + Lib/test/support/interpreters/__init__.py | 27 ++- Lib/test/test_interpreters/test_api.py | 48 +++++ Lib/test/test_interpreters/utils.py | 72 +++++++ Python/crossinterp.c | 218 ++++++++++++++++++++-- 5 files changed, 351 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index ce95979f8d343b..414e32b5155f62 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -188,6 +188,8 @@ typedef struct _excinfo { const char *module; } type; const char *msg; + const char *pickled; + Py_ssize_t pickled_len; } _PyXI_excinfo; diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 9cd1c3de0274d2..d619bea3e32f5d 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -34,17 +34,36 @@ def __getattr__(name): raise AttributeError(name) +_EXEC_FAILURE_STR = """ +{superstr} + +Uncaught in the interpreter: + +{formatted} +""".strip() + class ExecFailure(RuntimeError): def __init__(self, excinfo): msg = excinfo.formatted if not msg: - if excinfo.type and snapshot.msg: - msg = f'{snapshot.type.__name__}: {snapshot.msg}' + if excinfo.type and excinfo.msg: + msg = f'{excinfo.type.__name__}: {excinfo.msg}' else: - msg = snapshot.type.__name__ or snapshot.msg + msg = excinfo.type.__name__ or excinfo.msg super().__init__(msg) - self.snapshot = excinfo + self.excinfo = excinfo + + def __str__(self): + try: + formatted = ''.join(self.excinfo.tbexc.format()).rstrip() + except Exception: + return super().__str__() + else: + return _EXEC_FAILURE_STR.format( + superstr=super().__str__(), + formatted=formatted, + ) def create(): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index b702338c3de1ad..aefd326977095f 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -525,6 +525,54 @@ def test_failure(self): with self.assertRaises(interpreters.ExecFailure): interp.exec_sync('raise Exception') + def test_display_preserved_exception(self): + tempdir = self.temp_dir() + modfile = self.make_module('spam', tempdir, text=""" + def ham(): + raise RuntimeError('uh-oh!') + + def eggs(): + ham() + """) + scriptfile = self.make_script('script.py', tempdir, text=""" + from test.support import interpreters + + def script(): + import spam + spam.eggs() + + interp = interpreters.create() + interp.exec_sync(script) + """) + + stdout, stderr = self.assert_python_failure(scriptfile) + self.maxDiff = None + interpmod_line, = (l for l in stderr.splitlines() if ' exec_sync' in l) + # File "{interpreters.__file__}", line 179, in exec_sync + self.assertEqual(stderr, dedent(f"""\ + Traceback (most recent call last): + File "{scriptfile}", line 9, in + interp.exec_sync(script) + ~~~~~~~~~~~~~~~~^^^^^^^^ + {interpmod_line.strip()} + raise ExecFailure(excinfo) + test.support.interpreters.ExecFailure: RuntimeError: uh-oh! + + Uncaught in the interpreter: + + Traceback (most recent call last): + File "{scriptfile}", line 6, in script + spam.eggs() + ~~~~~~~~~^^ + File "{modfile}", line 6, in eggs + ham() + ~~~^^ + File "{modfile}", line 3, in ham + raise RuntimeError('uh-oh!') + RuntimeError: uh-oh! + """)) + self.assertEqual(stdout, '') + def test_in_thread(self): interp = interpreters.create() script, file = _captured_script('print("it worked!", end="")') diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 11b6f126dff0f4..3a37ed09dd8943 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,9 +1,16 @@ import contextlib import os +import os.path +import subprocess +import sys +import tempfile import threading from textwrap import dedent import unittest +from test import support +from test.support import os_helper + from test.support import interpreters @@ -71,5 +78,70 @@ def ensure_closed(fd): self.addCleanup(lambda: ensure_closed(w)) return r, w + def temp_dir(self): + tempdir = tempfile.mkdtemp() + tempdir = os.path.realpath(tempdir) + self.addCleanup(lambda: os_helper.rmtree(tempdir)) + return tempdir + + def make_script(self, filename, dirname=None, text=None): + if text: + text = dedent(text) + if dirname is None: + dirname = self.temp_dir() + filename = os.path.join(dirname, filename) + + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + def make_module(self, name, pathentry=None, text=None): + if text: + text = dedent(text) + if pathentry is None: + pathentry = self.temp_dir() + else: + os.makedirs(pathentry, exist_ok=True) + *subnames, basename = name.split('.') + + dirname = pathentry + for subname in subnames: + dirname = os.path.join(dirname, subname) + if os.path.isdir(dirname): + pass + elif os.path.exists(dirname): + raise Exception(dirname) + else: + os.mkdir(dirname) + initfile = os.path.join(dirname, '__init__.py') + if not os.path.exists(initfile): + with open(initfile, 'w'): + pass + filename = os.path.join(dirname, basename + '.py') + + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + @support.requires_subprocess() + def run_python(self, *argv): + proc = subprocess.run( + [sys.executable, *argv], + capture_output=True, + text=True, + ) + return proc.returncode, proc.stdout, proc.stderr + + def assert_python_ok(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 1) + return stdout, stderr + + def assert_python_failure(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 0) + return stdout, stderr + def tearDown(self): clean_up_interpreters() diff --git a/Python/crossinterp.c b/Python/crossinterp.c index a31b5ef4613dbd..edd61cf99f3f52 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -944,6 +944,26 @@ _xidregistry_fini(struct _xidregistry *registry) /* convenience utilities */ /*************************/ +static const char * +_copy_raw_string(const char *str, Py_ssize_t len) +{ + size_t size = len + 1; + if (len <= 0) { + size = strlen(str) + 1; + } + char *copied = PyMem_RawMalloc(size); + if (copied == NULL) { + return NULL; + } + if (len <= 0) { + strcpy(copied, str); + } + else { + memcpy(copied, str, size); + } + return copied; +} + static const char * _copy_string_obj_raw(PyObject *strobj) { @@ -961,6 +981,80 @@ _copy_string_obj_raw(PyObject *strobj) return copied; } + +static int +_pickle_object(PyObject *obj, const char **p_pickled, Py_ssize_t *p_len) +{ + assert(!PyErr_Occurred()); + PyObject *picklemod = PyImport_ImportModule("_pickle"); + if (picklemod == NULL) { + PyErr_Clear(); + picklemod = PyImport_ImportModule("pickle"); + if (picklemod == NULL) { + return -1; + } + } + PyObject *dumps = PyObject_GetAttrString(picklemod, "dumps"); + Py_DECREF(picklemod); + if (dumps == NULL) { + return -1; + } + PyObject *pickledobj = PyObject_CallOneArg(dumps, obj); + Py_DECREF(dumps); + if (pickledobj == NULL) { + return -1; + } + + char *pickled = NULL; + Py_ssize_t len = 0; + if (PyBytes_AsStringAndSize(pickledobj, &pickled, &len) < 0) { + Py_DECREF(pickledobj); + return -1; + } + const char *copied = _copy_raw_string(pickled, len); + Py_DECREF(pickledobj); + if (copied == NULL) { + return -1; + } + + *p_pickled = copied; + *p_len = len; + return 0; +} + +static int +_unpickle_object(const char *pickled, Py_ssize_t size, PyObject **p_obj) +{ + assert(!PyErr_Occurred()); + PyObject *picklemod = PyImport_ImportModule("_pickle"); + if (picklemod == NULL) { + PyErr_Clear(); + picklemod = PyImport_ImportModule("pickle"); + if (picklemod == NULL) { + return -1; + } + } + PyObject *loads = PyObject_GetAttrString(picklemod, "loads"); + Py_DECREF(picklemod); + if (loads == NULL) { + return -1; + } + PyObject *pickledobj = PyBytes_FromStringAndSize(pickled, size); + if (pickledobj == NULL) { + Py_DECREF(loads); + return -1; + } + PyObject *obj = PyObject_CallOneArg(loads, pickledobj); + Py_DECREF(loads); + Py_DECREF(pickledobj); + if (obj == NULL) { + return -1; + } + *p_obj = obj; + return 0; +} + + static int _release_xid_data(_PyCrossInterpreterData *data, int rawfree) { @@ -1094,6 +1188,9 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info) if (info->msg != NULL) { PyMem_RawFree((void *)info->msg); } + if (info->pickled != NULL) { + PyMem_RawFree((void *)info->pickled); + } *info = (_PyXI_excinfo){{NULL}}; } @@ -1129,6 +1226,63 @@ _PyXI_excinfo_format(_PyXI_excinfo *info) } } +static int +_convert_exc_to_TracebackException(PyObject *exc, PyObject **p_tbexc) +{ + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *create = NULL; + + // This is inspired by _PyErr_Display(). + PyObject *tbmod = PyImport_ImportModule("traceback"); + if (tbmod == NULL) { + return -1; + } + PyObject *tbexc_type = PyObject_GetAttrString(tbmod, "TracebackException"); + Py_DECREF(tbmod); + if (tbexc_type == NULL) { + return -1; + } + create = PyObject_GetAttrString(tbexc_type, "from_exception"); + Py_DECREF(tbexc_type); + if (create == NULL) { + return -1; + } + + args = PyTuple_Pack(1, exc); + if (args == NULL) { + goto error; + } + + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto error; + } + if (PyDict_SetItemString(kwargs, "save_exc_type", Py_False) < 0) { + goto error; + } + if (PyDict_SetItemString(kwargs, "lookup_lines", Py_False) < 0) { + goto error; + } + + PyObject *tbexc = PyObject_Call(create, args, kwargs); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(create); + if (tbexc == NULL) { + goto error; + } + + *p_tbexc = tbexc; + return 0; + +error: + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(create); + return -1; +} + static const char * _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) { @@ -1158,6 +1312,24 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) goto error; } + // Pickle a traceback.TracebackException. + PyObject *tbexc = NULL; + if (_convert_exc_to_TracebackException(exc, &tbexc) < 0) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored while creating TracebackException"); +#endif + PyErr_Clear(); + } + else { + if (_pickle_object(tbexc, &info->pickled, &info->pickled_len) < 0) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored while pickling TracebackException"); +#endif + PyErr_Clear(); + } + Py_DECREF(tbexc); + } + return NULL; error: @@ -1169,9 +1341,28 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) static void _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) { + PyObject *tbexc = NULL; + if (info->pickled != NULL) { + if (_unpickle_object(info->pickled, info->pickled_len, &tbexc) < 0) { + PyErr_Clear(); + } + } + PyObject *formatted = _PyXI_excinfo_format(info); PyErr_SetObject(exctype, formatted); Py_DECREF(formatted); + + if (tbexc != NULL) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyObject_SetAttrString(exc, "_tbexc", tbexc) < 0) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored when setting _tbexc"); +#endif + PyErr_Clear(); + } + Py_DECREF(tbexc); + PyErr_SetRaisedException(exc); + } } static PyObject * @@ -1277,6 +1468,20 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) goto error; } + if (info->pickled != NULL) { + PyObject *tbexc = NULL; + if (_unpickle_object(info->pickled, info->pickled_len, &tbexc) < 0) { + PyErr_Clear(); + } + else { + res = PyObject_SetAttrString(ns, "tbexc", tbexc); + Py_DECREF(tbexc); + if (res < 0) { + goto error; + } + } + } + return ns; error: @@ -1983,6 +2188,7 @@ _capture_current_exception(_PyXI_session *session) } else { failure = _PyXI_InitError(err, excval, _PyXI_ERR_UNCAUGHT_EXCEPTION); + Py_DECREF(excval); if (failure == NULL && override != NULL) { err->code = errcode; } @@ -1997,18 +2203,6 @@ _capture_current_exception(_PyXI_session *session) err = NULL; } - // a temporary hack (famous last words) - if (excval != NULL) { - // XXX Store the traceback info (or rendered traceback) on - // _PyXI_excinfo, attach it to the exception when applied, - // and teach PyErr_Display() to print it. -#ifdef Py_DEBUG - // XXX Drop this once _Py_excinfo picks up the slack. - PyErr_Display(NULL, excval, NULL); -#endif - Py_DECREF(excval); - } - // Finished! assert(!PyErr_Occurred()); session->error = err; From a3c031884d2f16d84aacc3f733c047b3a6cae208 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 12 Dec 2023 19:20:21 -0500 Subject: [PATCH 33/95] gh-112723: Call `PyThreadState_Clear()` from the correct interpreter (#112776) The `PyThreadState_Clear()` function must only be called with the GIL held and must be called from the same interpreter as the passed in thread state. Otherwise, any Python objects on the thread state may be destroyed using the wrong interpreter, leading to memory corruption. This is also important for `Py_GIL_DISABLED` builds because free lists will be associated with PyThreadStates and cleared in `PyThreadState_Clear()`. This fixes two places that called `PyThreadState_Clear()` from the wrong interpreter and adds an assertion to `PyThreadState_Clear()`. --- Include/internal/pycore_ceval.h | 2 +- Include/internal/pycore_pylifecycle.h | 2 +- Modules/_xxsubinterpretersmodule.c | 2 + Python/ceval_gil.c | 4 +- Python/pylifecycle.c | 72 ++++++++++----------------- Python/pystate.c | 7 ++- 6 files changed, 34 insertions(+), 55 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 64fb4034669e19..a357bfa3a26064 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -124,7 +124,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject *kwnames); extern int _PyEval_ThreadsInitialized(void); -extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); +extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_AcquireLock(PyThreadState *tstate); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index daf7cb77dcc63a..c675098685764c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -63,7 +63,7 @@ extern void _PyArg_Fini(void); extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *); extern PyStatus _PyGILState_Init(PyInterpreterState *interp); -extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); +extern void _PyGILState_SetTstate(PyThreadState *tstate); extern void _PyGILState_Fini(PyInterpreterState *interp); extern void _PyGC_DumpShutdownStats(PyInterpreterState *interp); diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 4bb54c93b0a61b..4e9e13457a9eb3 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -547,7 +547,9 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + PyThreadState_Swap(tstate); PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); PyThreadState_Delete(tstate); _PyInterpreterState_RequireIDRef(interp, 1); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 7581daa55b5e46..d70abbc27606b4 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -447,7 +447,7 @@ init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) interp->ceval.own_gil = 1; } -PyStatus +void _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); @@ -466,8 +466,6 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil) // Lock the GIL and mark the current thread as attached. _PyThreadState_Attach(tstate); - - return _PyStatus_OK(); } void diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b5c7dc5da596de..0ec29846b0850b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -576,44 +576,33 @@ init_interp_settings(PyInterpreterState *interp, interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; } - /* We check "gil" in init_interp_create_gil(). */ + switch (config->gil) { + case PyInterpreterConfig_DEFAULT_GIL: break; + case PyInterpreterConfig_SHARED_GIL: break; + case PyInterpreterConfig_OWN_GIL: break; + default: + return _PyStatus_ERR("invalid interpreter config 'gil' value"); + } return _PyStatus_OK(); } -static PyStatus +static void init_interp_create_gil(PyThreadState *tstate, int gil) { - PyStatus status; - /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is only called here. */ // XXX This is broken with a per-interpreter GIL. _PyEval_FiniGIL(tstate->interp); /* Auto-thread-state API */ - status = _PyGILState_SetTstate(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + _PyGILState_SetTstate(tstate); - int own_gil; - switch (gil) { - case PyInterpreterConfig_DEFAULT_GIL: own_gil = 0; break; - case PyInterpreterConfig_SHARED_GIL: own_gil = 0; break; - case PyInterpreterConfig_OWN_GIL: own_gil = 1; break; - default: - return _PyStatus_ERR("invalid interpreter config 'gil' value"); - } + int own_gil = (gil == PyInterpreterConfig_OWN_GIL); /* Create the GIL and take it */ - status = _PyEval_InitGIL(tstate, own_gil); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - return _PyStatus_OK(); + _PyEval_InitGIL(tstate, own_gil); } @@ -657,10 +646,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, } _PyThreadState_Bind(tstate); - status = init_interp_create_gil(tstate, config.gil); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + init_interp_create_gil(tstate, config.gil); *tstate_p = tstate; return _PyStatus_OK(); @@ -2099,28 +2085,21 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) return _PyStatus_OK(); } - PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INTERP); - if (tstate == NULL) { - PyInterpreterState_Delete(interp); - *tstate_p = NULL; - return _PyStatus_OK(); - } - _PyThreadState_Bind(tstate); - + // XXX Might new_interpreter() have been called without the GIL held? PyThreadState *save_tstate = _PyThreadState_GET(); - int has_gil = 0; + PyThreadState *tstate = NULL; /* From this point until the init_interp_create_gil() call, we must not do anything that requires that the GIL be held (or otherwise exist). That applies whether or not the new interpreter has its own GIL (e.g. the main interpreter). */ + if (save_tstate != NULL) { + _PyThreadState_Detach(save_tstate); + } /* Copy the current interpreter config into the new interpreter */ const PyConfig *src_config; if (save_tstate != NULL) { - // XXX Might new_interpreter() have been called without the GIL held? - _PyThreadState_Detach(save_tstate); src_config = _PyInterpreterState_GetConfig(save_tstate->interp); } else @@ -2142,11 +2121,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } - status = init_interp_create_gil(tstate, config->gil); - if (_PyStatus_EXCEPTION(status)) { + tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP); + if (tstate == NULL) { + status = _PyStatus_NO_MEMORY(); goto error; } - has_gil = 1; + + _PyThreadState_Bind(tstate); + init_interp_create_gil(tstate, config->gil); /* No objects have been created yet. */ @@ -2165,16 +2147,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) error: *tstate_p = NULL; - - /* Oops, it didn't work. Undo it all. */ - if (has_gil) { + if (tstate != NULL) { + PyThreadState_Clear(tstate); _PyThreadState_Detach(tstate); + PyThreadState_Delete(tstate); } if (save_tstate != NULL) { _PyThreadState_Attach(save_tstate); } - PyThreadState_Clear(tstate); - PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); return status; diff --git a/Python/pystate.c b/Python/pystate.c index f0c5259967d907..e18eb0186d0010 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1454,6 +1454,7 @@ void PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); + assert(current_fast_get(&_PyRuntime)->interp == tstate->interp); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -2150,7 +2151,7 @@ _PyGILState_Fini(PyInterpreterState *interp) // XXX Drop this. -PyStatus +void _PyGILState_SetTstate(PyThreadState *tstate) { /* must init with valid states */ @@ -2160,7 +2161,7 @@ _PyGILState_SetTstate(PyThreadState *tstate) if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, PyGILState is shared by all interpreters. The main * interpreter is responsible to initialize it. */ - return _PyStatus_OK(); + return; } #ifndef NDEBUG @@ -2170,8 +2171,6 @@ _PyGILState_SetTstate(PyThreadState *tstate) assert(gilstate_tss_get(runtime) == tstate); assert(tstate->gilstate_counter == 1); #endif - - return _PyStatus_OK(); } PyInterpreterState * From 7e2d93f30b157e414924c32232bb748c8f66c828 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 Dec 2023 14:29:21 -1000 Subject: [PATCH 34/95] gh-112989: asyncio: Reduce overhead to connect sockets with SelectorEventLoop (#112991) _ensure_fd_no_transport had a KeyError in the success path --- Lib/asyncio/selector_events.py | 14 +++++--------- .../2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst | 1 + 2 files changed, 6 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index d521b4e2e255a9..dcd5e0aa345029 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -261,15 +261,11 @@ def _ensure_fd_no_transport(self, fd): except (AttributeError, TypeError, ValueError): # This code matches selectors._fileobj_to_fd function. raise ValueError(f"Invalid file object: {fd!r}") from None - try: - transport = self._transports[fileno] - except KeyError: - pass - else: - if not transport.is_closing(): - raise RuntimeError( - f'File descriptor {fd!r} is used by transport ' - f'{transport!r}') + transport = self._transports.get(fileno) + if transport and not transport.is_closing(): + raise RuntimeError( + f'File descriptor {fd!r} is used by transport ' + f'{transport!r}') def _add_reader(self, fd, callback, *args): self._check_closed() diff --git a/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst b/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst new file mode 100644 index 00000000000000..ceeab8cc7d6bec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst @@ -0,0 +1 @@ +Reduce overhead to connect sockets with :mod:`asyncio` SelectorEventLoop. From c6e614fd81d7dca436fe640d63a307c7dc9f6f3b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 12 Dec 2023 17:31:30 -0700 Subject: [PATCH 35/95] gh-76785: Avoid Pickled TracebackException for Propagated Subinterpreter Exceptions (gh-113036) We need the TracebackException of uncaught exceptions for a single purpose: the error display. Thus we only need to pass the formatted error display between interpreters. Passing a pickled TracebackException is overkill. --- Include/internal/pycore_crossinterp.h | 3 +- Lib/test/support/interpreters/__init__.py | 2 +- Python/crossinterp.c | 239 ++++++++-------------- 3 files changed, 90 insertions(+), 154 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 414e32b5155f62..d6e297a7e8e6db 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -188,8 +188,7 @@ typedef struct _excinfo { const char *module; } type; const char *msg; - const char *pickled; - Py_ssize_t pickled_len; + const char *errdisplay; } _PyXI_excinfo; diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index d619bea3e32f5d..15a908e9663593 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -56,7 +56,7 @@ def __init__(self, excinfo): def __str__(self): try: - formatted = ''.join(self.excinfo.tbexc.format()).rstrip() + formatted = self.excinfo.errdisplay except Exception: return super().__str__() else: diff --git a/Python/crossinterp.c b/Python/crossinterp.c index edd61cf99f3f52..c6ed7daeb1074a 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -945,113 +945,105 @@ _xidregistry_fini(struct _xidregistry *registry) /*************************/ static const char * -_copy_raw_string(const char *str, Py_ssize_t len) +_copy_string_obj_raw(PyObject *strobj, Py_ssize_t *p_size) { - size_t size = len + 1; - if (len <= 0) { - size = strlen(str) + 1; - } - char *copied = PyMem_RawMalloc(size); - if (copied == NULL) { - return NULL; - } - if (len <= 0) { - strcpy(copied, str); - } - else { - memcpy(copied, str, size); - } - return copied; -} - -static const char * -_copy_string_obj_raw(PyObject *strobj) -{ - const char *str = PyUnicode_AsUTF8(strobj); + Py_ssize_t size = -1; + const char *str = PyUnicode_AsUTF8AndSize(strobj, &size); if (str == NULL) { return NULL; } - char *copied = PyMem_RawMalloc(strlen(str)+1); + char *copied = PyMem_RawMalloc(size+1); if (copied == NULL) { PyErr_NoMemory(); return NULL; } strcpy(copied, str); + if (p_size != NULL) { + *p_size = size; + } return copied; } static int -_pickle_object(PyObject *obj, const char **p_pickled, Py_ssize_t *p_len) +_convert_exc_to_TracebackException(PyObject *exc, PyObject **p_tbexc) { - assert(!PyErr_Occurred()); - PyObject *picklemod = PyImport_ImportModule("_pickle"); - if (picklemod == NULL) { - PyErr_Clear(); - picklemod = PyImport_ImportModule("pickle"); - if (picklemod == NULL) { - return -1; - } + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *create = NULL; + + // This is inspired by _PyErr_Display(). + PyObject *tbmod = PyImport_ImportModule("traceback"); + if (tbmod == NULL) { + return -1; } - PyObject *dumps = PyObject_GetAttrString(picklemod, "dumps"); - Py_DECREF(picklemod); - if (dumps == NULL) { + PyObject *tbexc_type = PyObject_GetAttrString(tbmod, "TracebackException"); + Py_DECREF(tbmod); + if (tbexc_type == NULL) { return -1; } - PyObject *pickledobj = PyObject_CallOneArg(dumps, obj); - Py_DECREF(dumps); - if (pickledobj == NULL) { + create = PyObject_GetAttrString(tbexc_type, "from_exception"); + Py_DECREF(tbexc_type); + if (create == NULL) { return -1; } - char *pickled = NULL; - Py_ssize_t len = 0; - if (PyBytes_AsStringAndSize(pickledobj, &pickled, &len) < 0) { - Py_DECREF(pickledobj); - return -1; + args = PyTuple_Pack(1, exc); + if (args == NULL) { + goto error; } - const char *copied = _copy_raw_string(pickled, len); - Py_DECREF(pickledobj); - if (copied == NULL) { - return -1; + + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto error; + } + if (PyDict_SetItemString(kwargs, "save_exc_type", Py_False) < 0) { + goto error; + } + if (PyDict_SetItemString(kwargs, "lookup_lines", Py_False) < 0) { + goto error; + } + + PyObject *tbexc = PyObject_Call(create, args, kwargs); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(create); + if (tbexc == NULL) { + goto error; } - *p_pickled = copied; - *p_len = len; + *p_tbexc = tbexc; return 0; + +error: + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(create); + return -1; } -static int -_unpickle_object(const char *pickled, Py_ssize_t size, PyObject **p_obj) + +static const char * +_format_TracebackException(PyObject *tbexc) { - assert(!PyErr_Occurred()); - PyObject *picklemod = PyImport_ImportModule("_pickle"); - if (picklemod == NULL) { - PyErr_Clear(); - picklemod = PyImport_ImportModule("pickle"); - if (picklemod == NULL) { - return -1; - } - } - PyObject *loads = PyObject_GetAttrString(picklemod, "loads"); - Py_DECREF(picklemod); - if (loads == NULL) { - return -1; - } - PyObject *pickledobj = PyBytes_FromStringAndSize(pickled, size); - if (pickledobj == NULL) { - Py_DECREF(loads); - return -1; + PyObject *lines = PyObject_CallMethod(tbexc, "format", NULL); + if (lines == NULL) { + return NULL; } - PyObject *obj = PyObject_CallOneArg(loads, pickledobj); - Py_DECREF(loads); - Py_DECREF(pickledobj); - if (obj == NULL) { - return -1; + PyObject *formatted_obj = PyUnicode_Join(&_Py_STR(empty), lines); + Py_DECREF(lines); + if (formatted_obj == NULL) { + return NULL; } - *p_obj = obj; - return 0; + + Py_ssize_t size = -1; + const char *formatted = _copy_string_obj_raw(formatted_obj, &size); + Py_DECREF(formatted_obj); + // We remove trailing the newline added by TracebackException.format(). + assert(formatted[size-1] == '\n'); + ((char *)formatted)[size-1] = '\0'; + return formatted; } @@ -1101,7 +1093,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) if (strobj == NULL) { return -1; } - info->name = _copy_string_obj_raw(strobj); + info->name = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); if (info->name == NULL) { return -1; @@ -1112,7 +1104,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) if (strobj == NULL) { return -1; } - info->qualname = _copy_string_obj_raw(strobj); + info->qualname = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); if (info->name == NULL) { return -1; @@ -1123,7 +1115,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) if (strobj == NULL) { return -1; } - info->module = _copy_string_obj_raw(strobj); + info->module = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); if (info->name == NULL) { return -1; @@ -1188,8 +1180,8 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info) if (info->msg != NULL) { PyMem_RawFree((void *)info->msg); } - if (info->pickled != NULL) { - PyMem_RawFree((void *)info->pickled); + if (info->errdisplay != NULL) { + PyMem_RawFree((void *)info->errdisplay); } *info = (_PyXI_excinfo){{NULL}}; } @@ -1226,63 +1218,6 @@ _PyXI_excinfo_format(_PyXI_excinfo *info) } } -static int -_convert_exc_to_TracebackException(PyObject *exc, PyObject **p_tbexc) -{ - PyObject *args = NULL; - PyObject *kwargs = NULL; - PyObject *create = NULL; - - // This is inspired by _PyErr_Display(). - PyObject *tbmod = PyImport_ImportModule("traceback"); - if (tbmod == NULL) { - return -1; - } - PyObject *tbexc_type = PyObject_GetAttrString(tbmod, "TracebackException"); - Py_DECREF(tbmod); - if (tbexc_type == NULL) { - return -1; - } - create = PyObject_GetAttrString(tbexc_type, "from_exception"); - Py_DECREF(tbexc_type); - if (create == NULL) { - return -1; - } - - args = PyTuple_Pack(1, exc); - if (args == NULL) { - goto error; - } - - kwargs = PyDict_New(); - if (kwargs == NULL) { - goto error; - } - if (PyDict_SetItemString(kwargs, "save_exc_type", Py_False) < 0) { - goto error; - } - if (PyDict_SetItemString(kwargs, "lookup_lines", Py_False) < 0) { - goto error; - } - - PyObject *tbexc = PyObject_Call(create, args, kwargs); - Py_DECREF(args); - Py_DECREF(kwargs); - Py_DECREF(create); - if (tbexc == NULL) { - goto error; - } - - *p_tbexc = tbexc; - return 0; - -error: - Py_XDECREF(args); - Py_XDECREF(kwargs); - Py_XDECREF(create); - return -1; -} - static const char * _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) { @@ -1305,7 +1240,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) failure = "error while formatting exception"; goto error; } - info->msg = _copy_string_obj_raw(msgobj); + info->msg = _copy_string_obj_raw(msgobj, NULL); Py_DECREF(msgobj); if (info->msg == NULL) { failure = "error while copying exception message"; @@ -1321,13 +1256,14 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) PyErr_Clear(); } else { - if (_pickle_object(tbexc, &info->pickled, &info->pickled_len) < 0) { + info->errdisplay = _format_TracebackException(tbexc); + Py_DECREF(tbexc); + if (info->errdisplay == NULL) { #ifdef Py_DEBUG - PyErr_FormatUnraisable("Exception ignored while pickling TracebackException"); + PyErr_FormatUnraisable("Exception ignored while formating TracebackException"); #endif PyErr_Clear(); } - Py_DECREF(tbexc); } return NULL; @@ -1342,8 +1278,9 @@ static void _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) { PyObject *tbexc = NULL; - if (info->pickled != NULL) { - if (_unpickle_object(info->pickled, info->pickled_len, &tbexc) < 0) { + if (info->errdisplay != NULL) { + tbexc = PyUnicode_FromString(info->errdisplay); + if (tbexc == NULL) { PyErr_Clear(); } } @@ -1354,9 +1291,9 @@ _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) if (tbexc != NULL) { PyObject *exc = PyErr_GetRaisedException(); - if (PyObject_SetAttrString(exc, "_tbexc", tbexc) < 0) { + if (PyObject_SetAttrString(exc, "_errdisplay", tbexc) < 0) { #ifdef Py_DEBUG - PyErr_FormatUnraisable("Exception ignored when setting _tbexc"); + PyErr_FormatUnraisable("Exception ignored when setting _errdisplay"); #endif PyErr_Clear(); } @@ -1468,13 +1405,13 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) goto error; } - if (info->pickled != NULL) { - PyObject *tbexc = NULL; - if (_unpickle_object(info->pickled, info->pickled_len, &tbexc) < 0) { + if (info->errdisplay != NULL) { + PyObject *tbexc = PyUnicode_FromString(info->errdisplay); + if (tbexc == NULL) { PyErr_Clear(); } else { - res = PyObject_SetAttrString(ns, "tbexc", tbexc); + res = PyObject_SetAttrString(ns, "errdisplay", tbexc); Py_DECREF(tbexc); if (res < 0) { goto error; @@ -1646,7 +1583,7 @@ _sharednsitem_is_initialized(_PyXI_namespace_item *item) static int _sharednsitem_init(_PyXI_namespace_item *item, PyObject *key) { - item->name = _copy_string_obj_raw(key); + item->name = _copy_string_obj_raw(key, NULL); if (item->name == NULL) { assert(!_sharednsitem_is_initialized(item)); return -1; From a3a1cb48456c809f7b1ab6a6ffe83e8b3f69be0f Mon Sep 17 00:00:00 2001 From: Jamie <101677823+ordinary-jamie@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:26:40 +1100 Subject: [PATCH 36/95] gh-112622: Pass name to loop create_task method (#112623) This affects task creation through either `asyncio.create_task()` or `TaskGroup.create_task()` -- the redundant call to `task.set_name()` is skipped. We still call `set_name()` when a task factory is involved, because the task factory call signature (unfortunately) doesn't take a `name` argument. --- Lib/asyncio/taskgroups.py | 6 +++--- Lib/asyncio/tasks.py | 5 ++--- .../Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst | 2 ++ 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 91be0decc41c42..cb9c1ce4d7d1d2 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -158,10 +158,10 @@ def create_task(self, coro, *, name=None, context=None): if self._aborting: raise RuntimeError(f"TaskGroup {self!r} is shutting down") if context is None: - task = self._loop.create_task(coro) + task = self._loop.create_task(coro, name=name) else: - task = self._loop.create_task(coro, context=context) - task.set_name(name) + task = self._loop.create_task(coro, name=name, context=context) + # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), # and skip scheduling a done callback diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index e84b21390557be..fafee3e738f6aa 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -404,11 +404,10 @@ def create_task(coro, *, name=None, context=None): loop = events.get_running_loop() if context is None: # Use legacy API if context is not needed - task = loop.create_task(coro) + task = loop.create_task(coro, name=name) else: - task = loop.create_task(coro, context=context) + task = loop.create_task(coro, name=name, context=context) - task.set_name(name) return task diff --git a/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst b/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst new file mode 100644 index 00000000000000..91c88bac334dcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst @@ -0,0 +1,2 @@ +Ensure ``name`` parameter is passed to event loop in +:func:`asyncio.create_task`. From 3aea6c4823e90172c9bc36cd20dc51b295d8a3c4 Mon Sep 17 00:00:00 2001 From: beavailable Date: Wed, 13 Dec 2023 11:23:29 +0800 Subject: [PATCH 37/95] gh-101336: Add keep_alive keyword arg for asyncio create_server() (#112485) --- Doc/library/asyncio-eventloop.rst | 8 ++++++++ Lib/asyncio/base_events.py | 4 ++++ Lib/asyncio/events.py | 4 ++++ .../2023-11-28-02-39-30.gh-issue-101336.ya433z.rst | 1 + 4 files changed, 17 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index ea1d146f06cf2b..828e506a72c937 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -671,6 +671,7 @@ Creating network servers flags=socket.AI_PASSIVE, \ sock=None, backlog=100, ssl=None, \ reuse_address=None, reuse_port=None, \ + keep_alive=None, \ ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, \ start_serving=True) @@ -735,6 +736,13 @@ Creating network servers set this flag when being created. This option is not supported on Windows. + * *keep_alive* set to ``True`` keeps connections active by enabling the + periodic transmission of messages. + + .. versionchanged:: 3.13 + + Added the *keep_alive* parameter. + * *ssl_handshake_timeout* is (for a TLS server) the time in seconds to wait for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 416c732298d9a9..a8870b636d1df5 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1496,6 +1496,7 @@ async def create_server( ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -1569,6 +1570,9 @@ async def create_server( socket.SOL_SOCKET, socket.SO_REUSEADDR, True) if reuse_port: _set_reuseport(sock) + if keep_alive: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) # Disable IPv4/IPv6 dual stack support (enabled by # default on Linux) which makes a single socket # listen on both address families. diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0ccf85105e6673..ebc3836bdc0c4d 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -316,6 +316,7 @@ async def create_server( *, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -354,6 +355,9 @@ async def create_server( they all set this flag when being created. This option is not supported on Windows. + keep_alive set to True keeps connections active by enabling the + periodic transmission of messages. + ssl_handshake_timeout is the time in seconds that an SSL server will wait for completion of the SSL handshake before aborting the connection. Default is 60s. diff --git a/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst b/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst new file mode 100644 index 00000000000000..c222febae6b554 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst @@ -0,0 +1 @@ +Add ``keep_alive`` keyword parameter for :meth:`AbstractEventLoop.create_server` and :meth:`BaseEventLoop.create_server`. From 3531ea441b8b76bff90d2ecc062335da65fd3341 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 13 Dec 2023 09:24:55 +0200 Subject: [PATCH 38/95] gh-101100: Fix Sphinx warning in references with asterisks (#113029) Co-authored-by: Alex Waygood --- Doc/library/bdb.rst | 2 +- Doc/library/cmd.rst | 14 +++++++------- Doc/library/configparser.rst | 2 +- Doc/library/csv.rst | 6 ++++-- Doc/library/http.server.rst | 8 ++++---- Doc/library/locale.rst | 2 +- Doc/library/os.rst | 8 ++++---- Doc/library/resource.rst | 4 ++-- Doc/library/socket.rst | 8 ++++---- Doc/library/unittest.rst | 14 +++++++------- Doc/library/urllib.request.rst | 8 ++++---- Doc/library/xml.dom.rst | 2 +- Doc/whatsnew/2.3.rst | 2 +- Doc/whatsnew/2.4.rst | 2 +- Doc/whatsnew/2.5.rst | 4 ++-- Doc/whatsnew/2.7.rst | 2 +- Doc/whatsnew/3.4.rst | 2 +- Misc/NEWS.d/3.10.0a1.rst | 4 ++-- Misc/NEWS.d/3.12.0a1.rst | 2 +- Misc/NEWS.d/3.12.0b1.rst | 2 +- Misc/NEWS.d/3.6.0a2.rst | 2 +- Misc/NEWS.d/3.9.0a1.rst | 10 +++++----- 22 files changed, 56 insertions(+), 54 deletions(-) diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index d201dc963b5995..4ce5c9bcde38ff 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -294,7 +294,7 @@ The :mod:`bdb` module also defines two classes: .. method:: set_quit() Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in - the next call to one of the :meth:`dispatch_\*` methods. + the next call to one of the :meth:`!dispatch_\*` methods. Derived classes and clients can call the following methods to manipulate diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 1318ffe5a48d53..a79882ed1cca4f 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -83,7 +83,7 @@ A :class:`Cmd` instance has the following methods: This method will return when the :meth:`postcmd` method returns a true value. The *stop* argument to :meth:`postcmd` is the return value from the command's - corresponding :meth:`do_\*` method. + corresponding :meth:`!do_\*` method. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling :meth:`complete_foo` with @@ -98,7 +98,7 @@ A :class:`Cmd` instance has the following methods: :meth:`help_bar`, and if that is not present, prints the docstring of :meth:`do_bar`, if available. With no argument, :meth:`do_help` lists all available help topics (that is, all commands with corresponding - :meth:`help_\*` methods or commands that have docstrings), and also lists any + :meth:`!help_\*` methods or commands that have docstrings), and also lists any undocumented commands. @@ -108,7 +108,7 @@ A :class:`Cmd` instance has the following methods: This may be overridden, but should not normally need to be; see the :meth:`precmd` and :meth:`postcmd` methods for useful execution hooks. The return value is a flag indicating whether interpretation of commands by the - interpreter should stop. If there is a :meth:`do_\*` method for the command + interpreter should stop. If there is a :meth:`!do_\*` method for the command *str*, the return value of that method is returned, otherwise the return value from the :meth:`default` method is returned. @@ -128,7 +128,7 @@ A :class:`Cmd` instance has the following methods: .. method:: Cmd.completedefault(text, line, begidx, endidx) Method called to complete an input line when no command-specific - :meth:`complete_\*` method is available. By default, it returns an empty list. + :meth:`!complete_\*` method is available. By default, it returns an empty list. .. method:: Cmd.columnize(list, displaywidth=80) @@ -209,14 +209,14 @@ Instances of :class:`Cmd` subclasses have some public instance variables: .. attribute:: Cmd.misc_header The header to issue if the help output has a section for miscellaneous help - topics (that is, there are :meth:`help_\*` methods without corresponding - :meth:`do_\*` methods). + topics (that is, there are :meth:`!help_\*` methods without corresponding + :meth:`!do_\*` methods). .. attribute:: Cmd.undoc_header The header to issue if the help output has a section for undocumented commands - (that is, there are :meth:`do_\*` methods without corresponding :meth:`help_\*` + (that is, there are :meth:`!do_\*` methods without corresponding :meth:`!help_\*` methods). diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb282428c5fffc..12eee47613d186 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -955,7 +955,7 @@ ConfigParser Objects When *converters* is given, it should be a dictionary where each key represents the name of a type converter and each value is a callable implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`get*()` method on the parser + converter gets its own corresponding :meth:`!get*()` method on the parser object and section proxies. .. versionchanged:: 3.1 diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index aba398b8ee1e54..4d52254e6d6db5 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -309,6 +309,8 @@ An example for :class:`Sniffer` use:: # ... process CSV file contents here ... +.. _csv-constants: + The :mod:`csv` module defines the following constants: .. data:: QUOTE_ALL @@ -432,8 +434,8 @@ Dialects support the following attributes: .. attribute:: Dialect.quoting Controls when quotes should be generated by the writer and recognised by the - reader. It can take on any of the :const:`QUOTE_\*` constants (see section - :ref:`csv-contents`) and defaults to :const:`QUOTE_MINIMAL`. + reader. It can take on any of the :ref:`QUOTE_\* constants ` + and defaults to :const:`QUOTE_MINIMAL`. .. attribute:: Dialect.skipinitialspace diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 6f79b222790094..64bddd23f82933 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -65,10 +65,10 @@ provides three different variants: The handler will parse the request and the headers, then call a method specific to the request type. The method name is constructed from the - request. For example, for the request method ``SPAM``, the :meth:`do_SPAM` + request. For example, for the request method ``SPAM``, the :meth:`!do_SPAM` method will be called with no arguments. All of the relevant information is stored in instance variables of the handler. Subclasses should not need to - override or extend the :meth:`__init__` method. + override or extend the :meth:`!__init__` method. :class:`BaseHTTPRequestHandler` has the following instance variables: @@ -187,13 +187,13 @@ provides three different variants: Calls :meth:`handle_one_request` once (or, if persistent connections are enabled, multiple times) to handle incoming HTTP requests. You should - never need to override it; instead, implement appropriate :meth:`do_\*` + never need to override it; instead, implement appropriate :meth:`!do_\*` methods. .. method:: handle_one_request() This method will parse and dispatch the request to the appropriate - :meth:`do_\*` method. You should never need to override it. + :meth:`!do_\*` method. You should never need to override it. .. method:: handle_expect_100() diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 0d48892fcdab97..a7201199191215 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -309,7 +309,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: getlocale(category=LC_CTYPE) Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`LC_\*` values + *language code*, *encoding*. *category* may be one of the :const:`!LC_\*` values except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index fe573f188ab066..e5ac9afc3c6fd9 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4170,7 +4170,7 @@ to be ignored. The "l" and "v" variants of the :func:`exec\* ` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the - individual parameters simply become additional parameters to the :func:`execl\*` + individual parameters simply become additional parameters to the :func:`!execl\*` functions. The "v" variants are good when the number of parameters is variable, with the arguments being passed in a list or tuple as the *args* parameter. In either case, the arguments to the child process should start with @@ -4708,7 +4708,7 @@ written in Python, such as a mail server's external command delivery program. command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the individual parameters simply become additional parameters to the - :func:`spawnl\*` functions. The "v" variants are good when the number of + :func:`!spawnl\*` functions. The "v" variants are good when the number of parameters is variable, with the arguments being passed in a list or tuple as the *args* parameter. In either case, the arguments to the child process must start with the name of the command being run. @@ -4758,7 +4758,7 @@ written in Python, such as a mail server's external command delivery program. P_NOWAITO Possible values for the *mode* parameter to the :func:`spawn\* ` family of - functions. If either of these values is given, the :func:`spawn\*` functions + functions. If either of these values is given, the :func:`spawn\* ` functions will return as soon as the new process has been created, with the process id as the return value. @@ -4768,7 +4768,7 @@ written in Python, such as a mail server's external command delivery program. .. data:: P_WAIT Possible value for the *mode* parameter to the :func:`spawn\* ` family of - functions. If this is given as *mode*, the :func:`spawn\*` functions will not + functions. If this is given as *mode*, the :func:`spawn\* ` functions will not return until the new process has run to completion and will return the exit code of the process the run is successful, or ``-signal`` if a signal kills the process. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index ef65674d1b0a78..4e58b043f1da31 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -277,7 +277,7 @@ These functions are used to retrieve resource usage information: This function returns an object that describes the resources consumed by either the current process or its children, as specified by the *who* parameter. The - *who* parameter should be specified using one of the :const:`RUSAGE_\*` + *who* parameter should be specified using one of the :const:`!RUSAGE_\*` constants described below. A simple example:: @@ -353,7 +353,7 @@ These functions are used to retrieve resource usage information: Returns the number of bytes in a system page. (This need not be the same as the hardware page size.) -The following :const:`RUSAGE_\*` symbols are passed to the :func:`getrusage` +The following :const:`!RUSAGE_\*` symbols are passed to the :func:`getrusage` function to specify which processes information should be provided for. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index e0a75304ef1606..4bfb0d8c2cfeac 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -311,7 +311,7 @@ Exceptions The accompanying value is a pair ``(error, string)`` representing an error returned by a library call. *string* represents the description of *error*, as returned by the :c:func:`gai_strerror` C function. The - numeric *error* value will match one of the :const:`EAI_\*` constants + numeric *error* value will match one of the :const:`!EAI_\*` constants defined in this module. .. versionchanged:: 3.3 @@ -1517,7 +1517,7 @@ to sockets. .. method:: socket.getsockopt(level, optname[, buflen]) Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:const:`SO_\*` etc.) + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) are defined in this module. If *buflen* is absent, an integer option is assumed and its integer value is returned by the function. If *buflen* is present, it specifies the maximum length of the buffer used to receive the option in, and @@ -1937,8 +1937,8 @@ to sockets. .. index:: pair: module; struct Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the - :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer, + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. `). The value can be an integer, ``None`` or a :term:`bytes-like object` representing a buffer. In the later case it is up to the caller to ensure that the bytestring contains the proper bits (see the optional built-in module :mod:`struct` for a way to diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 02b72cb9f6b8aa..c04e6c378cc3b1 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -390,8 +390,8 @@ testing code:: widget = Widget('The widget') self.assertEqual(widget.size(), (50, 50)) -Note that in order to test something, we use one of the :meth:`assert\*` -methods provided by the :class:`TestCase` base class. If the test fails, an +Note that in order to test something, we use one of the :ref:`assert\* methods ` +provided by the :class:`TestCase` base class. If the test fails, an exception will be raised with an explanatory message, and :mod:`unittest` will identify the test case as a :dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`. @@ -1940,14 +1940,14 @@ Loading and running tests String giving the prefix of method names which will be interpreted as test methods. The default value is ``'test'``. - This affects :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` + This affects :meth:`getTestCaseNames` and all the ``loadTestsFrom*`` methods. .. attribute:: sortTestMethodsUsing Function to be used to compare method names when sorting them in - :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` methods. + :meth:`getTestCaseNames` and all the ``loadTestsFrom*`` methods. .. attribute:: suiteClass @@ -1956,7 +1956,7 @@ Loading and running tests methods on the resulting object are needed. The default value is the :class:`TestSuite` class. - This affects all the :meth:`loadTestsFrom\*` methods. + This affects all the ``loadTestsFrom*`` methods. .. attribute:: testNamePatterns @@ -1969,7 +1969,7 @@ Loading and running tests so unlike patterns passed to the ``-k`` option, simple substring patterns will have to be converted using ``*`` wildcards. - This affects all the :meth:`loadTestsFrom\*` methods. + This affects all the ``loadTestsFrom*`` methods. .. versionadded:: 3.7 @@ -2003,7 +2003,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :ref:`assert\* methods `. .. attribute:: skipped diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index bf3af1bef0714c..040e28e9124014 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -712,8 +712,8 @@ The following attribute and methods should only be used by classes derived from .. note:: The convention has been adopted that subclasses defining - :meth:`_request` or :meth:`_response` methods are named - :class:`\*Processor`; all others are named :class:`\*Handler`. + :meth:`!_request` or :meth:`!_response` methods are named + :class:`!\*Processor`; all others are named :class:`!\*Handler`. .. attribute:: BaseHandler.parent @@ -833,9 +833,9 @@ HTTPRedirectHandler Objects .. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) Return a :class:`Request` or ``None`` in response to a redirect. This is called - by the default implementations of the :meth:`http_error_30\*` methods when a + by the default implementations of the :meth:`!http_error_30\*` methods when a redirection is received from the server. If a redirection should take place, - return a new :class:`Request` to allow :meth:`http_error_30\*` to perform the + return a new :class:`Request` to allow :meth:`!http_error_30\*` to perform the redirect to *newurl*. Otherwise, raise :exc:`~urllib.error.HTTPError` if no other handler should try to handle this URL, or return ``None`` if you can't but another handler might. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index b387240a3716cc..d0e1b248d595d1 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -734,7 +734,7 @@ NamedNodeMap Objects attribute node. Get its value with the :attr:`value` attribute. There are also experimental methods that give this class more mapping behavior. -You can use them or you can use the standardized :meth:`getAttribute\*` family +You can use them or you can use the standardized :meth:`!getAttribute\*` family of methods on the :class:`Element` objects. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index c989e6d3fa5787..0c77b339a182c9 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1362,7 +1362,7 @@ complete list of changes, or look through the CVS logs for all the details. :mod:`os` module. (Contributed by Gustavo Niemeyer, Geert Jansen, and Denis S. Otkidach.) -* In the :mod:`os` module, the :func:`\*stat` family of functions can now report +* In the :mod:`os` module, the :func:`!\*stat` family of functions can now report fractions of a second in a timestamp. Such time stamps are represented as floats, similar to the value returned by :func:`time.time`. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index a7b0c6e5d76a6f..bc748dd44f5f8e 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1164,7 +1164,7 @@ complete list of changes, or look through the CVS logs for all the details. * A number of functions were added to the :mod:`locale` module, such as :func:`bind_textdomain_codeset` to specify a particular encoding and a family of - :func:`l\*gettext` functions that return messages in the chosen encoding. + :func:`!l\*gettext` functions that return messages in the chosen encoding. (Contributed by Gustavo Niemeyer.) * Some keyword arguments were added to the :mod:`logging` package's diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 64b951da3fd5b8..627c918dd6d8b4 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1167,10 +1167,10 @@ marked in the following list. * It's now illegal to mix iterating over a file with ``for line in file`` and calling the file object's :meth:`read`/:meth:`readline`/:meth:`readlines` - methods. Iteration uses an internal buffer and the :meth:`read\*` methods + methods. Iteration uses an internal buffer and the :meth:`!read\*` methods don't use that buffer. Instead they would return the data following the buffer, causing the data to appear out of order. Mixing iteration and these - methods will now trigger a :exc:`ValueError` from the :meth:`read\*` method. + methods will now trigger a :exc:`ValueError` from the :meth:`!read\*` method. (Implemented by Thomas Wouters.) .. Patch 1397960 diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index cf6d26859bb6a2..5af700bd5d3506 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1417,7 +1417,7 @@ changes, or look through the Subversion logs for all the details. :func:`~math.lgamma` for the natural log of the Gamma function. (Contributed by Mark Dickinson and nirinA raseliarison; :issue:`3366`.) -* The :mod:`multiprocessing` module's :class:`Manager*` classes +* The :mod:`multiprocessing` module's :class:`!Manager*` classes can now be passed a callable that will be called whenever a subprocess is started, along with a set of arguments that will be passed to the callable. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 9f9cf7fafdc19e..72d12461d8f730 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -1936,7 +1936,7 @@ Other Improvements * The :ref:`python ` command has a new :ref:`option `, ``-I``, which causes it to run in "isolated mode", which means that :data:`sys.path` contains neither the script's directory nor - the user's ``site-packages`` directory, and all :envvar:`PYTHON*` environment + the user's ``site-packages`` directory, and all :envvar:`!PYTHON*` environment variables are ignored (it implies both ``-s`` and ``-E``). Other restrictions may also be applied in the future, with the goal being to isolate the execution of a script from the user's environment. This is diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index a9f25b482508ba..731eed3447d2bc 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -605,8 +605,8 @@ Opt out serialization/deserialization for _random.Random .. nonce: jxJ4yn .. section: Core and Builtins -Rename `PyPegen*` functions to `PyParser*`, so that we can remove the old -set of `PyParser*` functions that were using the old parser, but keep +Rename ``PyPegen*`` functions to ``PyParser*``, so that we can remove the old +set of ``PyParser*`` functions that were using the old parser, but keep everything backwards-compatible. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 633738de92bef7..29d04fa0e175bf 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -1913,7 +1913,7 @@ Stinner. .. nonce: Uxc9al .. section: Library -Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +Allow :mod:`venv` to pass along :envvar:`!PYTHON*` variables to ``ensurepip`` and ``pip`` when they do not impact path resolution .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 0944dfd0e90ab9..007a6ad4ffd4d4 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -880,7 +880,7 @@ Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. .. section: Library Make :mod:`dis` display the names of the args for -:opcode:`CALL_INTRINSIC_*`. +:opcode:`!CALL_INTRINSIC_*`. .. diff --git a/Misc/NEWS.d/3.6.0a2.rst b/Misc/NEWS.d/3.6.0a2.rst index 1b336d7bc5137a..05b3d9f0463c1c 100644 --- a/Misc/NEWS.d/3.6.0a2.rst +++ b/Misc/NEWS.d/3.6.0a2.rst @@ -603,7 +603,7 @@ configuring text widget colors to a new function. .. nonce: RbyFuV .. section: IDLE -Rename many `idlelib/*.py` and `idle_test/test_*.py` files. Edit files to +Rename many ``idlelib/*.py`` and ``idle_test/test_*.py`` files. Edit files to replace old names with new names when the old name referred to the module rather than the class it contained. See the issue and IDLE section in What's New in 3.6 for more. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 9818c17705074b..0444b53895b166 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -2534,7 +2534,7 @@ object when `self._spec_signature` exists. Patch by Elizabeth Uselton .. nonce: iXGuoi .. section: Library -Make `from tkinter import *` import only the expected objects. +Make ``from tkinter import *`` import only the expected objects. .. @@ -3117,9 +3117,9 @@ Ensure cookies with ``expires`` attribute are handled in .. section: Library Fix an unintended ValueError from :func:`subprocess.run` when checking for -conflicting `input` and `stdin` or `capture_output` and `stdout` or `stderr` -args when they were explicitly provided but with `None` values within a -passed in `**kwargs` dict rather than as passed directly by name. Patch +conflicting *input* and *stdin* or *capture_output* and *stdout* or *stderr* +args when they were explicitly provided but with ``None`` values within a +passed in ``**kwargs`` dict rather than as passed directly by name. Patch contributed by Rémi Lapeyre. .. @@ -3546,7 +3546,7 @@ Patch by Stein Karlsen. .. nonce: XaJDei .. section: Library -lib2to3 now recognizes expressions after ``*`` and `**` like in ``f(*[] or +lib2to3 now recognizes expressions after ``*`` and ``**`` like in ``f(*[] or [])``. .. From 428c9812cb4ff3521e5904719224fe63fba5370a Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:00:21 +0000 Subject: [PATCH 39/95] gh-112962: in dis module, put cache information in the Instruction instead of creating fake Instructions to represent it (#113016) --- Doc/library/dis.rst | 18 ++++- Lib/dis.py | 73 +++++++++++-------- Lib/test/support/bytecode_helper.py | 12 +++ Lib/test/test_code.py | 8 +- Lib/test/test_compile.py | 5 +- Lib/test/test_dis.py | 41 +++++++++-- ...-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst | 3 + 7 files changed, 114 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 0d93bc9f5da774..5647021d6a9ba6 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -328,13 +328,17 @@ operation is being performed, so the intermediate analysis object isn't useful: source line information (if any) is taken directly from the disassembled code object. - The *show_caches* and *adaptive* parameters work as they do in :func:`dis`. + The *adaptive* parameter works as it does in :func:`dis`. .. versionadded:: 3.4 .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + The *show_caches* parameter is deprecated and has no effect. The *cache_info* + field of each instruction is populated regardless of its value. + .. function:: findlinestarts(code) @@ -482,6 +486,14 @@ details of bytecode instructions as :class:`Instruction` instances: :class:`dis.Positions` object holding the start and end locations that are covered by this instruction. + .. data::cache_info + + Information about the cache entries of this instruction, as + triplets of the form ``(name, size, data)``, where the ``name`` + and ``size`` describe the cache format and data is the contents + of the cache. ``cache_info`` is ``None`` if the instruction does not have + caches. + .. versionadded:: 3.4 .. versionchanged:: 3.11 @@ -493,8 +505,8 @@ details of bytecode instructions as :class:`Instruction` instances: Changed field ``starts_line``. Added fields ``start_offset``, ``cache_offset``, ``end_offset``, - ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, and - ``line_number``. + ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, + ``line_number`` and ``cache_info``. .. class:: Positions diff --git a/Lib/dis.py b/Lib/dis.py index efa935c5a6a0b6..183091cb0d6098 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -267,9 +267,10 @@ def show_code(co, *, file=None): 'starts_line', 'line_number', 'label', - 'positions' + 'positions', + 'cache_info', ], - defaults=[None, None] + defaults=[None, None, None] ) _Instruction.opname.__doc__ = "Human readable name for operation" @@ -286,6 +287,7 @@ def show_code(co, *, file=None): _Instruction.line_number.__doc__ = "source line number associated with this opcode (if any), otherwise None" _Instruction.label.__doc__ = "A label (int > 0) if this instruction is a jump target, otherwise None" _Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction" +_Instruction.cache_info.__doc__ = "list of (name, size, data), one for each cache entry of the instruction" _ExceptionTableEntryBase = collections.namedtuple("_ExceptionTableEntryBase", "start end target depth lasti") @@ -334,6 +336,8 @@ class Instruction(_Instruction): label - A label if this instruction is a jump target, otherwise None positions - Optional dis.Positions object holding the span of source code covered by this instruction + cache_info - information about the format and content of the instruction's cache + entries (if any) """ @property @@ -570,7 +574,6 @@ def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False): linestarts=linestarts, line_offset=line_offset, co_positions=co.co_positions(), - show_caches=show_caches, original_code=original_code, arg_resolver=arg_resolver) @@ -645,8 +648,7 @@ def _is_backward_jump(op): 'ENTER_EXECUTOR') def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, - show_caches=False, original_code=None, labels_map=None, - arg_resolver=None): + original_code=None, labels_map=None, arg_resolver=None): """Iterate over the instructions in a bytecode string. Generates a sequence of Instruction namedtuples giving the details of each @@ -682,32 +684,28 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N else: argval, argrepr = arg, repr(arg) + instr = Instruction(_all_opname[op], op, arg, argval, argrepr, + offset, start_offset, starts_line, line_number, + labels_map.get(offset, None), positions) + + caches = _get_cache_size(_all_opname[deop]) + # Advance the co_positions iterator: + for _ in range(caches): + next(co_positions, ()) + + if caches: + cache_info = [] + for name, size in _cache_format[opname[deop]].items(): + data = code[offset + 2: offset + 2 + 2 * size] + cache_info.append((name, size, data)) + else: + cache_info = None + yield Instruction(_all_opname[op], op, arg, argval, argrepr, offset, start_offset, starts_line, line_number, - labels_map.get(offset, None), positions) + labels_map.get(offset, None), positions, cache_info) + - caches = _get_cache_size(_all_opname[deop]) - if not caches: - continue - if not show_caches: - # We still need to advance the co_positions iterator: - for _ in range(caches): - next(co_positions, ()) - continue - for name, size in _cache_format[opname[deop]].items(): - for i in range(size): - offset += 2 - # Only show the fancy argrepr for a CACHE instruction when it's - # the first entry for a particular cache value: - if i == 0: - data = code[offset: offset + 2 * size] - argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" - else: - argrepr = "" - yield Instruction( - "CACHE", CACHE, 0, None, argrepr, offset, offset, False, None, None, - Positions(*next(co_positions, ())) - ) def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, show_offsets=False): @@ -787,7 +785,6 @@ def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None, instrs = _get_instructions_bytes(code, linestarts=linestarts, line_offset=line_offset, co_positions=co_positions, - show_caches=show_caches, original_code=original_code, labels_map=labels_map, arg_resolver=arg_resolver) @@ -805,6 +802,23 @@ def print_instructions(instrs, exception_entries, formatter, show_caches=False, is_current_instr = instr.offset <= lasti \ <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) formatter.print_instruction(instr, is_current_instr) + deop = _deoptop(instr.opcode) + if show_caches and instr.cache_info: + offset = instr.offset + for name, size, data in instr.cache_info: + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + formatter.print_instruction( + Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions), + is_current_instr) + formatter.print_exception_table(exception_entries) def _disassemble_str(source, **kwargs): @@ -952,7 +966,6 @@ def __iter__(self): linestarts=self._linestarts, line_offset=self._line_offset, co_positions=co.co_positions(), - show_caches=self.show_caches, original_code=original_code, labels_map=labels_map, arg_resolver=arg_resolver) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 388d1266773c8a..a4845065a5322e 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -7,6 +7,18 @@ _UNSPECIFIED = object() +def instructions_with_positions(instrs, co_positions): + # Return (instr, positions) pairs from the instrs list and co_positions + # iterator. The latter contains items for cache lines and the former + # doesn't, so those need to be skipped. + + co_positions = co_positions or iter(()) + for instr in instrs: + yield instr, next(co_positions, ()) + for _, size, _ in (instr.cache_info or ()): + for i in range(size): + next(co_positions, ()) + class BytecodeTestCase(unittest.TestCase): """Custom assertion methods for inspecting bytecode.""" diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index a961ddbe17a3d3..d8fb826edeb681 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -144,6 +144,8 @@ gc_collect) from test.support.script_helper import assert_python_ok from test.support import threading_helper +from test.support.bytecode_helper import (BytecodeTestCase, + instructions_with_positions) from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -384,10 +386,8 @@ def test_co_positions_artificial_instructions(self): code = traceback.tb_frame.f_code artificial_instructions = [] - for instr, positions in zip( - dis.get_instructions(code, show_caches=True), - code.co_positions(), - strict=True + for instr, positions in instructions_with_positions( + dis.get_instructions(code), code.co_positions() ): # If any of the positions is None, then all have to # be None as well for the case above. There are still diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index df6e5e4b55f728..f681d125db7d7a 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -12,6 +12,7 @@ from test import support from test.support import (script_helper, requires_debug_ranges, requires_specialization, Py_C_RECURSION_LIMIT) +from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath class TestSpecifics(unittest.TestCase): @@ -1346,8 +1347,8 @@ def generic_visit(self, node): def assertOpcodeSourcePositionIs(self, code, opcode, line, end_line, column, end_column, occurrence=1): - for instr, position in zip( - dis.Bytecode(code, show_caches=True), code.co_positions(), strict=True + for instr, position in instructions_with_positions( + dis.Bytecode(code), code.co_positions() ): if instr.opname == opcode: occurrence -= 1 diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 0ea4dc4566a4a4..12e2c57e50b0ba 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -13,6 +13,7 @@ import opcode +CACHE = dis.opmap["CACHE"] def get_tb(): def _error(): @@ -1227,9 +1228,9 @@ def f(): else: # "copy" the code to un-quicken it: f.__code__ = f.__code__.replace() - for instruction in dis.get_instructions( + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( f, show_caches=True, adaptive=adaptive - ): + ), show_caches=True): if instruction.opname == "CACHE": yield instruction.argrepr @@ -1262,7 +1263,8 @@ def f(): # However, this might change in the future. So we explicitly try to find # a CACHE entry in the instructions. If we can't do that, fail the test - for inst in dis.get_instructions(f, show_caches=True): + for inst in _unroll_caches_as_Instructions( + dis.get_instructions(f, show_caches=True), show_caches=True): if inst.opname == "CACHE": op_offset = inst.offset - 2 cache_offset = inst.offset @@ -1775,8 +1777,8 @@ def simple(): pass class InstructionTestCase(BytecodeTestCase): def assertInstructionsEqual(self, instrs_1, instrs_2, /): - instrs_1 = [instr_1._replace(positions=None) for instr_1 in instrs_1] - instrs_2 = [instr_2._replace(positions=None) for instr_2 in instrs_2] + instrs_1 = [instr_1._replace(positions=None, cache_info=None) for instr_1 in instrs_1] + instrs_2 = [instr_2._replace(positions=None, cache_info=None) for instr_2 in instrs_2] self.assertEqual(instrs_1, instrs_2) class InstructionTests(InstructionTestCase): @@ -1890,9 +1892,9 @@ def roots(a, b, c): instruction.positions.col_offset, instruction.positions.end_col_offset, ) - for instruction in dis.get_instructions( + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( code, adaptive=adaptive, show_caches=show_caches - ) + ), show_caches=show_caches) ] self.assertEqual(co_positions, dis_positions) @@ -2233,6 +2235,31 @@ def get_disassembly(self, tb): dis.distb(tb, file=output) return output.getvalue() +def _unroll_caches_as_Instructions(instrs, show_caches=False): + # Cache entries are no longer reported by dis as fake instructions, + # but some tests assume that do. We should rewrite the tests to assume + # the new API, but it will be clearer to keep the tests working as + # before and do that in a separate PR. + + for instr in instrs: + yield instr + if not show_caches: + continue + + offset = instr.offset + for name, size, data in (instr.cache_info or ()): + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + + yield Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst b/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst new file mode 100644 index 00000000000000..b99e6bc90ae791 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst @@ -0,0 +1,3 @@ +:mod:`dis` module functions add cache information to the +:class:`~dis.Instruction` instance rather than creating fake +:class:`~dis.Instruction` instances to represent the cache entries. From 2a3c37c273762d2dc40bfdae3899cb09b7c88d4a Mon Sep 17 00:00:00 2001 From: Sequew <88668176+KrySeyt@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:04:17 +0300 Subject: [PATCH 40/95] [pprint]: Add docstring about `PrettyPrinter.underscore_numbers` parameter (#112963) Co-authored-by: Kirill Podoprigora --- Lib/pprint.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/pprint.py b/Lib/pprint.py index 34ed12637e2288..9314701db340c7 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -128,6 +128,9 @@ def __init__(self, indent=1, width=80, depth=None, stream=None, *, sort_dicts If true, dict keys are sorted. + underscore_numbers + If true, digit groups are separated with underscores. + """ indent = int(indent) width = int(width) From 9263173280d7ce949911965efa5b745287c581b2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 13 Dec 2023 12:31:41 +0000 Subject: [PATCH 41/95] Fix whitespace in generated code --- Python/executor_cases.c.h | 12 ++++++------ Tools/cases_generator/tier2_generator.py | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9dda3c9a743258..14d9dd6e95e533 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1842,7 +1842,7 @@ PyObject *null = NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject * descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2546,7 +2546,7 @@ PyObject *self = NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject * descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -2566,7 +2566,7 @@ PyObject *self = NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject * descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2585,7 +2585,7 @@ PyObject *attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject * descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2601,7 +2601,7 @@ PyObject *attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject * descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2630,7 +2630,7 @@ PyObject *self = NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject * descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index d2b503961aa7b9..a22fb6dd932503 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -123,10 +123,11 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: for cache in uop.caches: if cache.name != "unused": if cache.size == 4: - type = "PyObject *" + type = cast ="PyObject *" else: - type = f"uint{cache.size*16}_t" - out.emit(f"{type} {cache.name} = ({type})CURRENT_OPERAND();\n") + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS) if uop.properties.stores_sp: for i, var in enumerate(uop.stack.outputs): From 498a096a51a215cd3084845131e619222b906b3e Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 13 Dec 2023 14:00:34 +0000 Subject: [PATCH 42/95] gh-112205: Support `@setter` annotation from AC (gh-112922) --------- Co-authored-by: Erlend E. Aasland Co-authored-by: Alex Waygood --- Lib/test/clinic.test.c | 34 ++++++++++++-- Lib/test/test_clinic.py | 52 +++++++++++++++++++++ Modules/_io/bufferedio.c | 18 ++++---- Modules/_io/clinic/bufferedio.c.h | 26 ++++++++--- Modules/_io/clinic/stringio.c.h | 26 ++++++++--- Modules/_io/clinic/textio.c.h | 46 ++++++++++++++++++- Modules/_io/stringio.c | 6 +-- Modules/_io/textio.c | 43 ++++++++---------- Tools/clinic/clinic.py | 75 +++++++++++++++++++++++++++---- 9 files changed, 262 insertions(+), 64 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index ee4a4228fd28be..a6a21664bb82a1 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4956,8 +4956,12 @@ Test_meth_coexist_impl(TestObj *self) Test.property [clinic start generated code]*/ -#define TEST_PROPERTY_GETTERDEF \ - {"property", (getter)Test_property_get, NULL, NULL}, +#if defined(TEST_PROPERTY_GETSETDEF) +# undef TEST_PROPERTY_GETSETDEF +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, (setter)Test_property_set, NULL}, +#else +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, NULL, NULL}, +#endif static PyObject * Test_property_get_impl(TestObj *self); @@ -4970,8 +4974,32 @@ Test_property_get(TestObj *self, void *Py_UNUSED(context)) static PyObject * Test_property_get_impl(TestObj *self) -/*[clinic end generated code: output=892b6fb351ff85fd input=2d92b3449fbc7d2b]*/ +/*[clinic end generated code: output=af8140b692e0e2f1 input=2d92b3449fbc7d2b]*/ + +/*[clinic input] +@setter +Test.property +[clinic start generated code]*/ + +#if defined(TEST_PROPERTY_GETSETDEF) +# undef TEST_PROPERTY_GETSETDEF +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, (setter)Test_property_set, NULL}, +#else +# define TEST_PROPERTY_GETSETDEF {"property", NULL, (setter)Test_property_set, NULL}, +#endif + +static int +Test_property_set_impl(TestObj *self, PyObject *value); + +static int +Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + return Test_property_set_impl(self, value); +} +static int +Test_property_set_impl(TestObj *self, PyObject *value) +/*[clinic end generated code: output=f3eba6487d7550e2 input=3bc3f46a23c83a88]*/ /*[clinic input] output push diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f53e9481083106..d3dbde88dd82a9 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2197,6 +2197,58 @@ class Foo "" "" expected_error = err_template.format(invalid_kind) self.expect_failure(block, expected_error, lineno=3) + def test_invalid_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property -> int + """ + expected_error = f"{annotation} method cannot define a return type" + self.expect_failure(block, expected_error, lineno=3) + + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property + obj: int + / + """ + expected_error = f"{annotation} method cannot define parameters" + self.expect_failure(block, expected_error) + + def test_duplicate_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + {annotation} + Foo.property -> int + """ + expected_error = f"Cannot apply {annotation} twice to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getter_and_setter_disallowed_on_same_function(self): + dup_annotations = [("@getter", "@setter"), ("@setter", "@getter")] + for dup in dup_annotations: + with self.subTest(dup=dup): + block = f""" + module foo + class Foo "" "" + {dup[0]} + {dup[1]} + Foo.property -> int + """ + expected_error = "Cannot apply both @getter and @setter to the same function!" + self.expect_failure(block, expected_error, lineno=3) + def test_duplicate_coexist(self): err = "Called @coexist twice" block = """ diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 679626863c385c..f02207ace9f3d2 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2526,9 +2526,9 @@ static PyMemberDef bufferedreader_members[] = { }; static PyGetSetDef bufferedreader_getset[] = { - _IO__BUFFERED_CLOSED_GETTERDEF - _IO__BUFFERED_NAME_GETTERDEF - _IO__BUFFERED_MODE_GETTERDEF + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; @@ -2586,9 +2586,9 @@ static PyMemberDef bufferedwriter_members[] = { }; static PyGetSetDef bufferedwriter_getset[] = { - _IO__BUFFERED_CLOSED_GETTERDEF - _IO__BUFFERED_NAME_GETTERDEF - _IO__BUFFERED_MODE_GETTERDEF + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; @@ -2704,9 +2704,9 @@ static PyMemberDef bufferedrandom_members[] = { }; static PyGetSetDef bufferedrandom_getset[] = { - _IO__BUFFERED_CLOSED_GETTERDEF - _IO__BUFFERED_NAME_GETTERDEF - _IO__BUFFERED_MODE_GETTERDEF + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 69d28ad00c2ad5..ec46d5409a3d82 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -327,8 +327,12 @@ _io__Buffered_simple_flush(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#define _IO__BUFFERED_CLOSED_GETTERDEF \ - {"closed", (getter)_io__Buffered_closed_get, NULL, NULL}, +#if defined(_IO__BUFFERED_CLOSED_GETSETDEF) +# undef _IO__BUFFERED_CLOSED_GETSETDEF +# define _IO__BUFFERED_CLOSED_GETSETDEF {"closed", (getter)_io__Buffered_closed_get, (setter)_io__Buffered_closed_set, NULL}, +#else +# define _IO__BUFFERED_CLOSED_GETSETDEF {"closed", (getter)_io__Buffered_closed_get, NULL, NULL}, +#endif static PyObject * _io__Buffered_closed_get_impl(buffered *self); @@ -460,8 +464,12 @@ _io__Buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#define _IO__BUFFERED_NAME_GETTERDEF \ - {"name", (getter)_io__Buffered_name_get, NULL, NULL}, +#if defined(_IO__BUFFERED_NAME_GETSETDEF) +# undef _IO__BUFFERED_NAME_GETSETDEF +# define _IO__BUFFERED_NAME_GETSETDEF {"name", (getter)_io__Buffered_name_get, (setter)_io__Buffered_name_set, NULL}, +#else +# define _IO__BUFFERED_NAME_GETSETDEF {"name", (getter)_io__Buffered_name_get, NULL, NULL}, +#endif static PyObject * _io__Buffered_name_get_impl(buffered *self); @@ -478,8 +486,12 @@ _io__Buffered_name_get(buffered *self, void *Py_UNUSED(context)) return return_value; } -#define _IO__BUFFERED_MODE_GETTERDEF \ - {"mode", (getter)_io__Buffered_mode_get, NULL, NULL}, +#if defined(_IO__BUFFERED_MODE_GETSETDEF) +# undef _IO__BUFFERED_MODE_GETSETDEF +# define _IO__BUFFERED_MODE_GETSETDEF {"mode", (getter)_io__Buffered_mode_get, (setter)_io__Buffered_mode_set, NULL}, +#else +# define _IO__BUFFERED_MODE_GETSETDEF {"mode", (getter)_io__Buffered_mode_get, NULL, NULL}, +#endif static PyObject * _io__Buffered_mode_get_impl(buffered *self); @@ -1218,4 +1230,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=f21ed03255032b43 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0999c33f666dc692 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index ed505ae67589a8..fc2962d1c9c9a7 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -475,8 +475,12 @@ _io_StringIO___setstate__(stringio *self, PyObject *state) return return_value; } -#define _IO_STRINGIO_CLOSED_GETTERDEF \ - {"closed", (getter)_io_StringIO_closed_get, NULL, NULL}, +#if defined(_IO_STRINGIO_CLOSED_GETSETDEF) +# undef _IO_STRINGIO_CLOSED_GETSETDEF +# define _IO_STRINGIO_CLOSED_GETSETDEF {"closed", (getter)_io_StringIO_closed_get, (setter)_io_StringIO_closed_set, NULL}, +#else +# define _IO_STRINGIO_CLOSED_GETSETDEF {"closed", (getter)_io_StringIO_closed_get, NULL, NULL}, +#endif static PyObject * _io_StringIO_closed_get_impl(stringio *self); @@ -493,8 +497,12 @@ _io_StringIO_closed_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -#define _IO_STRINGIO_LINE_BUFFERING_GETTERDEF \ - {"line_buffering", (getter)_io_StringIO_line_buffering_get, NULL, NULL}, +#if defined(_IO_STRINGIO_LINE_BUFFERING_GETSETDEF) +# undef _IO_STRINGIO_LINE_BUFFERING_GETSETDEF +# define _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {"line_buffering", (getter)_io_StringIO_line_buffering_get, (setter)_io_StringIO_line_buffering_set, NULL}, +#else +# define _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {"line_buffering", (getter)_io_StringIO_line_buffering_get, NULL, NULL}, +#endif static PyObject * _io_StringIO_line_buffering_get_impl(stringio *self); @@ -511,8 +519,12 @@ _io_StringIO_line_buffering_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -#define _IO_STRINGIO_NEWLINES_GETTERDEF \ - {"newlines", (getter)_io_StringIO_newlines_get, NULL, NULL}, +#if defined(_IO_STRINGIO_NEWLINES_GETSETDEF) +# undef _IO_STRINGIO_NEWLINES_GETSETDEF +# define _IO_STRINGIO_NEWLINES_GETSETDEF {"newlines", (getter)_io_StringIO_newlines_get, (setter)_io_StringIO_newlines_set, NULL}, +#else +# define _IO_STRINGIO_NEWLINES_GETSETDEF {"newlines", (getter)_io_StringIO_newlines_get, NULL, NULL}, +#endif static PyObject * _io_StringIO_newlines_get_impl(stringio *self); @@ -528,4 +540,4 @@ _io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=3a92e8b6c322f61b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=27726751d98ab617 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 675e0ed2eab75e..a492f340c74c0d 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -1047,4 +1047,48 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) return return_value; } -/*[clinic end generated code: output=8781a91be6d99e2c input=a9049054013a1b77]*/ + +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) +# undef _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, NULL}, +#else +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, NULL, NULL}, +#endif + +static PyObject * +_io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper__CHUNK_SIZE_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper__CHUNK_SIZE_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) +# undef _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, NULL}, +#else +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", NULL, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, NULL}, +#endif + +static int +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value); + +static int +_io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper__CHUNK_SIZE_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=b312f2d2e2221580 input=a9049054013a1b77]*/ diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 74dcee23730306..06bc2679e8e227 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1037,15 +1037,15 @@ static struct PyMethodDef stringio_methods[] = { }; static PyGetSetDef stringio_getset[] = { - _IO_STRINGIO_CLOSED_GETTERDEF - _IO_STRINGIO_NEWLINES_GETTERDEF + _IO_STRINGIO_CLOSED_GETSETDEF + _IO_STRINGIO_NEWLINES_GETSETDEF /* (following comments straight off of the original Python wrapper:) XXX Cruft to support the TextIOWrapper API. This would only be meaningful if StringIO supported the buffer attribute. Hopefully, a better solution, than adding these pseudo-attributes, will be found. */ - _IO_STRINGIO_LINE_BUFFERING_GETTERDEF + _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {NULL} }; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 545f467b7f0257..c76d92cdd38b9a 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -3238,33 +3238,37 @@ textiowrapper_errors_get(textio *self, void *context) return result; } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper._CHUNK_SIZE +[clinic start generated code]*/ + static PyObject * -textiowrapper_chunk_size_get_impl(textio *self, void *context) +_io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self) +/*[clinic end generated code: output=039925cd2df375bc input=e9715b0e06ff0fa6]*/ { CHECK_ATTACHED(self); return PyLong_FromSsize_t(self->chunk_size); } -static PyObject * -textiowrapper_chunk_size_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_chunk_size_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@setter +_io.TextIOWrapper._CHUNK_SIZE +[clinic start generated code]*/ static int -textiowrapper_chunk_size_set_impl(textio *self, PyObject *arg, void *context) +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value) +/*[clinic end generated code: output=edb86d2db660a5ab input=32fc99861db02a0a]*/ { Py_ssize_t n; CHECK_ATTACHED_INT(self); - if (arg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - n = PyNumber_AsSsize_t(arg, PyExc_ValueError); + n = PyNumber_AsSsize_t(value, PyExc_ValueError); if (n == -1 && PyErr_Occurred()) return -1; if (n <= 0) { @@ -3276,16 +3280,6 @@ textiowrapper_chunk_size_set_impl(textio *self, PyObject *arg, void *context) return 0; } -static int -textiowrapper_chunk_size_set(textio *self, PyObject *arg, void *context) -{ - int result = 0; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_chunk_size_set_impl(self, arg, context); - Py_END_CRITICAL_SECTION(); - return result; -} - static PyMethodDef incrementalnewlinedecoder_methods[] = { _IO_INCREMENTALNEWLINEDECODER_DECODE_METHODDEF _IO_INCREMENTALNEWLINEDECODER_GETSTATE_METHODDEF @@ -3361,8 +3355,7 @@ static PyGetSetDef textiowrapper_getset[] = { */ {"newlines", (getter)textiowrapper_newlines_get, NULL, NULL}, {"errors", (getter)textiowrapper_errors_get, NULL, NULL}, - {"_CHUNK_SIZE", (getter)textiowrapper_chunk_size_get, - (setter)textiowrapper_chunk_size_set, NULL}, + _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {NULL} }; diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 816ce0e6efed61..5ec088765f3e01 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -850,6 +850,10 @@ class CLanguage(Language): static PyObject * {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) """) + PARSER_PROTOTYPE_SETTER: Final[str] = normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) + """) METH_O_PROTOTYPE: Final[str] = normalize_snippet(""" static PyObject * {c_basename}({impl_parameters}) @@ -870,8 +874,20 @@ class CLanguage(Language): {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, """) GETTERDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" - #define {getter_name} \ - {{"{name}", (getter){c_basename}, NULL, NULL}}, + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, NULL}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, NULL}}, + #endif + """) + SETTERDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, NULL}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, + #endif """) METHODDEF_PROTOTYPE_IFNDEF: Final[str] = normalize_snippet(""" #ifndef {methoddef_name} @@ -1172,6 +1188,10 @@ def output_templates( elif f.kind is GETTER: methoddef_define = self.GETTERDEF_PROTOTYPE_DEFINE docstring_prototype = docstring_definition = '' + elif f.kind is SETTER: + return_value_declaration = "int {return_value};" + methoddef_define = self.SETTERDEF_PROTOTYPE_DEFINE + docstring_prototype = docstring_prototype = docstring_definition = '' else: docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR @@ -1226,12 +1246,19 @@ def parser_body( limited_capi = False parsearg: str | None + if f.kind in {GETTER, SETTER} and parameters: + fail(f"@{f.kind.name.lower()} method cannot define parameters") + if not parameters: parser_code: list[str] | None if f.kind is GETTER: flags = "" # This should end up unused parser_prototype = self.PARSER_PROTOTYPE_GETTER parser_code = [] + elif f.kind is SETTER: + flags = "" + parser_prototype = self.PARSER_PROTOTYPE_SETTER + parser_code = [] elif not requires_defining_class: # no parameters, METH_NOARGS flags = "METH_NOARGS" @@ -1944,9 +1971,16 @@ def render_function( full_name = f.full_name template_dict = {'full_name': full_name} template_dict['name'] = f.displayname - if f.kind is GETTER: - template_dict['getter_name'] = f.c_basename.upper() + "_GETTERDEF" - template_dict['c_basename'] = f.c_basename + "_get" + if f.kind in {GETTER, SETTER}: + template_dict['getset_name'] = f.c_basename.upper() + template_dict['getset_basename'] = f.c_basename + if f.kind is GETTER: + template_dict['c_basename'] = f.c_basename + "_get" + elif f.kind is SETTER: + template_dict['c_basename'] = f.c_basename + "_set" + # Implicitly add the setter value parameter. + data.impl_parameters.append("PyObject *value") + data.impl_arguments.append("value") else: template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" template_dict['c_basename'] = f.c_basename @@ -1959,7 +1993,11 @@ def render_function( converter.set_template_dict(template_dict) f.return_converter.render(f, data) - template_dict['impl_return_type'] = f.return_converter.type + if f.kind is SETTER: + # All setters return an int. + template_dict['impl_return_type'] = 'int' + else: + template_dict['impl_return_type'] = f.return_converter.type template_dict['declarations'] = format_escape("\n".join(data.declarations)) template_dict['initializers'] = "\n\n".join(data.initializers) @@ -2954,6 +2992,7 @@ class FunctionKind(enum.Enum): METHOD_INIT = enum.auto() METHOD_NEW = enum.auto() GETTER = enum.auto() + SETTER = enum.auto() @functools.cached_property def new_or_init(self) -> bool: @@ -2970,6 +3009,7 @@ def __repr__(self) -> str: METHOD_INIT: Final = FunctionKind.METHOD_INIT METHOD_NEW: Final = FunctionKind.METHOD_NEW GETTER: Final = FunctionKind.GETTER +SETTER: Final = FunctionKind.SETTER ParamDict = dict[str, "Parameter"] ReturnConverterType = Callable[..., "CReturnConverter"] @@ -3056,7 +3096,7 @@ def methoddef_flags(self) -> str | None: case FunctionKind.STATIC_METHOD: flags.append('METH_STATIC') case _ as kind: - acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER} + acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} assert kind in acceptable_kinds, f"unknown kind: {kind!r}" if self.coexist: flags.append('METH_COEXIST') @@ -4702,7 +4742,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st def correct_name_for_self( f: Function ) -> tuple[str, str]: - if f.kind in {CALLABLE, METHOD_INIT, GETTER}: + if f.kind in {CALLABLE, METHOD_INIT, GETTER, SETTER}: if f.cls: return "PyObject *", "self" return "PyObject *", "module" @@ -5335,7 +5375,22 @@ def at_critical_section(self, *args: str) -> None: self.critical_section = True def at_getter(self) -> None: - self.kind = GETTER + match self.kind: + case FunctionKind.GETTER: + fail("Cannot apply @getter twice to the same function!") + case FunctionKind.SETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.GETTER + + def at_setter(self) -> None: + match self.kind: + case FunctionKind.SETTER: + fail("Cannot apply @setter twice to the same function!") + case FunctionKind.GETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.SETTER def at_staticmethod(self) -> None: if self.kind is not CALLABLE: @@ -5536,6 +5591,8 @@ def state_modulename_name(self, line: str) -> None: return_converter = None if returns: + if self.kind in {GETTER, SETTER}: + fail(f"@{self.kind.name.lower()} method cannot define a return type") ast_input = f"def x() -> {returns}: pass" try: module_node = ast.parse(ast_input) From 79dad03747fe17634136209f1bcaf346a8c10617 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 13 Dec 2023 15:38:45 +0000 Subject: [PATCH 43/95] gh-111650: Ensure pyconfig.h includes Py_GIL_DISABLED on Windows (GH-112778) --- .github/workflows/reusable-windows.yml | 6 ++-- Lib/sysconfig/__init__.py | 11 ++++++- Lib/test/test_sysconfig.py | 8 +++-- Lib/test/test_venv.py | 18 ++++++----- ...-12-05-22-56-30.gh-issue-111650.xlWmvM.rst | 3 ++ Modules/_ctypes/_ctypes_test.c | 2 -- Modules/_scproxy.c | 2 -- Modules/_stat.c | 2 -- Modules/_testcapi/heaptype_relative.c | 2 -- Modules/_testcapi/vectorcall_limited.c | 2 -- Modules/_testclinic_limited.c | 2 -- Modules/_testimportmultiple.c | 2 -- Modules/_uuidmodule.c | 2 -- Modules/errnomodule.c | 2 -- Modules/resource.c | 2 -- Modules/xxlimited.c | 2 -- Modules/xxlimited_35.c | 2 -- PC/layout/main.py | 8 +++-- PC/{pyconfig.h => pyconfig.h.in} | 3 ++ PC/winsound.c | 2 ++ PCbuild/_freeze_module.vcxproj | 30 ++++++++++++++++++ PCbuild/pyproject.props | 6 ++-- PCbuild/pythoncore.vcxproj | 31 ++++++++++++++++++- Tools/msi/dev/dev_files.wxs | 2 +- Tools/peg_generator/pegen/build.py | 4 +++ 25 files changed, 112 insertions(+), 44 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst rename PC/{pyconfig.h => pyconfig.h.in} (99%) diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 29e0a7e35b5450..47a3c10d2ca4c1 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -33,7 +33,7 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -50,4 +50,4 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }} diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 2a7fa45be079de..c60c9f3440615b 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -404,7 +404,16 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + # This ought to be as simple as dirname(sys._base_executable), but + # if a venv uses symlinks to a build in the source tree, then this + # fails. So instead we guess the subdirectory name from sys.winver + if sys.winver.endswith('-32'): + arch = 'win32' + elif sys.winver.endswith('-arm64'): + arch = 'arm64' + else: + arch = 'amd64' + inc_dir = os.path.join(_PROJECT_BASE, 'PCbuild', arch) else: inc_dir = _PROJECT_BASE else: diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2a6813f00bccc6..a19c04b1b2cde5 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -472,11 +472,15 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # /PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + # /PC/pyconfig.h.in always exists even if unused + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + # /pyconfig.h exists on Windows in a build tree + pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 890672c5d27eec..617d14dcb9c5fe 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -46,14 +46,18 @@ def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding=encoding) + stderr=subprocess.PIPE) out, err = p.communicate() if p.returncode: if verbose and err: - print(err.decode('utf-8', 'backslashreplace')) + print(err.decode(encoding or 'utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) + if encoding: + return ( + out.decode(encoding, 'backslashreplace'), + err.decode(encoding, 'backslashreplace'), + ) return out, err class BaseTest(unittest.TestCase): @@ -281,8 +285,8 @@ def test_sysconfig(self): ('get_config_h_filename()', sysconfig.get_config_h_filename())): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) @requireVenvCreate @unittest.skipUnless(can_symlink(), 'Needs symlinks') @@ -303,8 +307,8 @@ def test_sysconfig_symlinks(self): ('get_config_h_filename()', sysconfig.get_config_h_filename())): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst b/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst new file mode 100644 index 00000000000000..5a3493356e30be --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst @@ -0,0 +1,3 @@ +Ensures the ``Py_GIL_DISABLED`` preprocessor variable is defined in +:file:`pyconfig.h` so that extension modules written in C are able to use +it. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index fc9fc131f6249a..2681b9c58ecb9d 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 7920d2c2b8739d..fe82e918677f9a 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,9 +3,7 @@ * using the SystemConfiguration framework. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/_stat.c b/Modules/_stat.c index 1ef1e97f4b7dca..80f8a92668976b 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -11,9 +11,7 @@ * */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.13 for PyModule_Add() on Windows diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index 52286f05f7154c..52bda75736b316 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x030c0000 // 3.12 diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index 0a650f1b351d2d..d7b8d33b7f7162 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,8 +1,6 @@ /* Test Vectorcall in the limited API */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x030c0000 // 3.12 diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 61bc84134458da..ef595be0b626db 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -4,9 +4,7 @@ #undef Py_BUILD_CORE_MODULE #undef Py_BUILD_CORE_BUILTIN -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // For now, only limited C API 3.13 is supported diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index 245e81b2dce7f8..7e6556ad400cde 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -4,9 +4,7 @@ * foo, bar), only the first one is called the same as the compiled file. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x03020000 diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index d8b211c632eef1..4b6852c0d0ec73 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,9 +3,7 @@ * DCE compatible Universally Unique Identifier library. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 8287edbfb47f6c..1100e9f6094352 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,8 +1,6 @@ /* Errno module */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/resource.c b/Modules/resource.c index a4b8f648c329e3..19020b8cc1b6db 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.13 for PySys_Audit() diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 19f61216255cfa..0bb5e12d7c3dd9 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,9 +62,7 @@ pass */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 867820a6cb93fa..754a368f77e940 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -5,9 +5,7 @@ * See the xxlimited module for an extension module template. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x03050000 diff --git a/PC/layout/main.py b/PC/layout/main.py index cb2e4878da26b1..accfd51dd978fb 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -73,7 +73,10 @@ def copy_if_modified(src, dest): ) if do_copy: - shutil.copy2(src, dest) + try: + shutil.copy2(src, dest) + except FileNotFoundError: + raise FileNotFoundError(src) from None def get_lib_layout(ns): @@ -208,8 +211,7 @@ def _c(d): for dest, src in rglob(ns.source / "Include", "**/*.h"): yield "include/{}".format(dest), src - src = ns.source / "PC" / "pyconfig.h" - yield "include/pyconfig.h", src + yield "include/pyconfig.h", ns.build / "pyconfig.h" for dest, src in get_tcltk_lib(ns): yield dest, src diff --git a/PC/pyconfig.h b/PC/pyconfig.h.in similarity index 99% rename from PC/pyconfig.h rename to PC/pyconfig.h.in index e6b368caffe280..d8f0a6be69c21a 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h.in @@ -739,4 +739,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 +/* Define if you want to disable the GIL */ +#undef Py_GIL_DISABLED + #endif /* !Py_CONFIG_H */ diff --git a/PC/winsound.c b/PC/winsound.c index b0e416cfec4699..7e4ebd90f50c2e 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,6 +35,8 @@ winsound.PlaySound(None, 0) */ +#include "pyconfig.h" // Py_GIL_DISABLED + #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #define Py_LIMITED_API 0x030c0000 diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a1c37e183f21c7..f8c5fafa561efa 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -89,6 +89,7 @@ Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) + $(IntDir);%(AdditionalIncludeDirectories) Disabled false @@ -257,6 +258,9 @@ + + + @@ -414,6 +418,32 @@ + + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + + + + + diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 0acc7045c39a26..68c0550f7603b7 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,6 +10,8 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) + + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -38,9 +40,8 @@ - $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) + $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) - Py_GIL_DISABLED=1;%(PreprocessorDefinitions) _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed @@ -60,6 +61,7 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) -d2pattern-opt-disable:-932189325 %(AdditionalOptions) + /sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions) OnlyExplicitInline diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 778fc834c0db9c..90aa8cf28f8c5d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -378,7 +378,7 @@ - + @@ -646,6 +646,35 @@ + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + + + + + + + + + + + git diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be5..4357dc86d9d356 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ - + diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 30bfb31471c7b2..7df39a3b0ae48e 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -143,6 +143,10 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"), ] + if sys.platform == "win32": + # HACK: The location of pyconfig.h has moved within our build, and + # setuptools hasn't updated for it yet. So add the path manually for now + include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent) extension = Extension( extension_name, sources=[generated_source_path], From 6644ca45cde9ca1b80513a90dacccfeea2d98620 Mon Sep 17 00:00:00 2001 From: Diego Russo Date: Wed, 13 Dec 2023 16:08:15 +0000 Subject: [PATCH 44/95] gh-110190: Fix ctypes structs with array on PPCLE64 (GH-112959) Fix the same issue of PR #112604 on PPC64LE platform Refactor tests to make easier to add more platfroms if needed. --- Lib/test/test_ctypes/test_structures.py | 232 +++++++----------- ...-12-11-14-12-46.gh-issue-110190.e0iEUa.rst | 1 + Modules/_ctypes/_ctypes_test.c | 138 ++++++++--- Modules/_ctypes/stgdict.c | 45 ++-- 4 files changed, 209 insertions(+), 207 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 57ae9240b3c165..21039f04947507 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -9,6 +9,7 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) from struct import calcsize +from collections import namedtuple from test import support @@ -474,36 +475,53 @@ class X(Structure): def test_array_in_struct(self): # See bpo-22273 + # Load the shared library + dll = CDLL(_ctypes_test.__file__) + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c class Test2(Structure): _fields_ = [ ('data', c_ubyte * 16), ] - class Test3(Structure): + class Test3AParent(Structure): + _fields_ = [ + ('data', c_float * 2), + ] + + class Test3A(Test3AParent): + _fields_ = [ + ('more_data', c_float * 2), + ] + + class Test3B(Structure): _fields_ = [ ('data', c_double * 2), ] - class Test3A(Structure): + class Test3C(Structure): _fields_ = [ - ('data', c_float * 2), + ("data", c_double * 4) ] - class Test3B(Test3A): + class Test3D(Structure): _fields_ = [ - ('more_data', c_float * 2), + ("data", c_double * 8) + ] + + class Test3E(Structure): + _fields_ = [ + ("data", c_double * 9) ] - # Load the shared library - dll = CDLL(_ctypes_test.__file__) + # Tests for struct Test2 s = Test2() expected = 0 for i in range(16): s.data[i] = i expected += i - func = dll._testfunc_array_in_struct1 + func = dll._testfunc_array_in_struct2 func.restype = c_int func.argtypes = (Test2,) result = func(s) @@ -512,29 +530,16 @@ class Test3B(Test3A): for i in range(16): self.assertEqual(s.data[i], i) - s = Test3() - s.data[0] = 3.14159 - s.data[1] = 2.71828 - expected = 3.14159 + 2.71828 - func = dll._testfunc_array_in_struct2 - func.restype = c_double - func.argtypes = (Test3,) - result = func(s) - self.assertEqual(result, expected) - # check the passed-in struct hasn't changed - self.assertEqual(s.data[0], 3.14159) - self.assertEqual(s.data[1], 2.71828) - - s = Test3B() + # Tests for struct Test3A + s = Test3A() s.data[0] = 3.14159 s.data[1] = 2.71828 s.more_data[0] = -3.0 s.more_data[1] = -2.0 - - expected = 3.14159 + 2.71828 - 5.0 - func = dll._testfunc_array_in_struct2a + expected = 3.14159 + 2.71828 - 3.0 - 2.0 + func = dll._testfunc_array_in_struct3A func.restype = c_double - func.argtypes = (Test3B,) + func.argtypes = (Test3A,) result = func(s) self.assertAlmostEqual(result, expected, places=6) # check the passed-in struct hasn't changed @@ -543,129 +548,60 @@ class Test3B(Test3A): self.assertAlmostEqual(s.more_data[0], -3.0, places=6) self.assertAlmostEqual(s.more_data[1], -2.0, places=6) - @unittest.skipIf( - 'ppc64le' in platform.uname().machine, - "gh-110190: currently fails on ppc64le", - ) - def test_array_in_struct_registers(self): - dll = CDLL(_ctypes_test.__file__) - - class Test3C1(Structure): - _fields_ = [ - ("data", c_double * 4) - ] - - class DataType4(Array): - _type_ = c_double - _length_ = 4 - - class Test3C2(Structure): - _fields_ = [ - ("data", DataType4) - ] - - class Test3C3(Structure): - _fields_ = [ - ("x", c_double), - ("y", c_double), - ("z", c_double), - ("t", c_double) - ] - - class Test3D1(Structure): - _fields_ = [ - ("data", c_double * 5) - ] - - class DataType5(Array): - _type_ = c_double - _length_ = 5 - - class Test3D2(Structure): - _fields_ = [ - ("data", DataType5) - ] - - class Test3D3(Structure): - _fields_ = [ - ("x", c_double), - ("y", c_double), - ("z", c_double), - ("t", c_double), - ("u", c_double) - ] - - # Tests for struct Test3C - expected = (1.0, 2.0, 3.0, 4.0) - func = dll._testfunc_array_in_struct_set_defaults_3C - func.restype = Test3C1 - result = func() - # check the default values have been set properly - self.assertEqual( - (result.data[0], - result.data[1], - result.data[2], - result.data[3]), - expected + # Test3B, Test3C, Test3D, Test3E have the same logic with different + # sizes hence putting them in a loop. + StructCtype = namedtuple( + "StructCtype", + ["cls", "cfunc1", "cfunc2", "items"] ) - - func = dll._testfunc_array_in_struct_set_defaults_3C - func.restype = Test3C2 - result = func() - # check the default values have been set properly - self.assertEqual( - (result.data[0], - result.data[1], - result.data[2], - result.data[3]), - expected - ) - - func = dll._testfunc_array_in_struct_set_defaults_3C - func.restype = Test3C3 - result = func() - # check the default values have been set properly - self.assertEqual((result.x, result.y, result.z, result.t), expected) - - # Tests for struct Test3D - expected = (1.0, 2.0, 3.0, 4.0, 5.0) - func = dll._testfunc_array_in_struct_set_defaults_3D - func.restype = Test3D1 - result = func() - # check the default values have been set properly - self.assertEqual( - (result.data[0], - result.data[1], - result.data[2], - result.data[3], - result.data[4]), - expected - ) - - func = dll._testfunc_array_in_struct_set_defaults_3D - func.restype = Test3D2 - result = func() - # check the default values have been set properly - self.assertEqual( - (result.data[0], - result.data[1], - result.data[2], - result.data[3], - result.data[4]), - expected - ) - - func = dll._testfunc_array_in_struct_set_defaults_3D - func.restype = Test3D3 - result = func() - # check the default values have been set properly - self.assertEqual( - (result.x, - result.y, - result.z, - result.t, - result.u), - expected) + structs_to_test = [ + StructCtype( + Test3B, + dll._testfunc_array_in_struct3B, + dll._testfunc_array_in_struct3B_set_defaults, + 2), + StructCtype( + Test3C, + dll._testfunc_array_in_struct3C, + dll._testfunc_array_in_struct3C_set_defaults, + 4), + StructCtype( + Test3D, + dll._testfunc_array_in_struct3D, + dll._testfunc_array_in_struct3D_set_defaults, + 8), + StructCtype( + Test3E, + dll._testfunc_array_in_struct3E, + dll._testfunc_array_in_struct3E_set_defaults, + 9), + ] + + for sut in structs_to_test: + s = sut.cls() + + # Test for cfunc1 + expected = 0 + for i in range(sut.items): + float_i = float(i) + s.data[i] = float_i + expected += float_i + func = sut.cfunc1 + func.restype = c_double + func.argtypes = (sut.cls,) + result = func(s) + self.assertEqual(result, expected) + # check the passed-in struct hasn't changed + for i in range(sut.items): + self.assertEqual(s.data[i], float(i)) + + # Test for cfunc2 + func = sut.cfunc2 + func.restype = sut.cls + result = func() + # check if the default values have been set correctly + for i in range(sut.items): + self.assertEqual(result.data[i], float(i+1)) def test_38368(self): class U(Union): diff --git a/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst b/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst new file mode 100644 index 00000000000000..3bfed1e0f1dc91 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst @@ -0,0 +1 @@ +Fix ctypes structs with array on PPC64LE platform by setting ``MAX_STRUCT_SIZE`` to 64 in stgdict. Patch by Diego Russo. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 2681b9c58ecb9d..ecc60417790417 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -96,11 +96,10 @@ typedef struct { } Test2; EXPORT(int) -_testfunc_array_in_struct1(Test2 in) +_testfunc_array_in_struct2(Test2 in) { int result = 0; - - for (unsigned i = 0; i < 16; i++) + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) result += in.data[i]; /* As the structure is passed by value, changes to it shouldn't be * reflected in the caller. @@ -109,22 +108,25 @@ _testfunc_array_in_struct1(Test2 in) return result; } -typedef struct { - double data[2]; -} Test3; - +/* + * Test3A struct test the MAX_STRUCT_SIZE 16 with single precision floats. + * These structs should be passed via registers on all platforms and they are + * used for within bounds tests. + */ typedef struct { float data[2]; float more_data[2]; -} Test3B; +} Test3A; EXPORT(double) -_testfunc_array_in_struct2(Test3 in) +_testfunc_array_in_struct3A(Test3A in) { double result = 0; - for (unsigned i = 0; i < 2; i++) + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) result += in.data[i]; + for (unsigned i = 0; i < sizeof(in.more_data)/sizeof(in.more_data[0]); i++) + result += in.more_data[i]; /* As the structure is passed by value, changes to it shouldn't be * reflected in the caller. */ @@ -132,56 +134,116 @@ _testfunc_array_in_struct2(Test3 in) return result; } +/* The structs Test3B..Test3E use the same functions hence using the MACRO + * to define their implementation. + */ + +#define _TESTFUNC_ARRAY_IN_STRUCT_IMPL \ + double result = 0; \ + \ + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) \ + result += in.data[i]; \ + /* As the structure is passed by value, changes to it shouldn't be \ + * reflected in the caller. \ + */ \ + memset(in.data, 0, sizeof(in.data)); \ + return result; \ + +#define _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL \ + for (unsigned i = 0; i < sizeof(s.data)/sizeof(s.data[0]); i++) \ + s.data[i] = (double)i+1; \ + return s; \ + + +/* + * Test3B struct test the MAX_STRUCT_SIZE 16 with double precision floats. + * These structs should be passed via registers on all platforms and they are + * used for within bounds tests. + */ +typedef struct { + double data[2]; +} Test3B; + EXPORT(double) -_testfunc_array_in_struct2a(Test3B in) +_testfunc_array_in_struct3B(Test3B in) { - double result = 0; + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} - for (unsigned i = 0; i < 2; i++) - result += in.data[i]; - for (unsigned i = 0; i < 2; i++) - result += in.more_data[i]; - /* As the structure is passed by value, changes to it shouldn't be - * reflected in the caller. - */ - memset(in.data, 0, sizeof(in.data)); - return result; +EXPORT(Test3B) +_testfunc_array_in_struct3B_set_defaults(void) +{ + Test3B s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL } /* - * See gh-110190. structs containing arrays of up to four floating point types - * (max 32 bytes) are passed in registers on Arm. + * Test3C struct tests the MAX_STRUCT_SIZE 32. Structs containing arrays of up + * to four floating point types are passed in registers on Arm platforms. + * This struct is used for within bounds test on Arm platfroms and for an + * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 32. + * See gh-110190. */ - typedef struct { double data[4]; } Test3C; +EXPORT(double) +_testfunc_array_in_struct3C(Test3C in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + EXPORT(Test3C) -_testfunc_array_in_struct_set_defaults_3C(void) +_testfunc_array_in_struct3C_set_defaults(void) { Test3C s; - s.data[0] = 1.0; - s.data[1] = 2.0; - s.data[2] = 3.0; - s.data[3] = 4.0; - return s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL } +/* + * Test3D struct tests the MAX_STRUCT_SIZE 64. Structs containing arrays of up + * to eight floating point types are passed in registers on PPC64LE platforms. + * This struct is used for within bounds test on PPC64LE platfroms and for an + * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 64. + * See gh-110190. + */ typedef struct { - double data[5]; + double data[8]; } Test3D; +EXPORT(double) +_testfunc_array_in_struct3D(Test3D in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + EXPORT(Test3D) -_testfunc_array_in_struct_set_defaults_3D(void) +_testfunc_array_in_struct3D_set_defaults(void) { Test3D s; - s.data[0] = 1.0; - s.data[1] = 2.0; - s.data[2] = 3.0; - s.data[3] = 4.0; - s.data[4] = 5.0; - return s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL +} + +/* + * Test3E struct tests the out-of-bounds for all architectures. + * See gh-110190. + */ +typedef struct { + double data[9]; +} Test3E; + +EXPORT(double) +_testfunc_array_in_struct3E(Test3E in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + +EXPORT(Test3E) +_testfunc_array_in_struct3E_set_defaults(void) +{ + Test3E s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL } typedef union { diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 04dd9bae32cd5e..dfdb96b0e7258a 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -698,13 +698,12 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct stgdict->length = len; /* ADD ffi_ofs? */ /* - * On Arm platforms, structs with at most 4 elements of any floating point - * type values can be passed through registers. If the type is double the - * maximum size of the struct is 32 bytes. - * By Arm platforms it is meant both 32 and 64-bit. -*/ + * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. + */ #if defined(__aarch64__) || defined(__arm__) # define MAX_STRUCT_SIZE 32 +#elif defined(__powerpc64__) +# define MAX_STRUCT_SIZE 64 #else # define MAX_STRUCT_SIZE 16 #endif @@ -715,24 +714,29 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * pointers, which is fine when an array name is being passed as * parameter, but not when passing structures by value that contain * arrays. - * On x86_64 Linux and Arm platforms, small structures passed by - * value are passed in registers, and in order to do this, libffi needs - * to know the true type of the array members of structs. Treating them - * as pointers breaks things. + * Small structures passed by value are passed in registers, and in + * order to do this, libffi needs to know the true type of the array + * members of structs. Treating them as pointers breaks things. + * + * Small structures have different sizes depending on the platform + * where Python is running on: + * + * * x86-64: 16 bytes or less + * * Arm platforms (both 32 and 64 bit): 32 bytes or less + * * PowerPC 64 Little Endian: 64 bytes or less * - * By small structures, we mean ones that are 16 bytes or less on - * x86-64 and 32 bytes or less on Arm. In that case, there can't be - * more than 16 or 32 elements after unrolling arrays, as we (will) - * disallow bitfields. So we can collect the true ffi_type values in - * a fixed-size local array on the stack and, if any arrays were seen, - * replace the ffi_type_pointer.elements with a more accurate set, - * to allow libffi to marshal them into registers correctly. + * In that case, there can't be more than 16, 32 or 64 elements after + * unrolling arrays, as we (will) disallow bitfields. + * So we can collect the true ffi_type values in a fixed-size local + * array on the stack and, if any arrays were seen, replace the + * ffi_type_pointer.elements with a more accurate set, to allow + * libffi to marshal them into registers correctly. * It means one more loop over the fields, but if we got here, * the structure is small, so there aren't too many of those. * - * Although the passing in registers is specific to x86_64 Linux - * and Arm platforms, the array-in-struct vs. pointer problem is - * general. But we restrict the type transformation to small structs + * Although the passing in registers is specific to the above + * platforms, the array-in-struct vs. pointer problem is general. + * But we restrict the type transformation to small structs * nonetheless. * * Note that although a union may be small in terms of memory usage, it @@ -759,8 +763,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * struct { uint_32 e1; uint_32 e2; ... uint_32 e_4; } f6; * } * - * The same principle applies for a struct 32 bytes in size like in - * the case of Arm platforms. + * The same principle applies for a struct 32 or 64 bytes in size. * * So the struct/union needs setting up as follows: all non-array * elements copied across as is, and all array elements replaced with From 480b4b359d710c74f31e2ffd7fc51750dcec7012 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Dec 2023 09:29:40 -0700 Subject: [PATCH 45/95] gh-76785: Fix CODEOWNERS (gh-113038) In gh-112982 I made some changes to .github/CODEOWNERS. Later, @ezio-melotti pointed out that some of those changes were unnecessary. --- .github/CODEOWNERS | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f8291d8689dd95..db28c2a231ae04 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -40,6 +40,7 @@ Lib/test/test_patma.py @brandtbucher Lib/test/test_peepholer.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Tools/c-analyzer/ @ericsnowcurrently # Exceptions Lib/traceback.py @iritkatriel @@ -78,11 +79,6 @@ Python/traceback.c @iritkatriel **/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw -# Subinterpreters -Lib/test/support/interpreters/** @ericsnowcurrently -Modules/_xx*interp*module.c @ericsnowcurrently -Lib/test/test_interpreters/** @ericsnowcurrently - # Dates and times **/*datetime* @pganssle @abalkin **/*str*time* @pganssle @abalkin @@ -153,15 +149,7 @@ Doc/c-api/stable.rst @encukou **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger -Doc/**/*queue* @rhettinger -PCbuild/**/*queue* @rhettinger -Modules/_queuemodule.c @rhettinger -Lib/*queue*.py @rhettinger -Lib/asyncio/*queue*.py @rhettinger -Lib/multiprocessing/*queue*.py @rhettinger -Lib/test/*queue*.py @rhettinger -Lib/test_asyncio/*queue*.py @rhettinger -Lib/test_multiprocessing/*queue*.py @rhettinger +**/*queue* @rhettinger **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger @@ -201,6 +189,11 @@ Lib/test_multiprocessing/*queue*.py @rhettinger /Lib/test/test_clinic.py @erlend-aasland @AlexWaygood Doc/howto/clinic.rst @erlend-aasland +# Subinterpreters +Lib/test/support/interpreters/ @ericsnowcurrently +Modules/_xx*interp*module.c @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently + # WebAssembly /Tools/wasm/ @brettcannon From 2111795d0c2dea0ade67d5d76f839102d68caa23 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 13 Dec 2023 12:11:52 -0600 Subject: [PATCH 46/95] Use match/case in grouper() recipe (gh-113059) Use match/case in grouper() reciper --- Doc/library/itertools.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 8a4254cf15ebe2..56c66f670c74dd 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -914,14 +914,15 @@ which incur interpreter overhead. # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF args = [iter(iterable)] * n - if incomplete == 'fill': - return zip_longest(*args, fillvalue=fillvalue) - elif incomplete == 'strict': - return zip(*args, strict=True) - elif incomplete == 'ignore': - return zip(*args) - else: - raise ValueError('Expected fill, strict, or ignore') + match incomplete: + case 'fill': + return zip_longest(*args, fillvalue=fillvalue) + case 'strict': + return zip(*args, strict=True) + case 'ignore': + return zip(*args) + case _: + raise ValueError('Expected fill, strict, or ignore') def sliding_window(iterable, n): # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG From d05a180350fe20d5fde56c7e525e394a0b282703 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 13 Dec 2023 18:59:36 +0000 Subject: [PATCH 47/95] gh-101100: Improve docs on exception attributes (GH-113057) * Improve docs on exception attributes * thanks sphinx-lint * fix doctests * argh, okay, give up on doctests * Various improvements --- Doc/c-api/exceptions.rst | 16 +++--- Doc/library/exceptions.rst | 94 +++++++++++++++++++++------------- Doc/library/traceback.rst | 30 ++++++----- Doc/reference/datamodel.rst | 3 +- Doc/reference/simple_stmts.rst | 28 +++++++--- Doc/whatsnew/3.0.rst | 11 ++-- 6 files changed, 115 insertions(+), 67 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a3a63b38c432f2..284a9c71e420da 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -541,7 +541,8 @@ Querying the error indicator .. note:: - This function *does not* implicitly set the ``__traceback__`` + This function *does not* implicitly set the + :attr:`~BaseException.__traceback__` attribute on the exception value. If setting the traceback appropriately is desired, the following additional snippet is needed:: @@ -753,7 +754,8 @@ Exception Objects .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no + accessible from Python through the :attr:`~BaseException.__traceback__` + attribute. If there is no traceback associated, this returns ``NULL``. @@ -767,8 +769,8 @@ Exception Objects Return the context (another exception instance during whose handling *ex* was raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. + Python through the :attr:`~BaseException.__context__` attribute. + If there is no context associated, this returns ``NULL``. .. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) @@ -782,7 +784,8 @@ Exception Objects Return the cause (either an exception instance, or ``None``, set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. + reference, as accessible from Python through the + :attr:`~BaseException.__cause__` attribute. .. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) @@ -791,7 +794,8 @@ Exception Objects it. There is no type check to make sure that *cause* is either an exception instance or ``None``. This steals a reference to *cause*. - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + The :attr:`~BaseException.__suppress_context__` attribute is implicitly set + to ``True`` by this function. .. c:function:: PyObject* PyException_GetArgs(PyObject *ex) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index b67215b8b3a362..04686b6db0036c 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -38,36 +38,48 @@ information on defining exceptions is available in the Python Tutorial under Exception context ----------------- -When raising a new exception while another exception -is already being handled, the new exception's -:attr:`__context__` attribute is automatically set to the handled -exception. An exception may be handled when an :keyword:`except` or -:keyword:`finally` clause, or a :keyword:`with` statement, is used. - -This implicit exception context can be -supplemented with an explicit cause by using :keyword:`!from` with -:keyword:`raise`:: - - raise new_exc from original_exc - -The expression following :keyword:`from` must be an exception or ``None``. It -will be set as :attr:`__cause__` on the raised exception. Setting -:attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` -attribute to ``True``, so that using ``raise new_exc from None`` -effectively replaces the old exception with the new one for display -purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while -leaving the old exception available in :attr:`__context__` for introspection -when debugging. - -The default traceback display code shows these chained exceptions in -addition to the traceback for the exception itself. An explicitly chained -exception in :attr:`__cause__` is always shown when present. An implicitly -chained exception in :attr:`__context__` is shown only if :attr:`__cause__` -is :const:`None` and :attr:`__suppress_context__` is false. - -In either case, the exception itself is always shown after any chained -exceptions so that the final line of the traceback always shows the last -exception that was raised. +.. index:: pair: exception; chaining + __cause__ (exception attribute) + __context__ (exception attribute) + __suppress_context__ (exception attribute) + +Three attributes on exception objects provide information about the context in +which an the exception was raised: + +.. attribute:: BaseException.__context__ + BaseException.__cause__ + BaseException.__suppress_context__ + + When raising a new exception while another exception + is already being handled, the new exception's + :attr:`!__context__` attribute is automatically set to the handled + exception. An exception may be handled when an :keyword:`except` or + :keyword:`finally` clause, or a :keyword:`with` statement, is used. + + This implicit exception context can be + supplemented with an explicit cause by using :keyword:`!from` with + :keyword:`raise`:: + + raise new_exc from original_exc + + The expression following :keyword:`from` must be an exception or ``None``. It + will be set as :attr:`!__cause__` on the raised exception. Setting + :attr:`!__cause__` also implicitly sets the :attr:`!__suppress_context__` + attribute to ``True``, so that using ``raise new_exc from None`` + effectively replaces the old exception with the new one for display + purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while + leaving the old exception available in :attr:`!__context__` for introspection + when debugging. + + The default traceback display code shows these chained exceptions in + addition to the traceback for the exception itself. An explicitly chained + exception in :attr:`!__cause__` is always shown when present. An implicitly + chained exception in :attr:`!__context__` is shown only if :attr:`!__cause__` + is :const:`None` and :attr:`!__suppress_context__` is false. + + In either case, the exception itself is always shown after any chained + exceptions so that the final line of the traceback always shows the last + exception that was raised. Inheriting from built-in exceptions @@ -126,6 +138,12 @@ The following exceptions are used mostly as base classes for other exceptions. tb = sys.exception().__traceback__ raise OtherException(...).with_traceback(tb) + .. attribute:: __traceback__ + + A writable field that holds the + :ref:`traceback object ` associated with this + exception. See also: :ref:`raise`. + .. method:: add_note(note) Add the string ``note`` to the exception's notes which appear in the standard @@ -929,8 +947,10 @@ their subgroups based on the types of the contained exceptions. true for the exceptions that should be in the subgroup. The nesting structure of the current exception is preserved in the result, - as are the values of its :attr:`message`, :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields. + as are the values of its :attr:`message`, + :attr:`~BaseException.__traceback__`, :attr:`~BaseException.__cause__`, + :attr:`~BaseException.__context__` and + :attr:`~BaseException.__notes__` fields. Empty nested groups are omitted from the result. The condition is checked for all exceptions in the nested exception group, @@ -956,10 +976,14 @@ their subgroups based on the types of the contained exceptions. and :meth:`split` return instances of the subclass rather than :exc:`ExceptionGroup`. - :meth:`subgroup` and :meth:`split` copy the :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields from + :meth:`subgroup` and :meth:`split` copy the + :attr:`~BaseException.__traceback__`, + :attr:`~BaseException.__cause__`, :attr:`~BaseException.__context__` and + :attr:`~BaseException.__notes__` fields from the original exception group to the one returned by :meth:`derive`, so - these fields do not need to be updated by :meth:`derive`. :: + these fields do not need to be updated by :meth:`derive`. + + .. doctest:: >>> class MyGroup(ExceptionGroup): ... def derive(self, excs): diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 14cade690fc773..dfbd04de243a3c 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -67,7 +67,8 @@ The module defines the following functions: The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the - :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be + :attr:`~BaseException.__cause__` or :attr:`~BaseException.__context__` + attributes of the exception) will be printed as well, like the interpreter itself does when printing an unhandled exception. @@ -234,10 +235,11 @@ capture data for later printing in a lightweight fashion. Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. - If *compact* is true, only data that is required by :class:`TracebackException`'s - ``format`` method is saved in the class attributes. In particular, the - ``__context__`` field is calculated only if ``__cause__`` is ``None`` and - ``__suppress_context__`` is false. + If *compact* is true, only data that is required by + :class:`!TracebackException`'s :meth:`format` method + is saved in the class attributes. In particular, the + :attr:`__context__` field is calculated only if :attr:`__cause__` is + ``None`` and :attr:`__suppress_context__` is false. Note that when locals are captured, they are also shown in the traceback. @@ -255,27 +257,31 @@ capture data for later printing in a lightweight fashion. .. attribute:: __cause__ - A :class:`TracebackException` of the original ``__cause__``. + A :class:`!TracebackException` of the original + :attr:`~BaseException.__cause__`. .. attribute:: __context__ - A :class:`TracebackException` of the original ``__context__``. + A :class:`!TracebackException` of the original + :attr:`~BaseException.__context__`. .. attribute:: exceptions If ``self`` represents an :exc:`ExceptionGroup`, this field holds a list of - :class:`TracebackException` instances representing the nested exceptions. + :class:`!TracebackException` instances representing the nested exceptions. Otherwise it is ``None``. .. versionadded:: 3.11 .. attribute:: __suppress_context__ - The ``__suppress_context__`` value from the original exception. + The :attr:`~BaseException.__suppress_context__` value from the original + exception. .. attribute:: __notes__ - The ``__notes__`` value from the original exception, or ``None`` + The :attr:`~BaseException.__notes__` value from the original exception, + or ``None`` if the exception does not have any notes. If it is not ``None`` is it formatted in the traceback after the exception string. @@ -349,8 +355,8 @@ capture data for later printing in a lightweight fashion. Format the exception. - If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not - be formatted. + If *chain* is not ``True``, :attr:`__cause__` and :attr:`__context__` + will not be formatted. The return value is a generator of strings, each ending in a newline and some containing internal newlines. :func:`~traceback.print_exception` diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 16ee3e300c2b8b..5e3757e1f5c6f6 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1390,7 +1390,8 @@ unwinds the execution stack, at each unwound level a traceback object is inserted in front of the current traceback. When an exception handler is entered, the stack trace is made available to the program. (See section :ref:`try`.) It is accessible as the third item of the -tuple returned by :func:`sys.exc_info`, and as the ``__traceback__`` attribute +tuple returned by :func:`sys.exc_info`, and as the +:attr:`~BaseException.__traceback__` attribute of the caught exception. When the program contains no suitable diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a9e65be1eda340..34c3a620b87223 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -578,7 +578,7 @@ The :dfn:`type` of the exception is the exception instance's class, the .. index:: pair: object; traceback A traceback object is normally created automatically when an exception is raised -and attached to it as the :attr:`__traceback__` attribute, which is writable. +and attached to it as the :attr:`~BaseException.__traceback__` attribute. You can create an exception and set your own traceback in one step using the :meth:`~BaseException.with_traceback` exception method (which returns the same exception instance, with its traceback set to its argument), like so:: @@ -592,11 +592,13 @@ same exception instance, with its traceback set to its argument), like so:: The ``from`` clause is used for exception chaining: if given, the second *expression* must be another exception class or instance. If the second expression is an exception instance, it will be attached to the raised -exception as the :attr:`__cause__` attribute (which is writable). If the +exception as the :attr:`~BaseException.__cause__` attribute (which is writable). If the expression is an exception class, the class will be instantiated and the resulting exception instance will be attached to the raised exception as the -:attr:`__cause__` attribute. If the raised exception is not handled, both -exceptions will be printed:: +:attr:`!__cause__` attribute. If the raised exception is not handled, both +exceptions will be printed: + +.. code-block:: pycon >>> try: ... print(1 / 0) @@ -605,19 +607,24 @@ exceptions will be printed:: ... Traceback (most recent call last): File "", line 2, in + print(1 / 0) + ~~^~~ ZeroDivisionError: division by zero The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 4, in + raise RuntimeError("Something bad happened") from exc RuntimeError: Something bad happened A similar mechanism works implicitly if a new exception is raised when an exception is already being handled. An exception may be handled when an :keyword:`except` or :keyword:`finally` clause, or a :keyword:`with` statement, is used. The previous exception is then -attached as the new exception's :attr:`__context__` attribute:: +attached as the new exception's :attr:`~BaseException.__context__` attribute: + +.. code-block:: pycon >>> try: ... print(1 / 0) @@ -626,16 +633,21 @@ attached as the new exception's :attr:`__context__` attribute:: ... Traceback (most recent call last): File "", line 2, in + print(1 / 0) + ~~^~~ ZeroDivisionError: division by zero During handling of the above exception, another exception occurred: Traceback (most recent call last): File "", line 4, in + raise RuntimeError("Something bad happened") RuntimeError: Something bad happened Exception chaining can be explicitly suppressed by specifying :const:`None` in -the ``from`` clause:: +the ``from`` clause: + +.. doctest:: >>> try: ... print(1 / 0) @@ -653,8 +665,8 @@ and information about handling exceptions is in section :ref:`try`. :const:`None` is now permitted as ``Y`` in ``raise X from Y``. .. versionadded:: 3.3 - The ``__suppress_context__`` attribute to suppress automatic display of the - exception context. + The :attr:`~BaseException.__suppress_context__` attribute to suppress + automatic display of the exception context. .. versionchanged:: 3.11 If the traceback of the active exception is modified in an :keyword:`except` diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 5953b32c6aaa18..89e12062abaddd 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -711,7 +711,7 @@ new powerful features added: {Exception}({args})` instead of :samp:`raise {Exception}, {args}`. Additionally, you can no longer explicitly specify a traceback; instead, if you *have* to do this, you can assign directly to the - :attr:`__traceback__` attribute (see below). + :attr:`~BaseException.__traceback__` attribute (see below). * :pep:`3110`: Catching exceptions. You must now use :samp:`except {SomeException} as {variable}` instead @@ -725,7 +725,7 @@ new powerful features added: handler block. This usually happens due to a bug in the handler block; we call this a *secondary* exception. In this case, the original exception (that was being handled) is saved as the - :attr:`__context__` attribute of the secondary exception. + :attr:`~BaseException.__context__` attribute of the secondary exception. Explicit chaining is invoked with this syntax:: raise SecondaryException() from primary_exception @@ -733,14 +733,15 @@ new powerful features added: (where *primary_exception* is any expression that produces an exception object, probably an exception that was previously caught). In this case, the primary exception is stored on the - :attr:`__cause__` attribute of the secondary exception. The + :attr:`~BaseException.__cause__` attribute of the secondary exception. The traceback printed when an unhandled exception occurs walks the chain - of :attr:`__cause__` and :attr:`__context__` attributes and prints a + of :attr:`!__cause__` and :attr:`~BaseException.__context__` attributes and + prints a separate traceback for each component of the chain, with the primary exception at the top. (Java users may recognize this behavior.) * :pep:`3134`: Exception objects now store their traceback as the - :attr:`__traceback__` attribute. This means that an exception + :attr:`~BaseException.__traceback__` attribute. This means that an exception object now contains all the information pertaining to an exception, and there are fewer reasons to use :func:`sys.exc_info` (though the latter is not removed). From 85923cb377c4a13720c135da9ae3ed93465a81e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 13 Dec 2023 20:37:13 +0100 Subject: [PATCH 48/95] Docs: Fix external link to devguide.python.org (GH-112899) --- Doc/whatsnew/3.8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 4574702b1a600f..e4dcb9bf872e28 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -123,7 +123,7 @@ There is a new function parameter syntax ``/`` to indicate that some function parameters must be specified positionally and cannot be used as keyword arguments. This is the same notation shown by ``help()`` for C functions annotated with Larry Hastings' -`Argument Clinic `__ tool. +`Argument Clinic `__ tool. In the following example, parameters *a* and *b* are positional-only, while *c* or *d* can be positional or keyword, and *e* or *f* are From f14e3d59c955fb3cf89e5241727ec566164dcf42 Mon Sep 17 00:00:00 2001 From: Christoph Anton Mitterer Date: Wed, 13 Dec 2023 20:55:31 +0100 Subject: [PATCH 49/95] gh-107959: clarify Unix-availability of `os.lchmod()` (GH-107960) POSIX specifies that implementations are not required to support changing the file mode of symbolic links, but may do so. Consequently, `lchmod()` is not part of POSIX (but mentioned for implementations which do support the above). The current wording of the availability of `os.lchmod()` is rather vague and improved to clearly tell which POSIX/Unix/BSD-like support the function in general (those that support changing the file mode of symbolic links). Further, some examples of major implementations are added. Data for the BSDs taken from their online manpages. Signed-off-by: Christoph Anton Mitterer Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/os.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index e5ac9afc3c6fd9..9d2a3d65069253 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2160,9 +2160,12 @@ features: for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(path, mode, follow_symlinks=False)``. + ``lchmod()`` is not part of POSIX, but Unix implementations may have it if + changing the mode of symbolic links is supported. + .. audit-event:: os.chmod path,mode,dir_fd os.lchmod - .. availability:: Unix. + .. availability:: Unix, not Linux, FreeBSD >= 1.3, NetBSD >= 1.3, not OpenBSD .. versionchanged:: 3.6 Accepts a :term:`path-like object`. From f5c05e015c178975f24b77e5a8975a22d694e019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavol=20Babin=C4=8D=C3=A1k=E2=80=8F?= Date: Wed, 13 Dec 2023 21:23:13 +0100 Subject: [PATCH 50/95] bpo-40648: Test modes that file can get with chmod() on Windows (GH-20130) Order of tests matter second part makes testing file writable and possible to remove again. --- Lib/test/test_stat.py | 7 +++++++ .../next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst | 1 + 2 files changed, 8 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 0eced2fcf98376..a0d0f61e5a192c 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -148,12 +148,19 @@ def test_mode(self): self.assertEqual(modestr, '-r--r--r--') self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) else: + os.chmod(TESTFN, 0o500) + st_mode, modestr = self.get_mode() + self.assertEqual(modestr[:3], '-r-') + self.assertS_IS("REG", st_mode) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) + os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() self.assertEqual(modestr[:3], '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o666) @os_helper.skip_unless_working_chmod def test_directory(self): diff --git a/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst b/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst new file mode 100644 index 00000000000000..8fbe42d263feb9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst @@ -0,0 +1 @@ +Test modes that file can get with chmod() on Windows. From 41c18aacc7a6082fbd9b08697c4d6180a9b62265 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 13 Dec 2023 13:49:29 -0800 Subject: [PATCH 51/95] Move optimizer/executor tests to new file test_capi/test_opt.py (#113072) --- Lib/test/test_capi/test_misc.py | 535 ------------------------------- Lib/test/test_capi/test_opt.py | 544 ++++++++++++++++++++++++++++++++ 2 files changed, 544 insertions(+), 535 deletions(-) create mode 100644 Lib/test/test_capi/test_opt.py diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 776ee913a02216..123813b949fb7d 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -7,7 +7,6 @@ import importlib.machinery import importlib.util import json -import opcode import os import pickle import queue @@ -2483,540 +2482,6 @@ def func(): self.do_test(func, names) -@contextlib.contextmanager -def temporary_optimizer(opt): - old_opt = _testinternalcapi.get_optimizer() - _testinternalcapi.set_optimizer(opt) - try: - yield - finally: - _testinternalcapi.set_optimizer(old_opt) - - -@contextlib.contextmanager -def clear_executors(func): - # Clear executors in func before and after running a block - func.__code__ = func.__code__.replace() - try: - yield - finally: - func.__code__ = func.__code__.replace() - - -class TestOptimizerAPI(unittest.TestCase): - - def test_get_counter_optimizer_dealloc(self): - # See gh-108727 - def f(): - _testinternalcapi.get_counter_optimizer() - - f() - - def test_get_set_optimizer(self): - old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() - try: - _testinternalcapi.set_optimizer(opt) - self.assertEqual(_testinternalcapi.get_optimizer(), opt) - _testinternalcapi.set_optimizer(None) - self.assertEqual(_testinternalcapi.get_optimizer(), None) - finally: - _testinternalcapi.set_optimizer(old) - - - def test_counter_optimizer(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def loop(): - for _ in range(1000): - pass - """), ns, ns) - loop = ns['loop'] - - for repeat in range(5): - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - with clear_executors(loop): - loop() - self.assertEqual(opt.get_count(), 1000) - - def test_long_loop(self): - "Check that we aren't confused by EXTENDED_ARG" - - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def nop(): - pass - - def long_loop(): - for _ in range(10): - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - """), ns, ns) - long_loop = ns['long_loop'] - - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - long_loop() - self.assertEqual(opt.get_count(), 10) - - def test_code_restore_for_ENTER_EXECUTOR(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - code, replace_code = testfunc.__code__, testfunc.__code__.replace() - self.assertEqual(code, replace_code) - self.assertEqual(hash(code), hash(replace_code)) - - -def get_first_executor(func): - code = func.__code__ - co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] - for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass - return None - - -class TestExecutorInvalidation(unittest.TestCase): - - def setUp(self): - self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.get_counter_optimizer() - _testinternalcapi.set_optimizer(self.opt) - - def tearDown(self): - _testinternalcapi.set_optimizer(self.old) - - def test_invalidate_object(self): - # Generate a new set of functions at each call - ns = {} - func_src = "\n".join( - f""" - def f{n}(): - for _ in range(1000): - pass - """ for n in range(5) - ) - exec(textwrap.dedent(func_src), ns, ns) - funcs = [ ns[f'f{n}'] for n in range(5)] - objects = [object() for _ in range(5)] - - for f in funcs: - f() - executors = [get_first_executor(f) for f in funcs] - # Set things up so each executor depends on the objects - # with an equal or lower index. - for i, exe in enumerate(executors): - self.assertTrue(exe.is_valid()) - for obj in objects[:i+1]: - _testinternalcapi.add_executor_dependency(exe, obj) - self.assertTrue(exe.is_valid()) - # Assert that the correct executors are invalidated - # and check that nothing crashes when we invalidate - # an executor mutliple times. - for i in (4,3,2,1,0): - _testinternalcapi.invalidate_executors(objects[i]) - for exe in executors[i:]: - self.assertFalse(exe.is_valid()) - for exe in executors[:i]: - self.assertTrue(exe.is_valid()) - - def test_uop_optimizer_invalidation(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def f(): - for i in range(1000): - pass - """), ns, ns) - f = ns['f'] - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - f() - exe = get_first_executor(f) - self.assertTrue(exe.is_valid()) - _testinternalcapi.invalidate_executors(f.__code__) - self.assertFalse(exe.is_valid()) - -class TestUops(unittest.TestCase): - - def test_basic_loop(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_SET_IP", uops) - self.assertIn("LOAD_FAST", uops) - - def test_extended_arg(self): - "Check EXTENDED_ARG handling in superblock creation" - ns = {} - exec(textwrap.dedent(""" - def many_vars(): - # 260 vars, so z9 should have index 259 - a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 - b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 - c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 - d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 - e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 - f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 - g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 - h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 - i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 - j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 - k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 - l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 - m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 - n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 - o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 - p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 - q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 - r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 - s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 - t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 - u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 - v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 - w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 - x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 - y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 - while z9 > 0: - z9 = z9 - 1 - """), ns, ns) - many_vars = ns["many_vars"] - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - ex = get_first_executor(many_vars) - self.assertIsNone(ex) - many_vars() - - ex = get_first_executor(many_vars) - self.assertIsNotNone(ex) - self.assertIn(("LOAD_FAST", 259, 0), list(ex)) - - def test_unspecialized_unpack(self): - # An example of an unspecialized opcode - def testfunc(x): - i = 0 - while i < x: - i += 1 - a, b = {1: 2, 3: 3} - assert a == 1 and b == 3 - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_UNPACK_SEQUENCE", uops) - - def test_pop_jump_if_false(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_TRUE_POP", uops) - - def test_pop_jump_if_none(self): - def testfunc(a): - for x in a: - if x is None: - x = 0 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) - - def test_pop_jump_if_not_none(self): - def testfunc(a): - for x in a: - x = None - if x is not None: - x = 0 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_NONE_POP", uops) - - def test_pop_jump_if_true(self): - def testfunc(n): - i = 0 - while not i >= n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_FALSE_POP", uops) - - def test_jump_backward(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_JUMP_TO_TOP", uops) - - def test_jump_forward(self): - def testfunc(n): - a = 0 - while a < n: - if a < 0: - a = -a - else: - a = +a - a += 1 - return a - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - # Since there is no JUMP_FORWARD instruction, - # look for indirect evidence: the += operator - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_for_iter_range(self): - def testfunc(n): - total = 0 - for i in range(n): - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - total = testfunc(20) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_list(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = list(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_tuple(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = tuple(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_list_edge_case(self): - def testfunc(it): - for x in it: - pass - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = [1, 2, 3] - it = iter(a) - testfunc(it) - a.append(4) - with self.assertRaises(StopIteration): - next(it) - - def test_call_py_exact_args(self): - def testfunc(n): - def dummy(x): - return x+1 - for i in range(n): - dummy(i) - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_PUSH_FRAME", uops) - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_branch_taken(self): - def testfunc(n): - for i in range(n): - if i < 0: - i = 0 - else: - i = 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_FALSE_POP", uops) - - def test_for_iter_tier_two(self): - class MyIter: - def __init__(self, n): - self.n = n - def __iter__(self): - return self - def __next__(self): - self.n -= 1 - if self.n < 0: - raise StopIteration - return self.n - - def testfunc(n, m): - x = 0 - for i in range(m): - for j in MyIter(n): - x += 1000*i + j - return x - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - x = testfunc(10, 10) - - self.assertEqual(x, sum(range(10)) * 10010) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_FOR_ITER_TIER_TWO", uops) - - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - if i&0x20: - bits += 1 - return bits - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - x = testfunc(20) - - self.assertEqual(x, 40) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = [opname for opname, _, _ in ex] - count = ops.count("_GUARD_IS_TRUE_POP") - # Because Each 'if' halves the score, the second branch is - # too much already. - self.assertEqual(count, 1) - - @unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') class TestPyThreadId(unittest.TestCase): def test_py_thread_id(self): diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py new file mode 100644 index 00000000000000..9f4731103c9413 --- /dev/null +++ b/Lib/test/test_capi/test_opt.py @@ -0,0 +1,544 @@ +import contextlib +import opcode +import textwrap +import unittest + +import _testinternalcapi + + +@contextlib.contextmanager +def temporary_optimizer(opt): + old_opt = _testinternalcapi.get_optimizer() + _testinternalcapi.set_optimizer(opt) + try: + yield + finally: + _testinternalcapi.set_optimizer(old_opt) + + +@contextlib.contextmanager +def clear_executors(func): + # Clear executors in func before and after running a block + func.__code__ = func.__code__.replace() + try: + yield + finally: + func.__code__ = func.__code__.replace() + + +class TestOptimizerAPI(unittest.TestCase): + + def test_get_counter_optimizer_dealloc(self): + # See gh-108727 + def f(): + _testinternalcapi.get_counter_optimizer() + + f() + + def test_get_set_optimizer(self): + old = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.get_counter_optimizer() + try: + _testinternalcapi.set_optimizer(opt) + self.assertEqual(_testinternalcapi.get_optimizer(), opt) + _testinternalcapi.set_optimizer(None) + self.assertEqual(_testinternalcapi.get_optimizer(), None) + finally: + _testinternalcapi.set_optimizer(old) + + + def test_counter_optimizer(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def loop(): + for _ in range(1000): + pass + """), ns, ns) + loop = ns['loop'] + + for repeat in range(5): + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + self.assertEqual(opt.get_count(), 0) + with clear_executors(loop): + loop() + self.assertEqual(opt.get_count(), 1000) + + def test_long_loop(self): + "Check that we aren't confused by EXTENDED_ARG" + + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def nop(): + pass + + def long_loop(): + for _ in range(10): + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + """), ns, ns) + long_loop = ns['long_loop'] + + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + self.assertEqual(opt.get_count(), 0) + long_loop() + self.assertEqual(opt.get_count(), 10) + + def test_code_restore_for_ENTER_EXECUTOR(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + code, replace_code = testfunc.__code__, testfunc.__code__.replace() + self.assertEqual(code, replace_code) + self.assertEqual(hash(code), hash(replace_code)) + + +def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] + for i in range(0, len(co_code), 2): + if co_code[i] == JUMP_BACKWARD: + try: + return _testinternalcapi.get_executor(code, i) + except ValueError: + pass + return None + + +class TestExecutorInvalidation(unittest.TestCase): + + def setUp(self): + self.old = _testinternalcapi.get_optimizer() + self.opt = _testinternalcapi.get_counter_optimizer() + _testinternalcapi.set_optimizer(self.opt) + + def tearDown(self): + _testinternalcapi.set_optimizer(self.old) + + def test_invalidate_object(self): + # Generate a new set of functions at each call + ns = {} + func_src = "\n".join( + f""" + def f{n}(): + for _ in range(1000): + pass + """ for n in range(5) + ) + exec(textwrap.dedent(func_src), ns, ns) + funcs = [ ns[f'f{n}'] for n in range(5)] + objects = [object() for _ in range(5)] + + for f in funcs: + f() + executors = [get_first_executor(f) for f in funcs] + # Set things up so each executor depends on the objects + # with an equal or lower index. + for i, exe in enumerate(executors): + self.assertTrue(exe.is_valid()) + for obj in objects[:i+1]: + _testinternalcapi.add_executor_dependency(exe, obj) + self.assertTrue(exe.is_valid()) + # Assert that the correct executors are invalidated + # and check that nothing crashes when we invalidate + # an executor mutliple times. + for i in (4,3,2,1,0): + _testinternalcapi.invalidate_executors(objects[i]) + for exe in executors[i:]: + self.assertFalse(exe.is_valid()) + for exe in executors[:i]: + self.assertTrue(exe.is_valid()) + + def test_uop_optimizer_invalidation(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def f(): + for i in range(1000): + pass + """), ns, ns) + f = ns['f'] + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertTrue(exe.is_valid()) + _testinternalcapi.invalidate_executors(f.__code__) + self.assertFalse(exe.is_valid()) + +class TestUops(unittest.TestCase): + + def test_basic_loop(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_SET_IP", uops) + self.assertIn("LOAD_FAST", uops) + + def test_extended_arg(self): + "Check EXTENDED_ARG handling in superblock creation" + ns = {} + exec(textwrap.dedent(""" + def many_vars(): + # 260 vars, so z9 should have index 259 + a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 + b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 + c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 + d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 + e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 + f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 + g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 + h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 + i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 + j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 + k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 + l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 + m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 + n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 + o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 + p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 + q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 + s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 + t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 + u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 + v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 + w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 + x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 + y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + while z9 > 0: + z9 = z9 - 1 + """), ns, ns) + many_vars = ns["many_vars"] + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + ex = get_first_executor(many_vars) + self.assertIsNone(ex) + many_vars() + + ex = get_first_executor(many_vars) + self.assertIsNotNone(ex) + self.assertIn(("LOAD_FAST", 259, 0), list(ex)) + + def test_unspecialized_unpack(self): + # An example of an unspecialized opcode + def testfunc(x): + i = 0 + while i < x: + i += 1 + a, b = {1: 2, 3: 3} + assert a == 1 and b == 3 + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_UNPACK_SEQUENCE", uops) + + def test_pop_jump_if_false(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_TRUE_POP", uops) + + def test_pop_jump_if_none(self): + def testfunc(a): + for x in a: + if x is None: + x = 0 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(range(20)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) + + def test_pop_jump_if_not_none(self): + def testfunc(a): + for x in a: + x = None + if x is not None: + x = 0 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(range(20)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_NONE_POP", uops) + + def test_pop_jump_if_true(self): + def testfunc(n): + i = 0 + while not i >= n: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_jump_backward(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_JUMP_TO_TOP", uops) + + def test_jump_forward(self): + def testfunc(n): + a = 0 + while a < n: + if a < 0: + a = -a + else: + a = +a + a += 1 + return a + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + # Since there is no JUMP_FORWARD instruction, + # look for indirect evidence: the += operator + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_for_iter_range(self): + def testfunc(n): + total = 0 + for i in range(n): + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + total = testfunc(20) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_list(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = list(range(20)) + total = testfunc(a) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_tuple(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = tuple(range(20)) + total = testfunc(a) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_list_edge_case(self): + def testfunc(it): + for x in it: + pass + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = [1, 2, 3] + it = iter(a) + testfunc(it) + a.append(4) + with self.assertRaises(StopIteration): + next(it) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_branch_taken(self): + def testfunc(n): + for i in range(n): + if i < 0: + i = 0 + else: + i = 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_for_iter_tier_two(self): + class MyIter: + def __init__(self, n): + self.n = n + def __iter__(self): + return self + def __next__(self): + self.n -= 1 + if self.n < 0: + raise StopIteration + return self.n + + def testfunc(n, m): + x = 0 + for i in range(m): + for j in MyIter(n): + x += 1000*i + j + return x + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + x = testfunc(10, 10) + + self.assertEqual(x, sum(range(10)) * 10010) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_FOR_ITER_TIER_TWO", uops) + + def test_confidence_score(self): + def testfunc(n): + bits = 0 + for i in range(n): + if i & 0x01: + bits += 1 + if i & 0x02: + bits += 1 + if i&0x04: + bits += 1 + if i&0x08: + bits += 1 + if i&0x10: + bits += 1 + if i&0x20: + bits += 1 + return bits + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + x = testfunc(20) + + self.assertEqual(x, 40) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + ops = [opname for opname, _, _ in ex] + count = ops.count("_GUARD_IS_TRUE_POP") + # Because Each 'if' halves the score, the second branch is + # too much already. + self.assertEqual(count, 1) + + +if __name__ == "__main__": + unittest.main() From fddc829236d7b29a522a2160e57b2d7ca23b9b95 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 13 Dec 2023 23:41:43 +0000 Subject: [PATCH 52/95] gh-86179: Implement realpath() on Windows for getpath.py calculations (GH-113033) --- Lib/sysconfig/__init__.py | 11 +----- Lib/test/test_venv.py | 23 ++++++++++- ...3-12-12-20-58-09.gh-issue-86179.YYSk_6.rst | 1 + Modules/getpath.c | 39 +++++++++++++++++++ 4 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index c60c9f3440615b..deb438c705f3a0 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -404,16 +404,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - # This ought to be as simple as dirname(sys._base_executable), but - # if a venv uses symlinks to a build in the source tree, then this - # fails. So instead we guess the subdirectory name from sys.winver - if sys.winver.endswith('-32'): - arch = 'win32' - elif sys.winver.endswith('-arm64'): - arch = 'arm64' - else: - arch = 'amd64' - inc_dir = os.path.join(_PROJECT_BASE, 'PCbuild', arch) + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 617d14dcb9c5fe..8ecb23ff384362 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -46,7 +46,8 @@ def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + env={**os.environ, "PYTHONHOME": ""}) out, err = p.communicate() if p.returncode: if verbose and err: @@ -287,6 +288,16 @@ def test_sysconfig(self): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd, encoding='utf-8') self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) @requireVenvCreate @unittest.skipUnless(can_symlink(), 'Needs symlinks') @@ -309,6 +320,16 @@ def test_sysconfig_symlinks(self): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd, encoding='utf-8') self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst b/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst new file mode 100644 index 00000000000000..c1d96792bdae0b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst @@ -0,0 +1 @@ +Fixes path calculations when launching Python on Windows through a symlink. diff --git a/Modules/getpath.c b/Modules/getpath.c index 6c1078b8914522..422056b1fb6de4 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -502,6 +502,45 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) PyMem_Free((void *)path); PyMem_Free((void *)narrow); return r; +#elif defined(MS_WINDOWS) + HANDLE hFile; + wchar_t resolved[MAXPATHLEN+1]; + int len = 0, err; + PyObject *result; + + wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + hFile = CreateFileW(path, 0, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hFile != INVALID_HANDLE_VALUE) { + len = GetFinalPathNameByHandleW(hFile, resolved, MAXPATHLEN, VOLUME_NAME_DOS); + err = len ? 0 : GetLastError(); + CloseHandle(hFile); + } else { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + + if (err) { + PyErr_SetFromWindowsErr(err); + result = NULL; + } else if (len <= MAXPATHLEN) { + const wchar_t *p = resolved; + if (0 == wcsncmp(p, L"\\\\?\\", 4)) { + if (GetFileAttributesW(&p[4]) != INVALID_FILE_ATTRIBUTES) { + p += 4; + len -= 4; + } + } + result = PyUnicode_FromWideChar(p, len); + } else { + result = Py_NewRef(pathobj); + } + PyMem_Free(path); + return result; #endif return Py_NewRef(pathobj); From b4f2c89118d5a48ce6c495ba931d7f6493422029 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Dec 2023 09:16:06 +0200 Subject: [PATCH 53/95] gh-113086: Add tests for os.chmod() and os.lchmod() (GH-113087) Also make test_copymode_symlink_to_symlink in test_shutil more strict. --- Lib/test/test_posix.py | 116 ++++++++++++++++++++++++++++++++++++++++ Lib/test/test_shutil.py | 3 +- 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 1722c84727bbd8..4eb3cacc23768f 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -935,6 +935,122 @@ def test_utime(self): posix.utime(os_helper.TESTFN, (int(now), int(now))) posix.utime(os_helper.TESTFN, (now, now)) + def check_chmod(self, chmod_func, target, **kwargs): + mode = os.stat(target).st_mode + try: + new_mode = mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + try: + with open(target, 'wb+'): + pass + except PermissionError: + pass + new_mode = mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + with open(target, 'wb+'): + pass + finally: + posix.chmod(target, mode) + + def test_chmod_file(self): + self.check_chmod(posix.chmod, os_helper.TESTFN) + + def tempdir(self): + target = os_helper.TESTFN + 'd' + posix.mkdir(target) + self.addCleanup(posix.rmdir, target) + return target + + def test_chmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.chmod, target) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_file(self): + self.check_chmod(posix.lchmod, os_helper.TESTFN) + self.check_chmod(posix.chmod, os_helper.TESTFN, follow_symlinks=False) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.lchmod, target) + self.check_chmod(posix.chmod, target, follow_symlinks=False) + + def check_chmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + try: + new_mode = target_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + new_mode = target_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + finally: + posix.chmod(target, target_mode) + + def check_lchmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + new_mode = link_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + new_mode = link_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + + @os_helper.skip_unless_symlink + def test_chmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @os_helper.skip_unless_symlink + def test_chmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link, target_is_directory=True) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) self.assertTrue(hasattr(st, 'st_flags')) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index b29d316352f219..5ce8e5d77fbbf3 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1132,10 +1132,11 @@ def test_copymode_symlink_to_symlink(self): os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) # link to link os.lchmod(dst_link, stat.S_IRWXO) + old_mode = os.stat(dst).st_mode shutil.copymode(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.lstat(src_link).st_mode, os.lstat(dst_link).st_mode) - self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + self.assertEqual(os.stat(dst).st_mode, old_mode) # src link - use chmod os.lchmod(dst_link, stat.S_IRWXO) shutil.copymode(src_link, dst, follow_symlinks=False) From 4d5d9acb22ee8c6908ebd4fdc3d17472f8b286e3 Mon Sep 17 00:00:00 2001 From: Stephen Gildea Date: Wed, 13 Dec 2023 23:53:08 -0800 Subject: [PATCH 54/95] gh-90890: Reorder mailbox.Maildir method documentation (GH-113071) When new mailbox.Maildir methods were added for 3.13.0a2, their documentation was added at the end of the mailbox.Maildir section instead of grouping them with other methods Maildir adds to Mailbox. This commit moves the new methods' documentation adjacent to documentation for existing Maildir-specific methods, so that the "special remarks" for common methods remains at the end. --- Doc/library/mailbox.rst | 80 ++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 05ffaf6c9b336e..fd60d163378f07 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -383,46 +383,6 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. last 36 hours. The Maildir specification says that mail-reading programs should do this occasionally. - Some :class:`Mailbox` methods implemented by :class:`Maildir` deserve special - remarks: - - - .. method:: add(message) - __setitem__(key, message) - update(arg) - - .. warning:: - - These methods generate unique file names based upon the current process - ID. When using multiple threads, undetected name clashes may occur and - cause corruption of the mailbox unless threads are coordinated to avoid - using these methods to manipulate the same mailbox simultaneously. - - - .. method:: flush() - - All changes to Maildir mailboxes are immediately applied, so this method - does nothing. - - - .. method:: lock() - unlock() - - Maildir mailboxes do not support (or require) locking, so these methods do - nothing. - - - .. method:: close() - - :class:`Maildir` instances do not keep any open files and the underlying - mailboxes do not support locking, so this method does nothing. - - - .. method:: get_file(key) - - Depending upon the host platform, it may not be possible to modify or - remove the underlying message while the returned file remains open. - .. method:: get_flags(key) @@ -525,6 +485,46 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. versionadded:: 3.13 + Some :class:`Mailbox` methods implemented by :class:`Maildir` deserve special + remarks: + + + .. method:: add(message) + __setitem__(key, message) + update(arg) + + .. warning:: + + These methods generate unique file names based upon the current process + ID. When using multiple threads, undetected name clashes may occur and + cause corruption of the mailbox unless threads are coordinated to avoid + using these methods to manipulate the same mailbox simultaneously. + + + .. method:: flush() + + All changes to Maildir mailboxes are immediately applied, so this method + does nothing. + + + .. method:: lock() + unlock() + + Maildir mailboxes do not support (or require) locking, so these methods do + nothing. + + + .. method:: close() + + :class:`Maildir` instances do not keep any open files and the underlying + mailboxes do not support locking, so this method does nothing. + + + .. method:: get_file(key) + + Depending upon the host platform, it may not be possible to modify or + remove the underlying message while the returned file remains open. + .. seealso:: From bb36f72efcc6a656e0907ffa83620a1e44044895 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Dec 2023 12:04:23 +0200 Subject: [PATCH 55/95] gh-111049: Fix crash during garbage collection of the BytesIO buffer object (GH-111221) --- Lib/test/test_memoryio.py | 21 +++++++++++++++++++ ...-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst | 2 ++ Modules/_io/bytesio.c | 14 ++++--------- 3 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 731299294e6877..8192502a40791b 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -6,10 +6,12 @@ import unittest from test import support +import gc import io import _pyio as pyio import pickle import sys +import weakref class IntLike: def __init__(self, num): @@ -477,6 +479,25 @@ def test_getbuffer_empty(self): buf2.release() memio.write(b'x') + def test_getbuffer_gc_collect(self): + memio = self.ioclass(b"1234567890") + buf = memio.getbuffer() + memiowr = weakref.ref(memio) + bufwr = weakref.ref(buf) + # Create a reference loop. + a = [buf] + a.append(a) + # The Python implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + del memio + del buf + del a + # The C implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + gc.collect() + self.assertIsNone(memiowr()) + self.assertIsNone(bufwr()) + def test_read1(self): buf = self.buftype("1234567890") self.assertEqual(self.ioclass(buf).read1(), buf) diff --git a/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst b/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst new file mode 100644 index 00000000000000..b1de348bea0a58 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst @@ -0,0 +1,2 @@ +Fix crash during garbage collection of the :class:`io.BytesIO` buffer +object. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 16b8ac600ace79..4a15c8e841f25f 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -990,7 +990,9 @@ static int bytesio_clear(bytesio *self) { Py_CLEAR(self->dict); - Py_CLEAR(self->buf); + if (self->exports == 0) { + Py_CLEAR(self->buf); + } return 0; } @@ -1095,13 +1097,6 @@ bytesiobuf_releasebuffer(bytesiobuf *obj, Py_buffer *view) b->exports--; } -static int -bytesiobuf_clear(bytesiobuf *self) -{ - Py_CLEAR(self->source); - return 0; -} - static int bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) { @@ -1116,7 +1111,7 @@ bytesiobuf_dealloc(bytesiobuf *self) PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - (void)bytesiobuf_clear(self); + Py_CLEAR(self->source); tp->tp_free(self); Py_DECREF(tp); } @@ -1124,7 +1119,6 @@ bytesiobuf_dealloc(bytesiobuf *self) static PyType_Slot bytesiobuf_slots[] = { {Py_tp_dealloc, bytesiobuf_dealloc}, {Py_tp_traverse, bytesiobuf_traverse}, - {Py_tp_clear, bytesiobuf_clear}, // Buffer protocol {Py_bf_getbuffer, bytesiobuf_getbuffer}, From b3c21265fa48a137a4c6a35e425b2b7f94f96cf4 Mon Sep 17 00:00:00 2001 From: Daniel Wysocki Date: Thu, 14 Dec 2023 04:07:37 -0600 Subject: [PATCH 56/95] Fixing typo in DocTestRunner docs (GH-112326) --- Doc/library/doctest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index fa1b850c531346..8c28e4478bb70e 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1502,7 +1502,7 @@ DocTestRunner objects :attr:`failures` and :attr:`skips` attributes. The :meth:`run` and :meth:`summarize` methods return a :class:`TestResults` instance. - :class:`DocTestParser` defines the following methods: + :class:`DocTestRunner` defines the following methods: .. method:: report_start(out, test, example) From 23a5711100271e5a8b9dd9ab48b10807627ef4fb Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Thu, 14 Dec 2023 10:26:46 +0000 Subject: [PATCH 57/95] gh-112205: Update textio module to use `@getter` as possible. (gh-113095) --- Modules/_io/clinic/textio.c.h | 90 ++++++++++++++++++++++++++++++++++- Modules/_io/textio.c | 84 ++++++++++++++------------------ 2 files changed, 125 insertions(+), 49 deletions(-) diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index a492f340c74c0d..f24f65f0c1d4f9 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -1048,6 +1048,94 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(_IO_TEXTIOWRAPPER_NAME_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_NAME_GETSETDEF +# define _IO_TEXTIOWRAPPER_NAME_GETSETDEF {"name", (getter)_io_TextIOWrapper_name_get, (setter)_io_TextIOWrapper_name_set, NULL}, +#else +# define _IO_TEXTIOWRAPPER_NAME_GETSETDEF {"name", (getter)_io_TextIOWrapper_name_get, NULL, NULL}, +#endif + +static PyObject * +_io_TextIOWrapper_name_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_name_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_name_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_IO_TEXTIOWRAPPER_CLOSED_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF +# define _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF {"closed", (getter)_io_TextIOWrapper_closed_get, (setter)_io_TextIOWrapper_closed_set, NULL}, +#else +# define _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF {"closed", (getter)_io_TextIOWrapper_closed_get, NULL, NULL}, +#endif + +static PyObject * +_io_TextIOWrapper_closed_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_closed_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_closed_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF +# define _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF {"newlines", (getter)_io_TextIOWrapper_newlines_get, (setter)_io_TextIOWrapper_newlines_set, NULL}, +#else +# define _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF {"newlines", (getter)_io_TextIOWrapper_newlines_get, NULL, NULL}, +#endif + +static PyObject * +_io_TextIOWrapper_newlines_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_newlines_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_newlines_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_IO_TEXTIOWRAPPER_ERRORS_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF +# define _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF {"errors", (getter)_io_TextIOWrapper_errors_get, (setter)_io_TextIOWrapper_errors_set, NULL}, +#else +# define _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF {"errors", (getter)_io_TextIOWrapper_errors_get, NULL, NULL}, +#endif + +static PyObject * +_io_TextIOWrapper_errors_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_errors_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_errors_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + #if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) # undef _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF # define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, NULL}, @@ -1091,4 +1179,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=b312f2d2e2221580 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7af87bf848a5d3f3 input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index c76d92cdd38b9a..702336ca2aeb06 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1475,7 +1475,7 @@ textiowrapper_traverse(textio *self, visitproc visit, void *arg) } static PyObject * -textiowrapper_closed_get(textio *self, void *context); +_io_TextIOWrapper_closed_get_impl(textio *self); /* This macro takes some shortcuts to make the common case faster. */ #define CHECK_CLOSED(self) \ @@ -1486,7 +1486,7 @@ textiowrapper_closed_get(textio *self, void *context); if (self->raw != NULL) \ r = _PyFileIO_closed(self->raw); \ else { \ - _res = textiowrapper_closed_get(self, NULL); \ + _res = _io_TextIOWrapper_closed_get_impl(self); \ if (_res == NULL) \ return NULL; \ r = PyObject_IsTrue(_res); \ @@ -3090,7 +3090,7 @@ _io_TextIOWrapper_close_impl(textio *self) int r; CHECK_ATTACHED(self); - res = textiowrapper_closed_get(self, NULL); + res = _io_TextIOWrapper_closed_get_impl(self); if (res == NULL) return NULL; r = PyObject_IsTrue(res); @@ -3164,42 +3164,43 @@ textiowrapper_iternext(textio *self) return line; } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.name +[clinic start generated code]*/ + static PyObject * -textiowrapper_name_get_impl(textio *self, void *context) +_io_TextIOWrapper_name_get_impl(textio *self) +/*[clinic end generated code: output=8c2f1d6d8756af40 input=26ecec9b39e30e07]*/ { CHECK_ATTACHED(self); return PyObject_GetAttr(self->buffer, &_Py_ID(name)); } -static PyObject * -textiowrapper_name_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_name_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.closed +[clinic start generated code]*/ static PyObject * -textiowrapper_closed_get_impl(textio *self, void *context) +_io_TextIOWrapper_closed_get_impl(textio *self) +/*[clinic end generated code: output=b49b68f443a85e3c input=7dfcf43f63c7003d]*/ { CHECK_ATTACHED(self); return PyObject_GetAttr(self->buffer, &_Py_ID(closed)); } -static PyObject * -textiowrapper_closed_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_closed_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.newlines +[clinic start generated code]*/ static PyObject * -textiowrapper_newlines_get_impl(textio *self, void *context) +_io_TextIOWrapper_newlines_get_impl(textio *self) +/*[clinic end generated code: output=53aa03ac35573180 input=610df647e514b3e8]*/ { PyObject *res; CHECK_ATTACHED(self); @@ -3211,33 +3212,20 @@ textiowrapper_newlines_get_impl(textio *self, void *context) return res; } -static PyObject * -textiowrapper_newlines_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_newlines_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.errors +[clinic start generated code]*/ static PyObject * -textiowrapper_errors_get_impl(textio *self, void *context) +_io_TextIOWrapper_errors_get_impl(textio *self) +/*[clinic end generated code: output=dca3a3ef21b09484 input=b45f983e6d43c4d8]*/ { CHECK_INITIALIZED(self); return Py_NewRef(self->errors); } -static PyObject * -textiowrapper_errors_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_errors_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} - /*[clinic input] @critical_section @getter @@ -3349,12 +3337,12 @@ static PyMemberDef textiowrapper_members[] = { }; static PyGetSetDef textiowrapper_getset[] = { - {"name", (getter)textiowrapper_name_get, NULL, NULL}, - {"closed", (getter)textiowrapper_closed_get, NULL, NULL}, + _IO_TEXTIOWRAPPER_NAME_GETSETDEF + _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF /* {"mode", (getter)TextIOWrapper_mode_get, NULL, NULL}, */ - {"newlines", (getter)textiowrapper_newlines_get, NULL, NULL}, - {"errors", (getter)textiowrapper_errors_get, NULL, NULL}, + _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF + _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {NULL} }; From c6e953be125c491226adc1a5bc9c804ce256aa17 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Dec 2023 13:27:43 +0200 Subject: [PATCH 58/95] gh-113090: Fix test.support.os_support.can_chmod() on Windows (GH-113091) --- Lib/test/support/os_helper.py | 10 +++++++--- Lib/test/test_os.py | 2 +- Lib/test/test_posix.py | 4 +++- Lib/test/test_tarfile.py | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 7a67d87fb9e846..20f38fd36a8876 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -247,15 +247,15 @@ def can_chmod(): global _can_chmod if _can_chmod is not None: return _can_chmod - if not hasattr(os, "chown"): + if not hasattr(os, "chmod"): _can_chmod = False return _can_chmod try: with open(TESTFN, "wb") as f: try: - os.chmod(TESTFN, 0o777) + os.chmod(TESTFN, 0o555) mode1 = os.stat(TESTFN).st_mode - os.chmod(TESTFN, 0o666) + os.chmod(TESTFN, 0o777) mode2 = os.stat(TESTFN).st_mode except OSError as e: can = False @@ -302,6 +302,10 @@ def can_dac_override(): else: _can_dac_override = True finally: + try: + os.chmod(TESTFN, 0o700) + except OSError: + pass unlink(TESTFN) return _can_dac_override diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d4680ef0f0e03f..c66c5797471413 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1743,7 +1743,7 @@ def tearDown(self): os.removedirs(path) -@os_helper.skip_unless_working_chmod +@unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") class ChownFileTests(unittest.TestCase): @classmethod diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 4eb3cacc23768f..887420f8caccfb 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -791,7 +791,7 @@ def check_stat(uid, gid): self.assertRaises(TypeError, chown_func, first_param, uid, t(gid)) check_stat(uid, gid) - @os_helper.skip_unless_working_chmod + @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") @unittest.skipIf(support.is_emscripten, "getgid() is a stub") def test_chown(self): # raise an OSError if the file does not exist @@ -956,6 +956,7 @@ def check_chmod(self, chmod_func, target, **kwargs): finally: posix.chmod(target, mode) + @os_helper.skip_unless_working_chmod def test_chmod_file(self): self.check_chmod(posix.chmod, os_helper.TESTFN) @@ -965,6 +966,7 @@ def tempdir(self): self.addCleanup(posix.rmdir, target) return target + @os_helper.skip_unless_working_chmod def test_chmod_dir(self): target = self.tempdir() self.check_chmod(posix.chmod, target) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 761560bfbf8b53..edfeac6d6a5edf 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3452,7 +3452,7 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, path = pathlib.Path(os.path.normpath(self.destdir / name)) self.assertIn(path, self.expected_paths) self.expected_paths.remove(path) - if mode is not None and os_helper.can_chmod(): + if mode is not None and os_helper.can_chmod() and os.name != 'nt': got = stat.filemode(stat.S_IMODE(path.stat().st_mode)) self.assertEqual(got, mode) if type is None and isinstance(name, str) and name.endswith('/'): From 29f7eb4859bfc27a4c93f36449ca7d810e13288b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Dec 2023 13:28:37 +0200 Subject: [PATCH 59/95] gh-59616: Support os.chmod(follow_symlinks=True) and os.lchmod() on Windows (GH-113049) --- Doc/library/os.rst | 9 +- Doc/whatsnew/3.13.rst | 6 ++ Lib/os.py | 1 + Lib/tempfile.py | 2 +- Lib/test/test_posix.py | 4 +- ...3-12-13-17-08-21.gh-issue-59616.JNlWSs.rst | 3 + Modules/clinic/posixmodule.c.h | 11 +-- Modules/posixmodule.c | 83 +++++++++++++++---- Tools/clinic/clinic.py | 2 +- 9 files changed, 93 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 9d2a3d65069253..f4566a6684e14c 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2062,6 +2062,7 @@ features: Although Windows supports :func:`chmod`, you can only set the file's read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` constants or a corresponding integer value). All other bits are ignored. + The default value of *follow_symlinks* is ``False`` on Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. @@ -2075,6 +2076,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Added support for the *follow_symlinks* argument on Windows. + .. function:: chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True) @@ -2165,11 +2169,14 @@ features: .. audit-event:: os.chmod path,mode,dir_fd os.lchmod - .. availability:: Unix, not Linux, FreeBSD >= 1.3, NetBSD >= 1.3, not OpenBSD + .. availability:: Unix, Windows, not Linux, FreeBSD >= 1.3, NetBSD >= 1.3, not OpenBSD .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Added support on Windows. + .. function:: lchown(path, uid, gid) Change the owner and group id of *path* to the numeric *uid* and *gid*. This diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d599ba9ae6fac8..bd2ae653100a43 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -261,6 +261,12 @@ os CPU resources of a container system without having to modify the container (application code). (Contributed by Donghee Na in :gh:`109595`) +* Add support of :func:`os.lchmod` and the *follow_symlinks* argument + in :func:`os.chmod` on Windows. + Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is + ``False`` on Windows. + (Contributed by Serhiy Storchaka in :gh:`59616`) + pathlib ------- diff --git a/Lib/os.py b/Lib/os.py index a17946750ea7e7..8c4b93250918eb 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -171,6 +171,7 @@ def _add(str, fn): _add("HAVE_FSTATAT", "stat") _add("HAVE_LCHFLAGS", "chflags") _add("HAVE_LCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") if _exists("lchown"): # mac os x10.3 _add("HAVE_LCHOWN", "chown") _add("HAVE_LINKAT", "link") diff --git a/Lib/tempfile.py b/Lib/tempfile.py index cbfc172a789b25..b5a15f7b72c872 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -273,7 +273,7 @@ def _dont_follow_symlinks(func, path, *args): # Pass follow_symlinks=False, unless not supported on this platform. if func in _os.supports_follow_symlinks: func(path, *args, follow_symlinks=False) - elif _os.name == 'nt' or not _os.path.islink(path): + elif not _os.path.islink(path): func(path, *args) def _resetperms(path): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 887420f8caccfb..55cc5e4c6e4f03 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1019,7 +1019,7 @@ def test_chmod_file_symlink(self): self.check_lchmod_link(posix.chmod, target, link) else: self.check_chmod_link(posix.chmod, target, link) - self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) @os_helper.skip_unless_symlink def test_chmod_dir_symlink(self): @@ -1031,7 +1031,7 @@ def test_chmod_dir_symlink(self): self.check_lchmod_link(posix.chmod, target, link) else: self.check_chmod_link(posix.chmod, target, link) - self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') @os_helper.skip_unless_symlink diff --git a/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst b/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst new file mode 100644 index 00000000000000..793ae63b4c1ff5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst @@ -0,0 +1,3 @@ +Add support of :func:`os.lchmod` and the *follow_symlinks* argument in +:func:`os.chmod` on Windows. Note that the default value of *follow_symlinks* +in :func:`!os.lchmod` is ``False`` on Windows. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 9d6cd337f4a2f4..f36872a1eb7a0f 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -493,7 +493,8 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #endif /* defined(HAVE_FCHDIR) */ PyDoc_STRVAR(os_chmod__doc__, -"chmod($module, /, path, mode, *, dir_fd=None, follow_symlinks=True)\n" +"chmod($module, /, path, mode, *, dir_fd=None,\n" +" follow_symlinks=(os.name != \'nt\'))\n" "--\n" "\n" "Change the access permissions of a file.\n" @@ -562,7 +563,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw path_t path = PATH_T_INITIALIZE("chmod", "path", 0, PATH_HAVE_FCHMOD); int mode; int dir_fd = DEFAULT_DIR_FD; - int follow_symlinks = 1; + int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); if (!args) { @@ -677,7 +678,7 @@ os_fchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #endif /* defined(HAVE_FCHMOD) */ -#if defined(HAVE_LCHMOD) +#if (defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) PyDoc_STRVAR(os_lchmod__doc__, "lchmod($module, /, path, mode)\n" @@ -747,7 +748,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#endif /* defined(HAVE_LCHMOD) */ +#endif /* (defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) */ #if defined(HAVE_CHFLAGS) @@ -12421,4 +12422,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=ff0ec3371de19904 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1be15e60a553b40d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ddbb4cd43babfc..b464a28e63b8ac 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3309,6 +3309,29 @@ os_fchdir_impl(PyObject *module, int fd) } #endif /* HAVE_FCHDIR */ +#ifdef MS_WINDOWS +# define CHMOD_DEFAULT_FOLLOW_SYMLINKS 0 +#else +# define CHMOD_DEFAULT_FOLLOW_SYMLINKS 1 +#endif + +#ifdef MS_WINDOWS +static int +win32_lchmod(LPCWSTR path, int mode) +{ + DWORD attr = GetFileAttributesW(path); + if (attr == INVALID_FILE_ATTRIBUTES) { + return 0; + } + if (mode & _S_IWRITE) { + attr &= ~FILE_ATTRIBUTE_READONLY; + } + else { + attr |= FILE_ATTRIBUTE_READONLY; + } + return SetFileAttributesW(path, attr); +} +#endif /*[clinic input] os.chmod @@ -3331,7 +3354,8 @@ os.chmod and path should be relative; path will then be relative to that directory. - follow_symlinks: bool = True + follow_symlinks: bool(c_default="CHMOD_DEFAULT_FOLLOW_SYMLINKS", \ + py_default="(os.name != 'nt')") = CHMOD_DEFAULT_FOLLOW_SYMLINKS If False, and the last element of the path is a symbolic link, chmod will modify the symbolic link itself instead of the file the link points to. @@ -3348,20 +3372,16 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=674a14bc998de09d]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=fcf115d174b9f3d8]*/ { int result; -#ifdef MS_WINDOWS - DWORD attr; -#endif - #ifdef HAVE_FCHMODAT int fchmodat_nofollow_unsupported = 0; int fchmodat_unsupported = 0; #endif -#if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD)) +#if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) if (follow_symlinks_specified("chmod", follow_symlinks)) return NULL; #endif @@ -3372,19 +3392,36 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, } #ifdef MS_WINDOWS + result = 0; Py_BEGIN_ALLOW_THREADS - attr = GetFileAttributesW(path->wide); - if (attr == INVALID_FILE_ATTRIBUTES) - result = 0; + if (follow_symlinks) { + HANDLE hfile; + FILE_BASIC_INFO info; + + hfile = CreateFileW(path->wide, + FILE_READ_ATTRIBUTES|FILE_WRITE_ATTRIBUTES, + 0, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + if (GetFileInformationByHandleEx(hfile, FileBasicInfo, + &info, sizeof(info))) + { + if (mode & _S_IWRITE) { + info.FileAttributes &= ~FILE_ATTRIBUTE_READONLY; + } + else { + info.FileAttributes |= FILE_ATTRIBUTE_READONLY; + } + result = SetFileInformationByHandle(hfile, FileBasicInfo, + &info, sizeof(info)); + } + (void)CloseHandle(hfile); + } + } else { - if (mode & _S_IWRITE) - attr &= ~FILE_ATTRIBUTE_READONLY; - else - attr |= FILE_ATTRIBUTE_READONLY; - result = SetFileAttributesW(path->wide, attr); + result = win32_lchmod(path->wide, mode); } Py_END_ALLOW_THREADS - if (!result) { return path_error(path); } @@ -3514,7 +3551,7 @@ os_fchmod_impl(PyObject *module, int fd, int mode) #endif /* HAVE_FCHMOD */ -#ifdef HAVE_LCHMOD +#if defined(HAVE_LCHMOD) || defined(MS_WINDOWS) /*[clinic input] os.lchmod @@ -3535,6 +3572,15 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { return NULL; } +#ifdef MS_WINDOWS + Py_BEGIN_ALLOW_THREADS + res = win32_lchmod(path->wide, mode); + Py_END_ALLOW_THREADS + if (!res) { + path_error(path); + return NULL; + } +#else /* MS_WINDOWS */ Py_BEGIN_ALLOW_THREADS res = lchmod(path->narrow, mode); Py_END_ALLOW_THREADS @@ -3542,9 +3588,10 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) path_error(path); return NULL; } +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } -#endif /* HAVE_LCHMOD */ +#endif /* HAVE_LCHMOD || MS_WINDOWS */ #ifdef HAVE_CHFLAGS diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 5ec088765f3e01..a9bf110291eadd 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3737,7 +3737,7 @@ def converter_init(self, *, accept: TypeSet = {object}) -> None: self.format_unit = 'i' elif accept != {object}: fail(f"bool_converter: illegal 'accept' argument {accept!r}") - if self.default is not unspecified: + if self.default is not unspecified and self.default is not unknown: self.default = bool(self.default) self.c_default = str(int(self.default)) From 12f0bbd6e08bcc1e7165f2641716f7685c1db35c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Dec 2023 13:36:48 +0200 Subject: [PATCH 60/95] gh-112730: Update docs for colour env vars (#112837) --- Doc/using/cmdline.rst | 5 ++++- Doc/whatsnew/3.13.rst | 3 ++- Misc/python.man | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index dac4956b551dd3..e032a1971bc6d6 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -610,7 +610,9 @@ Miscellaneous options .. versionadded:: 3.13 The ``-X presite`` option. -Controlling Color +.. _using-on-controlling-color: + +Controlling color ~~~~~~~~~~~~~~~~~ The Python interpreter is configured by default to use colors to highlight @@ -1133,6 +1135,7 @@ conflict. If this variable is set to ``1``, the interpreter will colorize various kinds of output. Setting it to ``0`` deactivates this behavior. + See also :ref:`using-on-controlling-color`. .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bd2ae653100a43..e22257853d8333 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -91,7 +91,8 @@ Improved Error Messages * The interpreter now colorizes error messages when displaying tracebacks by default. This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment - variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) + variables. See also :ref:`using-on-controlling-color`. + (Contributed by Pablo Galindo Salgado in :gh:`112730`.) Other Language Changes ====================== diff --git a/Misc/python.man b/Misc/python.man index 9f89c94adf5028..14cbd85c60bfa9 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -601,6 +601,9 @@ show how long each import takes. This is exactly equivalent to setting .IP PYTHONBREAKPOINT If this environment variable is set to 0, it disables the default debugger. It can be set to the callable of your debugger of choice. +.IP PYTHON_COLORS +If this variable is set to 1, the interpreter will colorize various kinds of +output. Setting it to 0 deactivates this behavior. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the From 1161c14e8c68296fc465cd48970b32be9bee012e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Dec 2023 14:24:24 +0200 Subject: [PATCH 61/95] gh-112716: Fix SystemError when __builtins__ is not a dict (GH-112770) It was raised in two cases: * in the import statement when looking up __import__ * in pickling some builtin type when looking up built-ins iter, getattr, etc. --- Lib/test/test_builtin.py | 26 +++++++++++++++++++ ...-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst | 2 ++ Python/ceval.c | 4 +-- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 5e66d58fd2cb18..e15492783aeec1 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -837,6 +837,32 @@ class customdict(dict): # this one should not do anything fancy self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", exec, code, {'__builtins__': customdict()}) + def test_eval_builtins_mapping(self): + code = compile("superglobal", "test", "eval") + # works correctly + ns = {'__builtins__': types.MappingProxyType({'superglobal': 1})} + self.assertEqual(eval(code, ns), 1) + # custom builtins mapping is missing key + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", + eval, code, ns) + + def test_exec_builtins_mapping_import(self): + code = compile("import foo.bar", "test", "exec") + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns) + ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})} + exec(code, ns) + self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) + + def test_eval_builtins_mapping_reduce(self): + # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter") + code = compile("x.__reduce__()", "test", "eval") + ns = {'__builtins__': types.MappingProxyType({}), 'x': iter([1, 2])} + self.assertRaisesRegex(AttributeError, "iter", eval, code, ns) + ns = {'__builtins__': types.MappingProxyType({'iter': iter}), 'x': iter([1, 2])} + self.assertEqual(eval(code, ns), (iter, ([1, 2],), 0)) + def test_exec_redirected(self): savestdout = sys.stdout sys.stdout = None # Whatever that cannot flush() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst new file mode 100644 index 00000000000000..44d63269c5424a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst @@ -0,0 +1,2 @@ +Fix SystemError in the ``import`` statement and in ``__reduce__()`` methods +of builtin types when ``__builtins__`` is not a dict. diff --git a/Python/ceval.c b/Python/ceval.c index d92ab926f84963..8e0be7056919ea 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2417,7 +2417,7 @@ PyObject * _PyEval_GetBuiltin(PyObject *name) { PyObject *attr; - if (PyDict_GetItemRef(PyEval_GetBuiltins(), name, &attr) == 0) { + if (PyMapping_GetOptionalItem(PyEval_GetBuiltins(), name, &attr) == 0) { PyErr_SetObject(PyExc_AttributeError, name); } return attr; @@ -2570,7 +2570,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name, PyObject *fromlist, PyObject *level) { PyObject *import_func; - if (PyDict_GetItemRef(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { + if (PyMapping_GetOptionalItem(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { return NULL; } if (import_func == NULL) { From 4b3cb082da82da744f5db0b7315aa80558c51557 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 14 Dec 2023 13:30:57 +0000 Subject: [PATCH 62/95] gh-101100: Fix Sphinx nitpicks in `library/inspect.rst` and `reference/simple_stmts.rst` (#113107) --- Doc/conf.py | 4 ++++ Doc/reference/simple_stmts.rst | 6 +++--- Doc/tools/.nitignore | 2 -- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index f2d36fdc70430c..0d7c0b553eaa74 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -245,6 +245,10 @@ # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), + # Attributes that definitely should be documented better, + # but are deferred for now: + ('py:attr', '__annotations__'), + ('py:attr', '__wrapped__'), ] # gh-106948: Copy standard C types declared in the "c:type" domain to the diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 34c3a620b87223..04132c78ce77a6 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -214,7 +214,7 @@ Assignment of an object to a single target is recursively defined as follows. object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). - For user-defined objects, the :meth:`__setitem__` method is called with + For user-defined objects, the :meth:`~object.__setitem__` method is called with appropriate arguments. .. index:: pair: slicing; assignment @@ -351,7 +351,7 @@ If the right hand side is present, an annotated assignment performs the actual assignment before evaluating annotations (where applicable). If the right hand side is not present for an expression target, then the interpreter evaluates the target except for the last -:meth:`__setitem__` or :meth:`__setattr__` call. +:meth:`~object.__setitem__` or :meth:`~object.__setattr__` call. .. seealso:: @@ -932,7 +932,7 @@ That is not a future statement; it's an ordinary import statement with no special semantics or syntax restrictions. Code compiled by calls to the built-in functions :func:`exec` and :func:`compile` -that occur in a module :mod:`M` containing a future statement will, by default, +that occur in a module :mod:`!M` containing a future statement will, by default, use the new syntax or semantics associated with the future statement. This can be controlled by optional arguments to :func:`compile` --- see the documentation of that function for details. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 75d50fee33a064..c49fcf71dd3326 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -59,7 +59,6 @@ Doc/library/http.client.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst Doc/library/importlib.rst -Doc/library/inspect.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst @@ -116,7 +115,6 @@ Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/reference/expressions.rst Doc/reference/import.rst -Doc/reference/simple_stmts.rst Doc/tutorial/datastructures.rst Doc/using/windows.rst Doc/whatsnew/2.0.rst From d9e1b5794a8fade21773d18f91a07f80b55c839c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 14 Dec 2023 14:10:35 +0000 Subject: [PATCH 63/95] gh-101100: Fix Sphinx nitpicks in `library/traceback.rst` (#113106) --- Doc/library/exceptions.rst | 2 +- Doc/library/traceback.rst | 110 +++++++++++++++++++++++++------------ Doc/tools/.nitignore | 1 - 3 files changed, 75 insertions(+), 38 deletions(-) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 04686b6db0036c..f7891f2732bdb1 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -44,7 +44,7 @@ Exception context __suppress_context__ (exception attribute) Three attributes on exception objects provide information about the context in -which an the exception was raised: +which the exception was raised: .. attribute:: BaseException.__context__ BaseException.__cause__ diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index dfbd04de243a3c..ab83e0df10b709 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -18,7 +18,8 @@ interpreter. The module uses :ref:`traceback objects ` --- these are objects of type :class:`types.TracebackType`, -which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. +which are assigned to the :attr:`~BaseException.__traceback__` field of +:class:`BaseException` instances. .. seealso:: @@ -32,11 +33,13 @@ The module defines the following functions: .. function:: print_tb(tb, limit=None, file=None) - Print up to *limit* stack trace entries from traceback object *tb* (starting + Print up to *limit* stack trace entries from + :ref:`traceback object ` *tb* (starting from the caller's frame) if *limit* is positive. Otherwise, print the last ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are printed. If *file* is omitted or ``None``, the output goes to - ``sys.stderr``; otherwise it should be an open file or file-like object to + :data:`sys.stderr`; otherwise it should be an open + :term:`file ` or :term:`file-like object` to receive the output. .. versionchanged:: 3.5 @@ -46,7 +49,8 @@ The module defines the following functions: .. function:: print_exception(exc, /[, value, tb], limit=None, \ file=None, chain=True) - Print exception information and stack trace entries from traceback object + Print exception information and stack trace entries from + :ref:`traceback object ` *tb* to *file*. This differs from :func:`print_tb` in the following ways: @@ -98,7 +102,8 @@ The module defines the following functions: Print up to *limit* stack trace entries (starting from the invocation point) if *limit* is positive. Otherwise, print the last ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are printed. - The optional *f* argument can be used to specify an alternate stack frame + The optional *f* argument can be used to specify an alternate + :ref:`stack frame ` to start. The optional *file* argument has the same meaning as for :func:`print_tb`. @@ -109,20 +114,20 @@ The module defines the following functions: .. function:: extract_tb(tb, limit=None) Return a :class:`StackSummary` object representing a list of "pre-processed" - stack trace entries extracted from the traceback object *tb*. It is useful + stack trace entries extracted from the + :ref:`traceback object ` *tb*. It is useful for alternate formatting of stack traces. The optional *limit* argument has the same meaning as for :func:`print_tb`. A "pre-processed" stack trace entry is a :class:`FrameSummary` object containing attributes :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the - information that is usually printed for a stack trace. The - :attr:`~FrameSummary.line` is a string with leading and trailing - whitespace stripped; if the source is not available it is ``None``. + information that is usually printed for a stack trace. .. function:: extract_stack(f=None, limit=None) - Extract the raw traceback from the current stack frame. The return value has + Extract the raw traceback from the current + :ref:`stack frame `. The return value has the same format as for :func:`extract_tb`. The optional *f* and *limit* arguments have the same meaning as for :func:`print_stack`. @@ -140,7 +145,7 @@ The module defines the following functions: .. function:: format_exception_only(exc, /[, value], *, show_group=False) Format the exception part of a traceback using an exception value such as - given by ``sys.last_value``. The return value is a list of strings, each + given by :data:`sys.last_value`. The return value is a list of strings, each ending in a newline. The list contains the exception's message, which is normally a single string; however, for :exc:`SyntaxError` exceptions, it contains several lines that (when printed) display detailed information @@ -160,7 +165,8 @@ The module defines the following functions: positional-only. .. versionchanged:: 3.11 - The returned list now includes any notes attached to the exception. + The returned list now includes any + :attr:`notes ` attached to the exception. .. versionchanged:: 3.13 *show_group* parameter was added. @@ -199,14 +205,17 @@ The module defines the following functions: .. function:: clear_frames(tb) - Clears the local variables of all the stack frames in a traceback *tb* - by calling the :meth:`clear` method of each frame object. + Clears the local variables of all the stack frames in a + :ref:`traceback ` *tb* + by calling the :meth:`~frame.clear` method of each + :ref:`frame object `. .. versionadded:: 3.4 .. function:: walk_stack(f) - Walk a stack following ``f.f_back`` from the given frame, yielding the frame + Walk a stack following :attr:`f.f_back ` from the given frame, + yielding the frame and line number for each frame. If *f* is ``None``, the current stack is used. This helper is used with :meth:`StackSummary.extract`. @@ -222,12 +231,12 @@ The module defines the following functions: The module also defines the following classes: -:class:`TracebackException` Objects ------------------------------------ +:class:`!TracebackException` Objects +------------------------------------ .. versionadded:: 3.5 -:class:`TracebackException` objects are created from actual exceptions to +:class:`!TracebackException` objects are created from actual exceptions to capture data for later printing in a lightweight fashion. .. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) @@ -379,33 +388,34 @@ capture data for later printing in a lightweight fashion. well, recursively, with indentation relative to their nesting depth. .. versionchanged:: 3.11 - The exception's notes are now included in the output. + The exception's :attr:`notes ` are now + included in the output. .. versionchanged:: 3.13 Added the *show_group* parameter. -:class:`StackSummary` Objects ------------------------------ +:class:`!StackSummary` Objects +------------------------------ .. versionadded:: 3.5 -:class:`StackSummary` objects represent a call stack ready for formatting. +:class:`!StackSummary` objects represent a call stack ready for formatting. .. class:: StackSummary .. classmethod:: extract(frame_gen, *, limit=None, lookup_lines=True, capture_locals=False) - Construct a :class:`StackSummary` object from a frame generator (such as + Construct a :class:`!StackSummary` object from a frame generator (such as is returned by :func:`~traceback.walk_stack` or :func:`~traceback.walk_tb`). If *limit* is supplied, only this many frames are taken from *frame_gen*. If *lookup_lines* is ``False``, the returned :class:`FrameSummary` objects will not have read their lines in yet, making the cost of - creating the :class:`StackSummary` cheaper (which may be valuable if it + creating the :class:`!StackSummary` cheaper (which may be valuable if it may not actually get formatted). If *capture_locals* is ``True`` the - local variables in each :class:`FrameSummary` are captured as object + local variables in each :class:`!FrameSummary` are captured as object representations. .. versionchanged:: 3.12 @@ -414,14 +424,16 @@ capture data for later printing in a lightweight fashion. .. classmethod:: from_list(a_list) - Construct a :class:`StackSummary` object from a supplied list of + Construct a :class:`!StackSummary` object from a supplied list of :class:`FrameSummary` objects or old-style list of tuples. Each tuple - should be a 4-tuple with filename, lineno, name, line as the elements. + should be a 4-tuple with *filename*, *lineno*, *name*, *line* as the + elements. .. method:: format() Returns a list of strings ready for printing. Each string in the - resulting list corresponds to a single frame from the stack. + resulting list corresponds to a single :ref:`frame ` from + the stack. Each string ends in a newline; the strings may contain internal newlines as well, for those items with source text lines. @@ -434,7 +446,8 @@ capture data for later printing in a lightweight fashion. .. method:: format_frame_summary(frame_summary) - Returns a string for printing one of the frames involved in the stack. + Returns a string for printing one of the :ref:`frames ` + involved in the stack. This method is called for each :class:`FrameSummary` object to be printed by :meth:`StackSummary.format`. If it returns ``None``, the frame is omitted from the output. @@ -442,25 +455,50 @@ capture data for later printing in a lightweight fashion. .. versionadded:: 3.11 -:class:`FrameSummary` Objects ------------------------------ +:class:`!FrameSummary` Objects +------------------------------ .. versionadded:: 3.5 -A :class:`FrameSummary` object represents a single frame in a traceback. +A :class:`!FrameSummary` object represents a single :ref:`frame ` +in a :ref:`traceback `. .. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None) - Represent a single frame in the traceback or stack that is being formatted - or printed. It may optionally have a stringified version of the frames + Represents a single :ref:`frame ` in the + :ref:`traceback ` or stack that is being formatted + or printed. It may optionally have a stringified version of the frame's locals included in it. If *lookup_line* is ``False``, the source code is not - looked up until the :class:`FrameSummary` has the :attr:`~FrameSummary.line` - attribute accessed (which also happens when casting it to a tuple). + looked up until the :class:`!FrameSummary` has the :attr:`~FrameSummary.line` + attribute accessed (which also happens when casting it to a :class:`tuple`). :attr:`~FrameSummary.line` may be directly provided, and will prevent line lookups happening at all. *locals* is an optional local variable dictionary, and if supplied the variable representations are stored in the summary for later display. + :class:`!FrameSummary` instances have the following attributes: + + .. attribute:: FrameSummary.filename + + The filename of the source code for this frame. Equivalent to accessing + :attr:`f.f_code.co_filename ` on a + :ref:`frame object ` *f*. + + .. attribute:: FrameSummary.lineno + + The line number of the source code for this frame. + + .. attribute:: FrameSummary.name + + Equivalent to accessing :attr:`f.f_code.co_name ` on + a :ref:`frame object ` *f*. + + .. attribute:: FrameSummary.line + + A string representing the source code for this frame, with leading and + trailing whitespace stripped. + If the source is not available, it is ``None``. + .. _traceback-example: Traceback Examples diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index c49fcf71dd3326..4d1d31d44fcf75 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -96,7 +96,6 @@ Doc/library/test.rst Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst Doc/library/tkinter.ttk.rst -Doc/library/traceback.rst Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst From 6873555955497e9adbc72fc0e38df5261844aafd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Dec 2023 14:26:44 +0000 Subject: [PATCH 64/95] GH-112354: Treat _EXIT_TRACE like an unconditional side exit (GH-113104) --- Include/internal/pycore_opcode_metadata.h | 2 +- Python/bytecodes.c | 2 +- Python/ceval.c | 19 ++----------------- Python/ceval_macros.h | 2 -- Python/executor_cases.c.h | 2 +- 5 files changed, 5 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 2c512d97c421c9..4670c34a833963 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1689,7 +1689,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, + [_EXIT_TRACE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_CHECK_VALIDITY] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1ae83422730f8f..68bb15c2b536eb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4027,7 +4027,7 @@ dummy_func( op(_EXIT_TRACE, (--)) { TIER_TWO_ONLY - GOTO_TIER_ONE(); + DEOPT_IF(1); } op(_INSERT, (unused[oparg], top -- top, unused[oparg])) { diff --git a/Python/ceval.c b/Python/ceval.c index 8e0be7056919ea..27304d31e27949 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1063,31 +1063,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from DEOPT_IF() deoptimize: - // On DEOPT_IF we just repeat the last instruction. - // This presumes nothing was popped from the stack (nor pushed). - frame->instr_ptr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); + next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, (int)(next_uop - current_executor->trace - 1), _PyOpcode_OpName[frame->instr_ptr->op.code]); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); - frame->return_offset = 0; // Dispatch to frame->instr_ptr - _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(current_executor); - // Fall through -// Jump here from ENTER_EXECUTOR -enter_tier_one: - next_instr = frame->instr_ptr; - goto resume_frame; + DISPATCH(); -// Jump here from _EXIT_TRACE -exit_trace: - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); - Py_DECREF(current_executor); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - goto enter_tier_one; } #if defined(__GNUC__) # pragma GCC diagnostic pop diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ac44aecae046d8..a3606b17b71c62 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -392,8 +392,6 @@ stack_pointer = _PyFrame_GetStackPointer(frame); #define GOTO_TIER_TWO() goto enter_tier_two; -#define GOTO_TIER_ONE() goto exit_trace; - #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND() (next_uop[-1].operand) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 14d9dd6e95e533..2519a4ee546a5e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3516,7 +3516,7 @@ case _EXIT_TRACE: { TIER_TWO_ONLY - GOTO_TIER_ONE(); + if (1) goto deoptimize; break; } From fd81afc624402816e3455fd8e932ac7702e4d988 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 14 Dec 2023 15:16:39 +0000 Subject: [PATCH 65/95] gh-86179: Avoid making case-only changes when calculating realpath() during initialization (GH-113077) --- Modules/getpath.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index 422056b1fb6de4..afa9273ebc59b7 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -506,12 +506,17 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) HANDLE hFile; wchar_t resolved[MAXPATHLEN+1]; int len = 0, err; + Py_ssize_t pathlen; PyObject *result; - wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL); + wchar_t *path = PyUnicode_AsWideCharString(pathobj, &pathlen); if (!path) { return NULL; } + if (wcslen(path) != pathlen) { + PyErr_SetString(PyExc_ValueError, "path contains embedded nulls"); + return NULL; + } Py_BEGIN_ALLOW_THREADS hFile = CreateFileW(path, 0, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); @@ -535,7 +540,11 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) len -= 4; } } - result = PyUnicode_FromWideChar(p, len); + if (CompareStringOrdinal(path, (int)pathlen, p, len, TRUE) == CSTR_EQUAL) { + result = Py_NewRef(pathobj); + } else { + result = PyUnicode_FromWideChar(p, len); + } } else { result = Py_NewRef(pathobj); } From fb4cb7ce310e28e25d3adf33b29cb97c637097ed Mon Sep 17 00:00:00 2001 From: jeremy-dolan <129558107+jeremy-dolan@users.noreply.github.com> Date: Thu, 14 Dec 2023 10:40:24 -0500 Subject: [PATCH 66/95] gh-113113: doc: use less ambiguously named variable (gh-113114) --- Doc/tutorial/controlflow.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index aa9caa101da40a..77444f9cb8358d 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -559,10 +559,10 @@ defined to allow. For example:: def ask_ok(prompt, retries=4, reminder='Please try again!'): while True: - ok = input(prompt) - if ok in ('y', 'ye', 'yes'): + reply = input(prompt) + if reply in {'y', 'ye', 'yes'}: return True - if ok in ('n', 'no', 'nop', 'nope'): + if reply in {'n', 'no', 'nop', 'nope'}: return False retries = retries - 1 if retries < 0: From e24eccbc1cf5f22743cd5cda733cd04891155d54 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Dec 2023 16:41:52 +0000 Subject: [PATCH 67/95] GH-111485: Sort metadata tables for easier checking of future diffs (GH-113101) --- Include/internal/pycore_opcode_metadata.h | 3042 ++++++++++----------- Tools/cases_generator/generate_cases.py | 14 +- 2 files changed, 1529 insertions(+), 1527 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4670c34a833963..36b6cd52d2b272 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -31,636 +31,636 @@ extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { switch(opcode) { - case NOP: - return 0; - case RESUME: - return 0; - case RESUME_CHECK: - return 0; - case INSTRUMENTED_RESUME: - return 0; - case LOAD_CLOSURE: - return 0; - case LOAD_FAST_CHECK: - return 0; - case LOAD_FAST: - return 0; - case LOAD_FAST_AND_CLEAR: - return 0; - case LOAD_FAST_LOAD_FAST: - return 0; - case LOAD_CONST: - return 0; - case STORE_FAST: - return 1; - case STORE_FAST_MAYBE_NULL: - return 1; - case STORE_FAST_LOAD_FAST: + case BEFORE_ASYNC_WITH: return 1; - case STORE_FAST_STORE_FAST: - return 2; - case POP_TOP: + case BEFORE_WITH: return 1; - case PUSH_NULL: - return 0; - case END_FOR: - return 2; - case INSTRUMENTED_END_FOR: - return 2; - case END_SEND: + case BINARY_OP: return 2; - case INSTRUMENTED_END_SEND: + case BINARY_OP_ADD_FLOAT: return 2; - case UNARY_NEGATIVE: - return 1; - case UNARY_NOT: - return 1; - case _SPECIALIZE_TO_BOOL: - return 1; - case _TO_BOOL: - return 1; - case TO_BOOL: - return 1; - case TO_BOOL_BOOL: - return 1; - case TO_BOOL_INT: - return 1; - case TO_BOOL_LIST: - return 1; - case TO_BOOL_NONE: - return 1; - case TO_BOOL_STR: - return 1; - case TO_BOOL_ALWAYS_TRUE: - return 1; - case UNARY_INVERT: - return 1; - case _GUARD_BOTH_INT: + case BINARY_OP_ADD_INT: return 2; - case _BINARY_OP_MULTIPLY_INT: + case BINARY_OP_ADD_UNICODE: return 2; - case _BINARY_OP_ADD_INT: + case BINARY_OP_INPLACE_ADD_UNICODE: return 2; - case _BINARY_OP_SUBTRACT_INT: + case BINARY_OP_MULTIPLY_FLOAT: return 2; case BINARY_OP_MULTIPLY_INT: return 2; - case BINARY_OP_ADD_INT: + case BINARY_OP_SUBTRACT_FLOAT: return 2; case BINARY_OP_SUBTRACT_INT: return 2; - case _GUARD_BOTH_FLOAT: - return 2; - case _BINARY_OP_MULTIPLY_FLOAT: - return 2; - case _BINARY_OP_ADD_FLOAT: - return 2; - case _BINARY_OP_SUBTRACT_FLOAT: - return 2; - case BINARY_OP_MULTIPLY_FLOAT: + case BINARY_SLICE: + return 3; + case BINARY_SUBSCR: return 2; - case BINARY_OP_ADD_FLOAT: + case BINARY_SUBSCR_DICT: return 2; - case BINARY_OP_SUBTRACT_FLOAT: + case BINARY_SUBSCR_GETITEM: return 2; - case _GUARD_BOTH_UNICODE: + case BINARY_SUBSCR_LIST_INT: return 2; - case _BINARY_OP_ADD_UNICODE: + case BINARY_SUBSCR_STR_INT: return 2; - case BINARY_OP_ADD_UNICODE: + case BINARY_SUBSCR_TUPLE_INT: return 2; - case _BINARY_OP_INPLACE_ADD_UNICODE: + case BUILD_CONST_KEY_MAP: + return oparg + 1; + case BUILD_LIST: + return oparg; + case BUILD_MAP: + return oparg*2; + case BUILD_SET: + return oparg; + case BUILD_SLICE: + return ((oparg == 3) ? 1 : 0) + 2; + case BUILD_STRING: + return oparg; + case BUILD_TUPLE: + return oparg; + case CACHE: + return 0; + case CALL: + return oparg + 2; + case CALL_ALLOC_AND_ENTER_INIT: + return oparg + 2; + case CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case CALL_BUILTIN_CLASS: + return oparg + 2; + case CALL_BUILTIN_FAST: + return oparg + 2; + case CALL_BUILTIN_FAST_WITH_KEYWORDS: + return oparg + 2; + case CALL_BUILTIN_O: + return oparg + 2; + case CALL_FUNCTION_EX: + return ((oparg & 1) ? 1 : 0) + 3; + case CALL_INTRINSIC_1: + return 1; + case CALL_INTRINSIC_2: return 2; - case BINARY_OP_INPLACE_ADD_UNICODE: + case CALL_ISINSTANCE: + return oparg + 2; + case CALL_KW: + return oparg + 3; + case CALL_LEN: + return oparg + 2; + case CALL_LIST_APPEND: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_FAST: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_NOARGS: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_O: + return oparg + 2; + case CALL_PY_EXACT_ARGS: + return oparg + 2; + case CALL_PY_WITH_DEFAULTS: + return oparg + 2; + case CALL_STR_1: + return oparg + 2; + case CALL_TUPLE_1: + return oparg + 2; + case CALL_TYPE_1: + return oparg + 2; + case CHECK_EG_MATCH: return 2; - case _SPECIALIZE_BINARY_SUBSCR: + case CHECK_EXC_MATCH: return 2; - case _BINARY_SUBSCR: + case CLEANUP_THROW: + return 3; + case COMPARE_OP: return 2; - case BINARY_SUBSCR: + case COMPARE_OP_FLOAT: return 2; - case BINARY_SLICE: - return 3; - case STORE_SLICE: - return 4; - case BINARY_SUBSCR_LIST_INT: + case COMPARE_OP_INT: return 2; - case BINARY_SUBSCR_STR_INT: + case COMPARE_OP_STR: return 2; - case BINARY_SUBSCR_TUPLE_INT: + case CONTAINS_OP: return 2; - case BINARY_SUBSCR_DICT: + case CONVERT_VALUE: + return 1; + case COPY: + return (oparg-1) + 1; + case COPY_FREE_VARS: + return 0; + case DELETE_ATTR: + return 1; + case DELETE_DEREF: + return 0; + case DELETE_FAST: + return 0; + case DELETE_GLOBAL: + return 0; + case DELETE_NAME: + return 0; + case DELETE_SUBSCR: return 2; - case BINARY_SUBSCR_GETITEM: + case DICT_MERGE: + return (oparg - 1) + 5; + case DICT_UPDATE: + return (oparg - 1) + 2; + case END_ASYNC_FOR: return 2; - case LIST_APPEND: - return (oparg-1) + 2; - case SET_ADD: - return (oparg-1) + 2; - case _SPECIALIZE_STORE_SUBSCR: + case END_FOR: return 2; - case _STORE_SUBSCR: - return 3; - case STORE_SUBSCR: - return 3; - case STORE_SUBSCR_LIST_INT: - return 3; - case STORE_SUBSCR_DICT: - return 3; - case DELETE_SUBSCR: + case END_SEND: return 2; - case CALL_INTRINSIC_1: + case ENTER_EXECUTOR: + return 0; + case EXIT_INIT_CHECK: return 1; - case CALL_INTRINSIC_2: + case EXTENDED_ARG: + return 0; + case FORMAT_SIMPLE: + return 1; + case FORMAT_WITH_SPEC: return 2; - case RAISE_VARARGS: - return oparg; - case INTERPRETER_EXIT: + case FOR_ITER: return 1; - case _POP_FRAME: + case FOR_ITER_GEN: return 1; - case RETURN_VALUE: + case FOR_ITER_LIST: return 1; - case INSTRUMENTED_RETURN_VALUE: + case FOR_ITER_RANGE: + return 1; + case FOR_ITER_TUPLE: return 1; - case RETURN_CONST: - return 0; - case INSTRUMENTED_RETURN_CONST: - return 0; case GET_AITER: return 1; case GET_ANEXT: return 1; case GET_AWAITABLE: return 1; - case _SPECIALIZE_SEND: - return 2; - case _SEND: - return 2; - case SEND: - return 2; - case SEND_GEN: - return 2; - case INSTRUMENTED_YIELD_VALUE: + case GET_ITER: return 1; - case YIELD_VALUE: + case GET_LEN: return 1; - case POP_EXCEPT: + case GET_YIELD_FROM_ITER: return 1; - case RERAISE: - return oparg + 1; - case END_ASYNC_FOR: + case IMPORT_FROM: + return 1; + case IMPORT_NAME: return 2; - case CLEANUP_THROW: + case INSTRUMENTED_CALL: + return 0; + case INSTRUMENTED_CALL_FUNCTION_EX: + return 0; + case INSTRUMENTED_CALL_KW: + return 0; + case INSTRUMENTED_END_FOR: + return 2; + case INSTRUMENTED_END_SEND: + return 2; + case INSTRUMENTED_FOR_ITER: + return 0; + case INSTRUMENTED_INSTRUCTION: + return 0; + case INSTRUMENTED_JUMP_BACKWARD: + return 0; + case INSTRUMENTED_JUMP_FORWARD: + return 0; + case INSTRUMENTED_LOAD_SUPER_ATTR: return 3; - case LOAD_ASSERTION_ERROR: + case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; - case LOAD_BUILD_CLASS: + case INSTRUMENTED_POP_JUMP_IF_NONE: return 0; - case STORE_NAME: + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_TRUE: + return 0; + case INSTRUMENTED_RESUME: + return 0; + case INSTRUMENTED_RETURN_CONST: + return 0; + case INSTRUMENTED_RETURN_VALUE: return 1; - case DELETE_NAME: + case INSTRUMENTED_YIELD_VALUE: + return 1; + case INTERPRETER_EXIT: + return 1; + case IS_OP: + return 2; + case JUMP: return 0; - case _SPECIALIZE_UNPACK_SEQUENCE: + case JUMP_BACKWARD: + return 0; + case JUMP_BACKWARD_NO_INTERRUPT: + return 0; + case JUMP_FORWARD: + return 0; + case JUMP_NO_INTERRUPT: + return 0; + case LIST_APPEND: + return (oparg-1) + 2; + case LIST_EXTEND: + return (oparg-1) + 2; + case LOAD_ASSERTION_ERROR: + return 0; + case LOAD_ATTR: return 1; - case _UNPACK_SEQUENCE: + case LOAD_ATTR_CLASS: return 1; - case UNPACK_SEQUENCE: + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; - case UNPACK_SEQUENCE_TWO_TUPLE: + case LOAD_ATTR_INSTANCE_VALUE: return 1; - case UNPACK_SEQUENCE_TUPLE: + case LOAD_ATTR_METHOD_LAZY_DICT: return 1; - case UNPACK_SEQUENCE_LIST: + case LOAD_ATTR_METHOD_NO_DICT: return 1; - case UNPACK_EX: + case LOAD_ATTR_METHOD_WITH_VALUES: return 1; - case _SPECIALIZE_STORE_ATTR: + case LOAD_ATTR_MODULE: return 1; - case _STORE_ATTR: - return 2; - case STORE_ATTR: - return 2; - case DELETE_ATTR: + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; - case STORE_GLOBAL: + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; - case DELETE_GLOBAL: - return 0; - case LOAD_LOCALS: - return 0; - case LOAD_FROM_DICT_OR_GLOBALS: + case LOAD_ATTR_PROPERTY: return 1; - case LOAD_NAME: + case LOAD_ATTR_SLOT: + return 1; + case LOAD_ATTR_WITH_HINT: + return 1; + case LOAD_BUILD_CLASS: return 0; - case _SPECIALIZE_LOAD_GLOBAL: + case LOAD_CLOSURE: return 0; - case _LOAD_GLOBAL: + case LOAD_CONST: return 0; - case LOAD_GLOBAL: + case LOAD_DEREF: return 0; - case _GUARD_GLOBALS_VERSION: + case LOAD_FAST: return 0; - case _GUARD_BUILTINS_VERSION: + case LOAD_FAST_AND_CLEAR: return 0; - case _LOAD_GLOBAL_MODULE: + case LOAD_FAST_CHECK: return 0; - case _LOAD_GLOBAL_BUILTINS: + case LOAD_FAST_LOAD_FAST: return 0; - case LOAD_GLOBAL_MODULE: + case LOAD_FROM_DICT_OR_DEREF: + return 1; + case LOAD_FROM_DICT_OR_GLOBALS: + return 1; + case LOAD_GLOBAL: return 0; case LOAD_GLOBAL_BUILTIN: return 0; - case DELETE_FAST: - return 0; - case MAKE_CELL: - return 0; - case DELETE_DEREF: + case LOAD_GLOBAL_MODULE: return 0; - case LOAD_FROM_DICT_OR_DEREF: - return 1; - case LOAD_DEREF: + case LOAD_LOCALS: return 0; - case STORE_DEREF: + case LOAD_METHOD: return 1; - case COPY_FREE_VARS: - return 0; - case BUILD_STRING: - return oparg; - case BUILD_TUPLE: - return oparg; - case BUILD_LIST: - return oparg; - case LIST_EXTEND: - return (oparg-1) + 2; - case SET_UPDATE: - return (oparg-1) + 2; - case BUILD_SET: - return oparg; - case BUILD_MAP: - return oparg*2; - case SETUP_ANNOTATIONS: + case LOAD_NAME: return 0; - case BUILD_CONST_KEY_MAP: - return oparg + 1; - case DICT_UPDATE: - return (oparg - 1) + 2; - case DICT_MERGE: - return (oparg - 1) + 5; - case MAP_ADD: - return (oparg - 1) + 3; - case INSTRUMENTED_LOAD_SUPER_ATTR: - return 3; - case _SPECIALIZE_LOAD_SUPER_ATTR: + case LOAD_SUPER_ATTR: return 3; - case _LOAD_SUPER_ATTR: + case LOAD_SUPER_ATTR_ATTR: return 3; - case LOAD_SUPER_ATTR: + case LOAD_SUPER_ATTR_METHOD: return 3; case LOAD_SUPER_METHOD: return 3; - case LOAD_ZERO_SUPER_METHOD: - return 3; case LOAD_ZERO_SUPER_ATTR: return 3; - case LOAD_SUPER_ATTR_ATTR: - return 3; - case LOAD_SUPER_ATTR_METHOD: + case LOAD_ZERO_SUPER_METHOD: return 3; - case _SPECIALIZE_LOAD_ATTR: + case MAKE_CELL: + return 0; + case MAKE_FUNCTION: return 1; - case _LOAD_ATTR: + case MAP_ADD: + return (oparg - 1) + 3; + case MATCH_CLASS: + return 3; + case MATCH_KEYS: + return 2; + case MATCH_MAPPING: return 1; - case LOAD_ATTR: + case MATCH_SEQUENCE: return 1; - case LOAD_METHOD: + case NOP: + return 0; + case POP_BLOCK: + return 0; + case POP_EXCEPT: return 1; - case _GUARD_TYPE_VERSION: + case POP_JUMP_IF_FALSE: return 1; - case _CHECK_MANAGED_OBJECT_HAS_VALUES: + case POP_JUMP_IF_NONE: return 1; - case _LOAD_ATTR_INSTANCE_VALUE: + case POP_JUMP_IF_NOT_NONE: return 1; - case LOAD_ATTR_INSTANCE_VALUE: + case POP_JUMP_IF_TRUE: return 1; - case _CHECK_ATTR_MODULE: + case POP_TOP: return 1; - case _LOAD_ATTR_MODULE: + case PUSH_EXC_INFO: return 1; - case LOAD_ATTR_MODULE: + case PUSH_NULL: + return 0; + case RAISE_VARARGS: + return oparg; + case RERAISE: + return oparg + 1; + case RESERVED: + return 0; + case RESUME: + return 0; + case RESUME_CHECK: + return 0; + case RETURN_CONST: + return 0; + case RETURN_GENERATOR: + return 0; + case RETURN_VALUE: return 1; - case _CHECK_ATTR_WITH_HINT: + case SEND: + return 2; + case SEND_GEN: + return 2; + case SETUP_ANNOTATIONS: + return 0; + case SETUP_CLEANUP: + return 0; + case SETUP_FINALLY: + return 0; + case SETUP_WITH: + return 0; + case SET_ADD: + return (oparg-1) + 2; + case SET_FUNCTION_ATTRIBUTE: + return 2; + case SET_UPDATE: + return (oparg-1) + 2; + case STORE_ATTR: + return 2; + case STORE_ATTR_INSTANCE_VALUE: + return 2; + case STORE_ATTR_SLOT: + return 2; + case STORE_ATTR_WITH_HINT: + return 2; + case STORE_DEREF: return 1; - case _LOAD_ATTR_WITH_HINT: + case STORE_FAST: return 1; - case LOAD_ATTR_WITH_HINT: + case STORE_FAST_LOAD_FAST: return 1; - case _LOAD_ATTR_SLOT: + case STORE_FAST_MAYBE_NULL: return 1; - case LOAD_ATTR_SLOT: + case STORE_FAST_STORE_FAST: + return 2; + case STORE_GLOBAL: return 1; - case _CHECK_ATTR_CLASS: + case STORE_NAME: return 1; - case _LOAD_ATTR_CLASS: + case STORE_SLICE: + return 4; + case STORE_SUBSCR: + return 3; + case STORE_SUBSCR_DICT: + return 3; + case STORE_SUBSCR_LIST_INT: + return 3; + case SWAP: + return (oparg-2) + 2; + case TO_BOOL: return 1; - case LOAD_ATTR_CLASS: + case TO_BOOL_ALWAYS_TRUE: return 1; - case LOAD_ATTR_PROPERTY: + case TO_BOOL_BOOL: return 1; - case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + case TO_BOOL_INT: return 1; - case _GUARD_DORV_VALUES: + case TO_BOOL_LIST: return 1; - case _STORE_ATTR_INSTANCE_VALUE: - return 2; - case STORE_ATTR_INSTANCE_VALUE: - return 2; - case STORE_ATTR_WITH_HINT: - return 2; - case _STORE_ATTR_SLOT: - return 2; - case STORE_ATTR_SLOT: - return 2; - case _SPECIALIZE_COMPARE_OP: - return 2; - case _COMPARE_OP: + case TO_BOOL_NONE: + return 1; + case TO_BOOL_STR: + return 1; + case UNARY_INVERT: + return 1; + case UNARY_NEGATIVE: + return 1; + case UNARY_NOT: + return 1; + case UNPACK_EX: + return 1; + case UNPACK_SEQUENCE: + return 1; + case UNPACK_SEQUENCE_LIST: + return 1; + case UNPACK_SEQUENCE_TUPLE: + return 1; + case UNPACK_SEQUENCE_TWO_TUPLE: + return 1; + case WITH_EXCEPT_START: + return 4; + case YIELD_VALUE: + return 1; + case _BINARY_OP: return 2; - case COMPARE_OP: + case _BINARY_OP_ADD_FLOAT: return 2; - case COMPARE_OP_FLOAT: + case _BINARY_OP_ADD_INT: return 2; - case COMPARE_OP_INT: + case _BINARY_OP_ADD_UNICODE: return 2; - case COMPARE_OP_STR: + case _BINARY_OP_INPLACE_ADD_UNICODE: return 2; - case IS_OP: + case _BINARY_OP_MULTIPLY_FLOAT: return 2; - case CONTAINS_OP: + case _BINARY_OP_MULTIPLY_INT: return 2; - case CHECK_EG_MATCH: + case _BINARY_OP_SUBTRACT_FLOAT: return 2; - case CHECK_EXC_MATCH: + case _BINARY_OP_SUBTRACT_INT: return 2; - case IMPORT_NAME: + case _BINARY_SUBSCR: return 2; - case IMPORT_FROM: - return 1; - case JUMP_FORWARD: - return 0; - case JUMP_BACKWARD: - return 0; - case JUMP: - return 0; - case JUMP_NO_INTERRUPT: - return 0; - case ENTER_EXECUTOR: - return 0; - case _POP_JUMP_IF_FALSE: - return 1; - case _POP_JUMP_IF_TRUE: - return 1; - case _IS_NONE: + case _CALL: + return oparg + 2; + case _CHECK_ATTR_CLASS: return 1; - case POP_JUMP_IF_TRUE: + case _CHECK_ATTR_METHOD_LAZY_DICT: return 1; - case POP_JUMP_IF_FALSE: + case _CHECK_ATTR_MODULE: return 1; - case POP_JUMP_IF_NONE: + case _CHECK_ATTR_WITH_HINT: return 1; - case POP_JUMP_IF_NOT_NONE: + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _CHECK_FUNCTION_EXACT_ARGS: + return oparg + 2; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: return 1; - case JUMP_BACKWARD_NO_INTERRUPT: + case _CHECK_PEP_523: return 0; - case GET_LEN: - return 1; - case MATCH_CLASS: - return 3; - case MATCH_MAPPING: - return 1; - case MATCH_SEQUENCE: - return 1; - case MATCH_KEYS: + case _CHECK_STACK_SPACE: + return oparg + 2; + case _CHECK_VALIDITY: + return 0; + case _COMPARE_OP: return 2; - case GET_ITER: - return 1; - case GET_YIELD_FROM_ITER: - return 1; - case _SPECIALIZE_FOR_ITER: - return 1; + case _EXIT_TRACE: + return 0; case _FOR_ITER: return 1; case _FOR_ITER_TIER_TWO: return 1; - case FOR_ITER: + case _GUARD_BOTH_FLOAT: + return 2; + case _GUARD_BOTH_INT: + return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _GUARD_BUILTINS_VERSION: + return 0; + case _GUARD_DORV_VALUES: return 1; - case INSTRUMENTED_FOR_ITER: + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_GLOBALS_VERSION: return 0; - case _ITER_CHECK_LIST: + case _GUARD_IS_FALSE_POP: return 1; - case _ITER_JUMP_LIST: + case _GUARD_IS_NONE_POP: return 1; - case _GUARD_NOT_EXHAUSTED_LIST: + case _GUARD_IS_NOT_NONE_POP: return 1; - case _ITER_NEXT_LIST: + case _GUARD_IS_TRUE_POP: return 1; - case FOR_ITER_LIST: + case _GUARD_KEYS_VERSION: return 1; - case _ITER_CHECK_TUPLE: + case _GUARD_NOT_EXHAUSTED_LIST: return 1; - case _ITER_JUMP_TUPLE: + case _GUARD_NOT_EXHAUSTED_RANGE: return 1; case _GUARD_NOT_EXHAUSTED_TUPLE: return 1; - case _ITER_NEXT_TUPLE: + case _GUARD_TYPE_VERSION: return 1; - case FOR_ITER_TUPLE: + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: + return oparg + 2; + case _INSERT: + return oparg + 1; + case _IS_NONE: + return 1; + case _ITER_CHECK_LIST: return 1; case _ITER_CHECK_RANGE: return 1; - case _ITER_JUMP_RANGE: + case _ITER_CHECK_TUPLE: return 1; - case _GUARD_NOT_EXHAUSTED_RANGE: + case _ITER_JUMP_LIST: return 1; - case _ITER_NEXT_RANGE: + case _ITER_JUMP_RANGE: return 1; - case FOR_ITER_RANGE: + case _ITER_JUMP_TUPLE: return 1; - case FOR_ITER_GEN: + case _ITER_NEXT_LIST: return 1; - case BEFORE_ASYNC_WITH: + case _ITER_NEXT_RANGE: return 1; - case BEFORE_WITH: + case _ITER_NEXT_TUPLE: return 1; - case WITH_EXCEPT_START: - return 4; - case SETUP_FINALLY: - return 0; - case SETUP_CLEANUP: - return 0; - case SETUP_WITH: - return 0; - case POP_BLOCK: + case _JUMP_TO_TOP: return 0; - case PUSH_EXC_INFO: - return 1; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + case _LOAD_ATTR: return 1; - case _GUARD_KEYS_VERSION: + case _LOAD_ATTR_CLASS: return 1; - case _LOAD_ATTR_METHOD_WITH_VALUES: + case _LOAD_ATTR_INSTANCE_VALUE: return 1; - case LOAD_ATTR_METHOD_WITH_VALUES: + case _LOAD_ATTR_METHOD_LAZY_DICT: return 1; case _LOAD_ATTR_METHOD_NO_DICT: return 1; - case LOAD_ATTR_METHOD_NO_DICT: + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case _LOAD_ATTR_MODULE: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; - case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case _LOAD_ATTR_SLOT: return 1; - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case _LOAD_ATTR_WITH_HINT: return 1; - case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case _LOAD_GLOBAL: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_SUPER_ATTR: + return 3; + case _POP_FRAME: return 1; - case _CHECK_ATTR_METHOD_LAZY_DICT: + case _POP_JUMP_IF_FALSE: return 1; - case _LOAD_ATTR_METHOD_LAZY_DICT: + case _POP_JUMP_IF_TRUE: return 1; - case LOAD_ATTR_METHOD_LAZY_DICT: + case _PUSH_FRAME: return 1; - case INSTRUMENTED_CALL: + case _SAVE_RETURN_OFFSET: return 0; + case _SEND: + return 2; + case _SET_IP: + return 0; + case _SPECIALIZE_BINARY_OP: + return 2; + case _SPECIALIZE_BINARY_SUBSCR: + return 2; case _SPECIALIZE_CALL: return oparg + 2; - case _CALL: - return oparg + 2; - case CALL: - return oparg + 2; - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _CHECK_PEP_523: - return 0; - case _CHECK_FUNCTION_EXACT_ARGS: - return oparg + 2; - case _CHECK_STACK_SPACE: - return oparg + 2; - case _INIT_CALL_PY_EXACT_ARGS: - return oparg + 2; - case _PUSH_FRAME: + case _SPECIALIZE_COMPARE_OP: + return 2; + case _SPECIALIZE_FOR_ITER: return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case CALL_PY_EXACT_ARGS: - return oparg + 2; - case CALL_PY_WITH_DEFAULTS: - return oparg + 2; - case CALL_TYPE_1: - return oparg + 2; - case CALL_STR_1: - return oparg + 2; - case CALL_TUPLE_1: - return oparg + 2; - case CALL_ALLOC_AND_ENTER_INIT: - return oparg + 2; - case EXIT_INIT_CHECK: + case _SPECIALIZE_LOAD_ATTR: return 1; - case CALL_BUILTIN_CLASS: - return oparg + 2; - case CALL_BUILTIN_O: - return oparg + 2; - case CALL_BUILTIN_FAST: - return oparg + 2; - case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return oparg + 2; - case CALL_LEN: - return oparg + 2; - case CALL_ISINSTANCE: - return oparg + 2; - case CALL_LIST_APPEND: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_O: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_NOARGS: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_FAST: - return oparg + 2; - case INSTRUMENTED_CALL_KW: - return 0; - case CALL_KW: - return oparg + 3; - case INSTRUMENTED_CALL_FUNCTION_EX: + case _SPECIALIZE_LOAD_GLOBAL: return 0; - case CALL_FUNCTION_EX: - return ((oparg & 1) ? 1 : 0) + 3; - case MAKE_FUNCTION: + case _SPECIALIZE_LOAD_SUPER_ATTR: + return 3; + case _SPECIALIZE_SEND: + return 2; + case _SPECIALIZE_STORE_ATTR: return 1; - case SET_FUNCTION_ATTRIBUTE: + case _SPECIALIZE_STORE_SUBSCR: return 2; - case RETURN_GENERATOR: - return 0; - case BUILD_SLICE: - return ((oparg == 3) ? 1 : 0) + 2; - case CONVERT_VALUE: + case _SPECIALIZE_TO_BOOL: return 1; - case FORMAT_SIMPLE: + case _SPECIALIZE_UNPACK_SEQUENCE: return 1; - case FORMAT_WITH_SPEC: - return 2; - case COPY: - return (oparg-1) + 1; - case _SPECIALIZE_BINARY_OP: + case _STORE_ATTR: return 2; - case _BINARY_OP: + case _STORE_ATTR_INSTANCE_VALUE: return 2; - case BINARY_OP: + case _STORE_ATTR_SLOT: return 2; - case SWAP: - return (oparg-2) + 2; - case INSTRUMENTED_INSTRUCTION: - return 0; - case INSTRUMENTED_JUMP_FORWARD: - return 0; - case INSTRUMENTED_JUMP_BACKWARD: - return 0; - case INSTRUMENTED_POP_JUMP_IF_TRUE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_FALSE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NONE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: - return 0; - case EXTENDED_ARG: - return 0; - case CACHE: - return 0; - case RESERVED: - return 0; - case _GUARD_IS_TRUE_POP: - return 1; - case _GUARD_IS_FALSE_POP: - return 1; - case _GUARD_IS_NONE_POP: + case _STORE_SUBSCR: + return 3; + case _TO_BOOL: return 1; - case _GUARD_IS_NOT_NONE_POP: + case _UNPACK_SEQUENCE: return 1; - case _JUMP_TO_TOP: - return 0; - case _SET_IP: - return 0; - case _SAVE_RETURN_OFFSET: - return 0; - case _EXIT_TRACE: - return 0; - case _INSERT: - return oparg + 1; - case _CHECK_VALIDITY: - return 0; default: return -1; } @@ -671,636 +671,636 @@ extern int _PyOpcode_num_pushed(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { switch(opcode) { - case NOP: - return 0; - case RESUME: - return 0; - case RESUME_CHECK: - return 0; - case INSTRUMENTED_RESUME: - return 0; - case LOAD_CLOSURE: - return 1; - case LOAD_FAST_CHECK: + case BEFORE_ASYNC_WITH: + return 2; + case BEFORE_WITH: + return 2; + case BINARY_OP: return 1; - case LOAD_FAST: + case BINARY_OP_ADD_FLOAT: return 1; - case LOAD_FAST_AND_CLEAR: + case BINARY_OP_ADD_INT: return 1; - case LOAD_FAST_LOAD_FAST: - return 2; - case LOAD_CONST: + case BINARY_OP_ADD_UNICODE: return 1; - case STORE_FAST: - return 0; - case STORE_FAST_MAYBE_NULL: + case BINARY_OP_INPLACE_ADD_UNICODE: return 0; - case STORE_FAST_LOAD_FAST: + case BINARY_OP_MULTIPLY_FLOAT: return 1; - case STORE_FAST_STORE_FAST: - return 0; - case POP_TOP: - return 0; - case PUSH_NULL: + case BINARY_OP_MULTIPLY_INT: return 1; - case END_FOR: - return 0; - case INSTRUMENTED_END_FOR: - return 0; - case END_SEND: + case BINARY_OP_SUBTRACT_FLOAT: return 1; - case INSTRUMENTED_END_SEND: + case BINARY_OP_SUBTRACT_INT: return 1; - case UNARY_NEGATIVE: + case BINARY_SLICE: return 1; - case UNARY_NOT: + case BINARY_SUBSCR: return 1; - case _SPECIALIZE_TO_BOOL: + case BINARY_SUBSCR_DICT: return 1; - case _TO_BOOL: + case BINARY_SUBSCR_GETITEM: return 1; - case TO_BOOL: + case BINARY_SUBSCR_LIST_INT: return 1; - case TO_BOOL_BOOL: + case BINARY_SUBSCR_STR_INT: return 1; - case TO_BOOL_INT: + case BINARY_SUBSCR_TUPLE_INT: return 1; - case TO_BOOL_LIST: + case BUILD_CONST_KEY_MAP: return 1; - case TO_BOOL_NONE: + case BUILD_LIST: return 1; - case TO_BOOL_STR: + case BUILD_MAP: return 1; - case TO_BOOL_ALWAYS_TRUE: + case BUILD_SET: return 1; - case UNARY_INVERT: + case BUILD_SLICE: return 1; - case _GUARD_BOTH_INT: - return 2; - case _BINARY_OP_MULTIPLY_INT: + case BUILD_STRING: return 1; - case _BINARY_OP_ADD_INT: + case BUILD_TUPLE: return 1; - case _BINARY_OP_SUBTRACT_INT: + case CACHE: + return 0; + case CALL: return 1; - case BINARY_OP_MULTIPLY_INT: + case CALL_ALLOC_AND_ENTER_INIT: return 1; - case BINARY_OP_ADD_INT: + case CALL_BOUND_METHOD_EXACT_ARGS: + return 0; + case CALL_BUILTIN_CLASS: return 1; - case BINARY_OP_SUBTRACT_INT: + case CALL_BUILTIN_FAST: return 1; - case _GUARD_BOTH_FLOAT: - return 2; - case _BINARY_OP_MULTIPLY_FLOAT: + case CALL_BUILTIN_FAST_WITH_KEYWORDS: return 1; - case _BINARY_OP_ADD_FLOAT: + case CALL_BUILTIN_O: return 1; - case _BINARY_OP_SUBTRACT_FLOAT: + case CALL_FUNCTION_EX: return 1; - case BINARY_OP_MULTIPLY_FLOAT: + case CALL_INTRINSIC_1: return 1; - case BINARY_OP_ADD_FLOAT: + case CALL_INTRINSIC_2: return 1; - case BINARY_OP_SUBTRACT_FLOAT: + case CALL_ISINSTANCE: return 1; - case _GUARD_BOTH_UNICODE: - return 2; - case _BINARY_OP_ADD_UNICODE: + case CALL_KW: return 1; - case BINARY_OP_ADD_UNICODE: + case CALL_LEN: return 1; - case _BINARY_OP_INPLACE_ADD_UNICODE: - return 0; - case BINARY_OP_INPLACE_ADD_UNICODE: - return 0; - case _SPECIALIZE_BINARY_SUBSCR: - return 2; - case _BINARY_SUBSCR: + case CALL_LIST_APPEND: return 1; - case BINARY_SUBSCR: + case CALL_METHOD_DESCRIPTOR_FAST: return 1; - case BINARY_SLICE: + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 1; - case STORE_SLICE: - return 0; - case BINARY_SUBSCR_LIST_INT: + case CALL_METHOD_DESCRIPTOR_NOARGS: return 1; - case BINARY_SUBSCR_STR_INT: + case CALL_METHOD_DESCRIPTOR_O: return 1; - case BINARY_SUBSCR_TUPLE_INT: + case CALL_PY_EXACT_ARGS: + return 0; + case CALL_PY_WITH_DEFAULTS: return 1; - case BINARY_SUBSCR_DICT: + case CALL_STR_1: return 1; - case BINARY_SUBSCR_GETITEM: + case CALL_TUPLE_1: return 1; - case LIST_APPEND: - return (oparg-1) + 1; - case SET_ADD: - return (oparg-1) + 1; - case _SPECIALIZE_STORE_SUBSCR: + case CALL_TYPE_1: + return 1; + case CHECK_EG_MATCH: return 2; - case _STORE_SUBSCR: + case CHECK_EXC_MATCH: + return 2; + case CLEANUP_THROW: + return 2; + case COMPARE_OP: + return 1; + case COMPARE_OP_FLOAT: + return 1; + case COMPARE_OP_INT: + return 1; + case COMPARE_OP_STR: + return 1; + case CONTAINS_OP: + return 1; + case CONVERT_VALUE: + return 1; + case COPY: + return (oparg-1) + 2; + case COPY_FREE_VARS: return 0; - case STORE_SUBSCR: + case DELETE_ATTR: return 0; - case STORE_SUBSCR_LIST_INT: + case DELETE_DEREF: return 0; - case STORE_SUBSCR_DICT: + case DELETE_FAST: return 0; - case DELETE_SUBSCR: + case DELETE_GLOBAL: return 0; - case CALL_INTRINSIC_1: - return 1; - case CALL_INTRINSIC_2: - return 1; - case RAISE_VARARGS: + case DELETE_NAME: return 0; - case INTERPRETER_EXIT: + case DELETE_SUBSCR: return 0; - case _POP_FRAME: + case DICT_MERGE: + return (oparg - 1) + 4; + case DICT_UPDATE: + return (oparg - 1) + 1; + case END_ASYNC_FOR: return 0; - case RETURN_VALUE: + case END_FOR: return 0; - case INSTRUMENTED_RETURN_VALUE: + case END_SEND: + return 1; + case ENTER_EXECUTOR: return 0; - case RETURN_CONST: + case EXIT_INIT_CHECK: return 0; - case INSTRUMENTED_RETURN_CONST: + case EXTENDED_ARG: return 0; - case GET_AITER: + case FORMAT_SIMPLE: return 1; - case GET_ANEXT: - return 2; - case GET_AWAITABLE: + case FORMAT_WITH_SPEC: return 1; - case _SPECIALIZE_SEND: + case FOR_ITER: return 2; - case _SEND: + case FOR_ITER_GEN: return 2; - case SEND: + case FOR_ITER_LIST: return 2; - case SEND_GEN: + case FOR_ITER_RANGE: return 2; - case INSTRUMENTED_YIELD_VALUE: + case FOR_ITER_TUPLE: + return 2; + case GET_AITER: return 1; - case YIELD_VALUE: + case GET_ANEXT: + return 2; + case GET_AWAITABLE: return 1; - case POP_EXCEPT: - return 0; - case RERAISE: - return oparg; - case END_ASYNC_FOR: - return 0; - case CLEANUP_THROW: + case GET_ITER: + return 1; + case GET_LEN: return 2; - case LOAD_ASSERTION_ERROR: + case GET_YIELD_FROM_ITER: return 1; - case LOAD_BUILD_CLASS: + case IMPORT_FROM: + return 2; + case IMPORT_NAME: return 1; - case STORE_NAME: + case INSTRUMENTED_CALL: return 0; - case DELETE_NAME: + case INSTRUMENTED_CALL_FUNCTION_EX: return 0; - case _SPECIALIZE_UNPACK_SEQUENCE: - return 1; - case _UNPACK_SEQUENCE: - return oparg; - case UNPACK_SEQUENCE: - return oparg; - case UNPACK_SEQUENCE_TWO_TUPLE: - return oparg; - case UNPACK_SEQUENCE_TUPLE: - return oparg; - case UNPACK_SEQUENCE_LIST: - return oparg; - case UNPACK_EX: - return (oparg & 0xFF) + (oparg >> 8) + 1; - case _SPECIALIZE_STORE_ATTR: - return 1; - case _STORE_ATTR: + case INSTRUMENTED_CALL_KW: return 0; - case STORE_ATTR: + case INSTRUMENTED_END_FOR: return 0; - case DELETE_ATTR: + case INSTRUMENTED_END_SEND: + return 1; + case INSTRUMENTED_FOR_ITER: return 0; - case STORE_GLOBAL: + case INSTRUMENTED_INSTRUCTION: return 0; - case DELETE_GLOBAL: + case INSTRUMENTED_JUMP_BACKWARD: return 0; - case LOAD_LOCALS: - return 1; - case LOAD_FROM_DICT_OR_GLOBALS: - return 1; - case LOAD_NAME: - return 1; - case _SPECIALIZE_LOAD_GLOBAL: + case INSTRUMENTED_JUMP_FORWARD: return 0; - case _LOAD_GLOBAL: + case INSTRUMENTED_LOAD_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_GLOBAL: - return (oparg & 1 ? 1 : 0) + 1; - case _GUARD_GLOBALS_VERSION: + case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; - case _GUARD_BUILTINS_VERSION: + case INSTRUMENTED_POP_JUMP_IF_NONE: return 0; - case _LOAD_GLOBAL_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; - case _LOAD_GLOBAL_BUILTINS: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_GLOBAL_MODULE: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_GLOBAL_BUILTIN: - return (oparg & 1 ? 1 : 0) + 1; - case DELETE_FAST: + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: return 0; - case MAKE_CELL: + case INSTRUMENTED_POP_JUMP_IF_TRUE: return 0; - case DELETE_DEREF: + case INSTRUMENTED_RESUME: return 0; - case LOAD_FROM_DICT_OR_DEREF: - return 1; - case LOAD_DEREF: - return 1; - case STORE_DEREF: + case INSTRUMENTED_RETURN_CONST: return 0; - case COPY_FREE_VARS: + case INSTRUMENTED_RETURN_VALUE: return 0; - case BUILD_STRING: - return 1; - case BUILD_TUPLE: + case INSTRUMENTED_YIELD_VALUE: return 1; - case BUILD_LIST: + case INTERPRETER_EXIT: + return 0; + case IS_OP: return 1; - case LIST_EXTEND: + case JUMP: + return 0; + case JUMP_BACKWARD: + return 0; + case JUMP_BACKWARD_NO_INTERRUPT: + return 0; + case JUMP_FORWARD: + return 0; + case JUMP_NO_INTERRUPT: + return 0; + case LIST_APPEND: return (oparg-1) + 1; - case SET_UPDATE: + case LIST_EXTEND: return (oparg-1) + 1; - case BUILD_SET: - return 1; - case BUILD_MAP: - return 1; - case SETUP_ANNOTATIONS: - return 0; - case BUILD_CONST_KEY_MAP: + case LOAD_ASSERTION_ERROR: return 1; - case DICT_UPDATE: - return (oparg - 1) + 1; - case DICT_MERGE: - return (oparg - 1) + 4; - case MAP_ADD: - return (oparg - 1) + 1; - case INSTRUMENTED_LOAD_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case _SPECIALIZE_LOAD_SUPER_ATTR: - return 3; - case _LOAD_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_SUPER_ATTR: + case LOAD_ATTR: return (oparg & 1 ? 1 : 0) + 1; - case LOAD_SUPER_METHOD: + case LOAD_ATTR_CLASS: return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ZERO_SUPER_METHOD: + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + return 1; + case LOAD_ATTR_INSTANCE_VALUE: return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ZERO_SUPER_ATTR: + case LOAD_ATTR_METHOD_LAZY_DICT: + return 2; + case LOAD_ATTR_METHOD_NO_DICT: + return 2; + case LOAD_ATTR_METHOD_WITH_VALUES: + return 2; + case LOAD_ATTR_MODULE: return (oparg & 1 ? 1 : 0) + 1; - case LOAD_SUPER_ATTR_ATTR: + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; - case LOAD_SUPER_ATTR_METHOD: - return 2; - case _SPECIALIZE_LOAD_ATTR: + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; - case _LOAD_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR: + case LOAD_ATTR_PROPERTY: + return 1; + case LOAD_ATTR_SLOT: return (oparg & 1 ? 1 : 0) + 1; - case LOAD_METHOD: + case LOAD_ATTR_WITH_HINT: return (oparg & 1 ? 1 : 0) + 1; - case _GUARD_TYPE_VERSION: + case LOAD_BUILD_CLASS: return 1; - case _CHECK_MANAGED_OBJECT_HAS_VALUES: + case LOAD_CLOSURE: return 1; - case _LOAD_ATTR_INSTANCE_VALUE: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_INSTANCE_VALUE: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_MODULE: + case LOAD_CONST: return 1; - case _LOAD_ATTR_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_MODULE: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_WITH_HINT: - return 1; - case _LOAD_ATTR_WITH_HINT: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_WITH_HINT: - return (oparg & 1 ? 1 : 0) + 1; - case _LOAD_ATTR_SLOT: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_SLOT: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_CLASS: + case LOAD_DEREF: return 1; - case _LOAD_ATTR_CLASS: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_CLASS: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ATTR_PROPERTY: + case LOAD_FAST: return 1; - case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + case LOAD_FAST_AND_CLEAR: return 1; - case _GUARD_DORV_VALUES: + case LOAD_FAST_CHECK: return 1; - case _STORE_ATTR_INSTANCE_VALUE: - return 0; - case STORE_ATTR_INSTANCE_VALUE: - return 0; - case STORE_ATTR_WITH_HINT: - return 0; - case _STORE_ATTR_SLOT: - return 0; - case STORE_ATTR_SLOT: - return 0; - case _SPECIALIZE_COMPARE_OP: + case LOAD_FAST_LOAD_FAST: return 2; - case _COMPARE_OP: + case LOAD_FROM_DICT_OR_DEREF: return 1; - case COMPARE_OP: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; - case COMPARE_OP_FLOAT: + case LOAD_GLOBAL: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_GLOBAL_BUILTIN: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_GLOBAL_MODULE: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_LOCALS: return 1; - case COMPARE_OP_INT: + case LOAD_METHOD: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_NAME: return 1; - case COMPARE_OP_STR: + case LOAD_SUPER_ATTR: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_SUPER_ATTR_ATTR: return 1; - case IS_OP: + case LOAD_SUPER_ATTR_METHOD: + return 2; + case LOAD_SUPER_METHOD: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_ZERO_SUPER_ATTR: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_ZERO_SUPER_METHOD: + return (oparg & 1 ? 1 : 0) + 1; + case MAKE_CELL: + return 0; + case MAKE_FUNCTION: return 1; - case CONTAINS_OP: + case MAP_ADD: + return (oparg - 1) + 1; + case MATCH_CLASS: return 1; - case CHECK_EG_MATCH: - return 2; - case CHECK_EXC_MATCH: + case MATCH_KEYS: + return 3; + case MATCH_MAPPING: return 2; - case IMPORT_NAME: - return 1; - case IMPORT_FROM: + case MATCH_SEQUENCE: return 2; - case JUMP_FORWARD: + case NOP: return 0; - case JUMP_BACKWARD: + case POP_BLOCK: return 0; - case JUMP: + case POP_EXCEPT: return 0; - case JUMP_NO_INTERRUPT: + case POP_JUMP_IF_FALSE: return 0; - case ENTER_EXECUTOR: + case POP_JUMP_IF_NONE: return 0; - case _POP_JUMP_IF_FALSE: + case POP_JUMP_IF_NOT_NONE: return 0; - case _POP_JUMP_IF_TRUE: + case POP_JUMP_IF_TRUE: return 0; - case _IS_NONE: + case POP_TOP: + return 0; + case PUSH_EXC_INFO: + return 2; + case PUSH_NULL: return 1; - case POP_JUMP_IF_TRUE: + case RAISE_VARARGS: return 0; - case POP_JUMP_IF_FALSE: + case RERAISE: + return oparg; + case RESERVED: return 0; - case POP_JUMP_IF_NONE: + case RESUME: return 0; - case POP_JUMP_IF_NOT_NONE: + case RESUME_CHECK: return 0; - case JUMP_BACKWARD_NO_INTERRUPT: + case RETURN_CONST: return 0; - case GET_LEN: - return 2; - case MATCH_CLASS: - return 1; - case MATCH_MAPPING: + case RETURN_GENERATOR: + return 0; + case RETURN_VALUE: + return 0; + case SEND: return 2; - case MATCH_SEQUENCE: + case SEND_GEN: return 2; - case MATCH_KEYS: - return 3; - case GET_ITER: - return 1; - case GET_YIELD_FROM_ITER: + case SETUP_ANNOTATIONS: + return 0; + case SETUP_CLEANUP: + return 0; + case SETUP_FINALLY: + return 0; + case SETUP_WITH: + return 0; + case SET_ADD: + return (oparg-1) + 1; + case SET_FUNCTION_ATTRIBUTE: return 1; - case _SPECIALIZE_FOR_ITER: + case SET_UPDATE: + return (oparg-1) + 1; + case STORE_ATTR: + return 0; + case STORE_ATTR_INSTANCE_VALUE: + return 0; + case STORE_ATTR_SLOT: + return 0; + case STORE_ATTR_WITH_HINT: + return 0; + case STORE_DEREF: + return 0; + case STORE_FAST: + return 0; + case STORE_FAST_LOAD_FAST: return 1; - case _FOR_ITER: - return 2; - case _FOR_ITER_TIER_TWO: - return 2; - case FOR_ITER: - return 2; - case INSTRUMENTED_FOR_ITER: + case STORE_FAST_MAYBE_NULL: return 0; - case _ITER_CHECK_LIST: + case STORE_FAST_STORE_FAST: + return 0; + case STORE_GLOBAL: + return 0; + case STORE_NAME: + return 0; + case STORE_SLICE: + return 0; + case STORE_SUBSCR: + return 0; + case STORE_SUBSCR_DICT: + return 0; + case STORE_SUBSCR_LIST_INT: + return 0; + case SWAP: + return (oparg-2) + 2; + case TO_BOOL: return 1; - case _ITER_JUMP_LIST: + case TO_BOOL_ALWAYS_TRUE: return 1; - case _GUARD_NOT_EXHAUSTED_LIST: + case TO_BOOL_BOOL: return 1; - case _ITER_NEXT_LIST: - return 2; - case FOR_ITER_LIST: - return 2; - case _ITER_CHECK_TUPLE: + case TO_BOOL_INT: return 1; - case _ITER_JUMP_TUPLE: + case TO_BOOL_LIST: return 1; - case _GUARD_NOT_EXHAUSTED_TUPLE: + case TO_BOOL_NONE: return 1; - case _ITER_NEXT_TUPLE: - return 2; - case FOR_ITER_TUPLE: - return 2; - case _ITER_CHECK_RANGE: + case TO_BOOL_STR: return 1; - case _ITER_JUMP_RANGE: + case UNARY_INVERT: return 1; - case _GUARD_NOT_EXHAUSTED_RANGE: + case UNARY_NEGATIVE: return 1; - case _ITER_NEXT_RANGE: - return 2; - case FOR_ITER_RANGE: - return 2; - case FOR_ITER_GEN: - return 2; - case BEFORE_ASYNC_WITH: - return 2; - case BEFORE_WITH: - return 2; + case UNARY_NOT: + return 1; + case UNPACK_EX: + return (oparg & 0xFF) + (oparg >> 8) + 1; + case UNPACK_SEQUENCE: + return oparg; + case UNPACK_SEQUENCE_LIST: + return oparg; + case UNPACK_SEQUENCE_TUPLE: + return oparg; + case UNPACK_SEQUENCE_TWO_TUPLE: + return oparg; case WITH_EXCEPT_START: return 5; - case SETUP_FINALLY: - return 0; - case SETUP_CLEANUP: - return 0; - case SETUP_WITH: - return 0; - case POP_BLOCK: - return 0; - case PUSH_EXC_INFO: - return 2; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + case YIELD_VALUE: return 1; - case _GUARD_KEYS_VERSION: + case _BINARY_OP: return 1; - case _LOAD_ATTR_METHOD_WITH_VALUES: - return 2; - case LOAD_ATTR_METHOD_WITH_VALUES: - return 2; - case _LOAD_ATTR_METHOD_NO_DICT: - return 2; - case LOAD_ATTR_METHOD_NO_DICT: - return 2; - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case _BINARY_OP_ADD_FLOAT: return 1; - case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case _BINARY_OP_ADD_INT: return 1; - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case _BINARY_OP_ADD_UNICODE: return 1; - case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case _BINARY_OP_INPLACE_ADD_UNICODE: + return 0; + case _BINARY_OP_MULTIPLY_FLOAT: return 1; - case _CHECK_ATTR_METHOD_LAZY_DICT: + case _BINARY_OP_MULTIPLY_INT: + return 1; + case _BINARY_OP_SUBTRACT_FLOAT: + return 1; + case _BINARY_OP_SUBTRACT_INT: + return 1; + case _BINARY_SUBSCR: return 1; - case _LOAD_ATTR_METHOD_LAZY_DICT: - return 2; - case LOAD_ATTR_METHOD_LAZY_DICT: - return 2; - case INSTRUMENTED_CALL: - return 0; - case _SPECIALIZE_CALL: - return oparg + 2; case _CALL: return 1; - case CALL: + case _CHECK_ATTR_CLASS: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _CHECK_ATTR_MODULE: + return 1; + case _CHECK_ATTR_WITH_HINT: return 1; case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: return oparg + 2; - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + case _CHECK_FUNCTION_EXACT_ARGS: return oparg + 2; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; case _CHECK_PEP_523: return 0; - case _CHECK_FUNCTION_EXACT_ARGS: - return oparg + 2; case _CHECK_STACK_SPACE: return oparg + 2; - case _INIT_CALL_PY_EXACT_ARGS: - return 1; - case _PUSH_FRAME: + case _CHECK_VALIDITY: return 0; - case CALL_BOUND_METHOD_EXACT_ARGS: + case _COMPARE_OP: + return 1; + case _EXIT_TRACE: return 0; - case CALL_PY_EXACT_ARGS: + case _FOR_ITER: + return 2; + case _FOR_ITER_TIER_TWO: + return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _GUARD_BOTH_INT: + return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _GUARD_BUILTINS_VERSION: return 0; - case CALL_PY_WITH_DEFAULTS: - return 1; - case CALL_TYPE_1: - return 1; - case CALL_STR_1: - return 1; - case CALL_TUPLE_1: + case _GUARD_DORV_VALUES: return 1; - case CALL_ALLOC_AND_ENTER_INIT: + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: return 1; - case EXIT_INIT_CHECK: + case _GUARD_GLOBALS_VERSION: return 0; - case CALL_BUILTIN_CLASS: - return 1; - case CALL_BUILTIN_O: - return 1; - case CALL_BUILTIN_FAST: + case _GUARD_IS_FALSE_POP: + return 0; + case _GUARD_IS_NONE_POP: + return 0; + case _GUARD_IS_NOT_NONE_POP: + return 0; + case _GUARD_IS_TRUE_POP: + return 0; + case _GUARD_KEYS_VERSION: return 1; - case CALL_BUILTIN_FAST_WITH_KEYWORDS: + case _GUARD_NOT_EXHAUSTED_LIST: return 1; - case CALL_LEN: + case _GUARD_NOT_EXHAUSTED_RANGE: return 1; - case CALL_ISINSTANCE: + case _GUARD_NOT_EXHAUSTED_TUPLE: return 1; - case CALL_LIST_APPEND: + case _GUARD_TYPE_VERSION: return 1; - case CALL_METHOD_DESCRIPTOR_O: + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: return 1; - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + case _INSERT: + return oparg + 1; + case _IS_NONE: return 1; - case CALL_METHOD_DESCRIPTOR_NOARGS: + case _ITER_CHECK_LIST: return 1; - case CALL_METHOD_DESCRIPTOR_FAST: + case _ITER_CHECK_RANGE: return 1; - case INSTRUMENTED_CALL_KW: - return 0; - case CALL_KW: + case _ITER_CHECK_TUPLE: return 1; - case INSTRUMENTED_CALL_FUNCTION_EX: - return 0; - case CALL_FUNCTION_EX: + case _ITER_JUMP_LIST: return 1; - case MAKE_FUNCTION: + case _ITER_JUMP_RANGE: return 1; - case SET_FUNCTION_ATTRIBUTE: + case _ITER_JUMP_TUPLE: return 1; - case RETURN_GENERATOR: + case _ITER_NEXT_LIST: + return 2; + case _ITER_NEXT_RANGE: + return 2; + case _ITER_NEXT_TUPLE: + return 2; + case _JUMP_TO_TOP: return 0; - case BUILD_SLICE: - return 1; - case CONVERT_VALUE: - return 1; - case FORMAT_SIMPLE: - return 1; - case FORMAT_WITH_SPEC: - return 1; - case COPY: - return (oparg-1) + 2; - case _SPECIALIZE_BINARY_OP: + case _LOAD_ATTR: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_ATTR_CLASS: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: return 2; - case _BINARY_OP: + case _LOAD_ATTR_METHOD_NO_DICT: + return 2; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 2; + case _LOAD_ATTR_MODULE: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; - case BINARY_OP: + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; - case SWAP: - return (oparg-2) + 2; - case INSTRUMENTED_INSTRUCTION: - return 0; - case INSTRUMENTED_JUMP_FORWARD: - return 0; - case INSTRUMENTED_JUMP_BACKWARD: - return 0; - case INSTRUMENTED_POP_JUMP_IF_TRUE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_FALSE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NONE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: - return 0; - case EXTENDED_ARG: - return 0; - case CACHE: + case _LOAD_ATTR_SLOT: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_ATTR_WITH_HINT: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_GLOBAL: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_GLOBAL_BUILTINS: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_GLOBAL_MODULE: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_SUPER_ATTR: + return ((oparg & 1) ? 1 : 0) + 1; + case _POP_FRAME: return 0; - case RESERVED: + case _POP_JUMP_IF_FALSE: return 0; - case _GUARD_IS_TRUE_POP: + case _POP_JUMP_IF_TRUE: return 0; - case _GUARD_IS_FALSE_POP: + case _PUSH_FRAME: return 0; - case _GUARD_IS_NONE_POP: + case _SAVE_RETURN_OFFSET: return 0; - case _GUARD_IS_NOT_NONE_POP: + case _SEND: + return 2; + case _SET_IP: return 0; - case _JUMP_TO_TOP: + case _SPECIALIZE_BINARY_OP: + return 2; + case _SPECIALIZE_BINARY_SUBSCR: + return 2; + case _SPECIALIZE_CALL: + return oparg + 2; + case _SPECIALIZE_COMPARE_OP: + return 2; + case _SPECIALIZE_FOR_ITER: + return 1; + case _SPECIALIZE_LOAD_ATTR: + return 1; + case _SPECIALIZE_LOAD_GLOBAL: return 0; - case _SET_IP: + case _SPECIALIZE_LOAD_SUPER_ATTR: + return 3; + case _SPECIALIZE_SEND: + return 2; + case _SPECIALIZE_STORE_ATTR: + return 1; + case _SPECIALIZE_STORE_SUBSCR: + return 2; + case _SPECIALIZE_TO_BOOL: + return 1; + case _SPECIALIZE_UNPACK_SEQUENCE: + return 1; + case _STORE_ATTR: return 0; - case _SAVE_RETURN_OFFSET: + case _STORE_ATTR_INSTANCE_VALUE: return 0; - case _EXIT_TRACE: + case _STORE_ATTR_SLOT: return 0; - case _INSERT: - return oparg + 1; - case _CHECK_VALIDITY: + case _STORE_SUBSCR: return 0; + case _TO_BOOL: + return 1; + case _UNPACK_SEQUENCE: + return oparg; default: return -1; } @@ -1377,484 +1377,484 @@ struct opcode_macro_expansion { extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { - [NOP] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_CLOSURE] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, 0 }, - [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, - [END_FOR] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, - [_SPECIALIZE_TO_BOOL] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_TO_BOOL] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_BOTH_INT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [_GUARD_BOTH_FLOAT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [_BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [_BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_BINARY_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_STORE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [_POP_FRAME] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, + [END_SEND] = { true, INSTR_FMT_IX, 0 }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_SEND] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_SEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [JUMP] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_NO_INTERRUPT] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_UNPACK_SEQUENCE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_STORE_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, - [_STORE_ATTR] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_CLOSURE] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_LOAD_GLOBAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_GLOBAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_GLOBAL_BUILTINS] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [_SPECIALIZE_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_ATTR] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_LOAD_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR] = { true, INSTR_FMT_IBC0000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_MODULE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_WITH_HINT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_CLASS] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_DORV_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_COMPARE_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [JUMP] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [JUMP_NO_INTERRUPT] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_IS_NONE] = { true, INSTR_FMT_IX, 0 }, - [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ZERO_SUPER_ATTR] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ZERO_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_FOR_ITER_TIER_TWO] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_GUARD_NOT_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, 0 }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [_ITER_CHECK_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_GUARD_NOT_EXHAUSTED_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, 0 }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [_ITER_CHECK_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_RANGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_GUARD_NOT_EXHAUSTED_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_RANGE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SETUP_FINALLY] = { true, 0, HAS_ARG_FLAG }, - [SETUP_CLEANUP] = { true, 0, HAS_ARG_FLAG }, - [SETUP_WITH] = { true, 0, HAS_ARG_FLAG }, + [NOP] = { true, INSTR_FMT_IX, 0 }, [POP_BLOCK] = { true, 0, 0 }, + [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_GUARD_KEYS_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_CALL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SETUP_CLEANUP] = { true, 0, HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, 0, HAS_ARG_FLAG }, + [SETUP_WITH] = { true, 0, HAS_ARG_FLAG }, + [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, + [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_BINARY_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [_BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [_CALL] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_CHECK_ATTR_CLASS] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_CHECK_ATTR_MODULE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_CHECK_ATTR_WITH_HINT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_CHECK_FUNCTION_EXACT_ARGS] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_PUSH_FRAME] = { true, INSTR_FMT_IX, 0 }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_SPECIALIZE_BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_BINARY_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [_GUARD_IS_TRUE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_CHECK_VALIDITY] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_COMPARE_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_EXIT_TRACE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_FOR_ITER_TIER_TWO] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_BOTH_FLOAT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_BOTH_INT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [_GUARD_DORV_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [_GUARD_IS_FALSE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_GUARD_IS_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_GUARD_IS_NOT_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_IS_TRUE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_KEYS_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_GUARD_NOT_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_NOT_EXHAUSTED_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_NOT_EXHAUSTED_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_IS_NONE] = { true, INSTR_FMT_IX, 0 }, + [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_CHECK_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_CHECK_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_ITER_JUMP_RANGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_ITER_JUMP_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, 0 }, + [_ITER_NEXT_RANGE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, 0 }, [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, - [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR] = { true, INSTR_FMT_IBC0000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [_LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_GLOBAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_GLOBAL_BUILTINS] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_POP_FRAME] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_PUSH_FRAME] = { true, INSTR_FMT_IX, 0 }, [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_EXIT_TRACE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_CHECK_VALIDITY] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_SEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_SPECIALIZE_CALL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_LOAD_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_LOAD_GLOBAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_SEND] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_SPECIALIZE_STORE_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_SPECIALIZE_TO_BOOL] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_SPECIALIZE_UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_STORE_ATTR] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_STORE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_TO_BOOL] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_UNPACK_SEQUENCE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, }; #endif // NEED_OPCODE_METADATA extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { - [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, - [RESUME_CHECK] = { .nuops = 1, .uops = { { RESUME_CHECK, 0, 0 } } }, - [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, - [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, - [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, - [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { LOAD_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, - [LOAD_CONST] = { .nuops = 1, .uops = { { LOAD_CONST, 0, 0 } } }, - [STORE_FAST] = { .nuops = 1, .uops = { { STORE_FAST, 0, 0 } } }, - [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, - [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { STORE_FAST, 6, 0 } } }, - [POP_TOP] = { .nuops = 1, .uops = { { POP_TOP, 0, 0 } } }, - [PUSH_NULL] = { .nuops = 1, .uops = { { PUSH_NULL, 0, 0 } } }, - [END_FOR] = { .nuops = 2, .uops = { { POP_TOP, 0, 0 }, { POP_TOP, 0, 0 } } }, - [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } }, - [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } }, - [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } }, - [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, - [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } }, - [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } }, - [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } }, - [TO_BOOL_NONE] = { .nuops = 1, .uops = { { TO_BOOL_NONE, 0, 0 } } }, - [TO_BOOL_STR] = { .nuops = 1, .uops = { { TO_BOOL_STR, 0, 0 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, - [UNARY_INVERT] = { .nuops = 1, .uops = { { UNARY_INVERT, 0, 0 } } }, - [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, + [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { BEFORE_ASYNC_WITH, 0, 0 } } }, + [BEFORE_WITH] = { .nuops = 1, .uops = { { BEFORE_WITH, 0, 0 } } }, + [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, + [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_SUBTRACT_INT, 0, 0 } } }, + [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, - [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, + [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, - [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, - [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_SUBTRACT_INT, 0, 0 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, - [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, + [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, + [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_STR_INT, 0, 0 } } }, [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, - [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, - [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, - [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, - [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, 0, 0 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, - [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, - [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, - [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, - [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, - [RETURN_CONST] = { .nuops = 2, .uops = { { LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, - [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, - [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, - [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, - [POP_EXCEPT] = { .nuops = 1, .uops = { { POP_EXCEPT, 0, 0 } } }, - [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { LOAD_ASSERTION_ERROR, 0, 0 } } }, - [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } }, - [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } }, - [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } }, - [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, 0, 0 } } }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, - [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, - [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, - [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, - [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, - [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, - [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, - [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, - [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, - [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, - [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, - [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, - [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, - [MAKE_CELL] = { .nuops = 1, .uops = { { MAKE_CELL, 0, 0 } } }, - [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, - [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, - [LOAD_DEREF] = { .nuops = 1, .uops = { { LOAD_DEREF, 0, 0 } } }, - [STORE_DEREF] = { .nuops = 1, .uops = { { STORE_DEREF, 0, 0 } } }, - [COPY_FREE_VARS] = { .nuops = 1, .uops = { { COPY_FREE_VARS, 0, 0 } } }, - [BUILD_STRING] = { .nuops = 1, .uops = { { BUILD_STRING, 0, 0 } } }, - [BUILD_TUPLE] = { .nuops = 1, .uops = { { BUILD_TUPLE, 0, 0 } } }, + [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { BUILD_CONST_KEY_MAP, 0, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { BUILD_LIST, 0, 0 } } }, - [LIST_EXTEND] = { .nuops = 1, .uops = { { LIST_EXTEND, 0, 0 } } }, - [SET_UPDATE] = { .nuops = 1, .uops = { { SET_UPDATE, 0, 0 } } }, - [BUILD_SET] = { .nuops = 1, .uops = { { BUILD_SET, 0, 0 } } }, [BUILD_MAP] = { .nuops = 1, .uops = { { BUILD_MAP, 0, 0 } } }, - [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { SETUP_ANNOTATIONS, 0, 0 } } }, - [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { BUILD_CONST_KEY_MAP, 0, 0 } } }, - [DICT_UPDATE] = { .nuops = 1, .uops = { { DICT_UPDATE, 0, 0 } } }, - [DICT_MERGE] = { .nuops = 1, .uops = { { DICT_MERGE, 0, 0 } } }, - [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, - [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, - [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, - [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [BUILD_SET] = { .nuops = 1, .uops = { { BUILD_SET, 0, 0 } } }, + [BUILD_SLICE] = { .nuops = 1, .uops = { { BUILD_SLICE, 0, 0 } } }, + [BUILD_STRING] = { .nuops = 1, .uops = { { BUILD_STRING, 0, 0 } } }, + [BUILD_TUPLE] = { .nuops = 1, .uops = { { BUILD_TUPLE, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { CALL_BUILTIN_CLASS, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { CALL_BUILTIN_O, 0, 0 } } }, + [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, + [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { CALL_ISINSTANCE, 0, 0 } } }, + [CALL_LEN] = { .nuops = 1, .uops = { { CALL_LEN, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, + [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, + [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { CHECK_EG_MATCH, 0, 0 } } }, + [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { CHECK_EXC_MATCH, 0, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, [COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } }, - [IS_OP] = { .nuops = 1, .uops = { { IS_OP, 0, 0 } } }, [CONTAINS_OP] = { .nuops = 1, .uops = { { CONTAINS_OP, 0, 0 } } }, - [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { CHECK_EG_MATCH, 0, 0 } } }, - [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { CHECK_EXC_MATCH, 0, 0 } } }, - [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, 0, 0 } } }, - [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 0, 0 } } }, - [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 0, 0 } } }, - [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 0, 0 } } }, - [GET_LEN] = { .nuops = 1, .uops = { { GET_LEN, 0, 0 } } }, - [MATCH_CLASS] = { .nuops = 1, .uops = { { MATCH_CLASS, 0, 0 } } }, - [MATCH_MAPPING] = { .nuops = 1, .uops = { { MATCH_MAPPING, 0, 0 } } }, - [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { MATCH_SEQUENCE, 0, 0 } } }, - [MATCH_KEYS] = { .nuops = 1, .uops = { { MATCH_KEYS, 0, 0 } } }, - [GET_ITER] = { .nuops = 1, .uops = { { GET_ITER, 0, 0 } } }, - [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, + [CONVERT_VALUE] = { .nuops = 1, .uops = { { CONVERT_VALUE, 0, 0 } } }, + [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } }, + [COPY_FREE_VARS] = { .nuops = 1, .uops = { { COPY_FREE_VARS, 0, 0 } } }, + [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, + [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, + [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, + [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, + [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } }, + [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, + [DICT_MERGE] = { .nuops = 1, .uops = { { DICT_MERGE, 0, 0 } } }, + [DICT_UPDATE] = { .nuops = 1, .uops = { { DICT_UPDATE, 0, 0 } } }, + [END_FOR] = { .nuops = 2, .uops = { { POP_TOP, 0, 0 }, { POP_TOP, 0, 0 } } }, + [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } }, + [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { EXIT_INIT_CHECK, 0, 0 } } }, + [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } }, + [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } }, [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, 0, 0 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, 0, 0 }, { _ITER_JUMP_LIST, 0, 0 }, { _ITER_NEXT_LIST, 0, 0 } } }, - [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 0, 0 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, 0, 0 }, { _ITER_JUMP_RANGE, 0, 0 }, { _ITER_NEXT_RANGE, 0, 0 } } }, - [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { BEFORE_ASYNC_WITH, 0, 0 } } }, - [BEFORE_WITH] = { .nuops = 1, .uops = { { BEFORE_WITH, 0, 0 } } }, - [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, - [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 0, 0 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, + [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, + [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, + [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, + [GET_ITER] = { .nuops = 1, .uops = { { GET_ITER, 0, 0 } } }, + [GET_LEN] = { .nuops = 1, .uops = { { GET_LEN, 0, 0 } } }, + [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, + [IS_OP] = { .nuops = 1, .uops = { { IS_OP, 0, 0 } } }, + [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, + [LIST_EXTEND] = { .nuops = 1, .uops = { { LIST_EXTEND, 0, 0 } } }, + [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { LOAD_ASSERTION_ERROR, 0, 0 } } }, + [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, - [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { EXIT_INIT_CHECK, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { CALL_BUILTIN_O, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_LEN] = { .nuops = 1, .uops = { { CALL_LEN, 0, 0 } } }, - [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { CALL_ISINSTANCE, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, + [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } }, + [LOAD_CONST] = { .nuops = 1, .uops = { { LOAD_CONST, 0, 0 } } }, + [LOAD_DEREF] = { .nuops = 1, .uops = { { LOAD_DEREF, 0, 0 } } }, + [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, + [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, + [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, + [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { LOAD_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, + [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, + [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, + [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, + [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, + [MAKE_CELL] = { .nuops = 1, .uops = { { MAKE_CELL, 0, 0 } } }, [MAKE_FUNCTION] = { .nuops = 1, .uops = { { MAKE_FUNCTION, 0, 0 } } }, + [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, + [MATCH_CLASS] = { .nuops = 1, .uops = { { MATCH_CLASS, 0, 0 } } }, + [MATCH_KEYS] = { .nuops = 1, .uops = { { MATCH_KEYS, 0, 0 } } }, + [MATCH_MAPPING] = { .nuops = 1, .uops = { { MATCH_MAPPING, 0, 0 } } }, + [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { MATCH_SEQUENCE, 0, 0 } } }, + [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, + [POP_EXCEPT] = { .nuops = 1, .uops = { { POP_EXCEPT, 0, 0 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 0, 0 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 0, 0 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 0, 0 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, 0, 0 } } }, + [POP_TOP] = { .nuops = 1, .uops = { { POP_TOP, 0, 0 } } }, + [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, + [PUSH_NULL] = { .nuops = 1, .uops = { { PUSH_NULL, 0, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { RESUME_CHECK, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 2, .uops = { { LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, + [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { SETUP_ANNOTATIONS, 0, 0 } } }, + [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, - [BUILD_SLICE] = { .nuops = 1, .uops = { { BUILD_SLICE, 0, 0 } } }, - [CONVERT_VALUE] = { .nuops = 1, .uops = { { CONVERT_VALUE, 0, 0 } } }, - [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } }, - [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } }, - [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } }, - [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, + [SET_UPDATE] = { .nuops = 1, .uops = { { SET_UPDATE, 0, 0 } } }, + [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [STORE_DEREF] = { .nuops = 1, .uops = { { STORE_DEREF, 0, 0 } } }, + [STORE_FAST] = { .nuops = 1, .uops = { { STORE_FAST, 0, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { STORE_FAST, 6, 0 } } }, + [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, + [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } }, + [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, + [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, 0, 0 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, + [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, + [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } }, + [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } }, + [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } }, + [TO_BOOL_NONE] = { .nuops = 1, .uops = { { TO_BOOL_NONE, 0, 0 } } }, + [TO_BOOL_STR] = { .nuops = 1, .uops = { { TO_BOOL_STR, 0, 0 } } }, + [UNARY_INVERT] = { .nuops = 1, .uops = { { UNARY_INVERT, 0, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } }, + [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } }, + [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, + [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, 0, 0 } } }, + [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, + [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, + [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA @@ -1863,153 +1863,124 @@ extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_EXIT_TRACE] = "_EXIT_TRACE", [_SET_IP] = "_SET_IP", - [_SPECIALIZE_TO_BOOL] = "_SPECIALIZE_TO_BOOL", - [_TO_BOOL] = "_TO_BOOL", - [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", - [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", - [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", - [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", - [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", + [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", - [_SPECIALIZE_BINARY_SUBSCR] = "_SPECIALIZE_BINARY_SUBSCR", + [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SUBSCR] = "_BINARY_SUBSCR", - [_SPECIALIZE_STORE_SUBSCR] = "_SPECIALIZE_STORE_SUBSCR", - [_STORE_SUBSCR] = "_STORE_SUBSCR", - [_POP_FRAME] = "_POP_FRAME", - [_SPECIALIZE_SEND] = "_SPECIALIZE_SEND", - [_SEND] = "_SEND", - [_SPECIALIZE_UNPACK_SEQUENCE] = "_SPECIALIZE_UNPACK_SEQUENCE", - [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", - [_SPECIALIZE_STORE_ATTR] = "_SPECIALIZE_STORE_ATTR", - [_STORE_ATTR] = "_STORE_ATTR", - [_SPECIALIZE_LOAD_GLOBAL] = "_SPECIALIZE_LOAD_GLOBAL", - [_LOAD_GLOBAL] = "_LOAD_GLOBAL", - [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", - [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", - [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", - [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", - [_SPECIALIZE_LOAD_SUPER_ATTR] = "_SPECIALIZE_LOAD_SUPER_ATTR", - [_LOAD_SUPER_ATTR] = "_LOAD_SUPER_ATTR", - [_SPECIALIZE_LOAD_ATTR] = "_SPECIALIZE_LOAD_ATTR", - [_LOAD_ATTR] = "_LOAD_ATTR", - [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", - [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_CALL] = "_CALL", + [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", - [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", - [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", - [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", - [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", - [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", - [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", - [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", - [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", - [_SPECIALIZE_COMPARE_OP] = "_SPECIALIZE_COMPARE_OP", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_VALIDITY] = "_CHECK_VALIDITY", [_COMPARE_OP] = "_COMPARE_OP", - [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", - [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", - [_IS_NONE] = "_IS_NONE", - [_SPECIALIZE_FOR_ITER] = "_SPECIALIZE_FOR_ITER", [_FOR_ITER] = "_FOR_ITER", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", - [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", - [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", + [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", + [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", + [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", + [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", + [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", - [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", - [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", - [_ITER_JUMP_TUPLE] = "_ITER_JUMP_TUPLE", + [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", - [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INSERT] = "_INSERT", + [_IS_NONE] = "_IS_NONE", + [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", [_ITER_JUMP_RANGE] = "_ITER_JUMP_RANGE", - [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_ITER_JUMP_TUPLE] = "_ITER_JUMP_TUPLE", + [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", - [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", - [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", - [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", - [_SPECIALIZE_CALL] = "_SPECIALIZE_CALL", - [_CALL] = "_CALL", - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", - [_CHECK_PEP_523] = "_CHECK_PEP_523", - [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", - [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", - [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_SUPER_ATTR] = "_LOAD_SUPER_ATTR", + [_POP_FRAME] = "_POP_FRAME", + [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", + [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [_PUSH_FRAME] = "_PUSH_FRAME", - [_SPECIALIZE_BINARY_OP] = "_SPECIALIZE_BINARY_OP", - [_BINARY_OP] = "_BINARY_OP", - [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", - [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", - [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", - [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", - [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", - [_INSERT] = "_INSERT", - [_CHECK_VALIDITY] = "_CHECK_VALIDITY", -}; -#endif // NEED_OPCODE_METADATA - -extern const char *const _PyOpcode_OpName[268]; -#ifdef NEED_OPCODE_METADATA -const char *const _PyOpcode_OpName[268] = { - [CACHE] = "CACHE", - [RESERVED] = "RESERVED", - [RESUME] = "RESUME", - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", - [BINARY_SLICE] = "BINARY_SLICE", - [BINARY_SUBSCR] = "BINARY_SUBSCR", - [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", - [CLEANUP_THROW] = "CLEANUP_THROW", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [END_ASYNC_FOR] = "END_ASYNC_FOR", - [END_FOR] = "END_FOR", - [END_SEND] = "END_SEND", - [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", - [FORMAT_SIMPLE] = "FORMAT_SIMPLE", - [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", - [GET_AITER] = "GET_AITER", - [GET_ANEXT] = "GET_ANEXT", - [GET_ITER] = "GET_ITER", - [GET_LEN] = "GET_LEN", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [INTERPRETER_EXIT] = "INTERPRETER_EXIT", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_LOCALS] = "LOAD_LOCALS", - [MAKE_FUNCTION] = "MAKE_FUNCTION", - [MATCH_KEYS] = "MATCH_KEYS", - [MATCH_MAPPING] = "MATCH_MAPPING", - [MATCH_SEQUENCE] = "MATCH_SEQUENCE", - [NOP] = "NOP", - [POP_EXCEPT] = "POP_EXCEPT", - [POP_TOP] = "POP_TOP", - [PUSH_EXC_INFO] = "PUSH_EXC_INFO", - [PUSH_NULL] = "PUSH_NULL", - [RETURN_GENERATOR] = "RETURN_GENERATOR", - [RETURN_VALUE] = "RETURN_VALUE", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [STORE_SLICE] = "STORE_SLICE", - [STORE_SUBSCR] = "STORE_SUBSCR", - [TO_BOOL] = "TO_BOOL", - [UNARY_INVERT] = "UNARY_INVERT", - [UNARY_NEGATIVE] = "UNARY_NEGATIVE", - [UNARY_NOT] = "UNARY_NOT", - [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [_SEND] = "_SEND", + [_SPECIALIZE_BINARY_OP] = "_SPECIALIZE_BINARY_OP", + [_SPECIALIZE_BINARY_SUBSCR] = "_SPECIALIZE_BINARY_SUBSCR", + [_SPECIALIZE_CALL] = "_SPECIALIZE_CALL", + [_SPECIALIZE_COMPARE_OP] = "_SPECIALIZE_COMPARE_OP", + [_SPECIALIZE_FOR_ITER] = "_SPECIALIZE_FOR_ITER", + [_SPECIALIZE_LOAD_ATTR] = "_SPECIALIZE_LOAD_ATTR", + [_SPECIALIZE_LOAD_GLOBAL] = "_SPECIALIZE_LOAD_GLOBAL", + [_SPECIALIZE_LOAD_SUPER_ATTR] = "_SPECIALIZE_LOAD_SUPER_ATTR", + [_SPECIALIZE_SEND] = "_SPECIALIZE_SEND", + [_SPECIALIZE_STORE_ATTR] = "_SPECIALIZE_STORE_ATTR", + [_SPECIALIZE_STORE_SUBSCR] = "_SPECIALIZE_STORE_SUBSCR", + [_SPECIALIZE_TO_BOOL] = "_SPECIALIZE_TO_BOOL", + [_SPECIALIZE_UNPACK_SEQUENCE] = "_SPECIALIZE_UNPACK_SEQUENCE", + [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_TO_BOOL] = "_TO_BOOL", + [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", +}; +#endif // NEED_OPCODE_METADATA + +extern const char *const _PyOpcode_OpName[268]; +#ifdef NEED_OPCODE_METADATA +const char *const _PyOpcode_OpName[268] = { + [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", + [BEFORE_WITH] = "BEFORE_WITH", [BINARY_OP] = "BINARY_OP", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_SLICE] = "BINARY_SLICE", + [BINARY_SUBSCR] = "BINARY_SUBSCR", + [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", + [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", + [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", + [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_LIST] = "BUILD_LIST", [BUILD_MAP] = "BUILD_MAP", @@ -2017,12 +1988,37 @@ const char *const _PyOpcode_OpName[268] = { [BUILD_SLICE] = "BUILD_SLICE", [BUILD_STRING] = "BUILD_STRING", [BUILD_TUPLE] = "BUILD_TUPLE", + [CACHE] = "CACHE", [CALL] = "CALL", + [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_O] = "CALL_BUILTIN_O", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", + [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_LEN] = "CALL_LEN", + [CALL_LIST_APPEND] = "CALL_LIST_APPEND", + [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", + [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_STR_1] = "CALL_STR_1", + [CALL_TUPLE_1] = "CALL_TUPLE_1", + [CALL_TYPE_1] = "CALL_TYPE_1", + [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", + [CLEANUP_THROW] = "CLEANUP_THROW", [COMPARE_OP] = "COMPARE_OP", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_STR] = "COMPARE_OP_STR", [CONTAINS_OP] = "CONTAINS_OP", [CONVERT_VALUE] = "CONVERT_VALUE", [COPY] = "COPY", @@ -2032,21 +2028,74 @@ const char *const _PyOpcode_OpName[268] = { [DELETE_FAST] = "DELETE_FAST", [DELETE_GLOBAL] = "DELETE_GLOBAL", [DELETE_NAME] = "DELETE_NAME", + [DELETE_SUBSCR] = "DELETE_SUBSCR", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [END_ASYNC_FOR] = "END_ASYNC_FOR", + [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", [ENTER_EXECUTOR] = "ENTER_EXECUTOR", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", [EXTENDED_ARG] = "EXTENDED_ARG", + [FORMAT_SIMPLE] = "FORMAT_SIMPLE", + [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", [FOR_ITER] = "FOR_ITER", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [GET_AITER] = "GET_AITER", + [GET_ANEXT] = "GET_ANEXT", [GET_AWAITABLE] = "GET_AWAITABLE", + [GET_ITER] = "GET_ITER", + [GET_LEN] = "GET_LEN", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [IS_OP] = "IS_OP", + [JUMP] = "JUMP", [JUMP_BACKWARD] = "JUMP_BACKWARD", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [JUMP_FORWARD] = "JUMP_FORWARD", + [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [LOAD_ATTR] = "LOAD_ATTR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_CLOSURE] = "LOAD_CLOSURE", [LOAD_CONST] = "LOAD_CONST", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", @@ -2056,133 +2105,84 @@ const char *const _PyOpcode_OpName[268] = { [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", [LOAD_GLOBAL] = "LOAD_GLOBAL", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_LOCALS] = "LOAD_LOCALS", + [LOAD_METHOD] = "LOAD_METHOD", [LOAD_NAME] = "LOAD_NAME", [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", + [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", + [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", + [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", [MAKE_CELL] = "MAKE_CELL", + [MAKE_FUNCTION] = "MAKE_FUNCTION", [MAP_ADD] = "MAP_ADD", [MATCH_CLASS] = "MATCH_CLASS", + [MATCH_KEYS] = "MATCH_KEYS", + [MATCH_MAPPING] = "MATCH_MAPPING", + [MATCH_SEQUENCE] = "MATCH_SEQUENCE", + [NOP] = "NOP", + [POP_BLOCK] = "POP_BLOCK", + [POP_EXCEPT] = "POP_EXCEPT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", + [POP_TOP] = "POP_TOP", + [PUSH_EXC_INFO] = "PUSH_EXC_INFO", + [PUSH_NULL] = "PUSH_NULL", [RAISE_VARARGS] = "RAISE_VARARGS", [RERAISE] = "RERAISE", + [RESERVED] = "RESERVED", + [RESUME] = "RESUME", + [RESUME_CHECK] = "RESUME_CHECK", [RETURN_CONST] = "RETURN_CONST", + [RETURN_GENERATOR] = "RETURN_GENERATOR", + [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", + [SEND_GEN] = "SEND_GEN", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [SETUP_CLEANUP] = "SETUP_CLEANUP", + [SETUP_FINALLY] = "SETUP_FINALLY", + [SETUP_WITH] = "SETUP_WITH", [SET_ADD] = "SET_ADD", [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", [SET_UPDATE] = "SET_UPDATE", [STORE_ATTR] = "STORE_ATTR", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_DEREF] = "STORE_DEREF", [STORE_FAST] = "STORE_FAST", [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", + [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", [STORE_GLOBAL] = "STORE_GLOBAL", [STORE_NAME] = "STORE_NAME", - [SWAP] = "SWAP", - [UNPACK_EX] = "UNPACK_EX", - [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", - [YIELD_VALUE] = "YIELD_VALUE", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", - [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", - [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", - [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", - [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", - [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", - [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", - [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_BUILTIN_O] = "CALL_BUILTIN_O", - [CALL_ISINSTANCE] = "CALL_ISINSTANCE", - [CALL_LEN] = "CALL_LEN", - [CALL_LIST_APPEND] = "CALL_LIST_APPEND", - [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", - [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_STR_1] = "CALL_STR_1", - [CALL_TUPLE_1] = "CALL_TUPLE_1", - [CALL_TYPE_1] = "CALL_TYPE_1", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", - [COMPARE_OP_INT] = "COMPARE_OP_INT", - [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [RESUME_CHECK] = "RESUME_CHECK", - [SEND_GEN] = "SEND_GEN", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_SLICE] = "STORE_SLICE", + [STORE_SUBSCR] = "STORE_SUBSCR", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [SWAP] = "SWAP", + [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", [TO_BOOL_BOOL] = "TO_BOOL_BOOL", [TO_BOOL_INT] = "TO_BOOL_INT", [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [UNARY_INVERT] = "UNARY_INVERT", + [UNARY_NEGATIVE] = "UNARY_NEGATIVE", + [UNARY_NOT] = "UNARY_NOT", + [UNPACK_EX] = "UNPACK_EX", + [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", - [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", - [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", - [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", - [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", - [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", - [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", - [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [JUMP] = "JUMP", - [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", - [LOAD_CLOSURE] = "LOAD_CLOSURE", - [LOAD_METHOD] = "LOAD_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", - [POP_BLOCK] = "POP_BLOCK", - [SETUP_CLEANUP] = "SETUP_CLEANUP", - [SETUP_FINALLY] = "SETUP_FINALLY", - [SETUP_WITH] = "SETUP_WITH", - [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", + [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [YIELD_VALUE] = "YIELD_VALUE", }; #endif // NEED_OPCODE_METADATA diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index c6ed5911b846bf..50bc14a57fc584 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -209,8 +209,9 @@ def write_function( "", ): with self.out.block("switch(opcode)"): - for instr, effect in data: - self.out.emit(f"case {instr.name}:") + effects = [(instr.name, effect) for instr, effect in data] + for name, effect in sorted(effects): + self.out.emit(f"case {name}:") self.out.emit(f" return {effect};") self.out.emit("default:") self.out.emit(" return -1;") @@ -433,7 +434,8 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No ";", ): # Write metadata for each instruction - for thing in self.everything: + sorted_things = sorted(self.everything, key = lambda t:t.name) + for thing in sorted_things: match thing: case parsing.InstDef(): self.write_metadata_for_inst(self.instrs[thing.name]) @@ -456,7 +458,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No ";", ): # Write macro expansion for each non-pseudo instruction - for mac in self.macro_instrs.values(): + for mac in sorted(self.macro_instrs.values(), key=lambda t: t.name): if is_super_instruction(mac): # Special-case the heck out of super-instructions self.write_super_expansions(mac.name) @@ -475,7 +477,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No "=", ";", ): - for name in self.opmap: + for name in sorted(self.opmap): self.out.emit(f'[{name}] = "{name}",') with self.metadata_item( @@ -589,7 +591,7 @@ def add(name: str) -> None: add("_EXIT_TRACE") add("_SET_IP") - for instr in self.instrs.values(): + for instr in sorted(self.instrs.values(), key=lambda t:t.name): # Skip ops that are also macros -- those are desugared inst()s if instr.name not in self.macros: add(instr.name) From a723a13bf135306cdc5999a959596bfb487e8f4f Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Thu, 14 Dec 2023 11:06:53 -0800 Subject: [PATCH 68/95] bpo-36796: Clean the error handling in _testcapimodule.c (GH-13085) --- Modules/_testcapi/datetime.c | 46 +++++++++++++++++++++++++++++------- Modules/_testcapimodule.c | 18 +++++++++----- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 88f992915fa8c1..b1796039f0d83a 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -85,17 +85,25 @@ make_timezones_capi(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, -18000, 0); PyObject *name = PyUnicode_FromString("EST"); + if (offset == NULL || name == NULL) { + Py_XDECREF(offset); + Py_XDECREF(name); + return NULL; + } PyObject *est_zone_capi = PyDateTimeAPI->TimeZone_FromTimeZone(offset, name); PyObject *est_zone_macro = PyTimeZone_FromOffsetAndName(offset, name); PyObject *est_zone_macro_noname = PyTimeZone_FromOffset(offset); - - Py_DecRef(offset); - Py_DecRef(name); - + Py_DECREF(offset); + Py_DECREF(name); + if (est_zone_capi == NULL || est_zone_macro == NULL || + est_zone_macro_noname == NULL) + { + goto error; + } PyObject *rv = PyTuple_New(3); if (rv == NULL) { - return NULL; + goto error; } PyTuple_SET_ITEM(rv, 0, est_zone_capi); @@ -103,6 +111,11 @@ make_timezones_capi(PyObject *self, PyObject *args) PyTuple_SET_ITEM(rv, 2, est_zone_macro_noname); return rv; +error: + Py_XDECREF(est_zone_capi); + Py_XDECREF(est_zone_macro); + Py_XDECREF(est_zone_macro_noname); + return NULL; } static PyObject * @@ -110,6 +123,11 @@ get_timezones_offset_zero(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, 0, 0); PyObject *name = PyUnicode_FromString(""); + if (offset == NULL || name == NULL) { + Py_XDECREF(offset); + Py_XDECREF(name); + return NULL; + } // These two should return the UTC singleton PyObject *utc_singleton_0 = PyTimeZone_FromOffset(offset); @@ -117,16 +135,28 @@ get_timezones_offset_zero(PyObject *self, PyObject *args) // This one will return +00:00 zone, but not the UTC singleton PyObject *non_utc_zone = PyTimeZone_FromOffsetAndName(offset, name); - - Py_DecRef(offset); - Py_DecRef(name); + Py_DECREF(offset); + Py_DECREF(name); + if (utc_singleton_0 == NULL || utc_singleton_1 == NULL || + non_utc_zone == NULL) + { + goto error; + } PyObject *rv = PyTuple_New(3); + if (rv == NULL) { + goto error; + } PyTuple_SET_ITEM(rv, 0, utc_singleton_0); PyTuple_SET_ITEM(rv, 1, utc_singleton_1); PyTuple_SET_ITEM(rv, 2, non_utc_zone); return rv; +error: + Py_XDECREF(utc_singleton_0); + Py_XDECREF(utc_singleton_1); + Py_XDECREF(non_utc_zone); + return NULL; } static PyObject * diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b3ddfae58e6fc0..3527dfa77279ac 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -196,11 +196,11 @@ test_dict_inner(int count) for (i = 0; i < count; i++) { v = PyLong_FromLong(i); if (v == NULL) { - return -1; + goto error; } if (PyDict_SetItem(dict, v, v) < 0) { Py_DECREF(v); - return -1; + goto error; } Py_DECREF(v); } @@ -214,11 +214,12 @@ test_dict_inner(int count) assert(v != UNINITIALIZED_PTR); i = PyLong_AS_LONG(v) + 1; o = PyLong_FromLong(i); - if (o == NULL) - return -1; + if (o == NULL) { + goto error; + } if (PyDict_SetItem(dict, k, o) < 0) { Py_DECREF(o); - return -1; + goto error; } Py_DECREF(o); k = v = UNINITIALIZED_PTR; @@ -236,6 +237,9 @@ test_dict_inner(int count) } else { return 0; } +error: + Py_DECREF(dict); + return -1; } @@ -1556,7 +1560,9 @@ test_structseq_newtype_doesnt_leak(PyObject *Py_UNUSED(self), descr.n_in_sequence = 1; PyTypeObject* structseq_type = PyStructSequence_NewType(&descr); - assert(structseq_type != NULL); + if (structseq_type == NULL) { + return NULL; + } assert(PyType_Check(structseq_type)); assert(PyType_FastSubclass(structseq_type, Py_TPFLAGS_TUPLE_SUBCLASS)); Py_DECREF(structseq_type); From 22511f77c2818a138a252e6ddae89725d082f8b0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 14 Dec 2023 20:14:50 +0100 Subject: [PATCH 69/95] gh-105912: document gotcha with using os.fork on macOS (#112871) * gh-105912: document gotcha with using os.fork on macOS Using ``fork(2)`` on macOS when also using higher-level system APIs in the parent proces can crash on macOS because those system APIs are not written to handle this usage pattern. There's nothing we can do about this other than documenting the problem. Co-authored-by: Carol Willing --- Doc/library/os.rst | 10 ++++++++++ Doc/library/pty.rst | 3 +++ Doc/library/urllib.request.rst | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f4566a6684e14c..2ff0b73560ac4c 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4383,6 +4383,11 @@ written in Python, such as a mail server's external command delivery program. If you use TLS sockets in an application calling ``fork()``, see the warning in the :mod:`ssl` documentation. + .. warning:: + + On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.8 Calling ``fork()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -4422,6 +4427,11 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.forkpty "" os.forkpty + .. warning:: + + On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.12 If Python is able to detect that your process has multiple threads, this now raises a :exc:`DeprecationWarning`. See the diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index af9378464edb9f..bd2f5ed45cb8b4 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -33,6 +33,9 @@ The :mod:`pty` module defines the following functions: file descriptor connected to the child's controlling terminal (and also to the child's standard input and output). + .. warning:: On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. function:: openpty() diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 040e28e9124014..0e18db73280a63 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -21,6 +21,14 @@ authentication, redirections, cookies and more. The `Requests package `_ is recommended for a higher-level HTTP client interface. +.. warning:: + + On macOS it is unsafe to use this module in programs using + :func:`os.fork` because the :func:`getproxies` implementation for + macOS uses a higher-level system API. Set the environment variable + ``no_proxy`` to ``*`` to avoid this problem + (e.g. ``os.environ["no_proxy"] = "*"``). + .. include:: ../includes/wasm-notavail.rst The :mod:`urllib.request` module defines the following functions: From 93cf7358d996e0e296046526bf9bc44755d597d1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 14 Dec 2023 13:15:29 -0600 Subject: [PATCH 70/95] Add recipe for totient() to demonstrate unique_justseen() and factor(). (gh-113131) --- Doc/library/itertools.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 56c66f670c74dd..83e2a9fdb7b464 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1128,6 +1128,14 @@ The following recipes have a more mathematical flavor: if n > 1: yield n + def totient(n): + "Count of natural numbers up to n that are coprime to n." + # https://mathworld.wolfram.com/TotientFunction.html + # totient(12) --> 4 because len([1, 5, 7, 11]) == 4 + for p in unique_justseen(factor(n)): + n = n // p * (p - 1) + return n + def nth_combination(iterable, r, index): "Equivalent to list(combinations(iterable, r))[index]" pool = tuple(iterable) @@ -1429,6 +1437,25 @@ The following recipes have a more mathematical flavor: >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True + >>> totient(0) # https://www.wolframalpha.com/input?i=totient+0 + 0 + >>> first_totients = [1, 1, 2, 2, 4, 2, 6, 4, 6, 4, 10, 4, 12, 6, 8, 8, 16, 6, + ... 18, 8, 12, 10, 22, 8, 20, 12, 18, 12, 28, 8, 30, 16, 20, 16, 24, 12, 36, 18, + ... 24, 16, 40, 12, 42, 20, 24, 22, 46, 16, 42, 20, 32, 24, 52, 18, 40, 24, 36, + ... 28, 58, 16, 60, 30, 36, 32, 48, 20, 66, 32, 44] # https://oeis.org/A000010 + ... + >>> list(map(totient, range(1, 70))) == first_totients + True + >>> reference_totient = lambda n: sum(math.gcd(t, n) == 1 for t in range(1, n+1)) + >>> all(totient(n) == reference_totient(n) for n in range(1000)) + True + >>> totient(128_884_753_939) == 128_884_753_938 # large prime + True + >>> totient(999953 * 999983) == 999952 * 999982 # large semiprime + True + >>> totient(6 ** 20) == 1 * 2**19 * 2 * 3**19 # repeated primes + True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] From 006355b2a966060541166608ecfec4c957b85f55 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 14 Dec 2023 19:25:55 +0000 Subject: [PATCH 71/95] gh-101100: Fix Sphinx nitpicks in `library/collections.abc.rst` (#113116) --- Doc/conf.py | 3 +- Doc/library/collections.abc.rst | 80 +++++++++++++++++---------------- Doc/tools/.nitignore | 1 - 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 0d7c0b553eaa74..4077eaf5a139b0 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -245,10 +245,11 @@ # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), - # Attributes that definitely should be documented better, + # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: ('py:attr', '__annotations__'), ('py:attr', '__wrapped__'), + ('py:meth', 'index'), # list.index, tuple.index, etc. ] # gh-106948: Copy standard C types declared in the "c:type" domain to the diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index edc078953290d7..e0c72ff9249ee7 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -22,7 +22,7 @@ This module provides :term:`abstract base classes ` that can be used to test whether a class provides a particular interface; for -example, whether it is :term:`hashable` or whether it is a mapping. +example, whether it is :term:`hashable` or whether it is a :term:`mapping`. An :func:`issubclass` or :func:`isinstance` test for an interface works in one of three ways. @@ -73,7 +73,7 @@ of the API: >>> isinstance(D(), Sequence) True -In this example, class :class:`D` does not need to define +In this example, class :class:`!D` does not need to define ``__contains__``, ``__iter__``, and ``__reversed__`` because the :ref:`in-operator `, the :term:`iteration ` logic, and the :func:`reversed` function automatically fall back to @@ -183,14 +183,14 @@ ABC Inherits from Abstract Methods Mi .. rubric:: Footnotes -.. [1] These ABCs override :meth:`object.__subclasshook__` to support +.. [1] These ABCs override :meth:`~abc.ABCMeta.__subclasshook__` to support testing an interface by verifying the required methods are present and have not been set to :const:`None`. This only works for simple interfaces. More complex interfaces require registration or direct subclassing. .. [2] Checking ``isinstance(obj, Iterable)`` detects classes that are - registered as :class:`Iterable` or that have an :meth:`__iter__` + registered as :class:`Iterable` or that have an :meth:`~container.__iter__` method, but it does not detect classes that iterate with the :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -202,26 +202,27 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Container - ABC for classes that provide the :meth:`__contains__` method. + ABC for classes that provide the :meth:`~object.__contains__` method. .. class:: Hashable - ABC for classes that provide the :meth:`__hash__` method. + ABC for classes that provide the :meth:`~object.__hash__` method. .. class:: Sized - ABC for classes that provide the :meth:`__len__` method. + ABC for classes that provide the :meth:`~object.__len__` method. .. class:: Callable - ABC for classes that provide the :meth:`__call__` method. + ABC for classes that provide the :meth:`~object.__call__` method. .. class:: Iterable - ABC for classes that provide the :meth:`__iter__` method. + ABC for classes that provide the :meth:`~container.__iter__` method. Checking ``isinstance(obj, Iterable)`` detects classes that are registered - as :class:`Iterable` or that have an :meth:`__iter__` method, but it does + as :class:`Iterable` or that have an :meth:`~container.__iter__` method, + but it does not detect classes that iterate with the :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -240,17 +241,17 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Reversible - ABC for iterable classes that also provide the :meth:`__reversed__` + ABC for iterable classes that also provide the :meth:`~object.__reversed__` method. .. versionadded:: 3.6 .. class:: Generator - ABC for generator classes that implement the protocol defined in - :pep:`342` that extends iterators with the :meth:`~generator.send`, + ABC for :term:`generator` classes that implement the protocol defined in + :pep:`342` that extends :term:`iterators ` with the + :meth:`~generator.send`, :meth:`~generator.throw` and :meth:`~generator.close` methods. - See also the definition of :term:`generator`. .. versionadded:: 3.5 @@ -261,7 +262,7 @@ Collections Abstract Base Classes -- Detailed Descriptions ABCs for read-only and mutable :term:`sequences `. Implementation note: Some of the mixin methods, such as - :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make + :meth:`~container.__iter__`, :meth:`~object.__reversed__` and :meth:`index`, make repeated calls to the underlying :meth:`~object.__getitem__` method. Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; @@ -282,7 +283,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Set MutableSet - ABCs for read-only and mutable sets. + ABCs for read-only and mutable :ref:`sets `. .. class:: Mapping MutableMapping @@ -299,16 +300,16 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Awaitable ABC for :term:`awaitable` objects, which can be used in :keyword:`await` - expressions. Custom implementations must provide the :meth:`__await__` - method. + expressions. Custom implementations must provide the + :meth:`~object.__await__` method. :term:`Coroutine ` objects and instances of the :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. + In CPython, generator-based coroutines (:term:`generators ` + decorated with :func:`@types.coroutine `) are + *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -316,17 +317,17 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Coroutine - ABC for coroutine compatible classes. These implement the + ABC for :term:`coroutine` compatible classes. These implement the following methods, defined in :ref:`coroutine-objects`: :meth:`~coroutine.send`, :meth:`~coroutine.throw`, and :meth:`~coroutine.close`. Custom implementations must also implement - :meth:`__await__`. All :class:`Coroutine` instances are also instances of - :class:`Awaitable`. See also the definition of :term:`coroutine`. + :meth:`~object.__await__`. All :class:`Coroutine` instances are also + instances of :class:`Awaitable`. .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. + In CPython, generator-based coroutines (:term:`generators ` + decorated with :func:`@types.coroutine `) are + *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -334,7 +335,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: AsyncIterable - ABC for classes that provide ``__aiter__`` method. See also the + ABC for classes that provide an ``__aiter__`` method. See also the definition of :term:`asynchronous iterable`. .. versionadded:: 3.5 @@ -348,7 +349,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: AsyncGenerator - ABC for asynchronous generator classes that implement the protocol + ABC for :term:`asynchronous generator` classes that implement the protocol defined in :pep:`525` and :pep:`492`. .. versionadded:: 3.6 @@ -373,9 +374,9 @@ particular functionality, for example:: Several of the ABCs are also useful as mixins that make it easier to develop classes supporting container APIs. For example, to write a class supporting the full :class:`Set` API, it is only necessary to supply the three underlying -abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`. -The ABC supplies the remaining methods such as :meth:`__and__` and -:meth:`isdisjoint`:: +abstract methods: :meth:`~object.__contains__`, :meth:`~container.__iter__`, and +:meth:`~object.__len__`. The ABC supplies the remaining methods such as +:meth:`!__and__` and :meth:`~frozenset.isdisjoint`:: class ListBasedSet(collections.abc.Set): ''' Alternate set implementation favoring space over speed @@ -403,23 +404,24 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: (1) Since some set operations create new sets, the default mixin methods need - a way to create new instances from an iterable. The class constructor is + a way to create new instances from an :term:`iterable`. The class constructor is assumed to have a signature in the form ``ClassName(iterable)``. - That assumption is factored-out to an internal classmethod called - :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set. + That assumption is factored-out to an internal :class:`classmethod` called + :meth:`!_from_iterable` which calls ``cls(iterable)`` to produce a new set. If the :class:`Set` mixin is being used in a class with a different - constructor signature, you will need to override :meth:`_from_iterable` + constructor signature, you will need to override :meth:`!_from_iterable` with a classmethod or regular method that can construct new instances from an iterable argument. (2) To override the comparisons (presumably for speed, as the - semantics are fixed), redefine :meth:`__le__` and :meth:`__ge__`, + semantics are fixed), redefine :meth:`~object.__le__` and + :meth:`~object.__ge__`, then the other operations will automatically follow suit. (3) - The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value - for the set; however, :meth:`__hash__` is not defined because not all sets + The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value + for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, inherit from both :meth:`Set` and :meth:`Hashable`, then define ``__hash__ = Set._hash``. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4d1d31d44fcf75..ca0cb84d850928 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -35,7 +35,6 @@ Doc/library/bdb.rst Doc/library/bisect.rst Doc/library/calendar.rst Doc/library/cmd.rst -Doc/library/collections.abc.rst Doc/library/collections.rst Doc/library/concurrent.futures.rst Doc/library/configparser.rst From becad9a2a1b5f3deaad24759daec95014218e0db Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 14 Dec 2023 14:36:40 -0600 Subject: [PATCH 72/95] Remove itertool recipe with low pedagogical value (gh-113138) --- Doc/library/itertools.rst | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 83e2a9fdb7b464..36cea9a835f302 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1136,24 +1136,6 @@ The following recipes have a more mathematical flavor: n = n // p * (p - 1) return n - def nth_combination(iterable, r, index): - "Equivalent to list(combinations(iterable, r))[index]" - pool = tuple(iterable) - n = len(pool) - c = math.comb(n, r) - if index < 0: - index += c - if index < 0 or index >= c: - raise IndexError - result = [] - while r: - c, n, r = c*r//n, n-1, r-1 - while index >= c: - index -= c - c, n = c*(n-r)//n, n-1 - result.append(pool[-1-n]) - return tuple(result) - .. doctest:: :hide: @@ -1577,20 +1559,6 @@ The following recipes have a more mathematical flavor: >>> first_true('ABC0DEF1', '9', str.isdigit) '0' - >>> population = 'ABCDEFGH' - >>> for r in range(len(population) + 1): - ... seq = list(combinations(population, r)) - ... for i in range(len(seq)): - ... assert nth_combination(population, r, i) == seq[i] - ... for i in range(-len(seq), 0): - ... assert nth_combination(population, r, i) == seq[i] - - >>> iterable = 'abcde' - >>> r = 3 - >>> combos = list(combinations(iterable, r)) - >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) - True - .. testcode:: :hide: @@ -1617,6 +1585,24 @@ The following recipes have a more mathematical flavor: for (a, _), (b, c) in pairwise(pairwise(iterable)): yield a, b, c + def nth_combination(iterable, r, index): + "Equivalent to list(combinations(iterable, r))[index]" + pool = tuple(iterable) + n = len(pool) + c = math.comb(n, r) + if index < 0: + index += c + if index < 0 or index >= c: + raise IndexError + result = [] + while r: + c, n, r = c*r//n, n-1, r-1 + while index >= c: + index -= c + c, n = c*(n-r)//n, n-1 + result.append(pool[-1-n]) + return tuple(result) + .. doctest:: :hide: @@ -1632,3 +1618,17 @@ The following recipes have a more mathematical flavor: >>> list(triplewise('ABCDEFG')) [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] + + >>> population = 'ABCDEFGH' + >>> for r in range(len(population) + 1): + ... seq = list(combinations(population, r)) + ... for i in range(len(seq)): + ... assert nth_combination(population, r, i) == seq[i] + ... for i in range(-len(seq), 0): + ... assert nth_combination(population, r, i) == seq[i] + + >>> iterable = 'abcde' + >>> r = 3 + >>> combos = list(combinations(iterable, r)) + >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) + True From 961f1043a022ddeff314f63480a232335a598d6a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Dec 2023 22:48:36 +0200 Subject: [PATCH 73/95] gh-101100: Fix Sphinx warnings in `whatsnew/2.3.rst` (#112373) --- Doc/tools/.nitignore | 1 - Doc/whatsnew/2.3.rst | 276 +++++++++++++++++++++---------------------- 2 files changed, 138 insertions(+), 139 deletions(-) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ca0cb84d850928..20580f78e07b5d 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -118,7 +118,6 @@ Doc/using/windows.rst Doc/whatsnew/2.0.rst Doc/whatsnew/2.1.rst Doc/whatsnew/2.2.rst -Doc/whatsnew/2.3.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 0c77b339a182c9..8ebcbfaf248551 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -40,10 +40,10 @@ new feature. PEP 218: A Standard Set Datatype ================================ -The new :mod:`sets` module contains an implementation of a set datatype. The +The new :mod:`!sets` module contains an implementation of a set datatype. The :class:`Set` class is for mutable sets, sets that can have members added and -removed. The :class:`ImmutableSet` class is for sets that can't be modified, -and instances of :class:`ImmutableSet` can therefore be used as dictionary keys. +removed. The :class:`!ImmutableSet` class is for sets that can't be modified, +and instances of :class:`!ImmutableSet` can therefore be used as dictionary keys. Sets are built on top of dictionaries, so the elements within a set must be hashable. @@ -63,10 +63,10 @@ Here's a simple example:: Set([1, 2, 5]) >>> -The union and intersection of sets can be computed with the :meth:`union` and -:meth:`intersection` methods; an alternative notation uses the bitwise operators +The union and intersection of sets can be computed with the :meth:`~frozenset.union` and +:meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`union_update` and :meth:`intersection_update`. :: +:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) @@ -97,7 +97,7 @@ in-place version with the ungainly name :meth:`symmetric_difference_update`. :: Set([1, 2, 5, 6]) >>> -There are also :meth:`issubset` and :meth:`issuperset` methods for checking +There are also :meth:`!issubset` and :meth:`!issuperset` methods for checking whether one set is a subset or superset of another:: >>> S1 = sets.Set([1,2,3]) @@ -166,7 +166,7 @@ statement isn't allowed inside the :keyword:`try` block of a :keyword:`!try`...\ :keyword:`!finally` statement; read :pep:`255` for a full explanation of the interaction between :keyword:`!yield` and exceptions.) -Here's a sample usage of the :func:`generate_ints` generator:: +Here's a sample usage of the :func:`!generate_ints` generator:: >>> gen = generate_ints(3) >>> gen @@ -227,7 +227,7 @@ like:: sentence := "Store it in the neighboring harbor" if (i := find("or", sentence)) > 5 then write(i) -In Icon the :func:`find` function returns the indexes at which the substring +In Icon the :func:`!find` function returns the indexes at which the substring "or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon retries it with the second value of 23. 23 is greater than 5, so the comparison @@ -345,7 +345,7 @@ Python now allows using arbitrary Unicode strings (within the limitations of the file system) for all functions that expect file names, most notably the :func:`open` built-in function. If a Unicode string is passed to :func:`os.listdir`, Python now returns a list of Unicode strings. A new -function, :func:`os.getcwdu`, returns the current directory as a Unicode string. +function, :func:`!os.getcwdu`, returns the current directory as a Unicode string. Byte strings still work as file names, and on Windows Python will transparently convert them to Unicode using the ``mbcs`` encoding. @@ -386,10 +386,10 @@ one followed by the platform on which Python is running. Opening a file with the mode ``'U'`` or ``'rU'`` will open a file for reading in :term:`universal newlines` mode. All three line ending conventions will be translated to a ``'\n'`` in the strings returned by the various file methods such as -:meth:`read` and :meth:`readline`. +:meth:`!read` and :meth:`!readline`. Universal newline support is also used when importing modules and when executing -a file with the :func:`execfile` function. This means that Python modules can +a file with the :func:`!execfile` function. This means that Python modules can be shared between all three operating systems without needing to convert the line-endings. @@ -450,16 +450,16 @@ records to standard error or to a file or socket, send them to the system log, or even e-mail them to a particular address; of course, it's also possible to write your own handler classes. -The :class:`Logger` class is the primary class. Most application code will deal -with one or more :class:`Logger` objects, each one used by a particular -subsystem of the application. Each :class:`Logger` is identified by a name, and +The :class:`~logging.Logger` class is the primary class. Most application code will deal +with one or more :class:`~logging.Logger` objects, each one used by a particular +subsystem of the application. Each :class:`~logging.Logger` is identified by a name, and names are organized into a hierarchy using ``.`` as the component separator. -For example, you might have :class:`Logger` instances named ``server``, +For example, you might have :class:`~logging.Logger` instances named ``server``, ``server.auth`` and ``server.network``. The latter two instances are below ``server`` in the hierarchy. This means that if you turn up the verbosity for ``server`` or direct ``server`` messages to a different handler, the changes will also apply to records logged to ``server.auth`` and ``server.network``. -There's also a root :class:`Logger` that's the parent of all other loggers. +There's also a root :class:`~logging.Logger` that's the parent of all other loggers. For simple uses, the :mod:`logging` package contains some convenience functions that always use the root log:: @@ -480,14 +480,14 @@ This produces the following output:: In the default configuration, informational and debugging messages are suppressed and the output is sent to standard error. You can enable the display -of informational and debugging messages by calling the :meth:`setLevel` method +of informational and debugging messages by calling the :meth:`~logging.Logger.setLevel` method on the root logger. -Notice the :func:`warning` call's use of string formatting operators; all of the +Notice the :func:`~logging.warning` call's use of string formatting operators; all of the functions for logging messages take the arguments ``(msg, arg1, arg2, ...)`` and log the string resulting from ``msg % (arg1, arg2, ...)``. -There's also an :func:`exception` function that records the most recent +There's also an :func:`~logging.exception` function that records the most recent traceback. Any of the other functions will also record the traceback if you specify a true value for the keyword argument *exc_info*. :: @@ -517,16 +517,16 @@ it if it doesn't exist yet. ``getLogger(None)`` returns the root logger. :: ... Log records are usually propagated up the hierarchy, so a message logged to -``server.auth`` is also seen by ``server`` and ``root``, but a :class:`Logger` -can prevent this by setting its :attr:`propagate` attribute to :const:`False`. +``server.auth`` is also seen by ``server`` and ``root``, but a :class:`~logging.Logger` +can prevent this by setting its :attr:`~logging.Logger.propagate` attribute to :const:`False`. There are more classes provided by the :mod:`logging` package that can be -customized. When a :class:`Logger` instance is told to log a message, it -creates a :class:`LogRecord` instance that is sent to any number of different -:class:`Handler` instances. Loggers and handlers can also have an attached list -of filters, and each filter can cause the :class:`LogRecord` to be ignored or +customized. When a :class:`~logging.Logger` instance is told to log a message, it +creates a :class:`~logging.LogRecord` instance that is sent to any number of different +:class:`~logging.Handler` instances. Loggers and handlers can also have an attached list +of filters, and each filter can cause the :class:`~logging.LogRecord` to be ignored or can modify the record before passing it along. When they're finally output, -:class:`LogRecord` instances are converted to text by a :class:`Formatter` +:class:`~logging.LogRecord` instances are converted to text by a :class:`~logging.Formatter` class. All of these classes can be replaced by your own specially written classes. @@ -550,7 +550,7 @@ PEP 285: A Boolean Type ======================= A Boolean type was added to Python 2.3. Two new constants were added to the -:mod:`__builtin__` module, :const:`True` and :const:`False`. (:const:`True` and +:mod:`!__builtin__` module, :const:`True` and :const:`False`. (:const:`True` and :const:`False` constants were added to the built-ins in Python 2.2.1, but the 2.2.1 versions are simply set to integer values of 1 and 0 and aren't a different type.) @@ -662,7 +662,7 @@ a central catalog server. The resulting catalog is available from https://pypi.org. To make the catalog a bit more useful, a new optional *classifiers* keyword -argument has been added to the Distutils :func:`setup` function. A list of +argument has been added to the Distutils :func:`!setup` function. A list of `Trove `_-style strings can be supplied to help classify the software. @@ -703,14 +703,14 @@ PEP 302: New Import Hooks ========================= While it's been possible to write custom import hooks ever since the -:mod:`ihooks` module was introduced in Python 1.3, no one has ever been really +:mod:`!ihooks` module was introduced in Python 1.3, no one has ever been really happy with it because writing new import hooks is difficult and messy. There -have been various proposed alternatives such as the :mod:`imputil` and :mod:`iu` +have been various proposed alternatives such as the :mod:`!imputil` and :mod:`!iu` modules, but none of them has ever gained much acceptance, and none of them were easily usable from C code. :pep:`302` borrows ideas from its predecessors, especially from Gordon -McMillan's :mod:`iu` module. Three new items are added to the :mod:`sys` +McMillan's :mod:`!iu` module. Three new items are added to the :mod:`sys` module: * ``sys.path_hooks`` is a list of callable objects; most often they'll be @@ -790,7 +790,7 @@ package is much simpler:: for line in reader: print line -The :func:`reader` function takes a number of different options. The field +The :func:`~csv.reader` function takes a number of different options. The field separator isn't limited to the comma and can be changed to any character, and so can the quoting and line-ending characters. @@ -814,7 +814,7 @@ of tuples or lists, quoting strings that contain the delimiter. PEP 307: Pickle Enhancements ============================ -The :mod:`pickle` and :mod:`cPickle` modules received some attention during the +The :mod:`pickle` and :mod:`!cPickle` modules received some attention during the 2.3 development cycle. In 2.2, new-style classes could be pickled without difficulty, but they weren't pickled very compactly; :pep:`307` quotes a trivial example where a new-style class results in a pickled string three times longer @@ -829,13 +829,13 @@ fanciest protocol available. Unpickling is no longer considered a safe operation. 2.2's :mod:`pickle` provided hooks for trying to prevent unsafe classes from being unpickled -(specifically, a :attr:`__safe_for_unpickling__` attribute), but none of this +(specifically, a :attr:`!__safe_for_unpickling__` attribute), but none of this code was ever audited and therefore it's all been ripped out in 2.3. You should not unpickle untrusted data in any version of Python. To reduce the pickling overhead for new-style classes, a new interface for customizing pickling was added using three special methods: -:meth:`__getstate__`, :meth:`__setstate__`, and :meth:`__getnewargs__`. Consult +:meth:`~object.__getstate__`, :meth:`~object.__setstate__`, and :meth:`~object.__getnewargs__`. Consult :pep:`307` for the full semantics of these methods. As a way to compress pickles yet further, it's now possible to use integer codes @@ -939,7 +939,7 @@ Or use slice objects directly in subscripts:: To simplify implementing sequences that support extended slicing, slice objects now have a method ``indices(length)`` which, given the length of a sequence, returns a ``(start, stop, step)`` tuple that can be passed directly to -:func:`range`. :meth:`indices` handles omitted and out-of-bounds indices in a +:func:`range`. :meth:`!indices` handles omitted and out-of-bounds indices in a manner consistent with regular slices (and this innocuous phrase hides a welter of confusing details!). The method is intended to be used like this:: @@ -1042,7 +1042,7 @@ Here are all of the changes that Python 2.3 makes to the core Python language. execute any assertions. * Most type objects are now callable, so you can use them to create new objects - such as functions, classes, and modules. (This means that the :mod:`new` module + such as functions, classes, and modules. (This means that the :mod:`!new` module can be deprecated in a future Python version, because you can now use the type objects available in the :mod:`types` module.) For example, you can create a new module object with the following code: @@ -1069,11 +1069,11 @@ Here are all of the changes that Python 2.3 makes to the core Python language. * Using ``None`` as a variable name will now result in a :exc:`SyntaxWarning` warning. In a future version of Python, ``None`` may finally become a keyword. -* The :meth:`xreadlines` method of file objects, introduced in Python 2.1, is no +* The :meth:`!xreadlines` method of file objects, introduced in Python 2.1, is no longer necessary because files now behave as their own iterator. - :meth:`xreadlines` was originally introduced as a faster way to loop over all + :meth:`!xreadlines` was originally introduced as a faster way to loop over all the lines in a file, but now you can simply write ``for line in file_obj``. - File objects also have a new read-only :attr:`encoding` attribute that gives the + File objects also have a new read-only :attr:`!encoding` attribute that gives the encoding used by the file; Unicode strings written to the file will be automatically converted to bytes using the given encoding. @@ -1096,12 +1096,12 @@ Here are all of the changes that Python 2.3 makes to the core Python language. switching overhead. Some multithreaded applications may suffer slower response time, but that's easily fixed by setting the limit back to a lower number using ``sys.setcheckinterval(N)``. The limit can be retrieved with the new - :func:`sys.getcheckinterval` function. + :func:`!sys.getcheckinterval` function. * One minor but far-reaching change is that the names of extension types defined by the modules included with Python now contain the module and a ``'.'`` in front of the type name. For example, in Python 2.2, if you created a socket and - printed its :attr:`__class__`, you'd get this output:: + printed its :attr:`!__class__`, you'd get this output:: >>> s = socket.socket() >>> s.__class__ @@ -1138,9 +1138,9 @@ String Changes True Note that this doesn't tell you where the substring starts; if you need that - information, use the :meth:`find` string method. + information, use the :meth:`~str.find` string method. -* The :meth:`strip`, :meth:`lstrip`, and :meth:`rstrip` string methods now have +* The :meth:`~str.strip`, :meth:`~str.lstrip`, and :meth:`~str.rstrip` string methods now have an optional argument for specifying the characters to strip. The default is still to remove all whitespace characters:: @@ -1156,13 +1156,13 @@ String Changes (Suggested by Simon Brunning and implemented by Walter Dörwald.) -* The :meth:`startswith` and :meth:`endswith` string methods now accept negative +* The :meth:`~str.startswith` and :meth:`~str.endswith` string methods now accept negative numbers for the *start* and *end* parameters. -* Another new string method is :meth:`zfill`, originally a function in the - :mod:`string` module. :meth:`zfill` pads a numeric string with zeros on the +* Another new string method is :meth:`~str.zfill`, originally a function in the + :mod:`string` module. :meth:`~str.zfill` pads a numeric string with zeros on the left until it's the specified width. Note that the ``%`` operator is still more - flexible and powerful than :meth:`zfill`. :: + flexible and powerful than :meth:`~str.zfill`. :: >>> '45'.zfill(4) '0045' @@ -1173,10 +1173,10 @@ String Changes (Contributed by Walter Dörwald.) -* A new type object, :class:`basestring`, has been added. Both 8-bit strings and +* A new type object, :class:`!basestring`, has been added. Both 8-bit strings and Unicode strings inherit from this type, so ``isinstance(obj, basestring)`` will return :const:`True` for either kind of string. It's a completely abstract - type, so you can't create :class:`basestring` instances. + type, so you can't create :class:`!basestring` instances. * Interned strings are no longer immortal and will now be garbage-collected in the usual way when the only reference to them is from the internal dictionary of @@ -1191,7 +1191,7 @@ Optimizations * The creation of new-style class instances has been made much faster; they're now faster than classic classes! -* The :meth:`sort` method of list objects has been extensively rewritten by Tim +* The :meth:`~list.sort` method of list objects has been extensively rewritten by Tim Peters, and the implementation is significantly faster. * Multiplication of large long integers is now much faster thanks to an @@ -1203,7 +1203,7 @@ Optimizations increase, depending on your compiler's idiosyncrasies. See section :ref:`23section-other` for a longer explanation. (Removed by Michael Hudson.) -* :func:`xrange` objects now have their own iterator, making ``for i in +* :func:`!xrange` objects now have their own iterator, making ``for i in xrange(n)`` slightly faster than ``for i in range(n)``. (Patch by Raymond Hettinger.) @@ -1230,21 +1230,21 @@ complete list of changes, or look through the CVS logs for all the details. operator to add another array's contents, and the ``*=`` assignment operator to repeat an array. (Contributed by Jason Orendorff.) -* The :mod:`bsddb` module has been replaced by version 4.1.6 of the `PyBSDDB +* The :mod:`!bsddb` module has been replaced by version 4.1.6 of the `PyBSDDB `_ package, providing a more complete interface to the transactional features of the BerkeleyDB library. - The old version of the module has been renamed to :mod:`bsddb185` and is no + The old version of the module has been renamed to :mod:`!bsddb185` and is no longer built automatically; you'll have to edit :file:`Modules/Setup` to enable - it. Note that the new :mod:`bsddb` package is intended to be compatible with + it. Note that the new :mod:`!bsddb` package is intended to be compatible with the old module, so be sure to file bugs if you discover any incompatibilities. When upgrading to Python 2.3, if the new interpreter is compiled with a new version of the underlying BerkeleyDB library, you will almost certainly have to convert your database files to the new version. You can do this fairly easily with the new scripts :file:`db2pickle.py` and :file:`pickle2db.py` which you will find in the distribution's :file:`Tools/scripts` directory. If you've - already been using the PyBSDDB package and importing it as :mod:`bsddb3`, you - will have to change your ``import`` statements to import it as :mod:`bsddb`. + already been using the PyBSDDB package and importing it as :mod:`!bsddb3`, you + will have to change your ``import`` statements to import it as :mod:`!bsddb`. * The new :mod:`bz2` module is an interface to the bz2 data compression library. bz2-compressed data is usually smaller than corresponding @@ -1253,11 +1253,11 @@ complete list of changes, or look through the CVS logs for all the details. * A set of standard date/time types has been added in the new :mod:`datetime` module. See the following section for more details. -* The Distutils :class:`Extension` class now supports an extra constructor +* The Distutils :class:`!Extension` class now supports an extra constructor argument named *depends* for listing additional source files that an extension depends on. This lets Distutils recompile the module if any of the dependency files are modified. For example, if :file:`sampmodule.c` includes the header - file :file:`sample.h`, you would create the :class:`Extension` object like + file :file:`sample.h`, you would create the :class:`!Extension` object like this:: ext = Extension("samp", @@ -1268,21 +1268,21 @@ complete list of changes, or look through the CVS logs for all the details. (Contributed by Jeremy Hylton.) * Other minor changes to Distutils: it now checks for the :envvar:`CC`, - :envvar:`CFLAGS`, :envvar:`CPP`, :envvar:`LDFLAGS`, and :envvar:`CPPFLAGS` + :envvar:`CFLAGS`, :envvar:`!CPP`, :envvar:`LDFLAGS`, and :envvar:`CPPFLAGS` environment variables, using them to override the settings in Python's configuration (contributed by Robert Weber). * Previously the :mod:`doctest` module would only search the docstrings of public methods and functions for test cases, but it now also examines private - ones as well. The :func:`DocTestSuite` function creates a + ones as well. The :func:`~doctest.DocTestSuite` function creates a :class:`unittest.TestSuite` object from a set of :mod:`doctest` tests. * The new ``gc.get_referents(object)`` function returns a list of all the objects referenced by *object*. -* The :mod:`getopt` module gained a new function, :func:`gnu_getopt`, that - supports the same arguments as the existing :func:`getopt` function but uses - GNU-style scanning mode. The existing :func:`getopt` stops processing options as +* The :mod:`getopt` module gained a new function, :func:`~getopt.gnu_getopt`, that + supports the same arguments as the existing :func:`~getopt.getopt` function but uses + GNU-style scanning mode. The existing :func:`~getopt.getopt` stops processing options as soon as a non-option argument is encountered, but in GNU-style mode processing continues, meaning that options and arguments can be mixed. For example:: @@ -1311,7 +1311,7 @@ complete list of changes, or look through the CVS logs for all the details. O(lg n). (See https://xlinux.nist.gov/dads//HTML/priorityque.html for more information about the priority queue data structure.) - The :mod:`heapq` module provides :func:`heappush` and :func:`heappop` functions + The :mod:`heapq` module provides :func:`~heapq.heappush` and :func:`~heapq.heappop` functions for adding and removing items while maintaining the heap property on top of some other mutable Python sequence type. Here's an example that uses a Python list:: @@ -1343,7 +1343,7 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`itertools` contains a number of useful functions for use with iterators, inspired by various functions provided by the ML and Haskell languages. For example, ``itertools.ifilter(predicate, iterator)`` returns all - elements in the iterator for which the function :func:`predicate` returns + elements in the iterator for which the function :func:`!predicate` returns :const:`True`, and ``itertools.repeat(obj, N)`` returns ``obj`` *N* times. There are a number of other functions in the module; see the package's reference documentation for details. @@ -1356,9 +1356,9 @@ complete list of changes, or look through the CVS logs for all the details. was added to :func:`math.log` to make it easier to compute logarithms for bases other than ``e`` and ``10``. (Contributed by Raymond Hettinger.) -* Several new POSIX functions (:func:`getpgid`, :func:`killpg`, :func:`lchown`, - :func:`loadavg`, :func:`major`, :func:`makedev`, :func:`minor`, and - :func:`mknod`) were added to the :mod:`posix` module that underlies the +* Several new POSIX functions (:func:`!getpgid`, :func:`!killpg`, :func:`!lchown`, + :func:`!loadavg`, :func:`!major`, :func:`!makedev`, :func:`!minor`, and + :func:`!mknod`) were added to the :mod:`posix` module that underlies the :mod:`os` module. (Contributed by Gustavo Niemeyer, Geert Jansen, and Denis S. Otkidach.) @@ -1368,9 +1368,9 @@ complete list of changes, or look through the CVS logs for all the details. During testing, it was found that some applications will break if time stamps are floats. For compatibility, when using the tuple interface of the - :class:`stat_result` time stamps will be represented as integers. When using + :class:`~os.stat_result` time stamps will be represented as integers. When using named fields (a feature first introduced in Python 2.2), time stamps are still - represented as integers, unless :func:`os.stat_float_times` is invoked to enable + represented as integers, unless :func:`!os.stat_float_times` is invoked to enable float return values:: >>> os.stat("/tmp").st_mtime @@ -1391,7 +1391,7 @@ complete list of changes, or look through the CVS logs for all the details. automatically generate a usage message. See the following section for more details. -* The old and never-documented :mod:`linuxaudiodev` module has been deprecated, +* The old and never-documented :mod:`!linuxaudiodev` module has been deprecated, and a new version named :mod:`!ossaudiodev` has been added. The module was renamed because the OSS sound drivers can be used on platforms other than Linux, and the interface has also been tidied and brought up to date in various ways. @@ -1402,14 +1402,14 @@ complete list of changes, or look through the CVS logs for all the details. functions for getting the architecture, CPU type, the Windows OS version, and even the Linux distribution version. (Contributed by Marc-André Lemburg.) -* The parser objects provided by the :mod:`pyexpat` module can now optionally +* The parser objects provided by the :mod:`pyexpat ` module can now optionally buffer character data, resulting in fewer calls to your character data handler and therefore faster performance. Setting the parser object's - :attr:`buffer_text` attribute to :const:`True` will enable buffering. + :attr:`~xml.parsers.expat.xmlparser.buffer_text` attribute to :const:`True` will enable buffering. * The ``sample(population, k)`` function was added to the :mod:`random` - module. *population* is a sequence or :class:`xrange` object containing the - elements of a population, and :func:`sample` chooses *k* elements from the + module. *population* is a sequence or :class:`!xrange` object containing the + elements of a population, and :func:`~random.sample` chooses *k* elements from the population without replacing chosen elements. *k* can be any value up to ``len(population)``. For example:: @@ -1436,20 +1436,20 @@ complete list of changes, or look through the CVS logs for all the details. (All changes contributed by Raymond Hettinger.) * The :mod:`readline` module also gained a number of new functions: - :func:`get_history_item`, :func:`get_current_history_length`, and - :func:`redisplay`. + :func:`~readline.get_history_item`, :func:`~readline.get_current_history_length`, and + :func:`~readline.redisplay`. -* The :mod:`rexec` and :mod:`Bastion` modules have been declared dead, and +* The :mod:`!rexec` and :mod:`!Bastion` modules have been declared dead, and attempts to import them will fail with a :exc:`RuntimeError`. New-style classes provide new ways to break out of the restricted execution environment provided - by :mod:`rexec`, and no one has interest in fixing them or time to do so. If - you have applications using :mod:`rexec`, rewrite them to use something else. + by :mod:`!rexec`, and no one has interest in fixing them or time to do so. If + you have applications using :mod:`!rexec`, rewrite them to use something else. (Sticking with Python 2.2 or 2.1 will not make your applications any safer - because there are known bugs in the :mod:`rexec` module in those versions. To - repeat: if you're using :mod:`rexec`, stop using it immediately.) + because there are known bugs in the :mod:`!rexec` module in those versions. To + repeat: if you're using :mod:`!rexec`, stop using it immediately.) -* The :mod:`rotor` module has been deprecated because the algorithm it uses for +* The :mod:`!rotor` module has been deprecated because the algorithm it uses for encryption is not believed to be secure. If you need encryption, use one of the several AES Python modules that are available separately. @@ -1474,9 +1474,9 @@ complete list of changes, or look through the CVS logs for all the details. * On Windows, the :mod:`socket` module now ships with Secure Sockets Layer (SSL) support. -* The value of the C :c:macro:`PYTHON_API_VERSION` macro is now exposed at the +* The value of the C :c:macro:`!PYTHON_API_VERSION` macro is now exposed at the Python level as ``sys.api_version``. The current exception can be cleared by - calling the new :func:`sys.exc_clear` function. + calling the new :func:`!sys.exc_clear` function. * The new :mod:`tarfile` module allows reading from and writing to :program:`tar`\ -format archive files. (Contributed by Lars Gustäbel.) @@ -1486,7 +1486,7 @@ complete list of changes, or look through the CVS logs for all the details. string and returns a list containing the text split into lines of no more than the chosen width. The ``fill(text, width)`` function returns a single string, reformatted to fit into lines no longer than the chosen width. (As you - can guess, :func:`fill` is built on top of :func:`wrap`. For example:: + can guess, :func:`~textwrap.fill` is built on top of :func:`~textwrap.wrap`. For example:: >>> import textwrap >>> paragraph = "Not a whit, we defy augury: ... more text ..." @@ -1503,15 +1503,15 @@ complete list of changes, or look through the CVS logs for all the details. it will come: the readiness is all. >>> - The module also contains a :class:`TextWrapper` class that actually implements - the text wrapping strategy. Both the :class:`TextWrapper` class and the - :func:`wrap` and :func:`fill` functions support a number of additional keyword + The module also contains a :class:`~textwrap.TextWrapper` class that actually implements + the text wrapping strategy. Both the :class:`~textwrap.TextWrapper` class and the + :func:`~textwrap.wrap` and :func:`~textwrap.fill` functions support a number of additional keyword arguments for fine-tuning the formatting; consult the module's documentation for details. (Contributed by Greg Ward.) -* The :mod:`thread` and :mod:`threading` modules now have companion modules, - :mod:`dummy_thread` and :mod:`dummy_threading`, that provide a do-nothing - implementation of the :mod:`thread` module's interface for platforms where +* The :mod:`!thread` and :mod:`threading` modules now have companion modules, + :mod:`!dummy_thread` and :mod:`!dummy_threading`, that provide a do-nothing + implementation of the :mod:`!thread` module's interface for platforms where threads are not supported. The intention is to simplify thread-aware modules (ones that *don't* rely on threads to run) by putting the following code at the top:: @@ -1521,26 +1521,26 @@ complete list of changes, or look through the CVS logs for all the details. except ImportError: import dummy_threading as _threading - In this example, :mod:`_threading` is used as the module name to make it clear + In this example, :mod:`!_threading` is used as the module name to make it clear that the module being used is not necessarily the actual :mod:`threading` - module. Code can call functions and use classes in :mod:`_threading` whether or + module. Code can call functions and use classes in :mod:`!_threading` whether or not threads are supported, avoiding an :keyword:`if` statement and making the code slightly clearer. This module will not magically make multithreaded code run without threads; code that waits for another thread to return or to do something will simply hang forever. -* The :mod:`time` module's :func:`strptime` function has long been an annoyance - because it uses the platform C library's :func:`strptime` implementation, and +* The :mod:`time` module's :func:`~time.strptime` function has long been an annoyance + because it uses the platform C library's :func:`~time.strptime` implementation, and different platforms sometimes have odd bugs. Brett Cannon contributed a portable implementation that's written in pure Python and should behave identically on all platforms. * The new :mod:`timeit` module helps measure how long snippets of Python code take to execute. The :file:`timeit.py` file can be run directly from the - command line, or the module's :class:`Timer` class can be imported and used + command line, or the module's :class:`~timeit.Timer` class can be imported and used directly. Here's a short example that figures out whether it's faster to convert an 8-bit string to Unicode by appending an empty Unicode string to it or - by using the :func:`unicode` function:: + by using the :func:`!unicode` function:: import timeit @@ -1558,46 +1558,46 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`!Tix` module has received various bug fixes and updates for the current version of the Tix package. -* The :mod:`Tkinter` module now works with a thread-enabled version of Tcl. +* The :mod:`!Tkinter` module now works with a thread-enabled version of Tcl. Tcl's threading model requires that widgets only be accessed from the thread in which they're created; accesses from another thread can cause Tcl to panic. For - certain Tcl interfaces, :mod:`Tkinter` will now automatically avoid this when a + certain Tcl interfaces, :mod:`!Tkinter` will now automatically avoid this when a widget is accessed from a different thread by marshalling a command, passing it to the correct thread, and waiting for the results. Other interfaces can't be - handled automatically but :mod:`Tkinter` will now raise an exception on such an + handled automatically but :mod:`!Tkinter` will now raise an exception on such an access so that you can at least find out about the problem. See https://mail.python.org/pipermail/python-dev/2002-December/031107.html for a more detailed explanation of this change. (Implemented by Martin von Löwis.) -* Calling Tcl methods through :mod:`_tkinter` no longer returns only strings. +* Calling Tcl methods through :mod:`!_tkinter` no longer returns only strings. Instead, if Tcl returns other objects those objects are converted to their - Python equivalent, if one exists, or wrapped with a :class:`_tkinter.Tcl_Obj` + Python equivalent, if one exists, or wrapped with a :class:`!_tkinter.Tcl_Obj` object if no Python equivalent exists. This behavior can be controlled through - the :meth:`wantobjects` method of :class:`tkapp` objects. + the :meth:`!wantobjects` method of :class:`!tkapp` objects. - When using :mod:`_tkinter` through the :mod:`Tkinter` module (as most Tkinter + When using :mod:`!_tkinter` through the :mod:`!Tkinter` module (as most Tkinter applications will), this feature is always activated. It should not cause compatibility problems, since Tkinter would always convert string results to Python types where possible. If any incompatibilities are found, the old behavior can be restored by setting - the :attr:`wantobjects` variable in the :mod:`Tkinter` module to false before - creating the first :class:`tkapp` object. :: + the :attr:`!wantobjects` variable in the :mod:`!Tkinter` module to false before + creating the first :class:`!tkapp` object. :: import Tkinter Tkinter.wantobjects = 0 Any breakage caused by this change should be reported as a bug. -* The :mod:`UserDict` module has a new :class:`DictMixin` class which defines +* The :mod:`!UserDict` module has a new :class:`!DictMixin` class which defines all dictionary methods for classes that already have a minimum mapping interface. This greatly simplifies writing classes that need to be substitutable for dictionaries, such as the classes in the :mod:`shelve` module. Adding the mix-in as a superclass provides the full dictionary interface - whenever the class defines :meth:`~object.__getitem__`, :meth:`__setitem__`, - :meth:`__delitem__`, and :meth:`keys`. For example:: + whenever the class defines :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, + :meth:`~object.__delitem__`, and :meth:`!keys`. For example:: >>> import UserDict >>> class SeqDict(UserDict.DictMixin): @@ -1640,15 +1640,15 @@ complete list of changes, or look through the CVS logs for all the details. * The DOM implementation in :mod:`xml.dom.minidom` can now generate XML output in a particular encoding by providing an optional encoding argument to the - :meth:`toxml` and :meth:`toprettyxml` methods of DOM nodes. + :meth:`~xml.dom.minidom.Node.toxml` and :meth:`~xml.dom.minidom.Node.toprettyxml` methods of DOM nodes. -* The :mod:`xmlrpclib` module now supports an XML-RPC extension for handling nil +* The :mod:`!xmlrpclib` module now supports an XML-RPC extension for handling nil data values such as Python's ``None``. Nil values are always supported on unmarshalling an XML-RPC response. To generate requests containing ``None``, you must supply a true value for the *allow_none* parameter when creating a - :class:`Marshaller` instance. + :class:`!Marshaller` instance. -* The new :mod:`DocXMLRPCServer` module allows writing self-documenting XML-RPC +* The new :mod:`!DocXMLRPCServer` module allows writing self-documenting XML-RPC servers. Run it in demo mode (as a program) to see it in action. Pointing the web browser to the RPC server produces pydoc-style documentation; pointing xmlrpclib to the server allows invoking the actual methods. (Contributed by @@ -1663,8 +1663,8 @@ complete list of changes, or look through the CVS logs for all the details. The :mod:`socket` module has also been extended to transparently convert Unicode hostnames to the ACE version before passing them to the C library. - Modules that deal with hostnames such as :mod:`httplib` and :mod:`ftplib`) - also support Unicode host names; :mod:`httplib` also sends HTTP ``Host`` + Modules that deal with hostnames such as :mod:`!httplib` and :mod:`ftplib`) + also support Unicode host names; :mod:`!httplib` also sends HTTP ``Host`` headers using the ACE version of the domain name. :mod:`urllib` supports Unicode URLs with non-ASCII host names as long as the ``path`` part of the URL is ASCII only. @@ -1682,17 +1682,17 @@ Date and time types suitable for expressing timestamps were added as the :mod:`datetime` module. The types don't support different calendars or many fancy features, and just stick to the basics of representing time. -The three primary types are: :class:`date`, representing a day, month, and year; +The three primary types are: :class:`~datetime.date`, representing a day, month, and year; :class:`~datetime.time`, consisting of hour, minute, and second; and :class:`~datetime.datetime`, -which contains all the attributes of both :class:`date` and :class:`~datetime.time`. -There's also a :class:`timedelta` class representing differences between two +which contains all the attributes of both :class:`~datetime.date` and :class:`~datetime.time`. +There's also a :class:`~datetime.timedelta` class representing differences between two points in time, and time zone logic is implemented by classes inheriting from -the abstract :class:`tzinfo` class. +the abstract :class:`~datetime.tzinfo` class. -You can create instances of :class:`date` and :class:`~datetime.time` by either supplying +You can create instances of :class:`~datetime.date` and :class:`~datetime.time` by either supplying keyword arguments to the appropriate constructor, e.g. ``datetime.date(year=1972, month=10, day=15)``, or by using one of a number of -class methods. For example, the :meth:`date.today` class method returns the +class methods. For example, the :meth:`~datetime.date.today` class method returns the current local date. Once created, instances of the date/time classes are all immutable. There are a @@ -1707,8 +1707,8 @@ number of methods for producing formatted strings from objects:: >>> now.strftime('%Y %d %b') '2002 30 Dec' -The :meth:`replace` method allows modifying one or more fields of a -:class:`date` or :class:`~datetime.datetime` instance, returning a new instance:: +The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a +:class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: >>> d = datetime.datetime.now() >>> d @@ -1718,10 +1718,10 @@ The :meth:`replace` method allows modifying one or more fields of a >>> Instances can be compared, hashed, and converted to strings (the result is the -same as that of :meth:`isoformat`). :class:`date` and :class:`~datetime.datetime` -instances can be subtracted from each other, and added to :class:`timedelta` +same as that of :meth:`~datetime.datetime.isoformat`). :class:`~datetime.date` and :class:`~datetime.datetime` +instances can be subtracted from each other, and added to :class:`~datetime.timedelta` instances. The largest missing feature is that there's no standard library -support for parsing strings and getting back a :class:`date` or +support for parsing strings and getting back a :class:`~datetime.date` or :class:`~datetime.datetime`. For more information, refer to the module's reference documentation. @@ -1739,7 +1739,7 @@ command-line parsing that follows the Unix conventions, automatically creates the output for :option:`!--help`, and can perform different actions for different options. -You start by creating an instance of :class:`OptionParser` and telling it what +You start by creating an instance of :class:`~optparse.OptionParser` and telling it what your program's options are. :: import sys @@ -1753,7 +1753,7 @@ your program's options are. :: action='store', type='int', dest='length', help='set maximum length of output') -Parsing a command line is then done by calling the :meth:`parse_args` method. :: +Parsing a command line is then done by calling the :meth:`~optparse.OptionParser.parse_args` method. :: options, args = op.parse_args(sys.argv[1:]) print options @@ -1925,7 +1925,7 @@ Changes to Python's build process and to the C API include: dependence on a system version or local installation of Expat. * If you dynamically allocate type objects in your extension, you should be - aware of a change in the rules relating to the :attr:`__module__` and + aware of a change in the rules relating to the :attr:`!__module__` and :attr:`~definition.__name__` attributes. In summary, you will want to ensure the type's dictionary contains a ``'__module__'`` key; making the module name the part of the type name leading up to the final period will no longer have the desired @@ -1940,7 +1940,7 @@ Port-Specific Changes Support for a port to IBM's OS/2 using the EMX runtime environment was merged into the main Python source tree. EMX is a POSIX emulation layer over the OS/2 system APIs. The Python port for EMX tries to support all the POSIX-like -capability exposed by the EMX runtime, and mostly succeeds; :func:`fork` and +capability exposed by the EMX runtime, and mostly succeeds; :func:`!fork` and :func:`fcntl` are restricted by the limitations of the underlying emulation layer. The standard OS/2 port, which uses IBM's Visual Age compiler, also gained support for case-sensitive import semantics as part of the integration of @@ -2031,9 +2031,9 @@ code: the file's encoding (UTF-8, Latin-1, or whatever) by adding a comment to the top of the file. See section :ref:`section-encodings` for more information. -* Calling Tcl methods through :mod:`_tkinter` no longer returns only strings. +* Calling Tcl methods through :mod:`!_tkinter` no longer returns only strings. Instead, if Tcl returns other objects those objects are converted to their - Python equivalent, if one exists, or wrapped with a :class:`_tkinter.Tcl_Obj` + Python equivalent, if one exists, or wrapped with a :class:`!_tkinter.Tcl_Obj` object if no Python equivalent exists. * Large octal and hex literals such as ``0xffffffff`` now trigger a @@ -2049,10 +2049,10 @@ code: * You can no longer disable assertions by assigning to ``__debug__``. -* The Distutils :func:`setup` function has gained various new keyword arguments +* The Distutils :func:`!setup` function has gained various new keyword arguments such as *depends*. Old versions of the Distutils will abort if passed unknown keywords. A solution is to check for the presence of the new - :func:`get_distutil_options` function in your :file:`setup.py` and only uses the + :func:`!get_distutil_options` function in your :file:`setup.py` and only uses the new keywords with a version of the Distutils that supports them:: from distutils import core From 25061f5c98a47691fdb70f550943167bda77f6e0 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 14 Dec 2023 21:10:26 +0000 Subject: [PATCH 74/95] gh-101100: Cleanup `mailbox` docs (#113124) --- Doc/library/mailbox.rst | 261 +++++++++++++++++++++------------------- Doc/tools/.nitignore | 1 - 2 files changed, 139 insertions(+), 123 deletions(-) diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index fd60d163378f07..c98496d1fff993 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -13,8 +13,8 @@ This module defines two classes, :class:`Mailbox` and :class:`Message`, for accessing and manipulating on-disk mailboxes and the messages they contain. -:class:`Mailbox` offers a dictionary-like mapping from keys to messages. -:class:`Message` extends the :mod:`email.message` module's +:class:`!Mailbox` offers a dictionary-like mapping from keys to messages. +:class:`!Message` extends the :mod:`email.message` module's :class:`~email.message.Message` class with format-specific state and behavior. Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. @@ -27,37 +27,38 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-objects: -:class:`Mailbox` objects ------------------------- +:class:`!Mailbox` objects +------------------------- .. class:: Mailbox A mailbox, which may be inspected and modified. - The :class:`Mailbox` class defines an interface and is not intended to be + The :class:`!Mailbox` class defines an interface and is not intended to be instantiated. Instead, format-specific subclasses should inherit from - :class:`Mailbox` and your code should instantiate a particular subclass. + :class:`!Mailbox` and your code should instantiate a particular subclass. - The :class:`Mailbox` interface is dictionary-like, with small keys - corresponding to messages. Keys are issued by the :class:`Mailbox` instance - with which they will be used and are only meaningful to that :class:`Mailbox` + The :class:`!Mailbox` interface is dictionary-like, with small keys + corresponding to messages. Keys are issued by the :class:`!Mailbox` instance + with which they will be used and are only meaningful to that :class:`!Mailbox` instance. A key continues to identify a message even if the corresponding message is modified, such as by replacing it with another message. - Messages may be added to a :class:`Mailbox` instance using the set-like + Messages may be added to a :class:`!Mailbox` instance using the set-like method :meth:`add` and removed using a ``del`` statement or the set-like methods :meth:`remove` and :meth:`discard`. - :class:`Mailbox` interface semantics differ from dictionary semantics in some + :class:`!Mailbox` interface semantics differ from dictionary semantics in some noteworthy ways. Each time a message is requested, a new representation (typically a :class:`Message` instance) is generated based upon the current state of the mailbox. Similarly, when a message is added to a - :class:`Mailbox` instance, the provided message representation's contents are + :class:`!Mailbox` instance, the provided message representation's contents are copied. In neither case is a reference to the message representation kept by - the :class:`Mailbox` instance. + the :class:`!Mailbox` instance. - The default :class:`Mailbox` iterator iterates over message representations, - not keys as the default dictionary iterator does. Moreover, modification of a + The default :class:`!Mailbox` :term:`iterator` iterates over message + representations, not keys as the default :class:`dictionary ` + iterator does. Moreover, modification of a mailbox during iteration is safe and well-defined. Messages added to the mailbox after an iterator is created will not be seen by the iterator. Messages removed from the mailbox before the iterator yields them @@ -69,14 +70,15 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Be very cautious when modifying mailboxes that might be simultaneously changed by some other process. The safest mailbox format to use for such - tasks is Maildir; try to avoid using single-file formats such as mbox for + tasks is :class:`Maildir`; try to avoid using single-file formats such as + :class:`mbox` for concurrent writing. If you're modifying a mailbox, you *must* lock it by calling the :meth:`lock` and :meth:`unlock` methods *before* reading any messages in the file or making any changes by adding or deleting a message. Failing to lock the mailbox runs the risk of losing messages or corrupting the entire mailbox. - :class:`Mailbox` instances have the following methods: + :class:`!Mailbox` instances have the following methods: .. method:: add(message) @@ -127,21 +129,23 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: iterkeys() - keys() - Return an iterator over all keys if called as :meth:`iterkeys` or return a - list of keys if called as :meth:`keys`. + Return an :term:`iterator` over all keys + + + .. method:: keys() + + The same as :meth:`iterkeys`, except that a :class:`list` is returned + rather than an :term:`iterator` .. method:: itervalues() __iter__() - values() - Return an iterator over representations of all messages if called as - :meth:`itervalues` or :meth:`__iter__` or return a list of such - representations if called as :meth:`values`. The messages are represented + Return an :term:`iterator` over representations of all messages. + The messages are represented as instances of the appropriate format-specific :class:`Message` subclass - unless a custom message factory was specified when the :class:`Mailbox` + unless a custom message factory was specified when the :class:`!Mailbox` instance was initialized. .. note:: @@ -150,15 +154,25 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. iterate over keys. + .. method:: values() + + The same as :meth:`itervalues`, except that a :class:`list` is returned + rather than an :term:`iterator` + + .. method:: iteritems() - items() - Return an iterator over (*key*, *message*) pairs, where *key* is a key and - *message* is a message representation, if called as :meth:`iteritems` or - return a list of such pairs if called as :meth:`items`. The messages are + Return an :term:`iterator` over (*key*, *message*) pairs, where *key* is + a key and *message* is a message representation. The messages are represented as instances of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. + + + .. method:: items() + + The same as :meth:`iteritems`, except that a :class:`list` of pairs is + returned rather than an :term:`iterator` of pairs. .. method:: get(key, default=None) @@ -167,9 +181,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Return a representation of the message corresponding to *key*. If no such message exists, *default* is returned if the method was called as :meth:`get` and a :exc:`KeyError` exception is raised if the method was - called as :meth:`~object.__getitem__`. The message is represented as an instance + called as :meth:`!__getitem__`. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a - custom message factory was specified when the :class:`Mailbox` instance + custom message factory was specified when the :class:`!Mailbox` instance was initialized. @@ -198,21 +212,23 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_file(key) - Return a file-like representation of the message corresponding to *key*, + Return a :term:`file-like ` representation of the + message corresponding to *key*, or raise a :exc:`KeyError` exception if no such message exists. The file-like object behaves as if open in binary mode. This file should be closed once it is no longer needed. .. versionchanged:: 3.2 - The file object really is a binary file; previously it was incorrectly - returned in text mode. Also, the file-like object now supports the - context management protocol: you can use a :keyword:`with` statement to - automatically close it. + The file object really is a :term:`binary file`; previously it was + incorrectly returned in text mode. Also, the :term:`file-like object` + now supports the :term:`context manager` protocol: you can use a + :keyword:`with` statement to automatically close it. .. note:: - Unlike other representations of messages, file-like representations are - not necessarily independent of the :class:`Mailbox` instance that + Unlike other representations of messages, + :term:`file-like ` representations are not + necessarily independent of the :class:`!Mailbox` instance that created them or of the underlying mailbox. More specific documentation is provided by each subclass. @@ -238,7 +254,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. the message. If no such message exists, return *default*. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. .. method:: popitem() @@ -248,7 +264,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message. If the mailbox is empty, raise a :exc:`KeyError` exception. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. .. method:: update(arg) @@ -259,7 +275,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. *message* as if by using :meth:`__setitem__`. As with :meth:`__setitem__`, each *key* must already correspond to a message in the mailbox or else a :exc:`KeyError` exception will be raised, so in general it is incorrect - for *arg* to be a :class:`Mailbox` instance. + for *arg* to be a :class:`!Mailbox` instance. .. note:: @@ -269,7 +285,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: flush() Write any pending changes to the filesystem. For some :class:`Mailbox` - subclasses, changes are always written immediately and :meth:`flush` does + subclasses, changes are always written immediately and :meth:`!flush` does nothing, but you should still make a habit of calling this method. @@ -290,13 +306,13 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() Flush the mailbox, unlock it if necessary, and close any open files. For - some :class:`Mailbox` subclasses, this method does nothing. + some :class:`!Mailbox` subclasses, this method does nothing. .. _mailbox-maildir: -:class:`Maildir` -^^^^^^^^^^^^^^^^ +:class:`!Maildir` objects +^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Maildir(dirname, factory=None, create=True) @@ -330,11 +346,11 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Folders of the style introduced by the Courier mail transfer agent are also supported. Any subdirectory of the main mailbox is considered a folder if ``'.'`` is the first character in its name. Folder names are represented by - :class:`Maildir` without the leading ``'.'``. Each folder is itself a Maildir + :class:`!Maildir` without the leading ``'.'``. Each folder is itself a Maildir mailbox but should not contain other folders. Instead, a logical nesting is indicated using ``'.'`` to delimit levels, e.g., "Archived.2005.07". - .. note:: + .. attribute:: Maildir.colon The Maildir specification requires the use of a colon (``':'``) in certain message file names. However, some operating systems do not permit this @@ -346,9 +362,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. import mailbox mailbox.Maildir.colon = '!' - The :attr:`colon` attribute may also be set on a per-instance basis. + The :attr:`!colon` attribute may also be set on a per-instance basis. - :class:`Maildir` instances have all of the methods of :class:`Mailbox` in + :class:`!Maildir` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -359,14 +375,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_folder(folder) - Return a :class:`Maildir` instance representing the folder whose name is + Return a :class:`!Maildir` instance representing the folder whose name is *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder does not exist. .. method:: add_folder(folder) - Create a folder whose name is *folder* and return a :class:`Maildir` + Create a folder whose name is *folder* and return a :class:`!Maildir` instance representing it. @@ -485,7 +501,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. versionadded:: 3.13 - Some :class:`Mailbox` methods implemented by :class:`Maildir` deserve special + Some :class:`Mailbox` methods implemented by :class:`!Maildir` deserve special remarks: @@ -516,7 +532,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() - :class:`Maildir` instances do not keep any open files and the underlying + :class:`!Maildir` instances do not keep any open files and the underlying mailboxes do not support locking, so this method does nothing. @@ -539,8 +555,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mbox: -:class:`mbox` -^^^^^^^^^^^^^ +:class:`!mbox` objects +^^^^^^^^^^^^^^^^^^^^^^ .. class:: mbox(path, factory=None, create=True) @@ -557,22 +573,22 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. each message indicated by a line whose first five characters are "From ". Several variations of the mbox format exist to address perceived shortcomings in - the original. In the interest of compatibility, :class:`mbox` implements the + the original. In the interest of compatibility, :class:`!mbox` implements the original format, which is sometimes referred to as :dfn:`mboxo`. This means that the :mailheader:`Content-Length` header, if present, is ignored and that any occurrences of "From " at the beginning of a line in a message body are transformed to ">From " when storing the message, although occurrences of ">From " are not transformed to "From " when reading the message. - Some :class:`Mailbox` methods implemented by :class:`mbox` deserve special + Some :class:`Mailbox` methods implemented by :class:`!mbox` deserve special remarks: .. method:: get_file(key) - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`mbox` instance may yield unpredictable results or raise an - exception. + Using the file after calling :meth:`~Mailbox.flush` or + :meth:`~Mailbox.close` on the :class:`!mbox` instance may yield + unpredictable results or raise an exception. .. method:: lock() @@ -596,8 +612,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mh: -:class:`MH` -^^^^^^^^^^^ +:class:`!MH` objects +^^^^^^^^^^^^^^^^^^^^ .. class:: MH(path, factory=None, create=True) @@ -617,12 +633,12 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. messages without moving them to sub-folders. Sequences are defined in a file called :file:`.mh_sequences` in each folder. - The :class:`MH` class manipulates MH mailboxes, but it does not attempt to + The :class:`!MH` class manipulates MH mailboxes, but it does not attempt to emulate all of :program:`mh`'s behaviors. In particular, it does not modify and is not affected by the :file:`context` or :file:`.mh_profile` files that are used by :program:`mh` to store its state and configuration. - :class:`MH` instances have all of the methods of :class:`Mailbox` in addition + :class:`!MH` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -633,14 +649,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_folder(folder) - Return an :class:`MH` instance representing the folder whose name is + Return an :class:`!MH` instance representing the folder whose name is *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder does not exist. .. method:: add_folder(folder) - Create a folder whose name is *folder* and return an :class:`MH` instance + Create a folder whose name is *folder* and return an :class:`!MH` instance representing it. @@ -674,7 +690,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Already-issued keys are invalidated by this operation and should not be subsequently used. - Some :class:`Mailbox` methods implemented by :class:`MH` deserve special + Some :class:`Mailbox` methods implemented by :class:`!MH` deserve special remarks: @@ -710,7 +726,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() - :class:`MH` instances do not keep any open files, so this method is + :class:`!MH` instances do not keep any open files, so this method is equivalent to :meth:`unlock`. @@ -726,8 +742,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-babyl: -:class:`Babyl` -^^^^^^^^^^^^^^ +:class:`!Babyl` objects +^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Babyl(path, factory=None, create=True) @@ -754,7 +770,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message, and a list of all user-defined labels found in the mailbox is kept in the Babyl options section. - :class:`Babyl` instances have all of the methods of :class:`Mailbox` in + :class:`!Babyl` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -769,7 +785,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. options section, but the Babyl section is updated whenever the mailbox is modified. - Some :class:`Mailbox` methods implemented by :class:`Babyl` deserve special + Some :class:`Mailbox` methods implemented by :class:`!Babyl` deserve special remarks: @@ -802,8 +818,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mmdf: -:class:`MMDF` -^^^^^^^^^^^^^ +:class:`!MMDF` objects +^^^^^^^^^^^^^^^^^^^^^^ .. class:: MMDF(path, factory=None, create=True) @@ -824,15 +840,15 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. ">From " when storing messages because the extra message separator lines prevent mistaking such occurrences for the starts of subsequent messages. - Some :class:`Mailbox` methods implemented by :class:`MMDF` deserve special + Some :class:`Mailbox` methods implemented by :class:`!MMDF` deserve special remarks: .. method:: get_file(key) - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`MMDF` instance may yield unpredictable results or raise an - exception. + Using the file after calling :meth:`~Mailbox.flush` or + :meth:`~Mailbox.close` on the :class:`!MMDF` instance may yield + unpredictable results or raise an exception. .. method:: lock() @@ -854,20 +870,20 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-message-objects: -:class:`Message` objects ------------------------- +:class:`!Message` objects +------------------------- .. class:: Message(message=None) A subclass of the :mod:`email.message` module's - :class:`~email.message.Message`. Subclasses of :class:`mailbox.Message` add + :class:`~email.message.Message`. Subclasses of :class:`!mailbox.Message` add mailbox-format-specific state and behavior. If *message* is omitted, the new instance is created in a default, empty state. If *message* is an :class:`email.message.Message` instance, its contents are copied; furthermore, any format-specific information is converted insofar as - possible if *message* is a :class:`Message` instance. If *message* is a string, + possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, or a file, it should contain an :rfc:`2822`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files @@ -882,18 +898,18 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. such as whether a message has been read by the user or marked as important is retained, because it applies to the message itself. - There is no requirement that :class:`Message` instances be used to represent + There is no requirement that :class:`!Message` instances be used to represent messages retrieved using :class:`Mailbox` instances. In some situations, the - time and memory required to generate :class:`Message` representations might - not be acceptable. For such situations, :class:`Mailbox` instances also + time and memory required to generate :class:`!Message` representations might + not be acceptable. For such situations, :class:`!Mailbox` instances also offer string and file-like representations, and a custom message factory may - be specified when a :class:`Mailbox` instance is initialized. + be specified when a :class:`!Mailbox` instance is initialized. .. _mailbox-maildirmessage: -:class:`MaildirMessage` -^^^^^^^^^^^^^^^^^^^^^^^ +:class:`!MaildirMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MaildirMessage(message=None) @@ -928,7 +944,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. | T | Trashed | Marked for subsequent deletion | +------+---------+--------------------------------+ - :class:`MaildirMessage` instances offer the following methods: + :class:`!MaildirMessage` instances offer the following methods: .. method:: get_subdir() @@ -1005,7 +1021,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Set "info" to *info*, which should be a string. -When a :class:`MaildirMessage` instance is created based upon an +When a :class:`!MaildirMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1025,7 +1041,7 @@ take place: | T flag | D flag | +--------------------+----------------------------------------------+ -When a :class:`MaildirMessage` instance is created based upon an +When a :class:`!MaildirMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------------------+--------------------------+ @@ -1040,7 +1056,7 @@ When a :class:`MaildirMessage` instance is created based upon an | R flag | "replied" sequence | +-------------------------------+--------------------------+ -When a :class:`MaildirMessage` instance is created based upon a +When a :class:`!MaildirMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------------------+-------------------------------+ @@ -1060,8 +1076,8 @@ When a :class:`MaildirMessage` instance is created based upon a .. _mailbox-mboxmessage: -:class:`mboxMessage` -^^^^^^^^^^^^^^^^^^^^ +:class:`!mboxMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: mboxMessage(message=None) @@ -1097,7 +1113,7 @@ When a :class:`MaildirMessage` instance is created based upon a "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The flags and headers typically appear in the order mentioned. - :class:`mboxMessage` instances offer the following methods: + :class:`!mboxMessage` instances offer the following methods: .. method:: get_from() @@ -1145,7 +1161,7 @@ When a :class:`MaildirMessage` instance is created based upon a remove more than one flag at a time, *flag* maybe a string of more than one character. -When an :class:`mboxMessage` instance is created based upon a +When an :class:`!mboxMessage` instance is created based upon a :class:`MaildirMessage` instance, a "From " line is generated based upon the :class:`MaildirMessage` instance's delivery date, and the following conversions take place: @@ -1164,7 +1180,7 @@ take place: | A flag | R flag | +-----------------+-------------------------------+ -When an :class:`mboxMessage` instance is created based upon an +When an :class:`!mboxMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------+--------------------------+ @@ -1179,7 +1195,7 @@ When an :class:`mboxMessage` instance is created based upon an | A flag | "replied" sequence | +-------------------+--------------------------+ -When an :class:`mboxMessage` instance is created based upon a +When an :class:`!mboxMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------+-----------------------------+ @@ -1194,7 +1210,8 @@ When an :class:`mboxMessage` instance is created based upon a | A flag | "answered" label | +-------------------+-----------------------------+ -When a :class:`Message` instance is created based upon an :class:`MMDFMessage` +When a :class:`!mboxMessage` instance is created based upon an +:class:`MMDFMessage` instance, the "From " line is copied and all flags directly correspond: +-----------------+----------------------------+ @@ -1214,8 +1231,8 @@ instance, the "From " line is copied and all flags directly correspond: .. _mailbox-mhmessage: -:class:`MHMessage` -^^^^^^^^^^^^^^^^^^ +:class:`!MHMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MHMessage(message=None) @@ -1239,7 +1256,7 @@ instance, the "From " line is copied and all flags directly correspond: | flagged | Marked as important | +----------+------------------------------------------+ - :class:`MHMessage` instances offer the following methods: + :class:`!MHMessage` instances offer the following methods: .. method:: get_sequences() @@ -1261,7 +1278,7 @@ instance, the "From " line is copied and all flags directly correspond: Remove *sequence* from the list of sequences that include this message. -When an :class:`MHMessage` instance is created based upon a +When an :class:`!MHMessage` instance is created based upon a :class:`MaildirMessage` instance, the following conversions take place: +--------------------+-------------------------------+ @@ -1274,7 +1291,7 @@ When an :class:`MHMessage` instance is created based upon a | "flagged" sequence | F flag | +--------------------+-------------------------------+ -When an :class:`MHMessage` instance is created based upon an +When an :class:`!MHMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1290,7 +1307,7 @@ take place: | "flagged" sequence | F flag | +--------------------+----------------------------------------------+ -When an :class:`MHMessage` instance is created based upon a +When an :class:`!MHMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +--------------------+-----------------------------+ @@ -1304,8 +1321,8 @@ When an :class:`MHMessage` instance is created based upon a .. _mailbox-babylmessage: -:class:`BabylMessage` -^^^^^^^^^^^^^^^^^^^^^ +:class:`!BabylMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: BabylMessage(message=None) @@ -1334,11 +1351,11 @@ When an :class:`MHMessage` instance is created based upon a | resent | Resent | +-----------+------------------------------------------+ - By default, Rmail displays only visible headers. The :class:`BabylMessage` + By default, Rmail displays only visible headers. The :class:`!BabylMessage` class, though, uses the original headers because they are more complete. Visible headers may be accessed explicitly if desired. - :class:`BabylMessage` instances offer the following methods: + :class:`!BabylMessage` instances offer the following methods: .. method:: get_labels() @@ -1377,7 +1394,7 @@ When an :class:`MHMessage` instance is created based upon a .. method:: update_visible() - When a :class:`BabylMessage` instance's original headers are modified, the + When a :class:`!BabylMessage` instance's original headers are modified, the visible headers are not automatically modified to correspond. This method updates the visible headers as follows: each visible header with a corresponding original header is set to the value of the original header, @@ -1387,7 +1404,7 @@ When an :class:`MHMessage` instance is created based upon a present in the original headers but not the visible headers are added to the visible headers. -When a :class:`BabylMessage` instance is created based upon a +When a :class:`!BabylMessage` instance is created based upon a :class:`MaildirMessage` instance, the following conversions take place: +-------------------+-------------------------------+ @@ -1402,7 +1419,7 @@ When a :class:`BabylMessage` instance is created based upon a | "forwarded" label | P flag | +-------------------+-------------------------------+ -When a :class:`BabylMessage` instance is created based upon an +When a :class:`!BabylMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1418,7 +1435,7 @@ take place: | "answered" label | A flag | +------------------+----------------------------------------------+ -When a :class:`BabylMessage` instance is created based upon an +When a :class:`!BabylMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +------------------+--------------------------+ @@ -1432,8 +1449,8 @@ When a :class:`BabylMessage` instance is created based upon an .. _mailbox-mmdfmessage: -:class:`MMDFMessage` -^^^^^^^^^^^^^^^^^^^^ +:class:`!MMDFMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MMDFMessage(message=None) @@ -1467,7 +1484,7 @@ When a :class:`BabylMessage` instance is created based upon an "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The flags and headers typically appear in the order mentioned. - :class:`MMDFMessage` instances offer the following methods, which are + :class:`!MMDFMessage` instances offer the following methods, which are identical to those offered by :class:`mboxMessage`: @@ -1516,7 +1533,7 @@ When a :class:`BabylMessage` instance is created based upon an remove more than one flag at a time, *flag* maybe a string of more than one character. -When an :class:`MMDFMessage` instance is created based upon a +When an :class:`!MMDFMessage` instance is created based upon a :class:`MaildirMessage` instance, a "From " line is generated based upon the :class:`MaildirMessage` instance's delivery date, and the following conversions take place: @@ -1535,7 +1552,7 @@ take place: | A flag | R flag | +-----------------+-------------------------------+ -When an :class:`MMDFMessage` instance is created based upon an +When an :class:`!MMDFMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------+--------------------------+ @@ -1550,7 +1567,7 @@ When an :class:`MMDFMessage` instance is created based upon an | A flag | "replied" sequence | +-------------------+--------------------------+ -When an :class:`MMDFMessage` instance is created based upon a +When an :class:`!MMDFMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------+-----------------------------+ @@ -1565,7 +1582,7 @@ When an :class:`MMDFMessage` instance is created based upon a | A flag | "answered" label | +-------------------+-----------------------------+ -When an :class:`MMDFMessage` instance is created based upon an +When an :class:`!MMDFMessage` instance is created based upon an :class:`mboxMessage` instance, the "From " line is copied and all flags directly correspond: @@ -1587,7 +1604,7 @@ correspond: Exceptions ---------- -The following exception classes are defined in the :mod:`mailbox` module: +The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 20580f78e07b5d..d9147aaeee12bd 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -62,7 +62,6 @@ Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst Doc/library/lzma.rst -Doc/library/mailbox.rst Doc/library/mmap.rst Doc/library/multiprocessing.rst Doc/library/multiprocessing.shared_memory.rst From 5f7d7353b47ccf634b9b65f933d3fdeeb395301f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 14 Dec 2023 17:27:39 -0600 Subject: [PATCH 75/95] Optimize unique_justseen() recipe for a common case. (gh-113147) --- Doc/library/itertools.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 36cea9a835f302..03127afe1b4460 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1017,6 +1017,8 @@ which incur interpreter overhead. "List unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B # unique_justseen('ABBcCAD', str.lower) --> A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) return map(next, map(operator.itemgetter(1), groupby(iterable, key))) From f34e22c6470d1c5d0e2a3ffb2272d375c22654b9 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 15 Dec 2023 02:42:33 +0300 Subject: [PATCH 76/95] gh-112535: Update _Py_ThreadId() to support RISC-V (gh-113084) Update _Py_ThreadId() to support RISC-V --- Include/object.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Include/object.h b/Include/object.h index bd576b0bd43211..d22e5c2b8be2a9 100644 --- a/Include/object.h +++ b/Include/object.h @@ -283,6 +283,13 @@ _Py_ThreadId(void) // Both GCC and Clang have supported __builtin_thread_pointer // for s390 from long time ago. tid = (uintptr_t)__builtin_thread_pointer(); +#elif defined(__riscv) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // tp is Thread Pointer provided by the RISC-V ABI. + __asm__ ("mv %0, tp" : "=r" (tid)); + #endif #else # error "define _Py_ThreadId for this platform" #endif From 7bb00f053e0a86bb8827cc464961a4fad9278e6d Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 15 Dec 2023 08:57:23 +0000 Subject: [PATCH 77/95] gh-101100: Fix Sphinx nitpicks in `library/rlcompleter.rst` (#113125) --- Doc/library/readline.rst | 2 ++ Doc/library/rlcompleter.rst | 40 ++++++++++++++++++++----------------- Doc/tools/.nitignore | 1 - 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 2e0f45ced30b9c..1adafcaa02eab9 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -218,6 +218,8 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. _readline-completion: + Completion ---------- diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 40b09ce897880e..8287699c5f013e 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -10,12 +10,14 @@ -------------- -The :mod:`rlcompleter` module defines a completion function suitable for the -:mod:`readline` module by completing valid Python identifiers and keywords. +The :mod:`!rlcompleter` module defines a completion function suitable to be +passed to :func:`~readline.set_completer` in the :mod:`readline` module. When this module is imported on a Unix platform with the :mod:`readline` module available, an instance of the :class:`Completer` class is automatically created -and its :meth:`complete` method is set as the :mod:`readline` completer. +and its :meth:`~Completer.complete` method is set as the +:ref:`readline completer `. The method provides +completion of valid Python :ref:`identifiers and keywords `. Example:: @@ -28,7 +30,7 @@ Example:: readline.__name__ readline.parse_and_bind( >>> readline. -The :mod:`rlcompleter` module is designed for use with Python's +The :mod:`!rlcompleter` module is designed for use with Python's :ref:`interactive mode `. Unless Python is run with the :option:`-S` option, the module is automatically imported and configured (see :ref:`rlcompleter-config`). @@ -39,23 +41,25 @@ this module can still be used for custom purposes. .. _completer-objects: -Completer Objects ------------------ +.. class:: Completer -Completer objects have the following method: + Completer objects have the following method: + .. method:: Completer.complete(text, state) -.. method:: Completer.complete(text, state) + Return the next possible completion for *text*. - Return the *state*\ th completion for *text*. + When called by the :mod:`readline` module, this method is called + successively with ``state == 0, 1, 2, ...`` until the method returns + ``None``. - If called for *text* that doesn't include a period character (``'.'``), it will - complete from names currently defined in :mod:`__main__`, :mod:`builtins` and - keywords (as defined by the :mod:`keyword` module). - - If called for a dotted name, it will try to evaluate anything without obvious - side-effects (functions will not be evaluated, but it can generate calls to - :meth:`__getattr__`) up to the last part, and find matches for the rest via the - :func:`dir` function. Any exception raised during the evaluation of the - expression is caught, silenced and :const:`None` is returned. + If called for *text* that doesn't include a period character (``'.'``), it will + complete from names currently defined in :mod:`__main__`, :mod:`builtins` and + keywords (as defined by the :mod:`keyword` module). + If called for a dotted name, it will try to evaluate anything without obvious + side-effects (functions will not be evaluated, but it can generate calls to + :meth:`~object.__getattr__`) up to the last part, and find matches for the + rest via the :func:`dir` function. Any exception raised during the + evaluation of the expression is caught, silenced and :const:`None` is + returned. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index d9147aaeee12bd..c91e698ff0753a 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -79,7 +79,6 @@ Doc/library/pyexpat.rst Doc/library/random.rst Doc/library/readline.rst Doc/library/resource.rst -Doc/library/rlcompleter.rst Doc/library/select.rst Doc/library/signal.rst Doc/library/smtplib.rst From 55ef998a8dead3874e8390284081290c1ccb46e2 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 15 Dec 2023 12:28:22 +0000 Subject: [PATCH 78/95] gh-112720: Move dis's cache output code to the Formatter, labels lookup to the arg_resolver. Reduce the number of parameters passed around. (#113108) --- Lib/dis.py | 168 +++++++++++++++++++++++-------------------- Lib/test/test_dis.py | 17 +++-- 2 files changed, 104 insertions(+), 81 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index 183091cb0d6098..1a2f1032d500af 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -113,7 +113,14 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, elif hasattr(x, 'co_code'): # Code object _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) elif isinstance(x, (bytes, bytearray)): # Raw bytecode - _disassemble_bytes(x, file=file, show_caches=show_caches, show_offsets=show_offsets) + labels_map = _make_labels_map(x) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + offset_width=len(str(max(len(x) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(labels_map=labels_map) + _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) else: @@ -394,23 +401,41 @@ def __str__(self): class Formatter: def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, - line_offset=0): + line_offset=0, show_caches=False): """Create a Formatter *file* where to write the output *lineno_width* sets the width of the line number field (0 omits it) *offset_width* sets the width of the instruction offset field *label_width* sets the width of the label field + *show_caches* is a boolean indicating whether to display cache lines - *line_offset* the line number (within the code unit) """ self.file = file self.lineno_width = lineno_width self.offset_width = offset_width self.label_width = label_width - + self.show_caches = show_caches def print_instruction(self, instr, mark_as_current=False): + self.print_instruction_line(instr, mark_as_current) + if self.show_caches and instr.cache_info: + offset = instr.offset + for name, size, data in instr.cache_info: + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + self.print_instruction_line( + Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions), + False) + + def print_instruction_line(self, instr, mark_as_current): """Format instruction details for inclusion in disassembly output.""" lineno_width = self.lineno_width offset_width = self.offset_width @@ -474,11 +499,14 @@ def print_exception_table(self, exception_entries): class ArgResolver: - def __init__(self, co_consts, names, varname_from_oparg, labels_map): + def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_map=None): self.co_consts = co_consts self.names = names self.varname_from_oparg = varname_from_oparg - self.labels_map = labels_map + self.labels_map = labels_map or {} + + def get_label_for_offset(self, offset): + return self.labels_map.get(offset, None) def get_argval_argrepr(self, op, arg, offset): get_name = None if self.names is None else self.names.__getitem__ @@ -547,8 +575,7 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = _intrinsic_2_descs[arg] return argval, argrepr - -def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False): +def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): """Iterator for the opcodes in methods, functions or code Generates a series of Instruction named tuples giving the details of @@ -567,9 +594,10 @@ def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False): line_offset = 0 original_code = co.co_code - labels_map = _make_labels_map(original_code) - arg_resolver = ArgResolver(co.co_consts, co.co_names, co._varname_from_oparg, - labels_map) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=_make_labels_map(original_code)) return _get_instructions_bytes(_get_code_array(co, adaptive), linestarts=linestarts, line_offset=line_offset, @@ -648,7 +676,7 @@ def _is_backward_jump(op): 'ENTER_EXECUTOR') def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, - original_code=None, labels_map=None, arg_resolver=None): + original_code=None, arg_resolver=None): """Iterate over the instructions in a bytecode string. Generates a sequence of Instruction namedtuples giving the details of each @@ -661,8 +689,6 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N original_code = original_code or code co_positions = co_positions or iter(()) - labels_map = labels_map or _make_labels_map(original_code) - starts_line = False local_line_number = None line_number = None @@ -684,10 +710,6 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N else: argval, argrepr = arg, repr(arg) - instr = Instruction(_all_opname[op], op, arg, argval, argrepr, - offset, start_offset, starts_line, line_number, - labels_map.get(offset, None), positions) - caches = _get_cache_size(_all_opname[deop]) # Advance the co_positions iterator: for _ in range(caches): @@ -701,10 +723,10 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N else: cache_info = None + label = arg_resolver.get_label_for_offset(offset) if arg_resolver else None yield Instruction(_all_opname[op], op, arg, argval, argrepr, offset, start_offset, starts_line, line_number, - labels_map.get(offset, None), positions, cache_info) - + label, positions, cache_info) def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, @@ -712,12 +734,20 @@ def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) exception_entries = _parse_exception_table(co) - _disassemble_bytes(_get_code_array(co, adaptive), - lasti, co._varname_from_oparg, - co.co_names, co.co_consts, linestarts, file=file, - exception_entries=exception_entries, - co_positions=co.co_positions(), show_caches=show_caches, - original_code=co.co_code, show_offsets=show_offsets) + labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + lineno_width=_get_lineno_width(linestarts), + offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(_get_code_array(co, adaptive), lasti, linestarts, + exception_entries=exception_entries, co_positions=co.co_positions(), + original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False): disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) @@ -764,60 +794,29 @@ def _get_lineno_width(linestarts): return lineno_width -def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None, - names=None, co_consts=None, linestarts=None, - *, file=None, line_offset=0, exception_entries=(), - co_positions=None, show_caches=False, original_code=None, - show_offsets=False): - - offset_width = len(str(max(len(code) - 2, 9999))) if show_offsets else 0 - - labels_map = _make_labels_map(original_code or code, exception_entries) - label_width = 4 + len(str(len(labels_map))) +def _disassemble_bytes(code, lasti=-1, linestarts=None, + *, line_offset=0, exception_entries=(), + co_positions=None, original_code=None, + arg_resolver=None, formatter=None): - formatter = Formatter(file=file, - lineno_width=_get_lineno_width(linestarts), - offset_width=offset_width, - label_width=label_width, - line_offset=line_offset) + assert formatter is not None + assert arg_resolver is not None - arg_resolver = ArgResolver(co_consts, names, varname_from_oparg, labels_map) instrs = _get_instructions_bytes(code, linestarts=linestarts, line_offset=line_offset, co_positions=co_positions, original_code=original_code, - labels_map=labels_map, arg_resolver=arg_resolver) - print_instructions(instrs, exception_entries, formatter, - show_caches=show_caches, lasti=lasti) + print_instructions(instrs, exception_entries, formatter, lasti=lasti) -def print_instructions(instrs, exception_entries, formatter, show_caches=False, lasti=-1): +def print_instructions(instrs, exception_entries, formatter, lasti=-1): for instr in instrs: - if show_caches: - is_current_instr = instr.offset == lasti - else: - # Each CACHE takes 2 bytes - is_current_instr = instr.offset <= lasti \ - <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) + # Each CACHE takes 2 bytes + is_current_instr = instr.offset <= lasti \ + <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) formatter.print_instruction(instr, is_current_instr) - deop = _deoptop(instr.opcode) - if show_caches and instr.cache_info: - offset = instr.offset - for name, size, data in instr.cache_info: - for i in range(size): - offset += 2 - # Only show the fancy argrepr for a CACHE instruction when it's - # the first entry for a particular cache value: - if i == 0: - argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" - else: - argrepr = "" - formatter.print_instruction( - Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, - False, None, None, instr.positions), - is_current_instr) formatter.print_exception_table(exception_entries) @@ -960,14 +959,15 @@ def __iter__(self): co = self.codeobj original_code = co.co_code labels_map = _make_labels_map(original_code, self.exception_entries) - arg_resolver = ArgResolver(co.co_consts, co.co_names, co._varname_from_oparg, - labels_map) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) return _get_instructions_bytes(_get_code_array(co, self.adaptive), linestarts=self._linestarts, line_offset=self._line_offset, co_positions=co.co_positions(), original_code=original_code, - labels_map=labels_map, arg_resolver=arg_resolver) def __repr__(self): @@ -995,18 +995,32 @@ def dis(self): else: offset = -1 with io.StringIO() as output: - _disassemble_bytes(_get_code_array(co, self.adaptive), - varname_from_oparg=co._varname_from_oparg, - names=co.co_names, co_consts=co.co_consts, + code = _get_code_array(co, self.adaptive) + offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 + + + labels_map = _make_labels_map(co.co_code, self.exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=output, + lineno_width=_get_lineno_width(self._linestarts), + offset_width=offset_width, + label_width=label_width, + line_offset=self._line_offset, + show_caches=self.show_caches) + + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(code, linestarts=self._linestarts, line_offset=self._line_offset, - file=output, lasti=offset, exception_entries=self.exception_entries, co_positions=co.co_positions(), - show_caches=self.show_caches, original_code=co.co_code, - show_offsets=self.show_offsets) + arg_resolver=arg_resolver, + formatter=formatter) return output.getvalue() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 12e2c57e50b0ba..0c7fd60f640854 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -2,6 +2,7 @@ import contextlib import dis +import functools import io import re import sys @@ -1982,19 +1983,27 @@ def f(opcode, oparg, offset, *init_args): self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<')) self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 'INTRINSIC_IMPORT_STAR')) + def get_instructions(self, code): + return dis._get_instructions_bytes(code) + def test_start_offset(self): # When no extended args are present, # start_offset should be equal to offset + instructions = list(dis.Bytecode(_f)) for instruction in instructions: self.assertEqual(instruction.offset, instruction.start_offset) + def last_item(iterable): + return functools.reduce(lambda a, b : b, iterable) + code = bytes([ opcode.opmap["LOAD_FAST"], 0x00, opcode.opmap["EXTENDED_ARG"], 0x01, opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, ]) - jump = list(dis._get_instructions_bytes(code))[-1] + labels_map = dis._make_labels_map(code) + jump = last_item(self.get_instructions(code)) self.assertEqual(4, jump.offset) self.assertEqual(2, jump.start_offset) @@ -2006,7 +2015,7 @@ def test_start_offset(self): opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, opcode.opmap["CACHE"], 0x00, ]) - jump = list(dis._get_instructions_bytes(code))[-1] + jump = last_item(self.get_instructions(code)) self.assertEqual(8, jump.offset) self.assertEqual(2, jump.start_offset) @@ -2021,7 +2030,7 @@ def test_start_offset(self): opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, opcode.opmap["CACHE"], 0x00, ]) - instructions = list(dis._get_instructions_bytes(code)) + instructions = list(self.get_instructions(code)) # 1st jump self.assertEqual(4, instructions[2].offset) self.assertEqual(2, instructions[2].start_offset) @@ -2042,7 +2051,7 @@ def test_cache_offset_and_end_offset(self): opcode.opmap["CACHE"], 0x00, opcode.opmap["CACHE"], 0x00 ]) - instructions = list(dis._get_instructions_bytes(code)) + instructions = list(self.get_instructions(code)) self.assertEqual(2, instructions[0].cache_offset) self.assertEqual(10, instructions[0].end_offset) self.assertEqual(12, instructions[1].cache_offset) From 737d23ffcd16909274f21c932215ec104140fb1c Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 15 Dec 2023 05:03:17 -0800 Subject: [PATCH 79/95] GH-111485: Mark some instructions as `TIER_ONE_ONLY` (GH-113155) --- Python/bytecodes.c | 7 ++ Python/executor_cases.c.h | 139 ------------------------------------- Python/generated_cases.c.h | 7 ++ 3 files changed, 14 insertions(+), 139 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 68bb15c2b536eb..19e2268046fcdc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -506,6 +506,7 @@ dummy_func( // specializations, but there is no output. // At the end we just skip over the STORE_FAST. op(_BINARY_OP_INPLACE_ADD_UNICODE, (unused/1, left, right --)) { + TIER_ONE_ONLY assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left); @@ -786,6 +787,7 @@ dummy_func( } inst(INTERPRETER_EXIT, (retval --)) { + TIER_ONE_ONLY assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -1072,6 +1074,7 @@ dummy_func( } inst(YIELD_VALUE, (retval -- unused)) { + TIER_ONE_ONLY // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -2297,6 +2300,7 @@ dummy_func( } inst(JUMP_FORWARD, (--)) { + TIER_ONE_ONLY JUMPBY(oparg); } @@ -2402,6 +2406,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = _IS_NONE + _POP_JUMP_IF_FALSE; inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + TIER_ONE_ONLY /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3454,6 +3459,7 @@ dummy_func( // This is secretly a super-instruction inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { + TIER_ONE_ONLY assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append); @@ -3792,6 +3798,7 @@ dummy_func( } inst(RETURN_GENERATOR, (--)) { + TIER_ONE_ONLY assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2519a4ee546a5e..7cb60cbc1dd3ff 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -375,38 +375,6 @@ break; } - case _BINARY_OP_INPLACE_ADD_UNICODE: { - PyObject *right; - PyObject *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - if (*target_local != left) goto deoptimize; - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error_tier_two; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - stack_pointer += -2; - break; - } - case _BINARY_SUBSCR: { PyObject *sub; PyObject *container; @@ -690,18 +658,6 @@ break; } - case _INTERPRETER_EXIT: { - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; - } - case _POP_FRAME: { PyObject *retval; retval = stack_pointer[-1]; @@ -846,33 +802,6 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ - case _YIELD_VALUE: { - PyObject *retval; - oparg = CURRENT_OPARG(); - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } - case _POP_EXCEPT: { PyObject *exc_value; exc_value = stack_pointer[-1]; @@ -2084,12 +2013,6 @@ break; } - case _JUMP_FORWARD: { - oparg = CURRENT_OPARG(); - JUMPBY(oparg); - break; - } - /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ @@ -2111,17 +2034,6 @@ break; } - case _JUMP_BACKWARD_NO_INTERRUPT: { - oparg = CURRENT_OPARG(); - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); - break; - } - case _GET_LEN: { PyObject *obj; PyObject *len_o; @@ -3060,32 +2972,6 @@ break; } - case _CALL_LIST_APPEND: { - PyObject **args; - PyObject *self; - PyObject *callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.list_append) goto deoptimize; - assert(self != NULL); - if (!PyList_Check(self)) goto deoptimize; - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - DISPATCH(); - } - case _CALL_METHOD_DESCRIPTOR_O: { PyObject **args; PyObject *self_or_null; @@ -3307,31 +3193,6 @@ break; } - case _RETURN_GENERATOR: { - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - GOTO_ERROR(error); - } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - case _BUILD_SLICE: { PyObject *step = NULL; PyObject *stop; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 65e6f11f68b38c..24f26722d7a745 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -238,6 +238,7 @@ } // _BINARY_OP_INPLACE_ADD_UNICODE { + TIER_ONE_ONLY assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); @@ -1446,6 +1447,7 @@ args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + TIER_ONE_ONLY assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); @@ -3182,6 +3184,7 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); PyObject *retval; retval = stack_pointer[-1]; + TIER_ONE_ONLY assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -3253,6 +3256,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + TIER_ONE_ONLY /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3266,6 +3270,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); + TIER_ONE_ONLY JUMPBY(oparg); DISPATCH(); } @@ -4793,6 +4798,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); + TIER_ONE_ONLY assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -5764,6 +5770,7 @@ INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; + TIER_ONE_ONLY // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. From 8f8f0f97e126db9ca470fd7e7b2944c150db6305 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 15 Dec 2023 15:24:30 +0200 Subject: [PATCH 80/95] gh-61648: Detect line numbers of properties in doctests (GH-113161) --- Lib/doctest.py | 2 ++ Lib/test/doctest_lineno.py | 16 ++++++++++++++++ Lib/test/test_doctest.py | 2 ++ ...2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst | 1 + 4 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index d109b6c9e37343..114aac62a34e95 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1136,6 +1136,8 @@ def _find_lineno(self, obj, source_lines): # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ + if isinstance(obj, property): + obj = obj.fget if inspect.isfunction(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. obj = obj.__code__ diff --git a/Lib/test/doctest_lineno.py b/Lib/test/doctest_lineno.py index 729a68aceaa990..677c569cf710eb 100644 --- a/Lib/test/doctest_lineno.py +++ b/Lib/test/doctest_lineno.py @@ -49,5 +49,21 @@ def method_with_doctest(self): 'method_with_doctest' """ + @classmethod + def classmethod_with_doctest(cls): + """ + This has a doctest! + >>> MethodWrapper.classmethod_with_doctest.__name__ + 'classmethod_with_doctest' + """ + + @property + def property_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.property_with_doctest.__name__ + 'property_with_doctest' + """ + # https://github.com/python/cpython/issues/99433 str_wrapper = object().__str__ diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 36328f8086c7ad..46a51007f9644d 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -670,9 +670,11 @@ def basics(): r""" 30 test.doctest_lineno.ClassWithDoctest None test.doctest_lineno.ClassWithoutDocstring None test.doctest_lineno.MethodWrapper + 53 test.doctest_lineno.MethodWrapper.classmethod_with_doctest 39 test.doctest_lineno.MethodWrapper.method_with_docstring 45 test.doctest_lineno.MethodWrapper.method_with_doctest None test.doctest_lineno.MethodWrapper.method_without_docstring + 61 test.doctest_lineno.MethodWrapper.property_with_doctest 4 test.doctest_lineno.func_with_docstring 12 test.doctest_lineno.func_with_doctest None test.doctest_lineno.func_without_docstring diff --git a/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst b/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst new file mode 100644 index 00000000000000..c841e5c7f7683a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst @@ -0,0 +1 @@ +Detect line numbers of properties in doctests. From d1a2adfb0820ee730fa3e4bbc4bd88a67aa50666 Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 15 Dec 2023 21:42:37 +0800 Subject: [PATCH 81/95] gh-112278: Add retry in WMI tests in case of slow initialization (GH-113154) --- Lib/test/test_wmi.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 3445702846d8a0..bf8c52e646dc18 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -1,17 +1,29 @@ # Test the internal _wmi module on Windows # This is used by the platform module, and potentially others +import time import unittest -from test.support import import_helper, requires_resource +from test.support import import_helper, requires_resource, LOOPBACK_TIMEOUT # Do this first so test will be skipped if module doesn't exist _wmi = import_helper.import_module('_wmi', required_on=['win']) +def wmi_exec_query(query): + # gh-112278: WMI maybe slow response when first call. + try: + return _wmi.exec_query(query) + except WindowsError as e: + if e.winerror != 258: + raise + time.sleep(LOOPBACK_TIMEOUT) + return _wmi.exec_query(query) + + class WmiTests(unittest.TestCase): def test_wmi_query_os_version(self): - r = _wmi.exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") + r = wmi_exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") self.assertEqual(1, len(r)) k, eq, v = r[0].partition("=") self.assertEqual("=", eq, r[0]) @@ -28,7 +40,7 @@ def test_wmi_query_repeated(self): def test_wmi_query_error(self): # Invalid queries fail with OSError try: - _wmi.exec_query("SELECT InvalidColumnName FROM InvalidTableName") + wmi_exec_query("SELECT InvalidColumnName FROM InvalidTableName") except OSError as ex: if ex.winerror & 0xFFFFFFFF == 0x80041010: # This is the expected error code. All others should fail the test @@ -42,7 +54,7 @@ def test_wmi_query_repeated_error(self): def test_wmi_query_not_select(self): # Queries other than SELECT are blocked to avoid potential exploits with self.assertRaises(ValueError): - _wmi.exec_query("not select, just in case someone tries something") + wmi_exec_query("not select, just in case someone tries something") @requires_resource('cpu') def test_wmi_query_overflow(self): @@ -50,11 +62,11 @@ def test_wmi_query_overflow(self): # Test multiple times to ensure consistency for _ in range(2): with self.assertRaises(OSError): - _wmi.exec_query("SELECT * FROM CIM_DataFile") + wmi_exec_query("SELECT * FROM CIM_DataFile") def test_wmi_query_multiple_rows(self): # Multiple instances should have an extra null separator - r = _wmi.exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") + r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") self.assertFalse(r.startswith("\0"), r) self.assertFalse(r.endswith("\0"), r) it = iter(r.split("\0")) @@ -69,6 +81,6 @@ def test_wmi_query_threads(self): from concurrent.futures import ThreadPoolExecutor query = "SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000" with ThreadPoolExecutor(4) as pool: - task = [pool.submit(_wmi.exec_query, query) for _ in range(32)] + task = [pool.submit(wmi_exec_query, query) for _ in range(32)] for t in task: self.assertRegex(t.result(), "ProcessId=") From 4026ad5b2c595b855a3605420cfa0e3d49e63db7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 15 Dec 2023 15:57:49 +0100 Subject: [PATCH 82/95] gh-113009: Fix multiprocessing Process.terminate() on Windows (#113128) On Windows, Process.terminate() no longer sets the returncode attribute to always call WaitForSingleObject() in Process.wait(). Previously, sometimes the process was still running after TerminateProcess() even if GetExitCodeProcess() is not STILL_ACTIVE. --- Lib/multiprocessing/popen_spawn_win32.py | 54 ++++++++++--------- ...-12-14-19-00-29.gh-issue-113009.6LNdjz.rst | 5 ++ 2 files changed, 35 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index af044305709e56..49d4c7eea22411 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -101,18 +101,20 @@ def duplicate_for_child(self, handle): return reduction.duplicate(handle, self.sentinel) def wait(self, timeout=None): - if self.returncode is None: - if timeout is None: - msecs = _winapi.INFINITE - else: - msecs = max(0, int(timeout * 1000 + 0.5)) - - res = _winapi.WaitForSingleObject(int(self._handle), msecs) - if res == _winapi.WAIT_OBJECT_0: - code = _winapi.GetExitCodeProcess(self._handle) - if code == TERMINATE: - code = -signal.SIGTERM - self.returncode = code + if self.returncode is not None: + return self.returncode + + if timeout is None: + msecs = _winapi.INFINITE + else: + msecs = max(0, int(timeout * 1000 + 0.5)) + + res = _winapi.WaitForSingleObject(int(self._handle), msecs) + if res == _winapi.WAIT_OBJECT_0: + code = _winapi.GetExitCodeProcess(self._handle) + if code == TERMINATE: + code = -signal.SIGTERM + self.returncode = code return self.returncode @@ -120,18 +122,22 @@ def poll(self): return self.wait(timeout=0) def terminate(self): - if self.returncode is None: - try: - _winapi.TerminateProcess(int(self._handle), TERMINATE) - except PermissionError: - # ERROR_ACCESS_DENIED (winerror 5) is received when the - # process already died. - code = _winapi.GetExitCodeProcess(int(self._handle)) - if code == _winapi.STILL_ACTIVE: - raise - self.returncode = code - else: - self.returncode = -signal.SIGTERM + if self.returncode is not None: + return + + try: + _winapi.TerminateProcess(int(self._handle), TERMINATE) + except PermissionError: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + code = _winapi.GetExitCodeProcess(int(self._handle)) + if code == _winapi.STILL_ACTIVE: + raise + + # gh-113009: Don't set self.returncode. Even if GetExitCodeProcess() + # returns an exit code different than STILL_ACTIVE, the process can + # still be running. Only set self.returncode once WaitForSingleObject() + # returns WAIT_OBJECT_0 in wait(). kill = terminate diff --git a/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst b/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst new file mode 100644 index 00000000000000..6fd7f7f9afdfa2 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst @@ -0,0 +1,5 @@ +:mod:`multiprocessing`: On Windows, fix a race condition in +``Process.terminate()``: no longer set the ``returncode`` attribute to +always call ``WaitForSingleObject()`` in ``Process.wait()``. Previously, +sometimes the process was still running after ``TerminateProcess()`` even if +``GetExitCodeProcess()`` is not ``STILL_ACTIVE``. Patch by Victor Stinner. From 4a153a1d3b18803a684cd1bcc2cdf3ede3dbae19 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 15 Dec 2023 16:10:40 +0100 Subject: [PATCH 83/95] [CVE-2023-27043] gh-102988: Reject malformed addresses in email.parseaddr() (#111116) Detect email address parsing errors and return empty tuple to indicate the parsing error (old API). Add an optional 'strict' parameter to getaddresses() and parseaddr() functions. Patch by Thomas Dwyer. Co-Authored-By: Thomas Dwyer --- Doc/library/email.utils.rst | 19 +- Doc/whatsnew/3.13.rst | 13 ++ Lib/email/utils.py | 151 +++++++++++++- Lib/test/test_email/test_email.py | 187 +++++++++++++++++- ...-10-20-15-28-08.gh-issue-102988.dStNO7.rst | 8 + 5 files changed, 357 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 345b64001c1ace..d693a9bc3933b5 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -58,13 +58,18 @@ of the new API. begins with angle brackets, they are stripped off. -.. function:: parseaddr(address) +.. function:: parseaddr(address, *, strict=True) Parse address -- which should be the value of some address-containing field such as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and *email address* parts. Returns a tuple of that information, unless the parse fails, in which case a 2-tuple of ``('', '')`` is returned. + If *strict* is true, use a strict parser which rejects malformed inputs. + + .. versionchanged:: 3.13 + Add *strict* optional parameter and reject malformed inputs by default. + .. function:: formataddr(pair, charset='utf-8') @@ -82,12 +87,15 @@ of the new API. Added the *charset* option. -.. function:: getaddresses(fieldvalues) +.. function:: getaddresses(fieldvalues, *, strict=True) This method returns a list of 2-tuples of the form returned by ``parseaddr()``. *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all `. Here's a simple - example that gets all the recipients of a message:: + :meth:`Message.get_all `. + + If *strict* is true, use a strict parser which rejects malformed inputs. + + Here's a simple example that gets all the recipients of a message:: from email.utils import getaddresses @@ -97,6 +105,9 @@ of the new API. resent_ccs = msg.get_all('resent-cc', []) all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs) + .. versionchanged:: 3.13 + Add *strict* optional parameter and reject malformed inputs by default. + .. function:: parsedate(date) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e22257853d8333..4f9643967d20cf 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -199,6 +199,19 @@ doctest :attr:`doctest.TestResults.skipped` attributes. (Contributed by Victor Stinner in :gh:`108794`.) +email +----- + +* :func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now return + ``('', '')`` 2-tuples in more situations where invalid email addresses are + encountered instead of potentially inaccurate values. Add optional *strict* + parameter to these two functions: use ``strict=False`` to get the old + behavior, accept malformed inputs. + ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to + check if the *strict* paramater is available. + (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve + the CVE-2023-27043 fix.) + glob ---- diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 9175f2fdb6e69e..103cef61a83538 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -43,6 +43,7 @@ specialsre = re.compile(r'[][\\()<>@,:;".]') escapesre = re.compile(r'[\\"]') + def _has_surrogates(s): """Return True if s may contain surrogate-escaped binary data.""" # This check is based on the fact that unless there are surrogates, utf8 @@ -103,12 +104,127 @@ def formataddr(pair, charset='utf-8'): return address +def _iter_escaped_chars(addr): + pos = 0 + escape = False + for pos, ch in enumerate(addr): + if escape: + yield (pos, '\\' + ch) + escape = False + elif ch == '\\': + escape = True + else: + yield (pos, ch) + if escape: + yield (pos, '\\') + + +def _strip_quoted_realnames(addr): + """Strip real names between quotes.""" + if '"' not in addr: + # Fast path + return addr + + start = 0 + open_pos = None + result = [] + for pos, ch in _iter_escaped_chars(addr): + if ch == '"': + if open_pos is None: + open_pos = pos + else: + if start != open_pos: + result.append(addr[start:open_pos]) + start = pos + 1 + open_pos = None + + if start < len(addr): + result.append(addr[start:]) + + return ''.join(result) -def getaddresses(fieldvalues): - """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" - all = COMMASPACE.join(str(v) for v in fieldvalues) - a = _AddressList(all) - return a.addresslist + +supports_strict_parsing = True + +def getaddresses(fieldvalues, *, strict=True): + """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue. + + When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in + its place. + + If strict is true, use a strict parser which rejects malformed inputs. + """ + + # If strict is true, if the resulting list of parsed addresses is greater + # than the number of fieldvalues in the input list, a parsing error has + # occurred and consequently a list containing a single empty 2-tuple [('', + # '')] is returned in its place. This is done to avoid invalid output. + # + # Malformed input: getaddresses(['alice@example.com ']) + # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')] + # Safe output: [('', '')] + + if not strict: + all = COMMASPACE.join(str(v) for v in fieldvalues) + a = _AddressList(all) + return a.addresslist + + fieldvalues = [str(v) for v in fieldvalues] + fieldvalues = _pre_parse_validation(fieldvalues) + addr = COMMASPACE.join(fieldvalues) + a = _AddressList(addr) + result = _post_parse_validation(a.addresslist) + + # Treat output as invalid if the number of addresses is not equal to the + # expected number of addresses. + n = 0 + for v in fieldvalues: + # When a comma is used in the Real Name part it is not a deliminator. + # So strip those out before counting the commas. + v = _strip_quoted_realnames(v) + # Expected number of addresses: 1 + number of commas + n += 1 + v.count(',') + if len(result) != n: + return [('', '')] + + return result + + +def _check_parenthesis(addr): + # Ignore parenthesis in quoted real names. + addr = _strip_quoted_realnames(addr) + + opens = 0 + for pos, ch in _iter_escaped_chars(addr): + if ch == '(': + opens += 1 + elif ch == ')': + opens -= 1 + if opens < 0: + return False + return (opens == 0) + + +def _pre_parse_validation(email_header_fields): + accepted_values = [] + for v in email_header_fields: + if not _check_parenthesis(v): + v = "('', '')" + accepted_values.append(v) + + return accepted_values + + +def _post_parse_validation(parsed_email_header_tuples): + accepted_values = [] + # The parser would have parsed a correctly formatted domain-literal + # The existence of an [ after parsing indicates a parsing failure + for v in parsed_email_header_tuples: + if '[' in v[1]: + v = ('', '') + accepted_values.append(v) + + return accepted_values def _format_timetuple_and_zone(timetuple, zone): @@ -207,16 +323,33 @@ def parsedate_to_datetime(data): tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) -def parseaddr(addr): +def parseaddr(addr, *, strict=True): """ Parse addr into its constituent realname and email address parts. Return a tuple of realname and email address, unless the parse fails, in which case return a 2-tuple of ('', ''). + + If strict is True, use a strict parser which rejects malformed inputs. """ - addrs = _AddressList(addr).addresslist - if not addrs: - return '', '' + if not strict: + addrs = _AddressList(addr).addresslist + if not addrs: + return ('', '') + return addrs[0] + + if isinstance(addr, list): + addr = addr[0] + + if not isinstance(addr, str): + return ('', '') + + addr = _pre_parse_validation([addr])[0] + addrs = _post_parse_validation(_AddressList(addr).addresslist) + + if not addrs or len(addrs) > 1: + return ('', '') + return addrs[0] diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 512464f87162cd..39d4ace8d4a1d8 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -16,6 +16,7 @@ import email import email.policy +import email.utils from email.charset import Charset from email.generator import Generator, DecodedGenerator, BytesGenerator @@ -3337,15 +3338,137 @@ def test_getaddresses_comma_in_name(self): ], ) + def test_parsing_errors(self): + """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056""" + alice = 'alice@example.org' + bob = 'bob@example.com' + empty = ('', '') + + # Test utils.getaddresses() and utils.parseaddr() on malformed email + # addresses: default behavior (strict=True) rejects malformed address, + # and strict=False which tolerates malformed address. + for invalid_separator, expected_non_strict in ( + ('(', [(f'<{bob}>', alice)]), + (')', [('', alice), empty, ('', bob)]), + ('<', [('', alice), empty, ('', bob), empty]), + ('>', [('', alice), empty, ('', bob)]), + ('[', [('', f'{alice}[<{bob}>]')]), + (']', [('', alice), empty, ('', bob)]), + ('@', [empty, empty, ('', bob)]), + (';', [('', alice), empty, ('', bob)]), + (':', [('', alice), ('', bob)]), + ('.', [('', alice + '.'), ('', bob)]), + ('"', [('', alice), ('', f'<{bob}>')]), + ): + address = f'{alice}{invalid_separator}<{bob}>' + with self.subTest(address=address): + self.assertEqual(utils.getaddresses([address]), + [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + expected_non_strict) + + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Comma (',') is treated differently depending on strict parameter. + # Comma without quotes. + address = f'{alice},<{bob}>' + self.assertEqual(utils.getaddresses([address]), + [('', alice), ('', bob)]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', alice), ('', bob)]) + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Real name between quotes containing comma. + address = '"Alice, alice@example.org" ' + expected_strict = ('Alice, alice@example.org', 'bob@example.com') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Valid parenthesis in comments. + address = 'alice@example.org (Alice)' + expected_strict = ('Alice', 'alice@example.org') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Invalid parenthesis in comments. + address = 'alice@example.org )Alice(' + self.assertEqual(utils.getaddresses([address]), [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', 'alice@example.org'), ('', ''), ('', 'Alice')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Two addresses with quotes separated by comma. + address = '"Jane Doe" , "John Doe" ' + self.assertEqual(utils.getaddresses([address]), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Test email.utils.supports_strict_parsing attribute + self.assertEqual(email.utils.supports_strict_parsing, True) + def test_getaddresses_nasty(self): - eq = self.assertEqual - eq(utils.getaddresses(['foo: ;']), [('', '')]) - eq(utils.getaddresses( - ['[]*-- =~$']), - [('', ''), ('', ''), ('', '*--')]) - eq(utils.getaddresses( - ['foo: ;', '"Jason R. Mastaler" ']), - [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]) + for addresses, expected in ( + (['"Sürname, Firstname" '], + [('Sürname, Firstname', 'to@example.com')]), + + (['foo: ;'], + [('', '')]), + + (['foo: ;', '"Jason R. Mastaler" '], + [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]), + + ([r'Pete(A nice \) chap) '], + [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]), + + (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'], + [('', '')]), + + (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'], + [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]), + + (['John Doe '], + [('John Doe (comment)', 'jdoe@machine.example')]), + + (['"Mary Smith: Personal Account" '], + [('Mary Smith: Personal Account', 'smith@home.example')]), + + (['Undisclosed recipients:;'], + [('', '')]), + + ([r', "Giant; \"Big\" Box" '], + [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]), + ): + with self.subTest(addresses=addresses): + self.assertEqual(utils.getaddresses(addresses), + expected) + self.assertEqual(utils.getaddresses(addresses, strict=False), + expected) + + addresses = ['[]*-- =~$'] + self.assertEqual(utils.getaddresses(addresses), + [('', '')]) + self.assertEqual(utils.getaddresses(addresses, strict=False), + [('', ''), ('', ''), ('', '*--')]) def test_getaddresses_embedded_comment(self): """Test proper handling of a nested comment""" @@ -3536,6 +3659,54 @@ def test_mime_classes_policy_argument(self): m = cls(*constructor, policy=email.policy.default) self.assertIs(m.policy, email.policy.default) + def test_iter_escaped_chars(self): + self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')), + [(0, 'a'), + (2, '\\\\'), + (3, 'b'), + (5, '\\"'), + (6, 'c'), + (8, '\\\\'), + (9, '"'), + (10, 'd')]) + self.assertEqual(list(utils._iter_escaped_chars('a\\')), + [(0, 'a'), (1, '\\')]) + + def test_strip_quoted_realnames(self): + def check(addr, expected): + self.assertEqual(utils._strip_quoted_realnames(addr), expected) + + check('"Jane Doe" , "John Doe" ', + ' , ') + check(r'"Jane \"Doe\"." ', + ' ') + + # special cases + check(r'before"name"after', 'beforeafter') + check(r'before"name"', 'before') + check(r'b"name"', 'b') # single char + check(r'"name"after', 'after') + check(r'"name"a', 'a') # single char + check(r'"name"', '') + + # no change + for addr in ( + 'Jane Doe , John Doe ', + 'lone " quote', + ): + self.assertEqual(utils._strip_quoted_realnames(addr), addr) + + + def test_check_parenthesis(self): + addr = 'alice@example.net' + self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)')) + self.assertFalse(utils._check_parenthesis(f'{addr} )Alice(')) + self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))')) + self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)')) + + # Ignore real name between quotes + self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}')) + # Test the iterator/generators class TestIterators(TestEmailBase): diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst new file mode 100644 index 00000000000000..3d0e9e4078c934 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst @@ -0,0 +1,8 @@ +:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now +return ``('', '')`` 2-tuples in more situations where invalid email +addresses are encountered instead of potentially inaccurate values. Add +optional *strict* parameter to these two functions: use ``strict=False`` to +get the old behavior, accept malformed inputs. +``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check +if the *strict* paramater is available. Patch by Thomas Dwyer and Victor +Stinner to improve the CVE-2023-27043 fix. From e365c943f24ab577cf7a4bf12bc0d2619ab9ae47 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 15 Dec 2023 17:36:25 +0200 Subject: [PATCH 84/95] gh-113172: Fix compiler warnings in Modules/_xxinterpqueuesmodule.c (GH-113173) Fix compiler waarnings in Modules/_xxinterpqueuesmodule.c --- Modules/_xxinterpqueuesmodule.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 2cc3a2ac5dc85f..537ba9188055dd 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -234,11 +234,13 @@ static int add_exctype(PyObject *mod, PyObject **p_state_field, const char *qualname, const char *doc, PyObject *base) { +#ifndef NDEBUG const char *dot = strrchr(qualname, '.'); assert(dot != NULL); const char *name = dot+1; assert(*p_state_field == NULL); assert(!PyObject_HasAttrStringWithError(mod, name)); +#endif PyObject *exctype = PyErr_NewExceptionWithDoc(qualname, doc, base, NULL); if (exctype == NULL) { return -1; @@ -1505,7 +1507,7 @@ queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) } int64_t qid = qidarg.id; - int is_full; + int is_full = 0; int err = queue_is_full(&_globals.queues, qid, &is_full); if (handle_queue_error(err, self, qid)) { return NULL; From c2c4879b0a535b2a43005ba4983408082ef94bd2 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 15 Dec 2023 17:05:12 +0000 Subject: [PATCH 85/95] gh-101100: Fix Sphinx nitpicks in `library/numbers.rst` (#113162) --- Doc/library/numbers.rst | 19 ++++++++++--------- Doc/tools/.nitignore | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 2a05b56db051f9..17d1a275f04c9b 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`numbers` module (:pep:`3141`) defines a hierarchy of numeric +The :mod:`!numbers` module (:pep:`3141`) defines a hierarchy of numeric :term:`abstract base classes ` which progressively define more operations. None of the types defined in this module are intended to be instantiated. @@ -45,7 +45,7 @@ The numeric tower .. class:: Real - To :class:`Complex`, :class:`Real` adds the operations that work on real + To :class:`Complex`, :class:`!Real` adds the operations that work on real numbers. In short, those are: a conversion to :class:`float`, :func:`math.trunc`, @@ -126,7 +126,8 @@ We want to implement the arithmetic operations so that mixed-mode operations either call an implementation whose author knew about the types of both arguments, or convert both to the nearest built in type and do the operation there. For subtypes of :class:`Integral`, this -means that :meth:`__add__` and :meth:`__radd__` should be defined as:: +means that :meth:`~object.__add__` and :meth:`~object.__radd__` should be +defined as:: class MyIntegral(Integral): @@ -160,15 +161,15 @@ refer to ``MyIntegral`` and ``OtherTypeIKnowAbout`` as of :class:`Complex` (``a : A <: Complex``), and ``b : B <: Complex``. I'll consider ``a + b``: -1. If ``A`` defines an :meth:`__add__` which accepts ``b``, all is +1. If ``A`` defines an :meth:`~object.__add__` which accepts ``b``, all is well. 2. If ``A`` falls back to the boilerplate code, and it were to - return a value from :meth:`__add__`, we'd miss the possibility - that ``B`` defines a more intelligent :meth:`__radd__`, so the + return a value from :meth:`~object.__add__`, we'd miss the possibility + that ``B`` defines a more intelligent :meth:`~object.__radd__`, so the boilerplate should return :const:`NotImplemented` from - :meth:`__add__`. (Or ``A`` may not implement :meth:`__add__` at + :meth:`!__add__`. (Or ``A`` may not implement :meth:`!__add__` at all.) -3. Then ``B``'s :meth:`__radd__` gets a chance. If it accepts +3. Then ``B``'s :meth:`~object.__radd__` gets a chance. If it accepts ``a``, all is well. 4. If it falls back to the boilerplate, there are no more possible methods to try, so this is where the default implementation @@ -180,7 +181,7 @@ Complex``. I'll consider ``a + b``: If ``A <: Complex`` and ``B <: Real`` without sharing any other knowledge, then the appropriate shared operation is the one involving the built -in :class:`complex`, and both :meth:`__radd__` s land there, so ``a+b +in :class:`complex`, and both :meth:`~object.__radd__` s land there, so ``a+b == b+a``. Because most of the operations on any given type will be very similar, diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index c91e698ff0753a..23d4d3e522b455 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -65,7 +65,6 @@ Doc/library/lzma.rst Doc/library/mmap.rst Doc/library/multiprocessing.rst Doc/library/multiprocessing.shared_memory.rst -Doc/library/numbers.rst Doc/library/optparse.rst Doc/library/os.rst Doc/library/pickle.rst From 1addde0c698f7f8eb716bcecf63d119e19e1ecda Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 15 Dec 2023 17:15:34 +0000 Subject: [PATCH 86/95] gh-101100: Fix various Sphinx warnings for dunder references in the `library/` directory (#113163) --- Doc/conf.py | 1 + Doc/library/datetime.rst | 3 ++- Doc/library/exceptions.rst | 4 ++-- Doc/library/test.rst | 3 ++- Doc/library/unittest.mock.rst | 23 +++++++++++++---------- Doc/library/unittest.rst | 4 ++-- Doc/library/wsgiref.rst | 11 +++++++---- Doc/library/xmlrpc.client.rst | 9 +++++---- 8 files changed, 34 insertions(+), 24 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 4077eaf5a139b0..dc09b0b51ca84c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -248,6 +248,7 @@ # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: ('py:attr', '__annotations__'), + ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), ('py:meth', 'index'), # list.index, tuple.index, etc. ] diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1bab1fb9bd9464..686b37754368c1 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1985,7 +1985,8 @@ Examples of working with a :class:`.time` object:: American EST and EDT. Special requirement for pickling: A :class:`tzinfo` subclass must have an - :meth:`__init__` method that can be called with no arguments, otherwise it can be + :meth:`~object.__init__` method that can be called with no arguments, + otherwise it can be pickled but possibly not unpickled again. This is a technical requirement that may be relaxed in the future. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index f7891f2732bdb1..f821776c286133 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -1009,9 +1009,9 @@ their subgroups based on the types of the contained exceptions. True - Note that :exc:`BaseExceptionGroup` defines :meth:`__new__`, so + Note that :exc:`BaseExceptionGroup` defines :meth:`~object.__new__`, so subclasses that need a different constructor signature need to - override that rather than :meth:`__init__`. For example, the following + override that rather than :meth:`~object.__init__`. For example, the following defines an exception group subclass which accepts an exit_code and and constructs the group's message from it. :: diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 6cbdc39b3c024d..9173db07fd0071 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1408,7 +1408,8 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. class:: FakePath(path) - Simple :term:`path-like object`. It implements the :meth:`__fspath__` + Simple :term:`path-like object`. It implements the + :meth:`~os.PathLike.__fspath__` method which just returns the *path* argument. If *path* is an exception, it will be raised in :meth:`!__fspath__`. diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index d6ac4d09300d9f..f1cc482c5cfe2a 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -824,8 +824,9 @@ apply to method calls on the mock object. .. class:: PropertyMock(*args, **kwargs) - A mock intended to be used as a property, or other descriptor, on a class. - :class:`PropertyMock` provides :meth:`__get__` and :meth:`__set__` methods + A mock intended to be used as a :class:`property`, or other + :term:`descriptor`, on a class. :class:`PropertyMock` provides + :meth:`~object.__get__` and :meth:`~object.__set__` methods so you can specify a return value when it is fetched. Fetching a :class:`PropertyMock` instance from an object calls the mock, with @@ -1707,8 +1708,9 @@ Keywords can be used in the :func:`patch.dict` call to set values in the diction :func:`patch.dict` can be used with dictionary like objects that aren't actually dictionaries. At the very minimum they must support item getting, setting, deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`~object.__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either -:meth:`__iter__` or :meth:`__contains__`. +magic methods :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, +:meth:`~object.__delitem__` and either :meth:`~container.__iter__` or +:meth:`~object.__contains__`. >>> class Container: ... def __init__(self): @@ -2171,7 +2173,7 @@ For example: >>> object() in mock False -The two equality methods, :meth:`__eq__` and :meth:`__ne__`, are special. +The two equality methods, :meth:`!__eq__` and :meth:`!__ne__`, are special. They do the default equality comparison on identity, using the :attr:`~Mock.side_effect` attribute, unless you change their return value to return something else:: @@ -2521,8 +2523,8 @@ mock_open *read_data* is now reset on each call to the *mock*. .. versionchanged:: 3.8 - Added :meth:`__iter__` to implementation so that iteration (such as in for - loops) correctly consumes *read_data*. + Added :meth:`~container.__iter__` to implementation so that iteration + (such as in for loops) correctly consumes *read_data*. Using :func:`open` as a context manager is a great way to ensure your file handles are closed properly and is becoming common:: @@ -2704,7 +2706,7 @@ able to use autospec. On the other hand it is much better to design your objects so that introspection is safe [#]_. A more serious problem is that it is common for instance attributes to be -created in the :meth:`__init__` method and not to exist on the class at all. +created in the :meth:`~object.__init__` method and not to exist on the class at all. *autospec* can't know about any dynamically created attributes and restricts the api to visible attributes. :: @@ -2745,8 +2747,9 @@ this particular scenario: AttributeError: Mock object has no attribute 'a' Probably the best way of solving the problem is to add class attributes as -default values for instance members initialised in :meth:`__init__`. Note that if -you are only setting default attributes in :meth:`__init__` then providing them via +default values for instance members initialised in :meth:`~object.__init__`. +Note that if +you are only setting default attributes in :meth:`!__init__` then providing them via class attributes (shared between instances of course) is faster too. e.g. .. code-block:: python diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c04e6c378cc3b1..9aad96bdf11bf8 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1733,7 +1733,7 @@ Grouping tests .. method:: __iter__() Tests grouped by a :class:`TestSuite` are always accessed by iteration. - Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note + Subclasses can lazily provide tests by overriding :meth:`!__iter__`. Note that this method may be called several times on a single suite (for example when counting tests or comparing for equality) so the tests returned by repeated iterations before :meth:`TestSuite.run` must be the @@ -1744,7 +1744,7 @@ Grouping tests .. versionchanged:: 3.2 In earlier versions the :class:`TestSuite` accessed tests directly rather - than through iteration, so overriding :meth:`__iter__` wasn't sufficient + than through iteration, so overriding :meth:`!__iter__` wasn't sufficient for providing tests. .. versionchanged:: 3.4 diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index be9e56b04c1fbf..c2b0ba7046967e 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -201,8 +201,9 @@ manipulation of WSGI response headers using a mapping-like interface. an empty list. :class:`Headers` objects support typical mapping operations including - :meth:`~object.__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, - :meth:`__delitem__` and :meth:`__contains__`. For each of + :meth:`~object.__getitem__`, :meth:`~dict.get`, :meth:`~object.__setitem__`, + :meth:`~dict.setdefault`, + :meth:`~object.__delitem__` and :meth:`~object.__contains__`. For each of these methods, the key is the header name (treated case-insensitively), and the value is the first value associated with that header name. Setting a header deletes any existing values for that header, then adds a new value at the end of @@ -520,8 +521,10 @@ input, output, and error streams. want to subclass this instead of :class:`BaseCGIHandler`. This class is a subclass of :class:`BaseHandler`. It overrides the - :meth:`__init__`, :meth:`get_stdin`, :meth:`get_stderr`, :meth:`add_cgi_vars`, - :meth:`_write`, and :meth:`_flush` methods to support explicitly setting the + :meth:`!__init__`, :meth:`~BaseHandler.get_stdin`, + :meth:`~BaseHandler.get_stderr`, :meth:`~BaseHandler.add_cgi_vars`, + :meth:`~BaseHandler._write`, and :meth:`~BaseHandler._flush` methods to + support explicitly setting the environment and streams via the constructor. The supplied environment and streams are stored in the :attr:`stdin`, :attr:`stdout`, :attr:`stderr`, and :attr:`environ` attributes. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 146c4fd768233b..f7f23007fb0522 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -269,8 +269,9 @@ DateTime Objects Write the XML-RPC encoding of this :class:`DateTime` item to the *out* stream object. - It also supports certain of Python's built-in operators through rich comparison - and :meth:`__repr__` methods. + It also supports certain of Python's built-in operators through + :meth:`rich comparison ` and :meth:`~object.__repr__` + methods. A working example follows. The server code:: @@ -334,8 +335,8 @@ Binary Objects which was the de facto standard base64 specification when the XML-RPC spec was written. - It also supports certain of Python's built-in operators through :meth:`__eq__` - and :meth:`__ne__` methods. + It also supports certain of Python's built-in operators through + :meth:`~object.__eq__` and :meth:`~object.__ne__` methods. Example usage of the binary objects. We're going to transfer an image over XMLRPC:: From d07483292b115a5a0e9b9b09f3ec1000ce879986 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 15 Dec 2023 09:27:57 -0800 Subject: [PATCH 87/95] GH-112383: Fix test_loop_quicken when an executor is installed (GH-113153) --- Lib/test/test_dis.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 0c7fd60f640854..e4c7e9ba4649f7 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1210,8 +1210,13 @@ def test_loop_quicken(self): got = self.get_disassembly(loop_test, adaptive=True) expected = dis_loop_test_quickened_code if _testinternalcapi.get_optimizer(): - # We *may* see ENTER_EXECUTOR in the disassembly - got = got.replace("ENTER_EXECUTOR", "JUMP_BACKWARD ") + # We *may* see ENTER_EXECUTOR in the disassembly. This is a + # temporary hack to keep the test working until dis is able to + # handle the instruction correctly (GH-112383): + got = got.replace( + "ENTER_EXECUTOR 16", + "JUMP_BACKWARD 16 (to L1)", + ) self.do_disassembly_compare(got, expected) @cpython_only From 00d2b6d1fca91e1a83f7f99a370685b095ed4928 Mon Sep 17 00:00:00 2001 From: Akshat Khandelwal <35228810+akshatgokul@users.noreply.github.com> Date: Sat, 16 Dec 2023 02:23:16 +0530 Subject: [PATCH 88/95] gh-110746: Improve markup in ``tkinter.ttk.rst`` (#111236) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gh-110746: Improve markup in tkinter.ttk.rst * gh-110746: Improve markup in tkinter.ttk.rst * 📜🤖 Added by blurb_it. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Ezio Melotti --- Doc/library/tkinter.ttk.rst | 31 ++++++++++--------- ...-10-23-23-43-43.gh-issue-110746.yg77IE.rst | 1 + 2 files changed, 17 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 6e01ec7b291255..1609dc2ce9218e 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -986,19 +986,19 @@ ttk.Treeview The valid options/values are: - id + *id* Returns the column name. This is a read-only option. - anchor: One of the standard Tk anchor values. + *anchor*: One of the standard Tk anchor values. Specifies how the text in this column should be aligned with respect to the cell. - minwidth: width + *minwidth*: width The minimum width of the column in pixels. The treeview widget will not make the column any smaller than specified by this option when the widget is resized or the user drags a column. - stretch: ``True``/``False`` + *stretch*: ``True``/``False`` Specifies whether the column's width should be adjusted when the widget is resized. - width: width + *width*: width The width of the column in pixels. To configure the tree column, call this with column = "#0" @@ -1041,14 +1041,14 @@ ttk.Treeview The valid options/values are: - text: text + *text*: text The text to display in the column heading. - image: imageName + *image*: imageName Specifies an image to display to the right of the column heading. - anchor: anchor + *anchor*: anchor Specifies how the heading text should be aligned. One of the standard Tk anchor values. - command: callback + *command*: callback A callback to be invoked when the heading label is pressed. To configure the tree column heading, call this with column = "#0". @@ -1573,23 +1573,24 @@ Layouts A layout can be just ``None``, if it takes no options, or a dict of options specifying how to arrange the element. The layout mechanism uses a simplified version of the pack geometry manager: given an -initial cavity, each element is allocated a parcel. Valid -options/values are: +initial cavity, each element is allocated a parcel. -side: whichside +The valid options/values are: + +*side*: whichside Specifies which side of the cavity to place the element; one of top, right, bottom or left. If omitted, the element occupies the entire cavity. -sticky: nswe +*sticky*: nswe Specifies where the element is placed inside its allocated parcel. -unit: 0 or 1 +*unit*: 0 or 1 If set to 1, causes the element and all of its descendants to be treated as a single element for the purposes of :meth:`Widget.identify` et al. It's used for things like scrollbar thumbs with grips. -children: [sublayout... ] +*children*: [sublayout... ] Specifies a list of elements to place inside the element. Each element is a tuple (or other sequence type) where the first item is the layout name, and the other is a `Layout`_. diff --git a/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst b/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst new file mode 100644 index 00000000000000..215db7beb75dcf --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst @@ -0,0 +1 @@ +Improved markup for valid options/values for methods ttk.treeview.column and ttk.treeview.heading, and for Layouts. From 40574da0196c7e51e1886f6ff9ed26447622ae58 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 15 Dec 2023 18:03:44 -0600 Subject: [PATCH 89/95] Add reshape() recipe to demonstrate a use case for batched() and chained.from_iterable() (gh-113198) --- Doc/library/itertools.rst | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 03127afe1b4460..6bcda307f256f2 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1036,10 +1036,15 @@ The following recipes have a more mathematical flavor: # sum_of_squares([10, 20, 30]) -> 1400 return math.sumprod(*tee(it)) - def transpose(it): - "Swap the rows and columns of the input." + def reshape(matrix, cols): + "Reshape a 2-D matrix to have a given number of columns." + # reshape([(0, 1), (2, 3), (4, 5)], 3) --> (0, 1, 2), (3, 4, 5) + return batched(chain.from_iterable(matrix), cols) + + def transpose(matrix): + "Swap the rows and columns of a 2-D matrix." # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) - return zip(*it, strict=True) + return zip(*matrix, strict=True) def matmul(m1, m2): "Multiply two matrices." @@ -1254,6 +1259,22 @@ The following recipes have a more mathematical flavor: >>> sum_of_squares([10, 20, 30]) 1400 + >>> list(reshape([(0, 1), (2, 3), (4, 5)], 3)) + [(0, 1, 2), (3, 4, 5)] + >>> M = [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] + >>> list(reshape(M, 1)) + [(0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,)] + >>> list(reshape(M, 2)) + [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11)] + >>> list(reshape(M, 3)) + [(0, 1, 2), (3, 4, 5), (6, 7, 8), (9, 10, 11)] + >>> list(reshape(M, 4)) + [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] + >>> list(reshape(M, 6)) + [(0, 1, 2, 3, 4, 5), (6, 7, 8, 9, 10, 11)] + >>> list(reshape(M, 12)) + [(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)] + >>> list(transpose([(1, 2, 3), (11, 22, 33)])) [(1, 11), (2, 22), (3, 33)] From 5ae75e1be24bd6b031a68040cfddb71732461f67 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 15 Dec 2023 20:56:55 -0500 Subject: [PATCH 90/95] gh-111964: Add _PyRWMutex a "readers-writer" lock (gh-112859) This adds `_PyRWMutex`, a "readers-writer" lock, which wil be used to serialize global stop-the-world pauses with per-interpreter pauses. --- Include/internal/pycore_lock.h | 39 ++++++++++ Modules/_testinternalcapi/test_lock.c | 99 ++++++++++++++++++++++++ Python/lock.c | 106 ++++++++++++++++++++++++++ 3 files changed, 244 insertions(+) diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 03ad1c9fd584b5..18a8896d97a548 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -213,6 +213,45 @@ _PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) return _PyOnceFlag_CallOnceSlow(flag, fn, arg); } +// A readers-writer (RW) lock. The lock supports multiple concurrent readers or +// a single writer. The lock is write-preferring: if a writer is waiting while +// the lock is read-locked then, new readers will be blocked. This avoids +// starvation of writers. +// +// In C++, the equivalent synchronization primitive is std::shared_mutex +// with shared ("read") and exclusive ("write") locking. +// +// The two least significant bits are used to indicate if the lock is +// write-locked and if there are parked threads (either readers or writers) +// waiting to acquire the lock. The remaining bits are used to indicate the +// number of readers holding the lock. +// +// 0b000..00000: unlocked +// 0bnnn..nnn00: nnn..nnn readers holding the lock +// 0bnnn..nnn10: nnn..nnn readers holding the lock and a writer is waiting +// 0b00000..010: unlocked with awoken writer about to acquire lock +// 0b00000..001: write-locked +// 0b00000..011: write-locked and readers or other writers are waiting +// +// Note that reader_count must be zero if the lock is held by a writer, and +// vice versa. The lock can only be held by readers or a writer, but not both. +// +// The design is optimized for simplicity of the implementation. The lock is +// not fair: if fairness is desired, use an additional PyMutex to serialize +// writers. The lock is also not reentrant. +typedef struct { + uintptr_t bits; +} _PyRWMutex; + +// Read lock (i.e., shared lock) +PyAPI_FUNC(void) _PyRWMutex_RLock(_PyRWMutex *rwmutex); +PyAPI_FUNC(void) _PyRWMutex_RUnlock(_PyRWMutex *rwmutex); + +// Write lock (i.e., exclusive lock) +PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex); +PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); + + #ifdef __cplusplus } #endif diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 418f71c1441995..83081f73a72f64 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -372,6 +372,104 @@ test_lock_once(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +struct test_rwlock_data { + Py_ssize_t nthreads; + _PyRWMutex rw; + PyEvent step1; + PyEvent step2; + PyEvent step3; + PyEvent done; +}; + +static void +rdlock_thread(void *arg) +{ + struct test_rwlock_data *test_data = arg; + + // Acquire the lock in read mode + _PyRWMutex_RLock(&test_data->rw); + PyEvent_Wait(&test_data->step1); + _PyRWMutex_RUnlock(&test_data->rw); + + _PyRWMutex_RLock(&test_data->rw); + PyEvent_Wait(&test_data->step3); + _PyRWMutex_RUnlock(&test_data->rw); + + if (_Py_atomic_add_ssize(&test_data->nthreads, -1) == 1) { + _PyEvent_Notify(&test_data->done); + } +} +static void +wrlock_thread(void *arg) +{ + struct test_rwlock_data *test_data = arg; + + // First acquire the lock in write mode + _PyRWMutex_Lock(&test_data->rw); + PyEvent_Wait(&test_data->step2); + _PyRWMutex_Unlock(&test_data->rw); + + if (_Py_atomic_add_ssize(&test_data->nthreads, -1) == 1) { + _PyEvent_Notify(&test_data->done); + } +} + +static void +wait_until(uintptr_t *ptr, uintptr_t value) +{ + // wait up to two seconds for *ptr == value + int iters = 0; + uintptr_t bits; + do { + pysleep(10); + bits = _Py_atomic_load_uintptr(ptr); + iters++; + } while (bits != value && iters < 200); +} + +static PyObject * +test_lock_rwlock(PyObject *self, PyObject *obj) +{ + struct test_rwlock_data test_data = {.nthreads = 3}; + + _PyRWMutex_Lock(&test_data.rw); + assert(test_data.rw.bits == 1); + + _PyRWMutex_Unlock(&test_data.rw); + assert(test_data.rw.bits == 0); + + // Start two readers + PyThread_start_new_thread(rdlock_thread, &test_data); + PyThread_start_new_thread(rdlock_thread, &test_data); + + // wait up to two seconds for the threads to attempt to read-lock "rw" + wait_until(&test_data.rw.bits, 8); + assert(test_data.rw.bits == 8); + + // start writer (while readers hold lock) + PyThread_start_new_thread(wrlock_thread, &test_data); + wait_until(&test_data.rw.bits, 10); + assert(test_data.rw.bits == 10); + + // readers release lock, writer should acquire it + _PyEvent_Notify(&test_data.step1); + wait_until(&test_data.rw.bits, 3); + assert(test_data.rw.bits == 3); + + // writer releases lock, readers acquire it + _PyEvent_Notify(&test_data.step2); + wait_until(&test_data.rw.bits, 8); + assert(test_data.rw.bits == 8); + + // readers release lock again + _PyEvent_Notify(&test_data.step3); + wait_until(&test_data.rw.bits, 0); + assert(test_data.rw.bits == 0); + + PyEvent_Wait(&test_data.done); + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"test_lock_basic", test_lock_basic, METH_NOARGS}, {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, @@ -380,6 +478,7 @@ static PyMethodDef test_methods[] = { _TESTINTERNALCAPI_BENCHMARK_LOCKS_METHODDEF {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, {"test_lock_once", test_lock_once, METH_NOARGS}, + {"test_lock_rwlock", test_lock_rwlock, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/lock.c b/Python/lock.c index e9279f0b92a5e7..f0ff1176941da8 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -353,3 +353,109 @@ _PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) v = _Py_atomic_load_uint8(&flag->v); } } + +#define _Py_WRITE_LOCKED 1 +#define _PyRWMutex_READER_SHIFT 2 +#define _Py_RWMUTEX_MAX_READERS (UINTPTR_MAX >> _PyRWMutex_READER_SHIFT) + +static uintptr_t +rwmutex_set_parked_and_wait(_PyRWMutex *rwmutex, uintptr_t bits) +{ + // Set _Py_HAS_PARKED and wait until we are woken up. + if ((bits & _Py_HAS_PARKED) == 0) { + uintptr_t newval = bits | _Py_HAS_PARKED; + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, newval)) { + return bits; + } + bits = newval; + } + + _PyParkingLot_Park(&rwmutex->bits, &bits, sizeof(bits), -1, NULL, 1); + return _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); +} + +// The number of readers holding the lock +static uintptr_t +rwmutex_reader_count(uintptr_t bits) +{ + return bits >> _PyRWMutex_READER_SHIFT; +} + +void +_PyRWMutex_RLock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); + for (;;) { + if ((bits & _Py_WRITE_LOCKED)) { + // A writer already holds the lock. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + continue; + } + else if ((bits & _Py_HAS_PARKED)) { + // Reader(s) hold the lock (or just gave up the lock), but there is + // at least one waiting writer. We can't grab the lock because we + // don't want to starve the writer. Instead, we park ourselves and + // wait for the writer to eventually wake us up. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + continue; + } + else { + // The lock is unlocked or read-locked. Try to grab it. + assert(rwmutex_reader_count(bits) < _Py_RWMUTEX_MAX_READERS); + uintptr_t newval = bits + (1 << _PyRWMutex_READER_SHIFT); + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, newval)) { + continue; + } + return; + } + } +} + +void +_PyRWMutex_RUnlock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_add_uintptr(&rwmutex->bits, -(1 << _PyRWMutex_READER_SHIFT)); + assert(rwmutex_reader_count(bits) > 0 && "lock was not read-locked"); + bits -= (1 << _PyRWMutex_READER_SHIFT); + + if (rwmutex_reader_count(bits) == 0 && (bits & _Py_HAS_PARKED)) { + _PyParkingLot_UnparkAll(&rwmutex->bits); + return; + } +} + +void +_PyRWMutex_Lock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); + for (;;) { + // If there are no active readers and it's not already write-locked, + // then we can grab the lock. + if ((bits & ~_Py_HAS_PARKED) == 0) { + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, + bits | _Py_WRITE_LOCKED)) { + continue; + } + return; + } + + // Otherwise, we have to wait. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + } +} + +void +_PyRWMutex_Unlock(_PyRWMutex *rwmutex) +{ + uintptr_t old_bits = _Py_atomic_exchange_uintptr(&rwmutex->bits, 0); + + assert((old_bits & _Py_WRITE_LOCKED) && "lock was not write-locked"); + assert(rwmutex_reader_count(old_bits) == 0 && "lock was read-locked"); + + if ((old_bits & _Py_HAS_PARKED) != 0) { + _PyParkingLot_UnparkAll(&rwmutex->bits); + } +} From 84df3172efe8767ddf5c28bdb6696b3f216bcaa6 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 16 Dec 2023 03:12:39 -0500 Subject: [PATCH 91/95] gh-113046: Revise csv.reader doc (#113207) Clarify nature of csvfile. --- Doc/library/csv.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 4d52254e6d6db5..7a5589e68b3052 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -55,10 +55,11 @@ The :mod:`csv` module defines the following functions: .. function:: reader(csvfile, dialect='excel', **fmtparams) - Return a reader object which will iterate over lines in the given *csvfile*. - *csvfile* can be any object which supports the :term:`iterator` protocol and returns a - string each time its :meth:`!__next__` method is called --- :term:`file objects - ` and list objects are both suitable. If *csvfile* is a file object, + Return a :ref:`reader object ` that will process + lines from the given *csvfile*. A csvfile must be an iterable of + strings, each in the reader's defined csv format. + A csvfile is most commonly a file-like object or list. + If *csvfile* is a file object, it should be opened with ``newline=''``. [1]_ An optional *dialect* parameter can be given which is used to define a set of parameters specific to a particular CSV dialect. It may be an instance of a subclass of @@ -449,6 +450,8 @@ Dialects support the following attributes: When ``True``, raise exception :exc:`Error` on bad CSV input. The default is ``False``. +.. _reader-objects: + Reader Objects -------------- From fe479fb8a979894224a4d279d1e46a5cdb108fa4 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 16 Dec 2023 10:58:31 +0000 Subject: [PATCH 92/95] gh-67790: Support basic formatting for Fraction (#111320) PR #100161 added fancy float-style formatting for the Fraction type, but left us in a state where basic formatting for fractions (alignment, fill, minimum width, thousands separators) still wasn't supported. This PR adds that support. --------- Co-authored-by: Serhiy Storchaka --- Doc/library/fractions.rst | 37 ++++++-- Doc/whatsnew/3.13.rst | 8 ++ Lib/fractions.py | 87 +++++++++++++++---- Lib/test/test_fractions.py | 52 +++++++++-- ...3-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst | 2 + 5 files changed, 155 insertions(+), 31 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 509c63686f5a7f..887c3844d20faa 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -106,6 +106,10 @@ another rational number, or from a string. presentation types ``"e"``, ``"E"``, ``"f"``, ``"F"``, ``"g"``, ``"G"`` and ``"%""``. + .. versionchanged:: 3.13 + Formatting of :class:`Fraction` instances without a presentation type + now supports fill, alignment, sign handling, minimum width and grouping. + .. attribute:: numerator Numerator of the Fraction in lowest term. @@ -201,17 +205,36 @@ another rational number, or from a string. .. method:: __format__(format_spec, /) - Provides support for float-style formatting of :class:`Fraction` - instances via the :meth:`str.format` method, the :func:`format` built-in - function, or :ref:`Formatted string literals `. The - presentation types ``"e"``, ``"E"``, ``"f"``, ``"F"``, ``"g"``, ``"G"`` - and ``"%"`` are supported. For these presentation types, formatting for a - :class:`Fraction` object ``x`` follows the rules outlined for - the :class:`float` type in the :ref:`formatspec` section. + Provides support for formatting of :class:`Fraction` instances via the + :meth:`str.format` method, the :func:`format` built-in function, or + :ref:`Formatted string literals `. + + If the ``format_spec`` format specification string does not end with one + of the presentation types ``'e'``, ``'E'``, ``'f'``, ``'F'``, ``'g'``, + ``'G'`` or ``'%'`` then formatting follows the general rules for fill, + alignment, sign handling, minimum width, and grouping as described in the + :ref:`format specification mini-language `. The "alternate + form" flag ``'#'`` is supported: if present, it forces the output string + to always include an explicit denominator, even when the value being + formatted is an exact integer. The zero-fill flag ``'0'`` is not + supported. + + If the ``format_spec`` format specification string ends with one of + the presentation types ``'e'``, ``'E'``, ``'f'``, ``'F'``, ``'g'``, + ``'G'`` or ``'%'`` then formatting follows the rules outlined for the + :class:`float` type in the :ref:`formatspec` section. Here are some examples:: >>> from fractions import Fraction + >>> format(Fraction(103993, 33102), '_') + '103_993/33_102' + >>> format(Fraction(1, 7), '.^+10') + '...+1/7...' + >>> format(Fraction(3, 1), '') + '3' + >>> format(Fraction(3, 1), '#') + '3/1' >>> format(Fraction(1, 7), '.40g') '0.1428571428571428571428571428571428571429' >>> format(Fraction('1234567.855'), '_.2f') diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 4f9643967d20cf..ce4f66b97a0be4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -212,6 +212,14 @@ email (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the CVE-2023-27043 fix.) +fractions +--------- + +* Formatting for objects of type :class:`fractions.Fraction` now supports + the standard format specification mini-language rules for fill, alignment, + sign handling, minimum width and grouping. (Contributed by Mark Dickinson + in :gh:`111320`) + glob ---- diff --git a/Lib/fractions.py b/Lib/fractions.py index c95db0730e5b6d..6532d5d54e3c35 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -139,6 +139,23 @@ def _round_to_figures(n, d, figures): return sign, significand, exponent +# Pattern for matching non-float-style format specifications. +_GENERAL_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + # Alt flag forces a slash and denominator in the output, even for + # integer-valued Fraction objects. + (?P\#)? + # We don't implement the zeropad flag since there's no single obvious way + # to interpret it. + (?P0|[1-9][0-9]*)? + (?P[,_])? +""", re.DOTALL | re.VERBOSE).fullmatch + + # Pattern for matching float-style format specifications; # supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types. _FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" @@ -414,27 +431,42 @@ def __str__(self): else: return '%s/%s' % (self._numerator, self._denominator) - def __format__(self, format_spec, /): - """Format this fraction according to the given format specification.""" - - # Backwards compatiblility with existing formatting. - if not format_spec: - return str(self) + def _format_general(self, match): + """Helper method for __format__. + Handles fill, alignment, signs, and thousands separators in the + case of no presentation type. + """ # Validate and parse the format specifier. - match = _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec) - if match is None: - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}" - ) - elif match["align"] is not None and match["zeropad"] is not None: - # Avoid the temptation to guess. - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}; " - "can't use explicit alignment when zero-padding" - ) + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + alternate_form = bool(match["alt"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] or '' + + # Determine the body and sign representation. + n, d = self._numerator, self._denominator + if d > 1 or alternate_form: + body = f"{abs(n):{thousands_sep}}/{d:{thousands_sep}}" + else: + body = f"{abs(n):{thousands_sep}}" + sign = '-' if n < 0 else pos_sign + + # Pad with fill character if necessary and return. + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body + + def _format_float_style(self, match): + """Helper method for __format__; handles float presentation types.""" fill = match["fill"] or " " align = match["align"] or ">" pos_sign = "" if match["sign"] == "-" else match["sign"] @@ -530,6 +562,23 @@ def __format__(self, format_spec, /): else: # align == "=" return sign + padding + body + def __format__(self, format_spec, /): + """Format this fraction according to the given format specification.""" + + if match := _GENERAL_FORMAT_SPECIFICATION_MATCHER(format_spec): + return self._format_general(match) + + if match := _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec): + # Refuse the temptation to guess if both alignment _and_ + # zero padding are specified. + if match["align"] is None or match["zeropad"] is None: + return self._format_float_style(match) + + raise ValueError( + f"Invalid format specifier {format_spec!r} " + f"for object of type {type(self).__name__!r}" + ) + def _operator_fallbacks(monomorphic_operator, fallback_operator): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 499e3b6e656faa..84779526ce0eb0 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -849,12 +849,50 @@ def denominator(self): self.assertEqual(type(f.denominator), myint) def test_format_no_presentation_type(self): - # Triples (fraction, specification, expected_result) + # Triples (fraction, specification, expected_result). testcases = [ - (F(1, 3), '', '1/3'), - (F(-1, 3), '', '-1/3'), - (F(3), '', '3'), - (F(-3), '', '-3'), + # Explicit sign handling + (F(2, 3), '+', '+2/3'), + (F(-2, 3), '+', '-2/3'), + (F(3), '+', '+3'), + (F(-3), '+', '-3'), + (F(2, 3), ' ', ' 2/3'), + (F(-2, 3), ' ', '-2/3'), + (F(3), ' ', ' 3'), + (F(-3), ' ', '-3'), + (F(2, 3), '-', '2/3'), + (F(-2, 3), '-', '-2/3'), + (F(3), '-', '3'), + (F(-3), '-', '-3'), + # Padding + (F(0), '5', ' 0'), + (F(2, 3), '5', ' 2/3'), + (F(-2, 3), '5', ' -2/3'), + (F(2, 3), '0', '2/3'), + (F(2, 3), '1', '2/3'), + (F(2, 3), '2', '2/3'), + # Alignment + (F(2, 3), '<5', '2/3 '), + (F(2, 3), '>5', ' 2/3'), + (F(2, 3), '^5', ' 2/3 '), + (F(2, 3), '=5', ' 2/3'), + (F(-2, 3), '<5', '-2/3 '), + (F(-2, 3), '>5', ' -2/3'), + (F(-2, 3), '^5', '-2/3 '), + (F(-2, 3), '=5', '- 2/3'), + # Fill + (F(2, 3), 'X>5', 'XX2/3'), + (F(-2, 3), '.<5', '-2/3.'), + (F(-2, 3), '\n^6', '\n-2/3\n'), + # Thousands separators + (F(1234, 5679), ',', '1,234/5,679'), + (F(-1234, 5679), '_', '-1_234/5_679'), + (F(1234567), '_', '1_234_567'), + (F(-1234567), ',', '-1,234,567'), + # Alternate form forces a slash in the output + (F(123), '#', '123/1'), + (F(-123), '#', '-123/1'), + (F(0), '#', '0/1'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1218,6 +1256,10 @@ def test_invalid_formats(self): '.%', # Z instead of z for negative zero suppression 'Z.2f' + # z flag not supported for general formatting + 'z', + # zero padding not supported for general formatting + '05', ] for spec in invalid_specs: with self.subTest(spec=spec): diff --git a/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst b/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst new file mode 100644 index 00000000000000..44c5702a6551b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst @@ -0,0 +1,2 @@ +Implement basic formatting support (minimum width, alignment, fill) for +:class:`fractions.Fraction`. From 1583c40be938d2caf363c126976bc8757df90b13 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 16 Dec 2023 09:13:50 -0600 Subject: [PATCH 93/95] gh-113202: Add a strict option to itertools.batched() (gh-113203) --- Doc/library/itertools.rst | 18 +++++++++-- Lib/test/test_itertools.py | 4 +++ ...-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst | 1 + Modules/clinic/itertoolsmodule.c.h | 32 +++++++++++++------ Modules/itertoolsmodule.c | 29 ++++++++++------- 5 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 6bcda307f256f2..c016fb76bfd0a0 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -164,11 +164,14 @@ loops that truncate the stream. Added the optional *initial* parameter. -.. function:: batched(iterable, n) +.. function:: batched(iterable, n, *, strict=False) Batch data from the *iterable* into tuples of length *n*. The last batch may be shorter than *n*. + If *strict* is true, will raise a :exc:`ValueError` if the final + batch is shorter than *n*. + Loops over the input iterable and accumulates data into tuples up to size *n*. The input is consumed lazily, just enough to fill a batch. The result is yielded as soon as the batch is full or when the input @@ -190,16 +193,21 @@ loops that truncate the stream. Roughly equivalent to:: - def batched(iterable, n): + def batched(iterable, n, *, strict=False): # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: raise ValueError('n must be at least one') it = iter(iterable) while batch := tuple(islice(it, n)): + if strict and len(batch) != n: + raise ValueError('batched(): incomplete batch') yield batch .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *strict* option. + .. function:: chain(*iterables) @@ -1039,7 +1047,7 @@ The following recipes have a more mathematical flavor: def reshape(matrix, cols): "Reshape a 2-D matrix to have a given number of columns." # reshape([(0, 1), (2, 3), (4, 5)], 3) --> (0, 1, 2), (3, 4, 5) - return batched(chain.from_iterable(matrix), cols) + return batched(chain.from_iterable(matrix), cols, strict=True) def transpose(matrix): "Swap the rows and columns of a 2-D matrix." @@ -1270,6 +1278,10 @@ The following recipes have a more mathematical flavor: [(0, 1, 2), (3, 4, 5), (6, 7, 8), (9, 10, 11)] >>> list(reshape(M, 4)) [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] + >>> list(reshape(M, 5)) + Traceback (most recent call last): + ... + ValueError: batched(): incomplete batch >>> list(reshape(M, 6)) [(0, 1, 2, 3, 4, 5), (6, 7, 8, 9, 10, 11)] >>> list(reshape(M, 12)) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 705e880d98685e..9af0730ea98004 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -187,7 +187,11 @@ def test_batched(self): [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 1)), [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)]) + self.assertEqual(list(batched('ABCDEF', 2, strict=True)), + [('A', 'B'), ('C', 'D'), ('E', 'F')]) + with self.assertRaises(ValueError): # Incomplete batch when strict + list(batched('ABCDEFG', 3, strict=True)) with self.assertRaises(TypeError): # Too few arguments list(batched('ABCDEFG')) with self.assertRaises(TypeError): diff --git a/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst b/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst new file mode 100644 index 00000000000000..44f26aef60a33a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst @@ -0,0 +1 @@ +Add a ``strict`` option to ``batched()`` in the ``itertools`` module. diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index fa2c5e0e922387..3ec479943a83d4 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -10,7 +10,7 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(batched_new__doc__, -"batched(iterable, n)\n" +"batched(iterable, n, *, strict=False)\n" "--\n" "\n" "Batch data into tuples of length n. The last batch may be shorter than n.\n" @@ -25,10 +25,14 @@ PyDoc_STRVAR(batched_new__doc__, " ...\n" " (\'A\', \'B\', \'C\')\n" " (\'D\', \'E\', \'F\')\n" -" (\'G\',)"); +" (\'G\',)\n" +"\n" +"If \"strict\" is True, raises a ValueError if the final batch is shorter\n" +"than n."); static PyObject * -batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n); +batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, + int strict); static PyObject * batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -36,14 +40,14 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(n), }, + .ob_item = { &_Py_ID(iterable), &_Py_ID(n), &_Py_ID(strict), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -52,18 +56,20 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"iterable", "n", NULL}; + static const char * const _keywords[] = {"iterable", "n", "strict", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "batched", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; PyObject *iterable; Py_ssize_t n; + int strict = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); if (!fastargs) { @@ -82,7 +88,15 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } n = ival; } - return_value = batched_new_impl(type, iterable, n); + if (!noptargs) { + goto skip_optional_kwonly; + } + strict = PyObject_IsTrue(fastargs[2]); + if (strict < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = batched_new_impl(type, iterable, n, strict); exit: return return_value; @@ -914,4 +928,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=782fe7e30733779b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6a515f765da86b5 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ab99fa4d873bf5..164741495c7baf 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -105,20 +105,11 @@ class itertools.pairwise "pairwiseobject *" "clinic_state()->pairwise_type" /* batched object ************************************************************/ -/* Note: The built-in zip() function includes a "strict" argument - that was needed because that function would silently truncate data, - and there was no easy way for a user to detect the data loss. - The same reasoning does not apply to batched() which never drops data. - Instead, batched() produces a shorter tuple which can be handled - as the user sees fit. If requested, it would be reasonable to add - "fillvalue" support which had demonstrated value in zip_longest(). - For now, the API is kept simple and clean. - */ - typedef struct { PyObject_HEAD PyObject *it; Py_ssize_t batch_size; + bool strict; } batchedobject; /*[clinic input] @@ -126,6 +117,9 @@ typedef struct { itertools.batched.__new__ as batched_new iterable: object n: Py_ssize_t + * + strict: bool = False + Batch data into tuples of length n. The last batch may be shorter than n. Loops over the input iterable and accumulates data into tuples @@ -140,11 +134,15 @@ or when the input iterable is exhausted. ('D', 'E', 'F') ('G',) +If "strict" is True, raises a ValueError if the final batch is shorter +than n. + [clinic start generated code]*/ static PyObject * -batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) -/*[clinic end generated code: output=7ebc954d655371b6 input=ffd70726927c5129]*/ +batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, + int strict) +/*[clinic end generated code: output=c6de11b061529d3e input=7814b47e222f5467]*/ { PyObject *it; batchedobject *bo; @@ -170,6 +168,7 @@ batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) } bo->batch_size = n; bo->it = it; + bo->strict = (bool) strict; return (PyObject *)bo; } @@ -233,6 +232,12 @@ batched_next(batchedobject *bo) Py_DECREF(result); return NULL; } + if (bo->strict) { + Py_CLEAR(bo->it); + Py_DECREF(result); + PyErr_SetString(PyExc_ValueError, "batched(): incomplete batch"); + return NULL; + } _PyTuple_Resize(&result, i); return result; } From d91e43ed7839b601cbeadd137d89e69e2cc3efda Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 16 Dec 2023 19:04:33 +0000 Subject: [PATCH 94/95] GH-110109: Move tests for pathlib ABCs to new module. (#112904) --- Lib/test/test_pathlib/__init__.py | 5 + Lib/test/{ => test_pathlib}/test_pathlib.py | 1896 +----------------- Lib/test/test_pathlib/test_pathlib_abc.py | 1918 +++++++++++++++++++ Makefile.pre.in | 1 + 4 files changed, 1927 insertions(+), 1893 deletions(-) create mode 100644 Lib/test/test_pathlib/__init__.py rename Lib/test/{ => test_pathlib}/test_pathlib.py (50%) create mode 100644 Lib/test/test_pathlib/test_pathlib_abc.py diff --git a/Lib/test/test_pathlib/__init__.py b/Lib/test/test_pathlib/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_pathlib/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py similarity index 50% rename from Lib/test/test_pathlib.py rename to Lib/test/test_pathlib/test_pathlib.py index a1acba406ea37c..304786fc6f7fd9 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1,11 +1,9 @@ -import collections.abc import io import os import sys import errno import pathlib import pickle -import posixpath import socket import stat import tempfile @@ -14,10 +12,10 @@ from urllib.request import pathname2url from test.support import import_helper -from test.support import set_recursion_limit from test.support import is_emscripten, is_wasi from test.support import os_helper from test.support.os_helper import TESTFN, FakePath +from test.test_pathlib import test_pathlib_abc try: import grp, pwd @@ -25,12 +23,6 @@ grp = pwd = None -class UnsupportedOperationTest(unittest.TestCase): - def test_is_notimplemented(self): - self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) - - # Make sure any symbolic links in the base test path are resolved. BASE = os.path.realpath(TESTFN) join = lambda *x: os.path.join(BASE, *x) @@ -49,654 +41,7 @@ def test_is_notimplemented(self): # Tests for the pure classes. # - -class PurePathBaseTest(unittest.TestCase): - cls = pathlib._abc.PurePathBase - - def test_magic_methods(self): - P = self.cls - self.assertFalse(hasattr(P, '__fspath__')) - self.assertFalse(hasattr(P, '__bytes__')) - self.assertIs(P.__reduce__, object.__reduce__) - self.assertIs(P.__hash__, object.__hash__) - self.assertIs(P.__eq__, object.__eq__) - self.assertIs(P.__lt__, object.__lt__) - self.assertIs(P.__le__, object.__le__) - self.assertIs(P.__gt__, object.__gt__) - self.assertIs(P.__ge__, object.__ge__) - - -class DummyPurePath(pathlib._abc.PurePathBase): - def __eq__(self, other): - if not isinstance(other, DummyPurePath): - return NotImplemented - return str(self) == str(other) - - def __hash__(self): - return hash(str(self)) - - -class DummyPurePathTest(unittest.TestCase): - cls = DummyPurePath - - # Keys are canonical paths, values are list of tuples of arguments - # supposed to produce equal paths. - equivalences = { - 'a/b': [ - ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), - ('a/b/',), ('a//b',), ('a//b//',), - # Empty components get removed. - ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), - ], - '/b/c/d': [ - ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), - # Empty components get removed. - ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), - ], - } - - def setUp(self): - p = self.cls('a') - self.pathmod = p.pathmod - self.sep = self.pathmod.sep - self.altsep = self.pathmod.altsep - - def test_constructor_common(self): - P = self.cls - p = P('a') - self.assertIsInstance(p, P) - P('a', 'b', 'c') - P('/a', 'b', 'c') - P('a/b/c') - P('/a/b/c') - - def test_concrete_class(self): - if self.cls is pathlib.PurePath: - expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath - else: - expected = self.cls - p = self.cls('a') - self.assertIs(type(p), expected) - - def test_different_pathmods_unequal(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') - self.assertNotEqual(p, q) - - def test_different_pathmods_unordered(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') - with self.assertRaises(TypeError): - p < q - with self.assertRaises(TypeError): - p <= q - with self.assertRaises(TypeError): - p > q - with self.assertRaises(TypeError): - p >= q - - def _check_str_subclass(self, *args): - # Issue #21127: it should be possible to construct a PurePath object - # from a str subclass instance, and it then gets converted to - # a pure str object. - class StrSubclass(str): - pass - P = self.cls - p = P(*(StrSubclass(x) for x in args)) - self.assertEqual(p, P(*args)) - for part in p.parts: - self.assertIs(type(part), str) - - def test_str_subclass_common(self): - self._check_str_subclass('') - self._check_str_subclass('.') - self._check_str_subclass('a') - self._check_str_subclass('a/b.txt') - self._check_str_subclass('/a/b.txt') - - def test_with_segments_common(self): - class P(self.cls): - def __init__(self, *pathsegments, session_id): - super().__init__(*pathsegments) - self.session_id = session_id - - def with_segments(self, *pathsegments): - return type(self)(*pathsegments, session_id=self.session_id) - p = P('foo', 'bar', session_id=42) - self.assertEqual(42, (p / 'foo').session_id) - self.assertEqual(42, ('foo' / p).session_id) - self.assertEqual(42, p.joinpath('foo').session_id) - self.assertEqual(42, p.with_name('foo').session_id) - self.assertEqual(42, p.with_stem('foo').session_id) - self.assertEqual(42, p.with_suffix('.foo').session_id) - self.assertEqual(42, p.with_segments('foo').session_id) - self.assertEqual(42, p.relative_to('foo').session_id) - self.assertEqual(42, p.parent.session_id) - for parent in p.parents: - self.assertEqual(42, parent.session_id) - - def _check_parse_path(self, raw_path, *expected): - sep = self.pathmod.sep - actual = self.cls._parse_path(raw_path.replace('/', sep)) - self.assertEqual(actual, expected) - if altsep := self.pathmod.altsep: - actual = self.cls._parse_path(raw_path.replace('/', altsep)) - self.assertEqual(actual, expected) - - def test_parse_path_common(self): - check = self._check_parse_path - sep = self.pathmod.sep - check('', '', '', []) - check('a', '', '', ['a']) - check('a/', '', '', ['a']) - check('a/b', '', '', ['a', 'b']) - check('a/b/', '', '', ['a', 'b']) - check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) - check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) - check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) - check('.', '', '', []) - check('././b', '', '', ['b']) - check('a/./b', '', '', ['a', 'b']) - check('a/./.', '', '', ['a']) - check('/a/b', '', sep, ['a', 'b']) - - def test_join_common(self): - P = self.cls - p = P('a/b') - pp = p.joinpath('c') - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p.joinpath('c', 'd') - self.assertEqual(pp, P('a/b/c/d')) - pp = p.joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div_common(self): - # Basically the same as joinpath(). - P = self.cls - p = P('a/b') - pp = p / 'c' - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p / 'c/d' - self.assertEqual(pp, P('a/b/c/d')) - pp = p / 'c' / 'd' - self.assertEqual(pp, P('a/b/c/d')) - pp = 'c' / p / 'd' - self.assertEqual(pp, P('c/a/b/d')) - pp = p/ '/c' - self.assertEqual(pp, P('/c')) - - def _check_str(self, expected, args): - p = self.cls(*args) - self.assertEqual(str(p), expected.replace('/', self.sep)) - - def test_str_common(self): - # Canonicalized paths roundtrip. - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self._check_str(pathstr, (pathstr,)) - # Special case for the empty path. - self._check_str('.', ('',)) - # Other tests for str() are in test_equivalences(). - - def test_as_posix_common(self): - P = self.cls - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self.assertEqual(P(pathstr).as_posix(), pathstr) - # Other tests for as_posix() are in test_equivalences(). - - def test_repr_common(self): - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - with self.subTest(pathstr=pathstr): - p = self.cls(pathstr) - clsname = p.__class__.__name__ - r = repr(p) - # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) - inner = r[len(clsname) + 1 : -1] - self.assertEqual(eval(inner), p.as_posix()) - - def test_eq_common(self): - P = self.cls - self.assertEqual(P('a/b'), P('a/b')) - self.assertEqual(P('a/b'), P('a', 'b')) - self.assertNotEqual(P('a/b'), P('a')) - self.assertNotEqual(P('a/b'), P('/a/b')) - self.assertNotEqual(P('a/b'), P()) - self.assertNotEqual(P('/a/b'), P('/')) - self.assertNotEqual(P(), P('/')) - self.assertNotEqual(P(), "") - self.assertNotEqual(P(), {}) - self.assertNotEqual(P(), int) - - def test_match_common(self): - P = self.cls - self.assertRaises(ValueError, P('a').match, '') - self.assertRaises(ValueError, P('a').match, '.') - # Simple relative pattern. - self.assertTrue(P('b.py').match('b.py')) - self.assertTrue(P('a/b.py').match('b.py')) - self.assertTrue(P('/a/b.py').match('b.py')) - self.assertFalse(P('a.py').match('b.py')) - self.assertFalse(P('b/py').match('b.py')) - self.assertFalse(P('/a.py').match('b.py')) - self.assertFalse(P('b.py/c').match('b.py')) - # Wildcard relative pattern. - self.assertTrue(P('b.py').match('*.py')) - self.assertTrue(P('a/b.py').match('*.py')) - self.assertTrue(P('/a/b.py').match('*.py')) - self.assertFalse(P('b.pyc').match('*.py')) - self.assertFalse(P('b./py').match('*.py')) - self.assertFalse(P('b.py/c').match('*.py')) - # Multi-part relative pattern. - self.assertTrue(P('ab/c.py').match('a*/*.py')) - self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) - self.assertFalse(P('a.py').match('a*/*.py')) - self.assertFalse(P('/dab/c.py').match('a*/*.py')) - self.assertFalse(P('ab/c.py/d').match('a*/*.py')) - # Absolute pattern. - self.assertTrue(P('/b.py').match('/*.py')) - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('a/b.py').match('/*.py')) - self.assertFalse(P('/a/b.py').match('/*.py')) - # Multi-part absolute pattern. - self.assertTrue(P('/a/b.py').match('/a/*.py')) - self.assertFalse(P('/ab.py').match('/a/*.py')) - self.assertFalse(P('/a/b/c.py').match('/a/*.py')) - # Multi-part glob-style pattern. - self.assertTrue(P('a').match('**')) - self.assertTrue(P('c.py').match('**')) - self.assertTrue(P('a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('/**')) - self.assertTrue(P('/a/b/c.py').match('**/')) - self.assertTrue(P('/a/b/c.py').match('/a/**')) - self.assertTrue(P('/a/b/c.py').match('**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/b/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/**/**/**/*.py')) - self.assertFalse(P('c.py').match('**/a.py')) - self.assertFalse(P('c.py').match('c/**')) - self.assertFalse(P('a/b/c.py').match('**/a')) - self.assertFalse(P('a/b/c.py').match('**/a/b')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c.')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('/a/b/c.py/**')) - self.assertFalse(P('a/b/c.py').match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').match, '**a/b/c') - self.assertRaises(ValueError, P('a').match, 'a/b/c**') - # Case-sensitive flag - self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) - self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) - self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) - self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) - # Matching against empty path - self.assertFalse(P().match('*')) - self.assertTrue(P().match('**')) - self.assertFalse(P().match('**/*')) - - def test_parts_common(self): - # `parts` returns a tuple. - sep = self.sep - P = self.cls - p = P('a/b') - parts = p.parts - self.assertEqual(parts, ('a', 'b')) - # When the path is absolute, the anchor is a separate part. - p = P('/a/b') - parts = p.parts - self.assertEqual(parts, (sep, 'a', 'b')) - - def test_equivalences(self): - for k, tuples in self.equivalences.items(): - canon = k.replace('/', self.sep) - posix = k.replace(self.sep, '/') - if canon != posix: - tuples = tuples + [ - tuple(part.replace('/', self.sep) for part in t) - for t in tuples - ] - tuples.append((posix, )) - pcanon = self.cls(canon) - for t in tuples: - p = self.cls(*t) - self.assertEqual(p, pcanon, "failed with args {}".format(t)) - self.assertEqual(hash(p), hash(pcanon)) - self.assertEqual(str(p), canon) - self.assertEqual(p.as_posix(), posix) - - def test_parent_common(self): - # Relative - P = self.cls - p = P('a/b/c') - self.assertEqual(p.parent, P('a/b')) - self.assertEqual(p.parent.parent, P('a')) - self.assertEqual(p.parent.parent.parent, P()) - self.assertEqual(p.parent.parent.parent.parent, P()) - # Anchored - p = P('/a/b/c') - self.assertEqual(p.parent, P('/a/b')) - self.assertEqual(p.parent.parent, P('/a')) - self.assertEqual(p.parent.parent.parent, P('/')) - self.assertEqual(p.parent.parent.parent.parent, P('/')) - - def test_parents_common(self): - # Relative - P = self.cls - p = P('a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('a/b')) - self.assertEqual(par[1], P('a')) - self.assertEqual(par[2], P('.')) - self.assertEqual(par[-1], P('.')) - self.assertEqual(par[-2], P('a')) - self.assertEqual(par[-3], P('a/b')) - self.assertEqual(par[0:1], (P('a/b'),)) - self.assertEqual(par[:2], (P('a/b'), P('a'))) - self.assertEqual(par[:-1], (P('a/b'), P('a'))) - self.assertEqual(par[1:], (P('a'), P('.'))) - self.assertEqual(par[::2], (P('a/b'), P('.'))) - self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) - self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] - with self.assertRaises(TypeError): - par[0] = p - # Anchored - p = P('/a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('/a/b')) - self.assertEqual(par[1], P('/a')) - self.assertEqual(par[2], P('/')) - self.assertEqual(par[-1], P('/')) - self.assertEqual(par[-2], P('/a')) - self.assertEqual(par[-3], P('/a/b')) - self.assertEqual(par[0:1], (P('/a/b'),)) - self.assertEqual(par[:2], (P('/a/b'), P('/a'))) - self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) - self.assertEqual(par[1:], (P('/a'), P('/'))) - self.assertEqual(par[::2], (P('/a/b'), P('/'))) - self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) - self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] - - def test_drive_common(self): - P = self.cls - self.assertEqual(P('a/b').drive, '') - self.assertEqual(P('/a/b').drive, '') - self.assertEqual(P('').drive, '') - - def test_root_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').root, '') - self.assertEqual(P('a/b').root, '') - self.assertEqual(P('/').root, sep) - self.assertEqual(P('/a/b').root, sep) - - def test_anchor_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').anchor, '') - self.assertEqual(P('a/b').anchor, '') - self.assertEqual(P('/').anchor, sep) - self.assertEqual(P('/a/b').anchor, sep) - - def test_name_common(self): - P = self.cls - self.assertEqual(P('').name, '') - self.assertEqual(P('.').name, '') - self.assertEqual(P('/').name, '') - self.assertEqual(P('a/b').name, 'b') - self.assertEqual(P('/a/b').name, 'b') - self.assertEqual(P('/a/b/.').name, 'b') - self.assertEqual(P('a/b.py').name, 'b.py') - self.assertEqual(P('/a/b.py').name, 'b.py') - - def test_suffix_common(self): - P = self.cls - self.assertEqual(P('').suffix, '') - self.assertEqual(P('.').suffix, '') - self.assertEqual(P('..').suffix, '') - self.assertEqual(P('/').suffix, '') - self.assertEqual(P('a/b').suffix, '') - self.assertEqual(P('/a/b').suffix, '') - self.assertEqual(P('/a/b/.').suffix, '') - self.assertEqual(P('a/b.py').suffix, '.py') - self.assertEqual(P('/a/b.py').suffix, '.py') - self.assertEqual(P('a/.hgrc').suffix, '') - self.assertEqual(P('/a/.hgrc').suffix, '') - self.assertEqual(P('a/.hg.rc').suffix, '.rc') - self.assertEqual(P('/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') - - def test_suffixes_common(self): - P = self.cls - self.assertEqual(P('').suffixes, []) - self.assertEqual(P('.').suffixes, []) - self.assertEqual(P('/').suffixes, []) - self.assertEqual(P('a/b').suffixes, []) - self.assertEqual(P('/a/b').suffixes, []) - self.assertEqual(P('/a/b/.').suffixes, []) - self.assertEqual(P('a/b.py').suffixes, ['.py']) - self.assertEqual(P('/a/b.py').suffixes, ['.py']) - self.assertEqual(P('a/.hgrc').suffixes, []) - self.assertEqual(P('/a/.hgrc').suffixes, []) - self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem_common(self): - P = self.cls - self.assertEqual(P('').stem, '') - self.assertEqual(P('.').stem, '') - self.assertEqual(P('..').stem, '..') - self.assertEqual(P('/').stem, '') - self.assertEqual(P('a/b').stem, 'b') - self.assertEqual(P('a/b.py').stem, 'b') - self.assertEqual(P('a/.hgrc').stem, '.hgrc') - self.assertEqual(P('a/.hg.rc').stem, '.hg') - self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name_common(self): - P = self.cls - self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) - self.assertRaises(ValueError, P('').with_name, 'd.xml') - self.assertRaises(ValueError, P('.').with_name, 'd.xml') - self.assertRaises(ValueError, P('/').with_name, 'd.xml') - self.assertRaises(ValueError, P('a/b').with_name, '') - self.assertRaises(ValueError, P('a/b').with_name, '.') - self.assertRaises(ValueError, P('a/b').with_name, '/c') - self.assertRaises(ValueError, P('a/b').with_name, 'c/') - self.assertRaises(ValueError, P('a/b').with_name, 'c/d') - - def test_with_stem_common(self): - P = self.cls - self.assertEqual(P('a/b').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) - self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) - self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) - self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) - self.assertRaises(ValueError, P('').with_stem, 'd') - self.assertRaises(ValueError, P('.').with_stem, 'd') - self.assertRaises(ValueError, P('/').with_stem, 'd') - self.assertRaises(ValueError, P('a/b').with_stem, '') - self.assertRaises(ValueError, P('a/b').with_stem, '.') - self.assertRaises(ValueError, P('a/b').with_stem, '/c') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') - - def test_with_suffix_common(self): - P = self.cls - self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) - self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) - # Stripping suffix. - self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) - self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') - self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') - self.assertRaises(ValueError, P('a/b').with_suffix, './.d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') - - def test_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P()), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P()) - self.assertEqual(p.relative_to('a/b'), P()) - self.assertEqual(p.relative_to(P(), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P()) - self.assertEqual(p.relative_to('/a/b'), P()) - self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - - def test_is_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P())) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) - - -class PurePathTest(DummyPurePathTest): +class PurePathTest(test_pathlib_abc.DummyPurePathTest): cls = pathlib.PurePath def test_constructor_nested(self): @@ -1581,1246 +926,11 @@ class cls(pathlib.PurePath): test_repr_roundtrips = None -# -# Tests for the virtual classes. -# - -class PathBaseTest(PurePathBaseTest): - cls = pathlib._abc.PathBase - - def test_unsupported_operation(self): - P = self.cls - p = self.cls() - e = pathlib.UnsupportedOperation - self.assertRaises(e, p.stat) - self.assertRaises(e, p.lstat) - self.assertRaises(e, p.exists) - self.assertRaises(e, p.samefile, 'foo') - self.assertRaises(e, p.is_dir) - self.assertRaises(e, p.is_file) - self.assertRaises(e, p.is_mount) - self.assertRaises(e, p.is_symlink) - self.assertRaises(e, p.is_block_device) - self.assertRaises(e, p.is_char_device) - self.assertRaises(e, p.is_fifo) - self.assertRaises(e, p.is_socket) - self.assertRaises(e, p.open) - self.assertRaises(e, p.read_bytes) - self.assertRaises(e, p.read_text) - self.assertRaises(e, p.write_bytes, b'foo') - self.assertRaises(e, p.write_text, 'foo') - self.assertRaises(e, p.iterdir) - self.assertRaises(e, p.glob, '*') - self.assertRaises(e, p.rglob, '*') - self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.absolute) - self.assertRaises(e, P.cwd) - self.assertRaises(e, p.expanduser) - self.assertRaises(e, p.home) - self.assertRaises(e, p.readlink) - self.assertRaises(e, p.symlink_to, 'foo') - self.assertRaises(e, p.hardlink_to, 'foo') - self.assertRaises(e, p.mkdir) - self.assertRaises(e, p.touch) - self.assertRaises(e, p.rename, 'foo') - self.assertRaises(e, p.replace, 'foo') - self.assertRaises(e, p.chmod, 0o755) - self.assertRaises(e, p.lchmod, 0o755) - self.assertRaises(e, p.unlink) - self.assertRaises(e, p.rmdir) - self.assertRaises(e, p.owner) - self.assertRaises(e, p.group) - self.assertRaises(e, p.as_uri) - - def test_as_uri_common(self): - e = pathlib.UnsupportedOperation - self.assertRaises(e, self.cls().as_uri) - - def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls()) - - def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls()) - - def test_matches_path_api(self): - our_names = {name for name in dir(self.cls) if name[0] != '_'} - path_names = {name for name in dir(pathlib.Path) if name[0] != '_'} - self.assertEqual(our_names, path_names) - for attr_name in our_names: - our_attr = getattr(self.cls, attr_name) - path_attr = getattr(pathlib.Path, attr_name) - self.assertEqual(our_attr.__doc__, path_attr.__doc__) - - -class DummyPathIO(io.BytesIO): - """ - Used by DummyPath to implement `open('w')` - """ - - def __init__(self, files, path): - super().__init__() - self.files = files - self.path = path - - def close(self): - self.files[self.path] = self.getvalue() - super().close() - - -class DummyPath(pathlib._abc.PathBase): - """ - Simple implementation of PathBase that keeps files and directories in - memory. - """ - _files = {} - _directories = {} - _symlinks = {} - - def __eq__(self, other): - if not isinstance(other, DummyPath): - return NotImplemented - return str(self) == str(other) - - def __hash__(self): - return hash(str(self)) - - def stat(self, *, follow_symlinks=True): - if follow_symlinks: - path = str(self.resolve()) - else: - path = str(self.parent.resolve() / self.name) - if path in self._files: - st_mode = stat.S_IFREG - elif path in self._directories: - st_mode = stat.S_IFDIR - elif path in self._symlinks: - st_mode = stat.S_IFLNK - else: - raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) - return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - if buffering != -1: - raise NotImplementedError - path_obj = self.resolve() - path = str(path_obj) - name = path_obj.name - parent = str(path_obj.parent) - if path in self._directories: - raise IsADirectoryError(errno.EISDIR, "Is a directory", path) - - text = 'b' not in mode - mode = ''.join(c for c in mode if c not in 'btU') - if mode == 'r': - if path not in self._files: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - stream = io.BytesIO(self._files[path]) - elif mode == 'w': - if parent not in self._directories: - raise FileNotFoundError(errno.ENOENT, "File not found", parent) - stream = DummyPathIO(self._files, path) - self._files[path] = b'' - self._directories[parent].add(name) - else: - raise NotImplementedError - if text: - stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) - return stream - - def iterdir(self): - path = str(self.resolve()) - if path in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path in self._directories: - return (self / name for name in self._directories[path]) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - path = str(self.resolve()) - if path in self._directories: - if exist_ok: - return - else: - raise FileExistsError(errno.EEXIST, "File exists", path) - try: - if self.name: - self._directories[str(self.parent)].add(self.name) - self._directories[path] = set() - except KeyError: - if not parents: - raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - - -class DummyPathTest(DummyPurePathTest): - """Tests for PathBase methods that use stat(), open() and iterdir().""" - - cls = DummyPath - can_symlink = False - - # (BASE) - # | - # |-- brokenLink -> non-existing - # |-- dirA - # | `-- linkC -> ../dirB - # |-- dirB - # | |-- fileB - # | `-- linkD -> ../dirB - # |-- dirC - # | |-- dirD - # | | `-- fileD - # | `-- fileC - # | `-- novel.txt - # |-- dirE # No permissions - # |-- fileA - # |-- linkA -> fileA - # |-- linkB -> dirB - # `-- brokenLinkLoop -> brokenLinkLoop - # - - def setUp(self): - super().setUp() - pathmod = self.cls.pathmod - p = self.cls(BASE) - p.mkdir(parents=True) - p.joinpath('dirA').mkdir() - p.joinpath('dirB').mkdir() - p.joinpath('dirC').mkdir() - p.joinpath('dirC', 'dirD').mkdir() - p.joinpath('dirE').mkdir() - with p.joinpath('fileA').open('wb') as f: - f.write(b"this is file A\n") - with p.joinpath('dirB', 'fileB').open('wb') as f: - f.write(b"this is file B\n") - with p.joinpath('dirC', 'fileC').open('wb') as f: - f.write(b"this is file C\n") - with p.joinpath('dirC', 'novel.txt').open('wb') as f: - f.write(b"this is a novel\n") - with p.joinpath('dirC', 'dirD', 'fileD').open('wb') as f: - f.write(b"this is file D\n") - if self.can_symlink: - p.joinpath('linkA').symlink_to('fileA') - p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') - - def tearDown(self): - cls = self.cls - cls._files.clear() - cls._directories.clear() - cls._symlinks.clear() - - def tempdir(self): - path = self.cls(BASE).with_name('tmp-dirD') - path.mkdir() - return path - - def assertFileNotFound(self, func, *args, **kwargs): - with self.assertRaises(FileNotFoundError) as cm: - func(*args, **kwargs) - self.assertEqual(cm.exception.errno, errno.ENOENT) - - def assertEqualNormCase(self, path_a, path_b): - self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) - - def test_samefile(self): - fileA_path = os.path.join(BASE, 'fileA') - fileB_path = os.path.join(BASE, 'dirB', 'fileB') - p = self.cls(fileA_path) - pp = self.cls(fileA_path) - q = self.cls(fileB_path) - self.assertTrue(p.samefile(fileA_path)) - self.assertTrue(p.samefile(pp)) - self.assertFalse(p.samefile(fileB_path)) - self.assertFalse(p.samefile(q)) - # Test the non-existent file case - non_existent = os.path.join(BASE, 'foo') - r = self.cls(non_existent) - self.assertRaises(FileNotFoundError, p.samefile, r) - self.assertRaises(FileNotFoundError, p.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, p) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, r) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - - def test_empty_path(self): - # The empty path points to '.' - p = self.cls('') - self.assertEqual(str(p), '.') - - def test_exists(self): - P = self.cls - p = P(BASE) - self.assertIs(True, p.exists()) - self.assertIs(True, (p / 'dirA').exists()) - self.assertIs(True, (p / 'fileA').exists()) - self.assertIs(False, (p / 'fileA' / 'bah').exists()) - if self.can_symlink: - self.assertIs(True, (p / 'linkA').exists()) - self.assertIs(True, (p / 'linkB').exists()) - self.assertIs(True, (p / 'linkB' / 'fileB').exists()) - self.assertIs(False, (p / 'linkA' / 'bah').exists()) - self.assertIs(False, (p / 'brokenLink').exists()) - self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) - self.assertIs(False, (p / 'foo').exists()) - self.assertIs(False, P('/xyzzy').exists()) - self.assertIs(False, P(BASE + '\udfff').exists()) - self.assertIs(False, P(BASE + '\x00').exists()) - - def test_open_common(self): - p = self.cls(BASE) - with (p / 'fileA').open('r') as f: - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.read(), "this is file A\n") - with (p / 'fileA').open('rb') as f: - self.assertIsInstance(f, io.BufferedIOBase) - self.assertEqual(f.read().strip(), b"this is file A") - - def test_read_write_bytes(self): - p = self.cls(BASE) - (p / 'fileA').write_bytes(b'abcdefg') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - # Check that trying to write str does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - - def test_read_write_text(self): - p = self.cls(BASE) - (p / 'fileA').write_text('äbcdefg', encoding='latin-1') - self.assertEqual((p / 'fileA').read_text( - encoding='utf-8', errors='ignore'), 'bcdefg') - # Check that trying to write bytes does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') - self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') - - def test_read_text_with_newlines(self): - p = self.cls(BASE) - # Check that `\n` character change nothing - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\n'), - 'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\r'), - 'abcde\r\nfghlk\n\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\r\n'), - 'abcde\r\nfghlk\n\rmnopq') - - def test_write_text_with_newlines(self): - p = self.cls(BASE) - # Check that `\n` character change nothing - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\rfghlk\r\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\r\nfghlk\r\n\rmnopq') - # Check that no argument passed will change `\n` to `os.linesep` - os_linesep_byte = bytes(os.linesep, encoding='ascii') - (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') - - def test_iterdir(self): - P = self.cls - p = P(BASE) - it = p.iterdir() - paths = set(it) - expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] - if self.can_symlink: - expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] - self.assertEqual(paths, { P(BASE, q) for q in expected }) - - def test_iterdir_symlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") - # __iter__ on a symlink to a directory. - P = self.cls - p = P(BASE, 'linkB') - paths = set(p.iterdir()) - expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] } - self.assertEqual(paths, expected) - - def test_iterdir_nodir(self): - # __iter__ on something that is not a directory. - p = self.cls(BASE, 'fileA') - with self.assertRaises(OSError) as cm: - p.iterdir() - # ENOENT or EINVAL under Windows, ENOTDIR otherwise - # (see issue #12802). - self.assertIn(cm.exception.errno, (errno.ENOTDIR, - errno.ENOENT, errno.EINVAL)) - - def test_glob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - it = p.glob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.glob("fileB"), []) - _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) - if not self.can_symlink: - _check(p.glob("*A"), ['dirA', 'fileA']) - else: - _check(p.glob("*A"), ['dirA', 'fileA', 'linkA']) - if not self.can_symlink: - _check(p.glob("*B/*"), ['dirB/fileB']) - else: - _check(p.glob("*B/*"), ['dirB/fileB', 'dirB/linkD', - 'linkB/fileB', 'linkB/linkD']) - if not self.can_symlink: - _check(p.glob("*/fileB"), ['dirB/fileB']) - else: - _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) - if self.can_symlink: - _check(p.glob("brokenLink"), ['brokenLink']) - - if not self.can_symlink: - _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/"]) - else: - _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) - - def test_glob_empty_pattern(self): - p = self.cls() - with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): - list(p.glob('')) - - def test_glob_case_sensitive(self): - P = self.cls - def _check(path, pattern, case_sensitive, expected): - actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} - expected = {str(P(BASE, q)) for q in expected} - self.assertEqual(actual, expected) - path = P(BASE) - _check(path, "DIRB/FILE*", True, []) - _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) - _check(path, "dirb/file*", True, []) - _check(path, "dirb/file*", False, ["dirB/fileB"]) - - def test_glob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=True) - if "linkD" not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) - _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) - _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", - "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - - def test_glob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) - _check(p, "dir*/*/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**/", ["dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - - def test_rglob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), {P(BASE, q) for q in expected}) - P = self.cls - p = P(BASE) - it = p.rglob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.rglob("fileB"), ["dirB/fileB"]) - _check(p.rglob("**/fileB"), ["dirB/fileB"]) - _check(p.rglob("*/fileA"), []) - if not self.can_symlink: - _check(p.rglob("*/fileB"), ["dirB/fileB"]) - else: - _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", - "linkB/fileB", "dirA/linkC/fileB"]) - _check(p.rglob("file*"), ["fileA", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD"]) - if not self.can_symlink: - _check(p.rglob("*/"), [ - "dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/", - ]) - else: - _check(p.rglob("*/"), [ - "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", - "dirC/dirD/", "dirE/", "linkB/", - ]) - _check(p.rglob(""), ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) - - p = P(BASE, "dirC") - _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) - _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) - _check(p.rglob("*/"), ["dirC/dirD/"]) - _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) - _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p.rglob("*.txt"), ["dirC/novel.txt"]) - _check(p.rglob("*.*"), ["dirC/novel.txt"]) - - def test_rglob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=True) - if 'linkD' not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) - _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) - _check(p, "", ["./", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) - - p = P(BASE, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - def test_rglob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", ["dirB/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "", ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) - - p = P(BASE, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - def test_rglob_symlink_loop(self): - # Don't get fooled by symlink loops (Issue #26012). - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls - p = P(BASE) - given = set(p.rglob('*')) - expect = {'brokenLink', - 'dirA', 'dirA/linkC', - 'dirB', 'dirB/fileB', 'dirB/linkD', - 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', - 'dirC/fileC', 'dirC/novel.txt', - 'dirE', - 'fileA', - 'linkA', - 'linkB', - 'brokenLinkLoop', - } - self.assertEqual(given, {p / x for x in expect}) - - def test_glob_many_open_files(self): - depth = 30 - P = self.cls - p = base = P(BASE) / 'deep' - p.mkdir() - for _ in range(depth): - p /= 'd' - p.mkdir() - pattern = '/'.join(['*'] * depth) - iters = [base.glob(pattern) for j in range(100)] - for it in iters: - self.assertEqual(next(it), p) - iters = [base.rglob('d') for j in range(100)] - p = base - for i in range(depth): - p = p / 'd' - for it in iters: - self.assertEqual(next(it), p) - - def test_glob_dotdot(self): - # ".." is not special in globs. - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) - self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) - self.assertEqual(set(p.glob("dirA/../file*/..")), set()) - self.assertEqual(set(p.glob("../xyzzy")), set()) - self.assertEqual(set(p.glob("xyzzy/..")), set()) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) - - def test_glob_permissions(self): - # See bpo-38894 - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls - base = P(BASE) / 'permissions' - base.mkdir() - - for i in range(100): - link = base / f"link{i}" - if i % 2: - link.symlink_to(P(BASE, "dirE", "nonexistent")) - else: - link.symlink_to(P(BASE, "dirC")) - - self.assertEqual(len(set(base.glob("*"))), 100) - self.assertEqual(len(set(base.glob("*/"))), 50) - self.assertEqual(len(set(base.glob("*/fileC"))), 50) - self.assertEqual(len(set(base.glob("*/file*"))), 50) - - def test_glob_long_symlink(self): - # See gh-87695 - if not self.can_symlink: - self.skipTest("symlinks required") - base = self.cls(BASE) / 'long_symlink' - base.mkdir() - bad_link = base / 'bad_link' - bad_link.symlink_to("bad" * 200) - self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - - def test_glob_above_recursion_limit(self): - recursion_limit = 50 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(BASE, 'deep') - path = base.joinpath(*(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.glob('**/')) - - def test_glob_recursive_no_trailing_slash(self): - P = self.cls - p = P(BASE) - with self.assertWarns(FutureWarning): - p.glob('**') - with self.assertWarns(FutureWarning): - p.glob('*/**') - with self.assertWarns(FutureWarning): - p.rglob('**') - with self.assertWarns(FutureWarning): - p.rglob('*/**') - - - def test_readlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls(BASE) - self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) - self.assertEqual((P / 'brokenLink').readlink(), - self.cls('non-existing')) - self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) - self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) - with self.assertRaises(OSError): - (P / 'fileA').readlink() - - @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") - def test_readlink_unsupported(self): - P = self.cls(BASE) - p = P / 'fileA' - with self.assertRaises(pathlib.UnsupportedOperation): - q.readlink(p) - - def _check_resolve(self, p, expected, strict=True): - q = p.resolve(strict) - self.assertEqual(q, expected) - - # This can be used to check both relative and absolute resolutions. - _check_resolve_relative = _check_resolve_absolute = _check_resolve - - def test_resolve_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls - p = P(BASE, 'foo') - with self.assertRaises(OSError) as cm: - p.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ENOENT) - # Non-strict - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) - p = P(BASE, 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo', 'in', 'spam')) - p = P(BASE, '..', 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo', 'in', 'spam'))) - # These are all relative symlinks. - p = P(BASE, 'dirB', 'fileB') - self._check_resolve_relative(p, p) - p = P(BASE, 'linkA') - self._check_resolve_relative(p, P(BASE, 'fileA')) - p = P(BASE, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - p = P(BASE, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - # Non-strict - p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', - 'spam'), False) - p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', - 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) - # Now create absolute symlinks. - d = self.tempdir() - P(BASE, 'dirA', 'linkX').symlink_to(d) - P(BASE, str(d), 'linkY').symlink_to(join('dirB')) - p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) - # Non-strict - p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), - False) - p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) - - def test_resolve_dot(self): - # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE) - p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(os.path.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(os.path.join('1', '1'), target_is_directory=True) - q = p / '2' - self.assertEqual(q.resolve(strict=True), p) - r = q / '3' / '4' - self.assertRaises(FileNotFoundError, r.resolve, strict=True) - # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3' / '4') - - def _check_symlink_loop(self, *args): - path = self.cls(*args) - with self.assertRaises(OSError) as cm: - path.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ELOOP) - - def test_resolve_loop(self): - if not self.can_symlink: - self.skipTest("symlinks required") - if os.name == 'nt' and issubclass(self.cls, pathlib.Path): - self.skipTest("symlink loops work differently with concrete Windows paths") - # Loops with relative symlinks. - self.cls(BASE, 'linkX').symlink_to('linkX/inside') - self._check_symlink_loop(BASE, 'linkX') - self.cls(BASE, 'linkY').symlink_to('linkY') - self._check_symlink_loop(BASE, 'linkY') - self.cls(BASE, 'linkZ').symlink_to('linkZ/../linkZ') - self._check_symlink_loop(BASE, 'linkZ') - # Non-strict - p = self.cls(BASE, 'linkZ', 'foo') - self.assertEqual(p.resolve(strict=False), p) - # Loops with absolute symlinks. - self.cls(BASE, 'linkU').symlink_to(join('linkU/inside')) - self._check_symlink_loop(BASE, 'linkU') - self.cls(BASE, 'linkV').symlink_to(join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - self.cls(BASE, 'linkW').symlink_to(join('linkW/../linkW')) - self._check_symlink_loop(BASE, 'linkW') - # Non-strict - q = self.cls(BASE, 'linkW', 'foo') - self.assertEqual(q.resolve(strict=False), q) - - def test_stat(self): - statA = self.cls(BASE).joinpath('fileA').stat() - statB = self.cls(BASE).joinpath('dirB', 'fileB').stat() - statC = self.cls(BASE).joinpath('dirC').stat() - # st_mode: files are the same, directory differs. - self.assertIsInstance(statA.st_mode, int) - self.assertEqual(statA.st_mode, statB.st_mode) - self.assertNotEqual(statA.st_mode, statC.st_mode) - self.assertNotEqual(statB.st_mode, statC.st_mode) - # st_ino: all different, - self.assertIsInstance(statA.st_ino, int) - self.assertNotEqual(statA.st_ino, statB.st_ino) - self.assertNotEqual(statA.st_ino, statC.st_ino) - self.assertNotEqual(statB.st_ino, statC.st_ino) - # st_dev: all the same. - self.assertIsInstance(statA.st_dev, int) - self.assertEqual(statA.st_dev, statB.st_dev) - self.assertEqual(statA.st_dev, statC.st_dev) - # other attributes not used by pathlib. - - def test_stat_no_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE) / 'linkA' - st = p.stat() - self.assertNotEqual(st, p.stat(follow_symlinks=False)) - - def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.stat(follow_symlinks=False)) - - def test_lstat(self): - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE)/ 'linkA' - st = p.stat() - self.assertNotEqual(st, p.lstat()) - - def test_lstat_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.lstat()) - - def test_is_dir(self): - P = self.cls(BASE) - self.assertTrue((P / 'dirA').is_dir()) - self.assertFalse((P / 'fileA').is_dir()) - self.assertFalse((P / 'non-existing').is_dir()) - self.assertFalse((P / 'fileA' / 'bah').is_dir()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_dir()) - self.assertTrue((P / 'linkB').is_dir()) - self.assertFalse((P/ 'brokenLink').is_dir()) - self.assertFalse((P / 'dirA\udfff').is_dir()) - self.assertFalse((P / 'dirA\x00').is_dir()) - - def test_is_dir_no_follow_symlinks(self): - P = self.cls(BASE) - self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) - self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) - - def test_is_file(self): - P = self.cls(BASE) - self.assertTrue((P / 'fileA').is_file()) - self.assertFalse((P / 'dirA').is_file()) - self.assertFalse((P / 'non-existing').is_file()) - self.assertFalse((P / 'fileA' / 'bah').is_file()) - if self.can_symlink: - self.assertTrue((P / 'linkA').is_file()) - self.assertFalse((P / 'linkB').is_file()) - self.assertFalse((P/ 'brokenLink').is_file()) - self.assertFalse((P / 'fileA\udfff').is_file()) - self.assertFalse((P / 'fileA\x00').is_file()) - - def test_is_file_no_follow_symlinks(self): - P = self.cls(BASE) - self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) - self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) - - def test_is_mount(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_mount()) - self.assertFalse((P / 'dirA').is_mount()) - self.assertFalse((P / 'non-existing').is_mount()) - self.assertFalse((P / 'fileA' / 'bah').is_mount()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_mount()) - - def test_is_symlink(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_symlink()) - self.assertFalse((P / 'dirA').is_symlink()) - self.assertFalse((P / 'non-existing').is_symlink()) - self.assertFalse((P / 'fileA' / 'bah').is_symlink()) - if self.can_symlink: - self.assertTrue((P / 'linkA').is_symlink()) - self.assertTrue((P / 'linkB').is_symlink()) - self.assertTrue((P/ 'brokenLink').is_symlink()) - self.assertIs((P / 'fileA\udfff').is_file(), False) - self.assertIs((P / 'fileA\x00').is_file(), False) - if self.can_symlink: - self.assertIs((P / 'linkA\udfff').is_file(), False) - self.assertIs((P / 'linkA\x00').is_file(), False) - - def test_is_junction_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_junction()) - self.assertFalse((P / 'dirA').is_junction()) - self.assertFalse((P / 'non-existing').is_junction()) - self.assertFalse((P / 'fileA' / 'bah').is_junction()) - self.assertFalse((P / 'fileA\udfff').is_junction()) - self.assertFalse((P / 'fileA\x00').is_junction()) - - def test_is_fifo_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_fifo()) - self.assertFalse((P / 'dirA').is_fifo()) - self.assertFalse((P / 'non-existing').is_fifo()) - self.assertFalse((P / 'fileA' / 'bah').is_fifo()) - self.assertIs((P / 'fileA\udfff').is_fifo(), False) - self.assertIs((P / 'fileA\x00').is_fifo(), False) - - def test_is_socket_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_socket()) - self.assertFalse((P / 'dirA').is_socket()) - self.assertFalse((P / 'non-existing').is_socket()) - self.assertFalse((P / 'fileA' / 'bah').is_socket()) - self.assertIs((P / 'fileA\udfff').is_socket(), False) - self.assertIs((P / 'fileA\x00').is_socket(), False) - - def test_is_block_device_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_block_device()) - self.assertFalse((P / 'dirA').is_block_device()) - self.assertFalse((P / 'non-existing').is_block_device()) - self.assertFalse((P / 'fileA' / 'bah').is_block_device()) - self.assertIs((P / 'fileA\udfff').is_block_device(), False) - self.assertIs((P / 'fileA\x00').is_block_device(), False) - - def test_is_char_device_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_char_device()) - self.assertFalse((P / 'dirA').is_char_device()) - self.assertFalse((P / 'non-existing').is_char_device()) - self.assertFalse((P / 'fileA' / 'bah').is_char_device()) - self.assertIs((P / 'fileA\udfff').is_char_device(), False) - self.assertIs((P / 'fileA\x00').is_char_device(), False) - - def test_pickling_common(self): - p = self.cls(BASE, 'fileA') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertEqual(pp.stat(), p.stat()) - - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - - def _check_complex_symlinks(self, link0_target): - if not self.can_symlink: - self.skipTest("symlinks required") - - # Test solving a non-looping chain of symlinks (issue #19887). - P = self.cls(BASE) - P.joinpath('link1').symlink_to(os.path.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(os.path.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(os.path.join('link2', 'link2'), target_is_directory=True) - P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) - - # Resolve absolute paths. - p = (P / 'link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - - # Resolve relative paths. - try: - self.cls().absolute() - except pathlib.UnsupportedOperation: - return - old_path = os.getcwd() - os.chdir(BASE) - try: - p = self.cls('link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - finally: - os.chdir(old_path) - - def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(BASE) - - def test_complex_symlinks_relative(self): - self._check_complex_symlinks('.') - - def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(os.path.join('dirA', '..')) - - def setUpWalk(self): - # Build: - # TESTFN/ - # TEST1/ a file kid and two directory kids - # tmp1 - # SUB1/ a file kid and a directory kid - # tmp2 - # SUB11/ no kids - # SUB2/ a file kid and a dirsymlink kid - # tmp3 - # link/ a symlink to TEST2 - # broken_link - # broken_link2 - # TEST2/ - # tmp4 a lone file - self.walk_path = self.cls(BASE, "TEST1") - self.sub1_path = self.walk_path / "SUB1" - self.sub11_path = self.sub1_path / "SUB11" - self.sub2_path = self.walk_path / "SUB2" - tmp1_path = self.walk_path / "tmp1" - tmp2_path = self.sub1_path / "tmp2" - tmp3_path = self.sub2_path / "tmp3" - self.link_path = self.sub2_path / "link" - t2_path = self.cls(BASE, "TEST2") - tmp4_path = self.cls(BASE, "TEST2", "tmp4") - broken_link_path = self.sub2_path / "broken_link" - broken_link2_path = self.sub2_path / "broken_link2" - - self.sub11_path.mkdir(parents=True) - self.sub2_path.mkdir(parents=True) - t2_path.mkdir(parents=True) - - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: - with path.open("w", encoding='utf-8') as f: - f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") - - if self.can_symlink: - self.link_path.symlink_to(t2_path) - broken_link_path.symlink_to('broken') - broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) - self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) - else: - self.sub2_tree = (self.sub2_path, [], ["tmp3"]) - - def test_walk_topdown(self): - self.setUpWalk() - walker = self.walk_path.walk() - entry = next(walker) - entry[1].sort() # Ensure we visit SUB1 before SUB2 - self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"])) - entry = next(walker) - self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"])) - entry = next(walker) - self.assertEqual(entry, (self.sub11_path, [], [])) - entry = next(walker) - entry[1].sort() - entry[2].sort() - self.assertEqual(entry, self.sub2_tree) - with self.assertRaises(StopIteration): - next(walker) - - def test_walk_prune(self): - self.setUpWalk() - # Prune the search. - all = [] - for root, dirs, files in self.walk_path.walk(): - all.append((root, dirs, files)) - if 'SUB1' in dirs: - # Note that this also mutates the dirs we appended to all! - dirs.remove('SUB1') - - self.assertEqual(len(all), 2) - self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) - - all[1][-1].sort() - all[1][1].sort() - self.assertEqual(all[1], self.sub2_tree) - - def test_walk_bottom_up(self): - self.setUpWalk() - seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False - for path, dirnames, filenames in self.walk_path.walk(top_down=False): - if path == self.walk_path: - self.assertFalse(seen_testfn) - self.assertTrue(seen_sub1) - self.assertTrue(seen_sub2) - self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"]) - self.assertEqual(filenames, ["tmp1"]) - seen_testfn = True - elif path == self.sub1_path: - self.assertFalse(seen_testfn) - self.assertFalse(seen_sub1) - self.assertTrue(seen_sub11) - self.assertEqual(dirnames, ["SUB11"]) - self.assertEqual(filenames, ["tmp2"]) - seen_sub1 = True - elif path == self.sub11_path: - self.assertFalse(seen_sub1) - self.assertFalse(seen_sub11) - self.assertEqual(dirnames, []) - self.assertEqual(filenames, []) - seen_sub11 = True - elif path == self.sub2_path: - self.assertFalse(seen_testfn) - self.assertFalse(seen_sub2) - self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1])) - self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2])) - seen_sub2 = True - else: - raise AssertionError(f"Unexpected path: {path}") - self.assertTrue(seen_testfn) - - def test_walk_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") - self.setUpWalk() - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.link_path: - self.assertEqual(dirs, []) - self.assertEqual(files, ["tmp4"]) - break - else: - self.fail("Didn't follow symlink with follow_symlinks=True") - - def test_walk_symlink_location(self): - if not self.can_symlink: - self.skipTest("symlinks required") - self.setUpWalk() - # Tests whether symlinks end up in filenames or dirnames depending - # on the `follow_symlinks` argument. - walk_it = self.walk_path.walk(follow_symlinks=False) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", files) - break - else: - self.fail("symlink not found") - - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", dirs) - break - else: - self.fail("symlink not found") - - def test_walk_above_recursion_limit(self): - recursion_limit = 40 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(BASE, 'deep') - path = base.joinpath(*(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.walk()) - list(base.walk(top_down=False)) - -class DummyPathWithSymlinks(DummyPath): - def readlink(self): - path = str(self.parent.resolve() / self.name) - if path in self._symlinks: - return self.with_segments(self._symlinks[path]) - elif path in self._files or path in self._directories: - raise OSError(errno.EINVAL, "Not a symlink", path) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def symlink_to(self, target, target_is_directory=False): - self._directories[str(self.parent)].add(self.name) - self._symlinks[str(self)] = str(target) - - -class DummyPathWithSymlinksTest(DummyPathTest): - cls = DummyPathWithSymlinks - can_symlink = True - - # # Tests for the concrete classes. # -class PathTest(DummyPathTest, PurePathTest): +class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): """Tests for the FS-accessing functionalities of the Path classes.""" cls = pathlib.Path can_symlink = os_helper.can_symlink() diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py new file mode 100644 index 00000000000000..61ed3cb6a4a7f8 --- /dev/null +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -0,0 +1,1918 @@ +import collections.abc +import io +import os +import errno +import pathlib +import pickle +import posixpath +import stat +import unittest + +from test.support import set_recursion_limit +from test.support.os_helper import TESTFN + + +class UnsupportedOperationTest(unittest.TestCase): + def test_is_notimplemented(self): + self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) + self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + + +# Make sure any symbolic links in the base test path are resolved. +BASE = os.path.realpath(TESTFN) +join = lambda *x: os.path.join(BASE, *x) + +only_nt = unittest.skipIf(os.name != 'nt', + 'test requires a Windows-compatible system') +only_posix = unittest.skipIf(os.name == 'nt', + 'test requires a POSIX-compatible system') + + +# +# Tests for the pure classes. +# + + +class PurePathBaseTest(unittest.TestCase): + cls = pathlib._abc.PurePathBase + + def test_magic_methods(self): + P = self.cls + self.assertFalse(hasattr(P, '__fspath__')) + self.assertFalse(hasattr(P, '__bytes__')) + self.assertIs(P.__reduce__, object.__reduce__) + self.assertIs(P.__hash__, object.__hash__) + self.assertIs(P.__eq__, object.__eq__) + self.assertIs(P.__lt__, object.__lt__) + self.assertIs(P.__le__, object.__le__) + self.assertIs(P.__gt__, object.__gt__) + self.assertIs(P.__ge__, object.__ge__) + + +class DummyPurePath(pathlib._abc.PurePathBase): + def __eq__(self, other): + if not isinstance(other, DummyPurePath): + return NotImplemented + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + +class DummyPurePathTest(unittest.TestCase): + cls = DummyPurePath + + # Keys are canonical paths, values are list of tuples of arguments + # supposed to produce equal paths. + equivalences = { + 'a/b': [ + ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), + ('a/b/',), ('a//b',), ('a//b//',), + # Empty components get removed. + ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), + ], + '/b/c/d': [ + ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), + # Empty components get removed. + ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), + ], + } + + def setUp(self): + p = self.cls('a') + self.pathmod = p.pathmod + self.sep = self.pathmod.sep + self.altsep = self.pathmod.altsep + + def test_constructor_common(self): + P = self.cls + p = P('a') + self.assertIsInstance(p, P) + P('a', 'b', 'c') + P('/a', 'b', 'c') + P('a/b/c') + P('/a/b/c') + + def test_concrete_class(self): + if self.cls is pathlib.PurePath: + expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_different_pathmods_unequal(self): + p = self.cls('a') + if p.pathmod is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + self.assertNotEqual(p, q) + + def test_different_pathmods_unordered(self): + p = self.cls('a') + if p.pathmod is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + with self.assertRaises(TypeError): + p < q + with self.assertRaises(TypeError): + p <= q + with self.assertRaises(TypeError): + p > q + with self.assertRaises(TypeError): + p >= q + + def _check_str_subclass(self, *args): + # Issue #21127: it should be possible to construct a PurePath object + # from a str subclass instance, and it then gets converted to + # a pure str object. + class StrSubclass(str): + pass + P = self.cls + p = P(*(StrSubclass(x) for x in args)) + self.assertEqual(p, P(*args)) + for part in p.parts: + self.assertIs(type(part), str) + + def test_str_subclass_common(self): + self._check_str_subclass('') + self._check_str_subclass('.') + self._check_str_subclass('a') + self._check_str_subclass('a/b.txt') + self._check_str_subclass('/a/b.txt') + + def test_with_segments_common(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P('foo', 'bar', session_id=42) + self.assertEqual(42, (p / 'foo').session_id) + self.assertEqual(42, ('foo' / p).session_id) + self.assertEqual(42, p.joinpath('foo').session_id) + self.assertEqual(42, p.with_name('foo').session_id) + self.assertEqual(42, p.with_stem('foo').session_id) + self.assertEqual(42, p.with_suffix('.foo').session_id) + self.assertEqual(42, p.with_segments('foo').session_id) + self.assertEqual(42, p.relative_to('foo').session_id) + self.assertEqual(42, p.parent.session_id) + for parent in p.parents: + self.assertEqual(42, parent.session_id) + + def _check_parse_path(self, raw_path, *expected): + sep = self.pathmod.sep + actual = self.cls._parse_path(raw_path.replace('/', sep)) + self.assertEqual(actual, expected) + if altsep := self.pathmod.altsep: + actual = self.cls._parse_path(raw_path.replace('/', altsep)) + self.assertEqual(actual, expected) + + def test_parse_path_common(self): + check = self._check_parse_path + sep = self.pathmod.sep + check('', '', '', []) + check('a', '', '', ['a']) + check('a/', '', '', ['a']) + check('a/b', '', '', ['a', 'b']) + check('a/b/', '', '', ['a', 'b']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('.', '', '', []) + check('././b', '', '', ['b']) + check('a/./b', '', '', ['a', 'b']) + check('a/./.', '', '', ['a']) + check('/a/b', '', sep, ['a', 'b']) + + def test_join_common(self): + P = self.cls + p = P('a/b') + pp = p.joinpath('c') + self.assertEqual(pp, P('a/b/c')) + self.assertIs(type(pp), type(p)) + pp = p.joinpath('c', 'd') + self.assertEqual(pp, P('a/b/c/d')) + pp = p.joinpath('/c') + self.assertEqual(pp, P('/c')) + + def test_div_common(self): + # Basically the same as joinpath(). + P = self.cls + p = P('a/b') + pp = p / 'c' + self.assertEqual(pp, P('a/b/c')) + self.assertIs(type(pp), type(p)) + pp = p / 'c/d' + self.assertEqual(pp, P('a/b/c/d')) + pp = p / 'c' / 'd' + self.assertEqual(pp, P('a/b/c/d')) + pp = 'c' / p / 'd' + self.assertEqual(pp, P('c/a/b/d')) + pp = p/ '/c' + self.assertEqual(pp, P('/c')) + + def _check_str(self, expected, args): + p = self.cls(*args) + self.assertEqual(str(p), expected.replace('/', self.sep)) + + def test_str_common(self): + # Canonicalized paths roundtrip. + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self._check_str(pathstr, (pathstr,)) + # Special case for the empty path. + self._check_str('.', ('',)) + # Other tests for str() are in test_equivalences(). + + def test_as_posix_common(self): + P = self.cls + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self.assertEqual(P(pathstr).as_posix(), pathstr) + # Other tests for as_posix() are in test_equivalences(). + + def test_repr_common(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + clsname = p.__class__.__name__ + r = repr(p) + # The repr() is in the form ClassName("forward-slashes path"). + self.assertTrue(r.startswith(clsname + '('), r) + self.assertTrue(r.endswith(')'), r) + inner = r[len(clsname) + 1 : -1] + self.assertEqual(eval(inner), p.as_posix()) + + def test_eq_common(self): + P = self.cls + self.assertEqual(P('a/b'), P('a/b')) + self.assertEqual(P('a/b'), P('a', 'b')) + self.assertNotEqual(P('a/b'), P('a')) + self.assertNotEqual(P('a/b'), P('/a/b')) + self.assertNotEqual(P('a/b'), P()) + self.assertNotEqual(P('/a/b'), P('/')) + self.assertNotEqual(P(), P('/')) + self.assertNotEqual(P(), "") + self.assertNotEqual(P(), {}) + self.assertNotEqual(P(), int) + + def test_match_common(self): + P = self.cls + self.assertRaises(ValueError, P('a').match, '') + self.assertRaises(ValueError, P('a').match, '.') + # Simple relative pattern. + self.assertTrue(P('b.py').match('b.py')) + self.assertTrue(P('a/b.py').match('b.py')) + self.assertTrue(P('/a/b.py').match('b.py')) + self.assertFalse(P('a.py').match('b.py')) + self.assertFalse(P('b/py').match('b.py')) + self.assertFalse(P('/a.py').match('b.py')) + self.assertFalse(P('b.py/c').match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').match('*.py')) + self.assertTrue(P('a/b.py').match('*.py')) + self.assertTrue(P('/a/b.py').match('*.py')) + self.assertFalse(P('b.pyc').match('*.py')) + self.assertFalse(P('b./py').match('*.py')) + self.assertFalse(P('b.py/c').match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').match('a*/*.py')) + self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) + self.assertFalse(P('a.py').match('a*/*.py')) + self.assertFalse(P('/dab/c.py').match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').match('/*.py')) + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('a/b.py').match('/*.py')) + self.assertFalse(P('/a/b.py').match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').match('/a/*.py')) + self.assertFalse(P('/ab.py').match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertTrue(P('a').match('**')) + self.assertTrue(P('c.py').match('**')) + self.assertTrue(P('a/b/c.py').match('**')) + self.assertTrue(P('/a/b/c.py').match('**')) + self.assertTrue(P('/a/b/c.py').match('/**')) + self.assertTrue(P('/a/b/c.py').match('**/')) + self.assertTrue(P('/a/b/c.py').match('/a/**')) + self.assertTrue(P('/a/b/c.py').match('**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/a/b/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/**/**/**/**/*.py')) + self.assertFalse(P('c.py').match('**/a.py')) + self.assertFalse(P('c.py').match('c/**')) + self.assertFalse(P('a/b/c.py').match('**/a')) + self.assertFalse(P('a/b/c.py').match('**/a/b')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c.')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').match('/a/b/c.py/**')) + self.assertFalse(P('a/b/c.py').match('/**/a/b/c.py')) + self.assertRaises(ValueError, P('a').match, '**a/b/c') + self.assertRaises(ValueError, P('a').match, 'a/b/c**') + # Case-sensitive flag + self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P().match('*')) + self.assertTrue(P().match('**')) + self.assertFalse(P().match('**/*')) + + def test_parts_common(self): + # `parts` returns a tuple. + sep = self.sep + P = self.cls + p = P('a/b') + parts = p.parts + self.assertEqual(parts, ('a', 'b')) + # When the path is absolute, the anchor is a separate part. + p = P('/a/b') + parts = p.parts + self.assertEqual(parts, (sep, 'a', 'b')) + + def test_equivalences(self): + for k, tuples in self.equivalences.items(): + canon = k.replace('/', self.sep) + posix = k.replace(self.sep, '/') + if canon != posix: + tuples = tuples + [ + tuple(part.replace('/', self.sep) for part in t) + for t in tuples + ] + tuples.append((posix, )) + pcanon = self.cls(canon) + for t in tuples: + p = self.cls(*t) + self.assertEqual(p, pcanon, "failed with args {}".format(t)) + self.assertEqual(hash(p), hash(pcanon)) + self.assertEqual(str(p), canon) + self.assertEqual(p.as_posix(), posix) + + def test_parent_common(self): + # Relative + P = self.cls + p = P('a/b/c') + self.assertEqual(p.parent, P('a/b')) + self.assertEqual(p.parent.parent, P('a')) + self.assertEqual(p.parent.parent.parent, P()) + self.assertEqual(p.parent.parent.parent.parent, P()) + # Anchored + p = P('/a/b/c') + self.assertEqual(p.parent, P('/a/b')) + self.assertEqual(p.parent.parent, P('/a')) + self.assertEqual(p.parent.parent.parent, P('/')) + self.assertEqual(p.parent.parent.parent.parent, P('/')) + + def test_parents_common(self): + # Relative + P = self.cls + p = P('a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('a/b')) + self.assertEqual(par[1], P('a')) + self.assertEqual(par[2], P('.')) + self.assertEqual(par[-1], P('.')) + self.assertEqual(par[-2], P('a')) + self.assertEqual(par[-3], P('a/b')) + self.assertEqual(par[0:1], (P('a/b'),)) + self.assertEqual(par[:2], (P('a/b'), P('a'))) + self.assertEqual(par[:-1], (P('a/b'), P('a'))) + self.assertEqual(par[1:], (P('a'), P('.'))) + self.assertEqual(par[::2], (P('a/b'), P('.'))) + self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) + self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + with self.assertRaises(TypeError): + par[0] = p + # Anchored + p = P('/a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('/a/b')) + self.assertEqual(par[1], P('/a')) + self.assertEqual(par[2], P('/')) + self.assertEqual(par[-1], P('/')) + self.assertEqual(par[-2], P('/a')) + self.assertEqual(par[-3], P('/a/b')) + self.assertEqual(par[0:1], (P('/a/b'),)) + self.assertEqual(par[:2], (P('/a/b'), P('/a'))) + self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) + self.assertEqual(par[1:], (P('/a'), P('/'))) + self.assertEqual(par[::2], (P('/a/b'), P('/'))) + self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) + self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + + def test_drive_common(self): + P = self.cls + self.assertEqual(P('a/b').drive, '') + self.assertEqual(P('/a/b').drive, '') + self.assertEqual(P('').drive, '') + + def test_root_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').root, '') + self.assertEqual(P('a/b').root, '') + self.assertEqual(P('/').root, sep) + self.assertEqual(P('/a/b').root, sep) + + def test_anchor_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').anchor, '') + self.assertEqual(P('a/b').anchor, '') + self.assertEqual(P('/').anchor, sep) + self.assertEqual(P('/a/b').anchor, sep) + + def test_name_common(self): + P = self.cls + self.assertEqual(P('').name, '') + self.assertEqual(P('.').name, '') + self.assertEqual(P('/').name, '') + self.assertEqual(P('a/b').name, 'b') + self.assertEqual(P('/a/b').name, 'b') + self.assertEqual(P('/a/b/.').name, 'b') + self.assertEqual(P('a/b.py').name, 'b.py') + self.assertEqual(P('/a/b.py').name, 'b.py') + + def test_suffix_common(self): + P = self.cls + self.assertEqual(P('').suffix, '') + self.assertEqual(P('.').suffix, '') + self.assertEqual(P('..').suffix, '') + self.assertEqual(P('/').suffix, '') + self.assertEqual(P('a/b').suffix, '') + self.assertEqual(P('/a/b').suffix, '') + self.assertEqual(P('/a/b/.').suffix, '') + self.assertEqual(P('a/b.py').suffix, '.py') + self.assertEqual(P('/a/b.py').suffix, '.py') + self.assertEqual(P('a/.hgrc').suffix, '') + self.assertEqual(P('/a/.hgrc').suffix, '') + self.assertEqual(P('a/.hg.rc').suffix, '.rc') + self.assertEqual(P('/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + + def test_suffixes_common(self): + P = self.cls + self.assertEqual(P('').suffixes, []) + self.assertEqual(P('.').suffixes, []) + self.assertEqual(P('/').suffixes, []) + self.assertEqual(P('a/b').suffixes, []) + self.assertEqual(P('/a/b').suffixes, []) + self.assertEqual(P('/a/b/.').suffixes, []) + self.assertEqual(P('a/b.py').suffixes, ['.py']) + self.assertEqual(P('/a/b.py').suffixes, ['.py']) + self.assertEqual(P('a/.hgrc').suffixes, []) + self.assertEqual(P('/a/.hgrc').suffixes, []) + self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) + + def test_stem_common(self): + P = self.cls + self.assertEqual(P('').stem, '') + self.assertEqual(P('.').stem, '') + self.assertEqual(P('..').stem, '..') + self.assertEqual(P('/').stem, '') + self.assertEqual(P('a/b').stem, 'b') + self.assertEqual(P('a/b.py').stem, 'b') + self.assertEqual(P('a/.hgrc').stem, '.hgrc') + self.assertEqual(P('a/.hg.rc').stem, '.hg') + self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('a/Some name. Ending with a dot.').stem, + 'Some name. Ending with a dot.') + + def test_with_name_common(self): + P = self.cls + self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) + self.assertRaises(ValueError, P('').with_name, 'd.xml') + self.assertRaises(ValueError, P('.').with_name, 'd.xml') + self.assertRaises(ValueError, P('/').with_name, 'd.xml') + self.assertRaises(ValueError, P('a/b').with_name, '') + self.assertRaises(ValueError, P('a/b').with_name, '.') + self.assertRaises(ValueError, P('a/b').with_name, '/c') + self.assertRaises(ValueError, P('a/b').with_name, 'c/') + self.assertRaises(ValueError, P('a/b').with_name, 'c/d') + + def test_with_stem_common(self): + P = self.cls + self.assertEqual(P('a/b').with_stem('d'), P('a/d')) + self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) + self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) + self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + self.assertRaises(ValueError, P('').with_stem, 'd') + self.assertRaises(ValueError, P('.').with_stem, 'd') + self.assertRaises(ValueError, P('/').with_stem, 'd') + self.assertRaises(ValueError, P('a/b').with_stem, '') + self.assertRaises(ValueError, P('a/b').with_stem, '.') + self.assertRaises(ValueError, P('a/b').with_stem, '/c') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') + + def test_with_suffix_common(self): + P = self.cls + self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) + self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) + # Stripping suffix. + self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) + self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('.').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('a/b').with_suffix, '/') + self.assertRaises(ValueError, P('a/b').with_suffix, '.') + self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') + self.assertRaises(ValueError, P('a/b').with_suffix, './.d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') + + def test_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P()), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P()) + self.assertEqual(p.relative_to('a/b'), P()) + self.assertEqual(p.relative_to(P(), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P()) + self.assertEqual(p.relative_to('a/b', walk_up=True), P()) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # With several args. + with self.assertWarns(DeprecationWarning): + p.relative_to('a', 'b') + p.relative_to('a', 'b', walk_up=True) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P()) + self.assertEqual(p.relative_to('/a/b'), P()) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P()) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P()) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P())) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # With several args. + with self.assertWarns(DeprecationWarning): + p.is_relative_to('a', 'b') + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + +# +# Tests for the virtual classes. +# + +class PathBaseTest(PurePathBaseTest): + cls = pathlib._abc.PathBase + + def test_unsupported_operation(self): + P = self.cls + p = self.cls() + e = pathlib.UnsupportedOperation + self.assertRaises(e, p.stat) + self.assertRaises(e, p.lstat) + self.assertRaises(e, p.exists) + self.assertRaises(e, p.samefile, 'foo') + self.assertRaises(e, p.is_dir) + self.assertRaises(e, p.is_file) + self.assertRaises(e, p.is_mount) + self.assertRaises(e, p.is_symlink) + self.assertRaises(e, p.is_block_device) + self.assertRaises(e, p.is_char_device) + self.assertRaises(e, p.is_fifo) + self.assertRaises(e, p.is_socket) + self.assertRaises(e, p.open) + self.assertRaises(e, p.read_bytes) + self.assertRaises(e, p.read_text) + self.assertRaises(e, p.write_bytes, b'foo') + self.assertRaises(e, p.write_text, 'foo') + self.assertRaises(e, p.iterdir) + self.assertRaises(e, p.glob, '*') + self.assertRaises(e, p.rglob, '*') + self.assertRaises(e, lambda: list(p.walk())) + self.assertRaises(e, p.absolute) + self.assertRaises(e, P.cwd) + self.assertRaises(e, p.expanduser) + self.assertRaises(e, p.home) + self.assertRaises(e, p.readlink) + self.assertRaises(e, p.symlink_to, 'foo') + self.assertRaises(e, p.hardlink_to, 'foo') + self.assertRaises(e, p.mkdir) + self.assertRaises(e, p.touch) + self.assertRaises(e, p.rename, 'foo') + self.assertRaises(e, p.replace, 'foo') + self.assertRaises(e, p.chmod, 0o755) + self.assertRaises(e, p.lchmod, 0o755) + self.assertRaises(e, p.unlink) + self.assertRaises(e, p.rmdir) + self.assertRaises(e, p.owner) + self.assertRaises(e, p.group) + self.assertRaises(e, p.as_uri) + + def test_as_uri_common(self): + e = pathlib.UnsupportedOperation + self.assertRaises(e, self.cls().as_uri) + + def test_fspath_common(self): + self.assertRaises(TypeError, os.fspath, self.cls()) + + def test_as_bytes_common(self): + self.assertRaises(TypeError, bytes, self.cls()) + + def test_matches_path_api(self): + our_names = {name for name in dir(self.cls) if name[0] != '_'} + path_names = {name for name in dir(pathlib.Path) if name[0] != '_'} + self.assertEqual(our_names, path_names) + for attr_name in our_names: + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib.Path, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + + +class DummyPathIO(io.BytesIO): + """ + Used by DummyPath to implement `open('w')` + """ + + def __init__(self, files, path): + super().__init__() + self.files = files + self.path = path + + def close(self): + self.files[self.path] = self.getvalue() + super().close() + + +class DummyPath(pathlib._abc.PathBase): + """ + Simple implementation of PathBase that keeps files and directories in + memory. + """ + _files = {} + _directories = {} + _symlinks = {} + + def __eq__(self, other): + if not isinstance(other, DummyPath): + return NotImplemented + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + def stat(self, *, follow_symlinks=True): + if follow_symlinks: + path = str(self.resolve()) + else: + path = str(self.parent.resolve() / self.name) + if path in self._files: + st_mode = stat.S_IFREG + elif path in self._directories: + st_mode = stat.S_IFDIR + elif path in self._symlinks: + st_mode = stat.S_IFLNK + else: + raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) + return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + if buffering != -1: + raise NotImplementedError + path_obj = self.resolve() + path = str(path_obj) + name = path_obj.name + parent = str(path_obj.parent) + if path in self._directories: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + + text = 'b' not in mode + mode = ''.join(c for c in mode if c not in 'btU') + if mode == 'r': + if path not in self._files: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + stream = io.BytesIO(self._files[path]) + elif mode == 'w': + if parent not in self._directories: + raise FileNotFoundError(errno.ENOENT, "File not found", parent) + stream = DummyPathIO(self._files, path) + self._files[path] = b'' + self._directories[parent].add(name) + else: + raise NotImplementedError + if text: + stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) + return stream + + def iterdir(self): + path = str(self.resolve()) + if path in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path in self._directories: + return (self / name for name in self._directories[path]) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + path = str(self.resolve()) + if path in self._directories: + if exist_ok: + return + else: + raise FileExistsError(errno.EEXIST, "File exists", path) + try: + if self.name: + self._directories[str(self.parent)].add(self.name) + self._directories[path] = set() + except KeyError: + if not parents: + raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + + +class DummyPathTest(DummyPurePathTest): + """Tests for PathBase methods that use stat(), open() and iterdir().""" + + cls = DummyPath + can_symlink = False + + # (BASE) + # | + # |-- brokenLink -> non-existing + # |-- dirA + # | `-- linkC -> ../dirB + # |-- dirB + # | |-- fileB + # | `-- linkD -> ../dirB + # |-- dirC + # | |-- dirD + # | | `-- fileD + # | `-- fileC + # | `-- novel.txt + # |-- dirE # No permissions + # |-- fileA + # |-- linkA -> fileA + # |-- linkB -> dirB + # `-- brokenLinkLoop -> brokenLinkLoop + # + + def setUp(self): + super().setUp() + pathmod = self.cls.pathmod + p = self.cls(BASE) + p.mkdir(parents=True) + p.joinpath('dirA').mkdir() + p.joinpath('dirB').mkdir() + p.joinpath('dirC').mkdir() + p.joinpath('dirC', 'dirD').mkdir() + p.joinpath('dirE').mkdir() + with p.joinpath('fileA').open('wb') as f: + f.write(b"this is file A\n") + with p.joinpath('dirB', 'fileB').open('wb') as f: + f.write(b"this is file B\n") + with p.joinpath('dirC', 'fileC').open('wb') as f: + f.write(b"this is file C\n") + with p.joinpath('dirC', 'novel.txt').open('wb') as f: + f.write(b"this is a novel\n") + with p.joinpath('dirC', 'dirD', 'fileD').open('wb') as f: + f.write(b"this is file D\n") + if self.can_symlink: + p.joinpath('linkA').symlink_to('fileA') + p.joinpath('brokenLink').symlink_to('non-existing') + p.joinpath('linkB').symlink_to('dirB') + p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) + p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) + p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') + + def tearDown(self): + cls = self.cls + cls._files.clear() + cls._directories.clear() + cls._symlinks.clear() + + def tempdir(self): + path = self.cls(BASE).with_name('tmp-dirD') + path.mkdir() + return path + + def assertFileNotFound(self, func, *args, **kwargs): + with self.assertRaises(FileNotFoundError) as cm: + func(*args, **kwargs) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def assertEqualNormCase(self, path_a, path_b): + self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) + + def test_samefile(self): + fileA_path = os.path.join(BASE, 'fileA') + fileB_path = os.path.join(BASE, 'dirB', 'fileB') + p = self.cls(fileA_path) + pp = self.cls(fileA_path) + q = self.cls(fileB_path) + self.assertTrue(p.samefile(fileA_path)) + self.assertTrue(p.samefile(pp)) + self.assertFalse(p.samefile(fileB_path)) + self.assertFalse(p.samefile(q)) + # Test the non-existent file case + non_existent = os.path.join(BASE, 'foo') + r = self.cls(non_existent) + self.assertRaises(FileNotFoundError, p.samefile, r) + self.assertRaises(FileNotFoundError, p.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, p) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, r) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + + def test_empty_path(self): + # The empty path points to '.' + p = self.cls('') + self.assertEqual(str(p), '.') + + def test_exists(self): + P = self.cls + p = P(BASE) + self.assertIs(True, p.exists()) + self.assertIs(True, (p / 'dirA').exists()) + self.assertIs(True, (p / 'fileA').exists()) + self.assertIs(False, (p / 'fileA' / 'bah').exists()) + if self.can_symlink: + self.assertIs(True, (p / 'linkA').exists()) + self.assertIs(True, (p / 'linkB').exists()) + self.assertIs(True, (p / 'linkB' / 'fileB').exists()) + self.assertIs(False, (p / 'linkA' / 'bah').exists()) + self.assertIs(False, (p / 'brokenLink').exists()) + self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) + self.assertIs(False, (p / 'foo').exists()) + self.assertIs(False, P('/xyzzy').exists()) + self.assertIs(False, P(BASE + '\udfff').exists()) + self.assertIs(False, P(BASE + '\x00').exists()) + + def test_open_common(self): + p = self.cls(BASE) + with (p / 'fileA').open('r') as f: + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.read(), "this is file A\n") + with (p / 'fileA').open('rb') as f: + self.assertIsInstance(f, io.BufferedIOBase) + self.assertEqual(f.read().strip(), b"this is file A") + + def test_read_write_bytes(self): + p = self.cls(BASE) + (p / 'fileA').write_bytes(b'abcdefg') + self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') + # Check that trying to write str does not truncate the file. + self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') + self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') + + def test_read_write_text(self): + p = self.cls(BASE) + (p / 'fileA').write_text('äbcdefg', encoding='latin-1') + self.assertEqual((p / 'fileA').read_text( + encoding='utf-8', errors='ignore'), 'bcdefg') + # Check that trying to write bytes does not truncate the file. + self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') + self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') + + def test_read_text_with_newlines(self): + p = self.cls(BASE) + # Check that `\n` character change nothing + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\n'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r\n` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r\n'), + 'abcde\r\nfghlk\n\rmnopq') + + def test_write_text_with_newlines(self): + p = self.cls(BASE) + # Check that `\n` character change nothing + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\rfghlk\r\rmnopq') + # Check that `\r\n` character replaces `\n` + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\r\nfghlk\r\n\rmnopq') + # Check that no argument passed will change `\n` to `os.linesep` + os_linesep_byte = bytes(os.linesep, encoding='ascii') + (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') + + def test_iterdir(self): + P = self.cls + p = P(BASE) + it = p.iterdir() + paths = set(it) + expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] + if self.can_symlink: + expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] + self.assertEqual(paths, { P(BASE, q) for q in expected }) + + def test_iterdir_symlink(self): + if not self.can_symlink: + self.skipTest("symlinks required") + # __iter__ on a symlink to a directory. + P = self.cls + p = P(BASE, 'linkB') + paths = set(p.iterdir()) + expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] } + self.assertEqual(paths, expected) + + def test_iterdir_nodir(self): + # __iter__ on something that is not a directory. + p = self.cls(BASE, 'fileA') + with self.assertRaises(OSError) as cm: + p.iterdir() + # ENOENT or EINVAL under Windows, ENOTDIR otherwise + # (see issue #12802). + self.assertIn(cm.exception.errno, (errno.ENOTDIR, + errno.ENOENT, errno.EINVAL)) + + def test_glob_common(self): + def _check(glob, expected): + self.assertEqual(set(glob), { P(BASE, q) for q in expected }) + P = self.cls + p = P(BASE) + it = p.glob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + _check(it, ["fileA"]) + _check(p.glob("fileB"), []) + _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) + if not self.can_symlink: + _check(p.glob("*A"), ['dirA', 'fileA']) + else: + _check(p.glob("*A"), ['dirA', 'fileA', 'linkA']) + if not self.can_symlink: + _check(p.glob("*B/*"), ['dirB/fileB']) + else: + _check(p.glob("*B/*"), ['dirB/fileB', 'dirB/linkD', + 'linkB/fileB', 'linkB/linkD']) + if not self.can_symlink: + _check(p.glob("*/fileB"), ['dirB/fileB']) + else: + _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) + if self.can_symlink: + _check(p.glob("brokenLink"), ['brokenLink']) + + if not self.can_symlink: + _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/"]) + else: + _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + + def test_glob_empty_pattern(self): + p = self.cls() + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + + def test_glob_case_sensitive(self): + P = self.cls + def _check(path, pattern, case_sensitive, expected): + actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} + expected = {str(P(BASE, q)) for q in expected} + self.assertEqual(actual, expected) + path = P(BASE) + _check(path, "DIRB/FILE*", True, []) + _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) + _check(path, "dirb/file*", True, []) + _check(path, "dirb/file*", False, ["dirB/fileB"]) + + def test_glob_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, follow_symlinks=True) + if "linkD" not in path.parent.parts} # exclude symlink loop. + self.assertEqual(actual, { P(BASE, q) for q in expected }) + P = self.cls + p = P(BASE) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + def test_glob_no_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, follow_symlinks=False)} + self.assertEqual(actual, { P(BASE, q) for q in expected }) + P = self.cls + p = P(BASE) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) + _check(p, "dir*/*/..", ["dirC/dirD/.."]) + _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**/", ["dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + def test_rglob_common(self): + def _check(glob, expected): + self.assertEqual(set(glob), {P(BASE, q) for q in expected}) + P = self.cls + p = P(BASE) + it = p.rglob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + _check(it, ["fileA"]) + _check(p.rglob("fileB"), ["dirB/fileB"]) + _check(p.rglob("**/fileB"), ["dirB/fileB"]) + _check(p.rglob("*/fileA"), []) + if not self.can_symlink: + _check(p.rglob("*/fileB"), ["dirB/fileB"]) + else: + _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", + "linkB/fileB", "dirA/linkC/fileB"]) + _check(p.rglob("file*"), ["fileA", "dirB/fileB", + "dirC/fileC", "dirC/dirD/fileD"]) + if not self.can_symlink: + _check(p.rglob("*/"), [ + "dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/", + ]) + else: + _check(p.rglob("*/"), [ + "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", + "dirC/dirD/", "dirE/", "linkB/", + ]) + _check(p.rglob(""), ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + + p = P(BASE, "dirC") + _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) + _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) + _check(p.rglob("*/"), ["dirC/dirD/"]) + _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) + _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p.rglob("*.txt"), ["dirC/novel.txt"]) + _check(p.rglob("*.*"), ["dirC/novel.txt"]) + + def test_rglob_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, follow_symlinks=True) + if 'linkD' not in path.parent.parts} # exclude symlink loop. + self.assertEqual(actual, { P(BASE, q) for q in expected }) + P = self.cls + p = P(BASE) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["./", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) + + p = P(BASE, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + def test_rglob_no_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, follow_symlinks=False)} + self.assertEqual(actual, { P(BASE, q) for q in expected }) + P = self.cls + p = P(BASE) + _check(p, "fileB", ["dirB/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "", ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + + p = P(BASE, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + def test_rglob_symlink_loop(self): + # Don't get fooled by symlink loops (Issue #26012). + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls + p = P(BASE) + given = set(p.rglob('*')) + expect = {'brokenLink', + 'dirA', 'dirA/linkC', + 'dirB', 'dirB/fileB', 'dirB/linkD', + 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', + 'dirC/fileC', 'dirC/novel.txt', + 'dirE', + 'fileA', + 'linkA', + 'linkB', + 'brokenLinkLoop', + } + self.assertEqual(given, {p / x for x in expect}) + + def test_glob_many_open_files(self): + depth = 30 + P = self.cls + p = base = P(BASE) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() + pattern = '/'.join(['*'] * depth) + iters = [base.glob(pattern) for j in range(100)] + for it in iters: + self.assertEqual(next(it), p) + iters = [base.rglob('d') for j in range(100)] + p = base + for i in range(depth): + p = p / 'd' + for it in iters: + self.assertEqual(next(it), p) + + def test_glob_dotdot(self): + # ".." is not special in globs. + P = self.cls + p = P(BASE) + self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) + self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) + self.assertEqual(set(p.glob("../xyzzy")), set()) + self.assertEqual(set(p.glob("xyzzy/..")), set()) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) + + def test_glob_permissions(self): + # See bpo-38894 + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls + base = P(BASE) / 'permissions' + base.mkdir() + + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(BASE, "dirE", "nonexistent")) + else: + link.symlink_to(P(BASE, "dirC")) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) + + def test_glob_long_symlink(self): + # See gh-87695 + if not self.can_symlink: + self.skipTest("symlinks required") + base = self.cls(BASE) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + + def test_glob_above_recursion_limit(self): + recursion_limit = 50 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(BASE, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with set_recursion_limit(recursion_limit): + list(base.glob('**/')) + + def test_glob_recursive_no_trailing_slash(self): + P = self.cls + p = P(BASE) + with self.assertWarns(FutureWarning): + p.glob('**') + with self.assertWarns(FutureWarning): + p.glob('*/**') + with self.assertWarns(FutureWarning): + p.rglob('**') + with self.assertWarns(FutureWarning): + p.rglob('*/**') + + + def test_readlink(self): + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls(BASE) + self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) + self.assertEqual((P / 'brokenLink').readlink(), + self.cls('non-existing')) + self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) + with self.assertRaises(OSError): + (P / 'fileA').readlink() + + @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") + def test_readlink_unsupported(self): + P = self.cls(BASE) + p = P / 'fileA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.readlink(p) + + def _check_resolve(self, p, expected, strict=True): + q = p.resolve(strict) + self.assertEqual(q, expected) + + # This can be used to check both relative and absolute resolutions. + _check_resolve_relative = _check_resolve_absolute = _check_resolve + + def test_resolve_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls + p = P(BASE, 'foo') + with self.assertRaises(OSError) as cm: + p.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ENOENT) + # Non-strict + self.assertEqualNormCase(str(p.resolve(strict=False)), + os.path.join(BASE, 'foo')) + p = P(BASE, 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + os.path.join(BASE, 'foo', 'in', 'spam')) + p = P(BASE, '..', 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + os.path.abspath(os.path.join('foo', 'in', 'spam'))) + # These are all relative symlinks. + p = P(BASE, 'dirB', 'fileB') + self._check_resolve_relative(p, p) + p = P(BASE, 'linkA') + self._check_resolve_relative(p, P(BASE, 'fileA')) + p = P(BASE, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) + p = P(BASE, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) + # Non-strict + p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) + p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if os.name == 'nt' and isinstance(p, pathlib.Path): + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', + 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + # Now create absolute symlinks. + d = self.tempdir() + P(BASE, 'dirA', 'linkX').symlink_to(d) + P(BASE, str(d), 'linkY').symlink_to(join('dirB')) + p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) + # Non-strict + p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), + False) + p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if os.name == 'nt' and isinstance(p, pathlib.Path): + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + + def test_resolve_dot(self): + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ + if not self.can_symlink: + self.skipTest("symlinks required") + p = self.cls(BASE) + p.joinpath('0').symlink_to('.', target_is_directory=True) + p.joinpath('1').symlink_to(os.path.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(os.path.join('1', '1'), target_is_directory=True) + q = p / '2' + self.assertEqual(q.resolve(strict=True), p) + r = q / '3' / '4' + self.assertRaises(FileNotFoundError, r.resolve, strict=True) + # Non-strict + self.assertEqual(r.resolve(strict=False), p / '3' / '4') + + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + def test_resolve_loop(self): + if not self.can_symlink: + self.skipTest("symlinks required") + if os.name == 'nt' and issubclass(self.cls, pathlib.Path): + self.skipTest("symlink loops work differently with concrete Windows paths") + # Loops with relative symlinks. + self.cls(BASE, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(BASE, 'linkX') + self.cls(BASE, 'linkY').symlink_to('linkY') + self._check_symlink_loop(BASE, 'linkY') + self.cls(BASE, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(BASE, 'linkZ') + # Non-strict + p = self.cls(BASE, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(BASE, 'linkU').symlink_to(join('linkU/inside')) + self._check_symlink_loop(BASE, 'linkU') + self.cls(BASE, 'linkV').symlink_to(join('linkV')) + self._check_symlink_loop(BASE, 'linkV') + self.cls(BASE, 'linkW').symlink_to(join('linkW/../linkW')) + self._check_symlink_loop(BASE, 'linkW') + # Non-strict + q = self.cls(BASE, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + + def test_stat(self): + statA = self.cls(BASE).joinpath('fileA').stat() + statB = self.cls(BASE).joinpath('dirB', 'fileB').stat() + statC = self.cls(BASE).joinpath('dirC').stat() + # st_mode: files are the same, directory differs. + self.assertIsInstance(statA.st_mode, int) + self.assertEqual(statA.st_mode, statB.st_mode) + self.assertNotEqual(statA.st_mode, statC.st_mode) + self.assertNotEqual(statB.st_mode, statC.st_mode) + # st_ino: all different, + self.assertIsInstance(statA.st_ino, int) + self.assertNotEqual(statA.st_ino, statB.st_ino) + self.assertNotEqual(statA.st_ino, statC.st_ino) + self.assertNotEqual(statB.st_ino, statC.st_ino) + # st_dev: all the same. + self.assertIsInstance(statA.st_dev, int) + self.assertEqual(statA.st_dev, statB.st_dev) + self.assertEqual(statA.st_dev, statC.st_dev) + # other attributes not used by pathlib. + + def test_stat_no_follow_symlinks(self): + if not self.can_symlink: + self.skipTest("symlinks required") + p = self.cls(BASE) / 'linkA' + st = p.stat() + self.assertNotEqual(st, p.stat(follow_symlinks=False)) + + def test_stat_no_follow_symlinks_nosymlink(self): + p = self.cls(BASE) / 'fileA' + st = p.stat() + self.assertEqual(st, p.stat(follow_symlinks=False)) + + def test_lstat(self): + if not self.can_symlink: + self.skipTest("symlinks required") + p = self.cls(BASE)/ 'linkA' + st = p.stat() + self.assertNotEqual(st, p.lstat()) + + def test_lstat_nosymlink(self): + p = self.cls(BASE) / 'fileA' + st = p.stat() + self.assertEqual(st, p.lstat()) + + def test_is_dir(self): + P = self.cls(BASE) + self.assertTrue((P / 'dirA').is_dir()) + self.assertFalse((P / 'fileA').is_dir()) + self.assertFalse((P / 'non-existing').is_dir()) + self.assertFalse((P / 'fileA' / 'bah').is_dir()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir()) + self.assertTrue((P / 'linkB').is_dir()) + self.assertFalse((P/ 'brokenLink').is_dir()) + self.assertFalse((P / 'dirA\udfff').is_dir()) + self.assertFalse((P / 'dirA\x00').is_dir()) + + def test_is_dir_no_follow_symlinks(self): + P = self.cls(BASE) + self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) + + def test_is_file(self): + P = self.cls(BASE) + self.assertTrue((P / 'fileA').is_file()) + self.assertFalse((P / 'dirA').is_file()) + self.assertFalse((P / 'non-existing').is_file()) + self.assertFalse((P / 'fileA' / 'bah').is_file()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_file()) + self.assertFalse((P / 'linkB').is_file()) + self.assertFalse((P/ 'brokenLink').is_file()) + self.assertFalse((P / 'fileA\udfff').is_file()) + self.assertFalse((P / 'fileA\x00').is_file()) + + def test_is_file_no_follow_symlinks(self): + P = self.cls(BASE) + self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) + + def test_is_mount(self): + P = self.cls(BASE) + self.assertFalse((P / 'fileA').is_mount()) + self.assertFalse((P / 'dirA').is_mount()) + self.assertFalse((P / 'non-existing').is_mount()) + self.assertFalse((P / 'fileA' / 'bah').is_mount()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_mount()) + + def test_is_symlink(self): + P = self.cls(BASE) + self.assertFalse((P / 'fileA').is_symlink()) + self.assertFalse((P / 'dirA').is_symlink()) + self.assertFalse((P / 'non-existing').is_symlink()) + self.assertFalse((P / 'fileA' / 'bah').is_symlink()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_symlink()) + self.assertTrue((P / 'linkB').is_symlink()) + self.assertTrue((P/ 'brokenLink').is_symlink()) + self.assertIs((P / 'fileA\udfff').is_file(), False) + self.assertIs((P / 'fileA\x00').is_file(), False) + if self.can_symlink: + self.assertIs((P / 'linkA\udfff').is_file(), False) + self.assertIs((P / 'linkA\x00').is_file(), False) + + def test_is_junction_false(self): + P = self.cls(BASE) + self.assertFalse((P / 'fileA').is_junction()) + self.assertFalse((P / 'dirA').is_junction()) + self.assertFalse((P / 'non-existing').is_junction()) + self.assertFalse((P / 'fileA' / 'bah').is_junction()) + self.assertFalse((P / 'fileA\udfff').is_junction()) + self.assertFalse((P / 'fileA\x00').is_junction()) + + def test_is_fifo_false(self): + P = self.cls(BASE) + self.assertFalse((P / 'fileA').is_fifo()) + self.assertFalse((P / 'dirA').is_fifo()) + self.assertFalse((P / 'non-existing').is_fifo()) + self.assertFalse((P / 'fileA' / 'bah').is_fifo()) + self.assertIs((P / 'fileA\udfff').is_fifo(), False) + self.assertIs((P / 'fileA\x00').is_fifo(), False) + + def test_is_socket_false(self): + P = self.cls(BASE) + self.assertFalse((P / 'fileA').is_socket()) + self.assertFalse((P / 'dirA').is_socket()) + self.assertFalse((P / 'non-existing').is_socket()) + self.assertFalse((P / 'fileA' / 'bah').is_socket()) + self.assertIs((P / 'fileA\udfff').is_socket(), False) + self.assertIs((P / 'fileA\x00').is_socket(), False) + + def test_is_block_device_false(self): + P = self.cls(BASE) + self.assertFalse((P / 'fileA').is_block_device()) + self.assertFalse((P / 'dirA').is_block_device()) + self.assertFalse((P / 'non-existing').is_block_device()) + self.assertFalse((P / 'fileA' / 'bah').is_block_device()) + self.assertIs((P / 'fileA\udfff').is_block_device(), False) + self.assertIs((P / 'fileA\x00').is_block_device(), False) + + def test_is_char_device_false(self): + P = self.cls(BASE) + self.assertFalse((P / 'fileA').is_char_device()) + self.assertFalse((P / 'dirA').is_char_device()) + self.assertFalse((P / 'non-existing').is_char_device()) + self.assertFalse((P / 'fileA' / 'bah').is_char_device()) + self.assertIs((P / 'fileA\udfff').is_char_device(), False) + self.assertIs((P / 'fileA\x00').is_char_device(), False) + + def test_pickling_common(self): + p = self.cls(BASE, 'fileA') + for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): + dumped = pickle.dumps(p, proto) + pp = pickle.loads(dumped) + self.assertEqual(pp.stat(), p.stat()) + + def test_parts_interning(self): + P = self.cls + p = P('/usr/bin/foo') + q = P('/usr/local/bin') + # 'usr' + self.assertIs(p.parts[1], q.parts[1]) + # 'bin' + self.assertIs(p.parts[2], q.parts[3]) + + def _check_complex_symlinks(self, link0_target): + if not self.can_symlink: + self.skipTest("symlinks required") + + # Test solving a non-looping chain of symlinks (issue #19887). + P = self.cls(BASE) + P.joinpath('link1').symlink_to(os.path.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(os.path.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(os.path.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) + + # Resolve absolute paths. + p = (P / 'link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + p = (P / 'link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + p = (P / 'link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + p = (P / 'link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + + # Resolve relative paths. + try: + self.cls().absolute() + except pathlib.UnsupportedOperation: + return + old_path = os.getcwd() + os.chdir(BASE) + try: + p = self.cls('link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + p = self.cls('link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + p = self.cls('link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + p = self.cls('link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), BASE) + finally: + os.chdir(old_path) + + def test_complex_symlinks_absolute(self): + self._check_complex_symlinks(BASE) + + def test_complex_symlinks_relative(self): + self._check_complex_symlinks('.') + + def test_complex_symlinks_relative_dot_dot(self): + self._check_complex_symlinks(os.path.join('dirA', '..')) + + def setUpWalk(self): + # Build: + # TESTFN/ + # TEST1/ a file kid and two directory kids + # tmp1 + # SUB1/ a file kid and a directory kid + # tmp2 + # SUB11/ no kids + # SUB2/ a file kid and a dirsymlink kid + # tmp3 + # link/ a symlink to TEST2 + # broken_link + # broken_link2 + # TEST2/ + # tmp4 a lone file + self.walk_path = self.cls(BASE, "TEST1") + self.sub1_path = self.walk_path / "SUB1" + self.sub11_path = self.sub1_path / "SUB11" + self.sub2_path = self.walk_path / "SUB2" + tmp1_path = self.walk_path / "tmp1" + tmp2_path = self.sub1_path / "tmp2" + tmp3_path = self.sub2_path / "tmp3" + self.link_path = self.sub2_path / "link" + t2_path = self.cls(BASE, "TEST2") + tmp4_path = self.cls(BASE, "TEST2", "tmp4") + broken_link_path = self.sub2_path / "broken_link" + broken_link2_path = self.sub2_path / "broken_link2" + + self.sub11_path.mkdir(parents=True) + self.sub2_path.mkdir(parents=True) + t2_path.mkdir(parents=True) + + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: + with path.open("w", encoding='utf-8') as f: + f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + + if self.can_symlink: + self.link_path.symlink_to(t2_path) + broken_link_path.symlink_to('broken') + broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) + self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) + else: + self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + + def test_walk_topdown(self): + self.setUpWalk() + walker = self.walk_path.walk() + entry = next(walker) + entry[1].sort() # Ensure we visit SUB1 before SUB2 + self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"])) + entry = next(walker) + self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"])) + entry = next(walker) + self.assertEqual(entry, (self.sub11_path, [], [])) + entry = next(walker) + entry[1].sort() + entry[2].sort() + self.assertEqual(entry, self.sub2_tree) + with self.assertRaises(StopIteration): + next(walker) + + def test_walk_prune(self): + self.setUpWalk() + # Prune the search. + all = [] + for root, dirs, files in self.walk_path.walk(): + all.append((root, dirs, files)) + if 'SUB1' in dirs: + # Note that this also mutates the dirs we appended to all! + dirs.remove('SUB1') + + self.assertEqual(len(all), 2) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) + + all[1][-1].sort() + all[1][1].sort() + self.assertEqual(all[1], self.sub2_tree) + + def test_walk_bottom_up(self): + self.setUpWalk() + seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False + for path, dirnames, filenames in self.walk_path.walk(top_down=False): + if path == self.walk_path: + self.assertFalse(seen_testfn) + self.assertTrue(seen_sub1) + self.assertTrue(seen_sub2) + self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"]) + self.assertEqual(filenames, ["tmp1"]) + seen_testfn = True + elif path == self.sub1_path: + self.assertFalse(seen_testfn) + self.assertFalse(seen_sub1) + self.assertTrue(seen_sub11) + self.assertEqual(dirnames, ["SUB11"]) + self.assertEqual(filenames, ["tmp2"]) + seen_sub1 = True + elif path == self.sub11_path: + self.assertFalse(seen_sub1) + self.assertFalse(seen_sub11) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, []) + seen_sub11 = True + elif path == self.sub2_path: + self.assertFalse(seen_testfn) + self.assertFalse(seen_sub2) + self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1])) + self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2])) + seen_sub2 = True + else: + raise AssertionError(f"Unexpected path: {path}") + self.assertTrue(seen_testfn) + + def test_walk_follow_symlinks(self): + if not self.can_symlink: + self.skipTest("symlinks required") + self.setUpWalk() + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with follow_symlinks=True") + + def test_walk_symlink_location(self): + if not self.can_symlink: + self.skipTest("symlinks required") + self.setUpWalk() + # Tests whether symlinks end up in filenames or dirnames depending + # on the `follow_symlinks` argument. + walk_it = self.walk_path.walk(follow_symlinks=False) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", files) + break + else: + self.fail("symlink not found") + + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", dirs) + break + else: + self.fail("symlink not found") + + def test_walk_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(BASE, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with set_recursion_limit(recursion_limit): + list(base.walk()) + list(base.walk(top_down=False)) + +class DummyPathWithSymlinks(DummyPath): + def readlink(self): + path = str(self.parent.resolve() / self.name) + if path in self._symlinks: + return self.with_segments(self._symlinks[path]) + elif path in self._files or path in self._directories: + raise OSError(errno.EINVAL, "Not a symlink", path) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def symlink_to(self, target, target_is_directory=False): + self._directories[str(self.parent)].add(self.name) + self._symlinks[str(self)] = str(target) + + +class DummyPathWithSymlinksTest(DummyPathTest): + cls = DummyPathWithSymlinks + can_symlink = True + + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 5fb6ffc4e8f0cf..195fc0ddddecd3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2272,6 +2272,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/source \ test/test_json \ test/test_module \ + test/test_pathlib \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tkinter \ From 2f0ec7fa9450caeac820a6dc819d17d14fd16a4b Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 17 Dec 2023 00:07:32 +0000 Subject: [PATCH 95/95] GH-110109: pathlib tests: store base directory as test class attribute (#113221) Store the test base directory as a class attribute named `base` rather than module constants named `BASE`. The base directory is a local file path, and therefore not ideally suited to the pathlib ABC tests. In a future commit we'll change its value in `test_pathlib_abc.py` such that it points to a totally fictitious path, which will help to ensure we're not touching the local filesystem. --- Lib/test/test_pathlib/test_pathlib.py | 167 +++++++------- Lib/test/test_pathlib/test_pathlib_abc.py | 265 +++++++++++----------- 2 files changed, 210 insertions(+), 222 deletions(-) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 304786fc6f7fd9..00cfdd37e568a6 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -23,11 +23,6 @@ grp = pwd = None -# Make sure any symbolic links in the base test path are resolved. -BASE = os.path.realpath(TESTFN) -join = lambda *x: os.path.join(BASE, *x) -rel_join = lambda *x: os.path.join(TESTFN, *x) - only_nt = unittest.skipIf(os.name != 'nt', 'test requires a Windows-compatible system') only_posix = unittest.skipIf(os.name == 'nt', @@ -937,11 +932,11 @@ class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): def setUp(self): super().setUp() - os.chmod(join('dirE'), 0) + os.chmod(self.pathmod.join(self.base, 'dirE'), 0) def tearDown(self): - os.chmod(join('dirE'), 0o777) - os_helper.rmtree(BASE) + os.chmod(self.pathmod.join(self.base, 'dirE'), 0o777) + os_helper.rmtree(self.base) def tempdir(self): d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', @@ -978,23 +973,23 @@ def test_absolute_common(self): P = self.cls with mock.patch("os.getcwd") as getcwd: - getcwd.return_value = BASE + getcwd.return_value = self.base # Simple relative paths. - self.assertEqual(str(P().absolute()), BASE) - self.assertEqual(str(P('.').absolute()), BASE) - self.assertEqual(str(P('a').absolute()), os.path.join(BASE, 'a')) - self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(BASE, 'a', 'b', 'c')) + self.assertEqual(str(P().absolute()), self.base) + self.assertEqual(str(P('.').absolute()), self.base) + self.assertEqual(str(P('a').absolute()), os.path.join(self.base, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(self.base, 'a', 'b', 'c')) # Symlinks should not be resolved. - self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(BASE, 'linkB', 'fileB')) - self.assertEqual(str(P('brokenLink').absolute()), os.path.join(BASE, 'brokenLink')) - self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(BASE, 'brokenLinkLoop')) + self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(self.base, 'linkB', 'fileB')) + self.assertEqual(str(P('brokenLink').absolute()), os.path.join(self.base, 'brokenLink')) + self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(self.base, 'brokenLinkLoop')) # '..' entries should be preserved and not normalised. - self.assertEqual(str(P('..').absolute()), os.path.join(BASE, '..')) - self.assertEqual(str(P('a', '..').absolute()), os.path.join(BASE, 'a', '..')) - self.assertEqual(str(P('..', 'b').absolute()), os.path.join(BASE, '..', 'b')) + self.assertEqual(str(P('..').absolute()), os.path.join(self.base, '..')) + self.assertEqual(str(P('a', '..').absolute()), os.path.join(self.base, 'a', '..')) + self.assertEqual(str(P('..', 'b').absolute()), os.path.join(self.base, '..', 'b')) def _test_home(self, p): q = self.cls(os.path.expanduser('~')) @@ -1011,11 +1006,11 @@ def test_home(self): self._test_home(self.cls.home()) env.clear() - env['USERPROFILE'] = os.path.join(BASE, 'userprofile') + env['USERPROFILE'] = os.path.join(self.base, 'userprofile') self._test_home(self.cls.home()) # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = os.path.join(BASE, 'home') + env['HOME'] = os.path.join(self.base, 'home') self._test_home(self.cls.home()) @unittest.skipIf(is_wasi, "WASI has no user accounts.") @@ -1042,7 +1037,7 @@ def __init__(self, *pathsegments, session_id): def with_segments(self, *pathsegments): return type(self)(*pathsegments, session_id=self.session_id) - p = P(BASE, session_id=42) + p = P(self.base, session_id=42) self.assertEqual(42, p.absolute().session_id) self.assertEqual(42, p.resolve().session_id) if not is_wasi: # WASI has no user accounts. @@ -1061,7 +1056,7 @@ def with_segments(self, *pathsegments): self.assertEqual(42, dirpath.session_id) def test_open_unbuffered(self): - p = self.cls(BASE) + p = self.cls(self.base) with (p / 'fileA').open('rb', buffering=0) as f: self.assertIsInstance(f, io.RawIOBase) self.assertEqual(f.read().strip(), b"this is file A") @@ -1070,15 +1065,15 @@ def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') old_cwd = os.getcwd() - os.chdir(BASE) + os.chdir(self.base) try: - self.assertEqual(p.resolve(), self.cls(BASE, p)) + self.assertEqual(p.resolve(), self.cls(self.base, p)) finally: os.chdir(old_cwd) @os_helper.skip_unless_working_chmod def test_chmod(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' mode = p.stat().st_mode # Clear writable bit. new_mode = mode & ~0o222 @@ -1093,7 +1088,7 @@ def test_chmod(self): @only_posix @os_helper.skip_unless_working_chmod def test_chmod_follow_symlinks_true(self): - p = self.cls(BASE) / 'linkA' + p = self.cls(self.base) / 'linkA' q = p.resolve() mode = q.stat().st_mode # Clear writable bit. @@ -1116,7 +1111,7 @@ def _get_pw_name_or_skip_test(self, uid): @unittest.skipUnless(pwd, "the pwd module is needed for this test") def test_owner(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' expected_uid = p.stat().st_uid expected_name = self._get_pw_name_or_skip_test(expected_uid) @@ -1129,8 +1124,8 @@ def test_owner_no_follow_symlinks(self): if len(all_users) < 2: self.skipTest("test needs more than one user") - target = self.cls(BASE) / 'fileA' - link = self.cls(BASE) / 'linkA' + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' uid_1, uid_2 = all_users[:2] os.chown(target, uid_1, -1) @@ -1151,7 +1146,7 @@ def _get_gr_name_or_skip_test(self, gid): @unittest.skipUnless(grp, "the grp module is needed for this test") def test_group(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' expected_gid = p.stat().st_gid expected_name = self._get_gr_name_or_skip_test(expected_gid) @@ -1164,8 +1159,8 @@ def test_group_no_follow_symlinks(self): if len(all_groups) < 2: self.skipTest("test needs more than one group") - target = self.cls(BASE) / 'fileA' - link = self.cls(BASE) / 'linkA' + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' gid_1, gid_2 = all_groups[:2] os.chown(target, -1, gid_1) @@ -1178,18 +1173,18 @@ def test_group_no_follow_symlinks(self): self.assertEqual(expected_name, link.group(follow_symlinks=False)) def test_unlink(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' p.unlink() self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) def test_unlink_missing_ok(self): - p = self.cls(BASE) / 'fileAAA' + p = self.cls(self.base) / 'fileAAA' self.assertFileNotFound(p.unlink) p.unlink(missing_ok=True) def test_rmdir(self): - p = self.cls(BASE) / 'dirA' + p = self.cls(self.base) / 'dirA' for q in p.iterdir(): q.unlink() p.rmdir() @@ -1198,7 +1193,7 @@ def test_rmdir(self): @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") def test_hardlink_to(self): - P = self.cls(BASE) + P = self.cls(self.base) target = P / 'fileA' size = target.stat().st_size # linking to another path. @@ -1209,14 +1204,14 @@ def test_hardlink_to(self): self.assertTrue(target.exists()) # Linking to a str of a relative path. link2 = P / 'dirA' / 'fileAAA' - target2 = rel_join('fileA') + target2 = self.pathmod.join(TESTFN, 'fileA') link2.hardlink_to(target2) self.assertEqual(os.stat(target2).st_size, size) self.assertTrue(link2.exists()) @unittest.skipIf(hasattr(os, "link"), "os.link() is present") def test_hardlink_to_unsupported(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' # linking to another path. q = P / 'dirA' / 'fileAA' @@ -1224,7 +1219,7 @@ def test_hardlink_to_unsupported(self): q.hardlink_to(p) def test_rename(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' size = p.stat().st_size # Renaming to another path. @@ -1234,14 +1229,14 @@ def test_rename(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Renaming to a str of a relative path. - r = rel_join('fileAAA') + r = self.pathmod.join(TESTFN, 'fileAAA') renamed_q = q.rename(r) self.assertEqual(renamed_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) self.assertFileNotFound(q.stat) def test_replace(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' size = p.stat().st_size # Replacing a non-existing path. @@ -1251,14 +1246,14 @@ def test_replace(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Replacing another (existing) path. - r = rel_join('dirB', 'fileB') + r = self.pathmod.join(TESTFN, 'dirB', 'fileB') replaced_q = q.replace(r) self.assertEqual(replaced_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) self.assertFileNotFound(q.stat) def test_touch_common(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'newfileA' self.assertFalse(p.exists()) p.touch() @@ -1282,14 +1277,14 @@ def test_touch_common(self): self.assertRaises(OSError, p.touch, exist_ok=False) def test_touch_nochange(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' p.touch() with p.open('rb') as f: self.assertEqual(f.read().strip(), b"this is file A") def test_mkdir(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'newdirA' self.assertFalse(p.exists()) p.mkdir() @@ -1301,7 +1296,7 @@ def test_mkdir(self): def test_mkdir_parents(self): # Creating a chain of directories. - p = self.cls(BASE, 'newdirB', 'newdirC') + p = self.cls(self.base, 'newdirB', 'newdirC') self.assertFalse(p.exists()) with self.assertRaises(OSError) as cm: p.mkdir() @@ -1314,7 +1309,7 @@ def test_mkdir_parents(self): self.assertEqual(cm.exception.errno, errno.EEXIST) # Test `mode` arg. mode = stat.S_IMODE(p.stat().st_mode) # Default mode. - p = self.cls(BASE, 'newdirD', 'newdirE') + p = self.cls(self.base, 'newdirD', 'newdirE') p.mkdir(0o555, parents=True) self.assertTrue(p.exists()) self.assertTrue(p.is_dir()) @@ -1325,7 +1320,7 @@ def test_mkdir_parents(self): self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) def test_mkdir_exist_ok(self): - p = self.cls(BASE, 'dirB') + p = self.cls(self.base, 'dirB') st_ctime_first = p.stat().st_ctime self.assertTrue(p.exists()) self.assertTrue(p.is_dir()) @@ -1337,7 +1332,7 @@ def test_mkdir_exist_ok(self): self.assertEqual(p.stat().st_ctime, st_ctime_first) def test_mkdir_exist_ok_with_parent(self): - p = self.cls(BASE, 'dirC') + p = self.cls(self.base, 'dirC') self.assertTrue(p.exists()) with self.assertRaises(FileExistsError) as cm: p.mkdir() @@ -1371,7 +1366,7 @@ def test_mkdir_with_unknown_drive(self): (p / 'child' / 'path').mkdir(parents=True) def test_mkdir_with_child_file(self): - p = self.cls(BASE, 'dirB', 'fileB') + p = self.cls(self.base, 'dirB', 'fileB') self.assertTrue(p.exists()) # An exception is raised when the last path component is an existing # regular file, regardless of whether exist_ok is true or not. @@ -1383,7 +1378,7 @@ def test_mkdir_with_child_file(self): self.assertEqual(cm.exception.errno, errno.EEXIST) def test_mkdir_no_parents_file(self): - p = self.cls(BASE, 'fileA') + p = self.cls(self.base, 'fileA') self.assertTrue(p.exists()) # An exception is raised when the last path component is an existing # regular file, regardless of whether exist_ok is true or not. @@ -1396,7 +1391,7 @@ def test_mkdir_no_parents_file(self): def test_mkdir_concurrent_parent_creation(self): for pattern_num in range(32): - p = self.cls(BASE, 'dirCPC%d' % pattern_num) + p = self.cls(self.base, 'dirCPC%d' % pattern_num) self.assertFalse(p.exists()) real_mkdir = os.mkdir @@ -1427,7 +1422,7 @@ def my_mkdir(path, mode=0o777): def test_symlink_to(self): if not self.can_symlink: self.skipTest("symlinks required") - P = self.cls(BASE) + P = self.cls(self.base) target = P / 'fileA' # Symlinking a path target. link = P / 'dirA' / 'linkAA' @@ -1451,7 +1446,7 @@ def test_symlink_to(self): @unittest.skipIf(hasattr(os, "symlink"), "os.symlink() is present") def test_symlink_to_unsupported(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' # linking to another path. q = P / 'dirA' / 'fileAA' @@ -1459,7 +1454,7 @@ def test_symlink_to_unsupported(self): q.symlink_to(p) def test_is_junction(self): - P = self.cls(BASE) + P = self.cls(self.base) with mock.patch.object(P.pathmod, 'isjunction'): self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) @@ -1469,7 +1464,7 @@ def test_is_junction(self): @unittest.skipIf(sys.platform == "vxworks", "fifo requires special path on VxWorks") def test_is_fifo_true(self): - P = self.cls(BASE, 'myfifo') + P = self.cls(self.base, 'myfifo') try: os.mkfifo(str(P)) except PermissionError as e: @@ -1477,8 +1472,8 @@ def test_is_fifo_true(self): self.assertTrue(P.is_fifo()) self.assertFalse(P.is_socket()) self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'myfifo\udfff').is_fifo(), False) - self.assertIs(self.cls(BASE, 'myfifo\x00').is_fifo(), False) + self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) + self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") @unittest.skipIf( @@ -1488,7 +1483,7 @@ def test_is_fifo_true(self): is_wasi, "Cannot create socket on WASI." ) def test_is_socket_true(self): - P = self.cls(BASE, 'mysock') + P = self.cls(self.base, 'mysock') sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.addCleanup(sock.close) try: @@ -1500,8 +1495,8 @@ def test_is_socket_true(self): self.assertTrue(P.is_socket()) self.assertFalse(P.is_fifo()) self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'mysock\udfff').is_socket(), False) - self.assertIs(self.cls(BASE, 'mysock\x00').is_socket(), False) + self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) + self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) def test_is_char_device_true(self): # Under Unix, /dev/null should generally be a char device. @@ -1573,7 +1568,7 @@ def test_walk_bad_dir(self): def test_walk_many_open_files(self): depth = 30 - base = self.cls(BASE, 'deep') + base = self.cls(self.base, 'deep') path = self.cls(base, *(['d']*depth)) path.mkdir(parents=True) @@ -1615,15 +1610,15 @@ def test_absolute(self): def test_open_mode(self): old_mask = os.umask(0) self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) + p = self.cls(self.base) with (p / 'new_file').open('wb'): pass - st = os.stat(join('new_file')) + st = os.stat(self.pathmod.join(self.base, 'new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) os.umask(0o022) with (p / 'other_new_file').open('wb'): pass - st = os.stat(join('other_new_file')) + st = os.stat(self.pathmod.join(self.base, 'other_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) def test_resolve_root(self): @@ -1642,31 +1637,31 @@ def test_resolve_root(self): def test_touch_mode(self): old_mask = os.umask(0) self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) + p = self.cls(self.base) (p / 'new_file').touch() - st = os.stat(join('new_file')) + st = os.stat(self.pathmod.join(self.base, 'new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) os.umask(0o022) (p / 'other_new_file').touch() - st = os.stat(join('other_new_file')) + st = os.stat(self.pathmod.join(self.base, 'other_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(join('masked_new_file')) + st = os.stat(self.pathmod.join(self.base, 'masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) def test_glob(self): P = self.cls - p = P(BASE) + p = P(self.base) given = set(p.glob("FILEa")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given + expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given self.assertEqual(given, expect) self.assertEqual(set(p.glob("FILEa*")), set()) def test_rglob(self): P = self.cls - p = P(BASE, "dirC") + p = P(self.base, "dirC") given = set(p.rglob("FILEd")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given + expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given self.assertEqual(given, expect) self.assertEqual(set(p.rglob("FILEd*")), set()) @@ -1797,17 +1792,17 @@ def test_absolute(self): self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(share, 'a', 'b', 'c')) - drive = os.path.splitdrive(BASE)[0] - with os_helper.change_cwd(BASE): + drive = os.path.splitdrive(self.base)[0] + with os_helper.change_cwd(self.base): # Relative path with root self.assertEqual(str(P('\\').absolute()), drive + '\\') self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') # Relative path on current drive - self.assertEqual(str(P(drive).absolute()), BASE) - self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(BASE, 'foo')) + self.assertEqual(str(P(drive).absolute()), self.base) + self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(self.base, 'foo')) - with os_helper.subst_drive(BASE) as other_drive: + with os_helper.subst_drive(self.base) as other_drive: # Set the working directory on the substitute drive saved_cwd = os.getcwd() other_cwd = f'{other_drive}\\dirA' @@ -1820,18 +1815,18 @@ def test_absolute(self): def test_glob(self): P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) - self.assertEqual(set(p.glob("*a\\")), { P(BASE, "dirA/") }) - self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) + p = P(self.base) + self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) + self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) + self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) def test_rglob(self): P = self.cls - p = P(BASE, "dirC") - self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) - self.assertEqual(set(p.rglob("*\\")), { P(BASE, "dirC/dirD/") }) + p = P(self.base, "dirC") + self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) + self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) def test_expanduser(self): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 61ed3cb6a4a7f8..a272973d9c1d61 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -18,16 +18,6 @@ def test_is_notimplemented(self): self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) -# Make sure any symbolic links in the base test path are resolved. -BASE = os.path.realpath(TESTFN) -join = lambda *x: os.path.join(BASE, *x) - -only_nt = unittest.skipIf(os.name != 'nt', - 'test requires a Windows-compatible system') -only_posix = unittest.skipIf(os.name == 'nt', - 'test requires a POSIX-compatible system') - - # # Tests for the pure classes. # @@ -62,6 +52,9 @@ def __hash__(self): class DummyPurePathTest(unittest.TestCase): cls = DummyPurePath + # Make sure any symbolic links in the base test path are resolved. + base = os.path.realpath(TESTFN) + # Keys are canonical paths, values are list of tuples of arguments # supposed to produce equal paths. equivalences = { @@ -859,7 +852,7 @@ class DummyPathTest(DummyPurePathTest): cls = DummyPath can_symlink = False - # (BASE) + # (self.base) # | # |-- brokenLink -> non-existing # |-- dirA @@ -882,7 +875,7 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() pathmod = self.cls.pathmod - p = self.cls(BASE) + p = self.cls(self.base) p.mkdir(parents=True) p.joinpath('dirA').mkdir() p.joinpath('dirB').mkdir() @@ -914,7 +907,7 @@ def tearDown(self): cls._symlinks.clear() def tempdir(self): - path = self.cls(BASE).with_name('tmp-dirD') + path = self.cls(self.base).with_name('tmp-dirD') path.mkdir() return path @@ -927,8 +920,8 @@ def assertEqualNormCase(self, path_a, path_b): self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) def test_samefile(self): - fileA_path = os.path.join(BASE, 'fileA') - fileB_path = os.path.join(BASE, 'dirB', 'fileB') + fileA_path = os.path.join(self.base, 'fileA') + fileB_path = os.path.join(self.base, 'dirB', 'fileB') p = self.cls(fileA_path) pp = self.cls(fileA_path) q = self.cls(fileB_path) @@ -937,7 +930,7 @@ def test_samefile(self): self.assertFalse(p.samefile(fileB_path)) self.assertFalse(p.samefile(q)) # Test the non-existent file case - non_existent = os.path.join(BASE, 'foo') + non_existent = os.path.join(self.base, 'foo') r = self.cls(non_existent) self.assertRaises(FileNotFoundError, p.samefile, r) self.assertRaises(FileNotFoundError, p.samefile, non_existent) @@ -953,7 +946,7 @@ def test_empty_path(self): def test_exists(self): P = self.cls - p = P(BASE) + p = P(self.base) self.assertIs(True, p.exists()) self.assertIs(True, (p / 'dirA').exists()) self.assertIs(True, (p / 'fileA').exists()) @@ -967,11 +960,11 @@ def test_exists(self): self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) self.assertIs(False, (p / 'foo').exists()) self.assertIs(False, P('/xyzzy').exists()) - self.assertIs(False, P(BASE + '\udfff').exists()) - self.assertIs(False, P(BASE + '\x00').exists()) + self.assertIs(False, P(self.base + '\udfff').exists()) + self.assertIs(False, P(self.base + '\x00').exists()) def test_open_common(self): - p = self.cls(BASE) + p = self.cls(self.base) with (p / 'fileA').open('r') as f: self.assertIsInstance(f, io.TextIOBase) self.assertEqual(f.read(), "this is file A\n") @@ -980,7 +973,7 @@ def test_open_common(self): self.assertEqual(f.read().strip(), b"this is file A") def test_read_write_bytes(self): - p = self.cls(BASE) + p = self.cls(self.base) (p / 'fileA').write_bytes(b'abcdefg') self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') # Check that trying to write str does not truncate the file. @@ -988,7 +981,7 @@ def test_read_write_bytes(self): self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') def test_read_write_text(self): - p = self.cls(BASE) + p = self.cls(self.base) (p / 'fileA').write_text('äbcdefg', encoding='latin-1') self.assertEqual((p / 'fileA').read_text( encoding='utf-8', errors='ignore'), 'bcdefg') @@ -997,7 +990,7 @@ def test_read_write_text(self): self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') def test_read_text_with_newlines(self): - p = self.cls(BASE) + p = self.cls(self.base) # Check that `\n` character change nothing (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') self.assertEqual((p / 'fileA').read_text(newline='\n'), @@ -1012,7 +1005,7 @@ def test_read_text_with_newlines(self): 'abcde\r\nfghlk\n\rmnopq') def test_write_text_with_newlines(self): - p = self.cls(BASE) + p = self.cls(self.base) # Check that `\n` character change nothing (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') self.assertEqual((p / 'fileA').read_bytes(), @@ -1033,27 +1026,27 @@ def test_write_text_with_newlines(self): def test_iterdir(self): P = self.cls - p = P(BASE) + p = P(self.base) it = p.iterdir() paths = set(it) expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] if self.can_symlink: expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] - self.assertEqual(paths, { P(BASE, q) for q in expected }) + self.assertEqual(paths, { P(self.base, q) for q in expected }) def test_iterdir_symlink(self): if not self.can_symlink: self.skipTest("symlinks required") # __iter__ on a symlink to a directory. P = self.cls - p = P(BASE, 'linkB') + p = P(self.base, 'linkB') paths = set(p.iterdir()) - expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] } + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } self.assertEqual(paths, expected) def test_iterdir_nodir(self): # __iter__ on something that is not a directory. - p = self.cls(BASE, 'fileA') + p = self.cls(self.base, 'fileA') with self.assertRaises(OSError) as cm: p.iterdir() # ENOENT or EINVAL under Windows, ENOTDIR otherwise @@ -1063,9 +1056,9 @@ def test_iterdir_nodir(self): def test_glob_common(self): def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) + self.assertEqual(set(glob), { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) it = p.glob("fileA") self.assertIsInstance(it, collections.abc.Iterator) _check(it, ["fileA"]) @@ -1101,9 +1094,9 @@ def test_glob_case_sensitive(self): P = self.cls def _check(path, pattern, case_sensitive, expected): actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} - expected = {str(P(BASE, q)) for q in expected} + expected = {str(P(self.base, q)) for q in expected} self.assertEqual(actual, expected) - path = P(BASE) + path = P(self.base) _check(path, "DIRB/FILE*", True, []) _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) _check(path, "dirb/file*", True, []) @@ -1115,9 +1108,9 @@ def test_glob_follow_symlinks_common(self): def _check(path, glob, expected): actual = {path for path in path.glob(glob, follow_symlinks=True) if "linkD" not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) _check(p, "fileB", []) _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) _check(p, "*A", ["dirA", "fileA", "linkA"]) @@ -1140,9 +1133,9 @@ def test_glob_no_follow_symlinks_common(self): self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.glob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) _check(p, "fileB", []) _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) _check(p, "*A", ["dirA", "fileA", "linkA"]) @@ -1160,9 +1153,9 @@ def _check(path, glob, expected): def test_rglob_common(self): def _check(glob, expected): - self.assertEqual(set(glob), {P(BASE, q) for q in expected}) + self.assertEqual(set(glob), {P(self.base, q) for q in expected}) P = self.cls - p = P(BASE) + p = P(self.base) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) _check(it, ["fileA"]) @@ -1187,7 +1180,7 @@ def _check(glob, expected): ]) _check(p.rglob(""), ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) - p = P(BASE, "dirC") + p = P(self.base, "dirC") _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) @@ -1207,9 +1200,9 @@ def test_rglob_follow_symlinks_common(self): def _check(path, glob, expected): actual = {path for path in path.rglob(glob, follow_symlinks=True) if 'linkD' not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) _check(p, "*/fileA", []) _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) @@ -1220,7 +1213,7 @@ def _check(path, glob, expected): _check(p, "", ["./", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) - p = P(BASE, "dirC") + p = P(self.base, "dirC") _check(p, "*", ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) @@ -1236,9 +1229,9 @@ def test_rglob_no_follow_symlinks_common(self): self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.rglob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) _check(p, "fileB", ["dirB/fileB"]) _check(p, "*/fileA", []) _check(p, "*/fileB", ["dirB/fileB"]) @@ -1246,7 +1239,7 @@ def _check(path, glob, expected): _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "", ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) - p = P(BASE, "dirC") + p = P(self.base, "dirC") _check(p, "*", ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) @@ -1262,7 +1255,7 @@ def test_rglob_symlink_loop(self): if not self.can_symlink: self.skipTest("symlinks required") P = self.cls - p = P(BASE) + p = P(self.base) given = set(p.rglob('*')) expect = {'brokenLink', 'dirA', 'dirA/linkC', @@ -1280,7 +1273,7 @@ def test_rglob_symlink_loop(self): def test_glob_many_open_files(self): depth = 30 P = self.cls - p = base = P(BASE) / 'deep' + p = base = P(self.base) / 'deep' p.mkdir() for _ in range(depth): p /= 'd' @@ -1299,30 +1292,30 @@ def test_glob_many_open_files(self): def test_glob_dotdot(self): # ".." is not special in globs. P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) - self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) self.assertEqual(set(p.glob("dirA/../file*/..")), set()) self.assertEqual(set(p.glob("../xyzzy")), set()) self.assertEqual(set(p.glob("xyzzy/..")), set()) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) def test_glob_permissions(self): # See bpo-38894 if not self.can_symlink: self.skipTest("symlinks required") P = self.cls - base = P(BASE) / 'permissions' + base = P(self.base) / 'permissions' base.mkdir() for i in range(100): link = base / f"link{i}" if i % 2: - link.symlink_to(P(BASE, "dirE", "nonexistent")) + link.symlink_to(P(self.base, "dirE", "nonexistent")) else: - link.symlink_to(P(BASE, "dirC")) + link.symlink_to(P(self.base, "dirC")) self.assertEqual(len(set(base.glob("*"))), 100) self.assertEqual(len(set(base.glob("*/"))), 50) @@ -1333,7 +1326,7 @@ def test_glob_long_symlink(self): # See gh-87695 if not self.can_symlink: self.skipTest("symlinks required") - base = self.cls(BASE) / 'long_symlink' + base = self.cls(self.base) / 'long_symlink' base.mkdir() bad_link = base / 'bad_link' bad_link.symlink_to("bad" * 200) @@ -1343,7 +1336,7 @@ def test_glob_above_recursion_limit(self): recursion_limit = 50 # directory_depth > recursion_limit directory_depth = recursion_limit + 10 - base = self.cls(BASE, 'deep') + base = self.cls(self.base, 'deep') path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) @@ -1352,7 +1345,7 @@ def test_glob_above_recursion_limit(self): def test_glob_recursive_no_trailing_slash(self): P = self.cls - p = P(BASE) + p = P(self.base) with self.assertWarns(FutureWarning): p.glob('**') with self.assertWarns(FutureWarning): @@ -1366,7 +1359,7 @@ def test_glob_recursive_no_trailing_slash(self): def test_readlink(self): if not self.can_symlink: self.skipTest("symlinks required") - P = self.cls(BASE) + P = self.cls(self.base) self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) self.assertEqual((P / 'brokenLink').readlink(), self.cls('non-existing')) @@ -1377,7 +1370,7 @@ def test_readlink(self): @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") def test_readlink_unsupported(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' with self.assertRaises(pathlib.UnsupportedOperation): q.readlink(p) @@ -1393,53 +1386,53 @@ def test_resolve_common(self): if not self.can_symlink: self.skipTest("symlinks required") P = self.cls - p = P(BASE, 'foo') + p = P(self.base, 'foo') with self.assertRaises(OSError) as cm: p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) # Non-strict self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) - p = P(BASE, 'foo', 'in', 'spam') + os.path.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo', 'in', 'spam')) - p = P(BASE, '..', 'foo', 'in', 'spam') + os.path.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), os.path.abspath(os.path.join('foo', 'in', 'spam'))) # These are all relative symlinks. - p = P(BASE, 'dirB', 'fileB') + p = P(self.base, 'dirB', 'fileB') self._check_resolve_relative(p, p) - p = P(BASE, 'linkA') - self._check_resolve_relative(p, P(BASE, 'fileA')) - p = P(BASE, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - p = P(BASE, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) # Non-strict - p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) - p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') if os.name == 'nt' and isinstance(p, pathlib.Path): # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) # Now create absolute symlinks. d = self.tempdir() - P(BASE, 'dirA', 'linkX').symlink_to(d) - P(BASE, str(d), 'linkY').symlink_to(join('dirB')) - p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.pathmod.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) # Non-strict - p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), False) - p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') if os.name == 'nt' and isinstance(p, pathlib.Path): # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. @@ -1447,13 +1440,13 @@ def test_resolve_common(self): else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) def test_resolve_dot(self): # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ if not self.can_symlink: self.skipTest("symlinks required") - p = self.cls(BASE) + p = self.cls(self.base) p.joinpath('0').symlink_to('.', target_is_directory=True) p.joinpath('1').symlink_to(os.path.join('0', '0'), target_is_directory=True) p.joinpath('2').symlink_to(os.path.join('1', '1'), target_is_directory=True) @@ -1476,30 +1469,30 @@ def test_resolve_loop(self): if os.name == 'nt' and issubclass(self.cls, pathlib.Path): self.skipTest("symlink loops work differently with concrete Windows paths") # Loops with relative symlinks. - self.cls(BASE, 'linkX').symlink_to('linkX/inside') - self._check_symlink_loop(BASE, 'linkX') - self.cls(BASE, 'linkY').symlink_to('linkY') - self._check_symlink_loop(BASE, 'linkY') - self.cls(BASE, 'linkZ').symlink_to('linkZ/../linkZ') - self._check_symlink_loop(BASE, 'linkZ') + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') # Non-strict - p = self.cls(BASE, 'linkZ', 'foo') + p = self.cls(self.base, 'linkZ', 'foo') self.assertEqual(p.resolve(strict=False), p) # Loops with absolute symlinks. - self.cls(BASE, 'linkU').symlink_to(join('linkU/inside')) - self._check_symlink_loop(BASE, 'linkU') - self.cls(BASE, 'linkV').symlink_to(join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - self.cls(BASE, 'linkW').symlink_to(join('linkW/../linkW')) - self._check_symlink_loop(BASE, 'linkW') + self.cls(self.base, 'linkU').symlink_to(self.pathmod.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.pathmod.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.pathmod.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') # Non-strict - q = self.cls(BASE, 'linkW', 'foo') + q = self.cls(self.base, 'linkW', 'foo') self.assertEqual(q.resolve(strict=False), q) def test_stat(self): - statA = self.cls(BASE).joinpath('fileA').stat() - statB = self.cls(BASE).joinpath('dirB', 'fileB').stat() - statC = self.cls(BASE).joinpath('dirC').stat() + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() # st_mode: files are the same, directory differs. self.assertIsInstance(statA.st_mode, int) self.assertEqual(statA.st_mode, statB.st_mode) @@ -1519,29 +1512,29 @@ def test_stat(self): def test_stat_no_follow_symlinks(self): if not self.can_symlink: self.skipTest("symlinks required") - p = self.cls(BASE) / 'linkA' + p = self.cls(self.base) / 'linkA' st = p.stat() self.assertNotEqual(st, p.stat(follow_symlinks=False)) def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' st = p.stat() self.assertEqual(st, p.stat(follow_symlinks=False)) def test_lstat(self): if not self.can_symlink: self.skipTest("symlinks required") - p = self.cls(BASE)/ 'linkA' + p = self.cls(self.base)/ 'linkA' st = p.stat() self.assertNotEqual(st, p.lstat()) def test_lstat_nosymlink(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' st = p.stat() self.assertEqual(st, p.lstat()) def test_is_dir(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir()) self.assertFalse((P / 'fileA').is_dir()) self.assertFalse((P / 'non-existing').is_dir()) @@ -1554,7 +1547,7 @@ def test_is_dir(self): self.assertFalse((P / 'dirA\x00').is_dir()) def test_is_dir_no_follow_symlinks(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) @@ -1567,7 +1560,7 @@ def test_is_dir_no_follow_symlinks(self): self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) def test_is_file(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'fileA').is_file()) self.assertFalse((P / 'dirA').is_file()) self.assertFalse((P / 'non-existing').is_file()) @@ -1580,7 +1573,7 @@ def test_is_file(self): self.assertFalse((P / 'fileA\x00').is_file()) def test_is_file_no_follow_symlinks(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) @@ -1593,7 +1586,7 @@ def test_is_file_no_follow_symlinks(self): self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) def test_is_mount(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_mount()) self.assertFalse((P / 'dirA').is_mount()) self.assertFalse((P / 'non-existing').is_mount()) @@ -1602,7 +1595,7 @@ def test_is_mount(self): self.assertFalse((P / 'linkA').is_mount()) def test_is_symlink(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_symlink()) self.assertFalse((P / 'dirA').is_symlink()) self.assertFalse((P / 'non-existing').is_symlink()) @@ -1618,7 +1611,7 @@ def test_is_symlink(self): self.assertIs((P / 'linkA\x00').is_file(), False) def test_is_junction_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_junction()) self.assertFalse((P / 'dirA').is_junction()) self.assertFalse((P / 'non-existing').is_junction()) @@ -1627,7 +1620,7 @@ def test_is_junction_false(self): self.assertFalse((P / 'fileA\x00').is_junction()) def test_is_fifo_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_fifo()) self.assertFalse((P / 'dirA').is_fifo()) self.assertFalse((P / 'non-existing').is_fifo()) @@ -1636,7 +1629,7 @@ def test_is_fifo_false(self): self.assertIs((P / 'fileA\x00').is_fifo(), False) def test_is_socket_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_socket()) self.assertFalse((P / 'dirA').is_socket()) self.assertFalse((P / 'non-existing').is_socket()) @@ -1645,7 +1638,7 @@ def test_is_socket_false(self): self.assertIs((P / 'fileA\x00').is_socket(), False) def test_is_block_device_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_block_device()) self.assertFalse((P / 'dirA').is_block_device()) self.assertFalse((P / 'non-existing').is_block_device()) @@ -1654,7 +1647,7 @@ def test_is_block_device_false(self): self.assertIs((P / 'fileA\x00').is_block_device(), False) def test_is_char_device_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_char_device()) self.assertFalse((P / 'dirA').is_char_device()) self.assertFalse((P / 'non-existing').is_char_device()) @@ -1663,7 +1656,7 @@ def test_is_char_device_false(self): self.assertIs((P / 'fileA\x00').is_char_device(), False) def test_pickling_common(self): - p = self.cls(BASE, 'fileA') + p = self.cls(self.base, 'fileA') for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): dumped = pickle.dumps(p, proto) pp = pickle.loads(dumped) @@ -1683,7 +1676,7 @@ def _check_complex_symlinks(self, link0_target): self.skipTest("symlinks required") # Test solving a non-looping chain of symlinks (issue #19887). - P = self.cls(BASE) + P = self.cls(self.base) P.joinpath('link1').symlink_to(os.path.join('link0', 'link0'), target_is_directory=True) P.joinpath('link2').symlink_to(os.path.join('link1', 'link1'), target_is_directory=True) P.joinpath('link3').symlink_to(os.path.join('link2', 'link2'), target_is_directory=True) @@ -1692,16 +1685,16 @@ def _check_complex_symlinks(self, link0_target): # Resolve absolute paths. p = (P / 'link0').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = (P / 'link1').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = (P / 'link2').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = (P / 'link3').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) # Resolve relative paths. try: @@ -1709,25 +1702,25 @@ def _check_complex_symlinks(self, link0_target): except pathlib.UnsupportedOperation: return old_path = os.getcwd() - os.chdir(BASE) + os.chdir(self.base) try: p = self.cls('link0').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = self.cls('link1').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = self.cls('link2').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = self.cls('link3').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) finally: os.chdir(old_path) def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(BASE) + self._check_complex_symlinks(self.base) def test_complex_symlinks_relative(self): self._check_complex_symlinks('.') @@ -1750,7 +1743,7 @@ def setUpWalk(self): # broken_link2 # TEST2/ # tmp4 a lone file - self.walk_path = self.cls(BASE, "TEST1") + self.walk_path = self.cls(self.base, "TEST1") self.sub1_path = self.walk_path / "SUB1" self.sub11_path = self.sub1_path / "SUB11" self.sub2_path = self.walk_path / "SUB2" @@ -1758,8 +1751,8 @@ def setUpWalk(self): tmp2_path = self.sub1_path / "tmp2" tmp3_path = self.sub2_path / "tmp3" self.link_path = self.sub2_path / "link" - t2_path = self.cls(BASE, "TEST2") - tmp4_path = self.cls(BASE, "TEST2", "tmp4") + t2_path = self.cls(self.base, "TEST2") + tmp4_path = self.cls(self.base, "TEST2", "tmp4") broken_link_path = self.sub2_path / "broken_link" broken_link2_path = self.sub2_path / "broken_link2" @@ -1886,7 +1879,7 @@ def test_walk_above_recursion_limit(self): recursion_limit = 40 # directory_depth > recursion_limit directory_depth = recursion_limit + 10 - base = self.cls(BASE, 'deep') + base = self.cls(self.base, 'deep') path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True)