From 6e1e78054060ad326f26dd8dbf12adfedbb52883 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 13 Jan 2025 10:40:52 +0300 Subject: [PATCH 01/61] gh-128759: fix data race in `type_modified_unlocked` (#128764) --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 680846f1c0b865..0f5ebc6f90773d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1038,7 +1038,7 @@ type_modified_unlocked(PyTypeObject *type) We don't assign new version tags eagerly, but only as needed. */ - if (type->tp_version_tag == 0) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag) == 0) { return; } // Cannot modify static builtin types. From 6ecb620a0cae1b0b6183d1e6d0ae722013ca8cd2 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 13 Jan 2025 02:38:28 -0600 Subject: [PATCH 02/61] gh-90905: Allow cross-compilation on macOS (#128385) --- .../2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst | 1 + configure | 12 ++++++++++++ configure.ac | 12 ++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst diff --git a/Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst b/Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst new file mode 100644 index 00000000000000..31bb5fe005379a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst @@ -0,0 +1 @@ +Add support for cross-compiling to x86_64 on aarch64/arm64 macOS. diff --git a/configure b/configure index bb77c558abda5a..70581e11b60682 100755 --- a/configure +++ b/configure @@ -4097,6 +4097,9 @@ then *-apple-ios*) ac_sys_system=iOS ;; + *-*-darwin*) + ac_sys_system=Darwin + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -4591,6 +4594,15 @@ printf "%s\n" "$IPHONEOS_DEPLOYMENT_TARGET" >&6; } ;; esac ;; + *-*-darwin*) + case "$host_cpu" in + arm*) + _host_ident=arm + ;; + *) + _host_ident=$host_cpu + esac + ;; *-*-vxworks*) _host_ident=$host_cpu ;; diff --git a/configure.ac b/configure.ac index 653cd3f6c531b6..d7c3920d049d67 100644 --- a/configure.ac +++ b/configure.ac @@ -330,6 +330,9 @@ then *-apple-ios*) ac_sys_system=iOS ;; + *-*-darwin*) + ac_sys_system=Darwin + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -790,6 +793,15 @@ if test "$cross_compiling" = yes; then ;; esac ;; + *-*-darwin*) + case "$host_cpu" in + arm*) + _host_ident=arm + ;; + *) + _host_ident=$host_cpu + esac + ;; *-*-vxworks*) _host_ident=$host_cpu ;; From b00e1254fc00941bf91e41138940e73fd22e1cbf Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 13 Jan 2025 04:02:18 -0500 Subject: [PATCH 03/61] gh-128690: temporarily do not use test_embed in PGO profile builds (#128771) Temporarily do not use test_embed in PGO profile builds until the problem with test_init_pyvenv_cfg failing in some configurations is resolved. --- Lib/test/libregrtest/pgo.py | 1 - .../next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py index f762345c88cde3..04803ddf64453c 100644 --- a/Lib/test/libregrtest/pgo.py +++ b/Lib/test/libregrtest/pgo.py @@ -19,7 +19,6 @@ 'test_datetime', 'test_decimal', 'test_difflib', - 'test_embed', 'test_float', 'test_fstring', 'test_functools', diff --git a/Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst b/Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst new file mode 100644 index 00000000000000..fb84334ca77214 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst @@ -0,0 +1,2 @@ +Temporarily do not use test_embed in PGO profile builds until the problem +with test_init_pyvenv_cfg failing in some configurations is resolved. From 29fe8072cf404b891dde9c1d415095edddbe19de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:21:22 +0100 Subject: [PATCH 04/61] gh-123345: improve `fnmatch` docs (#123346) --- Doc/library/fnmatch.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index fda44923f204fc..5cb47777ae527d 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -46,9 +46,15 @@ module. See module :mod:`glob` for pathname expansion (:mod:`glob` uses a period are not special for this module, and are matched by the ``*`` and ``?`` patterns. -Also note that :func:`functools.lru_cache` with the *maxsize* of 32768 is used to -cache the compiled regex patterns in the following functions: :func:`fnmatch`, -:func:`fnmatchcase`, :func:`.filter`. +Unless stated otherwise, "filename string" and "pattern string" either refer to +:class:`str` or ``ISO-8859-1`` encoded :class:`bytes` objects. Note that the +functions documented below do not allow to mix a :class:`!bytes` pattern with +a :class:`!str` filename, and vice-versa. + +Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 +is used to cache the (typed) compiled regex patterns in the following +functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. + .. function:: fnmatch(name, pat) @@ -78,8 +84,8 @@ cache the compiled regex patterns in the following functions: :func:`fnmatch`, .. function:: filter(names, pat) - Construct a list from those elements of the :term:`iterable` *names* - that match pattern *pat*. + Construct a list from those elements of the :term:`iterable` of filename + strings *names* that match the pattern string *pat*. It is the same as ``[n for n in names if fnmatch(n, pat)]``, but implemented more efficiently. @@ -87,7 +93,7 @@ cache the compiled regex patterns in the following functions: :func:`fnmatch`, .. function:: translate(pat) Return the shell-style pattern *pat* converted to a regular expression for - using with :func:`re.match`. + using with :func:`re.match`. The pattern is expected to be a :class:`str`. Example: From ddd959987c557beaf823b681bf5e5e573ad657ac Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Jan 2025 10:30:28 +0000 Subject: [PATCH 05/61] GH-128685: Specialize (rather than quicken) LOAD_CONST into LOAD_CONST_[IM]MORTAL (GH-128708) --- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 16 +++++++-- Include/internal/pycore_uop_ids.h | 1 + Include/internal/pycore_uop_metadata.h | 6 ++-- Include/opcode_ids.h | 41 +++++++++++----------- Lib/_opcode_metadata.py | 42 ++++++++++++----------- Lib/test/test_dis.py | 4 +-- Python/bytecodes.c | 18 ++++++++-- Python/executor_cases.c.h | 7 ++-- Python/generated_cases.c.h | 26 +++++++++++++- Python/opcode_targets.h | 2 +- Python/optimizer_bytecodes.c | 7 ++++ Python/optimizer_cases.c.h | 4 ++- Python/specialize.c | 9 ----- 14 files changed, 121 insertions(+), 65 deletions(-) diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index f9f71d7453331e..8b3d6285c1e4e7 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -265,6 +265,7 @@ Known values: Python 3.14a4 3610 (Add VALUE_WITH_FAKE_GLOBALS format to annotationlib) Python 3.14a4 3611 (Add NOT_TAKEN instruction) Python 3.14a4 3612 (Add POP_ITER and INSTRUMENTED_POP_ITER) + Python 3.14a4 3613 (Add LOAD_CONST_MORTAL instruction) Python 3.15 will start with 3650 @@ -277,7 +278,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3612 +#define PYC_MAGIC_NUMBER 3613 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 90d5e277d8d6ce..51d6e435948225 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -321,6 +321,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case LOAD_CONST_IMMORTAL: return 0; + case LOAD_CONST_MORTAL: + return 0; case LOAD_DEREF: return 0; case LOAD_FAST: @@ -788,6 +790,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case LOAD_CONST_IMMORTAL: return 1; + case LOAD_CONST_MORTAL: + return 1; case LOAD_DEREF: return 1; case LOAD_FAST: @@ -1559,6 +1563,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = 1; return 0; } + case LOAD_CONST_MORTAL: { + *effect = 1; + return 0; + } case LOAD_DEREF: { *effect = 1; return 0; @@ -2109,8 +2117,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, + [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [LOAD_CONST_IMMORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [LOAD_CONST_MORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -2317,8 +2326,8 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } }, [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, 0, 0 } } }, - [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, 0, 0 } } }, [LOAD_CONST_IMMORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_IMMORTAL, 0, 0 } } }, + [LOAD_CONST_MORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_MORTAL, 0, 0 } } }, [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, 0, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { _LOAD_FAST_AND_CLEAR, 0, 0 } } }, @@ -2541,6 +2550,7 @@ const char *_PyOpcode_OpName[266] = { [LOAD_COMMON_CONSTANT] = "LOAD_COMMON_CONSTANT", [LOAD_CONST] = "LOAD_CONST", [LOAD_CONST_IMMORTAL] = "LOAD_CONST_IMMORTAL", + [LOAD_CONST_MORTAL] = "LOAD_CONST_MORTAL", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", @@ -2796,6 +2806,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_COMMON_CONSTANT] = LOAD_COMMON_CONSTANT, [LOAD_CONST] = LOAD_CONST, [LOAD_CONST_IMMORTAL] = LOAD_CONST, + [LOAD_CONST_MORTAL] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, @@ -2912,7 +2923,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 228: \ case 229: \ case 230: \ case 231: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 21690a28839565..3841363b411eed 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -206,6 +206,7 @@ extern "C" { #define _LOAD_CONST_INLINE_BORROW 430 #define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431 #define _LOAD_CONST_INLINE_WITH_NULL 432 +#define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF #define _LOAD_FAST 433 #define _LOAD_FAST_0 434 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 83e578cdd76fbd..942e5eb25b76b4 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -35,7 +35,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, + [_LOAD_CONST_MORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_CONST_IMMORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_SMALL_INT_0] = 0, [_LOAD_SMALL_INT_1] = 0, @@ -474,12 +474,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", - [_LOAD_CONST] = "_LOAD_CONST", [_LOAD_CONST_IMMORTAL] = "_LOAD_CONST_IMMORTAL", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL", [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", + [_LOAD_CONST_MORTAL] = "_LOAD_CONST_MORTAL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", [_LOAD_FAST_0] = "_LOAD_FAST_0", @@ -617,7 +617,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_FAST_LOAD_FAST: return 0; - case _LOAD_CONST: + case _LOAD_CONST_MORTAL: return 0; case _LOAD_CONST_IMMORTAL: return 0; diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 09e261fadd5544..f2d8963a1813c5 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -187,26 +187,27 @@ extern "C" { #define LOAD_ATTR_SLOT 205 #define LOAD_ATTR_WITH_HINT 206 #define LOAD_CONST_IMMORTAL 207 -#define LOAD_GLOBAL_BUILTIN 208 -#define LOAD_GLOBAL_MODULE 209 -#define LOAD_SUPER_ATTR_ATTR 210 -#define LOAD_SUPER_ATTR_METHOD 211 -#define RESUME_CHECK 212 -#define SEND_GEN 213 -#define STORE_ATTR_INSTANCE_VALUE 214 -#define STORE_ATTR_SLOT 215 -#define STORE_ATTR_WITH_HINT 216 -#define STORE_SUBSCR_DICT 217 -#define STORE_SUBSCR_LIST_INT 218 -#define TO_BOOL_ALWAYS_TRUE 219 -#define TO_BOOL_BOOL 220 -#define TO_BOOL_INT 221 -#define TO_BOOL_LIST 222 -#define TO_BOOL_NONE 223 -#define TO_BOOL_STR 224 -#define UNPACK_SEQUENCE_LIST 225 -#define UNPACK_SEQUENCE_TUPLE 226 -#define UNPACK_SEQUENCE_TWO_TUPLE 227 +#define LOAD_CONST_MORTAL 208 +#define LOAD_GLOBAL_BUILTIN 209 +#define LOAD_GLOBAL_MODULE 210 +#define LOAD_SUPER_ATTR_ATTR 211 +#define LOAD_SUPER_ATTR_METHOD 212 +#define RESUME_CHECK 213 +#define SEND_GEN 214 +#define STORE_ATTR_INSTANCE_VALUE 215 +#define STORE_ATTR_SLOT 216 +#define STORE_ATTR_WITH_HINT 217 +#define STORE_SUBSCR_DICT 218 +#define STORE_SUBSCR_LIST_INT 219 +#define TO_BOOL_ALWAYS_TRUE 220 +#define TO_BOOL_BOOL 221 +#define TO_BOOL_INT 222 +#define TO_BOOL_LIST 223 +#define TO_BOOL_NONE 224 +#define TO_BOOL_STR 225 +#define UNPACK_SEQUENCE_LIST 226 +#define UNPACK_SEQUENCE_TUPLE 227 +#define UNPACK_SEQUENCE_TWO_TUPLE 228 #define INSTRUMENTED_END_FOR 235 #define INSTRUMENTED_POP_ITER 236 #define INSTRUMENTED_END_SEND 237 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 64ee56fd10556f..d93e8d8df8fe4b 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -7,6 +7,7 @@ "RESUME_CHECK", ], "LOAD_CONST": [ + "LOAD_CONST_MORTAL", "LOAD_CONST_IMMORTAL", ], "TO_BOOL": [ @@ -178,26 +179,27 @@ 'LOAD_ATTR_SLOT': 205, 'LOAD_ATTR_WITH_HINT': 206, 'LOAD_CONST_IMMORTAL': 207, - 'LOAD_GLOBAL_BUILTIN': 208, - 'LOAD_GLOBAL_MODULE': 209, - 'LOAD_SUPER_ATTR_ATTR': 210, - 'LOAD_SUPER_ATTR_METHOD': 211, - 'RESUME_CHECK': 212, - 'SEND_GEN': 213, - 'STORE_ATTR_INSTANCE_VALUE': 214, - 'STORE_ATTR_SLOT': 215, - 'STORE_ATTR_WITH_HINT': 216, - 'STORE_SUBSCR_DICT': 217, - 'STORE_SUBSCR_LIST_INT': 218, - 'TO_BOOL_ALWAYS_TRUE': 219, - 'TO_BOOL_BOOL': 220, - 'TO_BOOL_INT': 221, - 'TO_BOOL_LIST': 222, - 'TO_BOOL_NONE': 223, - 'TO_BOOL_STR': 224, - 'UNPACK_SEQUENCE_LIST': 225, - 'UNPACK_SEQUENCE_TUPLE': 226, - 'UNPACK_SEQUENCE_TWO_TUPLE': 227, + 'LOAD_CONST_MORTAL': 208, + 'LOAD_GLOBAL_BUILTIN': 209, + 'LOAD_GLOBAL_MODULE': 210, + 'LOAD_SUPER_ATTR_ATTR': 211, + 'LOAD_SUPER_ATTR_METHOD': 212, + 'RESUME_CHECK': 213, + 'SEND_GEN': 214, + 'STORE_ATTR_INSTANCE_VALUE': 215, + 'STORE_ATTR_SLOT': 216, + 'STORE_ATTR_WITH_HINT': 217, + 'STORE_SUBSCR_DICT': 218, + 'STORE_SUBSCR_LIST_INT': 219, + 'TO_BOOL_ALWAYS_TRUE': 220, + 'TO_BOOL_BOOL': 221, + 'TO_BOOL_INT': 222, + 'TO_BOOL_LIST': 223, + 'TO_BOOL_NONE': 224, + 'TO_BOOL_STR': 225, + 'UNPACK_SEQUENCE_LIST': 226, + 'UNPACK_SEQUENCE_TUPLE': 227, + 'UNPACK_SEQUENCE_TWO_TUPLE': 228, } opmap = { diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 76d9b5401d7d8e..ed8bd6fa20880b 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -892,7 +892,7 @@ def loop_test(): %3d RESUME_CHECK 0 %3d BUILD_LIST 0 - LOAD_CONST 0 ((1, 2, 3)) + LOAD_CONST_MORTAL 0 ((1, 2, 3)) LIST_EXTEND 1 LOAD_SMALL_INT 3 BINARY_OP 5 (*) @@ -2548,7 +2548,7 @@ def test_specialized_code(self): expect = ''' 0 RESUME 0 - 1 LOAD_CONST_IMMORTAL 0 (None) + 1 LOAD_CONST 0 (None) RETURN_VALUE ''' for flag in ['-S', '--specialized']: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8bab4ea16b629b..657e2fb4e1c850 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -285,11 +285,25 @@ dummy_func( } family(LOAD_CONST, 0) = { + LOAD_CONST_MORTAL, LOAD_CONST_IMMORTAL, }; - pure inst(LOAD_CONST, (-- value)) { - value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + inst(LOAD_CONST, (-- value)) { + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); +#if ENABLE_SPECIALIZATION + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + } +#endif + } + + inst(LOAD_CONST_MORTAL, (-- value)) { + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); } inst(LOAD_CONST_IMMORTAL, (-- value)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e40fa88be89172..e90d6b5ec103ae 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -209,10 +209,13 @@ break; } - case _LOAD_CONST: { + /* _LOAD_CONST is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _LOAD_CONST_MORTAL: { _PyStackRef value; oparg = CURRENT_OPARG(); - value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7028ba52faae96..70d0814d682205 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5913,8 +5913,18 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); PREDICTED(LOAD_CONST); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; _PyStackRef value; - value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + } + #endif stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5936,6 +5946,20 @@ DISPATCH(); } + TARGET(LOAD_CONST_MORTAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_MORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + TARGET(LOAD_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 00d55d3a82edc1..c5c008fcbe574e 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -207,6 +207,7 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_CONST_IMMORTAL, + &&TARGET_LOAD_CONST_MORTAL, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_SUPER_ATTR_ATTR, @@ -233,7 +234,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 86394480f76bb8..788adecca8af80 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -479,6 +479,13 @@ dummy_func(void) { value = sym_new_const(ctx, val); } + op(_LOAD_CONST_MORTAL, (-- value)) { + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); + } + op(_LOAD_CONST_IMMORTAL, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c72ae7b6281e80..a64dd465467122 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -58,7 +58,9 @@ break; } - case _LOAD_CONST: { + /* _LOAD_CONST is not a viable micro-op for tier 2 */ + + case _LOAD_CONST_MORTAL: { _Py_UopsSymbol *value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; diff --git a/Python/specialize.c b/Python/specialize.c index c9325c39210874..897005c4f1078d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -478,15 +478,6 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, } i += caches; } - else if (opcode == LOAD_CONST) { - /* We can't do this in the bytecode compiler as - * marshalling can intern strings and make them immortal. */ - - PyObject *obj = PyTuple_GET_ITEM(consts, oparg); - if (_Py_IsImmortal(obj)) { - instructions[i].op.code = LOAD_CONST_IMMORTAL; - } - } if (opcode != EXTENDED_ARG) { oparg = 0; } From afb9dc887c6e8ae17b6a54c6124399e8bdc82253 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:05:02 +0200 Subject: [PATCH 06/61] gh-128595: Add test class helper to force no terminal colour (#128687) Co-authored-by: Erlend E. Aasland --- Lib/test/support/__init__.py | 47 ++++++++++++++++++--------- Lib/test/test_code_module.py | 3 +- Lib/test/test_exceptions.py | 1 + Lib/test/test_traceback.py | 9 ++++- Lib/test/test_unittest/test_result.py | 11 ++++--- 5 files changed, 49 insertions(+), 22 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 42e7b876594fa7..ee9520a8838625 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -60,6 +60,7 @@ "skip_on_s390x", "without_optimizer", "force_not_colorized", + "force_not_colorized_test_class", "BrokenIter", "in_systemd_nspawn_sync_suppressed", "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", @@ -2832,30 +2833,44 @@ def is_slot_wrapper(name, value): yield name, True +@contextlib.contextmanager +def no_color(): + import _colorize + from .os_helper import EnvironmentVarGuard + + with ( + swap_attr(_colorize, "can_colorize", lambda: False), + EnvironmentVarGuard() as env, + ): + for var in {"FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS"}: + env.unset(var) + env.set("NO_COLOR", "1") + yield + + def force_not_colorized(func): """Force the terminal not to be colorized.""" @functools.wraps(func) def wrapper(*args, **kwargs): - import _colorize - original_fn = _colorize.can_colorize - variables: dict[str, str | None] = { - "PYTHON_COLORS": None, "FORCE_COLOR": None, "NO_COLOR": None - } - try: - for key in variables: - variables[key] = os.environ.pop(key, None) - os.environ["NO_COLOR"] = "1" - _colorize.can_colorize = lambda: False + with no_color(): return func(*args, **kwargs) - finally: - _colorize.can_colorize = original_fn - del os.environ["NO_COLOR"] - for key, value in variables.items(): - if value is not None: - os.environ[key] = value return wrapper +def force_not_colorized_test_class(cls): + """Force the terminal not to be colorized for the entire test class.""" + original_setUpClass = cls.setUpClass + + @classmethod + @functools.wraps(cls.setUpClass) + def new_setUpClass(cls): + cls.enterClassContext(no_color()) + original_setUpClass() + + cls.setUpClass = new_setUpClass + return cls + + def initialized_with_pyrepl(): """Detect whether PyREPL was used during Python initialization.""" # If the main module has a __file__ attribute it's a Python module, which means PyREPL. diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 37c7bc772ed8c7..20b960ce8d1e02 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -5,9 +5,9 @@ from textwrap import dedent from contextlib import ExitStack from unittest import mock +from test.support import force_not_colorized_test_class from test.support import import_helper - code = import_helper.import_module('code') @@ -30,6 +30,7 @@ def mock_sys(self): del self.sysmod.ps2 +@force_not_colorized_test_class class TestInteractiveConsole(unittest.TestCase, MockSys): maxDiff = None diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 6ccfa9575f8569..206e22e791e02a 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2274,6 +2274,7 @@ def test_range_of_offsets(self): self.assertIn(expected, err.getvalue()) the_exception = exc + @force_not_colorized def test_subclass(self): class MySyntaxError(SyntaxError): pass diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 31f0a61d6a9d59..abdfc4638f2e9c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -21,7 +21,7 @@ from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok, assert_python_failure from test.support.import_helper import forget -from test.support import force_not_colorized +from test.support import force_not_colorized, force_not_colorized_test_class import json import textwrap @@ -1712,6 +1712,7 @@ def f(): @requires_debug_ranges() +@force_not_colorized_test_class class PurePythonTracebackErrorCaretTests( PurePythonExceptionFormattingMixin, TracebackErrorLocationCaretTestBase, @@ -1725,6 +1726,7 @@ class PurePythonTracebackErrorCaretTests( @cpython_only @requires_debug_ranges() +@force_not_colorized_test_class class CPythonTracebackErrorCaretTests( CAPIExceptionFormattingMixin, TracebackErrorLocationCaretTestBase, @@ -1736,6 +1738,7 @@ class CPythonTracebackErrorCaretTests( @cpython_only @requires_debug_ranges() +@force_not_colorized_test_class class CPythonTracebackLegacyErrorCaretTests( CAPIExceptionFormattingLegacyMixin, TracebackErrorLocationCaretTestBase, @@ -2149,10 +2152,12 @@ def test_print_exception_bad_type_python(self): boundaries = re.compile( '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) +@force_not_colorized_test_class class TestTracebackFormat(unittest.TestCase, TracebackFormatMixin): pass @cpython_only +@force_not_colorized_test_class class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixin): DEBUG_RANGES = False def setUp(self) -> None: @@ -2940,6 +2945,7 @@ def f(): self.assertEqual(report, expected) +@force_not_colorized_test_class class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks reporting through the 'traceback' module, with both @@ -2956,6 +2962,7 @@ def get_report(self, e): return s +@force_not_colorized_test_class class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks built-in reporting by the interpreter. diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 746b9fa2677717..ad6f52d7e0260e 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -1,13 +1,15 @@ import io import sys import textwrap - -from test.support import warnings_helper, captured_stdout - import traceback import unittest from unittest.util import strclass -from test.support import force_not_colorized +from test.support import warnings_helper +from test.support import ( + captured_stdout, + force_not_colorized, + force_not_colorized_test_class, +) from test.test_unittest.support import BufferedWriter @@ -772,6 +774,7 @@ def testFoo(self): runner.run(Test('testFoo')) +@force_not_colorized_test_class class TestOutputBuffering(unittest.TestCase): def setUp(self): From 39fc7ef4fe211e8f7d3b5a6e392e475ecdfbce72 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Jan 2025 11:42:45 +0000 Subject: [PATCH 07/61] GH-124483: Mark `Py_DECREF`, etc. as escaping for the JIT (GH-128678) --- Include/internal/pycore_opcode_metadata.h | 44 ++++++------- Include/internal/pycore_uop_metadata.h | 64 +++++++++---------- Lib/test/test_capi/test_opt.py | 19 ++++++ ...-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst | 3 + Tools/cases_generator/analyzer.py | 18 ++++-- 5 files changed, 90 insertions(+), 58 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 51d6e435948225..8c4182ae384e5f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1985,7 +1985,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, @@ -1995,8 +1995,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2006,7 +2006,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [CACHE] = { true, INSTR_FMT_IX, 0 }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2021,7 +2021,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2031,7 +2031,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -2054,8 +2054,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, HAS_NO_SAVE_IP_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -2063,9 +2063,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2086,10 +2086,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IXC, 0 }, [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2109,7 +2109,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -2137,7 +2137,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2169,19 +2169,19 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, @@ -2209,7 +2209,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, }; #endif diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 942e5eb25b76b4..70328a0194fd8c 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -42,21 +42,21 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SMALL_INT_2] = 0, [_LOAD_SMALL_INT_3] = 0, [_LOAD_SMALL_INT] = HAS_ARG_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_POP_TOP] = HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, - [_END_FOR] = HAS_NO_SAVE_IP_FLAG, - [_END_SEND] = HAS_PURE_FLAG, + [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, + [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -81,20 +81,20 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, - [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, + [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -128,7 +128,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_GLOBAL_MODULE_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -155,7 +155,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_MODULE_PUSH_KEYS] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_MODULE_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_CHECK_ATTR_WITH_HINT] = HAS_EXIT_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, @@ -167,9 +167,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, - [_STORE_ATTR_INSTANCE_VALUE] = 0, + [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG, + [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -212,16 +212,16 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, - [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, + [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_EXPAND_METHOD] = HAS_ARG_FLAG, + [_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_NON_PY_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -232,7 +232,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = 0, - [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -244,16 +244,16 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG, + [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_EXPAND_METHOD_KW] = HAS_ARG_FLAG, + [_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CALLARGS_A_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -269,8 +269,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, @@ -279,7 +279,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, @@ -288,7 +288,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, - [_START_EXECUTOR] = 0, + [_START_EXECUTOR] = HAS_ESCAPES_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d84702411afe41..12542d8b7fa62e 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1555,6 +1555,25 @@ def f(l: complex, r: complex) -> None: with self.subTest(l=l, r=r, x=x, y=y): script_helper.assert_python_ok("-c", s) + def test_decref_escapes(self): + class Convert9999ToNone: + def __del__(self): + ns = sys._getframe(1).f_locals + if ns["i"] == 9999: + ns["i"] = None + + def crash_addition(): + try: + for i in range(10000): + n = Convert9999ToNone() + i + i # Remove guards for i. + n = None # Change i. + i + i # This crashed when we didn't treat DECREF as escaping (gh-124483) + except TypeError: + pass + + crash_addition() + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst new file mode 100644 index 00000000000000..6f2e7265d62321 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst @@ -0,0 +1,3 @@ +Treat ``Py_DECREF`` and variants as escaping when generating opcode and uop +metadata. This prevents the possibility of a ``__del__`` method causing the JIT +to behave incorrectly. diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 679beca3ec3a9d..4e1d47281fdbbf 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -8,6 +8,7 @@ @dataclass class Properties: escaping_calls: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] + escapes: bool error_with_pop: bool error_without_pop: bool deopts: bool @@ -45,6 +46,7 @@ def from_list(properties: list["Properties"]) -> "Properties": escaping_calls.update(p.escaping_calls) return Properties( escaping_calls=escaping_calls, + escapes = any(p.escapes for p in properties), error_with_pop=any(p.error_with_pop for p in properties), error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), @@ -68,12 +70,9 @@ def from_list(properties: list["Properties"]) -> "Properties": def infallible(self) -> bool: return not self.error_with_pop and not self.error_without_pop - @property - def escapes(self) -> bool: - return bool(self.escaping_calls) - SKIP_PROPERTIES = Properties( escaping_calls={}, + escapes=False, error_with_pop=False, error_without_pop=False, deopts=False, @@ -815,8 +814,19 @@ def compute_properties(op: parser.InstDef) -> Properties: ) error_with_pop = has_error_with_pop(op) error_without_pop = has_error_without_pop(op) + escapes = ( + bool(escaping_calls) or + variable_used(op, "Py_DECREF") or + variable_used(op, "Py_XDECREF") or + variable_used(op, "Py_CLEAR") or + variable_used(op, "PyStackRef_CLOSE") or + variable_used(op, "PyStackRef_XCLOSE") or + variable_used(op, "PyStackRef_CLEAR") or + variable_used(op, "SETLOCAL") + ) return Properties( escaping_calls=escaping_calls, + escapes=escapes, error_with_pop=error_with_pop, error_without_pop=error_without_pop, deopts=deopts_if, From 6ff8f82f92a8af363b2bdd8bbaba5845eef430fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:46:13 +0100 Subject: [PATCH 08/61] gh-128150: Improve performances of `uuid.uuid*` constructor functions. (#128151) We introduce a private constructor `UUID._from_int()` for RFC 4122/9562 UUIDs, which takes the integral UUID value as input. The latter must have correctly set its variant and version bits. We also make `UUID.__init__()` slightly more efficient. --- Doc/whatsnew/3.14.rst | 16 ++++ Lib/uuid.py | 85 +++++++++++++------ ...-12-21-11-12-50.gh-issue-128151.aq7vpG.rst | 2 + 3 files changed, 76 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 72abfebd46f2b9..474bd6a24b8ece 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -717,6 +717,22 @@ io file's bytes in full. (Contributed by Cody Maloney and Victor Stinner in :gh:`120754` and :gh:`90102`.) + +uuid +---- + +* Improve generation of :class:`~uuid.UUID` objects via their dedicated + functions: + + * :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are both roughly 40% faster + for 16-byte names and 20% faster for 1024-byte names. Performance for + longer names remains unchanged. + * :func:`~uuid.uuid4` and :func:`~uuid.uuid8` are 30% and 40% faster + respectively. + + (Contributed by Bénédikt Tran in :gh:`128150`.) + + Deprecated ========== diff --git a/Lib/uuid.py b/Lib/uuid.py index 9c6ad9643cf6d5..cd1f3530ab63e1 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -85,6 +85,17 @@ class SafeUUID: unknown = None +_UINT_128_MAX = (1 << 128) - 1 +# 128-bit mask to clear the variant and version bits of a UUID integral value +_RFC_4122_CLEARFLAGS_MASK = ~((0xf000 << 64) | (0xc000 << 48)) +# RFC 4122 variant bits and version bits to activate on a UUID integral value. +_RFC_4122_VERSION_1_FLAGS = ((1 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_3_FLAGS = ((3 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_4_FLAGS = ((4 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_5_FLAGS = ((5 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_8_FLAGS = ((8 << 76) | (0x8000 << 48)) + + class UUID: """Instances of the UUID class represent UUIDs as specified in RFC 4122. UUID objects are immutable, hashable, and usable as dictionary keys. @@ -174,57 +185,69 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, if [hex, bytes, bytes_le, fields, int].count(None) != 4: raise TypeError('one of the hex, bytes, bytes_le, fields, ' 'or int arguments must be given') - if hex is not None: + if int is not None: + pass + elif hex is not None: hex = hex.replace('urn:', '').replace('uuid:', '') hex = hex.strip('{}').replace('-', '') if len(hex) != 32: raise ValueError('badly formed hexadecimal UUID string') int = int_(hex, 16) - if bytes_le is not None: + elif bytes_le is not None: if len(bytes_le) != 16: raise ValueError('bytes_le is not a 16-char string') + assert isinstance(bytes_le, bytes_), repr(bytes_le) bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] + bytes_le[8-1:6-1:-1] + bytes_le[8:]) - if bytes is not None: + int = int_.from_bytes(bytes) # big endian + elif bytes is not None: if len(bytes) != 16: raise ValueError('bytes is not a 16-char string') assert isinstance(bytes, bytes_), repr(bytes) int = int_.from_bytes(bytes) # big endian - if fields is not None: + elif fields is not None: if len(fields) != 6: raise ValueError('fields is not a 6-tuple') (time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node) = fields - if not 0 <= time_low < 1<<32: + if not 0 <= time_low < (1 << 32): raise ValueError('field 1 out of range (need a 32-bit value)') - if not 0 <= time_mid < 1<<16: + if not 0 <= time_mid < (1 << 16): raise ValueError('field 2 out of range (need a 16-bit value)') - if not 0 <= time_hi_version < 1<<16: + if not 0 <= time_hi_version < (1 << 16): raise ValueError('field 3 out of range (need a 16-bit value)') - if not 0 <= clock_seq_hi_variant < 1<<8: + if not 0 <= clock_seq_hi_variant < (1 << 8): raise ValueError('field 4 out of range (need an 8-bit value)') - if not 0 <= clock_seq_low < 1<<8: + if not 0 <= clock_seq_low < (1 << 8): raise ValueError('field 5 out of range (need an 8-bit value)') - if not 0 <= node < 1<<48: + if not 0 <= node < (1 << 48): raise ValueError('field 6 out of range (need a 48-bit value)') clock_seq = (clock_seq_hi_variant << 8) | clock_seq_low int = ((time_low << 96) | (time_mid << 80) | (time_hi_version << 64) | (clock_seq << 48) | node) - if int is not None: - if not 0 <= int < 1<<128: - raise ValueError('int is out of range (need a 128-bit value)') + if not 0 <= int <= _UINT_128_MAX: + raise ValueError('int is out of range (need a 128-bit value)') if version is not None: if not 1 <= version <= 8: raise ValueError('illegal version number') + # clear the variant and the version number bits + int &= _RFC_4122_CLEARFLAGS_MASK # Set the variant to RFC 4122/9562. - int &= ~(0xc000 << 48) - int |= 0x8000 << 48 + int |= 0x8000_0000_0000_0000 # (0x8000 << 48) # Set the version number. - int &= ~(0xf000 << 64) int |= version << 76 object.__setattr__(self, 'int', int) object.__setattr__(self, 'is_safe', is_safe) + @classmethod + def _from_int(cls, value): + """Create a UUID from an integer *value*. Internal use only.""" + assert 0 <= value <= _UINT_128_MAX, repr(value) + self = object.__new__(cls) + object.__setattr__(self, 'int', value) + object.__setattr__(self, 'is_safe', SafeUUID.unknown) + return self + def __getstate__(self): d = {'int': self.int} if self.is_safe != SafeUUID.unknown: @@ -700,24 +723,30 @@ def uuid3(namespace, name): """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" if isinstance(name, str): name = bytes(name, "utf-8") - from hashlib import md5 - digest = md5( - namespace.bytes + name, - usedforsecurity=False - ).digest() - return UUID(bytes=digest[:16], version=3) + import hashlib + h = hashlib.md5(namespace.bytes + name, usedforsecurity=False) + int_uuid_3 = int.from_bytes(h.digest()) + int_uuid_3 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_3 |= _RFC_4122_VERSION_3_FLAGS + return UUID._from_int(int_uuid_3) def uuid4(): """Generate a random UUID.""" - return UUID(bytes=os.urandom(16), version=4) + int_uuid_4 = int.from_bytes(os.urandom(16)) + int_uuid_4 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_4 |= _RFC_4122_VERSION_4_FLAGS + return UUID._from_int(int_uuid_4) def uuid5(namespace, name): """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" if isinstance(name, str): name = bytes(name, "utf-8") - from hashlib import sha1 - hash = sha1(namespace.bytes + name).digest() - return UUID(bytes=hash[:16], version=5) + import hashlib + h = hashlib.sha1(namespace.bytes + name, usedforsecurity=False) + int_uuid_5 = int.from_bytes(h.digest()[:16]) + int_uuid_5 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_5 |= _RFC_4122_VERSION_5_FLAGS + return UUID._from_int(int_uuid_5) def uuid8(a=None, b=None, c=None): """Generate a UUID from three custom blocks. @@ -740,7 +769,9 @@ def uuid8(a=None, b=None, c=None): int_uuid_8 = (a & 0xffff_ffff_ffff) << 80 int_uuid_8 |= (b & 0xfff) << 64 int_uuid_8 |= c & 0x3fff_ffff_ffff_ffff - return UUID(int=int_uuid_8, version=8) + # by construction, the variant and version bits are already cleared + int_uuid_8 |= _RFC_4122_VERSION_8_FLAGS + return UUID._from_int(int_uuid_8) def main(): """Run the uuid command line interface.""" diff --git a/Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst b/Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst new file mode 100644 index 00000000000000..04c744fb2ba54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst @@ -0,0 +1,2 @@ +Improve generation of :class:`~uuid.UUID` objects version 3, 4, 5, and 8 +via their dedicated functions by 30%. Patch by Bénédikt Tran. From 517dc65ffcea8413e1a60c4cb5d63e5fa39e7f72 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Jan 2025 12:37:48 +0000 Subject: [PATCH 09/61] GH-128682: Stronger checking of `PyStackRef_CLOSE` and `DEAD`. (GH-128683) --- Include/internal/pycore_opcode_metadata.h | 6 +- Include/internal/pycore_uop_metadata.h | 6 +- Lib/test/test_generated_cases.py | 12 +++ Python/bytecodes.c | 74 ++++++++------- Python/executor_cases.c.h | 67 +++++++------- Python/generated_cases.c.h | 75 ++++++++------- Tools/cases_generator/generators_common.py | 103 ++++++++++++++++++--- 7 files changed, 220 insertions(+), 123 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8c4182ae384e5f..5365e2a5c6b8cd 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1985,7 +1985,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, @@ -2055,7 +2055,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -2136,7 +2136,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 70328a0194fd8c..721fa94da19615 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -56,7 +56,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_TOP] = HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, - [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -81,7 +81,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -146,7 +146,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 75cbd8dd94e9cb..4a1c99edacb71a 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1739,6 +1739,18 @@ def test_no_escaping_calls_in_branching_macros(self): with self.assertRaises(SyntaxError): self.run_cases_test(input, "") + def test_kill_in_wrong_order(self): + input = """ + inst(OP, (a, b -- c)) { + c = b; + PyStackRef_CLOSE(a); + PyStackRef_CLOSE(b); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: super().setUp() diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 657e2fb4e1c850..ef7086c3c020b0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -384,7 +384,7 @@ dummy_func( (void)receiver; val = value; DEAD(value); - PyStackRef_CLOSE(receiver); + DECREF_INPUTS(); } tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- val)) { @@ -681,8 +681,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -725,7 +725,7 @@ dummy_func( * that the string is safe to mutate. */ assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE(left); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); DEAD(left); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); @@ -822,8 +822,7 @@ dummy_func( err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -2082,11 +2081,8 @@ dummy_func( int method_found = 0; PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); if (attr_o == NULL) { - PyStackRef_CLOSE(self_st); - ERROR_IF(true, error); + ERROR_NO_POP(); } if (method_found) { self_or_null = self_st; // transfer ownership @@ -2095,6 +2091,8 @@ dummy_func( PyStackRef_CLOSE(self_st); self_or_null = PyStackRef_NULL; } + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(global_super_st); attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -2920,7 +2918,6 @@ dummy_func( else { /* `iterable` is not a generator. */ PyObject *iter_o = PyObject_GetIter(iterable_o); - DEAD(iterable); if (iter_o == NULL) { ERROR_NO_POP(); } @@ -3466,11 +3463,11 @@ dummy_func( /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(true, error); } PyObject *res_o = PyObject_Vectorcall( @@ -3496,11 +3493,11 @@ dummy_func( } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3636,11 +3633,11 @@ dummy_func( NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3865,28 +3862,29 @@ dummy_func( op(_CALL_BUILTIN_CLASS, (callable[1], self_or_null[1], args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - + DEOPT_IF(!PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } - DEAD(self_or_null); - DEOPT_IF(!PyType_Check(callable_o)); - PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); } + DEAD(self_or_null); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } + DEAD(args); PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -3939,21 +3937,22 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } - DEAD(self_or_null); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); } + DEAD(self_or_null); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable_o), args_o, @@ -3963,8 +3962,9 @@ dummy_func( /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } + DEAD(args); PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -4043,8 +4043,10 @@ dummy_func( if (res_o == NULL) { GOTO_ERROR(error); } - PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); + DEAD(args); + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4053,25 +4055,24 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable_o != interp->callable_cache.isinstance); STAT_INC(CALL, hit); - _PyStackRef cls_stackref = args[1]; - _PyStackRef inst_stackref = args[0]; + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); if (retval < 0) { ERROR_NO_POP(); } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(inst_stackref); - PyStackRef_CLOSE(cls_stackref); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); } // This is secretly a super-instruction @@ -4370,11 +4371,11 @@ dummy_func( } PyStackRef_CLOSE(kwnames); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4529,6 +4530,8 @@ dummy_func( PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; + kwargs_out = kwargs_in; + DEAD(kwargs_in); DEAD(callargs); } else { @@ -4540,11 +4543,11 @@ dummy_func( if (tuple_o == NULL) { ERROR_NO_POP(); } + kwargs_out = kwargs_in; + DEAD(kwargs_in); PyStackRef_CLOSE(callargs); tuple = PyStackRef_FromPyObjectSteal(tuple_o); } - kwargs_out = kwargs_in; - DEAD(kwargs_in); } op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { @@ -4720,8 +4723,7 @@ dummy_func( inst(FORMAT_WITH_SPEC, (value, fmt_spec -- res)) { PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e90d6b5ec103ae..0d3b9b269ab247 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -838,8 +838,8 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -882,7 +882,7 @@ * that the string is safe to mutate. */ assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE(left); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); @@ -2529,11 +2529,8 @@ PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); if (attr_o == NULL) { - PyStackRef_CLOSE(self_st); - if (true) JUMP_TO_ERROR(); + JUMP_TO_ERROR(); } if (method_found) { self_or_null = self_st; // transfer ownership @@ -2541,6 +2538,8 @@ PyStackRef_CLOSE(self_st); self_or_null = PyStackRef_NULL; } + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(global_super_st); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -4215,7 +4214,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4229,10 +4228,10 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4695,25 +4694,26 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } if (tp->tp_vectorcall == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4725,7 +4725,7 @@ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); @@ -4800,8 +4800,9 @@ /* Builtin METH_FASTCALL functions, without keywords */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } if (!PyCFunction_CheckExact(callable_o)) { @@ -4815,10 +4816,10 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4834,7 +4835,7 @@ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); @@ -4879,7 +4880,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -4942,8 +4943,8 @@ if (res_o == NULL) { GOTO_ERROR(error); } - PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4963,8 +4964,9 @@ /* isinstance(o, o2) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } if (total_args != 2) { @@ -4977,8 +4979,8 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - _PyStackRef cls_stackref = args[1]; - _PyStackRef inst_stackref = args[0]; + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; _PyFrame_SetStackPointer(frame, stack_pointer); int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4987,9 +4989,11 @@ } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(inst_stackref); - PyStackRef_CLOSE(cls_stackref); PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -5136,7 +5140,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -5257,7 +5261,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -5467,7 +5471,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -5512,6 +5516,7 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; + kwargs_out = kwargs_in; } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5526,10 +5531,10 @@ if (tuple_o == NULL) { JUMP_TO_ERROR(); } + kwargs_out = kwargs_in; PyStackRef_CLOSE(callargs); tuple = PyStackRef_FromPyObjectSteal(tuple_o); } - kwargs_out = kwargs_in; stack_pointer[-1 - (oparg & 1)] = tuple; if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 70d0814d682205..45423a2565c76d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -159,8 +159,8 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -215,7 +215,7 @@ * that the string is safe to mutate. */ assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE(left); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); @@ -962,10 +962,10 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1001,10 +1001,10 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1351,19 +1351,20 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1379,7 +1380,7 @@ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { @@ -1430,8 +1431,9 @@ /* Builtin METH_FASTCALL functions, without keywords */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); @@ -1439,10 +1441,10 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1462,7 +1464,7 @@ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { @@ -1529,7 +1531,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1670,6 +1672,7 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; + kwargs_out = kwargs_in; } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1684,10 +1687,10 @@ if (tuple_o == NULL) { goto error; } + kwargs_out = kwargs_in; PyStackRef_CLOSE(callargs); tuple = PyStackRef_FromPyObjectSteal(tuple_o); } - kwargs_out = kwargs_in; } // _DO_CALL_FUNCTION_EX { @@ -1872,16 +1875,17 @@ /* isinstance(o, o2) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - _PyStackRef cls_stackref = args[1]; - _PyStackRef inst_stackref = args[0]; + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; _PyFrame_SetStackPointer(frame, stack_pointer); int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1890,9 +1894,11 @@ } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(inst_stackref); - PyStackRef_CLOSE(cls_stackref); PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1997,7 +2003,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2039,10 +2045,10 @@ } PyStackRef_CLOSE(kwnames); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2198,7 +2204,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2375,8 +2381,8 @@ if (res_o == NULL) { GOTO_ERROR(error); } - PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2456,7 +2462,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2541,7 +2547,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2774,7 +2780,7 @@ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2792,10 +2798,10 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4480,10 +4486,10 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4519,10 +4525,10 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -6540,11 +6546,8 @@ PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); if (attr_o == NULL) { - PyStackRef_CLOSE(self_st); - goto pop_3_error; + goto error; } if (method_found) { self_or_null = self_st; // transfer ownership @@ -6552,6 +6555,8 @@ PyStackRef_CLOSE(self_st); self_or_null = PyStackRef_NULL; } + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(global_super_st); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 8df9a9cada92ae..0035deb02ffdf0 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -98,6 +98,11 @@ def always_true(tkn: Token | None) -> bool: return False return tkn.text in {"true", "1"} +NON_ESCAPING_DEALLOCS = { + "_PyFloat_ExactDealloc", + "_PyLong_ExactDealloc", + "_PyUnicode_ExactDealloc", +} class Emitter: out: CWriter @@ -116,7 +121,7 @@ def __init__(self, out: CWriter): "SAVE_STACK": self.save_stack, "RELOAD_STACK": self.reload_stack, "PyStackRef_CLOSE": self.stackref_close, - "PyStackRef_CLOSE_SPECIALIZED": self.stackref_close, + "PyStackRef_CLOSE_SPECIALIZED": self.stackref_close_specialized, "PyStackRef_AsPyObjectSteal": self.stackref_steal, "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, @@ -234,23 +239,26 @@ def decref_inputs( next(tkn_iter) next(tkn_iter) self.out.emit_at("", tkn) - for var in uop.stack.inputs: - if var.name == "unused" or var.name == "null" or var.peek: + for var in storage.inputs: + if not var.defined: + continue + if var.name == "null": continue + close = "PyStackRef_CLOSE" + if "null" in var.name or var.condition and var.condition != "1": + close = "PyStackRef_XCLOSE" if var.size: if var.size == "1": - self.out.emit(f"PyStackRef_CLOSE({var.name}[0]);\n") + self.out.emit(f"{close}({var.name}[0]);\n") else: self.out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - self.out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") + self.out.emit(f"{close}({var.name}[_i]);\n") self.out.emit("}\n") elif var.condition: - if var.condition == "1": - self.out.emit(f"PyStackRef_CLOSE({var.name});\n") - elif var.condition != "0": - self.out.emit(f"PyStackRef_XCLOSE({var.name});\n") + if var.condition != "0": + self.out.emit(f"{close}({var.name});\n") else: - self.out.emit(f"PyStackRef_CLOSE({var.name});\n") + self.out.emit(f"{close}({var.name});\n") for input in storage.inputs: input.defined = False return True @@ -291,6 +299,25 @@ def kill( raise analysis_error(f"'{name}' is not a live input-only variable", name_tkn) return True + def stackref_kill( + self, + name: Token, + storage: Storage, + escapes: bool + ) -> bool: + live = "" + for var in reversed(storage.inputs): + if var.name == name.text: + if live and escapes: + raise analysis_error( + f"Cannot close '{name.text}' when " + f"'{live}' is still live", name) + var.defined = False + break + if var.defined: + live = var.name + return True + def stackref_close( self, tkn: Token, @@ -306,14 +333,60 @@ def stackref_close( name = next(tkn_iter) self.out.emit(name) if name.kind == "IDENTIFIER": - for var in storage.inputs: - if var.name == name.text: - var.defined = False + return self.stackref_kill(name, storage, True) rparen = emit_to(self.out, tkn_iter, "RPAREN") self.emit(rparen) return True - stackref_steal = stackref_close + def stackref_close_specialized( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + + self.out.emit(tkn) + tkn = next(tkn_iter) + assert tkn.kind == "LPAREN" + self.out.emit(tkn) + name = next(tkn_iter) + self.out.emit(name) + comma = next(tkn_iter) + if comma.kind != "COMMA": + raise analysis_error("Expected comma", comma) + self.out.emit(comma) + dealloc = next(tkn_iter) + if dealloc.kind != "IDENTIFIER": + raise analysis_error("Expected identifier", dealloc) + self.out.emit(dealloc) + if name.kind == "IDENTIFIER": + escapes = dealloc.text not in NON_ESCAPING_DEALLOCS + return self.stackref_kill(name, storage, escapes) + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + return True + + def stackref_steal( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit(tkn) + tkn = next(tkn_iter) + assert tkn.kind == "LPAREN" + self.out.emit(tkn) + name = next(tkn_iter) + self.out.emit(name) + if name.kind == "IDENTIFIER": + return self.stackref_kill(name, storage, False) + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + return True def sync_sp( self, @@ -548,7 +621,7 @@ def emit_tokens( storage.push_outputs() self._print_storage(storage) except StackError as ex: - raise analysis_error(ex.args[0], rbrace) + raise analysis_error(ex.args[0], rbrace) from None return storage def emit(self, txt: str | Token) -> None: From 76ffaef729c91bb79da6df2ade48f3ec51118300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:55:09 +0100 Subject: [PATCH 10/61] gh-128078: Clear exception in `anext` before calling `_PyGen_SetStopIterationValue` (#128780) Co-authored-by: Kumar Aditya --- Lib/test/test_asyncgen.py | 17 +++++++++++++++++ ...25-01-13-12-48-30.gh-issue-128078.qOsl9B.rst | 2 ++ Objects/genobject.c | 1 + Objects/iterobject.c | 2 ++ 4 files changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 5bfd789185c675..3f631a03c9bf72 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1152,6 +1152,23 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_asyncio_anext_tuple_no_exceptions(self): + # StopAsyncIteration exceptions should be cleared. + # See: https://github.com/python/cpython/issues/128078. + + async def foo(): + if False: + yield (1, 2) + + async def run(): + it = foo().__aiter__() + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + res = await anext(it, ('a', 'b')) + self.assertEqual(res, ('a', 'b')) + + self.loop.run_until_complete(run()) + def test_async_gen_asyncio_anext_stopiteration(self): async def foo(): try: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst new file mode 100644 index 00000000000000..498864a0aa3145 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst @@ -0,0 +1,2 @@ +Fix a :exc:`SystemError` when using :func:`anext` with a default tuple +value. Patch by Bénédikt Tran. diff --git a/Objects/genobject.c b/Objects/genobject.c index e87f199c2504ba..a4872de296e2cf 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -633,6 +633,7 @@ gen_iternext(PyObject *self) int _PyGen_SetStopIterationValue(PyObject *value) { + assert(!PyErr_Occurred()); PyObject *e; if (value == NULL || diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 135ced9ea1f268..ebb342ff109222 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -384,6 +384,7 @@ anextawaitable_iternext(anextawaitableobject *obj) return result; } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + PyErr_Clear(); _PyGen_SetStopIterationValue(obj->default_value); } return NULL; @@ -407,6 +408,7 @@ anextawaitable_proxy(anextawaitableobject *obj, char *meth, PyObject *arg) { * exception we replace it with a `StopIteration(default)`, as if * it was the return value of `__anext__()` coroutine. */ + PyErr_Clear(); _PyGen_SetStopIterationValue(obj->default_value); } return NULL; From aa6579cb60b4fcd652d7bc83fed2668e4ae84db3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 13 Jan 2025 14:10:41 +0100 Subject: [PATCH 11/61] gh-127773: Disable attribute cache on incompatible MRO entries (GH-127924) --- Include/cpython/object.h | 12 ++++++++- Lib/test/test_metaclass.py | 27 +++++++++++++++++++ ...-12-13-15-21-45.gh-issue-127773.E-DZR4.rst | 1 + Objects/typeobject.c | 12 ++++++++- 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst diff --git a/Include/cpython/object.h b/Include/cpython/object.h index e4797029da431e..c8c6bc97fa32ee 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -221,7 +221,9 @@ struct _typeobject { PyObject *tp_weaklist; /* not used for static builtin types */ destructor tp_del; - /* Type attribute cache version tag. Added in version 2.6 */ + /* Type attribute cache version tag. Added in version 2.6. + * If zero, the cache is invalid and must be initialized. + */ unsigned int tp_version_tag; destructor tp_finalize; @@ -229,9 +231,17 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + + /* Number of tp_version_tag values used. + * Set to _Py_ATTR_CACHE_UNUSED if the attribute cache is + * disabled for this type (e.g. due to custom MRO entries). + * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). + */ uint16_t tp_versions_used; }; +#define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) + /* This struct is used by the specializer * It should be treated as an opaque blob * by code other than the specializer and interpreter. */ diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index b37b7defe84d1c..07a333f98fa0a9 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -254,6 +254,33 @@ [...] test.test_metaclass.ObscureException +Test setting attributes with a non-base type in mro() (gh-127773). + + >>> class Base: + ... value = 1 + ... + >>> class Meta(type): + ... def mro(cls): + ... return (cls, Base, object) + ... + >>> class WeirdClass(metaclass=Meta): + ... pass + ... + >>> Base.value + 1 + >>> WeirdClass.value + 1 + >>> Base.value = 2 + >>> Base.value + 2 + >>> WeirdClass.value + 2 + >>> Base.value = 3 + >>> Base.value + 3 + >>> WeirdClass.value + 3 + """ import sys diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst new file mode 100644 index 00000000000000..7e68b3fecd1c08 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst @@ -0,0 +1 @@ +Do not use the type attribute cache for types with incompatible :term:`MRO`. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0f5ebc6f90773d..d8f5f6d9cb2366 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -992,6 +992,7 @@ static void set_version_unlocked(PyTypeObject *tp, unsigned int version) { ASSERT_TYPE_LOCK_HELD(); + assert(version == 0 || (tp->tp_versions_used != _Py_ATTR_CACHE_UNUSED)); #ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); // lookup the old version and set to null @@ -1148,6 +1149,10 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { PyObject *b = PyTuple_GET_ITEM(bases, i); PyTypeObject *cls = _PyType_CAST(b); + if (cls->tp_versions_used >= _Py_ATTR_CACHE_UNUSED) { + goto clear; + } + if (!is_subtype_with_mro(lookup_tp_mro(type), type, cls)) { goto clear; } @@ -1156,7 +1161,8 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); - set_version_unlocked(type, 0); /* 0 is not a valid version tag */ + set_version_unlocked(type, 0); /* 0 is not a valid version tag */ + type->tp_versions_used = _Py_ATTR_CACHE_UNUSED; if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1208,6 +1214,9 @@ _PyType_GetVersionForCurrentState(PyTypeObject *tp) #define MAX_VERSIONS_PER_CLASS 1000 +#if _Py_ATTR_CACHE_UNUSED < MAX_VERSIONS_PER_CLASS +#error "_Py_ATTR_CACHE_UNUSED must be bigger than max" +#endif static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) @@ -1225,6 +1234,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) return 0; } if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) { + /* (this includes `tp_versions_used == _Py_ATTR_CACHE_UNUSED`) */ return 0; } From 5044c2245c1519e117ba3977a4c8f608785326e7 Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Mon, 13 Jan 2025 05:14:59 -0800 Subject: [PATCH 12/61] gh-67748: DOC:Add summary table for str methods in stdtypes.rst (GH-1709) Co-authored-by: Ezio Melotti Co-authored-by: Oleg Iarygin Co-authored-by: Carol Willing --- Doc/library/stdtypes.rst | 94 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 191827526e890f..6050784264707b 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1548,6 +1548,100 @@ objects that compare equal might have different :attr:`~range.start`, single: str (built-in class); (see also string) pair: object; string +.. _text-methods-summary: + +Text and Binary Sequence Type Methods Summary +============================================= +The following table summarizes the text and binary sequence types methods by +category. + + ++--------------------------+-------------------------------------------+---------------------------------------------------+ +| Category | :class:`str` methods | :class:`bytes` and :class:`bytearray` methods | ++==========================+===========================================+===================================================+ +| Formatting | :meth:`str.format` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.format_map` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :ref:`f-strings` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :ref:`old-string-formatting` | :ref:`bytes-formatting` | ++--------------------------+------------------+------------------------+--------------------+------------------------------+ +| Searching and Replacing | :meth:`str.find` | :meth:`str.rfind` | :meth:`bytes.find` | :meth:`bytes.rfind` | +| +------------------+------------------------+--------------------+------------------------------+ +| | :meth:`str.index`| :meth:`str.rindex` | :meth:`bytes.index`| :meth:`bytes.rindex` | +| +------------------+------------------------+--------------------+------------------------------+ +| | :meth:`str.startswith` | :meth:`bytes.startswith` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.endswith` | :meth:`bytes.endswith` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.count` | :meth:`bytes.count` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.replace` | :meth:`bytes.replace` | ++--------------------------+-------------------+-----------------------+---------------------+-----------------------------+ +| Splitting and Joining | :meth:`str.split` | :meth:`str.rsplit` | :meth:`bytes.split` | :meth:`bytes.rsplit` | +| +-------------------+-----------------------+---------------------+-----------------------------+ +| | :meth:`str.splitlines` | :meth:`bytes.splitlines` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.partition` | :meth:`bytes.partition` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.rpartition` | :meth:`bytes.rpartition` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.join` | :meth:`bytes.join` | ++--------------------------+-------------------------------------------+---------------------------------------------------+ +| String Classification | :meth:`str.isalpha` | :meth:`bytes.isalpha` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isdecimal` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isdigit` | :meth:`bytes.isdigit` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isnumeric` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isalnum` | :meth:`bytes.isalnum` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isidentifier` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.islower` | :meth:`bytes.islower` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isupper` | :meth:`bytes.isupper` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.istitle` | :meth:`bytes.istitle` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isspace` | :meth:`bytes.isspace` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isprintable` | | ++--------------------------+-------------------------------------------+---------------------------------------------------+ +| Case Manipulation | :meth:`str.lower` | :meth:`bytes.lower` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.upper` | :meth:`bytes.upper` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.casefold` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.capitalize` | :meth:`bytes.capitalize` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.title` | :meth:`bytes.title` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.swapcase` | :meth:`bytes.swapcase` | ++--------------------------+-------------------+-----------------------+---------------------+-----------------------------+ +| Padding and Stripping | :meth:`str.ljust` | :meth:`str.rjust` | :meth:`bytes.ljust` | :meth:`bytes.rjust` | +| +-------------------+-----------------------+---------------------+-----------------------------+ +| | :meth:`str.center` | :meth:`bytes.center` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.expandtabs` | :meth:`bytes.expandtabs` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.strip` | :meth:`bytes.strip` | +| +--------------------+----------------------+----------------------+----------------------------+ +| | :meth:`str.lstrip` | :meth:`str.rstrip` | :meth:`bytes.lstrip` | :meth:`bytes.rstrip` | ++--------------------------+--------------------+----------------------+----------------------+----------------------------+ +| Translation and Encoding | :meth:`str.translate` | :meth:`bytes.translate` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.maketrans` | :meth:`bytes.maketrans` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.encode` | | +| +-------------------------------------------+---------------------------------------------------+ +| | | :meth:`bytes.decode` | ++--------------------------+-------------------------------------------+---------------------------------------------------+ + .. _textseq: Text Sequence Type --- :class:`str` From 8dfc743a1dae499d3b116b81ef6aad78708c45bc Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 13 Jan 2025 08:17:38 -0500 Subject: [PATCH 13/61] gh-128182: Add per-object memory access synchronization to `ctypes` (GH-128490) --- Doc/library/ctypes.rst | 30 +++ Lib/test/test_ctypes/test_arrays.py | 22 ++- ...-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst | 2 + Modules/_ctypes/_ctypes.c | 183 +++++++++++++----- Modules/_ctypes/ctypes.h | 44 +++++ 5 files changed, 234 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index f25bf94417c198..077a27919fc873 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -870,6 +870,36 @@ invalid non-\ ``NULL`` pointers would crash Python):: ValueError: NULL pointer access >>> +.. _ctypes-thread-safety: + +Thread safety without the GIL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.13, the :term:`GIL` may be disabled on :term:`experimental free threaded ` builds. +In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: + + .. code-block:: pycon + + >>> number = c_int(42) + >>> pointer_a = pointer(number) + >>> pointer_b = pointer(number) + +In the above, it's only safe for one object to read and write to the address at once if the GIL is disabled. +So, ``pointer_a`` can be shared and written to across multiple threads, but only if ``pointer_b`` +is not also attempting to do the same. If this is an issue, consider using a :class:`threading.Lock` +to synchronize access to memory: + + .. code-block:: pycon + + >>> import threading + >>> lock = threading.Lock() + >>> # Thread 1 + >>> with lock: + ... pointer_a.contents = 24 + >>> # Thread 2 + >>> with lock: + ... pointer_b.contents = 42 + .. _ctypes-type-conversions: diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index c80fdff5de685d..7f1f6cf58402c9 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -5,7 +5,7 @@ create_string_buffer, create_unicode_buffer, c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulonglong, c_float, c_double, c_longdouble) -from test.support import bigmemtest, _2G +from test.support import bigmemtest, _2G, threading_helper, Py_GIL_DISABLED from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -267,6 +267,26 @@ def test_bpo36504_signed_int_overflow(self): def test_large_array(self, size): c_char * size + @threading_helper.requires_working_threading() + @unittest.skipUnless(Py_GIL_DISABLED, "only meaningful if the GIL is disabled") + def test_thread_safety(self): + from threading import Thread + + buffer = (ctypes.c_char_p * 10)() + + def run(): + for i in range(100): + buffer.value = b"hello" + buffer[0] = b"j" + + with threading_helper.catch_threading_exception() as cm: + threads = (Thread(target=run) for _ in range(25)) + with threading_helper.start_threads(threads): + pass + + if cm.exc_value: + raise cm.exc_value + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst b/Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst new file mode 100644 index 00000000000000..038fecb5710436 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst @@ -0,0 +1,2 @@ +Fix crash when using :mod:`ctypes` pointers concurrently on the :term:`free +threaded ` build. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ede95bdf98bf76..c4d130a5ec1d52 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -598,7 +598,7 @@ StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) if (ptr == NULL) { return NULL; } - memcpy(ptr, self->b_ptr, self->b_size); + locked_memcpy_from(ptr, self, self->b_size); /* Create a Python object which calls PyMem_Free(ptr) in its deallocator. The object will be destroyed @@ -907,8 +907,7 @@ CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, result = generic_pycdata_new(st, (PyTypeObject *)type, NULL, NULL); if (result != NULL) { - memcpy(((CDataObject *)result)->b_ptr, - (char *)buffer->buf + offset, info->size); + locked_memcpy_to((CDataObject *) result, (char *)buffer->buf + offset, info->size); } return result; } @@ -1195,7 +1194,7 @@ PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) parg->tag = 'P'; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); - parg->value.p = *(void **)self->b_ptr; + parg->value.p = locked_deref(self); return parg; } @@ -1432,7 +1431,7 @@ CharArray_set_raw(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) goto fail; } - memcpy(self->b_ptr, ptr, size); + locked_memcpy_to(self, ptr, size); PyBuffer_Release(&view); return 0; @@ -1444,18 +1443,26 @@ CharArray_set_raw(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) static PyObject * CharArray_get_raw(CDataObject *self, void *Py_UNUSED(ignored)) { - return PyBytes_FromStringAndSize(self->b_ptr, self->b_size); + PyObject *res; + LOCK_PTR(self); + res = PyBytes_FromStringAndSize(self->b_ptr, self->b_size); + UNLOCK_PTR(self); + return res; } static PyObject * CharArray_get_value(CDataObject *self, void *Py_UNUSED(ignored)) { Py_ssize_t i; + PyObject *res; + LOCK_PTR(self); char *ptr = self->b_ptr; for (i = 0; i < self->b_size; ++i) if (*ptr++ == '\0') break; - return PyBytes_FromStringAndSize(self->b_ptr, i); + res = PyBytes_FromStringAndSize(self->b_ptr, i); + UNLOCK_PTR(self); + return res; } static int @@ -1486,9 +1493,11 @@ CharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored) } ptr = PyBytes_AS_STRING(value); + LOCK_PTR(self); memcpy(self->b_ptr, ptr, size); if (size < self->b_size) self->b_ptr[size] = '\0'; + UNLOCK_PTR(self); Py_DECREF(value); return 0; @@ -1506,11 +1515,15 @@ static PyObject * WCharArray_get_value(CDataObject *self, void *Py_UNUSED(ignored)) { Py_ssize_t i; + PyObject *res; wchar_t *ptr = (wchar_t *)self->b_ptr; + LOCK_PTR(self); for (i = 0; i < self->b_size/(Py_ssize_t)sizeof(wchar_t); ++i) if (*ptr++ == (wchar_t)0) break; - return PyUnicode_FromWideChar((wchar_t *)self->b_ptr, i); + res = PyUnicode_FromWideChar((wchar_t *)self->b_ptr, i); + UNLOCK_PTR(self); + return res; } static int @@ -1540,10 +1553,11 @@ WCharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored PyErr_SetString(PyExc_ValueError, "string too long"); return -1; } - if (PyUnicode_AsWideChar(value, (wchar_t *)self->b_ptr, size) < 0) { - return -1; - } - return 0; + Py_ssize_t rc; + LOCK_PTR(self); + rc = PyUnicode_AsWideChar(value, (wchar_t *)self->b_ptr, size); + UNLOCK_PTR(self); + return rc < 0 ? -1 : 0; } static PyGetSetDef WCharArray_getsets[] = { @@ -2053,6 +2067,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) parg->pffi_type = &ffi_type_pointer; parg->tag = 'P'; Py_INCREF(value); + // Function pointers don't change their contents, no need to lock parg->value.p = *(void **)func->b_ptr; parg->obj = value; return (PyObject *)parg; @@ -2079,7 +2094,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) parg->tag = 'Z'; parg->obj = Py_NewRef(value); /* Remember: b_ptr points to where the pointer is stored! */ - parg->value.p = *(void **)(((CDataObject *)value)->b_ptr); + parg->value.p = locked_deref((CDataObject *)value); return (PyObject *)parg; } } @@ -2196,7 +2211,7 @@ PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) parg->tag = fmt[0]; parg->pffi_type = fd->pffi_type; parg->obj = Py_NewRef(self); - memcpy(&parg->value, self->b_ptr, self->b_size); + locked_memcpy_from(&parg->value, self, self->b_size); return parg; } @@ -2697,7 +2712,7 @@ PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) parg->tag = 'P'; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); - parg->value.p = *(void **)self->b_ptr; + parg->value.p = locked_deref(self); return parg; } @@ -3017,8 +3032,12 @@ PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls) if (dict == NULL) { return NULL; } + PyObject *bytes; + LOCK_PTR(self); + bytes = PyBytes_FromStringAndSize(self->b_ptr, self->b_size); + UNLOCK_PTR(self); return Py_BuildValue("O(O(NN))", st->_unpickle, Py_TYPE(myself), dict, - PyBytes_FromStringAndSize(self->b_ptr, self->b_size)); + bytes); } static PyObject * @@ -3036,7 +3055,10 @@ PyCData_setstate(PyObject *myself, PyObject *args) } if (len > self->b_size) len = self->b_size; + // XXX Can we use locked_memcpy_to()? + LOCK_PTR(self); memmove(self->b_ptr, data, len); + UNLOCK_PTR(self); mydict = PyObject_GetAttrString(myself, "__dict__"); if (mydict == NULL) { return NULL; @@ -3094,6 +3116,12 @@ static PyType_Spec pycdata_spec = { static int PyCData_MallocBuffer(CDataObject *obj, StgInfo *info) { + /* We don't have to lock in this function, because it's only + * used in constructors and therefore does not have concurrent + * access. + */ + assert (Py_REFCNT(obj) == 1); + if ((size_t)info->size <= sizeof(obj->b_value)) { /* No need to call malloc, can use the default buffer */ obj->b_ptr = (char *)&obj->b_value; @@ -3219,15 +3247,28 @@ PyObject * PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *adr) { - if (getfunc) - return getfunc(adr, size); +#ifdef Py_GIL_DISABLED + // This isn't used if the GIL is enabled, so it causes a compiler warning. + CDataObject *cdata = (CDataObject *)src; +#endif + if (getfunc) { + PyObject *res; + LOCK_PTR(cdata); + res = getfunc(adr, size); + UNLOCK_PTR(cdata); + return res; + } assert(type); StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } if (info && info->getfunc && !_ctypes_simple_instance(st, type)) { - return info->getfunc(adr, size); + PyObject *res; + LOCK_PTR(cdata); + res = info->getfunc(adr, size); + UNLOCK_PTR(cdata); + return res; } return PyCData_FromBaseObj(st, type, src, index, adr); } @@ -3244,15 +3285,24 @@ _PyCData_set(ctypes_state *st, int err; if (setfunc) { - return setfunc(ptr, value, size); + PyObject *res; + LOCK_PTR(dst); + res = setfunc(ptr, value, size); + UNLOCK_PTR(dst); + return res; } if (!CDataObject_Check(st, value)) { StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - if (info && info->setfunc) - return info->setfunc(ptr, value, size); + if (info && info->setfunc) { + PyObject *res; + LOCK_PTR(dst); + res = info->setfunc(ptr, value, size); + UNLOCK_PTR(dst); + return res; + } /* If value is a tuple, we try to call the type with the tuple and use the result! @@ -3272,7 +3322,9 @@ _PyCData_set(ctypes_state *st, Py_DECREF(ob); return result; } else if (value == Py_None && PyCPointerTypeObject_Check(st, type)) { + LOCK_PTR(dst); *(void **)ptr = NULL; + UNLOCK_PTR(dst); Py_RETURN_NONE; } else { PyErr_Format(PyExc_TypeError, @@ -3288,9 +3340,7 @@ _PyCData_set(ctypes_state *st, if (err == -1) return NULL; if (err) { - memcpy(ptr, - src->b_ptr, - size); + locked_memcpy_from(ptr, src, size); if (PyCPointerTypeObject_Check(st, type)) { /* XXX */ @@ -3324,7 +3374,9 @@ _PyCData_set(ctypes_state *st, ((PyTypeObject *)type)->tp_name); return NULL; } + LOCK_PTR(src); *(void **)ptr = src->b_ptr; + UNLOCK_PTR(src); keep = GetKeepedObjects(src); if (keep == NULL) @@ -3891,6 +3943,8 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) self->paramflags = Py_XNewRef(paramflags); + // No other threads can have this object, no need to + // lock it. *(void **)self->b_ptr = address; Py_INCREF(dll); Py_DECREF(ftuple); @@ -4823,18 +4877,24 @@ Array_subscript(PyObject *myself, PyObject *item) if (slicelen <= 0) return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { - return PyBytes_FromStringAndSize(ptr + start, - slicelen); + PyObject *res; + LOCK_PTR(self); + res = PyBytes_FromStringAndSize(ptr + start, + slicelen); + UNLOCK_PTR(self); + return res; } dest = (char *)PyMem_Malloc(slicelen); if (dest == NULL) return PyErr_NoMemory(); + LOCK_PTR(self); for (cur = start, i = 0; i < slicelen; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyBytes_FromStringAndSize(dest, slicelen); PyMem_Free(dest); @@ -4847,8 +4907,12 @@ Array_subscript(PyObject *myself, PyObject *item) if (slicelen <= 0) return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { - return PyUnicode_FromWideChar(ptr + start, - slicelen); + PyObject *res; + LOCK_PTR(self); + res = PyUnicode_FromWideChar(ptr + start, + slicelen); + UNLOCK_PTR(self); + return res; } dest = PyMem_New(wchar_t, slicelen); @@ -4857,10 +4921,12 @@ Array_subscript(PyObject *myself, PyObject *item) return NULL; } + LOCK_PTR(self); for (cur = start, i = 0; i < slicelen; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyUnicode_FromWideChar(dest, slicelen); PyMem_Free(dest); @@ -5118,7 +5184,9 @@ Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) assert(info); /* Cannot be NULL for CDataObject instances */ assert(info->setfunc); + LOCK_PTR(self); result = info->setfunc(self->b_ptr, value, info->size); + UNLOCK_PTR(self); if (!result) return -1; @@ -5147,7 +5215,11 @@ Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) } assert(info); /* Cannot be NULL for CDataObject instances */ assert(info->getfunc); - return info->getfunc(self->b_ptr, self->b_size); + PyObject *res; + LOCK_PTR(self); + res = info->getfunc(self->b_ptr, self->b_size); + UNLOCK_PTR(self); + return res; } static PyGetSetDef Simple_getsets[] = { @@ -5183,7 +5255,11 @@ static PyMethodDef Simple_methods[] = { static int Simple_bool(CDataObject *self) { - return memcmp(self->b_ptr, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", self->b_size); + int cmp; + LOCK_PTR(self); + cmp = memcmp(self->b_ptr, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", self->b_size); + UNLOCK_PTR(self); + return cmp; } /* "%s(%s)" % (self.__class__.__name__, self.value) */ @@ -5239,8 +5315,9 @@ Pointer_item(PyObject *myself, Py_ssize_t index) Py_ssize_t size; Py_ssize_t offset; PyObject *proto; + void *deref = locked_deref(self); - if (*(void **)self->b_ptr == NULL) { + if (deref == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return NULL; @@ -5267,7 +5344,7 @@ Pointer_item(PyObject *myself, Py_ssize_t index) offset = index * iteminfo->size; return PyCData_get(st, proto, stginfo->getfunc, (PyObject *)self, - index, size, (*(char **)self->b_ptr) + offset); + index, size, (char *)((char *)deref + offset)); } static int @@ -5284,7 +5361,8 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - if (*(void **)self->b_ptr == NULL) { + void *deref = locked_deref(self); + if (deref == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return -1; @@ -5311,13 +5389,14 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) offset = index * iteminfo->size; return PyCData_set(st, (PyObject *)self, proto, stginfo->setfunc, value, - index, size, (*(char **)self->b_ptr) + offset); + index, size, ((char *)deref + offset)); } static PyObject * Pointer_get_contents(CDataObject *self, void *closure) { - if (*(void **)self->b_ptr == NULL) { + void *deref = locked_deref(self); + if (deref == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return NULL; @@ -5332,7 +5411,7 @@ Pointer_get_contents(CDataObject *self, void *closure) return PyCData_FromBaseObj(st, stginfo->proto, (PyObject *)self, 0, - *(void **)self->b_ptr); + deref); } static int @@ -5367,7 +5446,7 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) } dst = (CDataObject *)value; - *(void **)self->b_ptr = dst->b_ptr; + locked_deref_assign(self, dst->b_ptr); /* A Pointer instance must keep the value it points to alive. So, a @@ -5502,41 +5581,53 @@ Pointer_subscript(PyObject *myself, PyObject *item) } assert(iteminfo); if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - char *ptr = *(char **)self->b_ptr; + char *ptr = locked_deref(self); char *dest; if (len <= 0) return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { - return PyBytes_FromStringAndSize(ptr + start, - len); + PyObject *res; + LOCK_PTR(self); + res = PyBytes_FromStringAndSize(ptr + start, + len); + UNLOCK_PTR(self); + return res; } dest = (char *)PyMem_Malloc(len); if (dest == NULL) return PyErr_NoMemory(); + LOCK_PTR(self); for (cur = start, i = 0; i < len; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyBytes_FromStringAndSize(dest, len); PyMem_Free(dest); return np; } if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - wchar_t *ptr = *(wchar_t **)self->b_ptr; + wchar_t *ptr = locked_deref(self); wchar_t *dest; if (len <= 0) return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { - return PyUnicode_FromWideChar(ptr + start, - len); + PyObject *res; + LOCK_PTR(self); + res = PyUnicode_FromWideChar(ptr + start, + len); + UNLOCK_PTR(self); + return res; } dest = PyMem_New(wchar_t, len); if (dest == NULL) return PyErr_NoMemory(); + LOCK_PTR(self); for (cur = start, i = 0; i < len; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyUnicode_FromWideChar(dest, len); PyMem_Free(dest); return np; @@ -5562,7 +5653,7 @@ Pointer_subscript(PyObject *myself, PyObject *item) static int Pointer_bool(CDataObject *self) { - return (*(void **)self->b_ptr != NULL); + return locked_deref(self) != NULL; } static PyType_Slot pycpointer_slots[] = { @@ -5770,7 +5861,7 @@ cast(void *ptr, PyObject *src, PyObject *ctype) } } /* Should we assert that result is a pointer type? */ - memcpy(result->b_ptr, &ptr, sizeof(void *)); + locked_memcpy_to(result, &ptr, sizeof(void *)); return (PyObject *)result; failed: diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 45e00a538fb5a5..cc09639e21f7c2 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -543,3 +543,47 @@ PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) info->initialized = 1; return info; } + +/* See discussion in gh-128490. The plan here is to eventually use a per-object + * lock rather than a critical section, but that work is for later. */ +#ifdef Py_GIL_DISABLED +# define LOCK_PTR(self) Py_BEGIN_CRITICAL_SECTION(self) +# define UNLOCK_PTR(self) Py_END_CRITICAL_SECTION() +#else +# define LOCK_PTR(self) +# define UNLOCK_PTR(self) +#endif + +static inline void +locked_memcpy_to(CDataObject *self, void *buf, Py_ssize_t size) +{ + LOCK_PTR(self); + (void)memcpy(self->b_ptr, buf, size); + UNLOCK_PTR(self); +} + +static inline void +locked_memcpy_from(void *buf, CDataObject *self, Py_ssize_t size) +{ + LOCK_PTR(self); + (void)memcpy(buf, self->b_ptr, size); + UNLOCK_PTR(self); +} + +static inline void * +locked_deref(CDataObject *self) +{ + void *ptr; + LOCK_PTR(self); + ptr = *(void **)self->b_ptr; + UNLOCK_PTR(self); + return ptr; +} + +static inline void +locked_deref_assign(CDataObject *self, void *new_ptr) +{ + LOCK_PTR(self); + *(void **)self->b_ptr = new_ptr; + UNLOCK_PTR(self); +} From 4533036e50f1e604e8c06710d9e19dd700f89c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:25:04 +0100 Subject: [PATCH 14/61] gh-111178: fix UBSan failures in `Objects/codeobject.c` (GH-128240) --- Objects/codeobject.c | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index ae232cae86799b..15b36a868a47df 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -108,6 +108,8 @@ PyCode_ClearWatcher(int watcher_id) * generic helpers ******************/ +#define _PyCodeObject_CAST(op) (assert(PyCode_Check(op)), (PyCodeObject *)(op)) + static int should_intern_string(PyObject *o) { @@ -1865,11 +1867,12 @@ free_monitoring_data(_PyCoMonitoringData *data) } static void -code_dealloc(PyCodeObject *co) +code_dealloc(PyObject *self) { - _PyObject_ResurrectStart((PyObject *)co); + PyCodeObject *co = _PyCodeObject_CAST(self); + _PyObject_ResurrectStart(self); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); - if (_PyObject_ResurrectEnd((PyObject *)co)) { + if (_PyObject_ResurrectEnd(self)) { return; } @@ -1918,7 +1921,7 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co->_co_cached); } if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)co); + PyObject_ClearWeakRefs(self); } free_monitoring_data(co->_co_monitoring); #ifdef Py_GIL_DISABLED @@ -1939,7 +1942,7 @@ code_dealloc(PyCodeObject *co) static int code_traverse(PyObject *self, visitproc visit, void *arg) { - PyCodeObject *co = (PyCodeObject*)self; + PyCodeObject *co = _PyCodeObject_CAST(self); Py_VISIT(co->co_consts); return 0; } @@ -1948,7 +1951,7 @@ code_traverse(PyObject *self, visitproc visit, void *arg) static PyObject * code_repr(PyObject *self) { - PyCodeObject *co = (PyCodeObject*)self; + PyCodeObject *co = _PyCodeObject_CAST(self); int lineno; if (co->co_firstlineno != 0) lineno = co->co_firstlineno; @@ -2057,7 +2060,7 @@ code_richcompare(PyObject *self, PyObject *other, int op) static Py_hash_t code_hash(PyObject *self) { - PyCodeObject *co = (PyCodeObject*)self; + PyCodeObject *co = _PyCodeObject_CAST(self); Py_uhash_t uhash = 20221211; #define SCRAMBLE_IN(H) do { \ uhash ^= (Py_uhash_t)(H); \ @@ -2120,7 +2123,7 @@ static PyMemberDef code_memberlist[] = { static PyObject * code_getlnotab(PyObject *self, void *closure) { - PyCodeObject *code = (PyCodeObject*)self; + PyCodeObject *code = _PyCodeObject_CAST(self); if (PyErr_WarnEx(PyExc_DeprecationWarning, "co_lnotab is deprecated, use co_lines instead.", 1) < 0) { @@ -2132,28 +2135,28 @@ code_getlnotab(PyObject *self, void *closure) static PyObject * code_getvarnames(PyObject *self, void *closure) { - PyCodeObject *code = (PyCodeObject*)self; + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetVarnames(code); } static PyObject * code_getcellvars(PyObject *self, void *closure) { - PyCodeObject *code = (PyCodeObject*)self; + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetCellvars(code); } static PyObject * code_getfreevars(PyObject *self, void *closure) { - PyCodeObject *code = (PyCodeObject*)self; + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetFreevars(code); } static PyObject * code_getcodeadaptive(PyObject *self, void *closure) { - PyCodeObject *code = (PyCodeObject*)self; + PyCodeObject *code = _PyCodeObject_CAST(self); return PyBytes_FromStringAndSize(code->co_code_adaptive, _PyCode_NBYTES(code)); } @@ -2161,7 +2164,7 @@ code_getcodeadaptive(PyObject *self, void *closure) static PyObject * code_getcode(PyObject *self, void *closure) { - PyCodeObject *code = (PyCodeObject*)self; + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetCode(code); } @@ -2180,7 +2183,7 @@ static PyGetSetDef code_getsetlist[] = { static PyObject * code_sizeof(PyObject *self, PyObject *Py_UNUSED(args)) { - PyCodeObject *co = (PyCodeObject*)self; + PyCodeObject *co = _PyCodeObject_CAST(self); size_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; if (co_extra != NULL) { @@ -2193,13 +2196,14 @@ code_sizeof(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * code_linesiterator(PyObject *self, PyObject *Py_UNUSED(args)) { - PyCodeObject *code = (PyCodeObject*)self; + PyCodeObject *code = _PyCodeObject_CAST(self); return (PyObject *)new_linesiterator(code); } static PyObject * -code_branchesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) +code_branchesiterator(PyObject *self, PyObject *Py_UNUSED(args)) { + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyInstrumentation_BranchesIterator(code); } @@ -2343,7 +2347,7 @@ code__varname_from_oparg_impl(PyCodeObject *self, int oparg) static struct PyMethodDef code_methods[] = { {"__sizeof__", code_sizeof, METH_NOARGS}, {"co_lines", code_linesiterator, METH_NOARGS}, - {"co_branches", (PyCFunction)code_branchesiterator, METH_NOARGS}, + {"co_branches", code_branchesiterator, METH_NOARGS}, {"co_positions", code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF @@ -2358,7 +2362,7 @@ PyTypeObject PyCode_Type = { "code", offsetof(PyCodeObject, co_code_adaptive), sizeof(_Py_CODEUNIT), - (destructor)code_dealloc, /* tp_dealloc */ + code_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ From bf64a582f00a030fee11b7f89c6a02ea967990ca Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 13 Jan 2025 09:36:54 -0500 Subject: [PATCH 15/61] gh-128400: Only show the current thread in `Py_FatalError` on the free-threaded build (#128758) --- Lib/test/test_capi/test_misc.py | 14 ++++++++------ Lib/test/test_faulthandler.py | 14 ++++++++------ .../2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst | 2 ++ Python/pylifecycle.c | 4 ++++ 4 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index b62bc4c2ecd980..99d9a757759dcd 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -75,6 +75,8 @@ class InstanceMethod: id = _testcapi.instancemethod(id) testfunction = _testcapi.instancemethod(testfunction) +CURRENT_THREAD_REGEX = r'Current thread.*:\n' if not support.Py_GIL_DISABLED else r'Stack .*:\n' + class CAPITest(unittest.TestCase): def test_instancemethod(self): @@ -234,8 +236,8 @@ def test_return_null_without_error(self): r'Python runtime state: initialized\n' r'SystemError: ' r'returned NULL without setting an exception\n' - r'\n' - r'Current thread.*:\n' + r'\n' + + CURRENT_THREAD_REGEX + r' File .*", line 6 in \n') else: with self.assertRaises(SystemError) as cm: @@ -268,8 +270,8 @@ def test_return_result_with_error(self): r'SystemError: ' r'returned a result with an exception set\n' - r'\n' - r'Current thread.*:\n' + r'\n' + + CURRENT_THREAD_REGEX + r' File .*, line 6 in \n') else: with self.assertRaises(SystemError) as cm: @@ -298,8 +300,8 @@ def test_getitem_with_error(self): r'with an exception set\n' r'Python runtime state: initialized\n' r'ValueError: bug\n' - r'\n' - r'Current thread .* \(most recent call first\):\n' + r'\n' + + CURRENT_THREAD_REGEX + r' File .*, line 6 in \n' r'\n' r'Extension modules: _testcapi \(total: 1\)\n') diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 2088793cbb9387..75d303cd212c82 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -101,8 +101,7 @@ def check_error(self, code, lineno, fatal_error, *, Raise an error if the output doesn't match the expected format. """ all_threads_disabled = ( - (not py_fatal_error) - and all_threads + all_threads and (not sys._is_gil_enabled()) ) if all_threads and not all_threads_disabled: @@ -116,12 +115,15 @@ def check_error(self, code, lineno, fatal_error, *, if py_fatal_error: regex.append("Python runtime state: initialized") regex.append('') - if all_threads_disabled: + if all_threads_disabled and not py_fatal_error: regex.append("") regex.append(fr'{header} \(most recent call first\):') - if garbage_collecting and not all_threads_disabled: - regex.append(' Garbage-collecting') - regex.append(fr' File "", line {lineno} in {function}') + if support.Py_GIL_DISABLED and py_fatal_error and not know_current_thread: + regex.append(" ") + else: + if garbage_collecting and not all_threads_disabled: + regex.append(' Garbage-collecting') + regex.append(fr' File "", line {lineno} in {function}') regex = '\n'.join(regex) if other_regex: diff --git a/Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst b/Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst new file mode 100644 index 00000000000000..b9c117b269434c --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst @@ -0,0 +1,2 @@ +:c:func:`Py_FatalError` no longer shows all threads on the :term:`free +threaded ` build to prevent crashes. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8a15a09a01dbf7..9a8a92aaceb0be 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -3034,7 +3034,11 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PUTS(fd, "\n"); /* display the current Python stack */ +#ifndef Py_GIL_DISABLED _Py_DumpTracebackThreads(fd, interp, tstate); +#else + _Py_DumpTraceback(fd, tstate); +#endif } /* Print the current exception (if an exception is set) with its traceback, From 75214f87f1ddd1d7b9a5b663a9c688b1bd41c098 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Mon, 13 Jan 2025 20:08:33 +0530 Subject: [PATCH 16/61] gh-128421: make getters and setters of `BaseException` thread safe (#128728) --- Objects/clinic/exceptions.c.h | 311 ++++++++++++++++++++++++++++++++++ Objects/exceptions.c | 298 ++++++++++++++++++++++---------- 2 files changed, 520 insertions(+), 89 deletions(-) create mode 100644 Objects/clinic/exceptions.c.h diff --git a/Objects/clinic/exceptions.c.h b/Objects/clinic/exceptions.c.h new file mode 100644 index 00000000000000..caa5b0c63e53c5 --- /dev/null +++ b/Objects/clinic/exceptions.c.h @@ -0,0 +1,311 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_BadArgument() + +PyDoc_STRVAR(BaseException___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define BASEEXCEPTION___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)BaseException___reduce__, METH_NOARGS, BaseException___reduce____doc__}, + +static PyObject * +BaseException___reduce___impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___reduce__(PyBaseExceptionObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___reduce___impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseException___setstate____doc__, +"__setstate__($self, state, /)\n" +"--\n" +"\n"); + +#define BASEEXCEPTION___SETSTATE___METHODDEF \ + {"__setstate__", (PyCFunction)BaseException___setstate__, METH_O, BaseException___setstate____doc__}, + +static PyObject * +BaseException___setstate___impl(PyBaseExceptionObject *self, PyObject *state); + +static PyObject * +BaseException___setstate__(PyBaseExceptionObject *self, PyObject *state) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___setstate___impl(self, state); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseException_with_traceback__doc__, +"with_traceback($self, tb, /)\n" +"--\n" +"\n" +"Set self.__traceback__ to tb and return self."); + +#define BASEEXCEPTION_WITH_TRACEBACK_METHODDEF \ + {"with_traceback", (PyCFunction)BaseException_with_traceback, METH_O, BaseException_with_traceback__doc__}, + +static PyObject * +BaseException_with_traceback_impl(PyBaseExceptionObject *self, PyObject *tb); + +static PyObject * +BaseException_with_traceback(PyBaseExceptionObject *self, PyObject *tb) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_with_traceback_impl(self, tb); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseException_add_note__doc__, +"add_note($self, note, /)\n" +"--\n" +"\n" +"Add a note to the exception"); + +#define BASEEXCEPTION_ADD_NOTE_METHODDEF \ + {"add_note", (PyCFunction)BaseException_add_note, METH_O, BaseException_add_note__doc__}, + +static PyObject * +BaseException_add_note_impl(PyBaseExceptionObject *self, PyObject *note); + +static PyObject * +BaseException_add_note(PyBaseExceptionObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *note; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("add_note", "argument", "str", arg); + goto exit; + } + note = arg; + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_add_note_impl(self, note); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if !defined(BaseException_args_DOCSTR) +# define BaseException_args_DOCSTR NULL +#endif +#if defined(BASEEXCEPTION_ARGS_GETSETDEF) +# undef BASEEXCEPTION_ARGS_GETSETDEF +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", (getter)BaseException_args_get, (setter)BaseException_args_set, BaseException_args_DOCSTR}, +#else +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", (getter)BaseException_args_get, NULL, BaseException_args_DOCSTR}, +#endif + +static PyObject * +BaseException_args_get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException_args_get(PyBaseExceptionObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_args_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException_args_DOCSTR) +# define BaseException_args_DOCSTR NULL +#endif +#if defined(BASEEXCEPTION_ARGS_GETSETDEF) +# undef BASEEXCEPTION_ARGS_GETSETDEF +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", (getter)BaseException_args_get, (setter)BaseException_args_set, BaseException_args_DOCSTR}, +#else +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", NULL, (setter)BaseException_args_set, NULL}, +#endif + +static int +BaseException_args_set_impl(PyBaseExceptionObject *self, PyObject *value); + +static int +BaseException_args_set(PyBaseExceptionObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_args_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___traceback___DOCSTR) +# define BaseException___traceback___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___TRACEBACK___GETSETDEF) +# undef BASEEXCEPTION___TRACEBACK___GETSETDEF +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", (getter)BaseException___traceback___get, (setter)BaseException___traceback___set, BaseException___traceback___DOCSTR}, +#else +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", (getter)BaseException___traceback___get, NULL, BaseException___traceback___DOCSTR}, +#endif + +static PyObject * +BaseException___traceback___get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___traceback___get(PyBaseExceptionObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___traceback___get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___traceback___DOCSTR) +# define BaseException___traceback___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___TRACEBACK___GETSETDEF) +# undef BASEEXCEPTION___TRACEBACK___GETSETDEF +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", (getter)BaseException___traceback___get, (setter)BaseException___traceback___set, BaseException___traceback___DOCSTR}, +#else +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", NULL, (setter)BaseException___traceback___set, NULL}, +#endif + +static int +BaseException___traceback___set_impl(PyBaseExceptionObject *self, + PyObject *value); + +static int +BaseException___traceback___set(PyBaseExceptionObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___traceback___set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___context___DOCSTR) +# define BaseException___context___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CONTEXT___GETSETDEF) +# undef BASEEXCEPTION___CONTEXT___GETSETDEF +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", (getter)BaseException___context___get, (setter)BaseException___context___set, BaseException___context___DOCSTR}, +#else +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", (getter)BaseException___context___get, NULL, BaseException___context___DOCSTR}, +#endif + +static PyObject * +BaseException___context___get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___context___get(PyBaseExceptionObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___context___get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___context___DOCSTR) +# define BaseException___context___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CONTEXT___GETSETDEF) +# undef BASEEXCEPTION___CONTEXT___GETSETDEF +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", (getter)BaseException___context___get, (setter)BaseException___context___set, BaseException___context___DOCSTR}, +#else +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", NULL, (setter)BaseException___context___set, NULL}, +#endif + +static int +BaseException___context___set_impl(PyBaseExceptionObject *self, + PyObject *value); + +static int +BaseException___context___set(PyBaseExceptionObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___context___set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___cause___DOCSTR) +# define BaseException___cause___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CAUSE___GETSETDEF) +# undef BASEEXCEPTION___CAUSE___GETSETDEF +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", (getter)BaseException___cause___get, (setter)BaseException___cause___set, BaseException___cause___DOCSTR}, +#else +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", (getter)BaseException___cause___get, NULL, BaseException___cause___DOCSTR}, +#endif + +static PyObject * +BaseException___cause___get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___cause___get(PyBaseExceptionObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___cause___get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___cause___DOCSTR) +# define BaseException___cause___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CAUSE___GETSETDEF) +# undef BASEEXCEPTION___CAUSE___GETSETDEF +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", (getter)BaseException___cause___get, (setter)BaseException___cause___set, BaseException___cause___DOCSTR}, +#else +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", NULL, (setter)BaseException___cause___set, NULL}, +#endif + +static int +BaseException___cause___set_impl(PyBaseExceptionObject *self, + PyObject *value); + +static int +BaseException___cause___set(PyBaseExceptionObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___cause___set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=58afcfd60057fc39 input=a9049054013a1b77]*/ diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 714f8c828afbc1..4df89edfaf3953 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -16,6 +16,13 @@ #include "osdefs.h" // SEP +#include "clinic/exceptions.c.h" + +/*[clinic input] +class BaseException "PyBaseExceptionObject *" "&PyExc_BaseException" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=90558eb0fbf8a3d0]*/ + /* Compatibility aliases */ PyObject *PyExc_EnvironmentError = NULL; // borrowed ref @@ -152,30 +159,50 @@ BaseException_traverse(PyBaseExceptionObject *self, visitproc visit, void *arg) static PyObject * BaseException_str(PyBaseExceptionObject *self) { + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); switch (PyTuple_GET_SIZE(self->args)) { case 0: - return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + res = Py_GetConstant(Py_CONSTANT_EMPTY_STR); + break; case 1: - return PyObject_Str(PyTuple_GET_ITEM(self->args, 0)); + res = PyObject_Str(PyTuple_GET_ITEM(self->args, 0)); + break; default: - return PyObject_Str(self->args); + res = PyObject_Str(self->args); + break; } + Py_END_CRITICAL_SECTION(); + return res; } static PyObject * BaseException_repr(PyBaseExceptionObject *self) { + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); const char *name = _PyType_Name(Py_TYPE(self)); - if (PyTuple_GET_SIZE(self->args) == 1) - return PyUnicode_FromFormat("%s(%R)", name, + if (PyTuple_GET_SIZE(self->args) == 1) { + res = PyUnicode_FromFormat("%s(%R)", name, PyTuple_GET_ITEM(self->args, 0)); - else - return PyUnicode_FromFormat("%s%R", name, self->args); + } + else { + res = PyUnicode_FromFormat("%s%R", name, self->args); + } + Py_END_CRITICAL_SECTION(); + return res; } /* Pickling support */ + +/*[clinic input] +@critical_section +BaseException.__reduce__ +[clinic start generated code]*/ + static PyObject * -BaseException_reduce(PyBaseExceptionObject *self, PyObject *Py_UNUSED(ignored)) +BaseException___reduce___impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=af87c1247ef98748 input=283be5a10d9c964f]*/ { if (self->args && self->dict) return PyTuple_Pack(3, Py_TYPE(self), self->args, self->dict); @@ -188,8 +215,17 @@ BaseException_reduce(PyBaseExceptionObject *self, PyObject *Py_UNUSED(ignored)) * all their attributes in the __dict__. Code is taken from cPickle's * load_build function. */ + +/*[clinic input] +@critical_section +BaseException.__setstate__ + state: object + / +[clinic start generated code]*/ + static PyObject * -BaseException_setstate(PyObject *self, PyObject *state) +BaseException___setstate___impl(PyBaseExceptionObject *self, PyObject *state) +/*[clinic end generated code: output=f3834889950453ab input=5524b61cfe9b9856]*/ { PyObject *d_key, *d_value; Py_ssize_t i = 0; @@ -202,7 +238,7 @@ BaseException_setstate(PyObject *self, PyObject *state) while (PyDict_Next(state, &i, &d_key, &d_value)) { Py_INCREF(d_key); Py_INCREF(d_value); - int res = PyObject_SetAttr(self, d_key, d_value); + int res = PyObject_SetAttr((PyObject *)self, d_key, d_value); Py_DECREF(d_value); Py_DECREF(d_key); if (res < 0) { @@ -213,18 +249,26 @@ BaseException_setstate(PyObject *self, PyObject *state) Py_RETURN_NONE; } + +/*[clinic input] +@critical_section +BaseException.with_traceback + tb: object + / + +Set self.__traceback__ to tb and return self. +[clinic start generated code]*/ + static PyObject * -BaseException_with_traceback(PyObject *self, PyObject *tb) { - if (PyException_SetTraceback(self, tb)) +BaseException_with_traceback_impl(PyBaseExceptionObject *self, PyObject *tb) +/*[clinic end generated code: output=81e92f2387927f10 input=b5fb64d834717e36]*/ +{ + if (BaseException___traceback___set_impl(self, tb) < 0){ return NULL; - + } return Py_NewRef(self); } -PyDoc_STRVAR(with_traceback_doc, -"Exception.with_traceback(tb) --\n\ - set self.__traceback__ to tb and return self."); - static inline PyBaseExceptionObject* _PyBaseExceptionObject_cast(PyObject *exc) { @@ -232,18 +276,21 @@ _PyBaseExceptionObject_cast(PyObject *exc) return (PyBaseExceptionObject *)exc; } +/*[clinic input] +@critical_section +BaseException.add_note + note: object(subclass_of="&PyUnicode_Type") + / + +Add a note to the exception +[clinic start generated code]*/ + static PyObject * -BaseException_add_note(PyObject *self, PyObject *note) +BaseException_add_note_impl(PyBaseExceptionObject *self, PyObject *note) +/*[clinic end generated code: output=fb7cbcba611c187b input=e60a6b6e9596acaf]*/ { - if (!PyUnicode_Check(note)) { - PyErr_Format(PyExc_TypeError, - "note must be a str, not '%s'", - Py_TYPE(note)->tp_name); - return NULL; - } - PyObject *notes; - if (PyObject_GetOptionalAttr(self, &_Py_ID(__notes__), ¬es) < 0) { + if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(__notes__), ¬es) < 0) { return NULL; } if (notes == NULL) { @@ -251,7 +298,7 @@ BaseException_add_note(PyObject *self, PyObject *note) if (notes == NULL) { return NULL; } - if (PyObject_SetAttr(self, &_Py_ID(__notes__), notes) < 0) { + if (PyObject_SetAttr((PyObject *)self, &_Py_ID(__notes__), notes) < 0) { Py_DECREF(notes); return NULL; } @@ -269,22 +316,23 @@ BaseException_add_note(PyObject *self, PyObject *note) Py_RETURN_NONE; } -PyDoc_STRVAR(add_note_doc, -"Exception.add_note(note) --\n\ - add a note to the exception"); - static PyMethodDef BaseException_methods[] = { - {"__reduce__", (PyCFunction)BaseException_reduce, METH_NOARGS }, - {"__setstate__", (PyCFunction)BaseException_setstate, METH_O }, - {"with_traceback", (PyCFunction)BaseException_with_traceback, METH_O, - with_traceback_doc}, - {"add_note", (PyCFunction)BaseException_add_note, METH_O, - add_note_doc}, - {NULL, NULL, 0, NULL}, + BASEEXCEPTION___REDUCE___METHODDEF + BASEEXCEPTION___SETSTATE___METHODDEF + BASEEXCEPTION_WITH_TRACEBACK_METHODDEF + BASEEXCEPTION_ADD_NOTE_METHODDEF + {NULL, NULL, 0, NULL}, }; +/*[clinic input] +@critical_section +@getter +BaseException.args +[clinic start generated code]*/ + static PyObject * -BaseException_get_args(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) +BaseException_args_get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=e02e34e35cf4d677 input=64282386e4d7822d]*/ { if (self->args == NULL) { Py_RETURN_NONE; @@ -292,23 +340,37 @@ BaseException_get_args(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) return Py_NewRef(self->args); } +/*[clinic input] +@critical_section +@setter +BaseException.args +[clinic start generated code]*/ + static int -BaseException_set_args(PyBaseExceptionObject *self, PyObject *val, void *Py_UNUSED(ignored)) +BaseException_args_set_impl(PyBaseExceptionObject *self, PyObject *value) +/*[clinic end generated code: output=331137e11d8f9e80 input=2400047ea5970a84]*/ { PyObject *seq; - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "args may not be deleted"); return -1; } - seq = PySequence_Tuple(val); + seq = PySequence_Tuple(value); if (!seq) return -1; Py_XSETREF(self->args, seq); return 0; } +/*[clinic input] +@critical_section +@getter +BaseException.__traceback__ +[clinic start generated code]*/ + static PyObject * -BaseException_get_tb(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) +BaseException___traceback___get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=17cf874a52339398 input=a2277f0de62170cf]*/ { if (self->traceback == NULL) { Py_RETURN_NONE; @@ -316,17 +378,26 @@ BaseException_get_tb(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) return Py_NewRef(self->traceback); } + +/*[clinic input] +@critical_section +@setter +BaseException.__traceback__ +[clinic start generated code]*/ + static int -BaseException_set_tb(PyBaseExceptionObject *self, PyObject *tb, void *Py_UNUSED(ignored)) +BaseException___traceback___set_impl(PyBaseExceptionObject *self, + PyObject *value) +/*[clinic end generated code: output=a82c86d9f29f48f0 input=12676035676badad]*/ { - if (tb == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "__traceback__ may not be deleted"); return -1; } - if (PyTraceBack_Check(tb)) { - Py_XSETREF(self->traceback, Py_NewRef(tb)); + if (PyTraceBack_Check(value)) { + Py_XSETREF(self->traceback, Py_NewRef(value)); } - else if (tb == Py_None) { + else if (value == Py_None) { Py_CLEAR(self->traceback); } else { @@ -337,73 +408,100 @@ BaseException_set_tb(PyBaseExceptionObject *self, PyObject *tb, void *Py_UNUSED( return 0; } +/*[clinic input] +@critical_section +@getter +BaseException.__context__ +[clinic start generated code]*/ + static PyObject * -BaseException_get_context(PyObject *self, void *Py_UNUSED(ignored)) +BaseException___context___get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=6ec5d296ce8d1c93 input=b2d22687937e66ab]*/ { - PyObject *res = PyException_GetContext(self); - if (res) - return res; /* new reference already returned above */ - Py_RETURN_NONE; + if (self->context == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(self->context); } +/*[clinic input] +@critical_section +@setter +BaseException.__context__ +[clinic start generated code]*/ + static int -BaseException_set_context(PyObject *self, PyObject *arg, void *Py_UNUSED(ignored)) +BaseException___context___set_impl(PyBaseExceptionObject *self, + PyObject *value) +/*[clinic end generated code: output=b4cb52dcca1da3bd input=c0971adf47fa1858]*/ { - if (arg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "__context__ may not be deleted"); return -1; - } else if (arg == Py_None) { - arg = NULL; - } else if (!PyExceptionInstance_Check(arg)) { + } else if (value == Py_None) { + value = NULL; + } else if (!PyExceptionInstance_Check(value)) { PyErr_SetString(PyExc_TypeError, "exception context must be None " "or derive from BaseException"); return -1; } else { - /* PyException_SetContext steals this reference */ - Py_INCREF(arg); + Py_INCREF(value); } - PyException_SetContext(self, arg); + Py_XSETREF(self->context, value); return 0; } +/*[clinic input] +@critical_section +@getter +BaseException.__cause__ +[clinic start generated code]*/ + static PyObject * -BaseException_get_cause(PyObject *self, void *Py_UNUSED(ignored)) +BaseException___cause___get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=987f6c4d8a0bdbab input=40e0eac427b6e602]*/ { - PyObject *res = PyException_GetCause(self); - if (res) - return res; /* new reference already returned above */ - Py_RETURN_NONE; + if (self->cause == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(self->cause); } +/*[clinic input] +@critical_section +@setter +BaseException.__cause__ +[clinic start generated code]*/ + static int -BaseException_set_cause(PyObject *self, PyObject *arg, void *Py_UNUSED(ignored)) +BaseException___cause___set_impl(PyBaseExceptionObject *self, + PyObject *value) +/*[clinic end generated code: output=6161315398aaf541 input=e1b403c0bde3f62a]*/ { - if (arg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "__cause__ may not be deleted"); return -1; - } else if (arg == Py_None) { - arg = NULL; - } else if (!PyExceptionInstance_Check(arg)) { + } else if (value == Py_None) { + value = NULL; + } else if (!PyExceptionInstance_Check(value)) { PyErr_SetString(PyExc_TypeError, "exception cause must be None " "or derive from BaseException"); return -1; } else { /* PyException_SetCause steals this reference */ - Py_INCREF(arg); + Py_INCREF(value); } - PyException_SetCause(self, arg); + PyException_SetCause((PyObject *)self, value); return 0; } static PyGetSetDef BaseException_getset[] = { {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, - {"args", (getter)BaseException_get_args, (setter)BaseException_set_args}, - {"__traceback__", (getter)BaseException_get_tb, (setter)BaseException_set_tb}, - {"__context__", BaseException_get_context, - BaseException_set_context, PyDoc_STR("exception context")}, - {"__cause__", BaseException_get_cause, - BaseException_set_cause, PyDoc_STR("exception cause")}, + BASEEXCEPTION_ARGS_GETSETDEF + BASEEXCEPTION___TRACEBACK___GETSETDEF + BASEEXCEPTION___CONTEXT___GETSETDEF + BASEEXCEPTION___CAUSE___GETSETDEF {NULL}, }; @@ -411,59 +509,81 @@ static PyGetSetDef BaseException_getset[] = { PyObject * PyException_GetTraceback(PyObject *self) { - PyBaseExceptionObject *base_self = _PyBaseExceptionObject_cast(self); - return Py_XNewRef(base_self->traceback); + PyObject *traceback; + Py_BEGIN_CRITICAL_SECTION(self); + traceback = Py_XNewRef(_PyBaseExceptionObject_cast(self)->traceback); + Py_END_CRITICAL_SECTION(); + return traceback; } int PyException_SetTraceback(PyObject *self, PyObject *tb) { - return BaseException_set_tb(_PyBaseExceptionObject_cast(self), tb, NULL); + int res; + Py_BEGIN_CRITICAL_SECTION(self); + res = BaseException___traceback___set_impl(_PyBaseExceptionObject_cast(self), tb); + Py_END_CRITICAL_SECTION(); + return res; } PyObject * PyException_GetCause(PyObject *self) { - PyObject *cause = _PyBaseExceptionObject_cast(self)->cause; - return Py_XNewRef(cause); + PyObject *cause; + Py_BEGIN_CRITICAL_SECTION(self); + cause = Py_XNewRef(_PyBaseExceptionObject_cast(self)->cause); + Py_END_CRITICAL_SECTION(); + return cause; } /* Steals a reference to cause */ void PyException_SetCause(PyObject *self, PyObject *cause) { + Py_BEGIN_CRITICAL_SECTION(self); PyBaseExceptionObject *base_self = _PyBaseExceptionObject_cast(self); base_self->suppress_context = 1; Py_XSETREF(base_self->cause, cause); + Py_END_CRITICAL_SECTION(); } PyObject * PyException_GetContext(PyObject *self) { - PyObject *context = _PyBaseExceptionObject_cast(self)->context; - return Py_XNewRef(context); + PyObject *context; + Py_BEGIN_CRITICAL_SECTION(self); + context = Py_XNewRef(_PyBaseExceptionObject_cast(self)->context); + Py_END_CRITICAL_SECTION(); + return context; } /* Steals a reference to context */ void PyException_SetContext(PyObject *self, PyObject *context) { + Py_BEGIN_CRITICAL_SECTION(self); Py_XSETREF(_PyBaseExceptionObject_cast(self)->context, context); + Py_END_CRITICAL_SECTION(); } PyObject * PyException_GetArgs(PyObject *self) { - PyObject *args = _PyBaseExceptionObject_cast(self)->args; - return Py_NewRef(args); + PyObject *args; + Py_BEGIN_CRITICAL_SECTION(self); + args = Py_NewRef(_PyBaseExceptionObject_cast(self)->args); + Py_END_CRITICAL_SECTION(); + return args; } void PyException_SetArgs(PyObject *self, PyObject *args) { + Py_BEGIN_CRITICAL_SECTION(self); Py_INCREF(args); Py_XSETREF(_PyBaseExceptionObject_cast(self)->args, args); + Py_END_CRITICAL_SECTION(); } const char * @@ -4136,7 +4256,7 @@ _PyException_AddNote(PyObject *exc, PyObject *note) Py_TYPE(exc)->tp_name); return -1; } - PyObject *r = BaseException_add_note(exc, note); + PyObject *r = BaseException_add_note(_PyBaseExceptionObject_cast(exc), note); int res = r == NULL ? -1 : 0; Py_XDECREF(r); return res; From 3efe28a40b136164f0d33c4f84dfcef7e123d1a0 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Mon, 13 Jan 2025 21:06:55 +0530 Subject: [PATCH 17/61] gh-128002: add more thread safety tests for asyncio (#128480) --- Lib/test/test_asyncio/test_free_threading.py | 54 ++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py index 90bddbf3a9dda1..8f4bba5f3b97d9 100644 --- a/Lib/test/test_asyncio/test_free_threading.py +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -7,6 +7,11 @@ threading_helper.requires_working_threading(module=True) + +class MyException(Exception): + pass + + def tearDownModule(): asyncio._set_event_loop_policy(None) @@ -53,6 +58,55 @@ def runner(): with threading_helper.start_threads(threads): pass + def test_run_coroutine_threadsafe(self) -> None: + results = [] + + def in_thread(loop: asyncio.AbstractEventLoop): + coro = asyncio.sleep(0.1, result=42) + fut = asyncio.run_coroutine_threadsafe(coro, loop) + result = fut.result() + self.assertEqual(result, 42) + results.append(result) + + async def main(): + loop = asyncio.get_running_loop() + async with asyncio.TaskGroup() as tg: + for _ in range(10): + tg.create_task(asyncio.to_thread(in_thread, loop)) + self.assertEqual(results, [42] * 10) + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + def test_run_coroutine_threadsafe_exception(self) -> None: + async def coro(): + await asyncio.sleep(0) + raise MyException("test") + + def in_thread(loop: asyncio.AbstractEventLoop): + fut = asyncio.run_coroutine_threadsafe(coro(), loop) + return fut.result() + + async def main(): + loop = asyncio.get_running_loop() + tasks = [] + for _ in range(10): + task = loop.create_task(asyncio.to_thread(in_thread, loop)) + tasks.append(task) + results = await asyncio.gather(*tasks, return_exceptions=True) + + self.assertEqual(len(results), 10) + for result in results: + self.assertIsInstance(result, MyException) + self.assertEqual(str(result), "test") + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + class TestPyFreeThreading(TestFreeThreading, TestCase): all_tasks = staticmethod(asyncio.tasks._py_all_tasks) From 402b91da87052878b4e7e8946ba91bdf4ee4bebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:54:13 +0100 Subject: [PATCH 18/61] gh-128078: Use `PyErr_SetRaisedException` in `_PyGen_SetStopIterationValue` (#128287) Co-authored-by: Kumar Aditya --- Lib/test/test_asyncgen.py | 2 +- Objects/genobject.c | 32 ++++++++++---------------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 3f631a03c9bf72..b81187871753b9 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1165,7 +1165,7 @@ async def run(): with self.assertRaises(StopAsyncIteration): await it.__anext__() res = await anext(it, ('a', 'b')) - self.assertEqual(res, ('a', 'b')) + self.assertTupleEqual(res, ('a', 'b')) self.loop.run_until_complete(run()) diff --git a/Objects/genobject.c b/Objects/genobject.c index a4872de296e2cf..b32140766c4a38 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -634,30 +634,18 @@ int _PyGen_SetStopIterationValue(PyObject *value) { assert(!PyErr_Occurred()); - PyObject *e; - - if (value == NULL || - (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) - { - /* Delay exception instantiation if we can */ - PyErr_SetObject(PyExc_StopIteration, value); - return 0; - } - /* Construct an exception instance manually with - * PyObject_CallOneArg and pass it to PyErr_SetObject. - * - * We do this to handle a situation when "value" is a tuple, in which - * case PyErr_SetObject would set the value of StopIteration to - * the first element of the tuple. - * - * (See PyErr_SetObject/_PyErr_CreateException code for details.) - */ - e = PyObject_CallOneArg(PyExc_StopIteration, value); - if (e == NULL) { + // Construct an exception instance manually with PyObject_CallOneArg() + // but use PyErr_SetRaisedException() instead of PyErr_SetObject() as + // PyErr_SetObject(exc_type, value) has a fast path when 'value' + // is a tuple, where the value of the StopIteration exception would be + // set to 'value[0]' instead of 'value'. + PyObject *exc = value == NULL + ? PyObject_CallNoArgs(PyExc_StopIteration) + : PyObject_CallOneArg(PyExc_StopIteration, value); + if (exc == NULL) { return -1; } - PyErr_SetObject(PyExc_StopIteration, e); - Py_DECREF(e); + PyErr_SetRaisedException(exc /* stolen */); return 0; } From da8825ea95a7096bb4f933d33b212a94ade10f6e Mon Sep 17 00:00:00 2001 From: Zhikang Yan <2951256653@qq.com> Date: Mon, 13 Jan 2025 23:54:46 +0800 Subject: [PATCH 19/61] gh-128562: Fix generation of the tkinter widget names (GH-128604) There were possible conflicts if the widget class name ends with a digit. --- Lib/test/test_tkinter/test_misc.py | 10 +++++++++- Lib/tkinter/__init__.py | 2 ++ .../2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 3362169391818b..96ea3f0117ca03 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -31,12 +31,20 @@ def test_repr(self): self.assertEqual(repr(f), '') def test_generated_names(self): + class Button2(tkinter.Button): + pass + t = tkinter.Toplevel(self.root) f = tkinter.Frame(t) f2 = tkinter.Frame(t) + self.assertNotEqual(str(f), str(f2)) b = tkinter.Button(f2) - for name in str(b).split('.'): + b2 = Button2(f2) + for name in str(b).split('.') + str(b2).split('.'): self.assertFalse(name.isidentifier(), msg=repr(name)) + b3 = tkinter.Button(f2) + b4 = Button2(f2) + self.assertEqual(len({str(b), str(b2), str(b3), str(b4)}), 4) @requires_tk(8, 6, 6) def test_tk_busy(self): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index d494c0c9687cd1..0baed8b569e40f 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2741,6 +2741,8 @@ def _setup(self, master, cnf): del cnf['name'] if not name: name = self.__class__.__name__.lower() + if name[-1].isdigit(): + name += "!" # Avoid duplication when calculating names below if master._last_child_ids is None: master._last_child_ids = {} count = master._last_child_ids.get(name, 0) + 1 diff --git a/Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst b/Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst new file mode 100644 index 00000000000000..eb50dded67bea8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst @@ -0,0 +1 @@ +Fix possible conflicts in generated :mod:`tkinter` widget names if the widget class name ends with a digit. From 53e8942e6938df3a32b783815f1bd4b76eed3dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:14:59 +0100 Subject: [PATCH 20/61] Explicitly import `urllib.error` in `urllib.robotparser` (#128737) --- Lib/urllib/robotparser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 24ee198c355f56..409f2b2e48de6e 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -11,6 +11,7 @@ """ import collections +import urllib.error import urllib.parse import urllib.request From b70a567575db37846beecbe8b40fb56b875274db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:58:11 +0100 Subject: [PATCH 21/61] gh-125997: Increase test coverage for `time.sleep()` (#128751) - Add tests for durations of invalid types. - Add tests for `int` and `float` durations, including signed zeroes durations. - Add tests for nonzero very small durations and durations close to the clock resolution. --------- Co-authored-by: Victor Stinner --- Lib/test/test_time.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 1c540bed33c71e..1147997d8d86bf 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -158,10 +158,19 @@ def test_conversions(self): self.assertEqual(int(time.mktime(time.localtime(self.t))), int(self.t)) - def test_sleep(self): + def test_sleep_exceptions(self): + self.assertRaises(TypeError, time.sleep, []) + self.assertRaises(TypeError, time.sleep, "a") + self.assertRaises(TypeError, time.sleep, complex(0, 0)) + self.assertRaises(ValueError, time.sleep, -2) self.assertRaises(ValueError, time.sleep, -1) - time.sleep(1.2) + self.assertRaises(ValueError, time.sleep, -0.1) + + def test_sleep(self): + for value in [-0.0, 0, 0.0, 1e-100, 1e-9, 1e-6, 1, 1.2]: + with self.subTest(value=value): + time.sleep(value) def test_epoch(self): # bpo-43869: Make sure that Python use the same Epoch on all platforms: From 41f73501eca2ff8b42fa4811d918a81c052a758b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jan 2025 08:44:20 +0100 Subject: [PATCH 22/61] gh-106320: Document private C APIs promoted to public C API (#128788) --- Doc/whatsnew/3.14.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 474bd6a24b8ece..1dbd871f3e3e5e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1272,6 +1272,30 @@ Porting to Python 3.14 implementation details. (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) +* Private functions promoted to public C APIs: + + * ``_PyBytes_Join()``: :c:func:`PyBytes_Join`; + * ``_PyLong_IsNegative()``: :c:func:`PyLong_IsNegative`; + * ``_PyLong_IsPositive()``: :c:func:`PyLong_IsPositive`; + * ``_PyLong_IsZero()``: :c:func:`PyLong_IsZero`; + * ``_PyLong_Sign()``: :c:func:`PyLong_GetSign`; + * ``_PyUnicodeWriter_Dealloc()``: :c:func:`PyUnicodeWriter_Discard`; + * ``_PyUnicodeWriter_Finish()``: :c:func:`PyUnicodeWriter_Finish`; + * ``_PyUnicodeWriter_Init()``: :c:func:`PyUnicodeWriter_Create`; + * ``_PyUnicodeWriter_WriteChar()``: :c:func:`PyUnicodeWriter_WriteChar`; + * ``_PyUnicodeWriter_WriteStr()``: :c:func:`PyUnicodeWriter_WriteStr`; + * ``_PyUnicodeWriter_WriteSubstring()``: :c:func:`PyUnicodeWriter_WriteSubstring`; + * ``_PyUnicode_EQ()``: :c:func:`PyUnicode_Equal`; + * ``_PyUnicode_Equal()``: :c:func:`PyUnicode_Equal`; + * ``_Py_GetConfig()``: :c:func:`PyConfig_Get` and :c:func:`PyConfig_GetInt`; + * ``_Py_HashBytes()``: :c:func:`Py_HashBuffer`; + * ``_Py_fopen_obj()``: :c:func:`Py_fopen`. + + The `pythoncapi-compat project`_ can be used to get most of these new + functions on Python 3.13 and older. + +.. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ + Deprecated ---------- From 06cad77a5b345adde88609be9c3c470c5cd9f417 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Jan 2025 10:02:38 +0200 Subject: [PATCH 23/61] gh-71339: Add additional assertion methods for unittest (GH-128707) Add the following methods: * assertHasAttr() and assertNotHasAttr() * assertIsSubclass() and assertNotIsSubclass() * assertStartsWith() and assertNotStartsWith() * assertEndsWith() and assertNotEndsWith() Also improve error messages for assertIsInstance() and assertNotIsInstance(). --- Doc/library/unittest.rst | 61 ++++ Doc/whatsnew/3.14.rst | 17 + Lib/test/test_descr.py | 8 - Lib/test/test_gdb/util.py | 5 - .../resources/test_functional.py | 6 - Lib/test/test_pyclbr.py | 10 +- Lib/test/test_typing.py | 18 - Lib/test/test_unittest/test_case.py | 307 +++++++++++++++++- Lib/test/test_unittest/test_loader.py | 2 +- Lib/test/test_unittest/test_program.py | 10 +- Lib/test/test_unittest/test_result.py | 2 +- Lib/test/test_unittest/testmock/testasync.py | 14 +- .../test_unittest/testmock/testcallable.py | 14 +- .../test_unittest/testmock/testhelpers.py | 2 +- .../testmock/testmagicmethods.py | 12 +- Lib/test/test_unittest/testmock/testmock.py | 16 +- Lib/test/test_unittest/testmock/testpatch.py | 6 +- Lib/test/test_venv.py | 4 - Lib/unittest/case.py | 132 +++++++- ...5-01-10-15-06-45.gh-issue-71339.EKnpzw.rst | 9 + 20 files changed, 555 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 7f8b710f611002..0eead59a337ef9 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -883,6 +883,12 @@ Test cases | :meth:`assertNotIsInstance(a, b) | ``not isinstance(a, b)`` | 3.2 | | ` | | | +-----------------------------------------+-----------------------------+---------------+ + | :meth:`assertIsSubclass(a, b) | ``issubclass(a, b)`` | 3.14 | + | ` | | | + +-----------------------------------------+-----------------------------+---------------+ + | :meth:`assertNotIsSubclass(a, b) | ``not issubclass(a, b)`` | 3.14 | + | ` | | | + +-----------------------------------------+-----------------------------+---------------+ All the assert methods accept a *msg* argument that, if specified, is used as the error message on failure (see also :data:`longMessage`). @@ -961,6 +967,15 @@ Test cases .. versionadded:: 3.2 + .. method:: assertIsSubclass(cls, superclass, msg=None) + assertNotIsSubclass(cls, superclass, msg=None) + + Test that *cls* is (or is not) a subclass of *superclass* (which can be a + class or a tuple of classes, as supported by :func:`issubclass`). + To check for the exact type, use :func:`assertIs(cls, superclass) `. + + .. versionadded:: next + It is also possible to check the production of exceptions, warnings, and log messages using the following methods: @@ -1210,6 +1225,24 @@ Test cases | ` | elements in the same number, | | | | regardless of their order. | | +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertStartsWith(a, b) | ``a.startswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertNotStartsWith(a, b) | ``not a.startswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertEndsWith(a, b) | ``a.endswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertNotEndsWith(a, b) | ``not a.endswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertHasAttr(a, b) | ``hastattr(a, b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertNotHasAttr(a, b) | ``not hastattr(a, b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ .. method:: assertAlmostEqual(first, second, places=7, msg=None, delta=None) @@ -1279,6 +1312,34 @@ Test cases .. versionadded:: 3.2 + .. method:: assertStartsWith(s, prefix, msg=None) + .. method:: assertNotStartsWith(s, prefix, msg=None) + + Test that the Unicode or byte string *s* starts (or does not start) + with a *prefix*. + *prefix* can also be a tuple of strings to try. + + .. versionadded:: next + + + .. method:: assertEndsWith(s, suffix, msg=None) + .. method:: assertNotEndsWith(s, suffix, msg=None) + + Test that the Unicode or byte string *s* ends (or does not end) + with a *suffix*. + *suffix* can also be a tuple of strings to try. + + .. versionadded:: next + + + .. method:: assertHasAttr(obj, name, msg=None) + .. method:: assertNotHasAttr(obj, name, msg=None) + + Test that the object *obj* has (or has not) an attribute *name*. + + .. versionadded:: next + + .. _type-specific-methods: The :meth:`assertEqual` method dispatches the equality check for objects of diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1dbd871f3e3e5e..eedcc621c3c688 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -670,6 +670,23 @@ unittest directory again. It was removed in Python 3.11. (Contributed by Jacob Walls in :gh:`80958`.) +* A number of new methods were added in the :class:`~unittest.TestCase` class + that provide more specialized tests. + + - :meth:`~unittest.TestCase.assertHasAttr` and + :meth:`~unittest.TestCase.assertNotHasAttr` check whether the object + has a particular attribute. + - :meth:`~unittest.TestCase.assertIsSubclass` and + :meth:`~unittest.TestCase.assertNotIsSubclass` check whether the object + is a subclass of a particular class, or of one of a tuple of classes. + - :meth:`~unittest.TestCase.assertStartsWith`, + :meth:`~unittest.TestCase.assertNotStartsWith`, + :meth:`~unittest.TestCase.assertEndsWith` and + :meth:`~unittest.TestCase.assertNotEndsWith` check whether the Unicode + or byte string starts or ends with particular string(s). + + (Contributed by Serhiy Storchaka in :gh:`71339`.) + urllib ------ diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 168b78a477ee9c..51f97bb51f7bd2 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -405,14 +405,6 @@ def test_wrap_lenfunc_bad_cast(self): class ClassPropertiesAndMethods(unittest.TestCase): - def assertHasAttr(self, obj, name): - self.assertTrue(hasattr(obj, name), - '%r has no attribute %r' % (obj, name)) - - def assertNotHasAttr(self, obj, name): - self.assertFalse(hasattr(obj, name), - '%r has unexpected attribute %r' % (obj, name)) - def test_python_dicts(self): # Testing Python subclass of dict... self.assertTrue(issubclass(dict, dict)) diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index 8fe9cfc543395e..8097fd52ababe6 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -280,11 +280,6 @@ def get_stack_trace(self, source=None, script=None, return out - def assertEndsWith(self, actual, exp_end): - '''Ensure that the given "actual" string ends with "exp_end"''' - self.assertTrue(actual.endswith(exp_end), - msg='%r did not end with %r' % (actual, exp_end)) - def assertMultilineMatches(self, actual, pattern): m = re.match(pattern, actual, re.DOTALL) if not m: diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py index 4317abf3162c52..e8d25fa4d9faf0 100644 --- a/Lib/test/test_importlib/resources/test_functional.py +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -43,12 +43,6 @@ def _gen_resourcetxt_path_parts(self): with self.subTest(path_parts=path_parts): yield path_parts - def assertEndsWith(self, string, suffix): - """Assert that `string` ends with `suffix`. - - Used to ignore an architecture-specific UTF-16 byte-order mark.""" - self.assertEqual(string[-len(suffix) :], suffix) - def test_read_text(self): self.assertEqual( resources.read_text(self.anchor01, 'utf-8.file'), diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 4bf0576586cca5..25b313f6c25a4e 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -31,14 +31,6 @@ def assertListEq(self, l1, l2, ignore): print("l1=%r\nl2=%r\nignore=%r" % (l1, l2, ignore), file=sys.stderr) self.fail("%r missing" % missing.pop()) - def assertHasattr(self, obj, attr, ignore): - ''' succeed iff hasattr(obj,attr) or attr in ignore. ''' - if attr in ignore: return - if not hasattr(obj, attr): print("???", attr) - self.assertTrue(hasattr(obj, attr), - 'expected hasattr(%r, %r)' % (obj, attr)) - - def assertHaskey(self, obj, key, ignore): ''' succeed iff key in obj or key in ignore. ''' if key in ignore: return @@ -86,7 +78,7 @@ def ismethod(oclass, obj, name): for name, value in dict.items(): if name in ignore: continue - self.assertHasattr(module, name, ignore) + self.assertHasAttr(module, name, ignore) py_item = getattr(module, name) if isinstance(value, pyclbr.Function): self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c51ee763890af2..c98e6f820e8cf7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -59,20 +59,6 @@ class BaseTestCase(TestCase): - def assertIsSubclass(self, cls, class_or_tuple, msg=None): - if not issubclass(cls, class_or_tuple): - message = '%r is not a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) - - def assertNotIsSubclass(self, cls, class_or_tuple, msg=None): - if issubclass(cls, class_or_tuple): - message = '%r is a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) - def clear_caches(self): for f in typing._cleanups: f() @@ -1252,10 +1238,6 @@ class Gen[*Ts]: ... class TypeVarTupleTests(BaseTestCase): - def assertEndsWith(self, string, tail): - if not string.endswith(tail): - self.fail(f"String {string!r} does not end with {tail!r}") - def test_name(self): Ts = TypeVarTuple('Ts') self.assertEqual(Ts.__name__, 'Ts') diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index 621f8269a177ce..cd366496eedca3 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -10,6 +10,7 @@ import inspect import types +from collections import UserString from copy import deepcopy from test import support @@ -54,6 +55,10 @@ def tearDown(self): self.events.append('tearDown') +class List(list): + pass + + class Test_TestCase(unittest.TestCase, TestEquality, TestHashing): ### Set up attributes used by inherited tests @@ -85,7 +90,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test().id()[-13:], '.Test.runTest') + self.assertEndsWith(Test().id(), '.Test.runTest') # test that TestCase can be instantiated with no args # primarily for use at the interactive interpreter @@ -106,7 +111,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test('test').id()[-10:], '.Test.test') + self.assertEndsWith(Test('test').id(), '.Test.test') # "class TestCase([methodName])" # ... @@ -700,16 +705,120 @@ def testAssertIsNot(self): self.assertRaises(self.failureException, self.assertIsNot, thing, thing) def testAssertIsInstance(self): - thing = [] + thing = List() self.assertIsInstance(thing, list) - self.assertRaises(self.failureException, self.assertIsInstance, - thing, dict) + self.assertIsInstance(thing, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int) + self.assertEqual(str(cm.exception), + "[] is not an instance of ") + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, (int, float)) + self.assertEqual(str(cm.exception), + "[] is not an instance of any of (, )") + + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertNotIsInstance(self): - thing = [] - self.assertNotIsInstance(thing, dict) - self.assertRaises(self.failureException, self.assertNotIsInstance, - thing, list) + thing = List() + self.assertNotIsInstance(thing, int) + self.assertNotIsInstance(thing, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list) + self.assertEqual(str(cm.exception), + "[] is an instance of ") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, (int, list)) + self.assertEqual(str(cm.exception), + "[] is an instance of ") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertIsSubclass(self): + self.assertIsSubclass(List, list) + self.assertIsSubclass(List, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of ") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, (int, float)) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of any of (, )") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotIsSubclass(self): + self.assertNotIsSubclass(List, int) + self.assertNotIsSubclass(List, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of ") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, (int, list)) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of ") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertHasAttr(self): + a = List() + a.x = 1 + self.assertHasAttr(a, 'x') + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y') + self.assertEqual(str(cm.exception), + "List instance has no attribute 'y'") + + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotHasAttr(self): + a = List() + a.x = 1 + self.assertNotHasAttr(a, 'y') + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x') + self.assertEqual(str(cm.exception), + "List instance has unexpected attribute 'x'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertIn(self): animals = {'monkey': 'banana', 'cow': 'grass', 'seal': 'fish'} @@ -1864,6 +1973,186 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) + def testAssertStartswith(self): + self.assertStartsWith('ababahalamaha', 'ababa') + self.assertStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertStartsWith(UserString('ababahalamaha'), 'ababa') + self.assertStartsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertStartsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertStartsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertStartsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotStartswith(self): + self.assertNotStartsWith('ababahalamaha', 'amaha') + self.assertNotStartsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertNotStartsWith(UserString('ababahalamaha'), 'amaha') + self.assertNotStartsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertNotStartsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertNotStartsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertNotStartsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertNotStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertEndswith(self): + self.assertEndsWith('ababahalamaha', 'amaha') + self.assertEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEndsWith(UserString('ababahalamaha'), 'amaha') + self.assertEndsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertEndsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertEndsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertEndsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotEndswith(self): + self.assertNotEndsWith('ababahalamaha', 'ababa') + self.assertNotEndsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertNotEndsWith(UserString('ababahalamaha'), 'ababa') + self.assertNotEndsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertNotEndsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertNotEndsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertNotEndsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertNotEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + def testDeprecatedFailMethods(self): """Test that the deprecated fail* methods get removed in 3.12""" deprecated_names = [ diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index 83dd25ca54623f..cdff6d1a20c8df 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -76,7 +76,7 @@ def runTest(self): loader = unittest.TestLoader() # This has to be false for the test to succeed - self.assertFalse('runTest'.startswith(loader.testMethodPrefix)) + self.assertNotStartsWith('runTest', loader.testMethodPrefix) suite = loader.loadTestsFromTestCase(Foo) self.assertIsInstance(suite, loader.suiteClass) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 0b46f338ac77e1..58d0cef9708c95 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -128,14 +128,14 @@ def test_NonExit(self): argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), testLoader=self.TestLoader(self.FooBar)) - self.assertTrue(hasattr(program, 'result')) + self.assertHasAttr(program, 'result') out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) @force_not_colorized def test_Exit(self): @@ -153,7 +153,7 @@ def test_Exit(self): self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) @force_not_colorized def test_ExitAsDefault(self): @@ -169,7 +169,7 @@ def test_ExitAsDefault(self): self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) @force_not_colorized def test_ExitSkippedSuite(self): @@ -182,7 +182,7 @@ def test_ExitSkippedSuite(self): self.assertEqual(cm.exception.code, 0) out = stream.getvalue() expected = '\n\nOK (skipped=1)\n' - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) @force_not_colorized def test_ExitEmptySuite(self): diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index ad6f52d7e0260e..327b246452bedf 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -462,7 +462,7 @@ def test(result): self.assertTrue(result.failfast) result = runner.run(test) stream.flush() - self.assertTrue(stream.getvalue().endswith('\n\nOK\n')) + self.assertEndsWith(stream.getvalue(), '\n\nOK\n') class Test_TextTestResult(unittest.TestCase): diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index afc9d1f11da1e2..0791675b5401ca 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -586,16 +586,16 @@ def test_sync_magic_methods_return_magic_mocks(self): def test_magicmock_has_async_magic_methods(self): m_mock = MagicMock() - self.assertTrue(hasattr(m_mock, "__aenter__")) - self.assertTrue(hasattr(m_mock, "__aexit__")) - self.assertTrue(hasattr(m_mock, "__anext__")) + self.assertHasAttr(m_mock, "__aenter__") + self.assertHasAttr(m_mock, "__aexit__") + self.assertHasAttr(m_mock, "__anext__") def test_asyncmock_has_sync_magic_methods(self): a_mock = AsyncMock() - self.assertTrue(hasattr(a_mock, "__enter__")) - self.assertTrue(hasattr(a_mock, "__exit__")) - self.assertTrue(hasattr(a_mock, "__next__")) - self.assertTrue(hasattr(a_mock, "__len__")) + self.assertHasAttr(a_mock, "__enter__") + self.assertHasAttr(a_mock, "__exit__") + self.assertHasAttr(a_mock, "__next__") + self.assertHasAttr(a_mock, "__len__") def test_magic_methods_are_async_functions(self): m_mock = MagicMock() diff --git a/Lib/test/test_unittest/testmock/testcallable.py b/Lib/test/test_unittest/testmock/testcallable.py index ca88511f63959d..03cb983e447c70 100644 --- a/Lib/test/test_unittest/testmock/testcallable.py +++ b/Lib/test/test_unittest/testmock/testcallable.py @@ -23,21 +23,21 @@ def assertNotCallable(self, mock): def test_non_callable(self): for mock in NonCallableMagicMock(), NonCallableMock(): self.assertRaises(TypeError, mock) - self.assertFalse(hasattr(mock, '__call__')) + self.assertNotHasAttr(mock, '__call__') self.assertIn(mock.__class__.__name__, repr(mock)) def test_hierarchy(self): - self.assertTrue(issubclass(MagicMock, Mock)) - self.assertTrue(issubclass(NonCallableMagicMock, NonCallableMock)) + self.assertIsSubclass(MagicMock, Mock) + self.assertIsSubclass(NonCallableMagicMock, NonCallableMock) def test_attributes(self): one = NonCallableMock() - self.assertTrue(issubclass(type(one.one), Mock)) + self.assertIsSubclass(type(one.one), Mock) two = NonCallableMagicMock() - self.assertTrue(issubclass(type(two.two), MagicMock)) + self.assertIsSubclass(type(two.two), MagicMock) def test_subclasses(self): @@ -45,13 +45,13 @@ class MockSub(Mock): pass one = MockSub() - self.assertTrue(issubclass(type(one.one), MockSub)) + self.assertIsSubclass(type(one.one), MockSub) class MagicSub(MagicMock): pass two = MagicSub() - self.assertTrue(issubclass(type(two.two), MagicSub)) + self.assertIsSubclass(type(two.two), MagicSub) def test_patch_spec(self): diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index f260769eb8c35e..8d0f3ebc5cba88 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -951,7 +951,7 @@ def __getattr__(self, attribute): proxy = Foo() autospec = create_autospec(proxy) - self.assertFalse(hasattr(autospec, '__name__')) + self.assertNotHasAttr(autospec, '__name__') def test_autospec_signature_staticmethod(self): diff --git a/Lib/test/test_unittest/testmock/testmagicmethods.py b/Lib/test/test_unittest/testmock/testmagicmethods.py index 2a8aa11b3284f6..acdbd699d18134 100644 --- a/Lib/test/test_unittest/testmock/testmagicmethods.py +++ b/Lib/test/test_unittest/testmock/testmagicmethods.py @@ -10,13 +10,13 @@ class TestMockingMagicMethods(unittest.TestCase): def test_deleting_magic_methods(self): mock = Mock() - self.assertFalse(hasattr(mock, '__getitem__')) + self.assertNotHasAttr(mock, '__getitem__') mock.__getitem__ = Mock() - self.assertTrue(hasattr(mock, '__getitem__')) + self.assertHasAttr(mock, '__getitem__') del mock.__getitem__ - self.assertFalse(hasattr(mock, '__getitem__')) + self.assertNotHasAttr(mock, '__getitem__') def test_magicmock_del(self): @@ -252,12 +252,12 @@ def test_magicmock(self): self.assertEqual(list(mock), [1, 2, 3]) getattr(mock, '__bool__').return_value = False - self.assertFalse(hasattr(mock, '__nonzero__')) + self.assertNotHasAttr(mock, '__nonzero__') self.assertFalse(bool(mock)) for entry in _magics: - self.assertTrue(hasattr(mock, entry)) - self.assertFalse(hasattr(mock, '__imaginary__')) + self.assertHasAttr(mock, entry) + self.assertNotHasAttr(mock, '__imaginary__') def test_magic_mock_equality(self): diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index e1b108f81e513c..5d1bf4258afacd 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -2215,13 +2215,13 @@ def test_attach_mock_patch_autospec_signature(self): def test_attribute_deletion(self): for mock in (Mock(), MagicMock(), NonCallableMagicMock(), NonCallableMock()): - self.assertTrue(hasattr(mock, 'm')) + self.assertHasAttr(mock, 'm') del mock.m - self.assertFalse(hasattr(mock, 'm')) + self.assertNotHasAttr(mock, 'm') del mock.f - self.assertFalse(hasattr(mock, 'f')) + self.assertNotHasAttr(mock, 'f') self.assertRaises(AttributeError, getattr, mock, 'f') @@ -2230,18 +2230,18 @@ def test_mock_does_not_raise_on_repeated_attribute_deletion(self): for mock in (Mock(), MagicMock(), NonCallableMagicMock(), NonCallableMock()): mock.foo = 3 - self.assertTrue(hasattr(mock, 'foo')) + self.assertHasAttr(mock, 'foo') self.assertEqual(mock.foo, 3) del mock.foo - self.assertFalse(hasattr(mock, 'foo')) + self.assertNotHasAttr(mock, 'foo') mock.foo = 4 - self.assertTrue(hasattr(mock, 'foo')) + self.assertHasAttr(mock, 'foo') self.assertEqual(mock.foo, 4) del mock.foo - self.assertFalse(hasattr(mock, 'foo')) + self.assertNotHasAttr(mock, 'foo') def test_mock_raises_when_deleting_nonexistent_attribute(self): @@ -2259,7 +2259,7 @@ def test_reset_mock_does_not_raise_on_attr_deletion(self): mock.child = True del mock.child mock.reset_mock() - self.assertFalse(hasattr(mock, 'child')) + self.assertNotHasAttr(mock, 'child') def test_class_assignable(self): diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index 037c021e6eafcf..7c5fc3deed2ca2 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -366,7 +366,7 @@ def test(): self.assertEqual(SomeClass.frooble, sentinel.Frooble) test() - self.assertFalse(hasattr(SomeClass, 'frooble')) + self.assertNotHasAttr(SomeClass, 'frooble') def test_patch_wont_create_by_default(self): @@ -383,7 +383,7 @@ def test_patchobject_wont_create_by_default(self): @patch.object(SomeClass, 'ord', sentinel.Frooble) def test(): pass test() - self.assertFalse(hasattr(SomeClass, 'ord')) + self.assertNotHasAttr(SomeClass, 'ord') def test_patch_builtins_without_create(self): @@ -1477,7 +1477,7 @@ def test_patch_multiple_create(self): finally: patcher.stop() - self.assertFalse(hasattr(Foo, 'blam')) + self.assertNotHasAttr(Foo, 'blam') def test_patch_multiple_spec_set(self): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 0b09010c69d4ea..6e23097deaf221 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -111,10 +111,6 @@ def get_text_file_contents(self, *args, encoding='utf-8'): result = f.read() return result - def assertEndsWith(self, string, tail): - if not string.endswith(tail): - self.fail(f"String {string!r} does not end with {tail!r}") - class BasicTest(BaseTest): """Test venv module functionality.""" diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 55c79d353539ca..e9ef551d0b3ded 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -1321,13 +1321,67 @@ def assertIsInstance(self, obj, cls, msg=None): """Same as self.assertTrue(isinstance(obj, cls)), with a nicer default message.""" if not isinstance(obj, cls): - standardMsg = '%s is not an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + standardMsg = f'{safe_repr(obj)} is not an instance of any of {cls!r}' + else: + standardMsg = f'{safe_repr(obj)} is not an instance of {cls!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertNotIsInstance(self, obj, cls, msg=None): """Included for symmetry with assertIsInstance.""" if isinstance(obj, cls): - standardMsg = '%s is an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + for x in cls: + if isinstance(obj, x): + cls = x + break + standardMsg = f'{safe_repr(obj)} is an instance of {cls!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertIsSubclass(self, cls, superclass, msg=None): + try: + if issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + standardMsg = f'{cls!r} is not a subclass of any of {superclass!r}' + else: + standardMsg = f'{cls!r} is not a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotIsSubclass(self, cls, superclass, msg=None): + try: + if not issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + for x in superclass: + if issubclass(cls, x): + superclass = x + break + standardMsg = f'{cls!r} is a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertHasAttr(self, obj, name, msg=None): + if not hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has no attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__} instance has no attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotHasAttr(self, obj, name, msg=None): + if hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has unexpected attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__} instance has unexpected attribute {name!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertRaisesRegex(self, expected_exception, expected_regex, @@ -1391,6 +1445,80 @@ def assertNotRegex(self, text, unexpected_regex, msg=None): msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) + def _tail_type_check(self, s, tails, msg): + if not isinstance(tails, tuple): + tails = (tails,) + for tail in tails: + if isinstance(tail, str): + if not isinstance(s, str): + self.fail(self._formatMessage(msg, + f'Expected str, not {type(s).__name__}')) + elif isinstance(tail, (bytes, bytearray)): + if not isinstance(s, (bytes, bytearray)): + self.fail(self._formatMessage(msg, + f'Expected bytes, not {type(s).__name__}')) + + def assertStartsWith(self, s, prefix, msg=None): + try: + if s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(prefix) + if isinstance(prefix, tuple): + standardMsg = f"{a} doesn't start with any of {b}" + else: + standardMsg = f"{a} doesn't start with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotStartsWith(self, s, prefix, msg=None): + try: + if not s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + if isinstance(prefix, tuple): + for x in prefix: + if s.startswith(x): + prefix = x + break + a = safe_repr(s, short=True) + b = safe_repr(prefix) + self.fail(self._formatMessage(msg, f"{a} starts with {b}")) + + def assertEndsWith(self, s, suffix, msg=None): + try: + if s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(suffix) + if isinstance(suffix, tuple): + standardMsg = f"{a} doesn't end with any of {b}" + else: + standardMsg = f"{a} doesn't end with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotEndsWith(self, s, suffix, msg=None): + try: + if not s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + if isinstance(suffix, tuple): + for x in suffix: + if s.endswith(x): + suffix = x + break + a = safe_repr(s, short=True) + b = safe_repr(suffix) + self.fail(self._formatMessage(msg, f"{a} ends with {b}")) class FunctionTestCase(TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst b/Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst new file mode 100644 index 00000000000000..5f33a30bd5eae1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst @@ -0,0 +1,9 @@ +Add new assertion methods for :mod:`unittest`: +:meth:`~unittest.TestCase.assertHasAttr`, +:meth:`~unittest.TestCase.assertNotHasAttr`, +:meth:`~unittest.TestCase.assertIsSubclass`, +:meth:`~unittest.TestCase.assertNotIsSubclass` +:meth:`~unittest.TestCase.assertStartsWith`, +:meth:`~unittest.TestCase.assertNotStartsWith`, +:meth:`~unittest.TestCase.assertEndsWith` and +:meth:`~unittest.TestCase.assertNotEndsWith`. From 99327d124145d22cde8ff324754026c87f01a1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 14 Jan 2025 09:40:59 +0100 Subject: [PATCH 24/61] Skip CI expensive checks on `CODEOWNERS` update (#128754) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/reusable-change-detection.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 964bd87e815f42..c08c0cb8873f12 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -83,7 +83,22 @@ jobs: # into the PR branch anyway. # # https://github.com/python/core-workflow/issues/373 - git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run-tests=true" >> "$GITHUB_OUTPUT" || true + grep_ignore_args=( + # file extensions + -e '\.md$' + -e '\.rst$' + # top-level folders + -e '^Doc/' + -e '^Misc/' + # configuration files + -e '^\.github/CODEOWNERS$' + -e '^\.pre-commit-config\.yaml$' + -e '\.ruff\.toml$' + -e 'mypy\.ini$' + ) + git diff --name-only "origin/$GITHUB_BASE_REF.." \ + | grep -qvE "${grep_ignore_args[@]}" \ + && echo "run-tests=true" >> "$GITHUB_OUTPUT" || true fi # Check if we should run hypothesis tests From 98301326e4de507ae507b7c8eed76b964e4a2afc Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:53:03 +0200 Subject: [PATCH 25/61] gh-123299: Copyedit "What's New in Python 3.14" (#128814) --- Doc/deprecations/pending-removal-in-3.16.rst | 10 +++++----- Doc/whatsnew/3.14.rst | 21 ++++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index d093deb648baf7..41b30defdba0bc 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -57,6 +57,11 @@ Pending removal in Python 3.16 In the rare case that you need the bitwise inversion of the underlying integer, convert to ``int`` explicitly (``~int(x)``). +* :mod:`functools`: + + * Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. + * :mod:`shutil`: * The :class:`!ExecError` exception @@ -79,8 +84,3 @@ Pending removal in Python 3.16 * The undocumented and unused :attr:`!TarFile.tarfile` attribute has been deprecated since Python 3.13. - -* :mod:`functools`: - - * Calling the Python implementation of :func:`functools.reduce` with *function* - or *sequence* as keyword arguments has been deprecated since Python 3.14. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index eedcc621c3c688..1152c7a90b7712 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -67,6 +67,7 @@ Summary -- release highlights * :ref:`PEP 649: deferred evaluation of annotations ` * :ref:`PEP 741: Python Configuration C API ` +* :ref:`PEP 761: Discontinuation of PGP signatures ` New features @@ -583,13 +584,6 @@ pdb command when :mod:`pdb` is in ``inline`` mode. (Contributed by Tian Gao in :gh:`123757`.) -platform --------- - -* Add :func:`platform.invalidate_caches` to invalidate the cached results. - - (Contributed by Bénédikt Tran in :gh:`122549`.) - pickle ------ @@ -601,6 +595,15 @@ pickle of the error. (Contributed by Serhiy Storchaka in :gh:`122213`.) + +platform +-------- + +* Add :func:`platform.invalidate_caches` to invalidate the cached results. + + (Contributed by Bénédikt Tran in :gh:`122549`.) + + pydoc ----- @@ -1123,9 +1126,11 @@ Changes in the Python API Build changes ============= -* GNU Autoconf 2.72 is now required to generate :file:`!configure`. +* GNU Autoconf 2.72 is now required to generate :file:`configure`. (Contributed by Erlend Aasland in :gh:`115765`.) +.. _whatsnew314-pep761: + PEP 761: Discontinuation of PGP signatures ------------------------------------------ From eefd4a0bc764c0272c560f26dd10fb8fba0fb7d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:16:43 +0100 Subject: [PATCH 26/61] Update cryptographic primitives code owners. (#128747) --- .github/CODEOWNERS | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9e190d43b28ef9..1abcd455eca11b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -96,13 +96,14 @@ Doc/library/site.rst @FFY00 Lib/test/test_except*.py @iritkatriel Objects/exceptions.c @iritkatriel -# Hashing -**/*hashlib* @gpshead @tiran +# Hashing & cryptographic primitives +**/*hashlib* @gpshead @tiran @picnixz **/*pyhash* @gpshead @tiran -**/sha* @gpshead @tiran -Modules/md5* @gpshead @tiran -**/*blake* @gpshead @tiran +**/sha* @gpshead @tiran @picnixz +Modules/md5* @gpshead @tiran @picnixz +**/*blake* @gpshead @tiran @picnixz Modules/_hacl/** @gpshead +**/*hmac* @gpshead @picnixz # logging **/*logging* @vsajip From 24a8d920c1ec251a389fca6dedfb860d9f4bbc76 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jan 2025 11:17:20 +0100 Subject: [PATCH 27/61] gh-127787: Move _PyUnicodeError_GetParams() to the internal C API (#128803) --- Include/cpython/pyerrors.h | 6 ------ Include/internal/pycore_pyerrors.h | 9 +++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 49a6265e5eb02f..b36b4681f5dddb 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -94,12 +94,6 @@ PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); /* In exceptions.c */ -PyAPI_FUNC(int) _PyUnicodeError_GetParams( - PyObject *self, - PyObject **obj, Py_ssize_t *objlen, - Py_ssize_t *start, Py_ssize_t *end, - int as_bytes); - PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( PyObject *orig, PyObject *excs); diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 6f2fdda9a9f12f..8dea2d34117430 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -190,6 +190,15 @@ Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, P PyAPI_DATA(PyTypeObject) _PyExc_IncompleteInputError; #define PyExc_IncompleteInputError ((PyObject *)(&_PyExc_IncompleteInputError)) +extern int _PyUnicodeError_GetParams( + PyObject *self, + PyObject **obj, + Py_ssize_t *objlen, + Py_ssize_t *start, + Py_ssize_t *end, + int as_bytes); + + #ifdef __cplusplus } #endif From 1153e66e20124b8f3484bcaddbc0e252d31161a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jan 2025 11:18:52 +0100 Subject: [PATCH 28/61] gh-109959: Skip test_glob.test_selflink() flaky test (#128812) --- Lib/test/test_glob.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 00187a3fb3537d..1a836e34e8712f 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -511,6 +511,10 @@ def fn(pat): @skip_unless_symlink class SymlinkLoopGlobTests(unittest.TestCase): + # gh-109959: On Linux, glob._isdir() and glob._lexists() can return False + # randomly when checking the "link/" symbolic link. + # https://github.com/python/cpython/issues/109959#issuecomment-2577550700 + @unittest.skip("flaky test") def test_selflink(self): tempdir = TESTFN + "_dir" os.makedirs(tempdir) From ff3e145b2770ffe86c905b1092747ce3d8381319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:26:26 +0100 Subject: [PATCH 29/61] gh-118761: Improve import time of the `pickle` module. (#128732) Importing `pickle` is now roughly 25% faster. Importing the `re` module is no longer needed and thus `re` is no more implicitly exposed as `pickle.re`. --------- Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Lib/pickle.py | 3 +-- .../Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst diff --git a/Lib/pickle.py b/Lib/pickle.py index 1920973e3f83e9..8afb4aa4285f37 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -31,7 +31,6 @@ import sys from sys import maxsize from struct import pack, unpack -import re import io import codecs import _compat_pickle @@ -188,7 +187,7 @@ def __init__(self, value): NEXT_BUFFER = b'\x97' # push next out-of-band buffer READONLY_BUFFER = b'\x98' # make top of stack readonly -__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)]) +__all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) class _Framer: diff --git a/Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst b/Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst new file mode 100644 index 00000000000000..a0a0f891ca55d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst @@ -0,0 +1,3 @@ +Improve import time of :mod:`pickle` by 25% by removing an unnecessary +regular expression. As such, :mod:`re` is no more implicitly available +as ``pickle.re``. Patch by Bénédikt Tran. From f26daa9470925120f6336ca508f7ea193b00a3f4 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:41:39 +0200 Subject: [PATCH 30/61] Python 3.14.0a4 --- Doc/c-api/sys.rst | 4 +- Doc/library/asyncio-eventloop.rst | 2 +- Doc/library/asyncio-policy.rst | 12 +- Doc/library/calendar.rst | 8 +- Doc/library/ctypes.rst | 2 +- Doc/library/faulthandler.rst | 2 +- Doc/library/os.rst | 4 +- Doc/library/pyexpat.rst | 2 +- Doc/library/re.rst | 2 +- Doc/library/socket.rst | 2 +- Doc/library/ssl.rst | 2 +- Doc/library/turtle.rst | 2 +- Doc/library/unittest.rst | 8 +- Doc/using/configure.rst | 2 +- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 33 +- Misc/NEWS.d/3.14.0a4.rst | 830 ++++++++++++++++++ ...-12-16-16-16-35.gh-issue-127951.lpE13-.rst | 1 - ...-12-20-09-03-22.gh-issue-128104.m_SoVx.rst | 3 - ...-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst | 2 - ...-12-28-21-05-19.gh-issue-128321.0UvbXw.rst | 3 - ...4-12-31-17-09-37.gh-issue-90905.PjLNai.rst | 1 - ...-01-02-11-02-45.gh-issue-123925.TLlyUi.rst | 2 - ...-01-02-12-50-46.gh-issue-115765.jko7Fg.rst | 2 - ...-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst | 2 - ...-01-09-19-44-00.gh-issue-128627.mHzsEd.rst | 3 - ...-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst | 5 - ...-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst | 2 - ...-01-08-13-13-18.gh-issue-128629.gSmzyl.rst | 2 - ...-01-12-12-19-51.gh-issue-128400.OwoIDw.rst | 2 - ...-12-02-18-15-37.gh-issue-126862.fdIK7T.rst | 2 - ...-12-11-14-32-22.gh-issue-127809.0W8khe.rst | 2 - ...-12-13-14-17-24.gh-issue-127903.vemHSl.rst | 2 - ...-12-13-15-21-45.gh-issue-127773.E-DZR4.rst | 1 - ...-12-15-19-51-54.gh-issue-127970.vdUp-y.rst | 6 - ...4-12-15-21-11-26.gh-issue-66409.wv109z.rst | 3 - ...-12-17-13-45-33.gh-issue-127274.deNxNC.rst | 3 - ...-12-17-18-20-37.gh-issue-128035.JwqHdB.rst | 1 - ...-12-17-22-28-15.gh-issue-128030.H1ptOD.rst | 1 - ...-12-18-14-22-48.gh-issue-128079.SUD5le.rst | 5 - ...-12-20-12-25-16.gh-issue-127705.WmCz1z.rst | 4 - ...-12-20-23-07-33.gh-issue-114203.84NgoW.rst | 1 - ...-12-22-15-47-44.gh-issue-126868.RpjKez.rst | 1 - ...-12-23-11-14-07.gh-issue-128192.02mEhD.rst | 2 - ...-12-24-01-40-12.gh-issue-128137.gsTwr_.rst | 2 - ...-01-07-19-48-56.gh-issue-126703.0ISs-7.rst | 1 - ...-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst | 3 - ...-01-11-12-39-17.gh-issue-128717.i65d06.rst | 2 - ...-01-13-12-48-30.gh-issue-128078.qOsl9B.rst | 2 - ...2-07-28-12-32-59.gh-issue-95371.F24IFC.rst | 1 - ...-11-12-21-53-40.gh-issue-112015.2WPRxE.rst | 5 - ...-07-13-13-25-31.gh-issue-121676.KDLS11.rst | 3 - ...-08-28-16-10-37.gh-issue-123424.u96_i6.rst | 1 - ...-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst | 1 - ...-10-04-09-56-45.gh-issue-124761.N4pSD6.rst | 1 - ...-10-31-14-31-36.gh-issue-126225.vTxGXm.rst | 6 - ...-11-09-15-59-51.gh-issue-126624.bN53Va.rst | 2 - ...-11-11-07-56-03.gh-issue-126639.AmVSt-.rst | 1 - ...-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst | 4 - ...-11-24-14-53-35.gh-issue-127196.8CBkUa.rst | 2 - ...-11-28-14-24-12.gh-issue-127360.HVKt-c.rst | 4 - ...-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst | 4 - ...4-12-03-14-45-16.gh-issue-98188.GX9i2b.rst | 3 - ...-12-03-20-28-08.gh-issue-127586.zgotYF.rst | 3 - ...4-12-04-10-39-29.gh-issue-83662.CG1s3m.rst | 5 - ...-12-06-21-03-11.gh-issue-127688.NJqtc-.rst | 2 - ...-12-12-07-27-51.gh-issue-127847.ksfNKM.rst | 1 - ...-12-13-14-21-04.gh-issue-122548.hq3Vud.rst | 4 - ...-12-17-12-41-07.gh-issue-126742.l07qvT.rst | 3 - ...-12-17-13-21-52.gh-issue-127060.mv2bX6.rst | 2 - ...4-12-17-15-23-40.gh-issue-41872.31LjKY.rst | 3 - ...-12-18-00-07-50.gh-issue-128014.F3aUbz.rst | 2 - ...-12-18-10-18-55.gh-issue-128062.E9oU7-.rst | 2 - ...-12-19-20-46-01.gh-issue-127946.4lM3Op.rst | 2 - ...-12-20-10-57-10.gh-issue-128118.mYak8i.rst | 2 - ...-12-21-11-12-50.gh-issue-128151.aq7vpG.rst | 2 - ...-12-27-16-28-57.gh-issue-128302.2GMvyl.rst | 3 - ...-12-29-00-33-34.gh-issue-128317.WgFina.rst | 2 - ...-12-29-13-49-46.gh-issue-128302.psRpPN.rst | 2 - ...4-12-30-20-48-28.gh-issue-88834.RIvgwc.rst | 3 - ...-01-01-19-24-43.gh-issue-128388.8UdMz_.rst | 1 - ...-01-02-13-05-16.gh-issue-128400.5N43fF.rst | 2 - ...-01-02-15-20-17.gh-issue-128400.UMiG4f.rst | 2 - ...-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst | 2 - ...-01-05-11-46-14.gh-issue-128340.gKI0uU.rst | 1 - ...-01-06-18-41-08.gh-issue-128552.fV-f8j.rst | 1 - ...-01-06-21-35-00.gh-issue-128559.6fxcDM.rst | 1 - ...-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst | 1 - ...-01-09-12-06-52.gh-issue-128661.ixx_0z.rst | 2 - ...-01-10-13-34-33.gh-issue-118761.qRB8nS.rst | 3 - ...5-01-10-15-06-45.gh-issue-71339.EKnpzw.rst | 9 - ...-01-11-13-40-12.gh-issue-128731.qpKlai.rst | 1 - ...-01-13-01-29-08.gh-issue-128690.cPFVDb.rst | 2 - ...-01-03-23-51-07.gh-issue-128152.IhzElS.rst | 2 - ...-12-22-08-54-30.gh-issue-127592.iyuFCC.rst | 2 - README.rst | 2 +- 96 files changed, 880 insertions(+), 230 deletions(-) create mode 100644 Misc/NEWS.d/3.14.0a4.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-01-09-19-44-00.gh-issue-128627.mHzsEd.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-18-15-37.gh-issue-126862.fdIK7T.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-07-19-48-56.gh-issue-126703.0ISs-7.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-11-12-39-17.gh-issue-128717.i65d06.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-24-14-53-35.gh-issue-127196.8CBkUa.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-28-14-24-12.gh-issue-127360.HVKt-c.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-01-19-24-43.gh-issue-128388.8UdMz_.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-01-11-13-40-12.gh-issue-128731.qpKlai.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst delete mode 100644 Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 7a7d39aea20baf..4ab5df4ccccdbb 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -234,7 +234,7 @@ Operating System Utilities The caller must hold the GIL. - .. versionadded:: next + .. versionadded:: 3.14 .. c:function:: int Py_fclose(FILE *file) @@ -246,7 +246,7 @@ Operating System Utilities In either case, any further access (including another call to :c:func:`Py_fclose`) to the stream results in undefined behavior. - .. versionadded:: next + .. versionadded:: 3.14 .. _systemfunctions: diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 072ab206f25e4f..3bf38a2212c0e0 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -73,7 +73,7 @@ an event loop: Set *loop* as the current event loop for the current OS thread. - .. deprecated:: next + .. deprecated:: 3.14 The :func:`set_event_loop` function is deprecated and will be removed in Python 3.16. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 9f86234ce941d1..57f964912dd6ea 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -48,7 +48,7 @@ for the current process: Return the current process-wide policy. - .. deprecated:: next + .. deprecated:: 3.14 The :func:`get_event_loop_policy` function is deprecated and will be removed in Python 3.16. @@ -58,7 +58,7 @@ for the current process: If *policy* is set to ``None``, the default policy is restored. - .. deprecated:: next + .. deprecated:: 3.14 The :func:`set_event_loop_policy` function is deprecated and will be removed in Python 3.16. @@ -95,7 +95,7 @@ The abstract event loop policy base class is defined as follows: This method should never return ``None``. - .. deprecated:: next + .. deprecated:: 3.14 The :class:`AbstractEventLoopPolicy` class is deprecated and will be removed in Python 3.16. @@ -121,7 +121,7 @@ asyncio ships with the following built-in policies: The :meth:`get_event_loop` method of the default asyncio policy now raises a :exc:`RuntimeError` if there is no set event loop. - .. deprecated:: next + .. deprecated:: 3.14 The :class:`DefaultEventLoopPolicy` class is deprecated and will be removed in Python 3.16. @@ -133,7 +133,7 @@ asyncio ships with the following built-in policies: .. availability:: Windows. - .. deprecated:: next + .. deprecated:: 3.14 The :class:`WindowsSelectorEventLoopPolicy` class is deprecated and will be removed in Python 3.16. @@ -145,7 +145,7 @@ asyncio ships with the following built-in policies: .. availability:: Windows. - .. deprecated:: next + .. deprecated:: 3.14 The :class:`WindowsProactorEventLoopPolicy` class is deprecated and will be removed in Python 3.16. diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index f25930eb0d873e..1c6b5e03af3560 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -173,7 +173,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. - .. versionchanged:: next + .. versionchanged:: 3.14 If *highlight_day* is given, this date is highlighted in color. This can be :ref:`controlled using environment variables `. @@ -201,7 +201,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. - .. versionchanged:: next + .. versionchanged:: 3.14 If *highlight_day* is given, this date is highlighted in color. This can be :ref:`controlled using environment variables `. @@ -229,7 +229,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :meth:`setfirstweekday` method. The earliest year for which a calendar can be generated is platform-dependent. - .. versionchanged:: next + .. versionchanged:: 3.14 If *highlight_day* is given, this date is highlighted in color. This can be :ref:`controlled using environment variables `. @@ -727,7 +727,7 @@ The following options are accepted: The number of months printed per row. Defaults to 3. -.. versionchanged:: next +.. versionchanged:: 3.14 By default, today's date is highlighted in color and can be :ref:`controlled using environment variables `. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 077a27919fc873..164d706e7738e2 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2233,7 +2233,7 @@ Utility functions .. audit-event:: ctypes.memoryview_at address,size,readonly - .. versionadded:: next + .. versionadded:: 3.14 .. _ctypes-data-types: diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index b81da4af3cff58..b7df9f6b9bcf96 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -91,7 +91,7 @@ Fault handler state The dump now mentions if a garbage collector collection is running if *all_threads* is true. - .. versionchanged:: next + .. versionchanged:: 3.14 Only the current thread is dumped if the :term:`GIL` is disabled to prevent the risk of data races. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 69e6192038ab2b..6fec27bc6f68a2 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5424,7 +5424,7 @@ operating system. Scheduling policy for tasks with deadline constraints. - .. versionadded:: next + .. versionadded:: 3.14 .. data:: SCHED_IDLE @@ -5434,7 +5434,7 @@ operating system. Alias for :data:`SCHED_OTHER`. - .. versionadded:: next + .. versionadded:: 3.14 .. data:: SCHED_SPORADIC diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 0f3b58ef6ea5af..2d57cff10a9278 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -945,7 +945,7 @@ The ``errors`` module has the following attributes: The parser was tried to be stopped or suspended before it started. - .. versionadded:: next + .. versionadded:: 3.14 .. rubric:: Footnotes diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 29387a429b844c..e2a78dc95d4ae1 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -572,7 +572,7 @@ character ``'$'``. Word boundaries are determined by the current locale if the :py:const:`~re.LOCALE` flag is used. - .. versionchanged:: next + .. versionchanged:: 3.14 ``\B`` now matches empty input string. .. index:: single: \d; in regular expressions diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 8ba2bd1dcce8cc..b36acad29ecb00 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -678,7 +678,7 @@ Constants Constant to enable duplicate address and port bindings with load balancing. - .. versionadded:: next + .. versionadded:: 3.14 .. availability:: FreeBSD >= 12.0 diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 9d7b6aa66cd443..37ea32dc8a56e5 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -938,7 +938,7 @@ Constants Whether the OpenSSL library has built-in support for TLS-PHA. - .. versionadded:: next + .. versionadded:: 3.14 .. data:: CHANNEL_BINDING_TYPES diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 512647f5f6e01f..61e2d125445b5a 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2235,7 +2235,7 @@ Settings and special methods Add a turtle shape to TurtleScreen's shapelist. Only thusly registered shapes can be used by issuing the command ``shape(shapename)``. - .. versionchanged:: next + .. versionchanged:: 3.14 Added support for PNG, PGM, and PPM image formats. Both a shape name and an image file name can be specified. diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 0eead59a337ef9..5bb7f88d585e3b 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -974,7 +974,7 @@ Test cases class or a tuple of classes, as supported by :func:`issubclass`). To check for the exact type, use :func:`assertIs(cls, superclass) `. - .. versionadded:: next + .. versionadded:: 3.14 It is also possible to check the production of exceptions, warnings, and @@ -1319,7 +1319,7 @@ Test cases with a *prefix*. *prefix* can also be a tuple of strings to try. - .. versionadded:: next + .. versionadded:: 3.14 .. method:: assertEndsWith(s, suffix, msg=None) @@ -1329,7 +1329,7 @@ Test cases with a *suffix*. *suffix* can also be a tuple of strings to try. - .. versionadded:: next + .. versionadded:: 3.14 .. method:: assertHasAttr(obj, name, msg=None) @@ -1337,7 +1337,7 @@ Test cases Test that the object *obj* has (or has not) an attribute *name*. - .. versionadded:: next + .. versionadded:: 3.14 .. _type-specific-methods: diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 82df8dfc948ed2..b58532acc4b322 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -58,7 +58,7 @@ Features and minimum versions required to build CPython: .. versionchanged:: 3.13 Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. -.. versionchanged:: next +.. versionchanged:: 3.14 Autoconf 2.72 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform diff --git a/Include/patchlevel.h b/Include/patchlevel.h index eca2ca08a4337c..55af2c1e39cf40 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -21,10 +21,10 @@ #define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 3 +#define PY_RELEASE_SERIAL 4 /* Version as a string */ -#define PY_VERSION "3.14.0a3+" +#define PY_VERSION "3.14.0a4" /*--end constants--*/ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index aebcef2b81d43d..97fed45d5a998c 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Dec 17 11:49:52 2024 +# Autogenerated by Sphinx on Tue Jan 14 13:41:56 2025 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -2822,15 +2822,18 @@ ' enter = type(manager).__enter__\n' ' exit = type(manager).__exit__\n' ' value = enter(manager)\n' + ' hit_except = False\n' '\n' ' try:\n' ' TARGET = value\n' ' SUITE\n' ' except:\n' + ' hit_except = True\n' ' if not exit(manager, *sys.exc_info()):\n' ' raise\n' - ' else:\n' - ' exit(manager, None, None, None)\n' + ' finally:\n' + ' if not hit_except:\n' + ' exit(manager, None, None, None)\n' '\n' 'With more than one item, the context managers are processed as ' 'if\n' @@ -5286,15 +5289,16 @@ '\n' ' Added in version 3.14: The *commands* argument.\n' '\n' - 'pdb.post_mortem(traceback=None)\n' + 'pdb.post_mortem(t=None)\n' '\n' - ' Enter post-mortem debugging of the given *traceback* object. ' - 'If no\n' - ' *traceback* is given, it uses the one of the exception that ' - 'is\n' - ' currently being handled (an exception must be being handled ' - 'if the\n' - ' default is to be used).\n' + ' Enter post-mortem debugging of the given exception or ' + 'traceback\n' + ' object. If no value is given, it uses the exception that is\n' + ' currently being handled, or raises "ValueError" if there ' + 'isn’t one.\n' + '\n' + ' Changed in version 3.13: Support for exception objects was ' + 'added.\n' '\n' 'pdb.pm()\n' '\n' @@ -17403,15 +17407,18 @@ ' enter = type(manager).__enter__\n' ' exit = type(manager).__exit__\n' ' value = enter(manager)\n' + ' hit_except = False\n' '\n' ' try:\n' ' TARGET = value\n' ' SUITE\n' ' except:\n' + ' hit_except = True\n' ' if not exit(manager, *sys.exc_info()):\n' ' raise\n' - ' else:\n' - ' exit(manager, None, None, None)\n' + ' finally:\n' + ' if not hit_except:\n' + ' exit(manager, None, None, None)\n' '\n' 'With more than one item, the context managers are processed as if\n' 'multiple "with" statements were nested:\n' diff --git a/Misc/NEWS.d/3.14.0a4.rst b/Misc/NEWS.d/3.14.0a4.rst new file mode 100644 index 00000000000000..1e08b36020386c --- /dev/null +++ b/Misc/NEWS.d/3.14.0a4.rst @@ -0,0 +1,830 @@ +.. date: 2024-12-22-08-54-30 +.. gh-issue: 127592 +.. nonce: iyuFCC +.. release date: 2025-01-14 +.. section: macOS + +Usage of the unified Apple System Log APIs was disabled when the minimum +macOS version is earlier than 10.12. + +.. + +.. date: 2025-01-03-23-51-07 +.. gh-issue: 128152 +.. nonce: IhzElS +.. section: Tools/Demos + +Fix a bug where Argument Clinic's C pre-processor parser tried to parse +pre-processor directives inside C comments. Patch by Erlend Aasland. + +.. + +.. date: 2025-01-13-01-29-08 +.. gh-issue: 128690 +.. nonce: cPFVDb +.. section: Tests + +Temporarily do not use test_embed in PGO profile builds until the problem +with test_init_pyvenv_cfg failing in some configurations is resolved. + +.. + +.. date: 2025-01-11-13-40-12 +.. gh-issue: 128731 +.. nonce: qpKlai +.. section: Library + +Fix :exc:`ResourceWarning` in +:meth:`urllib.robotparser.RobotFileParser.read`. + +.. + +.. date: 2025-01-10-15-06-45 +.. gh-issue: 71339 +.. nonce: EKnpzw +.. section: Library + +Add new assertion methods for :mod:`unittest`: +:meth:`~unittest.TestCase.assertHasAttr`, +:meth:`~unittest.TestCase.assertNotHasAttr`, +:meth:`~unittest.TestCase.assertIsSubclass`, +:meth:`~unittest.TestCase.assertNotIsSubclass` +:meth:`~unittest.TestCase.assertStartsWith`, +:meth:`~unittest.TestCase.assertNotStartsWith`, +:meth:`~unittest.TestCase.assertEndsWith` and +:meth:`~unittest.TestCase.assertNotEndsWith`. + +.. + +.. date: 2025-01-10-13-34-33 +.. gh-issue: 118761 +.. nonce: qRB8nS +.. section: Library + +Improve import time of :mod:`pickle` by 25% by removing an unnecessary +regular expression. As such, :mod:`re` is no more implicitly available as +``pickle.re``. Patch by Bénédikt Tran. + +.. + +.. date: 2025-01-09-12-06-52 +.. gh-issue: 128661 +.. nonce: ixx_0z +.. section: Library + +Fixes :func:`typing.evaluate_forward_ref` not showing deprecation when +``type_params`` arg is not passed. + +.. + +.. date: 2025-01-08-03-09-29 +.. gh-issue: 128562 +.. nonce: Mlv-yO +.. section: Library + +Fix possible conflicts in generated :mod:`tkinter` widget names if the +widget class name ends with a digit. + +.. + +.. date: 2025-01-06-21-35-00 +.. gh-issue: 128559 +.. nonce: 6fxcDM +.. section: Library + +Improved import time of :mod:`asyncio`. + +.. + +.. date: 2025-01-06-18-41-08 +.. gh-issue: 128552 +.. nonce: fV-f8j +.. section: Library + +Fix cyclic garbage introduced by :meth:`asyncio.loop.create_task` and +:meth:`asyncio.TaskGroup.create_task` holding a reference to the created +task if it is eager. + +.. + +.. date: 2025-01-05-11-46-14 +.. gh-issue: 128340 +.. nonce: gKI0uU +.. section: Library + +Add internal thread safe handle to be used in +:meth:`asyncio.loop.call_soon_threadsafe` for thread safe cancellation. + +.. + +.. date: 2025-01-04-11-32-46 +.. gh-issue: 128182 +.. nonce: SJ2Zsa +.. section: Library + +Fix crash when using :mod:`ctypes` pointers concurrently on the :term:`free +threaded ` build. + +.. + +.. date: 2025-01-02-15-20-17 +.. gh-issue: 128400 +.. nonce: UMiG4f +.. section: Library + +Only show the current thread in :mod:`faulthandler` on the :term:`free +threaded ` build to prevent races. + +.. + +.. date: 2025-01-02-13-05-16 +.. gh-issue: 128400 +.. nonce: 5N43fF +.. section: Library + +Fix crash when using :func:`faulthandler.dump_traceback` while other threads +are active on the :term:`free threaded ` build. + +.. + +.. date: 2025-01-01-19-24-43 +.. gh-issue: 128388 +.. nonce: 8UdMz_ +.. section: Library + +Fix ``PyREPL`` on Windows to support more keybindings, like the +:kbd:`Control-←` and :kbd:`Control-→` word-skipping keybindings and those +with meta (i.e. :kbd:`Alt`), e.g. :kbd:`Alt-d` to ``kill-word`` or +:kbd:`Alt-Backspace` ``backward-kill-word``. + +.. + +.. date: 2024-12-30-20-48-28 +.. gh-issue: 88834 +.. nonce: RIvgwc +.. section: Library + +Unify the instance check for :class:`typing.Union` and +:class:`types.UnionType`: :class:`!Union` now uses the instance checks +against its parameters instead of the subclass checks. + +.. + +.. date: 2024-12-29-13-49-46 +.. gh-issue: 128302 +.. nonce: psRpPN +.. section: Library + +Fix :meth:`!xml.dom.xmlbuilder.DOMEntityResolver.resolveEntity`, which was +broken by the Python 3.0 transition. + +.. + +.. date: 2024-12-29-00-33-34 +.. gh-issue: 128317 +.. nonce: WgFina +.. section: Library + +Highlight today in colour in :mod:`calendar`'s CLI output. Patch by Hugo van +Kemenade. + +.. + +.. date: 2024-12-27-16-28-57 +.. gh-issue: 128302 +.. nonce: 2GMvyl +.. section: Library + +Allow :meth:`!xml.dom.xmlbuilder.DOMParser.parse` to correctly handle +:class:`!xml.dom.xmlbuilder.DOMInputSource` instances that only have a +:attr:`!systemId` attribute set. + +.. + +.. date: 2024-12-21-11-12-50 +.. gh-issue: 128151 +.. nonce: aq7vpG +.. section: Library + +Improve generation of :class:`~uuid.UUID` objects version 3, 4, 5, and 8 via +their dedicated functions by 30%. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-20-10-57-10 +.. gh-issue: 128118 +.. nonce: mYak8i +.. section: Library + +Improve performance of :func:`copy.copy` by 30% via a fast path for atomic +types and container types. + +.. + +.. date: 2024-12-19-20-46-01 +.. gh-issue: 127946 +.. nonce: 4lM3Op +.. section: Library + +Fix crash when modifying :class:`ctypes._CFuncPtr` objects concurrently on +the :term:`free threaded ` build. + +.. + +.. date: 2024-12-18-10-18-55 +.. gh-issue: 128062 +.. nonce: E9oU7- +.. section: Library + +Revert the font of :mod:`turtledemo`'s menu bar to its default value and +display the shortcut keys in the correct position. + +.. + +.. date: 2024-12-18-00-07-50 +.. gh-issue: 128014 +.. nonce: F3aUbz +.. section: Library + +Fix resetting the default window icon by passing ``default=''`` to the +:mod:`tkinter` method :meth:`!wm_iconbitmap`. + +.. + +.. date: 2024-12-17-15-23-40 +.. gh-issue: 41872 +.. nonce: 31LjKY +.. section: Library + +Fix quick extraction of module docstrings from a file in :mod:`pydoc`. It +now supports docstrings with single quotes, escape sequences, raw string +literals, and other Python syntax. + +.. + +.. date: 2024-12-17-13-21-52 +.. gh-issue: 127060 +.. nonce: mv2bX6 +.. section: Library + +Set TERM environment variable to "dumb" to disable traceback colors in IDLE, +since IDLE doesn't understand ANSI escape sequences. Patch by Victor +Stinner. + +.. + +.. date: 2024-12-17-12-41-07 +.. gh-issue: 126742 +.. nonce: l07qvT +.. section: Library + +Fix support of localized error messages reported by :manpage:`dlerror(3)` +and :manpage:`gdbm_strerror ` in :mod:`ctypes` and :mod:`dbm.gnu` +functions respectively. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-13-14-21-04 +.. gh-issue: 122548 +.. nonce: hq3Vud +.. section: Library + +Adds two new local events to sys.monitoring, ``BRANCH_LEFT`` and +``BRANCH_RIGHT``. This allows the two arms of the branch to be disabled +independently, which should hugely improve performance of branch-level +coverage tools. The old branch event, ``BRANCH`` is now deprecated. + +.. + +.. date: 2024-12-12-07-27-51 +.. gh-issue: 127847 +.. nonce: ksfNKM +.. section: Library + +Fix the position when doing interleaved seeks and reads in uncompressed, +unencrypted zip files returned by :meth:`zipfile.ZipFile.open`. + +.. + +.. date: 2024-12-06-21-03-11 +.. gh-issue: 127688 +.. nonce: NJqtc- +.. section: Library + +Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants to +the :mod:`os` module. + +.. + +.. date: 2024-12-04-10-39-29 +.. gh-issue: 83662 +.. nonce: CG1s3m +.. section: Library + +Add missing ``__class_getitem__`` method to the Python implementation of +:func:`functools.partial`, to make it compatible with the C version. This is +mainly relevant for alternative Python implementations like PyPy and +GraalPy, because CPython will usually use the C-implementation of that +function. + +.. + +.. date: 2024-12-03-20-28-08 +.. gh-issue: 127586 +.. nonce: zgotYF +.. section: Library + +:class:`multiprocessing.pool.Pool` now properly restores blocked signal +handlers of the parent thread when creating processes via either *spawn* or +*forkserver*. + +.. + +.. date: 2024-12-03-14-45-16 +.. gh-issue: 98188 +.. nonce: GX9i2b +.. section: Library + +Fix an issue in :meth:`email.message.Message.get_payload` where data cannot +be decoded if the Content Transfer Encoding mechanism contains trailing +whitespaces or additional junk text. Patch by Hui Liu. + +.. + +.. date: 2024-12-02-19-13-19 +.. gh-issue: 127529 +.. nonce: Pj1Xtf +.. section: Library + +Correct behavior of +:func:`!asyncio.selector_events.BaseSelectorEventLoop._accept_connection` in +handling :exc:`ConnectionAbortedError` in a loop. This improves performance +on OpenBSD. + +.. + +.. date: 2024-11-28-14-24-12 +.. gh-issue: 127360 +.. nonce: HVKt-c +.. section: Library + +When a descriptive error message cannot be provided for an +:exc:`ssl.SSLError`, the "unknown error" message now shows the internal +error code (as retrieved by ``ERR_get_error`` and similar OpenSSL +functions). + +.. + +.. date: 2024-11-24-14-53-35 +.. gh-issue: 127196 +.. nonce: 8CBkUa +.. section: Library + +Fix crash when dict with keys in invalid encoding were passed to several +functions in ``_interpreters`` module. + +.. + +.. date: 2024-11-19-10-46-57 +.. gh-issue: 124130 +.. nonce: OZ_vR5 +.. section: Library + +Fix a bug in matching regular expression ``\B`` in empty input string. Now +it is always the opposite of ``\b``. To get an old behavior, use +``(?!\A\Z)\B``. To get a new behavior in old Python versions, use +``(?!\b)``. + +.. + +.. date: 2024-11-11-07-56-03 +.. gh-issue: 126639 +.. nonce: AmVSt- +.. section: Library + +:class:`tempfile.NamedTemporaryFile` will now issue a :exc:`ResourceWarning` +when it is finalized by the garbage collector without being explicitly +closed. + +.. + +.. date: 2024-11-09-15-59-51 +.. gh-issue: 126624 +.. nonce: bN53Va +.. section: Library + +Expose error code :data:`~xml.parsers.expat.errors.XML_ERROR_NOT_STARTED` of +Expat >=2.6.4 in :mod:`xml.parsers.expat.errors`. + +.. + +.. date: 2024-10-31-14-31-36 +.. gh-issue: 126225 +.. nonce: vTxGXm +.. section: Library + +:mod:`getopt` and :mod:`optparse` are no longer marked as deprecated. There +are legitimate reasons to use one of these modules in preference to +:mod:`argparse`, and none of these modules are at risk of being removed from +the standard library. Of the three, ``argparse`` remains the recommended +default choice, *unless* one of the concerns noted at the top of the +``optparse`` module documentation applies. + +.. + +.. date: 2024-10-04-09-56-45 +.. gh-issue: 124761 +.. nonce: N4pSD6 +.. section: Library + +Add :data:`~socket.SO_REUSEPORT_LB` constant to :mod:`socket` for FreeBSD. + +.. + +.. date: 2024-09-04-14-13-14 +.. gh-issue: 121720 +.. nonce: z9hhXQ +.. section: Library + +:class:`enum.EnumDict` can now be used without resorting to private API. + +.. + +.. date: 2024-08-28-16-10-37 +.. gh-issue: 123424 +.. nonce: u96_i6 +.. section: Library + +Add :meth:`zipfile.ZipInfo._for_archive` setting default properties on +:class:`~zipfile.ZipInfo` objects. Patch by Bénédikt Tran and Jason R. +Coombs. + +.. + +.. date: 2024-07-13-13-25-31 +.. gh-issue: 121676 +.. nonce: KDLS11 +.. section: Library + +Deprecate calling the Python implementation of :meth:`functools.reduce` with +a ``function`` or ``sequence`` as a :term:`keyword argument`. This will be +forbidden in Python 3.16 in order to match the C implementation. + +.. + +.. date: 2023-11-12-21-53-40 +.. gh-issue: 112015 +.. nonce: 2WPRxE +.. section: Library + +:func:`ctypes.memoryview_at` now exists to create a :class:`memoryview` +object that refers to the supplied pointer and length. This works like +:func:`ctypes.string_at` except it avoids a buffer copy, and is typically +useful when implementing pure Python callback functions that are passed +dynamically-sized buffers. + +.. + +.. date: 2022-07-28-12-32-59 +.. gh-issue: 95371 +.. nonce: F24IFC +.. section: Library + +Added support for other image formats (PNG, PGM, and PPM) to the turtle +module. Patch by Shin-myoung-serp. + +.. + +.. date: 2025-01-13-12-48-30 +.. gh-issue: 128078 +.. nonce: qOsl9B +.. section: Core and Builtins + +Fix a :exc:`SystemError` when using :func:`anext` with a default tuple +value. Patch by Bénédikt Tran. + +.. + +.. date: 2025-01-11-12-39-17 +.. gh-issue: 128717 +.. nonce: i65d06 +.. section: Core and Builtins + +Fix a crash when setting the recursion limit while other threads are active +on the :term:`free threaded ` build. + +.. + +.. date: 2025-01-09-11-46-57 +.. gh-issue: 124483 +.. nonce: KRtBeQ +.. section: Core and Builtins + +Treat ``Py_DECREF`` and variants as escaping when generating opcode and uop +metadata. This prevents the possibility of a ``__del__`` method causing the +JIT to behave incorrectly. + +.. + +.. date: 2025-01-07-19-48-56 +.. gh-issue: 126703 +.. nonce: 0ISs-7 +.. section: Core and Builtins + +Improve performance of class methods by using a freelist. + +.. + +.. date: 2024-12-24-01-40-12 +.. gh-issue: 128137 +.. nonce: gsTwr_ +.. section: Core and Builtins + +Update :c:type:`PyASCIIObject` layout to handle interned field with the +atomic operation. Patch by Donghee Na. + +.. + +.. date: 2024-12-23-11-14-07 +.. gh-issue: 128192 +.. nonce: 02mEhD +.. section: Core and Builtins + +Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by +supporting SHA-256 digest authentication as specified in :rfc:`7616`. + +.. + +.. date: 2024-12-22-15-47-44 +.. gh-issue: 126868 +.. nonce: RpjKez +.. section: Core and Builtins + +Increase usage of freelist for :class:`int` allocation. + +.. + +.. date: 2024-12-20-23-07-33 +.. gh-issue: 114203 +.. nonce: 84NgoW +.. section: Core and Builtins + +Optimize ``Py_BEGIN_CRITICAL_SECTION`` for simple recursive calls. + +.. + +.. date: 2024-12-20-12-25-16 +.. gh-issue: 127705 +.. nonce: WmCz1z +.. section: Core and Builtins + +Adds stackref debugging when ``Py_STACKREF_DEBUG`` is set. Finds all +double-closes and leaks, logging the origin and last borrow. + +Inspired by HPy's debug mode. +https://docs.hpyproject.org/en/latest/debug-mode.html + +.. + +.. date: 2024-12-18-14-22-48 +.. gh-issue: 128079 +.. nonce: SUD5le +.. section: Core and Builtins + +Fix a bug where :keyword:`except* ` does not properly check the +return value of an :exc:`ExceptionGroup`'s :meth:`~BaseExceptionGroup.split` +function, leading to a crash in some cases. Now when +:meth:`~BaseExceptionGroup.split` returns an invalid object, +:keyword:`except* ` raises a :exc:`TypeError` with the original +raised :exc:`ExceptionGroup` object chained to it. + +.. + +.. date: 2024-12-17-22-28-15 +.. gh-issue: 128030 +.. nonce: H1ptOD +.. section: Core and Builtins + +Avoid error from calling ``PyModule_GetFilenameObject`` on a non-module +object when importing a non-existent symbol from a non-module object. + +.. + +.. date: 2024-12-17-18-20-37 +.. gh-issue: 128035 +.. nonce: JwqHdB +.. section: Core and Builtins + +Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports +TLSv1.3 post-handshake client authentication (PHA). Patch by Will +Childs-Klein. + +.. + +.. date: 2024-12-17-13-45-33 +.. gh-issue: 127274 +.. nonce: deNxNC +.. section: Core and Builtins + +Add a new flag, ``CO_METHOD``, to :attr:`~codeobject.co_flags` that +indicates whether the code object belongs to a function defined in class +scope. + +.. + +.. date: 2024-12-15-21-11-26 +.. gh-issue: 66409 +.. nonce: wv109z +.. section: Core and Builtins + +During the :ref:`path initialization `, we now check if +``base_exec_prefix`` is the same as ``base_prefix`` before falling back to +searching the Python interpreter directory. + +.. + +.. date: 2024-12-15-19-51-54 +.. gh-issue: 127970 +.. nonce: vdUp-y +.. section: Core and Builtins + +We now use the location of the ``libpython`` runtime library used in the +current proccess to determine :data:`sys.base_prefix` on all platforms +implementing the `dladdr +`_ +function defined by the UNIX standard — this includes Linux, Android, macOS, +iOS, FreeBSD, etc. This was already the case on Windows and macOS Framework +builds. + +.. + +.. date: 2024-12-13-15-21-45 +.. gh-issue: 127773 +.. nonce: E-DZR4 +.. section: Core and Builtins + +Do not use the type attribute cache for types with incompatible :term:`MRO`. + +.. + +.. date: 2024-12-13-14-17-24 +.. gh-issue: 127903 +.. nonce: vemHSl +.. section: Core and Builtins + +``Objects/unicodeobject.c``: fix a crash on DEBUG builds in +``_copy_characters`` when there is nothing to copy. + +.. + +.. date: 2024-12-11-14-32-22 +.. gh-issue: 127809 +.. nonce: 0W8khe +.. section: Core and Builtins + +Fix an issue where the experimental JIT may infer an incorrect result type +for exponentiation (``**`` and ``**=``), leading to bugs or crashes. + +.. + +.. date: 2024-12-02-18-15-37 +.. gh-issue: 126862 +.. nonce: fdIK7T +.. section: Core and Builtins + +Fix a possible overflow when a class inherits from an absurd number of +super-classes. Reported by Valery Fedorenko. Patch by Bénédikt Tran. + +.. + +.. date: 2025-01-12-12-19-51 +.. gh-issue: 128400 +.. nonce: OwoIDw +.. section: C API + +:c:func:`Py_FatalError` no longer shows all threads on the :term:`free +threaded ` build to prevent crashes. + +.. + +.. date: 2025-01-08-13-13-18 +.. gh-issue: 128629 +.. nonce: gSmzyl +.. section: C API + +Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for +bit-packing Python version numbers. + +.. + +.. date: 2024-12-16-21-59-06 +.. gh-issue: 128008 +.. nonce: fa9Jt0 +.. section: C API + +Add :c:func:`PyWeakref_IsDead` function, which tests if a weak reference is +dead. + +.. + +.. date: 2024-12-11-13-01-26 +.. gh-issue: 127350 +.. nonce: uEBZZ4 +.. section: C API + +Add :c:func:`Py_fopen` function to open a file. Similar to the +:c:func:`!fopen` function, but the *path* parameter is a Python object and +an exception is set on error. Add also :c:func:`Py_fclose` function to close +a file, function needed for Windows support. Patch by Victor Stinner. + +.. + +.. date: 2025-01-09-19-44-00 +.. gh-issue: 128627 +.. nonce: mHzsEd +.. section: Build + +For Emscripten builds the function pointer cast call trampoline now uses the +wasm-gc ref.test instruction if it's available instead of Wasm JS type +reflection. + +.. + +.. date: 2025-01-04-22-39-10 +.. gh-issue: 128472 +.. nonce: Wt5E6M +.. section: Build + +Skip BOLT optimization of functions using computed gotos, fixing errors on +build with LLVM 19. + +.. + +.. date: 2025-01-02-12-50-46 +.. gh-issue: 115765 +.. nonce: jko7Fg +.. section: Build + +GNU Autoconf 2.72 is now required to generate :file:`!configure`. Patch by +Erlend Aasland. + +.. + +.. date: 2025-01-02-11-02-45 +.. gh-issue: 123925 +.. nonce: TLlyUi +.. section: Build + +Fix building the :mod:`curses` module on platforms with libncurses but +without libncursesw. + +.. + +.. date: 2024-12-31-17-09-37 +.. gh-issue: 90905 +.. nonce: PjLNai +.. section: Build + +Add support for cross-compiling to x86_64 on aarch64/arm64 macOS. + +.. + +.. date: 2024-12-28-21-05-19 +.. gh-issue: 128321 +.. nonce: 0UvbXw +.. section: Build + +Set ``LIBS`` instead of ``LDFLAGS`` when checking if :mod:`sqlite3` library +functions are available. This fixes the ordering of linked libraries during +checks, which was incorrect when using a statically linked ``libsqlite3``. + +.. + +.. date: 2024-12-21-09-56-37 +.. gh-issue: 100384 +.. nonce: Ib-XrN +.. section: Build + +Error on ``unguarded-availability`` in macOS builds, preventing invalid use +of symbols that are not available in older versions of the OS. + +.. + +.. date: 2024-12-20-09-03-22 +.. gh-issue: 128104 +.. nonce: m_SoVx +.. section: Build + +Remove ``Py_STRFTIME_C99_SUPPORT`` conditions in favor of requiring C99 +:manpage:`strftime(3)` specifier support at build time. When +cross-compiling, there is no build time check and support is assumed. + +.. + +.. date: 2024-12-16-16-16-35 +.. gh-issue: 127951 +.. nonce: lpE13- +.. section: Build + +Add option ``--pystats`` to the Windows build to enable performance +statistics collection. diff --git a/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst b/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst deleted file mode 100644 index 0c1df0e6bd7baa..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst +++ /dev/null @@ -1 +0,0 @@ -Add option ``--pystats`` to the Windows build to enable performance statistics collection. diff --git a/Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst b/Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst deleted file mode 100644 index c3a47fbecd1dad..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove ``Py_STRFTIME_C99_SUPPORT`` conditions in favor of requiring C99 -:manpage:`strftime(3)` specifier support at build time. When cross-compiling, -there is no build time check and support is assumed. diff --git a/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst b/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst deleted file mode 100644 index 75c19fe3d8cef9..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Error on ``unguarded-availability`` in macOS builds, preventing invalid -use of symbols that are not available in older versions of the OS. diff --git a/Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst b/Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst deleted file mode 100644 index ed72cc8ab1449a..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst +++ /dev/null @@ -1,3 +0,0 @@ -Set ``LIBS`` instead of ``LDFLAGS`` when checking if :mod:`sqlite3` library -functions are available. This fixes the ordering of linked libraries during -checks, which was incorrect when using a statically linked ``libsqlite3``. diff --git a/Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst b/Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst deleted file mode 100644 index 31bb5fe005379a..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-12-31-17-09-37.gh-issue-90905.PjLNai.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for cross-compiling to x86_64 on aarch64/arm64 macOS. diff --git a/Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst b/Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst deleted file mode 100644 index a2a9c6fc7680b5..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix building the :mod:`curses` module on platforms with libncurses but -without libncursesw. diff --git a/Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst b/Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst deleted file mode 100644 index 34618c2c1288bc..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst +++ /dev/null @@ -1,2 +0,0 @@ -GNU Autoconf 2.72 is now required to generate :file:`!configure`. -Patch by Erlend Aasland. diff --git a/Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst b/Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst deleted file mode 100644 index c6233e1f2d8693..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip BOLT optimization of functions using computed gotos, fixing errors on -build with LLVM 19. diff --git a/Misc/NEWS.d/next/Build/2025-01-09-19-44-00.gh-issue-128627.mHzsEd.rst b/Misc/NEWS.d/next/Build/2025-01-09-19-44-00.gh-issue-128627.mHzsEd.rst deleted file mode 100644 index a8c80ab6804b02..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-01-09-19-44-00.gh-issue-128627.mHzsEd.rst +++ /dev/null @@ -1,3 +0,0 @@ -For Emscripten builds the function pointer cast call trampoline now uses the -wasm-gc ref.test instruction if it's available instead of Wasm JS type -reflection. diff --git a/Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst b/Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst deleted file mode 100644 index d1b528c673442f..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add :c:func:`Py_fopen` function to open a file. Similar to the :c:func:`!fopen` -function, but the *path* parameter is a Python object and an exception is set -on error. Add also :c:func:`Py_fclose` function to close a file, function -needed for Windows support. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst b/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst deleted file mode 100644 index 2349eccac8fedc..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:func:`PyWeakref_IsDead` function, which tests if a weak reference is -dead. diff --git a/Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst b/Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst deleted file mode 100644 index cde5bf38f754b6..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for -bit-packing Python version numbers. diff --git a/Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst b/Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst deleted file mode 100644 index b9c117b269434c..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-01-12-12-19-51.gh-issue-128400.OwoIDw.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`Py_FatalError` no longer shows all threads on the :term:`free -threaded ` build to prevent crashes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-18-15-37.gh-issue-126862.fdIK7T.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-18-15-37.gh-issue-126862.fdIK7T.rst deleted file mode 100644 index d930c2963e3632..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-18-15-37.gh-issue-126862.fdIK7T.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible overflow when a class inherits from an absurd number of -super-classes. Reported by Valery Fedorenko. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst deleted file mode 100644 index 19c8cc6e99c8c5..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where the experimental JIT may infer an incorrect result type -for exponentiation (``**`` and ``**=``), leading to bugs or crashes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst deleted file mode 100644 index ad479b52d1675c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst +++ /dev/null @@ -1,2 +0,0 @@ -``Objects/unicodeobject.c``: fix a crash on DEBUG builds in ``_copy_characters`` -when there is nothing to copy. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst deleted file mode 100644 index 7e68b3fecd1c08..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-15-21-45.gh-issue-127773.E-DZR4.rst +++ /dev/null @@ -1 +0,0 @@ -Do not use the type attribute cache for types with incompatible :term:`MRO`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst deleted file mode 100644 index e4dc7b5fe032d6..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst +++ /dev/null @@ -1,6 +0,0 @@ -We now use the location of the ``libpython`` runtime library used in the current -proccess to determine :data:`sys.base_prefix` on all platforms implementing the -`dladdr `_ -function defined by the UNIX standard — this includes Linux, Android, macOS, -iOS, FreeBSD, etc. This was already the case on Windows and macOS Framework -builds. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst deleted file mode 100644 index 0d70ad06c97968..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst +++ /dev/null @@ -1,3 +0,0 @@ -During the :ref:`path initialization `, we now check if -``base_exec_prefix`` is the same as ``base_prefix`` before falling back to -searching the Python interpreter directory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst deleted file mode 100644 index a4608fbbbf19ec..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add a new flag, ``CO_METHOD``, to :attr:`~codeobject.co_flags` that -indicates whether the code object belongs to a function defined in class -scope. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst deleted file mode 100644 index 27815d48425334..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst +++ /dev/null @@ -1 +0,0 @@ -Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports TLSv1.3 post-handshake client authentication (PHA). Patch by Will Childs-Klein. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst deleted file mode 100644 index 93d78632355b76..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid error from calling ``PyModule_GetFilenameObject`` on a non-module object when importing a non-existent symbol from a non-module object. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst deleted file mode 100644 index 8da4e677f068a3..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix a bug where :keyword:`except* ` does not properly check the -return value of an :exc:`ExceptionGroup`'s :meth:`~BaseExceptionGroup.split` -function, leading to a crash in some cases. Now when :meth:`~BaseExceptionGroup.split` -returns an invalid object, :keyword:`except* ` raises a :exc:`TypeError` -with the original raised :exc:`ExceptionGroup` object chained to it. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst deleted file mode 100644 index fde12b78ce0444..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst +++ /dev/null @@ -1,4 +0,0 @@ -Adds stackref debugging when ``Py_STACKREF_DEBUG`` is set. Finds all -double-closes and leaks, logging the origin and last borrow. - -Inspired by HPy's debug mode. https://docs.hpyproject.org/en/latest/debug-mode.html diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst deleted file mode 100644 index 6a9856e90c32bc..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize ``Py_BEGIN_CRITICAL_SECTION`` for simple recursive calls. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst deleted file mode 100644 index ede383deb4ad31..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst +++ /dev/null @@ -1 +0,0 @@ -Increase usage of freelist for :class:`int` allocation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst deleted file mode 100644 index b80ab715ffc7db..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by -supporting SHA-256 digest authentication as specified in :rfc:`7616`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst deleted file mode 100644 index a3b7cde7f67676..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :c:type:`PyASCIIObject` layout to handle interned field with the -atomic operation. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-07-19-48-56.gh-issue-126703.0ISs-7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-07-19-48-56.gh-issue-126703.0ISs-7.rst deleted file mode 100644 index ecb8eddb13c649..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-07-19-48-56.gh-issue-126703.0ISs-7.rst +++ /dev/null @@ -1 +0,0 @@ -Improve performance of class methods by using a freelist. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst deleted file mode 100644 index 6f2e7265d62321..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-09-11-46-57.gh-issue-124483.KRtBeQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Treat ``Py_DECREF`` and variants as escaping when generating opcode and uop -metadata. This prevents the possibility of a ``__del__`` method causing the JIT -to behave incorrectly. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-11-12-39-17.gh-issue-128717.i65d06.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-11-12-39-17.gh-issue-128717.i65d06.rst deleted file mode 100644 index 212c6d3cb97216..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-11-12-39-17.gh-issue-128717.i65d06.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash when setting the recursion limit while other threads are active -on the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst deleted file mode 100644 index 498864a0aa3145..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-12-48-30.gh-issue-128078.qOsl9B.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a :exc:`SystemError` when using :func:`anext` with a default tuple -value. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst b/Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst deleted file mode 100644 index 4a62aaed78b425..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst +++ /dev/null @@ -1 +0,0 @@ -Added support for other image formats (PNG, PGM, and PPM) to the turtle module. Patch by Shin-myoung-serp. diff --git a/Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst b/Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst deleted file mode 100644 index 4b58ec9d219eff..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst +++ /dev/null @@ -1,5 +0,0 @@ -:func:`ctypes.memoryview_at` now exists to create a -:class:`memoryview` object that refers to the supplied pointer and -length. This works like :func:`ctypes.string_at` except it avoids a -buffer copy, and is typically useful when implementing pure Python -callback functions that are passed dynamically-sized buffers. diff --git a/Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst b/Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst deleted file mode 100644 index be589b727a1968..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst +++ /dev/null @@ -1,3 +0,0 @@ -Deprecate calling the Python implementation of :meth:`functools.reduce` -with a ``function`` or ``sequence`` as a :term:`keyword argument`. -This will be forbidden in Python 3.16 in order to match the C implementation. diff --git a/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst b/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst deleted file mode 100644 index 4df4bbf2ba2b73..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst +++ /dev/null @@ -1 +0,0 @@ -Add :meth:`zipfile.ZipInfo._for_archive` setting default properties on :class:`~zipfile.ZipInfo` objects. Patch by Bénédikt Tran and Jason R. Coombs. diff --git a/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst b/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst deleted file mode 100644 index 96da94a9f211af..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`enum.EnumDict` can now be used without resorting to private API. diff --git a/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst b/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst deleted file mode 100644 index 797dd31b368548..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst +++ /dev/null @@ -1 +0,0 @@ -Add :data:`~socket.SO_REUSEPORT_LB` constant to :mod:`socket` for FreeBSD. diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst deleted file mode 100644 index 13a1f213c7a58e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst +++ /dev/null @@ -1,6 +0,0 @@ -:mod:`getopt` and :mod:`optparse` are no longer marked as deprecated. -There are legitimate reasons to use one of these modules in preference to -:mod:`argparse`, and none of these modules are at risk of being removed -from the standard library. Of the three, ``argparse`` remains the -recommended default choice, *unless* one of the concerns noted at the top of -the ``optparse`` module documentation applies. diff --git a/Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst b/Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst deleted file mode 100644 index 468840a651c253..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst +++ /dev/null @@ -1,2 +0,0 @@ -Expose error code :data:`~xml.parsers.expat.errors.XML_ERROR_NOT_STARTED` -of Expat >=2.6.4 in :mod:`xml.parsers.expat.errors`. diff --git a/Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst b/Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst deleted file mode 100644 index 0b75e5858de731..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`tempfile.NamedTemporaryFile` will now issue a :exc:`ResourceWarning` when it is finalized by the garbage collector without being explicitly closed. diff --git a/Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst b/Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst deleted file mode 100644 index a1d4fc8ff4c22f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a bug in matching regular expression ``\B`` in empty input string. -Now it is always the opposite of ``\b``. -To get an old behavior, use ``(?!\A\Z)\B``. -To get a new behavior in old Python versions, use ``(?!\b)``. diff --git a/Misc/NEWS.d/next/Library/2024-11-24-14-53-35.gh-issue-127196.8CBkUa.rst b/Misc/NEWS.d/next/Library/2024-11-24-14-53-35.gh-issue-127196.8CBkUa.rst deleted file mode 100644 index 471f64d185deab..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-24-14-53-35.gh-issue-127196.8CBkUa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash when dict with keys in invalid encoding were passed to several -functions in ``_interpreters`` module. diff --git a/Misc/NEWS.d/next/Library/2024-11-28-14-24-12.gh-issue-127360.HVKt-c.rst b/Misc/NEWS.d/next/Library/2024-11-28-14-24-12.gh-issue-127360.HVKt-c.rst deleted file mode 100644 index 4772aef71e658f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-28-14-24-12.gh-issue-127360.HVKt-c.rst +++ /dev/null @@ -1,4 +0,0 @@ -When a descriptive error message cannot be provided for an -:exc:`ssl.SSLError`, the "unknown error" message now shows the internal -error code (as retrieved by ``ERR_get_error`` and similar OpenSSL -functions). diff --git a/Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst b/Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst deleted file mode 100644 index 26f2fd5923ab7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst +++ /dev/null @@ -1,4 +0,0 @@ -Correct behavior of -:func:`!asyncio.selector_events.BaseSelectorEventLoop._accept_connection` -in handling :exc:`ConnectionAbortedError` in a loop. This improves -performance on OpenBSD. diff --git a/Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst b/Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst deleted file mode 100644 index 30ab8cfc3f0bc6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue in :meth:`email.message.Message.get_payload` where data -cannot be decoded if the Content Transfer Encoding mechanism contains -trailing whitespaces or additional junk text. Patch by Hui Liu. diff --git a/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst deleted file mode 100644 index 80217bd4a10503..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`multiprocessing.pool.Pool` now properly restores blocked signal handlers -of the parent thread when creating processes via either *spawn* or -*forkserver*. diff --git a/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst b/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst deleted file mode 100644 index 5e39933047993c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add missing ``__class_getitem__`` method to the Python implementation of -:func:`functools.partial`, to make it compatible with the C version. This is -mainly relevant for alternative Python implementations like PyPy and -GraalPy, because CPython will usually use the C-implementation of that -function. diff --git a/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst b/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst deleted file mode 100644 index a22b136da72faf..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants -to the :mod:`os` module. diff --git a/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst b/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst deleted file mode 100644 index 3d6e36fb538bca..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst +++ /dev/null @@ -1 +0,0 @@ -Fix the position when doing interleaved seeks and reads in uncompressed, unencrypted zip files returned by :meth:`zipfile.ZipFile.open`. diff --git a/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst b/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst deleted file mode 100644 index 6cd13572ff1893..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst +++ /dev/null @@ -1,4 +0,0 @@ -Adds two new local events to sys.monitoring, ``BRANCH_LEFT`` and -``BRANCH_RIGHT``. This allows the two arms of the branch to be disabled -independently, which should hugely improve performance of branch-level -coverage tools. The old branch event, ``BRANCH`` is now deprecated. diff --git a/Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst b/Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst deleted file mode 100644 index 70f7cc129f66e3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix support of localized error messages reported by :manpage:`dlerror(3)` and -:manpage:`gdbm_strerror ` in :mod:`ctypes` and :mod:`dbm.gnu` -functions respectively. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst b/Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst deleted file mode 100644 index 1da89e7a282147..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst +++ /dev/null @@ -1,2 +0,0 @@ -Set TERM environment variable to "dumb" to disable traceback colors in IDLE, -since IDLE doesn't understand ANSI escape sequences. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst b/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst deleted file mode 100644 index b807dcb284c248..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix quick extraction of module docstrings from a file in :mod:`pydoc`. -It now supports docstrings with single quotes, escape sequences, -raw string literals, and other Python syntax. diff --git a/Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst b/Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst deleted file mode 100644 index ef339a291f0ddd..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix resetting the default window icon by passing ``default=''`` to the -:mod:`tkinter` method :meth:`!wm_iconbitmap`. diff --git a/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst b/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst deleted file mode 100644 index d8e262e0848077..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Revert the font of :mod:`turtledemo`'s menu bar to its default value and -display the shortcut keys in the correct position. diff --git a/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst b/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst deleted file mode 100644 index faf1ec042bc2b9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash when modifying :class:`ctypes._CFuncPtr` objects concurrently on -the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst b/Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst deleted file mode 100644 index bc2898edfda721..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :func:`copy.copy` by 30% via -a fast path for atomic types and container types. diff --git a/Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst b/Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst deleted file mode 100644 index 04c744fb2ba54f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-21-11-12-50.gh-issue-128151.aq7vpG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve generation of :class:`~uuid.UUID` objects version 3, 4, 5, and 8 -via their dedicated functions by 30%. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst b/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst deleted file mode 100644 index 56e2fe6f85f4bf..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst +++ /dev/null @@ -1,3 +0,0 @@ -Allow :meth:`!xml.dom.xmlbuilder.DOMParser.parse` to correctly handle -:class:`!xml.dom.xmlbuilder.DOMInputSource` instances that only have a -:attr:`!systemId` attribute set. diff --git a/Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst b/Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst deleted file mode 100644 index 4441108014569e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst +++ /dev/null @@ -1,2 +0,0 @@ -Highlight today in colour in :mod:`calendar`'s CLI output. Patch by Hugo van -Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst b/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst deleted file mode 100644 index 98c07297b06f8a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`!xml.dom.xmlbuilder.DOMEntityResolver.resolveEntity`, which was -broken by the Python 3.0 transition. diff --git a/Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst b/Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst deleted file mode 100644 index ca43f914880ba3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst +++ /dev/null @@ -1,3 +0,0 @@ -Unify the instance check for :class:`typing.Union` and -:class:`types.UnionType`: :class:`!Union` now uses the instance checks -against its parameters instead of the subclass checks. diff --git a/Misc/NEWS.d/next/Library/2025-01-01-19-24-43.gh-issue-128388.8UdMz_.rst b/Misc/NEWS.d/next/Library/2025-01-01-19-24-43.gh-issue-128388.8UdMz_.rst deleted file mode 100644 index 5bef0fd6bcac17..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-01-19-24-43.gh-issue-128388.8UdMz_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``PyREPL`` on Windows to support more keybindings, like the :kbd:`Control-←` and :kbd:`Control-→` word-skipping keybindings and those with meta (i.e. :kbd:`Alt`), e.g. :kbd:`Alt-d` to ``kill-word`` or :kbd:`Alt-Backspace` ``backward-kill-word``. diff --git a/Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst b/Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst deleted file mode 100644 index 4033dea4eaf7bf..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash when using :func:`faulthandler.dump_traceback` while other threads -are active on the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst b/Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst deleted file mode 100644 index f9d5f84224c8dc..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Only show the current thread in :mod:`faulthandler` on the :term:`free -threaded ` build to prevent races. diff --git a/Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst b/Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst deleted file mode 100644 index 038fecb5710436..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-04-11-32-46.gh-issue-128182.SJ2Zsa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash when using :mod:`ctypes` pointers concurrently on the :term:`free -threaded ` build. diff --git a/Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst b/Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst deleted file mode 100644 index 790400a19f334b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst +++ /dev/null @@ -1 +0,0 @@ -Add internal thread safe handle to be used in :meth:`asyncio.loop.call_soon_threadsafe` for thread safe cancellation. diff --git a/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst b/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst deleted file mode 100644 index 83816f775da9c5..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst +++ /dev/null @@ -1 +0,0 @@ -Fix cyclic garbage introduced by :meth:`asyncio.loop.create_task` and :meth:`asyncio.TaskGroup.create_task` holding a reference to the created task if it is eager. diff --git a/Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst b/Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst deleted file mode 100644 index 7f9380de17761b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst +++ /dev/null @@ -1 +0,0 @@ -Improved import time of :mod:`asyncio`. diff --git a/Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst b/Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst deleted file mode 100644 index eb50dded67bea8..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-08-03-09-29.gh-issue-128562.Mlv-yO.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible conflicts in generated :mod:`tkinter` widget names if the widget class name ends with a digit. diff --git a/Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst b/Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst deleted file mode 100644 index 6c52b3dcc0ed00..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes :func:`typing.evaluate_forward_ref` not showing deprecation when -``type_params`` arg is not passed. diff --git a/Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst b/Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst deleted file mode 100644 index a0a0f891ca55d9..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-10-13-34-33.gh-issue-118761.qRB8nS.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve import time of :mod:`pickle` by 25% by removing an unnecessary -regular expression. As such, :mod:`re` is no more implicitly available -as ``pickle.re``. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst b/Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst deleted file mode 100644 index 5f33a30bd5eae1..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-10-15-06-45.gh-issue-71339.EKnpzw.rst +++ /dev/null @@ -1,9 +0,0 @@ -Add new assertion methods for :mod:`unittest`: -:meth:`~unittest.TestCase.assertHasAttr`, -:meth:`~unittest.TestCase.assertNotHasAttr`, -:meth:`~unittest.TestCase.assertIsSubclass`, -:meth:`~unittest.TestCase.assertNotIsSubclass` -:meth:`~unittest.TestCase.assertStartsWith`, -:meth:`~unittest.TestCase.assertNotStartsWith`, -:meth:`~unittest.TestCase.assertEndsWith` and -:meth:`~unittest.TestCase.assertNotEndsWith`. diff --git a/Misc/NEWS.d/next/Library/2025-01-11-13-40-12.gh-issue-128731.qpKlai.rst b/Misc/NEWS.d/next/Library/2025-01-11-13-40-12.gh-issue-128731.qpKlai.rst deleted file mode 100644 index b23499ca2fea3a..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-11-13-40-12.gh-issue-128731.qpKlai.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :exc:`ResourceWarning` in :meth:`urllib.robotparser.RobotFileParser.read`. diff --git a/Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst b/Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst deleted file mode 100644 index fb84334ca77214..00000000000000 --- a/Misc/NEWS.d/next/Tests/2025-01-13-01-29-08.gh-issue-128690.cPFVDb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Temporarily do not use test_embed in PGO profile builds until the problem -with test_init_pyvenv_cfg failing in some configurations is resolved. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst b/Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst deleted file mode 100644 index 9657e138e9911b..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug where Argument Clinic's C pre-processor parser tried to parse -pre-processor directives inside C comments. Patch by Erlend Aasland. diff --git a/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst b/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst deleted file mode 100644 index dfe659294c712e..00000000000000 --- a/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Usage of the unified Apple System Log APIs was disabled when the minimum -macOS version is earlier than 10.12. diff --git a/README.rst b/README.rst index 02776205e6dcc9..6f74bc3f9ed7ef 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.14.0 alpha 3 +This is Python version 3.14.0 alpha 4 ===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push From 75bd42c73718ca54ed70d5bf90cbc76c2c1d5ddc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Jan 2025 15:18:25 +0200 Subject: [PATCH 31/61] gh-71339: Use new assertion methods in test_sqlite3 (GH-128830) --- Lib/test/test_sqlite3/test_cli.py | 16 ++++++------ Lib/test/test_sqlite3/test_dbapi.py | 37 ++++++++++----------------- Lib/test/test_sqlite3/test_factory.py | 2 +- 3 files changed, 22 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index d014a9ce841607..dcd90d11d46819 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -90,14 +90,14 @@ def test_interact(self): out, err = self.run_cli() self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn(self.MEMORY_DB_MSG, err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 1) self.assertEqual(out.count(self.PS2), 0) def test_interact_quit(self): out, err = self.run_cli(commands=(".quit",)) self.assertIn(self.MEMORY_DB_MSG, err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 1) self.assertEqual(out.count(self.PS2), 0) @@ -105,7 +105,7 @@ def test_interact_version(self): out, err = self.run_cli(commands=(".version",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn(sqlite3.sqlite_version + "\n", out) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 0) self.assertIn(sqlite3.sqlite_version, out) @@ -114,14 +114,14 @@ def test_interact_valid_sql(self): out, err = self.run_cli(commands=("SELECT 1;",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn("(1,)\n", out) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 0) def test_interact_incomplete_multiline_sql(self): out, err = self.run_cli(commands=("SELECT 1",)) self.assertIn(self.MEMORY_DB_MSG, err) - self.assertTrue(out.endswith(self.PS2)) + self.assertEndsWith(out, self.PS2) self.assertEqual(out.count(self.PS1), 1) self.assertEqual(out.count(self.PS2), 1) @@ -130,7 +130,7 @@ def test_interact_valid_multiline_sql(self): self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn(self.PS2, out) self.assertIn("(1,)\n", out) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 1) @@ -138,7 +138,7 @@ def test_interact_invalid_sql(self): out, err = self.run_cli(commands=("sel;",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn("OperationalError (SQLITE_ERROR)", err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 0) @@ -147,7 +147,7 @@ def test_interact_on_disk_file(self): out, err = self.run_cli(TESTFN, commands=("CREATE TABLE t(t);",)) self.assertIn(TESTFN, err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) out, _ = self.run_cli(TESTFN, commands=("SELECT count(t) FROM t;",)) self.assertIn("(0,)\n", out) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 488b401fb0054d..f5ffe2427430e2 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -59,45 +59,34 @@ def test_param_style(self): sqlite.paramstyle) def test_warning(self): - self.assertTrue(issubclass(sqlite.Warning, Exception), - "Warning is not a subclass of Exception") + self.assertIsSubclass(sqlite.Warning, Exception) def test_error(self): - self.assertTrue(issubclass(sqlite.Error, Exception), - "Error is not a subclass of Exception") + self.assertIsSubclass(sqlite.Error, Exception) def test_interface_error(self): - self.assertTrue(issubclass(sqlite.InterfaceError, sqlite.Error), - "InterfaceError is not a subclass of Error") + self.assertIsSubclass(sqlite.InterfaceError, sqlite.Error) def test_database_error(self): - self.assertTrue(issubclass(sqlite.DatabaseError, sqlite.Error), - "DatabaseError is not a subclass of Error") + self.assertIsSubclass(sqlite.DatabaseError, sqlite.Error) def test_data_error(self): - self.assertTrue(issubclass(sqlite.DataError, sqlite.DatabaseError), - "DataError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.DataError, sqlite.DatabaseError) def test_operational_error(self): - self.assertTrue(issubclass(sqlite.OperationalError, sqlite.DatabaseError), - "OperationalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.OperationalError, sqlite.DatabaseError) def test_integrity_error(self): - self.assertTrue(issubclass(sqlite.IntegrityError, sqlite.DatabaseError), - "IntegrityError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.IntegrityError, sqlite.DatabaseError) def test_internal_error(self): - self.assertTrue(issubclass(sqlite.InternalError, sqlite.DatabaseError), - "InternalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.InternalError, sqlite.DatabaseError) def test_programming_error(self): - self.assertTrue(issubclass(sqlite.ProgrammingError, sqlite.DatabaseError), - "ProgrammingError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.ProgrammingError, sqlite.DatabaseError) def test_not_supported_error(self): - self.assertTrue(issubclass(sqlite.NotSupportedError, - sqlite.DatabaseError), - "NotSupportedError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.NotSupportedError, sqlite.DatabaseError) def test_module_constants(self): consts = [ @@ -274,7 +263,7 @@ def test_module_constants(self): consts.append("SQLITE_IOERR_CORRUPTFS") for const in consts: with self.subTest(const=const): - self.assertTrue(hasattr(sqlite, const)) + self.assertHasAttr(sqlite, const) def test_error_code_on_exception(self): err_msg = "unable to open database file" @@ -288,7 +277,7 @@ def test_error_code_on_exception(self): sqlite.connect(db) e = cm.exception self.assertEqual(e.sqlite_errorcode, err_code) - self.assertTrue(e.sqlite_errorname.startswith("SQLITE_CANTOPEN")) + self.assertStartsWith(e.sqlite_errorname, "SQLITE_CANTOPEN") def test_extended_error_code_on_exception(self): with memory_database() as con: @@ -425,7 +414,7 @@ def test_connection_exceptions(self): ] for exc in exceptions: with self.subTest(exc=exc): - self.assertTrue(hasattr(self.cx, exc)) + self.assertHasAttr(self.cx, exc) self.assertIs(getattr(sqlite, exc), getattr(self.cx, exc)) def test_interrupt_on_closed_db(self): diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 48d35b54a2e239..cc9f1ec5c4bec5 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -280,7 +280,7 @@ def test_custom(self): austria = "Österreich" row = self.con.execute("select ?", (austria,)).fetchone() self.assertEqual(type(row[0]), str, "type of row[0] must be unicode") - self.assertTrue(row[0].endswith("reich"), "column must contain original data") + self.assertEndsWith(row[0], "reich", "column must contain original data") class TextFactoryTestsWithEmbeddedZeroBytes(unittest.TestCase): From 859db49029b1021e24b5c99d4a9a0f57327d1b57 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Jan 2025 15:19:32 +0200 Subject: [PATCH 32/61] gh-71339: Use new assertion methods in test_typing (GH-128825) --- Lib/test/test_typing.py | 103 ++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c98e6f820e8cf7..f002d28df60e9c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -115,18 +115,18 @@ def test_errors(self): def test_can_subclass(self): class Mock(Any): pass - self.assertTrue(issubclass(Mock, Any)) + self.assertIsSubclass(Mock, Any) self.assertIsInstance(Mock(), Mock) class Something: pass - self.assertFalse(issubclass(Something, Any)) + self.assertNotIsSubclass(Something, Any) self.assertNotIsInstance(Something(), Mock) class MockSomething(Something, Mock): pass - self.assertTrue(issubclass(MockSomething, Any)) - self.assertTrue(issubclass(MockSomething, MockSomething)) - self.assertTrue(issubclass(MockSomething, Something)) - self.assertTrue(issubclass(MockSomething, Mock)) + self.assertIsSubclass(MockSomething, Any) + self.assertIsSubclass(MockSomething, MockSomething) + self.assertIsSubclass(MockSomething, Something) + self.assertIsSubclass(MockSomething, Mock) ms = MockSomething() self.assertIsInstance(ms, MockSomething) self.assertIsInstance(ms, Something) @@ -1997,11 +1997,11 @@ def test_basics(self): self.assertNotEqual(u, Union) def test_union_isinstance(self): - self.assertTrue(isinstance(42, Union[int, str])) - self.assertTrue(isinstance('abc', Union[int, str])) - self.assertFalse(isinstance(3.14, Union[int, str])) - self.assertTrue(isinstance(42, Union[int, list[int]])) - self.assertTrue(isinstance(42, Union[int, Any])) + self.assertIsInstance(42, Union[int, str]) + self.assertIsInstance('abc', Union[int, str]) + self.assertNotIsInstance(3.14, Union[int, str]) + self.assertIsInstance(42, Union[int, list[int]]) + self.assertIsInstance(42, Union[int, Any]) def test_union_isinstance_type_error(self): with self.assertRaises(TypeError): @@ -2018,9 +2018,9 @@ def test_union_isinstance_type_error(self): isinstance(42, Union[Any, str]) def test_optional_isinstance(self): - self.assertTrue(isinstance(42, Optional[int])) - self.assertTrue(isinstance(None, Optional[int])) - self.assertFalse(isinstance('abc', Optional[int])) + self.assertIsInstance(42, Optional[int]) + self.assertIsInstance(None, Optional[int]) + self.assertNotIsInstance('abc', Optional[int]) def test_optional_isinstance_type_error(self): with self.assertRaises(TypeError): @@ -2033,14 +2033,14 @@ def test_optional_isinstance_type_error(self): isinstance(None, Optional[Any]) def test_union_issubclass(self): - self.assertTrue(issubclass(int, Union[int, str])) - self.assertTrue(issubclass(str, Union[int, str])) - self.assertFalse(issubclass(float, Union[int, str])) - self.assertTrue(issubclass(int, Union[int, list[int]])) - self.assertTrue(issubclass(int, Union[int, Any])) - self.assertFalse(issubclass(int, Union[str, Any])) - self.assertTrue(issubclass(int, Union[Any, int])) - self.assertFalse(issubclass(int, Union[Any, str])) + self.assertIsSubclass(int, Union[int, str]) + self.assertIsSubclass(str, Union[int, str]) + self.assertNotIsSubclass(float, Union[int, str]) + self.assertIsSubclass(int, Union[int, list[int]]) + self.assertIsSubclass(int, Union[int, Any]) + self.assertNotIsSubclass(int, Union[str, Any]) + self.assertIsSubclass(int, Union[Any, int]) + self.assertNotIsSubclass(int, Union[Any, str]) def test_union_issubclass_type_error(self): with self.assertRaises(TypeError): @@ -2057,12 +2057,12 @@ def test_union_issubclass_type_error(self): issubclass(int, Union[list[int], str]) def test_optional_issubclass(self): - self.assertTrue(issubclass(int, Optional[int])) - self.assertTrue(issubclass(type(None), Optional[int])) - self.assertFalse(issubclass(str, Optional[int])) - self.assertTrue(issubclass(Any, Optional[Any])) - self.assertTrue(issubclass(type(None), Optional[Any])) - self.assertFalse(issubclass(int, Optional[Any])) + self.assertIsSubclass(int, Optional[int]) + self.assertIsSubclass(type(None), Optional[int]) + self.assertNotIsSubclass(str, Optional[int]) + self.assertIsSubclass(Any, Optional[Any]) + self.assertIsSubclass(type(None), Optional[Any]) + self.assertNotIsSubclass(int, Optional[Any]) def test_optional_issubclass_type_error(self): with self.assertRaises(TypeError): @@ -4050,8 +4050,8 @@ def test_generic_protocols_repr(self): class P(Protocol[T, S]): pass - self.assertTrue(repr(P[T, S]).endswith('P[~T, ~S]')) - self.assertTrue(repr(P[int, str]).endswith('P[int, str]')) + self.assertEndsWith(repr(P[T, S]), 'P[~T, ~S]') + self.assertEndsWith(repr(P[int, str]), 'P[int, str]') def test_generic_protocols_eq(self): T = TypeVar('T') @@ -4641,8 +4641,7 @@ class C(Generic[T]): self.assertNotEqual(Z, Y[int]) self.assertNotEqual(Z, Y[T]) - self.assertTrue(str(Z).endswith( - '.C[typing.Tuple[str, int]]')) + self.assertEndsWith(str(Z), '.C[typing.Tuple[str, int]]') def test_new_repr(self): T = TypeVar('T') @@ -4870,12 +4869,12 @@ class A(Generic[T]): self.assertNotEqual(typing.FrozenSet[A[str]], typing.FrozenSet[mod_generics_cache.B.A[str]]) - self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) - self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) - .endswith('mod_generics_cache.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) - .endswith('mod_generics_cache.B.A[str]]')) + self.assertEndsWith(repr(Tuple[A[str]]), '.A[str]]') + self.assertEndsWith(repr(Tuple[B.A[str]]), '.B.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.A[str]]), + 'mod_generics_cache.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.B.A[str]]), + 'mod_generics_cache.B.A[str]]') def test_extended_generic_rules_eq(self): T = TypeVar('T') @@ -5817,7 +5816,7 @@ def __call__(self, *args, **kwargs): @Wrapper def wrapped(): ... self.assertIsInstance(wrapped, Wrapper) - self.assertIs(False, hasattr(wrapped, "__final__")) + self.assertNotHasAttr(wrapped, "__final__") class Meta(type): @property @@ -5829,7 +5828,7 @@ class WithMeta(metaclass=Meta): ... # Builtin classes throw TypeError if you try to set an # attribute. final(int) - self.assertIs(False, hasattr(int, "__final__")) + self.assertNotHasAttr(int, "__final__") # Make sure it works with common builtin decorators class Methods: @@ -5910,19 +5909,19 @@ def static_method_bad_order(): self.assertEqual(Derived.class_method_good_order(), 42) self.assertIs(True, Derived.class_method_good_order.__override__) self.assertEqual(Derived.class_method_bad_order(), 42) - self.assertIs(False, hasattr(Derived.class_method_bad_order, "__override__")) + self.assertNotHasAttr(Derived.class_method_bad_order, "__override__") self.assertEqual(Derived.static_method_good_order(), 42) self.assertIs(True, Derived.static_method_good_order.__override__) self.assertEqual(Derived.static_method_bad_order(), 42) - self.assertIs(False, hasattr(Derived.static_method_bad_order, "__override__")) + self.assertNotHasAttr(Derived.static_method_bad_order, "__override__") # Base object is not changed: - self.assertIs(False, hasattr(Base.normal_method, "__override__")) - self.assertIs(False, hasattr(Base.class_method_good_order, "__override__")) - self.assertIs(False, hasattr(Base.class_method_bad_order, "__override__")) - self.assertIs(False, hasattr(Base.static_method_good_order, "__override__")) - self.assertIs(False, hasattr(Base.static_method_bad_order, "__override__")) + self.assertNotHasAttr(Base.normal_method, "__override__") + self.assertNotHasAttr(Base.class_method_good_order, "__override__") + self.assertNotHasAttr(Base.class_method_bad_order, "__override__") + self.assertNotHasAttr(Base.static_method_good_order, "__override__") + self.assertNotHasAttr(Base.static_method_bad_order, "__override__") def test_property(self): class Base: @@ -5947,8 +5946,8 @@ def wrong(self) -> int: self.assertEqual(instance.correct, 2) self.assertTrue(Child.correct.fget.__override__) self.assertEqual(instance.wrong, 2) - self.assertFalse(hasattr(Child.wrong, "__override__")) - self.assertFalse(hasattr(Child.wrong.fset, "__override__")) + self.assertNotHasAttr(Child.wrong, "__override__") + self.assertNotHasAttr(Child.wrong.fset, "__override__") def test_silent_failure(self): class CustomProp: @@ -5965,7 +5964,7 @@ def some(self): return 1 self.assertEqual(WithOverride.some, 1) - self.assertFalse(hasattr(WithOverride.some, "__override__")) + self.assertNotHasAttr(WithOverride.some, "__override__") def test_multiple_decorators(self): def with_wraps(f): # similar to `lru_cache` definition @@ -10474,7 +10473,7 @@ def test_special_attrs2(self): # to the variable name to which it is assigned". Thus, providing # __qualname__ is unnecessary. self.assertEqual(SpecialAttrsT.__name__, 'SpecialAttrsT') - self.assertFalse(hasattr(SpecialAttrsT, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsT, '__qualname__') self.assertEqual(SpecialAttrsT.__module__, __name__) # Module-level type variables are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -10483,7 +10482,7 @@ def test_special_attrs2(self): self.assertIs(SpecialAttrsT, loaded) self.assertEqual(SpecialAttrsP.__name__, 'SpecialAttrsP') - self.assertFalse(hasattr(SpecialAttrsP, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsP, '__qualname__') self.assertEqual(SpecialAttrsP.__module__, __name__) # Module-level ParamSpecs are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): From bbd3300ae82a71da483dbb0c345175ba090263c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 14 Jan 2025 14:25:33 +0100 Subject: [PATCH 33/61] gh-118761: substitute `re` import in `base64.b16decode` for a more efficient alternative (#128736) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Chris Markiewicz Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 9 +++++++++ Lib/base64.py | 3 +-- .../2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst | 5 +++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1152c7a90b7712..ac203b208f4a8a 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -728,6 +728,15 @@ asyncio reduces memory usage. (Contributed by Kumar Aditya in :gh:`107803`.) + +base64 +------ + +* Improve the performance of :func:`base64.b16decode` by up to ten times, + and reduce the import time of :mod:`base64` by up to six times. + (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) + + io --- * :mod:`io` which provides the built-in :func:`open` makes less system calls diff --git a/Lib/base64.py b/Lib/base64.py index 61be4fb856e92c..5d78cc09f40cd3 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -4,7 +4,6 @@ # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import re import struct import binascii @@ -284,7 +283,7 @@ def b16decode(s, casefold=False): s = _bytes_from_decode_data(s) if casefold: s = s.upper() - if re.search(b'[^0-9A-F]', s): + if s.translate(None, delete=b'0123456789ABCDEF'): raise binascii.Error('Non-base16 digit found') return binascii.unhexlify(s) diff --git a/Misc/NEWS.d/next/Library/2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst b/Misc/NEWS.d/next/Library/2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst new file mode 100644 index 00000000000000..37c25cb2efd034 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst @@ -0,0 +1,5 @@ +Improve the performance of :func:`base64.b16decode` by up to ten times +by more efficiently checking the byte-string for hexadecimal digits. +Reduce the import time of :mod:`base64` by up to six times, +by no longer importing :mod:`re`. +Patch by Bénédikt Tran, Chris Markiewicz, and Adam Turner. From f49a1df6f3b1249bcaedfda8e000e27c6cdffd99 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Jan 2025 15:08:56 +0000 Subject: [PATCH 34/61] GH-128682: Convert explicit loops closing arrays into `DECREF_INPUTS`. (GH-128822) * Mark Py_DECREF and Py_XDECREF as escaping * Remove explicit loops for clearing array inputs --- Python/bytecodes.c | 139 +++++-------- Python/executor_cases.c.h | 132 +++++++----- Python/generated_cases.c.h | 226 ++++++++++++--------- Python/optimizer_cases.c.h | 69 ++++--- Tools/cases_generator/analyzer.py | 3 - Tools/cases_generator/generators_common.py | 1 + 6 files changed, 305 insertions(+), 265 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ef7086c3c020b0..cec530fefffefb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1004,10 +1004,10 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! - Py_DECREF(old_value); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); } inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub -- )) { @@ -1885,9 +1885,8 @@ dummy_func( if (err == 0) { err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); } - PyStackRef_CLOSE(values[i]); } - DEAD(values); + DECREF_INPUTS(); if (err != 0) { Py_DECREF(set_o); ERROR_IF(true, error); @@ -2412,8 +2411,8 @@ dummy_func( _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); } macro(STORE_ATTR_INSTANCE_VALUE) = @@ -2457,9 +2456,9 @@ dummy_func( // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. - Py_XDECREF(old_value); STAT_INC(STORE_ATTR, hit); PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); } macro(STORE_ATTR_WITH_HINT) = @@ -2476,8 +2475,8 @@ dummy_func( PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); } macro(STORE_ATTR_SLOT) = @@ -3435,8 +3434,9 @@ dummy_func( // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } // Check if the call can be inlined or not @@ -3448,7 +3448,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, total_args, NULL, frame + arguments, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). SYNC_SP(); @@ -3461,13 +3461,9 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(true, error); } PyObject *res_o = PyObject_Vectorcall( @@ -3477,7 +3473,7 @@ dummy_func( STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -3493,11 +3489,7 @@ dummy_func( } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3617,12 +3609,13 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -3633,11 +3626,7 @@ dummy_func( NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3877,15 +3866,9 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(true, error); } - DEAD(self_or_null); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } - DEAD(args); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3952,20 +3935,13 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(true, error); } - DEAD(self_or_null); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable_o), args_o, total_args); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } - DEAD(args); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3981,34 +3957,28 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ + /* res = func(self, arguments, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); } PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4147,8 +4117,9 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -4156,12 +4127,12 @@ dummy_func( PyMethodDef *meth = method->d_method; EXIT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); EXIT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4171,13 +4142,7 @@ dummy_func( PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4231,8 +4196,9 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -4240,12 +4206,12 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; EXIT_IF(meth->ml_flags != METH_FASTCALL); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4255,13 +4221,7 @@ dummy_func( PyObject *res_o = cfunc(self, (args_o + 1), nargs); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4313,8 +4273,9 @@ dummy_func( // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -4327,7 +4288,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // Sync stack explicitly since we leave using DISPATCH_INLINED(). @@ -4342,7 +4303,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4354,7 +4315,7 @@ dummy_func( STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -4369,13 +4330,7 @@ dummy_func( } } } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4385,8 +4340,9 @@ dummy_func( // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -4396,7 +4352,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -4487,12 +4443,13 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4506,11 +4463,7 @@ dummy_func( PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - DEAD(self_or_null); - PyStackRef_CLOSE(callable[0]); + DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0d3b9b269ab247..1aa80f398d7470 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -943,8 +943,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -979,8 +979,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -1300,11 +1300,13 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! - Py_DECREF(old_value); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -1452,8 +1454,8 @@ "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -2092,7 +2094,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -2302,10 +2306,16 @@ err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(values[i]); + } + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } if (err != 0) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) JUMP_TO_ERROR(); } set = PyStackRef_FromPyObjectSteal(set_o); @@ -2367,12 +2377,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) JUMP_TO_ERROR(); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); } break; } @@ -2992,10 +3004,12 @@ _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -3059,11 +3073,13 @@ UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. - Py_XDECREF(old_value); STAT_INC(STORE_ATTR, hit); PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -3083,10 +3099,12 @@ PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -3111,8 +3129,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) JUMP_TO_ERROR(); res = res_bool ? PyStackRef_True : PyStackRef_False; } @@ -3859,7 +3877,9 @@ tb = Py_None; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(tb); + stack_pointer = _PyFrame_GetStackPointer(frame); } assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off @@ -4206,12 +4226,13 @@ #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -4228,10 +4249,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4723,11 +4745,11 @@ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4833,11 +4855,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4858,8 +4880,9 @@ /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } if (!PyCFunction_CheckExact(callable_o)) { @@ -4871,13 +4894,13 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ + /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -4891,11 +4914,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -5115,8 +5138,9 @@ callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -5130,14 +5154,14 @@ JUMP_TO_JUMP_TARGET(); } PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); if (!Py_IS_TYPE(self, d_type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -5153,11 +5177,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -5236,8 +5260,9 @@ callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -5251,14 +5276,14 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -5274,11 +5299,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -5334,8 +5359,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -5346,7 +5372,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); @@ -5463,12 +5489,13 @@ #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -5489,10 +5516,11 @@ PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3 - oparg] = res; @@ -6128,7 +6156,9 @@ case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND0(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(tstate->previous_executor); + stack_pointer = _PyFrame_GetStackPointer(frame); tstate->previous_executor = NULL; #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 45423a2565c76d..810beb61d0db5e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -418,8 +418,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -765,15 +765,17 @@ err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(values[i]); + } + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } if (err != 0) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; @@ -931,8 +933,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } // Check if the call can be inlined or not @@ -945,7 +948,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, total_args, NULL, frame + arguments, total_args, NULL, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -960,12 +963,13 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -981,7 +985,7 @@ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( @@ -1001,10 +1005,11 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1378,11 +1383,11 @@ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1462,11 +1467,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1515,20 +1520,21 @@ /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ + /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -1546,11 +1552,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1967,8 +1973,9 @@ PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -1983,7 +1990,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); @@ -2000,7 +2007,7 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -2024,7 +2031,7 @@ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( @@ -2043,12 +2050,12 @@ } } } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2115,8 +2122,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -2127,7 +2135,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); @@ -2196,12 +2204,13 @@ #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -2226,10 +2235,11 @@ PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2291,8 +2301,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -2303,7 +2314,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); @@ -2446,8 +2457,9 @@ callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -2455,11 +2467,11 @@ DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -2479,11 +2491,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2531,8 +2543,9 @@ callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -2540,11 +2553,11 @@ PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -2564,11 +2577,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2772,12 +2785,13 @@ #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -2798,10 +2812,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3279,8 +3294,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) goto error; res = res_bool ? PyStackRef_True : PyStackRef_False; } @@ -3602,7 +3617,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -4069,7 +4086,9 @@ #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } #endif /* Jump forward oparg, then skip following END_FOR instruction */ @@ -4159,7 +4178,9 @@ if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { if (seq != NULL) { it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); @@ -4220,8 +4241,8 @@ "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -4455,8 +4476,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } // Check if the call can be inlined or not @@ -4469,7 +4491,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, total_args, NULL, frame + arguments, total_args, NULL, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -4484,12 +4506,13 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4505,7 +4528,7 @@ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( @@ -4525,10 +4548,11 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -6470,8 +6494,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = PyObject_GetAttr(super, name); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); null = PyStackRef_NULL; @@ -6961,8 +6985,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } stack_pointer += 1; @@ -7307,12 +7331,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); } @@ -7472,11 +7498,13 @@ _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7507,11 +7535,13 @@ PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7570,12 +7600,14 @@ UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. - Py_XDECREF(old_value); STAT_INC(STORE_ATTR, hit); PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7723,8 +7755,8 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -7839,11 +7871,13 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! - Py_DECREF(old_value); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -8254,7 +8288,9 @@ tb = Py_None; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(tb); + stack_pointer = _PyFrame_GetStackPointer(frame); } assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a64dd465467122..1a7cc6becfefb6 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -279,16 +279,19 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { res = sym_new_type(ctx, &PyLong_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -309,16 +312,19 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { res = sym_new_type(ctx, &PyLong_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -339,16 +345,19 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { res = sym_new_type(ctx, &PyLong_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -401,16 +410,19 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -432,16 +444,19 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -463,16 +478,19 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -503,14 +521,17 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { res = sym_new_type(ctx, &PyUnicode_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = res; break; } @@ -527,15 +548,17 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { res = sym_new_type(ctx, &PyUnicode_Type); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } // _STORE_FAST: GETLOCAL(this_instr->operand0) = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4e1d47281fdbbf..99896f32fd2b08 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -567,7 +567,6 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "PyUnicode_READ_CHAR", "Py_ARRAY_LENGTH", "Py_CLEAR", - "Py_DECREF", "Py_FatalError", "Py_INCREF", "Py_IS_TYPE", @@ -577,7 +576,6 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "Py_TYPE", "Py_UNREACHABLE", "Py_Unicode_GET_LENGTH", - "Py_XDECREF", "_PyCode_CODE", "_PyDictValues_AddToInsertionOrder", "_PyErr_Occurred", @@ -620,7 +618,6 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "_PyUnicode_JoinArray", "_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY", "_Py_DECREF_NO_DEALLOC", - "_Py_DECREF_SPECIALIZED", "_Py_EnterRecursiveCallTstateUnchecked", "_Py_ID", "_Py_IsImmortal", diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 0035deb02ffdf0..69d84183f1c7e6 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -121,6 +121,7 @@ def __init__(self, out: CWriter): "SAVE_STACK": self.save_stack, "RELOAD_STACK": self.reload_stack, "PyStackRef_CLOSE": self.stackref_close, + "PyStackRef_XCLOSE": self.stackref_close, "PyStackRef_CLOSE_SPECIALIZED": self.stackref_close_specialized, "PyStackRef_AsPyObjectSteal": self.stackref_steal, "DISPATCH": self.dispatch, From aa805887cf23e5bff1598615d4f91369835f016f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:10:53 +0200 Subject: [PATCH 35/61] Post 3.14.0a4 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 55af2c1e39cf40..40e7d73b7a6634 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,7 +24,7 @@ #define PY_RELEASE_SERIAL 4 /* Version as a string */ -#define PY_VERSION "3.14.0a4" +#define PY_VERSION "3.14.0a4+" /*--end constants--*/ From 1598e18a659ef3a533caa0a79a489a3ed1d80bdd Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Tue, 14 Jan 2025 12:17:24 -0300 Subject: [PATCH 36/61] Fix a "doctest" block in `Doc/library/turtle.rst` (#128831) --- Doc/library/turtle.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 61e2d125445b5a..548b9d3543fb94 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -993,8 +993,8 @@ Settings for measurement >>> turtle.heading() 90.0 - Change angle measurement unit to grad (also known as gon, - grade, or gradian and equals 1/100-th of the right angle.) + >>> # Change angle measurement unit to grad (also known as gon, + >>> # grade, or gradian and equals 1/100-th of the right angle.) >>> turtle.degrees(400.0) >>> turtle.heading() 100.0 From 43ef9587ae869fd11711a3bb126e65afe9177342 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jan 2025 16:29:21 +0100 Subject: [PATCH 37/61] gh-106320: Document replacement for removed C API (#128787) --- Doc/whatsnew/3.13.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 6a0e483bd895d6..c2cfdf3186f86d 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -2706,6 +2706,33 @@ Changes in the C API Calling this function is redundant now that :c:func:`PyFrame_GetLocals` returns a write-through proxy for :term:`optimized scopes `. +* Python 3.13 removed many private functions. Some of them can be replaced using these + alternatives: + + * ``_PyDict_Pop()``: :c:func:`PyDict_Pop` or :c:func:`PyDict_PopString`; + * ``_PyDict_GetItemWithError()``: :c:func:`PyDict_GetItemRef`; + * ``_PyErr_WriteUnraisableMsg()``: :c:func:`PyErr_FormatUnraisable`; + * ``_PyEval_SetTrace()``: :c:func:`PyEval_SetTrace` or :c:func:`PyEval_SetTraceAllThreads`; + * ``_PyList_Extend()``: :c:func:`PyList_Extend`; + * ``_PyLong_AsInt()``: :c:func:`PyLong_AsInt`; + * ``_PyMem_RawStrdup()``: ``strdup()``; + * ``_PyMem_Strdup()``: ``strdup()``; + * ``_PyObject_ClearManagedDict()``: :c:func:`PyObject_ClearManagedDict`; + * ``_PyObject_VisitManagedDict()``: :c:func:`PyObject_VisitManagedDict`; + * ``_PyThreadState_UncheckedGet()``: :c:func:`PyThreadState_GetUnchecked()`; + * ``_PyTime_AsSecondsDouble()``: :c:func:`PyTime_AsSecondsDouble`; + * ``_PyTime_GetMonotonicClock()``: :c:func:`PyTime_Monotonic` or :c:func:`PyTime_MonotonicRaw`; + * ``_PyTime_GetPerfCounter()``: :c:func:`PyTime_PerfCounter` or :c:func:`PyTime_PerfCounterRaw`; + * ``_PyTime_GetSystemClock()``: :c:func:`PyTime_Time` or :c:func:`PyTime_TimeRaw`; + * ``_PyTime_MAX``: :c:var:`PyTime_MAX`; + * ``_PyTime_MIN``: :c:var:`PyTime_MIN`; + * ``_PyTime_t``: :c:type:`PyTime_t`; + * ``_Py_HashPointer()``: :c:func:`Py_HashPointer`; + * ``_Py_IsFinalizing()``: :c:func:`Py_IsFinalizing`. + + The `pythoncapi-compat project`_ can be used to get most of these new + functions on Python 3.12 and older. + Regression Test Changes ======================= From d786ac7f584f23c3206f4c86032bdabe83c17b51 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jan 2025 16:31:13 +0100 Subject: [PATCH 38/61] gh-59705: Document OS thread name change (#128800) --- Doc/library/threading.rst | 10 +++++++--- Doc/whatsnew/3.14.rst | 7 +++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index f183f3f535c4cb..00511df32e4388 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -380,6 +380,13 @@ since it is impossible to detect the termination of alien threads. This method will raise a :exc:`RuntimeError` if called more than once on the same thread object. + If supported, set the operating system thread name to + :attr:`threading.Thread.name`. The name can be truncated depending on the + operating system thread name limits. + + .. versionchanged:: 3.14 + Set the operating system thread name. + .. method:: run() Method representing the thread's activity. @@ -443,9 +450,6 @@ since it is impossible to detect the termination of alien threads. running thread is renamed. (Setting the *name* attribute of a different thread only updates the Python Thread object.) - .. versionchanged:: 3.14 - Set the operating system thread name. - .. method:: getName() setName() diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ac203b208f4a8a..9f7ef101e56478 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -645,6 +645,13 @@ sys.monitoring * Two new events are added: :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. +threading +--------- + +* :meth:`threading.Thread.start` now sets the operating system thread name + to :attr:`threading.Thread.name`. + (Contributed by Victor Stinner in :gh:`59705`.) + tkinter ------- From d906bde250d59c396d8dab92285b832c66cdec27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:07:37 +0100 Subject: [PATCH 39/61] gh-67206: Document that `string.printable` is not printable in the POSIX sense (#128820) --- Doc/library/string.rst | 13 ++++++++++--- .../2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst | 3 +++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 913672a3ff2270..09165c481b246e 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -59,11 +59,18 @@ The constants defined in this module are: String of ASCII characters which are considered punctuation characters in the ``C`` locale: ``!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~``. + .. data:: printable - String of ASCII characters which are considered printable. This is a - combination of :const:`digits`, :const:`ascii_letters`, :const:`punctuation`, - and :const:`whitespace`. + String of ASCII characters which are considered printable by Python. + This is a combination of :const:`digits`, :const:`ascii_letters`, + :const:`punctuation`, and :const:`whitespace`. + + .. note:: + + By design, :meth:`string.printable.isprintable() ` + returns :const:`False`. In particular, ``string.printable`` is not + printable in the POSIX sense (see :manpage:`LC_CTYPE `). .. data:: whitespace diff --git a/Misc/NEWS.d/next/Documentation/2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst b/Misc/NEWS.d/next/Documentation/2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst new file mode 100644 index 00000000000000..11fb617e98edc3 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst @@ -0,0 +1,3 @@ +Document that :const:`string.printable` is not printable in the POSIX sense. +In particular, :meth:`string.printable.isprintable() ` returns +:const:`False`. Patch by Bénédikt Tran. From 1c13c56a34fc4c4d8969f0b6dc93d5208a50d61b Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 14 Jan 2025 11:43:42 -0800 Subject: [PATCH 40/61] gh-128384: Add locking to warnings.py. (gh-128386) Co-authored-by: Kumar Aditya --- Include/internal/pycore_warnings.h | 2 +- Lib/test/test_warnings/__init__.py | 2 +- Lib/warnings.py | 188 ++++++++++-------- ...-01-06-10-37-27.gh-issue-128384.V0xzwH.rst | 5 + Python/_warnings.c | 118 +++++++---- Python/clinic/_warnings.c.h | 50 ++++- 6 files changed, 237 insertions(+), 128 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-06-10-37-27.gh-issue-128384.V0xzwH.rst diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index f9f6559312f4ef..672228cd6fbd19 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -14,7 +14,7 @@ struct _warnings_runtime_state { PyObject *filters; /* List */ PyObject *once_registry; /* Dict */ PyObject *default_action; /* String */ - PyMutex mutex; + _PyRecursiveMutex lock; long filters_version; }; diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 4e3c877896f295..4bd164b8a9a82b 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1521,7 +1521,7 @@ def test_late_resource_warning(self): self.assertTrue(err.startswith(expected), ascii(err)) -class DeprecatedTests(unittest.TestCase): +class DeprecatedTests(PyPublicAPITests): def test_dunder_deprecated(self): @deprecated("A will go away soon") class A: diff --git a/Lib/warnings.py b/Lib/warnings.py index e83cde37ab2d1a..f20b01372dd7a4 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -185,24 +185,32 @@ def simplefilter(action, category=Warning, lineno=0, append=False): raise ValueError("lineno must be an int >= 0") _add_filter(action, None, category, None, lineno, append=append) +def _filters_mutated(): + # Even though this function is not part of the public API, it's used by + # a fair amount of user code. + with _lock: + _filters_mutated_lock_held() + def _add_filter(*item, append): - # Remove possible duplicate filters, so new one will be placed - # in correct place. If append=True and duplicate exists, do nothing. - if not append: - try: - filters.remove(item) - except ValueError: - pass - filters.insert(0, item) - else: - if item not in filters: - filters.append(item) - _filters_mutated() + with _lock: + if not append: + # Remove possible duplicate filters, so new one will be placed + # in correct place. If append=True and duplicate exists, do nothing. + try: + filters.remove(item) + except ValueError: + pass + filters.insert(0, item) + else: + if item not in filters: + filters.append(item) + _filters_mutated_lock_held() def resetwarnings(): """Clear the list of warning filters, so that no filters are active.""" - filters[:] = [] - _filters_mutated() + with _lock: + filters[:] = [] + _filters_mutated_lock_held() class _OptionError(Exception): """Exception used by option processing helpers.""" @@ -353,11 +361,6 @@ def warn_explicit(message, category, filename, lineno, module = filename or "" if module[-3:].lower() == ".py": module = module[:-3] # XXX What about leading pathname? - if registry is None: - registry = {} - if registry.get('version', 0) != _filters_version: - registry.clear() - registry['version'] = _filters_version if isinstance(message, Warning): text = str(message) category = message.__class__ @@ -365,52 +368,59 @@ def warn_explicit(message, category, filename, lineno, text = message message = category(message) key = (text, category, lineno) - # Quick test for common case - if registry.get(key): - return - # Search the filters - for item in filters: - action, msg, cat, mod, ln = item - if ((msg is None or msg.match(text)) and - issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break - else: - action = defaultaction - # Early exit actions - if action == "ignore": - return + with _lock: + if registry is None: + registry = {} + if registry.get('version', 0) != _filters_version: + registry.clear() + registry['version'] = _filters_version + # Quick test for common case + if registry.get(key): + return + # Search the filters + for item in filters: + action, msg, cat, mod, ln = item + if ((msg is None or msg.match(text)) and + issubclass(category, cat) and + (mod is None or mod.match(module)) and + (ln == 0 or lineno == ln)): + break + else: + action = defaultaction + # Early exit actions + if action == "ignore": + return + + if action == "error": + raise message + # Other actions + if action == "once": + registry[key] = 1 + oncekey = (text, category) + if onceregistry.get(oncekey): + return + onceregistry[oncekey] = 1 + elif action in {"always", "all"}: + pass + elif action == "module": + registry[key] = 1 + altkey = (text, category, 0) + if registry.get(altkey): + return + registry[altkey] = 1 + elif action == "default": + registry[key] = 1 + else: + # Unrecognized actions are errors + raise RuntimeError( + "Unrecognized action (%r) in warnings.filters:\n %s" % + (action, item)) # Prime the linecache for formatting, in case the # "file" is actually in a zipfile or something. import linecache linecache.getlines(filename, module_globals) - if action == "error": - raise message - # Other actions - if action == "once": - registry[key] = 1 - oncekey = (text, category) - if onceregistry.get(oncekey): - return - onceregistry[oncekey] = 1 - elif action in {"always", "all"}: - pass - elif action == "module": - registry[key] = 1 - altkey = (text, category, 0) - if registry.get(altkey): - return - registry[altkey] = 1 - elif action == "default": - registry[key] = 1 - else: - # Unrecognized actions are errors - raise RuntimeError( - "Unrecognized action (%r) in warnings.filters:\n %s" % - (action, item)) # Print message and context msg = WarningMessage(message, category, filename, lineno, source) _showwarnmsg(msg) @@ -488,30 +498,32 @@ def __enter__(self): if self._entered: raise RuntimeError("Cannot enter %r twice" % self) self._entered = True - self._filters = self._module.filters - self._module.filters = self._filters[:] - self._module._filters_mutated() - self._showwarning = self._module.showwarning - self._showwarnmsg_impl = self._module._showwarnmsg_impl + with _lock: + self._filters = self._module.filters + self._module.filters = self._filters[:] + self._module._filters_mutated_lock_held() + self._showwarning = self._module.showwarning + self._showwarnmsg_impl = self._module._showwarnmsg_impl + if self._record: + log = [] + self._module._showwarnmsg_impl = log.append + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig + else: + log = None if self._filter is not None: simplefilter(*self._filter) - if self._record: - log = [] - self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig - return log - else: - return None + return log def __exit__(self, *exc_info): if not self._entered: raise RuntimeError("Cannot exit %r without entering first" % self) - self._module.filters = self._filters - self._module._filters_mutated() - self._module.showwarning = self._showwarning - self._module._showwarnmsg_impl = self._showwarnmsg_impl + with _lock: + self._module.filters = self._filters + self._module._filters_mutated_lock_held() + self._module.showwarning = self._showwarning + self._module._showwarnmsg_impl = self._showwarnmsg_impl class deprecated: @@ -701,18 +713,36 @@ def extract(): # If either if the compiled regexs are None, match anything. try: from _warnings import (filters, _defaultaction, _onceregistry, - warn, warn_explicit, _filters_mutated) + warn, warn_explicit, + _filters_mutated_lock_held, + _acquire_lock, _release_lock, + ) defaultaction = _defaultaction onceregistry = _onceregistry _warnings_defaults = True + + class _Lock: + def __enter__(self): + _acquire_lock() + return self + + def __exit__(self, *args): + _release_lock() + + _lock = _Lock() + except ImportError: filters = [] defaultaction = "default" onceregistry = {} + import _thread + + _lock = _thread.RLock() + _filters_version = 1 - def _filters_mutated(): + def _filters_mutated_lock_held(): global _filters_version _filters_version += 1 diff --git a/Misc/NEWS.d/next/Library/2025-01-06-10-37-27.gh-issue-128384.V0xzwH.rst b/Misc/NEWS.d/next/Library/2025-01-06-10-37-27.gh-issue-128384.V0xzwH.rst new file mode 100644 index 00000000000000..2ca592be20b681 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-06-10-37-27.gh-issue-128384.V0xzwH.rst @@ -0,0 +1,5 @@ +Add locking to :mod:`warnings` to avoid some data races when free-threading +is used. Change ``_warnings_runtime_state.mutex`` to be a recursive mutex +and expose it to :mod:`warnings`, via the :func:`!_acquire_lock` and +:func:`!_release_lock` functions. The lock is held when ``filters`` and +``_filters_version`` are updated. diff --git a/Python/_warnings.c b/Python/_warnings.c index e05ba99e8eaec4..283f203c72c9bf 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -232,6 +232,61 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import) return obj; } +static inline void +warnings_lock(PyInterpreterState *interp) +{ + WarningsState *st = warnings_get_state(interp); + assert(st != NULL); + _PyRecursiveMutex_Lock(&st->lock); +} + +static inline void +warnings_unlock(PyInterpreterState *interp) +{ + WarningsState *st = warnings_get_state(interp); + assert(st != NULL); + _PyRecursiveMutex_Unlock(&st->lock); +} + +static inline bool +warnings_lock_held(WarningsState *st) +{ + return PyMutex_IsLocked(&st->lock.mutex); +} + +/*[clinic input] +_acquire_lock as warnings_acquire_lock + +[clinic start generated code]*/ + +static PyObject * +warnings_acquire_lock_impl(PyObject *module) +/*[clinic end generated code: output=594313457d1bf8e1 input=46ec20e55acca52f]*/ +{ + PyInterpreterState *interp = get_current_interp(); + if (interp == NULL) { + return NULL; + } + warnings_lock(interp); + Py_RETURN_NONE; +} + +/*[clinic input] +_release_lock as warnings_release_lock + +[clinic start generated code]*/ + +static PyObject * +warnings_release_lock_impl(PyObject *module) +/*[clinic end generated code: output=d73d5a8789396750 input=ea01bb77870c5693]*/ +{ + PyInterpreterState *interp = get_current_interp(); + if (interp == NULL) { + return NULL; + } + warnings_unlock(interp); + Py_RETURN_NONE; +} static PyObject * get_once_registry(PyInterpreterState *interp) @@ -239,7 +294,7 @@ get_once_registry(PyInterpreterState *interp) WarningsState *st = warnings_get_state(interp); assert(st != NULL); - _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); + assert(warnings_lock_held(st)); PyObject *registry = GET_WARNINGS_ATTR(interp, onceregistry, 0); if (registry == NULL) { @@ -267,7 +322,7 @@ get_default_action(PyInterpreterState *interp) WarningsState *st = warnings_get_state(interp); assert(st != NULL); - _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); + assert(warnings_lock_held(st)); PyObject *default_action = GET_WARNINGS_ATTR(interp, defaultaction, 0); if (default_action == NULL) { @@ -299,7 +354,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, WarningsState *st = warnings_get_state(interp); assert(st != NULL); - _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); + assert(warnings_lock_held(st)); PyObject *warnings_filters = GET_WARNINGS_ATTR(interp, filters, 0); if (warnings_filters == NULL) { @@ -399,7 +454,7 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, WarningsState *st = warnings_get_state(interp); assert(st != NULL); - _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); + assert(warnings_lock_held(st)); PyObject *version_obj; if (PyDict_GetItemRef(registry, &_Py_ID(version), &version_obj) < 0) { @@ -994,15 +1049,10 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, &filename, &lineno, &module, ®istry)) return NULL; -#ifdef Py_GIL_DISABLED - WarningsState *st = warnings_get_state(tstate->interp); - assert(st != NULL); -#endif - - Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); + warnings_lock(tstate->interp); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, source); - Py_END_CRITICAL_SECTION(); + warnings_unlock(tstate->interp); Py_DECREF(filename); Py_DECREF(registry); Py_DECREF(module); @@ -1151,27 +1201,22 @@ warnings_warn_explicit_impl(PyObject *module, PyObject *message, } } -#ifdef Py_GIL_DISABLED - WarningsState *st = warnings_get_state(tstate->interp); - assert(st != NULL); -#endif - - Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); + warnings_lock(tstate->interp); returned = warn_explicit(tstate, category, message, filename, lineno, mod, registry, source_line, sourceobj); - Py_END_CRITICAL_SECTION(); + warnings_unlock(tstate->interp); Py_XDECREF(source_line); return returned; } /*[clinic input] -_filters_mutated as warnings_filters_mutated +_filters_mutated_lock_held as warnings_filters_mutated_lock_held [clinic start generated code]*/ static PyObject * -warnings_filters_mutated_impl(PyObject *module) -/*[clinic end generated code: output=8ce517abd12b88f4 input=35ecbf08ee2491b2]*/ +warnings_filters_mutated_lock_held_impl(PyObject *module) +/*[clinic end generated code: output=df5c84f044e856ec input=34208bf03d70e432]*/ { PyInterpreterState *interp = get_current_interp(); if (interp == NULL) { @@ -1181,14 +1226,17 @@ warnings_filters_mutated_impl(PyObject *module) WarningsState *st = warnings_get_state(interp); assert(st != NULL); - Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); + // Note that the lock must be held by the caller. + if (!warnings_lock_held(st)) { + PyErr_SetString(PyExc_RuntimeError, "warnings lock is not held"); + return NULL; + } + st->filters_version++; - Py_END_CRITICAL_SECTION(); Py_RETURN_NONE; } - /* Function to issue a warning message; may raise an exception. */ static int @@ -1303,15 +1351,10 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, return -1; } -#ifdef Py_GIL_DISABLED - WarningsState *st = warnings_get_state(tstate->interp); - assert(st != NULL); -#endif - - Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); + warnings_lock(tstate->interp); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, NULL); - Py_END_CRITICAL_SECTION(); + warnings_unlock(tstate->interp); if (res == NULL) return -1; Py_DECREF(res); @@ -1376,15 +1419,10 @@ PyErr_WarnExplicitFormat(PyObject *category, PyObject *res; PyThreadState *tstate = get_current_tstate(); if (tstate != NULL) { -#ifdef Py_GIL_DISABLED - WarningsState *st = warnings_get_state(tstate->interp); - assert(st != NULL); -#endif - - Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); + warnings_lock(tstate->interp); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, NULL); - Py_END_CRITICAL_SECTION(); + warnings_unlock(tstate->interp); Py_DECREF(message); if (res != NULL) { Py_DECREF(res); @@ -1464,7 +1502,9 @@ _PyErr_WarnUnawaitedCoroutine(PyObject *coro) static PyMethodDef warnings_functions[] = { WARNINGS_WARN_METHODDEF WARNINGS_WARN_EXPLICIT_METHODDEF - WARNINGS_FILTERS_MUTATED_METHODDEF + WARNINGS_FILTERS_MUTATED_LOCK_HELD_METHODDEF + WARNINGS_ACQUIRE_LOCK_METHODDEF + WARNINGS_RELEASE_LOCK_METHODDEF /* XXX(brett.cannon): add showwarning? */ /* XXX(brett.cannon): Reasonable to add formatwarning? */ {NULL, NULL} /* sentinel */ diff --git a/Python/clinic/_warnings.c.h b/Python/clinic/_warnings.c.h index 9a2c33f2ea8169..bcb4b344fa4370 100644 --- a/Python/clinic/_warnings.c.h +++ b/Python/clinic/_warnings.c.h @@ -9,6 +9,40 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(warnings_acquire_lock__doc__, +"_acquire_lock($module, /)\n" +"--\n" +"\n"); + +#define WARNINGS_ACQUIRE_LOCK_METHODDEF \ + {"_acquire_lock", (PyCFunction)warnings_acquire_lock, METH_NOARGS, warnings_acquire_lock__doc__}, + +static PyObject * +warnings_acquire_lock_impl(PyObject *module); + +static PyObject * +warnings_acquire_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return warnings_acquire_lock_impl(module); +} + +PyDoc_STRVAR(warnings_release_lock__doc__, +"_release_lock($module, /)\n" +"--\n" +"\n"); + +#define WARNINGS_RELEASE_LOCK_METHODDEF \ + {"_release_lock", (PyCFunction)warnings_release_lock, METH_NOARGS, warnings_release_lock__doc__}, + +static PyObject * +warnings_release_lock_impl(PyObject *module); + +static PyObject * +warnings_release_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return warnings_release_lock_impl(module); +} + PyDoc_STRVAR(warnings_warn__doc__, "warn($module, /, message, category=None, stacklevel=1, source=None, *,\n" " skip_file_prefixes=)\n" @@ -230,20 +264,20 @@ warnings_warn_explicit(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } -PyDoc_STRVAR(warnings_filters_mutated__doc__, -"_filters_mutated($module, /)\n" +PyDoc_STRVAR(warnings_filters_mutated_lock_held__doc__, +"_filters_mutated_lock_held($module, /)\n" "--\n" "\n"); -#define WARNINGS_FILTERS_MUTATED_METHODDEF \ - {"_filters_mutated", (PyCFunction)warnings_filters_mutated, METH_NOARGS, warnings_filters_mutated__doc__}, +#define WARNINGS_FILTERS_MUTATED_LOCK_HELD_METHODDEF \ + {"_filters_mutated_lock_held", (PyCFunction)warnings_filters_mutated_lock_held, METH_NOARGS, warnings_filters_mutated_lock_held__doc__}, static PyObject * -warnings_filters_mutated_impl(PyObject *module); +warnings_filters_mutated_lock_held_impl(PyObject *module); static PyObject * -warnings_filters_mutated(PyObject *module, PyObject *Py_UNUSED(ignored)) +warnings_filters_mutated_lock_held(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return warnings_filters_mutated_impl(module); + return warnings_filters_mutated_lock_held_impl(module); } -/*[clinic end generated code: output=ed02c0f521a03a37 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d9d32a8b59a30683 input=a9049054013a1b77]*/ From b5ee0258bf5bb60a5a5a65c64717853e06b64808 Mon Sep 17 00:00:00 2001 From: mpage Date: Tue, 14 Jan 2025 11:56:11 -0800 Subject: [PATCH 41/61] gh-115999: Specialize `LOAD_ATTR` for instance and class receivers in free-threaded builds (#128164) Finish specialization for LOAD_ATTR in the free-threaded build by adding support for class and instance receivers. --- Include/cpython/pystats.h | 2 +- Include/internal/pycore_dict.h | 10 + Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Lib/test/test_capi/test_type.py | 5 +- Lib/test/test_descr.py | 32 ++- Lib/test/test_generated_cases.py | 74 ++++--- Lib/test/test_opcache.py | 90 +++++++- Objects/dictobject.c | 67 ++++-- Python/bytecodes.c | 85 +++++--- Python/executor_cases.c.h | 141 +++++++++---- Python/generated_cases.c.h | 81 +++++--- Python/optimizer_bytecodes.c | 8 +- Python/optimizer_cases.c.h | 19 +- Python/specialize.c | 229 +++++++++++++-------- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/generators_common.py | 15 +- Tools/cases_generator/stack.py | 4 - 18 files changed, 608 insertions(+), 260 deletions(-) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 29ef0c0e4d4e72..ee8885cda7b60d 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -31,7 +31,7 @@ #define PYSTATS_MAX_UOP_ID 512 -#define SPECIALIZATION_FAILURE_KINDS 36 +#define SPECIALIZATION_FAILURE_KINDS 37 /* Stats for determining who is calling PyEval_EvalFrame */ #define EVAL_CALL_TOTAL 0 diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 71927006d1cd48..74ac8f2148174c 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -114,6 +114,16 @@ extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); + +/* Look up a string key in an all unicode dict keys, assign the keys object a version, and + * store it in version. + * + * Returns DKIX_ERROR if key is not a string or if the keys object is not all + * strings. + * + * Returns DKIX_EMPTY if the key is not present. + */ +extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5365e2a5c6b8cd..0c0a6145bdbb27 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1540,7 +1540,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { return 0; } case LOAD_ATTR_WITH_HINT: { - *effect = Py_MAX(0, (oparg & 1)); + *effect = Py_MAX(1, (oparg & 1)); return 0; } case LOAD_BUILD_CLASS: { diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 721fa94da19615..5670fe26f72071 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -862,7 +862,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_ATTR_WITH_HINT: return 0; case _LOAD_ATTR_WITH_HINT: - return 1; + return 2; case _LOAD_ATTR_SLOT_0: return 1; case _LOAD_ATTR_SLOT_1: diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py index 54c83e09f892a0..92d056e802eeed 100644 --- a/Lib/test/test_capi/test_type.py +++ b/Lib/test/test_capi/test_type.py @@ -1,4 +1,4 @@ -from test.support import import_helper +from test.support import import_helper, Py_GIL_DISABLED, refleak_helper import unittest _testcapi = import_helper.import_module('_testcapi') @@ -37,6 +37,9 @@ class D(A, C): pass # as well type_freeze(D) + @unittest.skipIf( + Py_GIL_DISABLED and refleak_helper.hunting_for_refleaks(), + "Specialization failure triggers gh-127773") def test_freeze_meta(self): """test PyType_Freeze() with overridden MRO""" type_freeze = _testcapi.type_freeze diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 51f97bb51f7bd2..a7ebc9e8be0294 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -7,6 +7,7 @@ import random import string import sys +import textwrap import types import unittest import warnings @@ -15,6 +16,7 @@ from copy import deepcopy from contextlib import redirect_stdout from test import support +from test.support.script_helper import assert_python_ok try: import _testcapi @@ -5222,6 +5224,7 @@ def test_type_lookup_mro_reference(self): # Issue #14199: _PyType_Lookup() has to keep a strong reference to # the type MRO because it may be modified during the lookup, if # __bases__ is set during the lookup for example. + code = textwrap.dedent(""" class MyKey(object): def __hash__(self): return hash('mykey') @@ -5237,12 +5240,29 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - with self.assertWarnsRegex(RuntimeWarning, 'X'): - X = type('X', (Base,), {MyKey(): 5}) - # mykey is read from Base - self.assertEqual(X.mykey, 'from Base') - # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ - self.assertEqual(X.mykey2, 'from Base2') + X = type('X', (Base,), {MyKey(): 5}) + + bases_before = ",".join([c.__name__ for c in X.__bases__]) + print(f"before={bases_before}") + + # mykey is initially read from Base, however, the lookup will be perfomed + # again if specialization fails. The second lookup will use the new + # mro set by __eq__. + print(X.mykey) + + bases_after = ",".join([c.__name__ for c in X.__bases__]) + print(f"after={bases_after}") + + # mykey2 is read from Base2 because MyKey.__eq__ has set __bases_ + print(f"mykey2={X.mykey2}") + """) + _, out, err = assert_python_ok("-c", code) + err = err.decode() + self.assertRegex(err, "RuntimeWarning: .*X") + out = out.decode() + self.assertRegex(out, "before=Base") + self.assertRegex(out, "after=Base2") + self.assertRegex(out, "mykey2=from Base2") class PicklingTests(unittest.TestCase): diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 4a1c99edacb71a..7a50a29bb0126c 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1639,12 +1639,16 @@ def test_escaping_call_next_to_cmacro(self): """ self.run_cases_test(input, output) - def test_pop_dead_inputs_all_live(self): + def test_pystackref_frompyobject_new_next_to_cmacro(self): input = """ - inst(OP, (a, b --)) { - POP_DEAD_INPUTS(); - HAM(a, b); - INPUTS_DEAD(); + inst(OP, (-- out1, out2)) { + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); } """ output = """ @@ -1652,22 +1656,28 @@ def test_pop_dead_inputs_all_live(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef a; - _PyStackRef b; - b = stack_pointer[-1]; - a = stack_pointer[-2]; - HAM(a, b); - stack_pointer += -2; + _PyStackRef out1; + _PyStackRef out2; + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = out1; + stack_pointer[1] = out2; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ self.run_cases_test(input, output) - def test_pop_dead_inputs_some_live(self): + def test_pop_input(self): input = """ - inst(OP, (a, b, c --)) { - POP_DEAD_INPUTS(); + inst(OP, (a, b --)) { + POP_INPUT(b); HAM(a); INPUTS_DEAD(); } @@ -1678,8 +1688,10 @@ def test_pop_dead_inputs_some_live(self): next_instr += 1; INSTRUCTION_STATS(OP); _PyStackRef a; - a = stack_pointer[-3]; - stack_pointer += -2; + _PyStackRef b; + b = stack_pointer[-1]; + a = stack_pointer[-2]; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); HAM(a); stack_pointer += -1; @@ -1689,29 +1701,23 @@ def test_pop_dead_inputs_some_live(self): """ self.run_cases_test(input, output) - def test_pop_dead_inputs_with_output(self): + def test_pop_input_with_empty_stack(self): input = """ - inst(OP, (a, b -- c)) { - POP_DEAD_INPUTS(); - c = SPAM(); + inst(OP, (--)) { + POP_INPUT(foo); } """ - output = """ - TARGET(OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef c; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - c = SPAM(); - stack_pointer[0] = c; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_pop_input_with_non_tos(self): + input = """ + inst(OP, (a, b --)) { + POP_INPUT(a); } """ - self.run_cases_test(input, output) + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") def test_no_escaping_calls_in_branching_macros(self): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index c7cd4c2e8a3146..b80ccbf17f1ee6 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -564,6 +564,16 @@ def instantiate(): instantiate() +def make_deferred_ref_count_obj(): + """Create an object that uses deferred reference counting. + + Only objects that use deferred refence counting may be stored in inline + caches in free-threaded builds. This constructs a new class named Foo, + which uses deferred reference counting. + """ + return type("Foo", (object,), {}) + + @threading_helper.requires_working_threading() class TestRacesDoNotCrash(TestBase): # Careful with these. Bigger numbers have a higher chance of catching bugs, @@ -714,11 +724,11 @@ def write(items): opname = "FOR_ITER_LIST" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_class(self): def get_items(): class C: - a = object() + a = make_deferred_ref_count_obj() items = [] for _ in range(self.ITEMS): @@ -739,12 +749,45 @@ def write(items): del item.a except AttributeError: pass - item.a = object() + item.a = make_deferred_ref_count_obj() opname = "LOAD_ATTR_CLASS" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft + def test_load_attr_class_with_metaclass_check(self): + def get_items(): + class Meta(type): + pass + + class C(metaclass=Meta): + a = make_deferred_ref_count_obj() + + items = [] + for _ in range(self.ITEMS): + item = C + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.a + except AttributeError: + pass + item.a = make_deferred_ref_count_obj() + + opname = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_getattribute_overridden(self): def get_items(): class C: @@ -774,7 +817,7 @@ def write(items): opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_instance_value(self): def get_items(): class C: @@ -798,7 +841,7 @@ def write(items): opname = "LOAD_ATTR_INSTANCE_VALUE" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_method_lazy_dict(self): def get_items(): class C(Exception): @@ -828,7 +871,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_LAZY_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_method_no_dict(self): def get_items(): class C: @@ -859,7 +902,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_NO_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_method_with_values(self): def get_items(): class C: @@ -914,7 +957,7 @@ def write(items): opname = "LOAD_ATTR_MODULE" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_property(self): def get_items(): class C: @@ -944,7 +987,34 @@ def write(items): opname = "LOAD_ATTR_PROPERTY" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft + def test_load_attr_slot(self): + def get_items(): + class C: + __slots__ = ["a", "b"] + + items = [] + for i in range(self.ITEMS): + item = C() + item.a = i + item.b = i + self.ITEMS + items.append(item) + return items + + def read(items): + for item in items: + item.a + item.b + + def write(items): + for item in items: + item.a = 100 + item.b = 200 + + opname = "LOAD_ATTR_SLOT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_with_hint(self): def get_items(): class C: diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 2a054c3f2ae0ff..82789d5e56f523 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1129,6 +1129,24 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P return do_lookup(mp, dk, key, hash, compare_generic); } +static bool +check_keys_unicode(PyDictKeysObject *dk, PyObject *key) +{ + return PyUnicode_CheckExact(key) && (dk->dk_kind != DICT_KEYS_GENERAL); +} + +static Py_ssize_t +hash_unicode_key(PyObject *key) +{ + assert(PyUnicode_CheckExact(key)); + Py_hash_t hash = unicode_get_hash(key); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(key); + assert(hash != -1); + } + return hash; +} + #ifdef Py_GIL_DISABLED static Py_ssize_t unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, @@ -1167,21 +1185,28 @@ unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) { - DictKeysKind kind = dk->dk_kind; - if (!PyUnicode_CheckExact(key) || kind == DICT_KEYS_GENERAL) { + if (!check_keys_unicode(dk, key)) { return DKIX_ERROR; } - Py_hash_t hash = unicode_get_hash(key); - if (hash == -1) { - hash = PyUnicode_Type.tp_hash(key); - if (hash == -1) { - PyErr_Clear(); - return DKIX_ERROR; - } - } + Py_hash_t hash = hash_unicode_key(key); return unicodekeys_lookup_unicode(dk, key, hash); } +Py_ssize_t +_PyDictKeys_StringLookupAndVersion(PyDictKeysObject *dk, PyObject *key, uint32_t *version) +{ + if (!check_keys_unicode(dk, key)) { + return DKIX_ERROR; + } + Py_ssize_t ix; + Py_hash_t hash = hash_unicode_key(key); + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + *version = _PyDictKeys_GetVersionForCurrentState(_PyInterpreterState_GET(), dk); + UNLOCK_KEYS(dk); + return ix; +} + /* Like _PyDictKeys_StringLookup() but only works on split keys. Note * that in free-threaded builds this locks the keys object as required. */ @@ -1926,6 +1951,16 @@ build_indices_unicode(PyDictKeysObject *keys, PyDictUnicodeEntry *ep, Py_ssize_t } } +static void +invalidate_and_clear_inline_values(PyDictValues *values) +{ + assert(values->embedded); + FT_ATOMIC_STORE_UINT8(values->valid, 0); + for (int i = 0; i < values->capacity; i++) { + FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL); + } +} + /* Restructure the table by allocating a new table and reinserting all items again. When entries have been deleted, the new table may @@ -2017,7 +2052,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, if (oldvalues->embedded) { assert(oldvalues->embedded == 1); assert(oldvalues->valid == 1); - FT_ATOMIC_STORE_UINT8(oldvalues->valid, 0); + invalidate_and_clear_inline_values(oldvalues); } else { free_values(oldvalues, IS_DICT_SHARED(mp)); @@ -7007,7 +7042,13 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr #ifdef Py_GIL_DISABLED PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]); - if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) { + if (value == NULL) { + if (FT_ATOMIC_LOAD_UINT8(values->valid)) { + *attr = NULL; + return true; + } + } + else if (_Py_TryIncrefCompare(&values->values[ix], value)) { *attr = value; return true; } @@ -7345,7 +7386,7 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) } mp->ma_values = values; - FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0); + invalidate_and_clear_inline_values(_PyObject_InlineValues(obj)); assert(_PyObject_InlineValuesConsistencyCheck(obj)); ASSERT_CONSISTENT(mp); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cec530fefffefb..a906ded365650c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2190,18 +2190,23 @@ dummy_func( PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)); } split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); DEOPT_IF(attr_o == NULL); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); DECREF_INPUTS(); } @@ -2227,9 +2232,8 @@ dummy_func( assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - DEAD(mod_keys); // Clear mod_keys from stack in case we need to deopt - POP_DEAD_INPUTS(); + POP_INPUT(mod_keys); DEOPT_IF(attr_o == NULL); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -2251,30 +2255,50 @@ dummy_func( _LOAD_ATTR_MODULE_FROM_KEYS + unused/5; - op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { + op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - EXIT_IF(dict == NULL); - assert(PyDict_CheckExact((PyObject *)dict)); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + EXIT_IF(dict_o == NULL); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) { PyObject *attr_o; + if (!LOCK_OBJECT(dict)) { + POP_INPUT(dict); + DEOPT_IF(true); + } - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL); + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); + DEAD(dict); null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2289,12 +2313,17 @@ dummy_func( split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); DEOPT_IF(attr_o == NULL); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); DECREF_INPUTS(); } @@ -2309,7 +2338,7 @@ dummy_func( EXIT_IF(!PyType_Check(owner_o)); assert(type_version != 0); - EXIT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version); + EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version); } split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { @@ -2363,7 +2392,7 @@ dummy_func( DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -3281,13 +3310,15 @@ dummy_func( op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid)); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version); } split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { @@ -3357,7 +3388,7 @@ dummy_func( op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1aa80f398d7470..cda01bb768c269 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2652,7 +2652,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner_o)->valid) { + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2668,15 +2668,23 @@ uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; @@ -2691,15 +2699,23 @@ uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; @@ -2778,55 +2794,88 @@ case _CHECK_ATTR_WITH_HINT: { _PyStackRef owner; + PyDictObject *dict; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + if (dict_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict)); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; + stack_pointer[0].bits = (uintptr_t)dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { + PyDictObject *dict; _PyStackRef owner; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + dict = (PyDictObject *)stack_pointer[-1].bits; + owner = stack_pointer[-2]; uint16_t hint = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (!LOCK_OBJECT(dict)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } if (hint >= (size_t)dict->ma_keys->dk_nentries) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (!DK_IS_UNICODE(dict->ma_keys)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } attr_o = ep->me_value; if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2839,15 +2888,23 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; @@ -2861,15 +2918,23 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; @@ -2890,7 +2955,7 @@ JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); - if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3924,7 +3989,8 @@ owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner_o)->valid) { + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3937,7 +4003,8 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4022,7 +4089,7 @@ owner = stack_pointer[-1]; uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ if (dict != NULL) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 810beb61d0db5e..81408380d6b2b8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5345,7 +5345,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5380,7 +5380,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5421,7 +5421,7 @@ DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -5463,19 +5463,24 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -5507,7 +5512,7 @@ { uint16_t dictoffset = read_u16(&this_instr[4].cache); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } @@ -5586,14 +5591,16 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5716,14 +5723,16 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5824,12 +5833,17 @@ { uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed, LOAD_ATTR); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -5846,6 +5860,7 @@ INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; + PyDictObject *dict; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ @@ -5861,26 +5876,40 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict_o == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + if (!LOCK_OBJECT(dict)) { + DEOPT_IF(true, LOAD_ATTR); + } + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 788adecca8af80..4d96ada5acf00f 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -582,11 +582,17 @@ dummy_func(void) { } } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { + op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) { + dict = sym_new_not_null(ctx); + (void)owner; + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; (void)owner; + (void)dict; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1a7cc6becfefb6..aff4493fdc4dd7 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1250,22 +1250,33 @@ } case _CHECK_ATTR_WITH_HINT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *dict; + owner = stack_pointer[-1]; + dict = sym_new_not_null(ctx); + (void)owner; + stack_pointer[0] = dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { + _Py_UopsSymbol *dict; _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; - owner = stack_pointer[-1]; + dict = stack_pointer[-1]; + owner = stack_pointer[-2]; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; (void)owner; - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + (void)dict; + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/specialize.c b/Python/specialize.c index 897005c4f1078d..8d9f19c8895187 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3,6 +3,7 @@ #include "opcode.h" #include "pycore_code.h" +#include "pycore_critical_section.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // DICT_KEYS_UNICODE #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() @@ -537,6 +538,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33 #define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34 #define SPEC_FAIL_ATTR_SPLIT_DICT 35 +#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36 /* Binary subscr and store subscr */ @@ -729,11 +731,8 @@ unspecialize(_Py_CODEUNIT *instr) } static int function_kind(PyCodeObject *code); -#ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -static uint32_t type_get_version(PyTypeObject *t, int opcode); -#endif static int specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) @@ -879,10 +878,11 @@ descriptor_is_class(PyObject *descriptor, PyObject *name) (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); } -#ifndef Py_GIL_DISABLED static DescriptorClassification -analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) { bool has_getattr = false; + bool have_ga_version = false; + unsigned int ga_version; getattrofunc getattro_slot = type->tp_getattro; if (getattro_slot == PyObject_GenericGetAttr) { /* Normal attribute lookup; */ @@ -892,24 +892,27 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { getattro_slot == _Py_slot_tp_getattro) { /* One or both of __getattribute__ or __getattr__ may have been overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__)); + PyObject *getattribute = _PyType_LookupRefAndVersion(type, + &_Py_ID(__getattribute__), &ga_version); + have_ga_version = true; PyInterpreterState *interp = _PyInterpreterState_GET(); bool has_custom_getattribute = getattribute != NULL && getattribute != interp->callable_cache.object__getattribute__; - PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__)); + PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); has_getattr = getattr != NULL; - Py_XDECREF(getattr); if (has_custom_getattribute) { if (getattro_slot == _Py_slot_tp_getattro && !has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; + *tp_version = ga_version; return GETATTRIBUTE_IS_PYTHON_FUNCTION; } /* Potentially both __getattr__ and __getattribute__ are set. Too complicated */ Py_DECREF(getattribute); *descr = NULL; + *tp_version = ga_version; return GETSET_OVERRIDDEN; } /* Potentially has __getattr__ but no custom __getattribute__. @@ -923,16 +926,18 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { } else { *descr = NULL; + *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag); return GETSET_OVERRIDDEN; } - PyObject *descriptor = _PyType_LookupRef(type, name); + unsigned int descr_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version); *descr = descriptor; + *tp_version = have_ga_version ? ga_version : descr_version; if (descriptor_is_class(descriptor, name)) { return DUNDER_CLASS; } return classify_descriptor(descriptor, has_getattr); } -#endif //!Py_GIL_DISABLED static DescriptorClassification analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) @@ -952,12 +957,13 @@ analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, u static int specialize_dict_access_inline( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, unsigned int tp_version, + PyObject *name, unsigned int tp_version, int base_op, int values_op) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner); Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); assert (index != DKIX_ERROR); if (index == DKIX_EMPTY) { @@ -965,6 +971,7 @@ specialize_dict_access_inline( return 0; } assert(index >= 0); + assert(_PyObject_InlineValues(owner)->valid); char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; Py_ssize_t offset = value_addr - (char *)owner; if (offset != (uint16_t)offset) { @@ -980,10 +987,13 @@ specialize_dict_access_inline( static int specialize_dict_access_hint( PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, unsigned int tp_version, + PyObject *name, unsigned int tp_version, int base_op, int hint_op) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + // We found an instance with a __dict__. if (_PyDict_HasSplitTable(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); @@ -1027,7 +1037,7 @@ specialize_dict_access( PyDictObject *dict = _PyObject_GetManagedDict(owner); if (dict == NULL) { // managed dict, not materialized, inline values valid - res = specialize_dict_access_inline(owner, instr, type, kind, name, + res = specialize_dict_access_inline(owner, instr, type, name, tp_version, base_op, values_op); } else { @@ -1047,16 +1057,19 @@ specialize_dict_access( int res; Py_BEGIN_CRITICAL_SECTION(dict); // materialized managed dict - res = specialize_dict_access_hint(dict, instr, type, kind, name, + res = specialize_dict_access_hint(dict, instr, type, name, tp_version, base_op, hint_op); Py_END_CRITICAL_SECTION(); return res; } } -#ifndef Py_GIL_DISABLED -static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, - PyObject* descr, DescriptorClassification kind, bool is_method); +static int +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version); static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); /* Returns true if instances of obj's class are @@ -1065,7 +1078,7 @@ static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyOb * For other objects, we check their actual dictionary. */ static bool -instance_has_key(PyObject *obj, PyObject* name) +instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version) { PyTypeObject *cls = Py_TYPE(obj); if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { @@ -1073,36 +1086,38 @@ instance_has_key(PyObject *obj, PyObject* name) } if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys; - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); + Py_ssize_t index = + _PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version); return index >= 0; } PyDictObject *dict = _PyObject_GetManagedDict(obj); if (dict == NULL || !PyDict_CheckExact(dict)) { return false; } + bool result; + Py_BEGIN_CRITICAL_SECTION(dict); if (dict->ma_values) { - return false; + result = false; } - Py_ssize_t index = _PyDict_LookupIndex(dict, name); - if (index < 0) { - return false; + else { + result = (_PyDict_LookupIndex(dict, name) >= 0); } - return true; + Py_END_CRITICAL_SECTION(); + return result; } static int -specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, + bool shadow, uint32_t shared_keys_version, + DescriptorClassification kind, PyObject *descr, unsigned int tp_version) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); - bool shadow = instance_has_key(owner, name); - PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor_load(type, name, &descr); - Py_XDECREF(descr); // turn strong ref into a borrowed ref - assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); - if (type_get_version(type, LOAD_ATTR) == 0) { + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } + uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1112,9 +1127,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (shadow) { goto try_instance; } - int oparg = instr->op.arg; if (oparg & 1) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) { + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, true, + shared_keys_version)) { return 0; } else { @@ -1140,7 +1156,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (!function_check_args(fget, 1, LOAD_ATTR)) { return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } @@ -1149,8 +1165,14 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } - assert(type->tp_version_tag != 0); - write_u32(lm_cache->type_version, type->tp_version_tag); + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(fget)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; + } + #endif + assert(tp_version != 0); + write_u32(lm_cache->type_version, tp_version); /* borrowed */ write_obj(lm_cache->descr, fget); specialize(instr, LOAD_ATTR_PROPERTY); @@ -1176,7 +1198,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); + write_u32(cache->version, tp_version); specialize(instr, LOAD_ATTR_SLOT); return 0; } @@ -1185,7 +1207,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na Py_ssize_t offset = offsetof(PyObject, ob_type); assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); + write_u32(cache->version, tp_version); specialize(instr, LOAD_ATTR_SLOT); return 0; } @@ -1200,13 +1222,18 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { + #ifndef Py_GIL_DISABLED + // In free-threaded builds it's possible for tp_getattro to change + // after the call to analyze_descriptor. That is fine: the version + // guard will fail. assert(type->tp_getattro == _Py_slot_tp_getattro); + #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } @@ -1219,10 +1246,16 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; + } + #endif write_u32(lm_cache->keys_version, version); /* borrowed */ write_obj(lm_cache->descr, descr); - write_u32(lm_cache->type_version, type->tp_version_tag); + write_u32(lm_cache->type_version, tp_version); specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); return 0; } @@ -1237,8 +1270,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (shadow) { goto try_instance; } - if ((instr->op.arg & 1) == 0) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) { + if ((oparg & 1) == 0) { + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, false, + shared_keys_version)) { return 0; } } @@ -1252,14 +1287,28 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } Py_UNREACHABLE(); try_instance: - if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag, + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { return 0; } return -1; } -#endif // Py_GIL_DISABLED + +static int +specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +{ + // 0 is not a valid version + uint32_t shared_keys_version = 0; + bool shadow = instance_has_key(owner, name, &shared_keys_version); + PyObject *descr = NULL; + unsigned int tp_version = 0; + PyTypeObject *type = Py_TYPE(owner); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version); + int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version); + Py_XDECREF(descr); + return result; +} void _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) @@ -1281,20 +1330,10 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam fail = specialize_module_load_attr(owner, instr, name); } else if (PyType_Check(owner)) { - #ifdef Py_GIL_DISABLED - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - fail = true; - #else fail = specialize_class_load_attr(owner, instr, name); - #endif } else { - #ifdef Py_GIL_DISABLED - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - fail = true; - #else fail = specialize_instance_load_attr(owner, instr, name); - #endif } if (fail) { @@ -1402,8 +1441,6 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na return; } -#ifndef Py_GIL_DISABLED - #ifdef Py_STATS static int load_attr_fail_kind(DescriptorClassification kind) @@ -1452,8 +1489,10 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN); return -1; } - PyObject *metadescriptor = _PyType_Lookup(Py_TYPE(cls), name); + unsigned int meta_version = 0; + PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name, &meta_version); DescriptorClassification metakind = classify_descriptor(metadescriptor, false); + Py_XDECREF(metadescriptor); switch (metakind) { case METHOD: case NON_DESCRIPTOR: @@ -1468,38 +1507,52 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor_load(cls, name, &descr); - Py_XDECREF(descr); // turn strong ref into a borrowed ref - if (type_get_version(cls, LOAD_ATTR) == 0) { + unsigned int tp_version = 0; + kind = analyze_descriptor_load(cls, name, &descr, &tp_version); + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); return -1; } bool metaclass_check = false; if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) { metaclass_check = true; - if (type_get_version(Py_TYPE(cls), LOAD_ATTR) == 0) { + if (meta_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); return -1; } } switch (kind) { case METHOD: case NON_DESCRIPTOR: - write_u32(cache->type_version, cls->tp_version_tag); + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + Py_XDECREF(descr); + return -1; + } + #endif + write_u32(cache->type_version, tp_version); write_obj(cache->descr, descr); if (metaclass_check) { - write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag); + write_u32(cache->keys_version, meta_version); specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); } else { specialize(instr, LOAD_ATTR_CLASS); } + Py_XDECREF(descr); return 0; #ifdef Py_STATS case ABSENT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + Py_XDECREF(descr); return -1; #endif default: SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind)); + Py_XDECREF(descr); return -1; } } @@ -1508,29 +1561,41 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, // can cause a significant drop in cache hits. A possible test is // python.exe -m test_typing test_re test_dis test_zlib. static int -specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, -PyObject *descr, DescriptorClassification kind, bool is_method) +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version) { _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyTypeObject *owner_cls = Py_TYPE(owner); assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); - if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { - PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - assert(_PyDictKeys_StringLookup(keys, name) < 0); - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( - _PyInterpreterState_GET(), keys); - if (keys_version == 0) { + + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return 0; + } + #endif + + unsigned long tp_flags = PyType_GetFlags(owner_cls); + if (tp_flags & Py_TPFLAGS_INLINE_VALUES) { + #ifndef Py_GIL_DISABLED + assert(_PyDictKeys_StringLookup( + ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0); + #endif + if (shared_keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } - write_u32(cache->keys_version, keys_version); + write_u32(cache->keys_version, shared_keys_version); specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { Py_ssize_t dictoffset; - if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + if (tp_flags & Py_TPFLAGS_MANAGED_DICT) { dictoffset = MANAGED_DICT_OFFSET; } else { @@ -1576,13 +1641,11 @@ PyObject *descr, DescriptorClassification kind, bool is_method) * PyType_Modified usages in typeobject.c). The MCACHE has been * working since Python 2.6 and it's battle-tested. */ - write_u32(cache->type_version, owner_cls->tp_version_tag); + write_u32(cache->type_version, tp_version); write_obj(cache->descr, descr); return 1; } -#endif // Py_GIL_DISABLED - static void specialize_load_global_lock_held( @@ -1729,7 +1792,6 @@ function_kind(PyCodeObject *code) { return SIMPLE_FUNCTION; } -#ifndef Py_GIL_DISABLED /* Returning false indicates a failure. */ static bool function_check_args(PyObject *o, int expected_argcount, int opcode) @@ -1763,19 +1825,6 @@ function_get_version(PyObject *o, int opcode) return version; } -/* Returning 0 indicates a failure. */ -static uint32_t -type_get_version(PyTypeObject *t, int opcode) -{ - uint32_t version = t->tp_version_tag; - if (version == 0) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); - return 0; - } - return version; -} -#endif // Py_GIL_DISABLED - void _Py_Specialize_BinarySubscr( _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 99896f32fd2b08..4013b503502df6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -386,7 +386,7 @@ def find_assignment_target(node: parser.InstDef, idx: int) -> list[lexer.Token]: """Find the tokens that make up the left-hand side of an assignment""" offset = 0 for tkn in reversed(node.block.tokens[: idx]): - if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + if tkn.kind in {"SEMI", "LBRACE", "RBRACE", "CMACRO"}: return node.block.tokens[idx - offset : idx] offset += 1 return [] diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 69d84183f1c7e6..f54afbb880d2fa 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -126,7 +126,7 @@ def __init__(self, out: CWriter): "PyStackRef_AsPyObjectSteal": self.stackref_steal, "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, - "POP_DEAD_INPUTS": self.pop_dead_inputs, + "POP_INPUT": self.pop_input, } self.out = out @@ -423,7 +423,7 @@ def save_stack( self.emit_save(storage) return True - def pop_dead_inputs( + def pop_input( self, tkn: Token, tkn_iter: TokenIterator, @@ -432,9 +432,18 @@ def pop_dead_inputs( inst: Instruction | None, ) -> bool: next(tkn_iter) + name_tkn = next(tkn_iter) + name = name_tkn.text next(tkn_iter) next(tkn_iter) - storage.pop_dead_inputs(self.out) + if not storage.inputs: + raise analysis_error("stack is empty", tkn) + tos = storage.inputs[-1] + if tos.name != name: + raise analysis_error(f"'{name} is not top of stack", name_tkn) + tos.defined = False + storage.clear_dead_inputs() + storage.flush(self.out) return True def emit_reload(self, storage: Storage) -> None: diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 9471fe0e56f7d8..286f47d0cfb11b 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -512,10 +512,6 @@ def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = self._push_defined_outputs() self.stack.flush(out, cast_type, extract_bits) - def pop_dead_inputs(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True) -> None: - self.clear_dead_inputs() - self.stack.flush(out, cast_type, extract_bits) - def save(self, out: CWriter) -> None: assert self.spilled >= 0 if self.spilled == 0: From f7ceb317aec498823555885a4b7fed5e0244f300 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Jan 2025 22:40:45 +0200 Subject: [PATCH 42/61] gh-71339: Use new assertion methods in test_logging (GH-128828) --- Lib/test/test_logging.py | 51 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 44c854f02a73c6..2e5f6475ae3b1e 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1134,7 +1134,7 @@ def test_basic(self): self.assertEqual(mailfrom, 'me') self.assertEqual(rcpttos, ['you']) self.assertIn('\nSubject: Log\n', data) - self.assertTrue(data.endswith('\n\nHello \u2713')) + self.assertEndsWith(data, '\n\nHello \u2713') h.close() def process_message(self, *args): @@ -3524,7 +3524,7 @@ def test_config14_ok(self): self.assertEqual(h.foo, 'bar') self.assertEqual(h.terminator, '!\n') logging.warning('Exclamation') - self.assertTrue(output.getvalue().endswith('Exclamation!\n')) + self.assertEndsWith(output.getvalue(), 'Exclamation!\n') def test_config15_ok(self): @@ -4281,7 +4281,7 @@ def test_queue_handler(self): msg = self.next_message() self.que_logger.warning(msg) data = self.queue.get_nowait() - self.assertTrue(isinstance(data, logging.LogRecord)) + self.assertIsInstance(data, logging.LogRecord) self.assertEqual(data.name, self.que_logger.name) self.assertEqual((data.msg, data.args), (msg, None)) @@ -4879,14 +4879,14 @@ def test_formatting(self): r.removeHandler(h) h.close() r = h.records[0] - self.assertTrue(r.exc_text.startswith('Traceback (most recent ' - 'call last):\n')) - self.assertTrue(r.exc_text.endswith('\nRuntimeError: ' - 'deliberate mistake')) - self.assertTrue(r.stack_info.startswith('Stack (most recent ' - 'call last):\n')) - self.assertTrue(r.stack_info.endswith('logging.exception(\'failed\', ' - 'stack_info=True)')) + self.assertStartsWith(r.exc_text, + 'Traceback (most recent call last):\n') + self.assertEndsWith(r.exc_text, + '\nRuntimeError: deliberate mistake') + self.assertStartsWith(r.stack_info, + 'Stack (most recent call last):\n') + self.assertEndsWith(r.stack_info, + "logging.exception('failed', stack_info=True)") class LastResortTest(BaseTest): @@ -5229,8 +5229,8 @@ class LogRecordTest(BaseTest): def test_str_rep(self): r = logging.makeLogRecord({}) s = str(r) - self.assertTrue(s.startswith('')) + self.assertStartsWith(s, '') def test_dict_arg(self): h = RecordingHandler() @@ -5880,14 +5880,14 @@ def test_extra_in_records(self): self.adapter.critical('foo should be here') self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertTrue(hasattr(record, 'foo')) + self.assertHasAttr(record, 'foo') self.assertEqual(record.foo, '1') def test_extra_not_merged_by_default(self): self.adapter.critical('foo should NOT be here', extra={'foo': 'nope'}) self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertFalse(hasattr(record, 'foo')) + self.assertNotHasAttr(record, 'foo') def test_extra_merged(self): self.adapter = logging.LoggerAdapter(logger=self.logger, @@ -5897,8 +5897,8 @@ def test_extra_merged(self): self.adapter.critical('foo and bar should be here', extra={'bar': '2'}) self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertTrue(hasattr(record, 'foo')) - self.assertTrue(hasattr(record, 'bar')) + self.assertHasAttr(record, 'foo') + self.assertHasAttr(record, 'bar') self.assertEqual(record.foo, '1') self.assertEqual(record.bar, '2') @@ -5910,7 +5910,7 @@ def test_extra_merged_log_call_has_precedence(self): self.adapter.critical('foo shall be min', extra={'foo': '2'}) self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertTrue(hasattr(record, 'foo')) + self.assertHasAttr(record, 'foo') self.assertEqual(record.foo, '2') @@ -6624,18 +6624,19 @@ def namer(filename): p = '%s.log.' % prefix for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.startswith(p)) + self.assertStartsWith(fn, p) elif prefix.startswith('d.e'): for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.endswith('.log'), fn) - self.assertTrue(fn.startswith(prefix + '.') and - fn[len(prefix) + 2].isdigit()) + self.assertEndsWith(fn, '.log') + self.assertStartsWith(fn, prefix + '.') + self.assertTrue(fn[len(prefix) + 2].isdigit()) elif prefix == 'g': for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.endswith('.oldlog')) - self.assertTrue(fn.startswith('g') and fn[1].isdigit()) + self.assertEndsWith(fn, '.oldlog') + self.assertStartsWith(fn, 'g') + self.assertTrue(fn[1].isdigit()) def test_compute_files_to_delete_same_filename_different_extensions(self): # See GH-93205 for background @@ -6673,7 +6674,7 @@ def test_compute_files_to_delete_same_filename_different_extensions(self): matcher = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\Z") for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.startswith(prefix+'.')) + self.assertStartsWith(fn, prefix+'.') suffix = fn[(len(prefix)+1):] self.assertRegex(suffix, matcher) From b52de22ac345ad8583bcc57f963e26b35c2ee527 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 15 Jan 2025 01:17:11 +0200 Subject: [PATCH 43/61] gh-71339: Use new assertion methods in the multiprocessing tests (GH-128847) --- Lib/test/_test_multiprocessing.py | 53 ++++++++++++++----------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 38a03f3391d31d..4b7c3e7fa8bdd7 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -319,7 +319,7 @@ def test_current(self): authkey = current.authkey self.assertTrue(current.is_alive()) - self.assertTrue(not current.daemon) + self.assertFalse(current.daemon) self.assertIsInstance(authkey, bytes) self.assertTrue(len(authkey) > 0) self.assertEqual(current.ident, os.getpid()) @@ -463,7 +463,7 @@ def test_process(self): self.assertEqual(p.is_alive(), False) self.assertEqual(p.daemon, True) self.assertNotIn(p, self.active_children()) - self.assertTrue(type(self.active_children()) is list) + self.assertIs(type(self.active_children()), list) self.assertEqual(p.exitcode, None) p.start() @@ -583,8 +583,8 @@ def test_cpu_count(self): cpus = multiprocessing.cpu_count() except NotImplementedError: cpus = 1 - self.assertTrue(type(cpus) is int) - self.assertTrue(cpus >= 1) + self.assertIsInstance(cpus, int) + self.assertGreaterEqual(cpus, 1) def test_active_children(self): self.assertEqual(type(self.active_children()), list) @@ -2382,14 +2382,14 @@ def test_getobj_getlock(self): self.assertEqual(lock, lock3) arr4 = self.Value('i', 5, lock=False) - self.assertFalse(hasattr(arr4, 'get_lock')) - self.assertFalse(hasattr(arr4, 'get_obj')) + self.assertNotHasAttr(arr4, 'get_lock') + self.assertNotHasAttr(arr4, 'get_obj') self.assertRaises(AttributeError, self.Value, 'i', 5, lock='navalue') arr5 = self.RawValue('i', 5) - self.assertFalse(hasattr(arr5, 'get_lock')) - self.assertFalse(hasattr(arr5, 'get_obj')) + self.assertNotHasAttr(arr5, 'get_lock') + self.assertNotHasAttr(arr5, 'get_obj') class _TestArray(BaseTestCase): @@ -2462,14 +2462,14 @@ def test_getobj_getlock_obj(self): self.assertEqual(lock, lock3) arr4 = self.Array('i', range(10), lock=False) - self.assertFalse(hasattr(arr4, 'get_lock')) - self.assertFalse(hasattr(arr4, 'get_obj')) + self.assertNotHasAttr(arr4, 'get_lock') + self.assertNotHasAttr(arr4, 'get_obj') self.assertRaises(AttributeError, self.Array, 'i', range(10), lock='notalock') arr5 = self.RawArray('i', range(10)) - self.assertFalse(hasattr(arr5, 'get_lock')) - self.assertFalse(hasattr(arr5, 'get_obj')) + self.assertNotHasAttr(arr5, 'get_lock') + self.assertNotHasAttr(arr5, 'get_obj') # # @@ -2657,8 +2657,8 @@ def test_namespace(self): self.assertEqual((n.name, n.job), ('Bob', 'Builder')) del n.job self.assertEqual(str(n), "Namespace(name='Bob')") - self.assertTrue(hasattr(n, 'name')) - self.assertTrue(not hasattr(n, 'job')) + self.assertHasAttr(n, 'name') + self.assertNotHasAttr(n, 'job') # # @@ -4938,13 +4938,9 @@ def test_import(self): for name in modules: __import__(name) mod = sys.modules[name] - self.assertTrue(hasattr(mod, '__all__'), name) - + self.assertHasAttr(mod, '__all__', name) for attr in mod.__all__: - self.assertTrue( - hasattr(mod, attr), - '%r does not have attribute %r' % (mod, attr) - ) + self.assertHasAttr(mod, attr) # # Quick test that logging works -- does not test logging output @@ -4957,7 +4953,7 @@ class _TestLogging(BaseTestCase): def test_enable_logging(self): logger = multiprocessing.get_logger() logger.setLevel(util.SUBWARNING) - self.assertTrue(logger is not None) + self.assertIsNotNone(logger) logger.debug('this will not be printed') logger.info('nor will this') logger.setLevel(LOG_LEVEL) @@ -5753,9 +5749,8 @@ def test_set_get(self): self.assertEqual(multiprocessing.get_start_method(), method) ctx = multiprocessing.get_context() self.assertEqual(ctx.get_start_method(), method) - self.assertTrue(type(ctx).__name__.lower().startswith(method)) - self.assertTrue( - ctx.Process.__name__.lower().startswith(method)) + self.assertStartsWith(type(ctx).__name__.lower(), method) + self.assertStartsWith(ctx.Process.__name__.lower(), method) self.check_context(multiprocessing) count += 1 finally: @@ -5956,9 +5951,9 @@ def check_resource_tracker_death(self, signum, should_die): if should_die: self.assertEqual(len(all_warn), 1) the_warn = all_warn[0] - self.assertTrue(issubclass(the_warn.category, UserWarning)) - self.assertTrue("resource_tracker: process died" - in str(the_warn.message)) + self.assertIsSubclass(the_warn.category, UserWarning) + self.assertIn("resource_tracker: process died", + str(the_warn.message)) else: self.assertEqual(len(all_warn), 0) @@ -6163,8 +6158,8 @@ def is_alive(self): Process=FailingForkProcess)) p.close() p.join() - self.assertFalse( - any(process.is_alive() for process in forked_processes)) + for process in forked_processes: + self.assertFalse(process.is_alive(), process) @hashlib_helper.requires_hashdigest('sha256') From bd3baa8b1a7755f17b2fc98c7fb7b872fec43af3 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Wed, 15 Jan 2025 01:48:46 +0100 Subject: [PATCH 44/61] gh-121604: Make sure all deprecated items in importlib raise DeprecationWarning (#128007) Co-authored-by: rashansmith Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap_external.py | 6 ++++ Lib/importlib/abc.py | 13 ++++++++ Lib/importlib/machinery.py | 27 ++++++++++++++-- Lib/inspect.py | 3 +- Lib/test/test_importlib/test_abc.py | 32 +++++++++++++++++++ Lib/test/test_importlib/test_api.py | 13 ++++++++ Lib/test/test_importlib/test_windows.py | 6 ++++ ...-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst | 1 + 8 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index fa36159711846f..697f7c55218a8f 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -716,6 +716,12 @@ def _search_registry(cls, fullname): @classmethod def find_spec(cls, fullname, path=None, target=None): + _warnings.warn('importlib.machinery.WindowsRegistryFinder is ' + 'deprecated; use site configuration instead. ' + 'Future versions of Python may not enable this ' + 'finder by default.', + DeprecationWarning, stacklevel=2) + filepath = cls._search_registry(fullname) if filepath is None: return None diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index eea6b38af6fa13..bb2837d38d83f1 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -70,6 +70,15 @@ class ResourceLoader(Loader): """ + def __init__(self): + import warnings + warnings.warn('importlib.abc.ResourceLoader is deprecated in ' + 'favour of supporting resource loading through ' + 'importlib.resources.abc.ResourceReader.', + DeprecationWarning, stacklevel=2) + super().__init__() + + @abc.abstractmethod def get_data(self, path): """Abstract method which when implemented should return the bytes for @@ -199,6 +208,10 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo def path_mtime(self, path): """Return the (int) modification time for the path (str).""" + import warnings + warnings.warn('SourceLoader.path_mtime is deprecated in favour of ' + 'SourceLoader.path_stats().', + DeprecationWarning, stacklevel=2) if self.path_stats.__func__ is SourceLoader.path_stats: raise OSError return int(self.path_stats(path)['mtime']) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 6e294d59bfdcb9..63d726445c3d96 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -3,9 +3,11 @@ from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter -from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, - OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, - EXTENSION_SUFFIXES) +from ._bootstrap_external import ( + SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, + DEBUG_BYTECODE_SUFFIXES as _DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES as _OPTIMIZED_BYTECODE_SUFFIXES +) from ._bootstrap_external import WindowsRegistryFinder from ._bootstrap_external import PathFinder from ._bootstrap_external import FileFinder @@ -27,3 +29,22 @@ def all_suffixes(): 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', 'WindowsRegistryFinder', 'all_suffixes'] + + +def __getattr__(name): + import warnings + + if name == 'DEBUG_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.DEBUG_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _DEBUG_BYTECODE_SUFFIXES + elif name == 'OPTIMIZED_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _OPTIMIZED_BYTECODE_SUFFIXES + + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/Lib/inspect.py b/Lib/inspect.py index 5b7c4df8927c87..facad478103668 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -858,8 +858,7 @@ def getsourcefile(object): Return None if no way can be identified to get the source. """ filename = getfile(object) - all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] - all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] + all_bytecode_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:] if any(filename.endswith(s) for s in all_bytecode_suffixes): filename = (os.path.splitext(filename)[0] + importlib.machinery.SOURCE_SUFFIXES[0]) diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 603125f6d926f6..00af2dd712425a 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -913,5 +913,37 @@ def test_universal_newlines(self): SourceOnlyLoaderMock=SPLIT_SOL) +class SourceLoaderDeprecationWarningsTests(unittest.TestCase): + """Tests SourceLoader deprecation warnings.""" + + def test_deprecated_path_mtime(self): + from importlib.abc import SourceLoader + class DummySourceLoader(SourceLoader): + def get_data(self, path): + return b'' + + def get_filename(self, fullname): + return 'foo.py' + + def path_stats(self, path): + return {'mtime': 1} + + loader = DummySourceLoader() + with self.assertWarns(DeprecationWarning): + loader.path_mtime('foo.py') + + +class ResourceLoaderDeprecationWarningsTests(unittest.TestCase): + """Tests ResourceLoader deprecation warnings.""" + + def test_deprecated_resource_loader(self): + from importlib.abc import ResourceLoader + class DummyLoader(ResourceLoader): + def get_data(self, path): + return b'' + + with self.assertWarns(DeprecationWarning): + DummyLoader() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 51ea5270b1a928..6035b2ca72efb9 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -492,5 +492,18 @@ def test_util(self): support.check__all__(self, util['Source'], extra=extra) +class TestDeprecations(unittest.TestCase): + def test_machinery_deprecated_attributes(self): + from importlib import machinery + attributes = ( + 'DEBUG_BYTECODE_SUFFIXES', + 'OPTIMIZED_BYTECODE_SUFFIXES', + ) + for attr in attributes: + with self.subTest(attr=attr): + with self.assertWarns(DeprecationWarning): + getattr(machinery, attr) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index 8a9a8fffcd10d4..f32680bdbeb9e3 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -104,6 +104,12 @@ def test_module_not_found(self): spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNone(spec) + def test_raises_deprecation_warning(self): + # WindowsRegistryFinder is not meant to be instantiated, so the + # deprecation warning is raised in the 'find_spec' method instead. + with self.assertWarns(DeprecationWarning): + self.machinery.WindowsRegistryFinder.find_spec('spam') + (Frozen_WindowsRegistryFinderTests, Source_WindowsRegistryFinderTests ) = test_util.test_both(WindowsRegistryFinderTests, machinery=machinery) diff --git a/Misc/NEWS.d/next/Library/2024-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst b/Misc/NEWS.d/next/Library/2024-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst new file mode 100644 index 00000000000000..9a6fce8647cc6b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst @@ -0,0 +1 @@ +Add missing Deprecation warnings for :attr:`importlib.machinery.DEBUG_BYTECODE_SUFFIXES`, :attr:`importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES`, :class:`importlib.machinery.WindowsRegistryFinder`, :class:`importlib.abc.ResourceLoader`, :meth:`importlib.abc.SourceLoader.path_mtime`. From 6e4f64109b0eb6c9f1b50eb7dc5f647a1d901ff4 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Tue, 14 Jan 2025 19:49:02 -0600 Subject: [PATCH 45/61] gh-128473: Skip segfaulting `test_embed` tests when BOLT instrumented (gh-128474) * Skip segfaulting `test_embed` tests when BOLT instrumented Co-authored-by: Gregory Szorc * NEWS --------- Co-authored-by: Gregory Szorc --- Lib/test/test_embed.py | 4 ++++ .../next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7110fb889f3c8e..1b55cd156d759d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -940,6 +940,7 @@ def check_all_configs(self, testname, expected_config=None, self.check_global_config(configs) return configs + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") def test_init_default_config(self): self.check_all_configs("test_init_initialize_config", api=API_COMPAT) @@ -1039,6 +1040,7 @@ def test_init_from_config(self): self.check_all_configs("test_init_from_config", config, preconfig, api=API_COMPAT) + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") def test_init_compat_env(self): preconfig = { 'allocator': ALLOCATOR_FOR_CONFIG, @@ -1074,6 +1076,7 @@ def test_init_compat_env(self): self.check_all_configs("test_init_compat_env", config, preconfig, api=API_COMPAT) + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") def test_init_python_env(self): preconfig = { 'allocator': ALLOCATOR_FOR_CONFIG, @@ -1772,6 +1775,7 @@ def test_init_set_config(self): self.check_all_configs("test_init_set_config", config, api=API_ISOLATED) + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") def test_initconfig_api(self): preconfig = { 'configure_locale': True, diff --git a/Misc/NEWS.d/next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst b/Misc/NEWS.d/next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst new file mode 100644 index 00000000000000..7dc807757c5ec0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst @@ -0,0 +1,2 @@ +Disable ``test_embed`` test cases that segfault on BOLT instrument binaries. +The tests are only disabled when BOLT is enabled. From ae7f621c33c854334c69cf97260c7516170dcf0d Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Wed, 15 Jan 2025 10:38:43 +0100 Subject: [PATCH 46/61] gh-128438: Use `EnvironmentVarGuard` for `test_{builtin,io,locale}.py` (#128476) Modifying locale-related environment variables in `Lib/test/test_builtin.py`, `Lib/test/test_io.py` and `Lib/test/test_locale.py` is now achieved by using an `EnvironmentVarGuard` context instead of an explicit `try-finally` block. --- Lib/test/test_builtin.py | 9 ++------- Lib/test/test_io.py | 9 ++------- Lib/test/test_locale.py | 23 +++++++---------------- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 5f4eac5267622f..73b139e405ae59 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1567,14 +1567,12 @@ def test_open(self): @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_open_default_encoding(self): - old_environ = dict(os.environ) - try: + with EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that open() uses the current locale # encoding and not the user preferred encoding for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset(key) self.write_testfile() current_locale_encoding = locale.getencoding() @@ -1583,9 +1581,6 @@ def test_open_default_encoding(self): fp = open(TESTFN, 'w') with fp: self.assertEqual(fp.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) @support.requires_subprocess() def test_open_non_inheritable(self): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 33e0161241e87e..8c79d2c24a140a 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2892,14 +2892,12 @@ def test_reconfigure_line_buffering(self): @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_default_encoding(self): - old_environ = dict(os.environ) - try: + with os_helper.EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that TextIOWrapper() uses the current # locale encoding and not the user preferred encoding for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset(key) current_locale_encoding = locale.getencoding() b = self.BytesIO() @@ -2907,9 +2905,6 @@ def test_default_encoding(self): warnings.simplefilter("ignore", EncodingWarning) t = self.TextIOWrapper(b) self.assertEqual(t.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) def test_encoding(self): # Check the encoding attribute is always set, and valid diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 00e93d8e78443d..c025ed4108fb58 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,5 +1,5 @@ from decimal import Decimal -from test.support import verbose, is_android, is_emscripten, is_wasi +from test.support import verbose, is_android, is_emscripten, is_wasi, os_helper from test.support.warnings_helper import check_warnings from test.support.import_helper import import_fresh_module from unittest import mock @@ -499,25 +499,16 @@ def test_defaults_UTF8(self): else: orig_getlocale = None - orig_env = {} try: - for key in ('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE'): - if key in os.environ: - orig_env[key] = os.environ[key] - del os.environ[key] + with os_helper.EnvironmentVarGuard() as env: + for key in ('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE'): + env.unset(key) - os.environ['LC_CTYPE'] = 'UTF-8' - - with check_warnings(('', DeprecationWarning)): - self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) + env.set('LC_CTYPE', 'UTF-8') + with check_warnings(('', DeprecationWarning)): + self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) finally: - for k in orig_env: - os.environ[k] = orig_env[k] - - if 'LC_CTYPE' not in orig_env: - del os.environ['LC_CTYPE'] - if orig_getlocale is not None: _locale._getdefaultlocale = orig_getlocale From 1a1056d394a489d229b26c06c7c79aa9c06696f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20Ran=20=28=E6=B1=AA=E7=84=B6=29?= Date: Wed, 15 Jan 2025 20:24:31 +0800 Subject: [PATCH 47/61] Fix typo in `Lib/asyncio/futures.py` (#128819) --- Lib/asyncio/futures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index c95fce035cd548..359b7a5e3f9eea 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -62,7 +62,7 @@ class Future: # that it is not compatible by setting this to None. # - It is set by __iter__() below so that Task.__step() can tell # the difference between - # `await Future()` or`yield from Future()` (correct) vs. + # `await Future()` or `yield from Future()` (correct) vs. # `yield Future()` (incorrect). _asyncio_future_blocking = False From 599be687ec7327c30c6469cf743aa4ee9e82232d Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 15 Jan 2025 13:05:59 +0000 Subject: [PATCH 48/61] gh-128816: Fix warnings in test_doctest (GH-128817) * Fix a deprecation warning for using importlib.resources.abc.ResourceReader. * Fix an import warning when importing readline (if it has not yet been imported). --- Lib/test/test_doctest/test_doctest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index b1e165fe16b54f..a4a49298bab3be 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2860,7 +2860,7 @@ def test_testfile(): r""" >>> _colorize.COLORIZE = save_colorize """ -class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): +class TestImporter(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path, target=None): return importlib.util.spec_from_file_location(fullname, path, loader=self) @@ -2869,6 +2869,12 @@ def get_data(self, path): with open(path, mode='rb') as f: return f.read() + def exec_module(self, module): + raise ImportError + + def create_module(self, spec): + return None + class TestHook: def __init__(self, pathdir): From 40a4d88a14c741172a158683c39d232c587c6f11 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Jan 2025 15:12:40 +0100 Subject: [PATCH 49/61] gh-128874: Fix the documentation for blurb 2.0 (#128875) --- Misc/NEWS.d/3.10.0b1.rst | 2 +- Misc/NEWS.d/3.11.0b1.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 25c6b827146e82..406a5d7853edc0 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -941,7 +941,7 @@ result from ``entry_points()`` as deprecated. .. -.. gh: 47383 +.. gh-issue: 47383 .. date: 2021-04-08-19-32-26 .. nonce: YI1hdL .. section: Library diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 85cb0f1b5cffbd..87442dbbbd17f5 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -570,7 +570,7 @@ planned). Patch by Alex Waygood. .. -.. gh: 78157 +.. gh-issue: 78157 .. date: 2022-05-05-20-40-45 .. nonce: IA_9na .. section: Library @@ -1289,7 +1289,7 @@ Deprecate the chunk module. .. -.. gh: 91498 +.. gh-issue: 91498 .. date: 2022-04-10-08-39-44 .. nonce: 8oII92 .. section: Library From 8d8b854824c4723d7c5924f1d5c6a397ea7214a5 Mon Sep 17 00:00:00 2001 From: Umar Butler <8473183+umarbutler@users.noreply.github.com> Date: Thu, 16 Jan 2025 04:00:54 +1100 Subject: [PATCH 50/61] gh-128016: Improved invalid escape sequence warning message (#128020) --- Lib/test/test_cmd_line_script.py | 3 +- Lib/test/test_codeop.py | 2 +- Lib/test/test_string_literals.py | 29 +++++++++++++------ Lib/test/test_unparse.py | 4 ++- ...-12-17-09-28-17.gh-issue-128016.DPqhah.rst | 1 + Objects/bytesobject.c | 7 +++-- Objects/unicodeobject.c | 6 ++-- Parser/string_parser.c | 24 +++++++++++---- Parser/tokenizer/helpers.c | 12 ++++++-- 9 files changed, 63 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f30107225ff612..527d51857fc904 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -659,7 +659,8 @@ def test_syntaxerror_invalid_escape_sequence_multi_line(self): stderr.splitlines()[-3:], [ b' foo = """\\q"""', b' ^^^^^^^^', - b'SyntaxError: invalid escape sequence \'\\q\'' + b'SyntaxError: "\\q" is an invalid escape sequence. ' + b'Did you mean "\\\\q"? A raw string is also an option.' ], ) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 787bd1b6a79e20..0eefc22d11bce0 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -282,7 +282,7 @@ def test_warning(self): # Test that the warning is only returned once. with warnings_helper.check_warnings( ('"is" with \'str\' literal', SyntaxWarning), - ("invalid escape sequence", SyntaxWarning), + ('"\\\\e" is an invalid escape sequence', SyntaxWarning), ) as w: compile_command(r"'\e' is 0") self.assertEqual(len(w.warnings), 2) diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index c7c6f684cd33f0..f56195ca27672c 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -116,7 +116,9 @@ def test_eval_str_invalid_escape(self): warnings.simplefilter('always', category=SyntaxWarning) eval("'''\n\\z'''") self.assertEqual(len(w), 1) - self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'") + self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -126,7 +128,8 @@ def test_eval_str_invalid_escape(self): eval("'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid escape sequence '\z'") + self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) self.assertEqual(exc.offset, 1) @@ -153,7 +156,9 @@ def test_eval_str_invalid_octal_escape(self): eval("'''\n\\407'''") self.assertEqual(len(w), 1) self.assertEqual(str(w[0].message), - r"invalid octal escape sequence '\407'") + r'"\407" is an invalid octal escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -163,7 +168,8 @@ def test_eval_str_invalid_octal_escape(self): eval("'''\n\\407'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'") + self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) self.assertEqual(exc.offset, 1) @@ -205,7 +211,9 @@ def test_eval_bytes_invalid_escape(self): warnings.simplefilter('always', category=SyntaxWarning) eval("b'''\n\\z'''") self.assertEqual(len(w), 1) - self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'") + self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -215,7 +223,8 @@ def test_eval_bytes_invalid_escape(self): eval("b'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid escape sequence '\z'") + self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) @@ -228,8 +237,9 @@ def test_eval_bytes_invalid_octal_escape(self): warnings.simplefilter('always', category=SyntaxWarning) eval("b'''\n\\407'''") self.assertEqual(len(w), 1) - self.assertEqual(str(w[0].message), - r"invalid octal escape sequence '\407'") + self.assertEqual(str(w[0].message), r'"\407" is an invalid octal escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -239,7 +249,8 @@ def test_eval_bytes_invalid_octal_escape(self): eval("b'''\n\\407'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'") + self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 35394f29fbe49d..332919540da4d6 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -651,7 +651,9 @@ def test_multiquote_joined_string(self): def test_backslash_in_format_spec(self): import re - msg = re.escape("invalid escape sequence '\\ '") + msg = re.escape('"\\ " is an invalid escape sequence. ' + 'Such sequences will not work in the future. ' + 'Did you mean "\\\\ "? A raw string is also an option.') with self.assertWarnsRegex(SyntaxWarning, msg): self.check_ast_roundtrip("""f"{x:\\ }" """) self.check_ast_roundtrip("""f"{x:\\n}" """) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst new file mode 100644 index 00000000000000..0832d777bc3251 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst @@ -0,0 +1 @@ +Improved the ``SyntaxWarning`` message for invalid escape sequences to clarify that such sequences will raise a ``SyntaxError`` in future Python releases. The new message also suggests a potential fix, i.e., ``Did you mean "\\e"?``. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 024653546563e6..b3d1c425ad18b7 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1205,7 +1205,8 @@ PyObject *PyBytes_DecodeEscape(const char *s, unsigned char c = *first_invalid_escape; if ('4' <= c && c <= '7') { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid octal escape sequence '\\%.3s'", + "b\"\\%.3s\" is an invalid octal escape sequence. " + "Such sequences will not work in the future. ", first_invalid_escape) < 0) { Py_DECREF(result); @@ -1214,7 +1215,8 @@ PyObject *PyBytes_DecodeEscape(const char *s, } else { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid escape sequence '\\%c'", + "b\"\\%c\" is an invalid escape sequence. " + "Such sequences will not work in the future. ", c) < 0) { Py_DECREF(result); @@ -1223,7 +1225,6 @@ PyObject *PyBytes_DecodeEscape(const char *s, } } return result; - } /* -------------------------------------------------------------------- */ /* object api */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 3eafa2381c1a4d..d9952f764bb178 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6853,7 +6853,8 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, unsigned char c = *first_invalid_escape; if ('4' <= c && c <= '7') { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid octal escape sequence '\\%.3s'", + "\"\\%.3s\" is an invalid octal escape sequence. " + "Such sequences will not work in the future. ", first_invalid_escape) < 0) { Py_DECREF(result); @@ -6862,7 +6863,8 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, } else { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid escape sequence '\\%c'", + "\"\\%c\" is an invalid escape sequence. " + "Such sequences will not work in the future. ", c) < 0) { Py_DECREF(result); diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 9537c543b0eb93..9dd8f9ef28bd4f 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -28,9 +28,16 @@ warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token int octal = ('4' <= c && c <= '7'); PyObject *msg = octal - ? PyUnicode_FromFormat("invalid octal escape sequence '\\%.3s'", - first_invalid_escape) - : PyUnicode_FromFormat("invalid escape sequence '\\%c'", c); + ? PyUnicode_FromFormat( + "\"\\%.3s\" is an invalid octal escape sequence. " + "Such sequences will not work in the future. " + "Did you mean \"\\\\%.3s\"? A raw string is also an option.", + first_invalid_escape, first_invalid_escape) + : PyUnicode_FromFormat( + "\"\\%c\" is an invalid escape sequence. " + "Such sequences will not work in the future. " + "Did you mean \"\\\\%c\"? A raw string is also an option.", + c, c); if (msg == NULL) { return -1; } @@ -53,11 +60,16 @@ warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token error location, if p->known_err_token is not set. */ p->known_err_token = t; if (octal) { - RAISE_SYNTAX_ERROR("invalid octal escape sequence '\\%.3s'", - first_invalid_escape); + RAISE_SYNTAX_ERROR( + "\"\\%.3s\" is an invalid octal escape sequence. " + "Did you mean \"\\\\%.3s\"? A raw string is also an option.", + first_invalid_escape, first_invalid_escape); } else { - RAISE_SYNTAX_ERROR("invalid escape sequence '\\%c'", c); + RAISE_SYNTAX_ERROR( + "\"\\%c\" is an invalid escape sequence. " + "Did you mean \"\\\\%c\"? A raw string is also an option.", + c, c); } } Py_DECREF(msg); diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 9c9d05bbef0f1a..5a416adb875aa1 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -113,7 +113,10 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval } PyObject *msg = PyUnicode_FromFormat( - "invalid escape sequence '\\%c'", + "\"\\%c\" is an invalid escape sequence. " + "Such sequences will not work in the future. " + "Did you mean \"\\\\%c\"? A raw string is also an option.", + (char) first_invalid_escape_char, (char) first_invalid_escape_char ); @@ -129,7 +132,12 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval /* Replace the SyntaxWarning exception with a SyntaxError to get a more accurate error report */ PyErr_Clear(); - return _PyTokenizer_syntaxerror(tok, "invalid escape sequence '\\%c'", (char) first_invalid_escape_char); + + return _PyTokenizer_syntaxerror(tok, + "\"\\%c\" is an invalid escape sequence. " + "Did you mean \"\\\\%c\"? A raw string is also an option.", + (char) first_invalid_escape_char, + (char) first_invalid_escape_char); } return -1; From 080f444a58dae1bb76a87ce746a09fd499792cfb Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Wed, 15 Jan 2025 11:27:28 -0800 Subject: [PATCH 51/61] gh-128807: Add marking phase for free-threaded cyclic GC (gh-128808) --- Include/internal/pycore_gc.h | 16 +- ...-01-13-17-03-49.gh-issue-128807.BGGBxD.rst | 6 + Python/gc_free_threading.c | 336 +++++++++++++++++- 3 files changed, 336 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 4ff34bf8ead7d0..b1806df2706097 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -45,12 +45,13 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { * the per-object lock. */ #ifdef Py_GIL_DISABLED -# define _PyGC_BITS_TRACKED (1) // Tracked by the GC -# define _PyGC_BITS_FINALIZED (2) // tp_finalize was called -# define _PyGC_BITS_UNREACHABLE (4) -# define _PyGC_BITS_FROZEN (8) -# define _PyGC_BITS_SHARED (16) -# define _PyGC_BITS_DEFERRED (64) // Use deferred reference counting +# define _PyGC_BITS_TRACKED (1<<0) // Tracked by the GC +# define _PyGC_BITS_FINALIZED (1<<1) // tp_finalize was called +# define _PyGC_BITS_UNREACHABLE (1<<2) +# define _PyGC_BITS_FROZEN (1<<3) +# define _PyGC_BITS_SHARED (1<<4) +# define _PyGC_BITS_ALIVE (1<<5) // Reachable from a known root. +# define _PyGC_BITS_DEFERRED (1<<6) // Use deferred reference counting #endif #ifdef Py_GIL_DISABLED @@ -330,6 +331,9 @@ struct _gc_runtime_state { collections, and are awaiting to undergo a full collection for the first time. */ Py_ssize_t long_lived_pending; + + /* True if gc.freeze() has been used. */ + int freeze_active; #endif }; diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst new file mode 100644 index 00000000000000..34952e9abb66e5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst @@ -0,0 +1,6 @@ +Add a marking phase to the free-threaded GC. This is similar to what was +done in GH-126491. Since the free-threaded GC does not have generations and +is not incremental, the marking phase looks for all objects reachable from +known roots. The roots are objects known to not be garbage, like the module +dictionary for :mod:`sys`. For most programs, this marking phase should +make the GC a bit faster since typically less work is done per object. diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f7f44407494e51..d1023d9351086f 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -17,6 +17,17 @@ #include "pydtrace.h" #include "pycore_uniqueid.h" // _PyObject_MergeThreadLocalRefcounts() + +// enable the "mark alive" pass of GC +#define GC_ENABLE_MARK_ALIVE 1 + +// include additional roots in "mark alive" pass +#define GC_MARK_ALIVE_EXTRA_ROOTS 1 + +// include Python stacks as set of known roots +#define GC_MARK_ALIVE_STACKS 1 + + #ifdef Py_GIL_DISABLED typedef struct _gc_runtime_state GCState; @@ -113,28 +124,66 @@ worklist_remove(struct worklist_iter *iter) iter->next = iter->ptr; } +static inline int +gc_has_bit(PyObject *op, uint8_t bit) +{ + return (op->ob_gc_bits & bit) != 0; +} + +static inline void +gc_set_bit(PyObject *op, uint8_t bit) +{ + op->ob_gc_bits |= bit; +} + +static inline void +gc_clear_bit(PyObject *op, uint8_t bit) +{ + op->ob_gc_bits &= ~bit; +} + static inline int gc_is_frozen(PyObject *op) { - return (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0; + return gc_has_bit(op, _PyGC_BITS_FROZEN); } static inline int gc_is_unreachable(PyObject *op) { - return (op->ob_gc_bits & _PyGC_BITS_UNREACHABLE) != 0; + return gc_has_bit(op, _PyGC_BITS_UNREACHABLE); } -static void +static inline void gc_set_unreachable(PyObject *op) { - op->ob_gc_bits |= _PyGC_BITS_UNREACHABLE; + gc_set_bit(op, _PyGC_BITS_UNREACHABLE); } -static void +static inline void gc_clear_unreachable(PyObject *op) { - op->ob_gc_bits &= ~_PyGC_BITS_UNREACHABLE; + gc_clear_bit(op, _PyGC_BITS_UNREACHABLE); +} + +static inline int +gc_is_alive(PyObject *op) +{ + return gc_has_bit(op, _PyGC_BITS_ALIVE); +} + +#ifdef GC_ENABLE_MARK_ALIVE +static inline void +gc_set_alive(PyObject *op) +{ + gc_set_bit(op, _PyGC_BITS_ALIVE); +} +#endif + +static inline void +gc_clear_alive(PyObject *op) +{ + gc_clear_bit(op, _PyGC_BITS_ALIVE); } // Initialize the `ob_tid` field to zero if the object is not already @@ -143,6 +192,7 @@ static void gc_maybe_init_refs(PyObject *op) { if (!gc_is_unreachable(op)) { + assert(!gc_is_alive(op)); gc_set_unreachable(op); op->ob_tid = 0; } @@ -264,9 +314,13 @@ static void gc_restore_refs(PyObject *op) { if (gc_is_unreachable(op)) { + assert(!gc_is_alive(op)); gc_restore_tid(op); gc_clear_unreachable(op); } + else { + gc_clear_alive(op); + } } // Given a mimalloc memory block return the PyObject stored in it or NULL if @@ -392,6 +446,119 @@ gc_visit_thread_stacks(PyInterpreterState *interp) _Py_FOR_EACH_TSTATE_END(interp); } +// Untrack objects that can never create reference cycles. +// Return true if the object was untracked. +static bool +gc_maybe_untrack(PyObject *op) +{ + // Currently we only check for tuples containing only non-GC objects. In + // theory we could check other immutable objects that contain references + // to non-GC objects. + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + return true; + } + } + return false; +} + +#ifdef GC_ENABLE_MARK_ALIVE +static int +mark_alive_stack_push(PyObject *op, _PyObjectStack *stack) +{ + if (op == NULL) { + return 0; + } + if (!_PyObject_GC_IS_TRACKED(op)) { + return 0; + } + if (gc_is_alive(op)) { + return 0; // already visited this object + } + if (gc_maybe_untrack(op)) { + return 0; // was untracked, don't visit it + } + + // Need to call tp_traverse on this object. Add to stack and mark it + // alive so we don't traverse it a second time. + gc_set_alive(op); + if (_PyObjectStack_Push(stack, op) < 0) { + return -1; + } + return 0; +} + +static bool +gc_clear_alive_bits(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + if (gc_is_alive(op)) { + gc_clear_alive(op); + } + return true; +} + +static void +gc_abort_mark_alive(PyInterpreterState *interp, + struct collection_state *state, + _PyObjectStack *stack) +{ + // We failed to allocate memory for "stack" while doing the "mark + // alive" phase. In that case, free the object stack and make sure + // that no objects have the alive bit set. + _PyObjectStack_Clear(stack); + gc_visit_heaps(interp, &gc_clear_alive_bits, &state->base); +} + +#ifdef GC_MARK_ALIVE_STACKS +static int +gc_visit_stackref_mark_alive(_PyObjectStack *stack, _PyStackRef stackref) +{ + // Note: we MUST check that it is deferred before checking the rest. + // Otherwise we might read into invalid memory due to non-deferred references + // being dead already. + if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) { + PyObject *op = PyStackRef_AsPyObjectBorrow(stackref); + if (mark_alive_stack_push(op, stack) < 0) { + return -1; + } + } + return 0; +} + +static int +gc_visit_thread_stacks_mark_alive(PyInterpreterState *interp, _PyObjectStack *stack) +{ + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + for (_PyInterpreterFrame *f = p->current_frame; f != NULL; f = f->previous) { + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (executable == NULL || !PyCode_Check(executable)) { + continue; + } + + PyCodeObject *co = (PyCodeObject *)executable; + int max_stack = co->co_nlocalsplus + co->co_stacksize; + if (gc_visit_stackref_mark_alive(stack, f->f_executable) < 0) { + return -1; + } + for (int i = 0; i < max_stack; i++) { + if (gc_visit_stackref_mark_alive(stack, f->localsplus[i]) < 0) { + return -1; + } + } + } + } + _Py_FOR_EACH_TSTATE_END(interp); + return 0; +} +#endif // GC_MARK_ALIVE_STACKS +#endif // GC_ENABLE_MARK_ALIVE + static void queue_untracked_obj_decref(PyObject *op, struct collection_state *state) { @@ -460,7 +627,8 @@ visit_decref(PyObject *op, void *arg) { if (_PyObject_GC_IS_TRACKED(op) && !_Py_IsImmortal(op) - && !gc_is_frozen(op)) + && !gc_is_frozen(op) + && !gc_is_alive(op)) { // If update_refs hasn't reached this object yet, mark it // as (tentatively) unreachable and initialize ob_tid to zero. @@ -482,6 +650,10 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + if (gc_is_alive(op)) { + return true; + } + // Exclude immortal objects from garbage collection if (_Py_IsImmortal(op)) { op->ob_tid = 0; @@ -497,14 +669,9 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, _PyObject_ASSERT(op, refcount >= 0); if (refcount > 0 && !_PyObject_HasDeferredRefcount(op)) { - // Untrack tuples and dicts as necessary in this pass, but not objects - // with zero refcount, which we will want to collect. - if (PyTuple_CheckExact(op)) { - _PyTuple_MaybeUntrack(op); - if (!_PyObject_GC_IS_TRACKED(op)) { - gc_restore_refs(op); - return true; - } + if (gc_maybe_untrack(op)) { + gc_restore_refs(op); + return true; } } @@ -553,6 +720,21 @@ mark_reachable(PyObject *op) } #ifdef GC_DEBUG +static bool +validate_alive_bits(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT_WITH_MSG(op, !gc_is_alive(op), + "object should not be marked as alive yet"); + + return true; +} + static bool validate_refcounts(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) @@ -586,6 +768,11 @@ validate_gc_objects(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + if (gc_is_alive(op)) { + _PyObject_ASSERT(op, !gc_is_unreachable(op)); + return true; + } + _PyObject_ASSERT(op, gc_is_unreachable(op)); _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, "refcount is too small"); @@ -602,6 +789,10 @@ mark_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + if (gc_is_alive(op)) { + return true; + } + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, "refcount is too small"); @@ -630,6 +821,7 @@ restore_refs(const mi_heap_t *heap, const mi_heap_area_t *area, } gc_restore_tid(op); gc_clear_unreachable(op); + gc_clear_alive(op); return true; } @@ -679,6 +871,7 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, // object is reachable, restore `ob_tid`; we're done with these objects gc_restore_tid(op); + gc_clear_alive(op); state->long_lived_total++; return true; } @@ -686,6 +879,89 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, static int move_legacy_finalizer_reachable(struct collection_state *state); +#ifdef GC_ENABLE_MARK_ALIVE +static int +propagate_alive_bits(_PyObjectStack *stack) +{ + for (;;) { + PyObject *op = _PyObjectStack_Pop(stack); + if (op == NULL) { + break; + } + assert(_PyObject_GC_IS_TRACKED(op)); + assert(gc_is_alive(op)); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + if (traverse(op, (visitproc)&mark_alive_stack_push, stack) < 0) { + return -1; + } + } + return 0; +} + +// Using tp_traverse, mark everything reachable from known root objects +// (which must be non-garbage) as alive (_PyGC_BITS_ALIVE is set). In +// most programs, this marks nearly all objects that are not actually +// unreachable. +// +// Actually alive objects can be missed in this pass if they are alive +// due to being referenced from an unknown root (e.g. an extension +// module global), some tp_traverse methods are either missing or not +// accurate, or objects that have been untracked. Objects that are only +// reachable from the aforementioned are also missed. +// +// If gc.freeze() is used, this pass is disabled since it is unlikely to +// help much. The next stages of cyclic GC will ignore objects with the +// alive bit set. +// +// Returns -1 on failure (out of memory). +static int +mark_alive_from_roots(PyInterpreterState *interp, + struct collection_state *state) +{ +#ifdef GC_DEBUG + // Check that all objects don't have alive bit set + gc_visit_heaps(interp, &validate_alive_bits, &state->base); +#endif + _PyObjectStack stack = { NULL }; + + #define STACK_PUSH(op) \ + if (mark_alive_stack_push(op, &stack) < 0) { \ + gc_abort_mark_alive(interp, state, &stack); \ + return -1; \ + } + STACK_PUSH(interp->sysdict); +#ifdef GC_MARK_ALIVE_EXTRA_ROOTS + STACK_PUSH(interp->builtins); + STACK_PUSH(interp->dict); + struct types_state *types = &interp->types; + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + STACK_PUSH(types->builtins.initialized[i].tp_dict); + STACK_PUSH(types->builtins.initialized[i].tp_subclasses); + } + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + STACK_PUSH(types->for_extensions.initialized[i].tp_dict); + STACK_PUSH(types->for_extensions.initialized[i].tp_subclasses); + } +#endif +#ifdef GC_MARK_ALIVE_STACKS + if (gc_visit_thread_stacks_mark_alive(interp, &stack) < 0) { + gc_abort_mark_alive(interp, state, &stack); + return -1; + } +#endif + #undef STACK_PUSH + + // Use tp_traverse to find everything reachable from roots. + if (propagate_alive_bits(&stack) < 0) { + gc_abort_mark_alive(interp, state, &stack); + return -1; + } + + return 0; +} +#endif // GC_ENABLE_MARK_ALIVE + + static int deduce_unreachable_heap(PyInterpreterState *interp, struct collection_state *state) @@ -1245,6 +1521,25 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, process_delayed_frees(interp, state); + #ifdef GC_ENABLE_MARK_ALIVE + // If gc.freeze() was used, it seems likely that doing this "mark alive" + // pass will not be a performance win. Typically the majority of alive + // objects will be marked as frozen and will be skipped anyhow, without + // doing this extra work. Doing this pass also defeats one of the + // purposes of using freeze: avoiding writes to objects that are frozen. + // So, we just skip this if gc.freeze() was used. + if (!state->gcstate->freeze_active) { + // Mark objects reachable from known roots as "alive". These will + // be ignored for rest of the GC pass. + int err = mark_alive_from_roots(interp, state); + if (err < 0) { + _PyEval_StartTheWorld(interp); + PyErr_NoMemory(); + return; + } + } + #endif + // Find unreachable objects int err = deduce_unreachable_heap(interp, state); if (err < 0) { @@ -1253,6 +1548,11 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, return; } +#ifdef GC_DEBUG + // At this point, no object should have the alive bit set + gc_visit_heaps(interp, &validate_alive_bits, &state->base); +#endif + // Print debugging information. if (interp->gc.debug & _PyGC_DEBUG_COLLECTABLE) { PyObject *op; @@ -1564,6 +1864,8 @@ _PyGC_Freeze(PyInterpreterState *interp) { struct visitor_args args; _PyEval_StopTheWorld(interp); + GCState *gcstate = get_gc_state(); + gcstate->freeze_active = true; gc_visit_heaps(interp, &visit_freeze, &args); _PyEval_StartTheWorld(interp); } @@ -1574,7 +1876,7 @@ visit_unfreeze(const mi_heap_t *heap, const mi_heap_area_t *area, { PyObject *op = op_from_block(block, args, true); if (op != NULL) { - op->ob_gc_bits &= ~_PyGC_BITS_FROZEN; + gc_clear_bit(op, _PyGC_BITS_FROZEN); } return true; } @@ -1584,6 +1886,8 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { struct visitor_args args; _PyEval_StopTheWorld(interp); + GCState *gcstate = get_gc_state(); + gcstate->freeze_active = false; gc_visit_heaps(interp, &visit_unfreeze, &args); _PyEval_StartTheWorld(interp); } From 36c5e3bcc25700645d19eba65dcdf22acd99a7a8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Jan 2025 21:22:44 +0100 Subject: [PATCH 52/61] gh-128679: Redesign tracemalloc locking (#128888) * Use TABLES_LOCK() to protect 'tracemalloc_config.tracing'. * Hold TABLES_LOCK() longer while accessing tables. * tracemalloc_realloc() and tracemalloc_free() no longer remove the trace on reentrant call. * _PyTraceMalloc_Stop() unregisters _PyTraceMalloc_TraceRef(). * _PyTraceMalloc_GetTraces() sets the reentrant flag. * tracemalloc_clear_traces_unlocked() sets the reentrant flag. --- Include/internal/pycore_object.h | 6 - Include/internal/pycore_tracemalloc.h | 10 +- Modules/_tracemalloc.c | 16 +- Python/pylifecycle.c | 7 +- Python/tracemalloc.c | 611 ++++++++++++-------------- 5 files changed, 294 insertions(+), 356 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e26cb7673f939c..322305bc8c664a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -299,12 +299,6 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); extern int _PyType_CheckConsistency(PyTypeObject *type); extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); -/* Update the Python traceback of an object. This function must be called - when a memory block is reused from a free list. - - Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*); - // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 7ddc5bac5d10af..9fed393ac9e60e 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -25,7 +25,7 @@ struct _PyTraceMalloc_Config { } initialized; /* Is tracemalloc tracing memory allocations? - Variable protected by the GIL */ + Variable protected by the TABLES_LOCK(). */ int tracing; /* limit of the number of frames in a traceback, 1 by default. @@ -85,14 +85,14 @@ struct _tracemalloc_runtime_state { size_t peak_traced_memory; /* Hash table used as a set to intern filenames: PyObject* => PyObject*. - Protected by the GIL */ + Protected by the TABLES_LOCK(). */ _Py_hashtable_t *filenames; /* Buffer to store a new traceback in traceback_new(). - Protected by the GIL. */ + Protected by the TABLES_LOCK(). */ struct tracemalloc_traceback *traceback; /* Hash table used as a set to intern tracebacks: traceback_t* => traceback_t* - Protected by the GIL */ + Protected by the TABLES_LOCK(). */ _Py_hashtable_t *tracebacks; /* pointer (void*) => trace (trace_t*). Protected by TABLES_LOCK(). */ @@ -144,7 +144,7 @@ extern PyObject* _PyTraceMalloc_GetTraces(void); extern PyObject* _PyTraceMalloc_GetObjectTraceback(PyObject *obj); /* Initialize tracemalloc */ -extern int _PyTraceMalloc_Init(void); +extern PyStatus _PyTraceMalloc_Init(void); /* Start tracemalloc */ extern int _PyTraceMalloc_Start(int max_nframe); diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 887a1e820e250e..be71fc9fc9c341 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -215,18 +215,14 @@ static struct PyModuleDef module_def = { PyMODINIT_FUNC PyInit__tracemalloc(void) { - PyObject *m; - m = PyModule_Create(&module_def); - if (m == NULL) + PyObject *mod = PyModule_Create(&module_def); + if (mod == NULL) { return NULL; + } + #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif - if (_PyTraceMalloc_Init() < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return mod; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9a8a92aaceb0be..f1ecee6a92e5a1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -707,7 +707,12 @@ pycore_create_interpreter(_PyRuntimeState *runtime, // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. if (_PyMem_init_obmalloc(interp) < 0) { - return _PyStatus_NO_MEMORY(); + return _PyStatus_NO_MEMORY(); + } + + status = _PyTraceMalloc_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; } PyThreadState *tstate = _PyThreadState_New(interp, diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index f661d69c0312fa..68b641c51868b9 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -2,6 +2,7 @@ #include "pycore_fileutils.h" // _Py_write_noraise() #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() #include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() @@ -19,6 +20,8 @@ _Py_DECLARE_STR(anon_unknown, ""); /* Forward declaration */ static void* raw_malloc(size_t size); static void raw_free(void *ptr); +static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, + void* Py_UNUSED(ignore)); #ifdef Py_DEBUG # define TRACE_DEBUG @@ -106,16 +109,16 @@ tracemalloc_error(const char *format, ...) static int get_reentrant(void) { - void *ptr; - assert(PyThread_tss_is_created(&tracemalloc_reentrant_key)); - ptr = PyThread_tss_get(&tracemalloc_reentrant_key); + + void *ptr = PyThread_tss_get(&tracemalloc_reentrant_key); if (ptr != NULL) { assert(ptr == REENTRANT); return 1; } - else + else { return 0; + } } static void @@ -252,6 +255,7 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { assert(PyStackRef_CodeCheck(pyframe->f_executable)); frame->filename = &_Py_STR(anon_unknown); + int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); if (lineno < 0) { lineno = 0; @@ -259,7 +263,6 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) frame->lineno = (unsigned int)lineno; PyObject *filename = filename = _PyFrame_GetCode(pyframe)->co_filename; - if (filename == NULL) { #ifdef TRACE_DEBUG tracemalloc_error("failed to get the filename of the code object"); @@ -275,7 +278,7 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) } if (!PyUnicode_IS_READY(filename)) { /* Don't make a Unicode string ready to avoid reentrant calls - to tracemalloc_malloc() or tracemalloc_realloc() */ + to tracemalloc_alloc() or tracemalloc_realloc() */ #ifdef TRACE_DEBUG tracemalloc_error("filename is not a ready unicode string"); #endif @@ -309,7 +312,7 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) static Py_uhash_t traceback_hash(traceback_t *traceback) { - /* code based on tuplehash() of Objects/tupleobject.c */ + /* code based on tuple_hash() of Objects/tupleobject.c */ Py_uhash_t x, y; /* Unsigned for defined overflow behavior. */ int len = traceback->nframe; Py_uhash_t mult = PyHASH_MULTIPLIER; @@ -440,7 +443,7 @@ tracemalloc_get_traces_table(unsigned int domain) static void -tracemalloc_remove_trace(unsigned int domain, uintptr_t ptr) +tracemalloc_remove_trace_unlocked(unsigned int domain, uintptr_t ptr) { assert(tracemalloc_config.tracing); @@ -459,12 +462,12 @@ tracemalloc_remove_trace(unsigned int domain, uintptr_t ptr) } #define REMOVE_TRACE(ptr) \ - tracemalloc_remove_trace(DEFAULT_DOMAIN, (uintptr_t)(ptr)) + tracemalloc_remove_trace_unlocked(DEFAULT_DOMAIN, (uintptr_t)(ptr)) static int -tracemalloc_add_trace(unsigned int domain, uintptr_t ptr, - size_t size) +tracemalloc_add_trace_unlocked(unsigned int domain, uintptr_t ptr, + size_t size) { assert(tracemalloc_config.tracing); @@ -519,82 +522,138 @@ tracemalloc_add_trace(unsigned int domain, uintptr_t ptr, } #define ADD_TRACE(ptr, size) \ - tracemalloc_add_trace(DEFAULT_DOMAIN, (uintptr_t)(ptr), size) + tracemalloc_add_trace_unlocked(DEFAULT_DOMAIN, (uintptr_t)(ptr), size) static void* -tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) +tracemalloc_alloc(int need_gil, int use_calloc, + void *ctx, size_t nelem, size_t elsize) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - void *ptr; - assert(elsize == 0 || nelem <= SIZE_MAX / elsize); - if (use_calloc) + int reentrant = get_reentrant(); + + // Ignore reentrant call. + // + // For example, PyObjet_Malloc() calls + // PyMem_Malloc() for allocations larger than 512 bytes: don't trace the + // same memory allocation twice. + // + // If reentrant calls are not ignored, PyGILState_Ensure() can call + // PyMem_RawMalloc() which would call PyGILState_Ensure() again in a loop. + if (!reentrant) { + set_reentrant(1); + } + + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + void *ptr; + if (use_calloc) { ptr = alloc->calloc(alloc->ctx, nelem, elsize); - else + } + else { ptr = alloc->malloc(alloc->ctx, nelem * elsize); - if (ptr == NULL) - return NULL; + } + if (ptr == NULL) { + goto done; + } + if (reentrant) { + goto done; + } + + PyGILState_STATE gil_state; + if (need_gil) { + gil_state = PyGILState_Ensure(); + } TABLES_LOCK(); + if (ADD_TRACE(ptr, nelem * elsize) < 0) { - /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); + // Failed to allocate a trace for the new memory block alloc->free(alloc->ctx, ptr); - return NULL; + ptr = NULL; } + TABLES_UNLOCK(); + if (need_gil) { + PyGILState_Release(gil_state); + } + +done: + if (!reentrant) { + set_reentrant(0); + } return ptr; } static void* -tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) +tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) { + int reentrant = get_reentrant(); + + // Ignore reentrant call. PyObjet_Realloc() calls PyMem_Realloc() for + // allocations larger than 512 bytes: don't trace the same memory block + // twice. + if (!reentrant) { + set_reentrant(1); + } + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - void *ptr2; + void *ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - if (ptr2 == NULL) - return NULL; + if (ptr2 == NULL) { + goto done; + } + if (reentrant) { + goto done; + } - if (ptr != NULL) { - /* an existing memory block has been resized */ + PyGILState_STATE gil_state; + if (need_gil) { + gil_state = PyGILState_Ensure(); + } + TABLES_LOCK(); - TABLES_LOCK(); + if (ptr != NULL) { + // An existing memory block has been resized - /* tracemalloc_add_trace() updates the trace if there is already - a trace at address ptr2 */ + // tracemalloc_add_trace_unlocked() updates the trace if there is + // already a trace at address ptr2. if (ptr2 != ptr) { REMOVE_TRACE(ptr); } if (ADD_TRACE(ptr2, new_size) < 0) { - /* Memory allocation failed. The error cannot be reported to - the caller, because realloc() may already have shrunk the - memory block and so removed bytes. - - This case is very unlikely: a hash entry has just been - released, so the hash table should have at least one free entry. - - The GIL and the table lock ensures that only one thread is - allocating memory. */ + // Memory allocation failed. The error cannot be reported to the + // caller, because realloc() already have shrunk the memory block + // and so removed bytes. + // + // This case is very unlikely: a hash entry has just been released, + // so the hash table should have at least one free entry. + // + // The GIL and the table lock ensures that only one thread is + // allocating memory. Py_FatalError("tracemalloc_realloc() failed to allocate a trace"); } - TABLES_UNLOCK(); } else { - /* new allocation */ + // New allocation - TABLES_LOCK(); if (ADD_TRACE(ptr2, new_size) < 0) { - /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); + // Failed to allocate a trace for the new memory block alloc->free(alloc->ctx, ptr2); - return NULL; + ptr2 = NULL; } - TABLES_UNLOCK(); + } + + TABLES_UNLOCK(); + if (need_gil) { + PyGILState_Release(gil_state); + } + +done: + if (!reentrant) { + set_reentrant(0); } return ptr2; } @@ -603,168 +662,63 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) static void tracemalloc_free(void *ctx, void *ptr) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - - if (ptr == NULL) + if (ptr == NULL) { return; + } - /* GIL cannot be locked in PyMem_RawFree() because it would introduce - a deadlock in _PyThreadState_DeleteCurrent(). */ - + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; alloc->free(alloc->ctx, ptr); - TABLES_LOCK(); - REMOVE_TRACE(ptr); - TABLES_UNLOCK(); -} - - -static void* -tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize) -{ - void *ptr; - if (get_reentrant()) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - if (use_calloc) - return alloc->calloc(alloc->ctx, nelem, elsize); - else - return alloc->malloc(alloc->ctx, nelem * elsize); + return; } - /* Ignore reentrant call. PyObjet_Malloc() calls PyMem_Malloc() for - allocations larger than 512 bytes, don't trace the same memory - allocation twice. */ - set_reentrant(1); - - ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize); - - set_reentrant(0); - return ptr; + TABLES_LOCK(); + REMOVE_TRACE(ptr); + TABLES_UNLOCK(); } static void* tracemalloc_malloc_gil(void *ctx, size_t size) { - return tracemalloc_alloc_gil(0, ctx, 1, size); + return tracemalloc_alloc(0, 0, ctx, 1, size); } static void* tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize) { - return tracemalloc_alloc_gil(1, ctx, nelem, elsize); + return tracemalloc_alloc(0, 1, ctx, nelem, elsize); } static void* tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) { - void *ptr2; - - if (get_reentrant()) { - /* Reentrant call to PyMem_Realloc() and PyMem_RawRealloc(). - Example: PyMem_RawRealloc() is called internally by pymalloc - (_PyObject_Malloc() and _PyObject_Realloc()) to allocate a new - arena (new_arena()). */ - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - - ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - if (ptr2 != NULL && ptr != NULL) { - TABLES_LOCK(); - REMOVE_TRACE(ptr); - TABLES_UNLOCK(); - } - return ptr2; - } - - /* Ignore reentrant call. PyObjet_Realloc() calls PyMem_Realloc() for - allocations larger than 512 bytes. Don't trace the same memory - allocation twice. */ - set_reentrant(1); - - ptr2 = tracemalloc_realloc(ctx, ptr, new_size); - - set_reentrant(0); - return ptr2; + return tracemalloc_realloc(0, ctx, ptr, new_size); } #ifdef TRACE_RAW_MALLOC -static void* -tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) -{ - PyGILState_STATE gil_state; - void *ptr; - - if (get_reentrant()) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - if (use_calloc) - return alloc->calloc(alloc->ctx, nelem, elsize); - else - return alloc->malloc(alloc->ctx, nelem * elsize); - } - - /* Ignore reentrant call. PyGILState_Ensure() may call PyMem_RawMalloc() - indirectly which would call PyGILState_Ensure() if reentrant are not - disabled. */ - set_reentrant(1); - - gil_state = PyGILState_Ensure(); - ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize); - PyGILState_Release(gil_state); - - set_reentrant(0); - return ptr; -} - - static void* tracemalloc_raw_malloc(void *ctx, size_t size) { - return tracemalloc_raw_alloc(0, ctx, 1, size); + return tracemalloc_alloc(1, 0, ctx, 1, size); } static void* tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize) { - return tracemalloc_raw_alloc(1, ctx, nelem, elsize); + return tracemalloc_alloc(1, 1, ctx, nelem, elsize); } static void* tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) { - PyGILState_STATE gil_state; - void *ptr2; - - if (get_reentrant()) { - /* Reentrant call to PyMem_RawRealloc(). */ - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - - ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - - if (ptr2 != NULL && ptr != NULL) { - TABLES_LOCK(); - REMOVE_TRACE(ptr); - TABLES_UNLOCK(); - } - return ptr2; - } - - /* Ignore reentrant call. PyGILState_Ensure() may call PyMem_RawMalloc() - indirectly which would call PyGILState_Ensure() if reentrant calls are - not disabled. */ - set_reentrant(1); - - gil_state = PyGILState_Ensure(); - ptr2 = tracemalloc_realloc(ctx, ptr, new_size); - PyGILState_Release(gil_state); - - set_reentrant(0); - return ptr2; + return tracemalloc_realloc(1, ctx, ptr, new_size); } #endif /* TRACE_RAW_MALLOC */ @@ -777,48 +731,36 @@ tracemalloc_clear_filename(void *value) } -/* reentrant flag must be set to call this function and GIL must be held */ static void -tracemalloc_clear_traces(void) +tracemalloc_clear_traces_unlocked(void) { - /* The GIL protects variables against concurrent access */ + // Clearing tracemalloc_filenames requires the GIL to call Py_DECREF() assert(PyGILState_Check()); - TABLES_LOCK(); + set_reentrant(1); + _Py_hashtable_clear(tracemalloc_traces); _Py_hashtable_clear(tracemalloc_domains); + _Py_hashtable_clear(tracemalloc_tracebacks); + _Py_hashtable_clear(tracemalloc_filenames); + tracemalloc_traced_memory = 0; tracemalloc_peak_traced_memory = 0; - TABLES_UNLOCK(); - - _Py_hashtable_clear(tracemalloc_tracebacks); - _Py_hashtable_clear(tracemalloc_filenames); + set_reentrant(0); } -int +PyStatus _PyTraceMalloc_Init(void) { - if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { - PyErr_SetString(PyExc_RuntimeError, - "the tracemalloc module has been unloaded"); - return -1; - } - - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) - return 0; + assert(tracemalloc_config.initialized == TRACEMALLOC_NOT_INITIALIZED); PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); #ifdef REENTRANT_THREADLOCAL if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) { -#ifdef MS_WINDOWS - PyErr_SetFromWindowsErr(0); -#else - PyErr_SetFromErrno(PyExc_OSError); -#endif - return -1; + return _PyStatus_NO_MEMORY(); } #endif @@ -826,8 +768,7 @@ _PyTraceMalloc_Init(void) if (tables_lock == NULL) { tables_lock = PyThread_allocate_lock(); if (tables_lock == NULL) { - PyErr_SetString(PyExc_RuntimeError, "cannot allocate lock"); - return -1; + return _PyStatus_NO_MEMORY(); } } #endif @@ -844,9 +785,9 @@ _PyTraceMalloc_Init(void) tracemalloc_domains = tracemalloc_create_domains_table(); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL - || tracemalloc_traces == NULL || tracemalloc_domains == NULL) { - PyErr_NoMemory(); - return -1; + || tracemalloc_traces == NULL || tracemalloc_domains == NULL) + { + return _PyStatus_NO_MEMORY(); } tracemalloc_empty_traceback.nframe = 1; @@ -857,7 +798,7 @@ _PyTraceMalloc_Init(void) tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; - return 0; + return _PyStatus_OK(); } @@ -892,9 +833,6 @@ tracemalloc_deinit(void) int _PyTraceMalloc_Start(int max_nframe) { - PyMemAllocatorEx alloc; - size_t size; - if (max_nframe < 1 || (unsigned long) max_nframe > MAX_NFRAME) { PyErr_Format(PyExc_ValueError, "the number of frames must be in range [1; %lu]", @@ -902,23 +840,15 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } - if (_PyTraceMalloc_Init() < 0) { - return -1; - } - - if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { - return -1; - } - - if (tracemalloc_config.tracing) { - /* hook already installed: do nothing */ + if (_PyTraceMalloc_IsTracing()) { + /* hooks already installed: do nothing */ return 0; } tracemalloc_config.max_nframe = max_nframe; /* allocate a buffer to store a new traceback */ - size = TRACEBACK_SIZE(max_nframe); + size_t size = TRACEBACK_SIZE(max_nframe); assert(tracemalloc_traceback == NULL); tracemalloc_traceback = raw_malloc(size); if (tracemalloc_traceback == NULL) { @@ -926,6 +856,7 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } + PyMemAllocatorEx alloc; #ifdef TRACE_RAW_MALLOC alloc.malloc = tracemalloc_raw_malloc; alloc.calloc = tracemalloc_raw_calloc; @@ -950,8 +881,14 @@ _PyTraceMalloc_Start(int max_nframe) PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { + return -1; + } + /* everything is ready: start tracing Python memory allocations */ + TABLES_LOCK(); tracemalloc_config.tracing = 1; + TABLES_UNLOCK(); return 0; } @@ -960,8 +897,11 @@ _PyTraceMalloc_Start(int max_nframe) void _PyTraceMalloc_Stop(void) { - if (!tracemalloc_config.tracing) - return; + TABLES_LOCK(); + + if (!tracemalloc_config.tracing) { + goto done; + } /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; @@ -973,11 +913,16 @@ _PyTraceMalloc_Stop(void) PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); - tracemalloc_clear_traces(); + tracemalloc_clear_traces_unlocked(); /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; + + (void)PyRefTracer_SetTracer(NULL, NULL); + +done: + TABLES_UNLOCK(); } @@ -985,15 +930,16 @@ _PyTraceMalloc_Stop(void) static PyObject* frame_to_pyobject(frame_t *frame) { - PyObject *frame_obj, *lineno_obj; + assert(get_reentrant()); - frame_obj = PyTuple_New(2); - if (frame_obj == NULL) + PyObject *frame_obj = PyTuple_New(2); + if (frame_obj == NULL) { return NULL; + } PyTuple_SET_ITEM(frame_obj, 0, Py_NewRef(frame->filename)); - lineno_obj = PyLong_FromUnsignedLong(frame->lineno); + PyObject *lineno_obj = PyLong_FromUnsignedLong(frame->lineno); if (lineno_obj == NULL) { Py_DECREF(frame_obj); return NULL; @@ -1008,7 +954,6 @@ static PyObject* traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) { PyObject *frames; - if (intern_table != NULL) { frames = _Py_hashtable_get(intern_table, (const void *)traceback); if (frames) { @@ -1017,8 +962,9 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) } frames = PyTuple_New(traceback->nframe); - if (frames == NULL) + if (frames == NULL) { return NULL; + } for (int i=0; i < traceback->nframe; i++) { PyObject *frame = frame_to_pyobject(&traceback->frames[i]); @@ -1046,14 +992,14 @@ static PyObject* trace_to_pyobject(unsigned int domain, const trace_t *trace, _Py_hashtable_t *intern_tracebacks) { - PyObject *trace_obj = NULL; - PyObject *obj; + assert(get_reentrant()); - trace_obj = PyTuple_New(4); - if (trace_obj == NULL) + PyObject *trace_obj = PyTuple_New(4); + if (trace_obj == NULL) { return NULL; + } - obj = PyLong_FromSize_t(domain); + PyObject *obj = PyLong_FromSize_t(domain); if (obj == NULL) { Py_DECREF(trace_obj); return NULL; @@ -1100,7 +1046,6 @@ tracemalloc_copy_trace(_Py_hashtable_t *traces, void *user_data) { _Py_hashtable_t *traces2 = (_Py_hashtable_t *)user_data; - trace_t *trace = (trace_t *)value; trace_t *trace2 = raw_malloc(sizeof(trace_t)); @@ -1141,7 +1086,6 @@ tracemalloc_copy_domain(_Py_hashtable_t *domains, void *user_data) { _Py_hashtable_t *domains2 = (_Py_hashtable_t *)user_data; - unsigned int domain = (unsigned int)FROM_PTR(key); _Py_hashtable_t *traces = (_Py_hashtable_t *)value; @@ -1182,7 +1126,6 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, void *user_data) { get_traces_t *get_traces = user_data; - const trace_t *trace = (const trace_t *)value; PyObject *tuple = trace_to_pyobject(get_traces->domain, trace, @@ -1196,7 +1139,6 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, if (res < 0) { return 1; } - return 0; } @@ -1207,7 +1149,6 @@ tracemalloc_get_traces_domain(_Py_hashtable_t *domains, void *user_data) { get_traces_t *get_traces = user_data; - unsigned int domain = (unsigned int)FROM_PTR(key); _Py_hashtable_t *traces = (_Py_hashtable_t *)value; @@ -1227,27 +1168,21 @@ tracemalloc_pyobject_decref(void *value) static traceback_t* -tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr) +tracemalloc_get_traceback_unlocked(unsigned int domain, uintptr_t ptr) { - - if (!tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) { return NULL; + } - trace_t *trace; - TABLES_LOCK(); _Py_hashtable_t *traces = tracemalloc_get_traces_table(domain); - if (traces) { - trace = _Py_hashtable_get(traces, TO_PTR(ptr)); - } - else { - trace = NULL; + if (!traces) { + return NULL; } - TABLES_UNLOCK(); + trace_t *trace = _Py_hashtable_get(traces, TO_PTR(ptr)); if (!trace) { return NULL; } - return trace->traceback; } @@ -1269,24 +1204,28 @@ _PyMem_DumpFrame(int fd, frame_t * frame) void _PyMem_DumpTraceback(int fd, const void *ptr) { - traceback_t *traceback; - int i; - + TABLES_LOCK(); if (!tracemalloc_config.tracing) { PUTS(fd, "Enable tracemalloc to get the memory block " "allocation traceback\n\n"); - return; + goto done; } - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (uintptr_t)ptr); - if (traceback == NULL) - return; + traceback_t *traceback; + traceback = tracemalloc_get_traceback_unlocked(DEFAULT_DOMAIN, + (uintptr_t)ptr); + if (traceback == NULL) { + goto done; + } PUTS(fd, "Memory block allocated at (most recent call first):\n"); - for (i=0; i < traceback->nframe; i++) { + for (int i=0; i < traceback->nframe; i++) { _PyMem_DumpFrame(fd, &traceback->frames[i]); } PUTS(fd, "\n"); + +done: + TABLES_UNLOCK(); } #undef PUTS @@ -1307,38 +1246,42 @@ int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { - int res; - PyGILState_STATE gil_state; + PyGILState_STATE gil_state = PyGILState_Ensure(); + TABLES_LOCK(); - if (!tracemalloc_config.tracing) { + int result; + if (tracemalloc_config.tracing) { + result = tracemalloc_add_trace_unlocked(domain, ptr, size); + } + else { /* tracemalloc is not tracing: do nothing */ - return -2; + result = -2; } - gil_state = PyGILState_Ensure(); - - TABLES_LOCK(); - res = tracemalloc_add_trace(domain, ptr, size); TABLES_UNLOCK(); - PyGILState_Release(gil_state); - return res; + + return result; } int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { - if (!tracemalloc_config.tracing) { + TABLES_LOCK(); + + int result; + if (tracemalloc_config.tracing) { + tracemalloc_remove_trace_unlocked(domain, ptr); + result = 0; + } + else { /* tracemalloc is not tracing: do nothing */ - return -2; + result = -2; } - TABLES_LOCK(); - tracemalloc_remove_trace(domain, ptr); TABLES_UNLOCK(); - - return 0; + return result; } @@ -1355,87 +1298,101 @@ _PyTraceMalloc_Fini(void) Do nothing if tracemalloc is not tracing memory allocations or if the object memory block is not already traced. */ -int -_PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ignore)) +static int +_PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, + void* Py_UNUSED(ignore)) { if (event != PyRefTracer_CREATE) { return 0; } + if (get_reentrant()) { + return 0; + } assert(PyGILState_Check()); + TABLES_LOCK(); - if (!tracemalloc_config.tracing) { - /* tracemalloc is not tracing: do nothing */ - return -1; - } + int result = -1; + assert(tracemalloc_config.tracing); PyTypeObject *type = Py_TYPE(op); const size_t presize = _PyType_PreHeaderSize(type); uintptr_t ptr = (uintptr_t)((char *)op - presize); - int res = -1; - - TABLES_LOCK(); trace_t *trace = _Py_hashtable_get(tracemalloc_traces, TO_PTR(ptr)); if (trace != NULL) { /* update the traceback of the memory block */ traceback_t *traceback = traceback_new(); if (traceback != NULL) { trace->traceback = traceback; - res = 0; + result = 0; } } /* else: cannot track the object, its memory block size is unknown */ - TABLES_UNLOCK(); - return res; + TABLES_UNLOCK(); + return result; } PyObject* _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) { - traceback_t *traceback; + TABLES_LOCK(); - traceback = tracemalloc_get_traceback(domain, ptr); - if (traceback == NULL) - Py_RETURN_NONE; + traceback_t *traceback = tracemalloc_get_traceback_unlocked(domain, ptr); + PyObject *result; + if (traceback) { + set_reentrant(1); + result = traceback_to_pyobject(traceback, NULL); + set_reentrant(0); + } + else { + result = Py_NewRef(Py_None); + } - return traceback_to_pyobject(traceback, NULL); + TABLES_UNLOCK(); + return result; } int _PyTraceMalloc_IsTracing(void) { - return tracemalloc_config.tracing; + TABLES_LOCK(); + int tracing = tracemalloc_config.tracing; + TABLES_UNLOCK(); + return tracing; } void _PyTraceMalloc_ClearTraces(void) { - - if (!tracemalloc_config.tracing) { - return; + TABLES_LOCK(); + if (tracemalloc_config.tracing) { + tracemalloc_clear_traces_unlocked(); } - set_reentrant(1); - tracemalloc_clear_traces(); - set_reentrant(0); + TABLES_UNLOCK(); } PyObject * _PyTraceMalloc_GetTraces(void) { + TABLES_LOCK(); + set_reentrant(1); + get_traces_t get_traces; get_traces.domain = DEFAULT_DOMAIN; get_traces.traces = NULL; get_traces.domains = NULL; get_traces.tracebacks = NULL; get_traces.list = PyList_New(0); - if (get_traces.list == NULL) - goto error; + if (get_traces.list == NULL) { + goto finally; + } - if (!tracemalloc_config.tracing) - return get_traces.list; + if (!tracemalloc_config.tracing) { + goto finally; + } /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ @@ -1449,24 +1406,17 @@ _PyTraceMalloc_GetTraces(void) // Copy all traces so tracemalloc_get_traces_fill() doesn't have to disable // temporarily tracemalloc which would impact other threads and so would // miss allocations while get_traces() is called. - TABLES_LOCK(); get_traces.traces = tracemalloc_copy_traces(tracemalloc_traces); - TABLES_UNLOCK(); - if (get_traces.traces == NULL) { goto no_memory; } - TABLES_LOCK(); get_traces.domains = tracemalloc_copy_domains(tracemalloc_domains); - TABLES_UNLOCK(); - if (get_traces.domains == NULL) { goto no_memory; } // Convert traces to a list of tuples - set_reentrant(1); int err = _Py_hashtable_foreach(get_traces.traces, tracemalloc_get_traces_fill, &get_traces); @@ -1475,20 +1425,22 @@ _PyTraceMalloc_GetTraces(void) tracemalloc_get_traces_domain, &get_traces); } - set_reentrant(0); + if (err) { - goto error; + Py_CLEAR(get_traces.list); + goto finally; } - goto finally; no_memory: PyErr_NoMemory(); - -error: Py_CLEAR(get_traces.list); + goto finally; finally: + set_reentrant(0); + TABLES_UNLOCK(); + if (get_traces.tracebacks != NULL) { _Py_hashtable_destroy(get_traces.tracebacks); } @@ -1506,31 +1458,21 @@ PyObject * _PyTraceMalloc_GetObjectTraceback(PyObject *obj) /*[clinic end generated code: output=41ee0553a658b0aa input=29495f1b21c53212]*/ { - PyTypeObject *type; - traceback_t *traceback; - - type = Py_TYPE(obj); + PyTypeObject *type = Py_TYPE(obj); const size_t presize = _PyType_PreHeaderSize(type); uintptr_t ptr = (uintptr_t)((char *)obj - presize); - - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); - if (traceback == NULL) { - Py_RETURN_NONE; - } - - return traceback_to_pyobject(traceback, NULL); + return _PyTraceMalloc_GetTraceback(DEFAULT_DOMAIN, ptr); } -int _PyTraceMalloc_GetTracebackLimit(void) { +int _PyTraceMalloc_GetTracebackLimit(void) +{ return tracemalloc_config.max_nframe; } size_t -_PyTraceMalloc_GetMemory(void) { - - size_t size; - - size = _Py_hashtable_size(tracemalloc_tracebacks); +_PyTraceMalloc_GetMemory(void) +{ + size_t size = _Py_hashtable_size(tracemalloc_tracebacks); size += _Py_hashtable_size(tracemalloc_filenames); TABLES_LOCK(); @@ -1545,26 +1487,27 @@ _PyTraceMalloc_GetMemory(void) { PyObject * _PyTraceMalloc_GetTracedMemory(void) { - Py_ssize_t size, peak_size; - - if (!tracemalloc_config.tracing) - return Py_BuildValue("ii", 0, 0); - TABLES_LOCK(); - size = tracemalloc_traced_memory; - peak_size = tracemalloc_peak_traced_memory; + Py_ssize_t traced, peak; + if (tracemalloc_config.tracing) { + traced = tracemalloc_traced_memory; + peak = tracemalloc_peak_traced_memory; + } + else { + traced = 0; + peak = 0; + } TABLES_UNLOCK(); - return Py_BuildValue("nn", size, peak_size); + return Py_BuildValue("nn", traced, peak); } void _PyTraceMalloc_ResetPeak(void) { - if (!tracemalloc_config.tracing) { - return; - } TABLES_LOCK(); - tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + if (tracemalloc_config.tracing) { + tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + } TABLES_UNLOCK(); } From 256d6d2131541b3ff8f06f42c8157f808fde464c Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Wed, 15 Jan 2025 21:47:36 +0100 Subject: [PATCH 53/61] gh-97850: Suggest `TraversableResources` as the alternative for `ResourceLoader` (GH-128601) Suggest TraversableResources as the alternative for ResourceLoader. Previously, ResourceReader was the suggested alternative, but it is itself deprecated in favour of TraversableResources. --- Doc/library/importlib.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 9e088a598a6c08..b935fc0e42a4bd 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -380,13 +380,15 @@ ABC hierarchy:: .. class:: ResourceLoader + *Superseded by TraversableResources* + An abstract base class for a :term:`loader` which implements the optional :pep:`302` protocol for loading arbitrary resources from the storage back-end. .. deprecated:: 3.7 This ABC is deprecated in favour of supporting resource loading - through :class:`importlib.resources.abc.ResourceReader`. + through :class:`importlib.resources.abc.TraversableResources`. .. abstractmethod:: get_data(path) From 5eee2fe2b05c1a2836184e047fbd4176945cbf10 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:02:32 +0000 Subject: [PATCH 54/61] gh-128891: add specialized opcodes to opcode.opname (#128892) --- Lib/opcode.py | 5 +++-- Lib/test/test__opcode.py | 7 +++++++ Lib/test/test_dis.py | 6 ++++-- .../Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst | 1 + 4 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst diff --git a/Lib/opcode.py b/Lib/opcode.py index 974f4d35e2a524..aba66153af1b2e 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -17,8 +17,9 @@ EXTENDED_ARG = opmap['EXTENDED_ARG'] opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] -for op, i in opmap.items(): - opname[i] = op +for m in (opmap, _specialized_opmap): + for op, i in m.items(): + opname[i] = op cmp_op = ('<', '<=', '==', '!=', '>', '>=') diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index d5cf014d40daf8..95e09500df51d0 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -38,6 +38,13 @@ def test_is_valid(self): opcodes = [dis.opmap[opname] for opname in names] self.check_bool_function_result(_opcode.is_valid, opcodes, True) + def test_opmaps(self): + def check_roundtrip(name, map): + return self.assertEqual(opcode.opname[map[name]], name) + + check_roundtrip('BINARY_OP', opcode.opmap) + check_roundtrip('BINARY_OP_ADD_INT', opcode._specialized_opmap) + def test_oplists(self): def check_function(self, func, expected): for op in [-10, 520]: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ed8bd6fa20880b..8afe9653f19f6e 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -999,12 +999,14 @@ def test_boundaries(self): def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', 'INSTRUMENTED_CALL_FUNCTION_EX']) - for opcode, opname in enumerate(dis.opname): + for op, opname in enumerate(dis.opname): if opname in long_opcodes or opname.startswith("INSTRUMENTED"): continue + if opname in opcode._specialized_opmap: + continue with self.subTest(opname=opname): width = dis._OPNAME_WIDTH - if opcode in dis.hasarg: + if op in dis.hasarg: width += 1 + dis._OPARG_WIDTH self.assertLessEqual(len(opname), width) diff --git a/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst b/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst new file mode 100644 index 00000000000000..79d845bbab7cfc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst @@ -0,0 +1 @@ +Add specialized opcodes to ``opcode.opname``. From d05140f9f77d7dfc753dd1e5ac3a5962aaa03eff Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 15 Jan 2025 21:13:59 +0000 Subject: [PATCH 55/61] gh-121604: fix ResourceLoader deprecation warning message (GH-128859) --- Lib/importlib/abc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index bb2837d38d83f1..29f01f77eff4a0 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -74,7 +74,7 @@ def __init__(self): import warnings warnings.warn('importlib.abc.ResourceLoader is deprecated in ' 'favour of supporting resource loading through ' - 'importlib.resources.abc.ResourceReader.', + 'importlib.resources.abc.TraversableResources.', DeprecationWarning, stacklevel=2) super().__init__() From 313b96eb8b8d0ad3bac58d633822a0a3705ce60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Srinivas=20Reddy=20Thatiparthy=20=28=E0=B0=A4=E0=B0=BE?= =?UTF-8?q?=E0=B0=9F=E0=B0=BF=E0=B0=AA=E0=B0=B0=E0=B1=8D=E0=B0=A4=E0=B0=BF?= =?UTF-8?q?=20=E0=B0=B6=E0=B1=8D=E0=B0=B0=E0=B1=80=E0=B0=A8=E0=B0=BF?= =?UTF-8?q?=E0=B0=B5=E0=B0=BE=E0=B0=B8=E0=B1=8D=20=20=E0=B0=B0=E0=B1=86?= =?UTF-8?q?=E0=B0=A1=E0=B1=8D=E0=B0=A1=E0=B0=BF=29?= Date: Thu, 16 Jan 2025 15:47:03 +0530 Subject: [PATCH 56/61] gh-128017: Make a note that sys variables are read-only (#128887) --- Doc/library/sys.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index dd6293c722e7ad..151fd60532048a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -8,7 +8,7 @@ This module provides access to some variables used or maintained by the interpreter and to functions that interact strongly with the interpreter. It is -always available. +always available. Unless explicitly noted otherwise, all variables are read-only. .. data:: abiflags From 3193cb5ef87b77158d5c9bef99ae445cb82727de Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 16 Jan 2025 13:53:18 +0100 Subject: [PATCH 57/61] gh-128679: Fix tracemalloc.stop() race conditions (#128893) tracemalloc_alloc(), tracemalloc_realloc(), tracemalloc_free(), _PyTraceMalloc_TraceRef() and _PyTraceMalloc_GetMemory() now check 'tracemalloc_config.tracing' after calling TABLES_LOCK(). _PyTraceMalloc_TraceRef() now always returns 0. --- Lib/test/test_tracemalloc.py | 10 +- ...-01-15-21-41-51.gh-issue-128679.tq10F2.rst | 3 + Modules/_testcapimodule.c | 99 +++++++++++++++++++ Python/tracemalloc.c | 51 +++++++--- 4 files changed, 146 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-15-21-41-51.gh-issue-128679.tq10F2.rst diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 5755f7697de91a..da2db28775578a 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -7,8 +7,9 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, interpreter_requires_environment) from test import support -from test.support import os_helper from test.support import force_not_colorized +from test.support import os_helper +from test.support import threading_helper try: import _testcapi @@ -952,7 +953,6 @@ def check_env_var_invalid(self, nframe): return self.fail(f"unexpected output: {stderr!a}") - def test_env_var_invalid(self): for nframe in INVALID_NFRAME: with self.subTest(nframe=nframe): @@ -1101,6 +1101,12 @@ def test_stop_untrack(self): with self.assertRaises(RuntimeError): self.untrack() + @unittest.skipIf(_testcapi is None, 'need _testcapi') + @threading_helper.requires_working_threading() + def test_tracemalloc_track_race(self): + # gh-128679: Test fix for tracemalloc.stop() race condition + _testcapi.tracemalloc_track_race() + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-15-21-41-51.gh-issue-128679.tq10F2.rst b/Misc/NEWS.d/next/Library/2025-01-15-21-41-51.gh-issue-128679.tq10F2.rst new file mode 100644 index 00000000000000..5c108da5703c00 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-15-21-41-51.gh-issue-128679.tq10F2.rst @@ -0,0 +1,3 @@ +:mod:`tracemalloc`: Fix race conditions when :func:`tracemalloc.stop` is +called by a thread, while other threads are tracing memory allocations. +Patch by Victor Stinner. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a0a1f8af6710a3..7d304add5999d1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3435,6 +3435,104 @@ code_offset_to_line(PyObject* self, PyObject* const* args, Py_ssize_t nargsf) return PyLong_FromInt32(PyCode_Addr2Line(code, offset)); } + +static void +tracemalloc_track_race_thread(void *data) +{ + PyTraceMalloc_Track(123, 10, 1); + + PyThread_type_lock lock = (PyThread_type_lock)data; + PyThread_release_lock(lock); +} + +// gh-128679: Test fix for tracemalloc.stop() race condition +static PyObject * +tracemalloc_track_race(PyObject *self, PyObject *args) +{ +#define NTHREAD 50 + PyObject *tracemalloc = NULL; + PyObject *stop = NULL; + PyThread_type_lock locks[NTHREAD]; + memset(locks, 0, sizeof(locks)); + + // Call tracemalloc.start() + tracemalloc = PyImport_ImportModule("tracemalloc"); + if (tracemalloc == NULL) { + goto error; + } + PyObject *start = PyObject_GetAttrString(tracemalloc, "start"); + if (start == NULL) { + goto error; + } + PyObject *res = PyObject_CallNoArgs(start); + Py_DECREF(start); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + stop = PyObject_GetAttrString(tracemalloc, "stop"); + Py_CLEAR(tracemalloc); + if (stop == NULL) { + goto error; + } + + // Start threads + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = PyThread_allocate_lock(); + if (!lock) { + PyErr_NoMemory(); + goto error; + } + locks[i] = lock; + PyThread_acquire_lock(lock, 1); + + unsigned long thread; + thread = PyThread_start_new_thread(tracemalloc_track_race_thread, + (void*)lock); + if (thread == (unsigned long)-1) { + PyErr_SetString(PyExc_RuntimeError, "can't start new thread"); + goto error; + } + } + + // Call tracemalloc.stop() while threads are running + res = PyObject_CallNoArgs(stop); + Py_CLEAR(stop); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + // Wait until threads complete with the GIL released + Py_BEGIN_ALLOW_THREADS + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_acquire_lock(lock, 1); + PyThread_release_lock(lock); + } + Py_END_ALLOW_THREADS + + // Free threads locks + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_free_lock(lock); + } + Py_RETURN_NONE; + +error: + Py_CLEAR(tracemalloc); + Py_CLEAR(stop); + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + if (lock) { + PyThread_free_lock(lock); + } + } + return NULL; +#undef NTHREAD +} + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3578,6 +3676,7 @@ static PyMethodDef TestMethods[] = { {"type_freeze", type_freeze, METH_VARARGS}, {"test_atexit", test_atexit, METH_NOARGS}, {"code_offset_to_line", _PyCFunction_CAST(code_offset_to_line), METH_FASTCALL}, + {"tracemalloc_track_race", tracemalloc_track_race, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 68b641c51868b9..919c564ee72967 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -567,11 +567,14 @@ tracemalloc_alloc(int need_gil, int use_calloc, } TABLES_LOCK(); - if (ADD_TRACE(ptr, nelem * elsize) < 0) { - // Failed to allocate a trace for the new memory block - alloc->free(alloc->ctx, ptr); - ptr = NULL; + if (tracemalloc_config.tracing) { + if (ADD_TRACE(ptr, nelem * elsize) < 0) { + // Failed to allocate a trace for the new memory block + alloc->free(alloc->ctx, ptr); + ptr = NULL; + } } + // else: gh-128679: tracemalloc.stop() was called by another thread TABLES_UNLOCK(); if (need_gil) { @@ -614,6 +617,11 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) } TABLES_LOCK(); + if (!tracemalloc_config.tracing) { + // gh-128679: tracemalloc.stop() was called by another thread + goto unlock; + } + if (ptr != NULL) { // An existing memory block has been resized @@ -646,6 +654,7 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) } } +unlock: TABLES_UNLOCK(); if (need_gil) { PyGILState_Release(gil_state); @@ -674,7 +683,12 @@ tracemalloc_free(void *ctx, void *ptr) } TABLES_LOCK(); - REMOVE_TRACE(ptr); + + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } + // else: gh-128679: tracemalloc.stop() was called by another thread + TABLES_UNLOCK(); } @@ -1312,8 +1326,9 @@ _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, assert(PyGILState_Check()); TABLES_LOCK(); - int result = -1; - assert(tracemalloc_config.tracing); + if (!tracemalloc_config.tracing) { + goto done; + } PyTypeObject *type = Py_TYPE(op); const size_t presize = _PyType_PreHeaderSize(type); @@ -1325,13 +1340,13 @@ _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, traceback_t *traceback = traceback_new(); if (traceback != NULL) { trace->traceback = traceback; - result = 0; } } /* else: cannot track the object, its memory block size is unknown */ +done: TABLES_UNLOCK(); - return result; + return 0; } @@ -1472,13 +1487,19 @@ int _PyTraceMalloc_GetTracebackLimit(void) size_t _PyTraceMalloc_GetMemory(void) { - size_t size = _Py_hashtable_size(tracemalloc_tracebacks); - size += _Py_hashtable_size(tracemalloc_filenames); - TABLES_LOCK(); - size += _Py_hashtable_size(tracemalloc_traces); - _Py_hashtable_foreach(tracemalloc_domains, - tracemalloc_get_tracemalloc_memory_cb, &size); + size_t size; + if (tracemalloc_config.tracing) { + size = _Py_hashtable_size(tracemalloc_tracebacks); + size += _Py_hashtable_size(tracemalloc_filenames); + + size += _Py_hashtable_size(tracemalloc_traces); + _Py_hashtable_foreach(tracemalloc_domains, + tracemalloc_get_tracemalloc_memory_cb, &size); + } + else { + size = 0; + } TABLES_UNLOCK(); return size; } From e81fe940c9bd092f6de558fa965100502b78da0f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:15:52 +0000 Subject: [PATCH 58/61] gh-119786: added InternalDocs/generators.md (#128524) --- InternalDocs/README.md | 2 +- InternalDocs/generators.md | 108 +++++++++++++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 6 deletions(-) diff --git a/InternalDocs/README.md b/InternalDocs/README.md index 794b4f3c6aad42..4502902307cd5c 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -25,7 +25,7 @@ Runtime Objects - [Code Objects](code_objects.md) -- [Generators (coming soon)](generators.md) +- [Generators](generators.md) - [Frames](frames.md) diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index afa8b8f4bb8040..608bd215aae65a 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -1,8 +1,106 @@ + +Generators and Coroutines +========================= + Generators -========== +---------- + +Generators in CPython are implemented with the struct `PyGenObject`. +They consist of a [`frame`](frames.md) and metadata about the generator's +execution state. + +A generator object resumes execution in its frame when its `send()` +method is called. This is analogous to a function executing in its own +fram when it is called, but a function returns to the calling frame only once, +while a generator "returns" execution to the caller's frame every time +it emits a new item with a +[`yield` expression](https://docs.python.org/dev/reference/expressions.html#yield-expressions). +This is implemented by the +[`YIELD_VALUE`](https://docs.python.org/dev/library/dis.html#opcode-YIELD_VALUE) +bytecode, which is similar to +[`RETURN_VALUE`](https://docs.python.org/dev/library/dis.html#opcode-RETURN_VALUE) +in the sense that it puts a value on the stack and returns execution to the +calling frame, but it also needs to perform additional work to leave the generator +frame in a state that allows it to be resumed. In particular, it updates the frame's +instruction pointer and stores the interpreter's exception state on the generator +object. When the generator is resumed, this exception state is copied back to the +interpreter state. + +The `frame` of a generator is embedded in the generator object struct as a +[`_PyInterpreterFrame`](frames.md) (see `_PyGenObject_HEAD` in +[`pycore_genobject.h`](../Include/internal/pycore_genobject.h)). +This means that we can get the frame from the generator or the generator +from the frame (see `_PyGen_GetGeneratorFromFrame` in the same file). +Other fields of the generator struct include metadata (such as the name of +the generator function) and runtime state information (such as whether its +frame is executing, suspended, cleared, etc.). + +Generator Object Creation and Destruction +----------------------------------------- + +The bytecode of a generator function begins with a +[`RETURN_GENERATOR`](https://docs.python.org/dev/library/dis.html#opcode-RETURN_GENERATOR) +instruction, which creates a generator object, including its embedded frame. +The generator's frame is initialized as a copy of the frame in which +`RETURN_GENERATOR` is executing, but its `owner` field is overwritten to indicate +that it is owned by a generator. Finally, `RETURN_GENERATOR` pushes the new generator +object to the stack and returns to the caller of the generator function (at +which time its frame is destroyed). When the generator is next resumed by +[`gen_send_ex2()`](../Objects/genobject.c), `_PyEval_EvalFrame()` is called +to continue executing the generator function, in the frame that is embedded in +the generator object. + +When a generator object is destroyed in [`gen_dealloc`](../Objects/genobject.c), +its embedded `_PyInterpreterFrame` field may need to be preserved, if it is exposed +to Python as part of a [`PyFrameObject`](frames.md#frame-objects). This is detected +in [`_PyFrame_ClearExceptCode`](../Python/frame.c) by the fact that the interpreter +frame's `frame_obj` field is set, and the frame object it points to has refcount +greater than 1. If so, the `take_ownership()` function is called to create a new +copy of the interpreter frame and transfer ownership of it from the generator to +the frame object. + +Iteration +--------- + +The [`FOR_ITER`](https://docs.python.org/dev/library/dis.html#opcode-FOR_ITER) +instruction calls `__next__` on the iterator which is on the top of the stack, +and pushes the result to the stack. It has [`specializations`](adaptive.md) +for a few common iterator types, including `FOR_ITER_GEN`, for iterating over +a generator. `FOR_ITER_GEN` bypasses the call to `__next__`, and instead +directly pushes the generator stack and resumes its execution from the +instruction that follows the last yield. + +Chained Generators +------------------ + +A `yield from` expression creates a generator that efficiently yields the +sequence created by another generator. This is implemented with the +[`SEND` instruction](https://docs.python.org/dev/library/dis.html#opcode-SEND), +which pushes the value of its arg to the stack of the generator's frame, sets +the exception state on this frame, and resumes execution of the chained generator. +On return from `SEND`, the value at the top of the stack is sent back up +the generator chain with a `YIELD_VALUE`. This sequence of `SEND` followed by +`YIELD_VALUE` is repeated in a loop, until a `StopIteration` exception is +raised to indicate that the generator has no more values to emit. + +The [`CLEANUP_THROW`](https://docs.python.org/dev/library/dis.html#opcode-CLEANUP_THROW) +instruction is used to handle exceptions raised from the send-yield loop. +Exceptions of type `StopIteration` is handled, their `value` field hold the +value to be returned by the generator's `close()` function. Any other +exception is re-raised by `CLEANUP_THROW`. + +Coroutines +---------- -Coming soon. +Coroutines are generators that use the value returned from a `yield` expression, +i.e., the argument that was passed to the `.send()` call that resumed it after +it yielded. This makes it possible for data to flow in both directions: from +the generator to the caller via the argument of the `yield` expression, and +from the caller to the generator via the send argument to the `send()` call. +A `yield from` expression passes the `send` argument to the chained generator, +so this data flow works along the chain (see `gen_send_ex2()` in +[`genobject.c`](../Objects/genobject.c)). - +Recall that a generator's `__next__` function simply calls `self.send(None)`, +so all this works the same in generators and coroutines, but only coroutines +use the value of the argument to `send`. From 3893a92d956363fa2443bc5e47d4bae3deddacef Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Jan 2025 15:22:13 +0000 Subject: [PATCH 59/61] gh-100239: specialize long tail of binary operations (#128722) --- Include/internal/pycore_code.h | 13 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 45 ++- Include/internal/pycore_uop_ids.h | 370 +++++++++--------- Include/internal/pycore_uop_metadata.h | 8 + Include/opcode_ids.h | 153 ++++---- Lib/_opcode_metadata.py | 154 ++++---- Lib/opcode.py | 1 + Lib/test/test_code.py | 10 +- Lib/test/test_dis.py | 182 ++++----- Lib/test/test_monitoring.py | 2 +- Lib/test/test_opcache.py | 25 ++ ...-01-10-23-54-16.gh-issue-100239.ijOOUs.rst | 2 + Python/bytecodes.c | 45 ++- Python/executor_cases.c.h | 45 +++ Python/generated_cases.c.h | 99 +++-- Python/opcode_targets.h | 2 +- Python/optimizer_cases.c.h | 13 + Python/specialize.c | 109 +++++- Tools/c-analyzer/cpython/ignored.tsv | 2 + Tools/cases_generator/parsing.py | 5 +- 21 files changed, 796 insertions(+), 492 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-23-54-16.gh-issue-100239.ijOOUs.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index d97fe81a2fc54a..01d41446fdb0cf 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -100,6 +100,7 @@ typedef struct { typedef struct { _Py_BackoffCounter counter; + uint16_t external_cache[4]; } _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) @@ -438,7 +439,7 @@ write_u64(uint16_t *p, uint64_t val) } static inline void -write_obj(uint16_t *p, PyObject *val) +write_ptr(uint16_t *p, void *val) { memcpy(p, &val, sizeof(val)); } @@ -576,6 +577,16 @@ adaptive_counter_backoff(_Py_BackoffCounter counter) { return restart_backoff_counter(counter); } +/* Specialization Extensions */ + +/* callbacks for an external specialization */ +typedef int (*binaryopguardfunc)(PyObject *lhs, PyObject *rhs); +typedef PyObject *(*binaryopactionfunc)(PyObject *lhs, PyObject *rhs); + +typedef struct { + binaryopguardfunc guard; + binaryopactionfunc action; +} _PyBinaryOpSpecializationDescr; /* Comparison bit masks. */ diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 8b3d6285c1e4e7..1dd155abf3babf 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -266,6 +266,7 @@ Known values: Python 3.14a4 3611 (Add NOT_TAKEN instruction) Python 3.14a4 3612 (Add POP_ITER and INSTRUMENTED_POP_ITER) Python 3.14a4 3613 (Add LOAD_CONST_MORTAL instruction) + Python 3.14a5 3614 (Add BINARY_OP_EXTEND) Python 3.15 will start with 3650 @@ -278,7 +279,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3613 +#define PYC_MAGIC_NUMBER 3614 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0c0a6145bdbb27..f7c23ad634d6be 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -43,6 +43,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_OP_ADD_UNICODE: return 2; + case BINARY_OP_EXTEND: + return 2; case BINARY_OP_INPLACE_ADD_UNICODE: return 2; case BINARY_OP_MULTIPLY_FLOAT: @@ -512,6 +514,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case BINARY_OP_ADD_UNICODE: return 1; + case BINARY_OP_EXTEND: + return 1; case BINARY_OP_INPLACE_ADD_UNICODE: return 0; case BINARY_OP_MULTIPLY_FLOAT: @@ -989,6 +993,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = 0; return 0; } + case BINARY_OP_EXTEND: { + *effect = 0; + return 0; + } case BINARY_OP_INPLACE_ADD_UNICODE: { *effect = 0; return 0; @@ -1919,11 +1927,13 @@ enum InstructionFormat { INSTR_FMT_IBC = 2, INSTR_FMT_IBC00 = 3, INSTR_FMT_IBC000 = 4, - INSTR_FMT_IBC00000000 = 5, - INSTR_FMT_IX = 6, - INSTR_FMT_IXC = 7, - INSTR_FMT_IXC00 = 8, - INSTR_FMT_IXC000 = 9, + INSTR_FMT_IBC0000 = 5, + INSTR_FMT_IBC00000000 = 6, + INSTR_FMT_IX = 7, + INSTR_FMT_IXC = 8, + INSTR_FMT_IXC00 = 9, + INSTR_FMT_IXC000 = 10, + INSTR_FMT_IXC0000 = 11, }; #define IS_VALID_OPCODE(OP) \ @@ -1981,15 +1991,16 @@ struct opcode_metadata { extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { - [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2227,6 +2238,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, + [BINARY_OP_EXTEND] = { .nuops = 2, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 } } }, [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, 0, 0 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, @@ -2411,6 +2423,7 @@ const char *_PyOpcode_OpName[266] = { [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_EXTEND] = "BINARY_OP_EXTEND", [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", @@ -2661,7 +2674,7 @@ const uint8_t _PyOpcode_Caches[256] = { [FOR_ITER] = 1, [CALL] = 3, [CALL_KW] = 3, - [BINARY_OP] = 1, + [BINARY_OP] = 5, }; #endif @@ -2672,6 +2685,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_EXTEND] = BINARY_OP, [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, [BINARY_OP_MULTIPLY_INT] = BINARY_OP, @@ -2923,7 +2937,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 229: \ case 230: \ case 231: \ case 232: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3841363b411eed..f95defbc364aed 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -15,16 +15,17 @@ extern "C" { #define _BINARY_OP_ADD_FLOAT 303 #define _BINARY_OP_ADD_INT 304 #define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_INPLACE_ADD_UNICODE 306 -#define _BINARY_OP_MULTIPLY_FLOAT 307 -#define _BINARY_OP_MULTIPLY_INT 308 -#define _BINARY_OP_SUBTRACT_FLOAT 309 -#define _BINARY_OP_SUBTRACT_INT 310 -#define _BINARY_SLICE 311 -#define _BINARY_SUBSCR 312 -#define _BINARY_SUBSCR_CHECK_FUNC 313 +#define _BINARY_OP_EXTEND 306 +#define _BINARY_OP_INPLACE_ADD_UNICODE 307 +#define _BINARY_OP_MULTIPLY_FLOAT 308 +#define _BINARY_OP_MULTIPLY_INT 309 +#define _BINARY_OP_SUBTRACT_FLOAT 310 +#define _BINARY_OP_SUBTRACT_INT 311 +#define _BINARY_SLICE 312 +#define _BINARY_SUBSCR 313 +#define _BINARY_SUBSCR_CHECK_FUNC 314 #define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT -#define _BINARY_SUBSCR_INIT_CALL 314 +#define _BINARY_SUBSCR_INIT_CALL 315 #define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT #define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT #define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT @@ -34,122 +35,123 @@ extern "C" { #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 315 -#define _CALL_BUILTIN_FAST 316 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 317 -#define _CALL_BUILTIN_O 318 +#define _CALL_BUILTIN_CLASS 316 +#define _CALL_BUILTIN_FAST 317 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 318 +#define _CALL_BUILTIN_O 319 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_KW_NON_PY 319 +#define _CALL_KW_NON_PY 320 #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 320 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 321 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 322 -#define _CALL_METHOD_DESCRIPTOR_O 323 -#define _CALL_NON_PY_GENERAL 324 -#define _CALL_STR_1 325 -#define _CALL_TUPLE_1 326 +#define _CALL_METHOD_DESCRIPTOR_FAST 321 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 322 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 323 +#define _CALL_METHOD_DESCRIPTOR_O 324 +#define _CALL_NON_PY_GENERAL 325 +#define _CALL_STR_1 326 +#define _CALL_TUPLE_1 327 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_AND_ALLOCATE_OBJECT 327 -#define _CHECK_ATTR_CLASS 328 -#define _CHECK_ATTR_METHOD_LAZY_DICT 329 -#define _CHECK_ATTR_MODULE_PUSH_KEYS 330 -#define _CHECK_ATTR_WITH_HINT 331 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 332 +#define _CHECK_AND_ALLOCATE_OBJECT 328 +#define _CHECK_ATTR_CLASS 329 +#define _CHECK_ATTR_METHOD_LAZY_DICT 330 +#define _CHECK_ATTR_MODULE_PUSH_KEYS 331 +#define _CHECK_ATTR_WITH_HINT 332 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 333 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 333 -#define _CHECK_FUNCTION_EXACT_ARGS 334 -#define _CHECK_FUNCTION_VERSION 335 -#define _CHECK_FUNCTION_VERSION_INLINE 336 -#define _CHECK_FUNCTION_VERSION_KW 337 -#define _CHECK_IS_NOT_PY_CALLABLE 338 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 339 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 340 -#define _CHECK_METHOD_VERSION 341 -#define _CHECK_METHOD_VERSION_KW 342 -#define _CHECK_PEP_523 343 -#define _CHECK_PERIODIC 344 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 345 -#define _CHECK_STACK_SPACE 346 -#define _CHECK_STACK_SPACE_OPERAND 347 -#define _CHECK_VALIDITY 348 -#define _CHECK_VALIDITY_AND_SET_IP 349 -#define _COMPARE_OP 350 -#define _COMPARE_OP_FLOAT 351 -#define _COMPARE_OP_INT 352 -#define _COMPARE_OP_STR 353 -#define _CONTAINS_OP 354 +#define _CHECK_FUNCTION 334 +#define _CHECK_FUNCTION_EXACT_ARGS 335 +#define _CHECK_FUNCTION_VERSION 336 +#define _CHECK_FUNCTION_VERSION_INLINE 337 +#define _CHECK_FUNCTION_VERSION_KW 338 +#define _CHECK_IS_NOT_PY_CALLABLE 339 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 340 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 341 +#define _CHECK_METHOD_VERSION 342 +#define _CHECK_METHOD_VERSION_KW 343 +#define _CHECK_PEP_523 344 +#define _CHECK_PERIODIC 345 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 346 +#define _CHECK_STACK_SPACE 347 +#define _CHECK_STACK_SPACE_OPERAND 348 +#define _CHECK_VALIDITY 349 +#define _CHECK_VALIDITY_AND_SET_IP 350 +#define _COMPARE_OP 351 +#define _COMPARE_OP_FLOAT 352 +#define _COMPARE_OP_INT 353 +#define _COMPARE_OP_STR 354 +#define _CONTAINS_OP 355 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 355 +#define _CREATE_INIT_FRAME 356 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 356 +#define _DEOPT 357 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 357 -#define _DO_CALL_FUNCTION_EX 358 -#define _DO_CALL_KW 359 -#define _DYNAMIC_EXIT 360 +#define _DO_CALL 358 +#define _DO_CALL_FUNCTION_EX 359 +#define _DO_CALL_KW 360 +#define _DYNAMIC_EXIT 361 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 361 +#define _ERROR_POP_N 362 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 362 -#define _EXPAND_METHOD_KW 363 -#define _FATAL_ERROR 364 +#define _EXPAND_METHOD 363 +#define _EXPAND_METHOD_KW 364 +#define _FATAL_ERROR 365 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 365 -#define _FOR_ITER_GEN_FRAME 366 -#define _FOR_ITER_TIER_TWO 367 +#define _FOR_ITER 366 +#define _FOR_ITER_GEN_FRAME 367 +#define _FOR_ITER_TIER_TWO 368 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 368 -#define _GUARD_BOTH_INT 369 -#define _GUARD_BOTH_UNICODE 370 -#define _GUARD_BUILTINS_VERSION_PUSH_KEYS 371 -#define _GUARD_DORV_NO_DICT 372 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 373 -#define _GUARD_GLOBALS_VERSION 374 -#define _GUARD_GLOBALS_VERSION_PUSH_KEYS 375 -#define _GUARD_IS_FALSE_POP 376 -#define _GUARD_IS_NONE_POP 377 -#define _GUARD_IS_NOT_NONE_POP 378 -#define _GUARD_IS_TRUE_POP 379 -#define _GUARD_KEYS_VERSION 380 -#define _GUARD_NOS_FLOAT 381 -#define _GUARD_NOS_INT 382 -#define _GUARD_NOT_EXHAUSTED_LIST 383 -#define _GUARD_NOT_EXHAUSTED_RANGE 384 -#define _GUARD_NOT_EXHAUSTED_TUPLE 385 -#define _GUARD_TOS_FLOAT 386 -#define _GUARD_TOS_INT 387 -#define _GUARD_TYPE_VERSION 388 -#define _GUARD_TYPE_VERSION_AND_LOCK 389 +#define _GUARD_BINARY_OP_EXTEND 369 +#define _GUARD_BOTH_FLOAT 370 +#define _GUARD_BOTH_INT 371 +#define _GUARD_BOTH_UNICODE 372 +#define _GUARD_BUILTINS_VERSION_PUSH_KEYS 373 +#define _GUARD_DORV_NO_DICT 374 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 375 +#define _GUARD_GLOBALS_VERSION 376 +#define _GUARD_GLOBALS_VERSION_PUSH_KEYS 377 +#define _GUARD_IS_FALSE_POP 378 +#define _GUARD_IS_NONE_POP 379 +#define _GUARD_IS_NOT_NONE_POP 380 +#define _GUARD_IS_TRUE_POP 381 +#define _GUARD_KEYS_VERSION 382 +#define _GUARD_NOS_FLOAT 383 +#define _GUARD_NOS_INT 384 +#define _GUARD_NOT_EXHAUSTED_LIST 385 +#define _GUARD_NOT_EXHAUSTED_RANGE 386 +#define _GUARD_NOT_EXHAUSTED_TUPLE 387 +#define _GUARD_TOS_FLOAT 388 +#define _GUARD_TOS_INT 389 +#define _GUARD_TYPE_VERSION 390 +#define _GUARD_TYPE_VERSION_AND_LOCK 391 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 390 -#define _INIT_CALL_PY_EXACT_ARGS 391 -#define _INIT_CALL_PY_EXACT_ARGS_0 392 -#define _INIT_CALL_PY_EXACT_ARGS_1 393 -#define _INIT_CALL_PY_EXACT_ARGS_2 394 -#define _INIT_CALL_PY_EXACT_ARGS_3 395 -#define _INIT_CALL_PY_EXACT_ARGS_4 396 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 392 +#define _INIT_CALL_PY_EXACT_ARGS 393 +#define _INIT_CALL_PY_EXACT_ARGS_0 394 +#define _INIT_CALL_PY_EXACT_ARGS_1 395 +#define _INIT_CALL_PY_EXACT_ARGS_2 396 +#define _INIT_CALL_PY_EXACT_ARGS_3 397 +#define _INIT_CALL_PY_EXACT_ARGS_4 398 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -162,144 +164,144 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 397 -#define _IS_NONE 398 +#define _INTERNAL_INCREMENT_OPT_COUNTER 399 +#define _IS_NONE 400 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 399 -#define _ITER_CHECK_RANGE 400 -#define _ITER_CHECK_TUPLE 401 -#define _ITER_JUMP_LIST 402 -#define _ITER_JUMP_RANGE 403 -#define _ITER_JUMP_TUPLE 404 -#define _ITER_NEXT_LIST 405 -#define _ITER_NEXT_RANGE 406 -#define _ITER_NEXT_TUPLE 407 -#define _JUMP_TO_TOP 408 +#define _ITER_CHECK_LIST 401 +#define _ITER_CHECK_RANGE 402 +#define _ITER_CHECK_TUPLE 403 +#define _ITER_JUMP_LIST 404 +#define _ITER_JUMP_RANGE 405 +#define _ITER_JUMP_TUPLE 406 +#define _ITER_NEXT_LIST 407 +#define _ITER_NEXT_RANGE 408 +#define _ITER_NEXT_TUPLE 409 +#define _JUMP_TO_TOP 410 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 409 -#define _LOAD_ATTR_CLASS 410 -#define _LOAD_ATTR_CLASS_0 411 -#define _LOAD_ATTR_CLASS_1 412 +#define _LOAD_ATTR 411 +#define _LOAD_ATTR_CLASS 412 +#define _LOAD_ATTR_CLASS_0 413 +#define _LOAD_ATTR_CLASS_1 414 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 413 -#define _LOAD_ATTR_INSTANCE_VALUE_0 414 -#define _LOAD_ATTR_INSTANCE_VALUE_1 415 -#define _LOAD_ATTR_METHOD_LAZY_DICT 416 -#define _LOAD_ATTR_METHOD_NO_DICT 417 -#define _LOAD_ATTR_METHOD_WITH_VALUES 418 -#define _LOAD_ATTR_MODULE 419 -#define _LOAD_ATTR_MODULE_FROM_KEYS 420 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 421 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 422 -#define _LOAD_ATTR_PROPERTY_FRAME 423 -#define _LOAD_ATTR_SLOT 424 -#define _LOAD_ATTR_SLOT_0 425 -#define _LOAD_ATTR_SLOT_1 426 -#define _LOAD_ATTR_WITH_HINT 427 +#define _LOAD_ATTR_INSTANCE_VALUE 415 +#define _LOAD_ATTR_INSTANCE_VALUE_0 416 +#define _LOAD_ATTR_INSTANCE_VALUE_1 417 +#define _LOAD_ATTR_METHOD_LAZY_DICT 418 +#define _LOAD_ATTR_METHOD_NO_DICT 419 +#define _LOAD_ATTR_METHOD_WITH_VALUES 420 +#define _LOAD_ATTR_MODULE 421 +#define _LOAD_ATTR_MODULE_FROM_KEYS 422 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 423 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 424 +#define _LOAD_ATTR_PROPERTY_FRAME 425 +#define _LOAD_ATTR_SLOT 426 +#define _LOAD_ATTR_SLOT_0 427 +#define _LOAD_ATTR_SLOT_1 428 +#define _LOAD_ATTR_WITH_HINT 429 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 428 +#define _LOAD_BYTECODE 430 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 429 -#define _LOAD_CONST_INLINE_BORROW 430 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431 -#define _LOAD_CONST_INLINE_WITH_NULL 432 +#define _LOAD_CONST_INLINE 431 +#define _LOAD_CONST_INLINE_BORROW 432 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 433 +#define _LOAD_CONST_INLINE_WITH_NULL 434 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 433 -#define _LOAD_FAST_0 434 -#define _LOAD_FAST_1 435 -#define _LOAD_FAST_2 436 -#define _LOAD_FAST_3 437 -#define _LOAD_FAST_4 438 -#define _LOAD_FAST_5 439 -#define _LOAD_FAST_6 440 -#define _LOAD_FAST_7 441 +#define _LOAD_FAST 435 +#define _LOAD_FAST_0 436 +#define _LOAD_FAST_1 437 +#define _LOAD_FAST_2 438 +#define _LOAD_FAST_3 439 +#define _LOAD_FAST_4 440 +#define _LOAD_FAST_5 441 +#define _LOAD_FAST_6 442 +#define _LOAD_FAST_7 443 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 442 -#define _LOAD_GLOBAL_BUILTINS 443 -#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 444 -#define _LOAD_GLOBAL_MODULE 445 -#define _LOAD_GLOBAL_MODULE_FROM_KEYS 446 +#define _LOAD_GLOBAL 444 +#define _LOAD_GLOBAL_BUILTINS 445 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 446 +#define _LOAD_GLOBAL_MODULE 447 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 448 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 447 -#define _LOAD_SMALL_INT_0 448 -#define _LOAD_SMALL_INT_1 449 -#define _LOAD_SMALL_INT_2 450 -#define _LOAD_SMALL_INT_3 451 +#define _LOAD_SMALL_INT 449 +#define _LOAD_SMALL_INT_0 450 +#define _LOAD_SMALL_INT_1 451 +#define _LOAD_SMALL_INT_2 452 +#define _LOAD_SMALL_INT_3 453 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 452 +#define _MAKE_CALLARGS_A_TUPLE 454 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 453 +#define _MAKE_WARM 455 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 454 -#define _MAYBE_EXPAND_METHOD_KW 455 -#define _MONITOR_CALL 456 -#define _MONITOR_JUMP_BACKWARD 457 -#define _MONITOR_RESUME 458 +#define _MAYBE_EXPAND_METHOD 456 +#define _MAYBE_EXPAND_METHOD_KW 457 +#define _MONITOR_CALL 458 +#define _MONITOR_JUMP_BACKWARD 459 +#define _MONITOR_RESUME 460 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 459 -#define _POP_JUMP_IF_TRUE 460 +#define _POP_JUMP_IF_FALSE 461 +#define _POP_JUMP_IF_TRUE 462 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 461 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 463 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 462 +#define _PUSH_FRAME 464 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 463 -#define _PY_FRAME_KW 464 -#define _QUICKEN_RESUME 465 -#define _REPLACE_WITH_TRUE 466 +#define _PY_FRAME_GENERAL 465 +#define _PY_FRAME_KW 466 +#define _QUICKEN_RESUME 467 +#define _REPLACE_WITH_TRUE 468 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 467 -#define _SEND 468 -#define _SEND_GEN_FRAME 469 +#define _SAVE_RETURN_OFFSET 469 +#define _SEND 470 +#define _SEND_GEN_FRAME 471 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 470 -#define _STORE_ATTR 471 -#define _STORE_ATTR_INSTANCE_VALUE 472 -#define _STORE_ATTR_SLOT 473 -#define _STORE_ATTR_WITH_HINT 474 +#define _START_EXECUTOR 472 +#define _STORE_ATTR 473 +#define _STORE_ATTR_INSTANCE_VALUE 474 +#define _STORE_ATTR_SLOT 475 +#define _STORE_ATTR_WITH_HINT 476 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 475 -#define _STORE_FAST_0 476 -#define _STORE_FAST_1 477 -#define _STORE_FAST_2 478 -#define _STORE_FAST_3 479 -#define _STORE_FAST_4 480 -#define _STORE_FAST_5 481 -#define _STORE_FAST_6 482 -#define _STORE_FAST_7 483 +#define _STORE_FAST 477 +#define _STORE_FAST_0 478 +#define _STORE_FAST_1 479 +#define _STORE_FAST_2 480 +#define _STORE_FAST_3 481 +#define _STORE_FAST_4 482 +#define _STORE_FAST_5 483 +#define _STORE_FAST_6 484 +#define _STORE_FAST_7 485 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 484 -#define _STORE_SUBSCR 485 +#define _STORE_SLICE 486 +#define _STORE_SUBSCR 487 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 486 -#define _TO_BOOL 487 +#define _TIER2_RESUME_CHECK 488 +#define _TO_BOOL 489 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -309,13 +311,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 488 +#define _UNPACK_SEQUENCE 490 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 488 +#define MAX_UOP_ID 490 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 5670fe26f72071..298e918b872c62 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -82,6 +82,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_GUARD_BINARY_OP_EXTEND] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -309,6 +311,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", @@ -410,6 +413,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GET_ITER] = "_GET_ITER", [_GET_LEN] = "_GET_LEN", [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", @@ -711,6 +715,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: return 2; + case _GUARD_BINARY_OP_EXTEND: + return 0; + case _BINARY_OP_EXTEND: + return 2; case _BINARY_SUBSCR: return 2; case _BINARY_SLICE: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index f2d8963a1813c5..06b207b347e504 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -132,82 +132,83 @@ extern "C" { #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 #define BINARY_OP_ADD_UNICODE 152 -#define BINARY_OP_MULTIPLY_FLOAT 153 -#define BINARY_OP_MULTIPLY_INT 154 -#define BINARY_OP_SUBTRACT_FLOAT 155 -#define BINARY_OP_SUBTRACT_INT 156 -#define BINARY_SUBSCR_DICT 157 -#define BINARY_SUBSCR_GETITEM 158 -#define BINARY_SUBSCR_LIST_INT 159 -#define BINARY_SUBSCR_STR_INT 160 -#define BINARY_SUBSCR_TUPLE_INT 161 -#define CALL_ALLOC_AND_ENTER_INIT 162 -#define CALL_BOUND_METHOD_EXACT_ARGS 163 -#define CALL_BOUND_METHOD_GENERAL 164 -#define CALL_BUILTIN_CLASS 165 -#define CALL_BUILTIN_FAST 166 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 -#define CALL_BUILTIN_O 168 -#define CALL_ISINSTANCE 169 -#define CALL_KW_BOUND_METHOD 170 -#define CALL_KW_NON_PY 171 -#define CALL_KW_PY 172 -#define CALL_LEN 173 -#define CALL_LIST_APPEND 174 -#define CALL_METHOD_DESCRIPTOR_FAST 175 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 176 -#define CALL_METHOD_DESCRIPTOR_NOARGS 177 -#define CALL_METHOD_DESCRIPTOR_O 178 -#define CALL_NON_PY_GENERAL 179 -#define CALL_PY_EXACT_ARGS 180 -#define CALL_PY_GENERAL 181 -#define CALL_STR_1 182 -#define CALL_TUPLE_1 183 -#define CALL_TYPE_1 184 -#define COMPARE_OP_FLOAT 185 -#define COMPARE_OP_INT 186 -#define COMPARE_OP_STR 187 -#define CONTAINS_OP_DICT 188 -#define CONTAINS_OP_SET 189 -#define FOR_ITER_GEN 190 -#define FOR_ITER_LIST 191 -#define FOR_ITER_RANGE 192 -#define FOR_ITER_TUPLE 193 -#define LOAD_ATTR_CLASS 194 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 195 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 196 -#define LOAD_ATTR_INSTANCE_VALUE 197 -#define LOAD_ATTR_METHOD_LAZY_DICT 198 -#define LOAD_ATTR_METHOD_NO_DICT 199 -#define LOAD_ATTR_METHOD_WITH_VALUES 200 -#define LOAD_ATTR_MODULE 201 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 202 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 203 -#define LOAD_ATTR_PROPERTY 204 -#define LOAD_ATTR_SLOT 205 -#define LOAD_ATTR_WITH_HINT 206 -#define LOAD_CONST_IMMORTAL 207 -#define LOAD_CONST_MORTAL 208 -#define LOAD_GLOBAL_BUILTIN 209 -#define LOAD_GLOBAL_MODULE 210 -#define LOAD_SUPER_ATTR_ATTR 211 -#define LOAD_SUPER_ATTR_METHOD 212 -#define RESUME_CHECK 213 -#define SEND_GEN 214 -#define STORE_ATTR_INSTANCE_VALUE 215 -#define STORE_ATTR_SLOT 216 -#define STORE_ATTR_WITH_HINT 217 -#define STORE_SUBSCR_DICT 218 -#define STORE_SUBSCR_LIST_INT 219 -#define TO_BOOL_ALWAYS_TRUE 220 -#define TO_BOOL_BOOL 221 -#define TO_BOOL_INT 222 -#define TO_BOOL_LIST 223 -#define TO_BOOL_NONE 224 -#define TO_BOOL_STR 225 -#define UNPACK_SEQUENCE_LIST 226 -#define UNPACK_SEQUENCE_TUPLE 227 -#define UNPACK_SEQUENCE_TWO_TUPLE 228 +#define BINARY_OP_EXTEND 153 +#define BINARY_OP_MULTIPLY_FLOAT 154 +#define BINARY_OP_MULTIPLY_INT 155 +#define BINARY_OP_SUBTRACT_FLOAT 156 +#define BINARY_OP_SUBTRACT_INT 157 +#define BINARY_SUBSCR_DICT 158 +#define BINARY_SUBSCR_GETITEM 159 +#define BINARY_SUBSCR_LIST_INT 160 +#define BINARY_SUBSCR_STR_INT 161 +#define BINARY_SUBSCR_TUPLE_INT 162 +#define CALL_ALLOC_AND_ENTER_INIT 163 +#define CALL_BOUND_METHOD_EXACT_ARGS 164 +#define CALL_BOUND_METHOD_GENERAL 165 +#define CALL_BUILTIN_CLASS 166 +#define CALL_BUILTIN_FAST 167 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 168 +#define CALL_BUILTIN_O 169 +#define CALL_ISINSTANCE 170 +#define CALL_KW_BOUND_METHOD 171 +#define CALL_KW_NON_PY 172 +#define CALL_KW_PY 173 +#define CALL_LEN 174 +#define CALL_LIST_APPEND 175 +#define CALL_METHOD_DESCRIPTOR_FAST 176 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 177 +#define CALL_METHOD_DESCRIPTOR_NOARGS 178 +#define CALL_METHOD_DESCRIPTOR_O 179 +#define CALL_NON_PY_GENERAL 180 +#define CALL_PY_EXACT_ARGS 181 +#define CALL_PY_GENERAL 182 +#define CALL_STR_1 183 +#define CALL_TUPLE_1 184 +#define CALL_TYPE_1 185 +#define COMPARE_OP_FLOAT 186 +#define COMPARE_OP_INT 187 +#define COMPARE_OP_STR 188 +#define CONTAINS_OP_DICT 189 +#define CONTAINS_OP_SET 190 +#define FOR_ITER_GEN 191 +#define FOR_ITER_LIST 192 +#define FOR_ITER_RANGE 193 +#define FOR_ITER_TUPLE 194 +#define LOAD_ATTR_CLASS 195 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 196 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 197 +#define LOAD_ATTR_INSTANCE_VALUE 198 +#define LOAD_ATTR_METHOD_LAZY_DICT 199 +#define LOAD_ATTR_METHOD_NO_DICT 200 +#define LOAD_ATTR_METHOD_WITH_VALUES 201 +#define LOAD_ATTR_MODULE 202 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 203 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 204 +#define LOAD_ATTR_PROPERTY 205 +#define LOAD_ATTR_SLOT 206 +#define LOAD_ATTR_WITH_HINT 207 +#define LOAD_CONST_IMMORTAL 208 +#define LOAD_CONST_MORTAL 209 +#define LOAD_GLOBAL_BUILTIN 210 +#define LOAD_GLOBAL_MODULE 211 +#define LOAD_SUPER_ATTR_ATTR 212 +#define LOAD_SUPER_ATTR_METHOD 213 +#define RESUME_CHECK 214 +#define SEND_GEN 215 +#define STORE_ATTR_INSTANCE_VALUE 216 +#define STORE_ATTR_SLOT 217 +#define STORE_ATTR_WITH_HINT 218 +#define STORE_SUBSCR_DICT 219 +#define STORE_SUBSCR_LIST_INT 220 +#define TO_BOOL_ALWAYS_TRUE 221 +#define TO_BOOL_BOOL 222 +#define TO_BOOL_INT 223 +#define TO_BOOL_LIST 224 +#define TO_BOOL_NONE 225 +#define TO_BOOL_STR 226 +#define UNPACK_SEQUENCE_LIST 227 +#define UNPACK_SEQUENCE_TUPLE 228 +#define UNPACK_SEQUENCE_TWO_TUPLE 229 #define INSTRUMENTED_END_FOR 235 #define INSTRUMENTED_POP_ITER 236 #define INSTRUMENTED_END_SEND 237 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index d93e8d8df8fe4b..7dd528ef74df33 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -26,6 +26,7 @@ "BINARY_OP_ADD_FLOAT", "BINARY_OP_SUBTRACT_FLOAT", "BINARY_OP_ADD_UNICODE", + "BINARY_OP_EXTEND", "BINARY_OP_INPLACE_ADD_UNICODE", ], "BINARY_SUBSCR": [ @@ -123,83 +124,84 @@ 'BINARY_OP_ADD_FLOAT': 150, 'BINARY_OP_ADD_INT': 151, 'BINARY_OP_ADD_UNICODE': 152, + 'BINARY_OP_EXTEND': 153, 'BINARY_OP_INPLACE_ADD_UNICODE': 3, - 'BINARY_OP_MULTIPLY_FLOAT': 153, - 'BINARY_OP_MULTIPLY_INT': 154, - 'BINARY_OP_SUBTRACT_FLOAT': 155, - 'BINARY_OP_SUBTRACT_INT': 156, - 'BINARY_SUBSCR_DICT': 157, - 'BINARY_SUBSCR_GETITEM': 158, - 'BINARY_SUBSCR_LIST_INT': 159, - 'BINARY_SUBSCR_STR_INT': 160, - 'BINARY_SUBSCR_TUPLE_INT': 161, - 'CALL_ALLOC_AND_ENTER_INIT': 162, - 'CALL_BOUND_METHOD_EXACT_ARGS': 163, - 'CALL_BOUND_METHOD_GENERAL': 164, - 'CALL_BUILTIN_CLASS': 165, - 'CALL_BUILTIN_FAST': 166, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, - 'CALL_BUILTIN_O': 168, - 'CALL_ISINSTANCE': 169, - 'CALL_KW_BOUND_METHOD': 170, - 'CALL_KW_NON_PY': 171, - 'CALL_KW_PY': 172, - 'CALL_LEN': 173, - 'CALL_LIST_APPEND': 174, - 'CALL_METHOD_DESCRIPTOR_FAST': 175, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 176, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 177, - 'CALL_METHOD_DESCRIPTOR_O': 178, - 'CALL_NON_PY_GENERAL': 179, - 'CALL_PY_EXACT_ARGS': 180, - 'CALL_PY_GENERAL': 181, - 'CALL_STR_1': 182, - 'CALL_TUPLE_1': 183, - 'CALL_TYPE_1': 184, - 'COMPARE_OP_FLOAT': 185, - 'COMPARE_OP_INT': 186, - 'COMPARE_OP_STR': 187, - 'CONTAINS_OP_DICT': 188, - 'CONTAINS_OP_SET': 189, - 'FOR_ITER_GEN': 190, - 'FOR_ITER_LIST': 191, - 'FOR_ITER_RANGE': 192, - 'FOR_ITER_TUPLE': 193, - 'LOAD_ATTR_CLASS': 194, - 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 195, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 196, - 'LOAD_ATTR_INSTANCE_VALUE': 197, - 'LOAD_ATTR_METHOD_LAZY_DICT': 198, - 'LOAD_ATTR_METHOD_NO_DICT': 199, - 'LOAD_ATTR_METHOD_WITH_VALUES': 200, - 'LOAD_ATTR_MODULE': 201, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 202, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 203, - 'LOAD_ATTR_PROPERTY': 204, - 'LOAD_ATTR_SLOT': 205, - 'LOAD_ATTR_WITH_HINT': 206, - 'LOAD_CONST_IMMORTAL': 207, - 'LOAD_CONST_MORTAL': 208, - 'LOAD_GLOBAL_BUILTIN': 209, - 'LOAD_GLOBAL_MODULE': 210, - 'LOAD_SUPER_ATTR_ATTR': 211, - 'LOAD_SUPER_ATTR_METHOD': 212, - 'RESUME_CHECK': 213, - 'SEND_GEN': 214, - 'STORE_ATTR_INSTANCE_VALUE': 215, - 'STORE_ATTR_SLOT': 216, - 'STORE_ATTR_WITH_HINT': 217, - 'STORE_SUBSCR_DICT': 218, - 'STORE_SUBSCR_LIST_INT': 219, - 'TO_BOOL_ALWAYS_TRUE': 220, - 'TO_BOOL_BOOL': 221, - 'TO_BOOL_INT': 222, - 'TO_BOOL_LIST': 223, - 'TO_BOOL_NONE': 224, - 'TO_BOOL_STR': 225, - 'UNPACK_SEQUENCE_LIST': 226, - 'UNPACK_SEQUENCE_TUPLE': 227, - 'UNPACK_SEQUENCE_TWO_TUPLE': 228, + 'BINARY_OP_MULTIPLY_FLOAT': 154, + 'BINARY_OP_MULTIPLY_INT': 155, + 'BINARY_OP_SUBTRACT_FLOAT': 156, + 'BINARY_OP_SUBTRACT_INT': 157, + 'BINARY_SUBSCR_DICT': 158, + 'BINARY_SUBSCR_GETITEM': 159, + 'BINARY_SUBSCR_LIST_INT': 160, + 'BINARY_SUBSCR_STR_INT': 161, + 'BINARY_SUBSCR_TUPLE_INT': 162, + 'CALL_ALLOC_AND_ENTER_INIT': 163, + 'CALL_BOUND_METHOD_EXACT_ARGS': 164, + 'CALL_BOUND_METHOD_GENERAL': 165, + 'CALL_BUILTIN_CLASS': 166, + 'CALL_BUILTIN_FAST': 167, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 168, + 'CALL_BUILTIN_O': 169, + 'CALL_ISINSTANCE': 170, + 'CALL_KW_BOUND_METHOD': 171, + 'CALL_KW_NON_PY': 172, + 'CALL_KW_PY': 173, + 'CALL_LEN': 174, + 'CALL_LIST_APPEND': 175, + 'CALL_METHOD_DESCRIPTOR_FAST': 176, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 177, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 178, + 'CALL_METHOD_DESCRIPTOR_O': 179, + 'CALL_NON_PY_GENERAL': 180, + 'CALL_PY_EXACT_ARGS': 181, + 'CALL_PY_GENERAL': 182, + 'CALL_STR_1': 183, + 'CALL_TUPLE_1': 184, + 'CALL_TYPE_1': 185, + 'COMPARE_OP_FLOAT': 186, + 'COMPARE_OP_INT': 187, + 'COMPARE_OP_STR': 188, + 'CONTAINS_OP_DICT': 189, + 'CONTAINS_OP_SET': 190, + 'FOR_ITER_GEN': 191, + 'FOR_ITER_LIST': 192, + 'FOR_ITER_RANGE': 193, + 'FOR_ITER_TUPLE': 194, + 'LOAD_ATTR_CLASS': 195, + 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 196, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 197, + 'LOAD_ATTR_INSTANCE_VALUE': 198, + 'LOAD_ATTR_METHOD_LAZY_DICT': 199, + 'LOAD_ATTR_METHOD_NO_DICT': 200, + 'LOAD_ATTR_METHOD_WITH_VALUES': 201, + 'LOAD_ATTR_MODULE': 202, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 203, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 204, + 'LOAD_ATTR_PROPERTY': 205, + 'LOAD_ATTR_SLOT': 206, + 'LOAD_ATTR_WITH_HINT': 207, + 'LOAD_CONST_IMMORTAL': 208, + 'LOAD_CONST_MORTAL': 209, + 'LOAD_GLOBAL_BUILTIN': 210, + 'LOAD_GLOBAL_MODULE': 211, + 'LOAD_SUPER_ATTR_ATTR': 212, + 'LOAD_SUPER_ATTR_METHOD': 213, + 'RESUME_CHECK': 214, + 'SEND_GEN': 215, + 'STORE_ATTR_INSTANCE_VALUE': 216, + 'STORE_ATTR_SLOT': 217, + 'STORE_ATTR_WITH_HINT': 218, + 'STORE_SUBSCR_DICT': 219, + 'STORE_SUBSCR_LIST_INT': 220, + 'TO_BOOL_ALWAYS_TRUE': 221, + 'TO_BOOL_BOOL': 222, + 'TO_BOOL_INT': 223, + 'TO_BOOL_LIST': 224, + 'TO_BOOL_NONE': 225, + 'TO_BOOL_STR': 226, + 'UNPACK_SEQUENCE_LIST': 227, + 'UNPACK_SEQUENCE_TUPLE': 228, + 'UNPACK_SEQUENCE_TWO_TUPLE': 229, } opmap = { diff --git a/Lib/opcode.py b/Lib/opcode.py index aba66153af1b2e..4ee0d64026bd0a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -52,6 +52,7 @@ }, "BINARY_OP": { "counter": 1, + "descr": 4, }, "UNPACK_SEQUENCE": { "counter": 1, diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 7ffa4eb8639add..69c1ee0690d269 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -429,14 +429,14 @@ def test_invalid_bytecode(self): def foo(): pass - # assert that opcode 229 is invalid - self.assertEqual(opname[229], '<229>') + # assert that opcode 135 is invalid + self.assertEqual(opname[135], '<135>') - # change first opcode to 0xeb (=229) + # change first opcode to 0x87 (=135) foo.__code__ = foo.__code__.replace( - co_code=b'\xe5' + foo.__code__.co_code[1:]) + co_code=b'\x87' + foo.__code__.co_code[1:]) - msg = "unknown opcode 229" + msg = "unknown opcode 135" with self.assertRaisesRegex(SystemError, msg): foo() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 8afe9653f19f6e..da57aad2f84fbd 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -438,7 +438,7 @@ def foo(a: int, b: str) -> str: LOAD_SMALL_INT 1 BINARY_OP 13 (+=) STORE_NAME 0 (x) - JUMP_BACKWARD 8 (to L1) + JUMP_BACKWARD 12 (to L1) """ dis_traceback = """\ @@ -843,7 +843,7 @@ def foo(x): L1: RESUME 0 LOAD_FAST 0 (.0) GET_ITER - L2: FOR_ITER 10 (to L3) + L2: FOR_ITER 14 (to L3) STORE_FAST 1 (z) LOAD_DEREF 2 (x) LOAD_FAST 1 (z) @@ -851,7 +851,7 @@ def foo(x): YIELD_VALUE 0 RESUME 5 POP_TOP - JUMP_BACKWARD 12 (to L2) + JUMP_BACKWARD 16 (to L2) L3: END_FOR POP_ITER LOAD_CONST 0 (None) @@ -1807,7 +1807,7 @@ def _prepare_test_cases(): Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST_CHECK', opcode=85, arg=0, argval='i', argrepr='i', offset=118, start_offset=118, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=36, argval=204, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=212, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=134, start_offset=134, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=144, start_offset=144, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), @@ -1815,93 +1815,93 @@ def _prepare_test_cases(): Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=154, start_offset=154, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=44, arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=164, start_offset=164, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=170, start_offset=170, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=184, argrepr='to L6', offset=174, start_offset=174, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=33, argval=118, argrepr='to L5', offset=180, start_offset=180, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=184, start_offset=184, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=188, start_offset=188, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=202, argrepr='to L7', offset=192, start_offset=192, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=196, start_offset=196, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=42, argval=118, argrepr='to L5', offset=198, start_offset=198, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_FORWARD', opcode=76, arg=11, argval=226, argrepr='to L9', offset=202, start_offset=202, starts_line=True, line_number=17, label=7, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=204, start_offset=204, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=214, start_offset=214, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=216, start_offset=216, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=228, start_offset=228, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=0, argval=0, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=44, arg=11, argval=11, argrepr='/', offset=232, start_offset=232, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=238, start_offset=238, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=1, argval=1, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=92, arg=1, argval=1, argrepr='__exit__', offset=242, start_offset=242, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=92, arg=0, argval=0, argrepr='__enter__', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=0, argval=0, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=258, start_offset=258, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=260, start_offset=260, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=270, start_offset=270, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=272, start_offset=272, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=280, start_offset=280, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=282, start_offset=282, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=286, start_offset=286, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=3, argval=3, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=298, start_offset=298, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=320, start_offset=320, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=322, start_offset=322, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=324, start_offset=324, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=43, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=344, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=29, argval=298, argrepr='to L10', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=356, start_offset=356, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=364, start_offset=364, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=410, argrepr='to L12', offset=376, start_offset=376, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=384, start_offset=384, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=56, argval=298, argrepr='to L10', offset=408, start_offset=408, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=410, start_offset=410, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=412, start_offset=412, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=416, start_offset=416, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=420, start_offset=420, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=430, start_offset=430, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=440, start_offset=440, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=442, start_offset=442, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=444, start_offset=444, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=44, arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=172, start_offset=172, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=176, start_offset=176, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=178, start_offset=178, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=192, argrepr='to L6', offset=182, start_offset=182, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=37, argval=118, argrepr='to L5', offset=188, start_offset=188, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=192, start_offset=192, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=194, start_offset=194, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=196, start_offset=196, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=210, argrepr='to L7', offset=200, start_offset=200, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=46, argval=118, argrepr='to L5', offset=206, start_offset=206, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=11, argval=234, argrepr='to L9', offset=210, start_offset=210, starts_line=True, line_number=17, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=212, start_offset=212, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=222, start_offset=222, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=0, argval=0, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=44, arg=11, argval=11, argrepr='/', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=254, start_offset=254, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=1, argval=1, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=1, argval=1, argrepr='__exit__', offset=258, start_offset=258, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=0, argval=0, argrepr='__enter__', offset=264, start_offset=264, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=0, argval=0, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=276, start_offset=276, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=286, start_offset=286, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=298, start_offset=298, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=302, start_offset=302, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=3, argval=3, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=314, start_offset=314, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=336, start_offset=336, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=43, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=360, argrepr='to L11', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=29, argval=314, argrepr='to L10', offset=370, start_offset=370, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=372, start_offset=372, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=380, start_offset=380, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=426, argrepr='to L12', offset=392, start_offset=392, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=410, start_offset=410, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=56, argval=314, argrepr='to L10', offset=424, start_offset=424, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=436, start_offset=436, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=446, start_offset=446, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=456, start_offset=456, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=460, start_offset=460, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 43e3e56639db62..bea0e48883bf9a 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1649,7 +1649,7 @@ def foo(n=0): return None in_loop = ('branch left', 'foo', 10, 16) - exit_loop = ('branch right', 'foo', 10, 32) + exit_loop = ('branch right', 'foo', 10, 40) self.check_events(foo, recorders = BRANCH_OFFSET_RECORDERS, expected = [ in_loop, in_loop, diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index b80ccbf17f1ee6..72b845fcc8fdbf 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1338,6 +1338,31 @@ def binary_op_add_unicode(): self.assert_specialized(binary_op_add_unicode, "BINARY_OP_ADD_UNICODE") self.assert_no_opcode(binary_op_add_unicode, "BINARY_OP") + def binary_op_add_extend(): + for _ in range(100): + a, b = 6, 3.0 + c = a + b + self.assertEqual(c, 9.0) + c = b + a + self.assertEqual(c, 9.0) + c = a - b + self.assertEqual(c, 3.0) + c = b - a + self.assertEqual(c, -3.0) + c = a * b + self.assertEqual(c, 18.0) + c = b * a + self.assertEqual(c, 18.0) + c = a / b + self.assertEqual(c, 2.0) + c = b / a + self.assertEqual(c, 0.5) + + binary_op_add_extend() + self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") + self.assert_no_opcode(binary_op_add_extend, "BINARY_OP") + + @cpython_only @requires_specialization_ft def test_load_super_attr(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-23-54-16.gh-issue-100239.ijOOUs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-23-54-16.gh-issue-100239.ijOOUs.rst new file mode 100644 index 00000000000000..f58c1fc767515e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-23-54-16.gh-issue-100239.ijOOUs.rst @@ -0,0 +1,2 @@ +Add opcode ``BINARY_OP_EXTEND`` which executes a pair of functions (guard and +specialization functions) accessed from the inline cache. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a906ded365650c..b1d61a8707bd4c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -522,6 +522,7 @@ dummy_func( BINARY_OP_SUBTRACT_FLOAT, BINARY_OP_ADD_UNICODE, // BINARY_OP_INPLACE_ADD_UNICODE, // See comments at that opcode. + BINARY_OP_EXTEND, }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { @@ -587,11 +588,11 @@ dummy_func( } macro(BINARY_OP_MULTIPLY_INT) = - _GUARD_BOTH_INT + unused/1 + _BINARY_OP_MULTIPLY_INT; + _GUARD_BOTH_INT + unused/5 + _BINARY_OP_MULTIPLY_INT; macro(BINARY_OP_ADD_INT) = - _GUARD_BOTH_INT + unused/1 + _BINARY_OP_ADD_INT; + _GUARD_BOTH_INT + unused/5 + _BINARY_OP_ADD_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; + _GUARD_BOTH_INT + unused/5 + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -659,11 +660,11 @@ dummy_func( } macro(BINARY_OP_MULTIPLY_FLOAT) = - _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_MULTIPLY_FLOAT; + _GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = - _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_ADD_FLOAT; + _GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT; macro(BINARY_OP_SUBTRACT_FLOAT) = - _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; + _GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -689,7 +690,7 @@ dummy_func( } macro(BINARY_OP_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_ADD_UNICODE; + _GUARD_BOTH_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE; // This is a subtle one. It's a super-instruction for // BINARY_OP_ADD_UNICODE followed by STORE_FAST @@ -741,8 +742,34 @@ dummy_func( #endif } + op(_GUARD_BINARY_OP_EXTEND, (descr/4, left, right -- left, right)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + int res = d->guard(left_o, right_o); + EXIT_IF(!res); + } + + pure op(_BINARY_OP_EXTEND, (descr/4, left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + + STAT_INC(BINARY_OP, hit); + + PyObject *res_o = d->action(left_o, right_o); + DECREF_INPUTS(); + res = PyStackRef_FromPyObjectSteal(res_o); + } + + macro(BINARY_OP_EXTEND) = + unused/1 + _GUARD_BINARY_OP_EXTEND + rewind/-4 + _BINARY_OP_EXTEND; + macro(BINARY_OP_INPLACE_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_INPLACE_ADD_UNICODE; + _GUARD_BOTH_UNICODE + unused/5 + _BINARY_OP_INPLACE_ADD_UNICODE; family(BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = { BINARY_SUBSCR_DICT, @@ -4742,7 +4769,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; + macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + unused/4 + _BINARY_OP; pure inst(SWAP, (bottom_in, unused[oparg-2], top_in -- top_out, unused[oparg-2], bottom_out)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index cda01bb768c269..ad825881c8228c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -899,6 +899,51 @@ break; } + case _GUARD_BINARY_OP_EXTEND: { + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _BINARY_OP_EXTEND: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_SUBSCR: { _PyStackRef sub; _PyStackRef container; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 81408380d6b2b8..dc90f75f2645e1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11,10 +11,10 @@ TARGET(BINARY_OP) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef lhs; _PyStackRef rhs; @@ -39,6 +39,7 @@ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); } + /* Skip 4 cache entries */ // _BINARY_OP { PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); @@ -60,9 +61,9 @@ TARGET(BINARY_OP_ADD_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -75,7 +76,7 @@ DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -98,9 +99,9 @@ TARGET(BINARY_OP_ADD_INT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -113,7 +114,7 @@ DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -135,9 +136,9 @@ TARGET(BINARY_OP_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -150,7 +151,7 @@ DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -170,11 +171,57 @@ DISPATCH(); } + TARGET(BINARY_OP_EXTEND) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(!res, BINARY_OP); + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; // _GUARD_BOTH_UNICODE @@ -186,7 +233,7 @@ DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -235,9 +282,9 @@ TARGET(BINARY_OP_MULTIPLY_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -250,7 +297,7 @@ DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -273,9 +320,9 @@ TARGET(BINARY_OP_MULTIPLY_INT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -288,7 +335,7 @@ DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -310,9 +357,9 @@ TARGET(BINARY_OP_SUBTRACT_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -325,7 +372,7 @@ DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -348,9 +395,9 @@ TARGET(BINARY_OP_SUBTRACT_INT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -363,7 +410,7 @@ DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c5c008fcbe574e..cb6c33f01d3598 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -152,6 +152,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_BINARY_OP_EXTEND, &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, @@ -233,7 +234,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index aff4493fdc4dd7..90838d274a5e87 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -562,6 +562,19 @@ break; } + case _GUARD_BINARY_OP_EXTEND: { + break; + } + + case _BINARY_OP_EXTEND: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_SUBSCR: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); diff --git a/Python/specialize.c b/Python/specialize.c index 8d9f19c8895187..09bfcd34b5a543 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1174,7 +1174,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* assert(tp_version != 0); write_u32(lm_cache->type_version, tp_version); /* borrowed */ - write_obj(lm_cache->descr, fget); + write_ptr(lm_cache->descr, fget); specialize(instr, LOAD_ATTR_PROPERTY); return 0; } @@ -1254,7 +1254,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* #endif write_u32(lm_cache->keys_version, version); /* borrowed */ - write_obj(lm_cache->descr, descr); + write_ptr(lm_cache->descr, descr); write_u32(lm_cache->type_version, tp_version); specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); return 0; @@ -1534,7 +1534,7 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } #endif write_u32(cache->type_version, tp_version); - write_obj(cache->descr, descr); + write_ptr(cache->descr, descr); if (metaclass_check) { write_u32(cache->keys_version, meta_version); specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); @@ -1642,7 +1642,7 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, * working since Python 2.6 and it's battle-tested. */ write_u32(cache->type_version, tp_version); - write_obj(cache->descr, descr); + write_ptr(cache->descr, descr); return 1; } @@ -2412,6 +2412,92 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) } #endif +/** Binary Op Specialization Extensions */ + +/* float-long */ + +static int +float_compactlong_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + PyFloat_CheckExact(lhs) && + PyLong_CheckExact(rhs) && + _PyLong_IsCompact((PyLongObject *)rhs) + ); +} + +#define FLOAT_LONG_ACTION(NAME, OP) \ + static PyObject * \ + (NAME)(PyObject *lhs, PyObject *rhs) \ + { \ + double lhs_val = PyFloat_AsDouble(lhs); \ + Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \ + return PyFloat_FromDouble(lhs_val OP rhs_val); \ + } +FLOAT_LONG_ACTION(float_compactlong_add, +) +FLOAT_LONG_ACTION(float_compactlong_subtract, -) +FLOAT_LONG_ACTION(float_compactlong_multiply, *) +FLOAT_LONG_ACTION(float_compactlong_true_div, /) +#undef FLOAT_LONG_ACTION + +/* long-float */ + +static int +compactlong_float_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + PyFloat_CheckExact(rhs) && + PyLong_CheckExact(lhs) && + _PyLong_IsCompact((PyLongObject *)lhs) + ); +} + +#define LONG_FLOAT_ACTION(NAME, OP) \ + static PyObject * \ + (NAME)(PyObject *lhs, PyObject *rhs) \ + { \ + double rhs_val = PyFloat_AsDouble(rhs); \ + Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \ + return PyFloat_FromDouble(lhs_val OP rhs_val); \ + } +LONG_FLOAT_ACTION(compactlong_float_add, +) +LONG_FLOAT_ACTION(compactlong_float_subtract, -) +LONG_FLOAT_ACTION(compactlong_float_multiply, *) +LONG_FLOAT_ACTION(compactlong_float_true_div, /) +#undef LONG_FLOAT_ACTION + +static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = { + [NB_ADD] = {float_compactlong_guard, float_compactlong_add}, + [NB_SUBTRACT] = {float_compactlong_guard, float_compactlong_subtract}, + [NB_TRUE_DIVIDE] = {float_compactlong_guard, float_compactlong_true_div}, + [NB_MULTIPLY] = {float_compactlong_guard, float_compactlong_multiply}, +}; + +static _PyBinaryOpSpecializationDescr compactlong_float_specs[NB_OPARG_LAST+1] = { + [NB_ADD] = {compactlong_float_guard, compactlong_float_add}, + [NB_SUBTRACT] = {compactlong_float_guard, compactlong_float_subtract}, + [NB_TRUE_DIVIDE] = {compactlong_float_guard, compactlong_float_true_div}, + [NB_MULTIPLY] = {compactlong_float_guard, compactlong_float_multiply}, +}; + +static int +binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg, + _PyBinaryOpSpecializationDescr **descr) +{ +#define LOOKUP_SPEC(TABLE, OPARG) \ + if ((TABLE)[(OPARG)].action) { \ + if ((TABLE)[(OPARG)].guard(lhs, rhs)) { \ + *descr = &((TABLE)[OPARG]); \ + return 1; \ + } \ + } + + LOOKUP_SPEC(compactlong_float_specs, oparg); + LOOKUP_SPEC(float_compactlong_specs, oparg); +#undef LOOKUP_SPEC + return 0; +} + void _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals) @@ -2420,6 +2506,12 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); + + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); + if (instr->op.code == BINARY_OP_EXTEND) { + write_ptr(cache->external_cache, NULL); + } + switch (oparg) { case NB_ADD: case NB_INPLACE_ADD: @@ -2474,8 +2566,17 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in } break; } + + _PyBinaryOpSpecializationDescr *descr; + if (binary_op_extended_specialization(lhs, rhs, oparg, &descr)) { + specialize(instr, BINARY_OP_EXTEND); + write_ptr(cache->external_cache, (void*)descr); + return; + } + SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); unspecialize(instr); + return; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c8c30a7985aa2e..213165b06866b5 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -378,6 +378,8 @@ Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF - Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - +Python/specialize.c - float_compactlong_specs - +Python/specialize.c - compactlong_float_specs - Python/stdlib_module_names.h - _Py_stdlib_module_names - Python/sysmodule.c - perf_map_state - Python/sysmodule.c - _PySys_ImplCacheTag - diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index de31d9b232f9df..41b36b6a546360 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -357,9 +357,12 @@ def uops(self) -> list[UOp] | None: def uop(self) -> UOp | None: if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.DIVIDE): + sign = 1 + if negate := self.expect(lx.MINUS): + sign = -1 if num := self.expect(lx.NUMBER): try: - size = int(num.text) + size = sign * int(num.text) except ValueError: raise self.make_syntax_error( f"Expected integer, got {num.text!r}" From f48702dade921beed3e227d2a5ac82a9ae2533d0 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 16 Jan 2025 18:41:40 +0300 Subject: [PATCH 60/61] gh-128910: Remove `_PyTrash_begin` and `_PyTrash_end` C-API functions (#128919) --- Include/cpython/object.h | 3 --- .../2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst diff --git a/Include/cpython/object.h b/Include/cpython/object.h index c8c6bc97fa32ee..ba31e2464abf84 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -475,9 +475,6 @@ partially-deallocated object. To check this, the tp_dealloc function must be passed as second argument to Py_TRASHCAN_BEGIN(). */ -/* Python 3.9 private API, invoked by the macros below. */ -PyAPI_FUNC(int) _PyTrash_begin(PyThreadState *tstate, PyObject *op); -PyAPI_FUNC(void) _PyTrash_end(PyThreadState *tstate); PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst new file mode 100644 index 00000000000000..e095ba9ebf6be4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst @@ -0,0 +1,2 @@ +Undocumented and unused private C-API functions ``_PyTrash_begin`` and +``_PyTrash_end`` are removed. From 211f41316b7f205d18eb65c1ececd7f7fb30b02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Srinivas=20Reddy=20Thatiparthy=20=28=E0=B0=A4=E0=B0=BE?= =?UTF-8?q?=E0=B0=9F=E0=B0=BF=E0=B0=AA=E0=B0=B0=E0=B1=8D=E0=B0=A4=E0=B0=BF?= =?UTF-8?q?=20=E0=B0=B6=E0=B1=8D=E0=B0=B0=E0=B1=80=E0=B0=A8=E0=B0=BF?= =?UTF-8?q?=E0=B0=B5=E0=B0=BE=E0=B0=B8=E0=B1=8D=20=20=E0=B0=B0=E0=B1=86?= =?UTF-8?q?=E0=B0=A1=E0=B1=8D=E0=B0=A1=E0=B0=BF=29?= Date: Thu, 16 Jan 2025 22:02:17 +0530 Subject: [PATCH 61/61] Fix typo in doc (#128917) --- InternalDocs/generators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index 608bd215aae65a..87fbb91236844b 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -11,7 +11,7 @@ execution state. A generator object resumes execution in its frame when its `send()` method is called. This is analogous to a function executing in its own -fram when it is called, but a function returns to the calling frame only once, +frame when it is called, but a function returns to the calling frame only once, while a generator "returns" execution to the caller's frame every time it emits a new item with a [`yield` expression](https://docs.python.org/dev/reference/expressions.html#yield-expressions).