pythongh-131185: Use a proper thread-local for cached thread states (pythongh-132510)

Switches over to a _Py_thread_local in place of autoTssKey, and also fixes a few other checks regarding PyGILState_Ensure after finalization.

Note that this doesn't fix concurrent use of PyGILState_Ensure with Py_Finalize; I'm pretty sure zapthreads doesn't work at all, and that needs to be fixed seperately.

Author: ZeroIntensity

Include/internal/pycore_runtime_init.h

@@ -61,9 +61,6 @@ extern PyTypeObject _PyExc_MemoryError;
                 }, \
             }, \
         }, \
-        /* A TSS key must be initialized with Py_tss_NEEDS_INIT \
-           in accordance with the specification. */ \
-        .autoTSSkey = Py_tss_NEEDS_INIT, \
         .parser = _parser_runtime_state_INIT, \
         .ceval = { \
             .pending_mainthread = { \
@@ -236,4 +233,4 @@ extern PyTypeObject _PyExc_MemoryError;
 #ifdef __cplusplus
 }
 #endif
-#endif /* !Py_INTERNAL_RUNTIME_INIT_H */
+#endif /* !Py_INTERNAL_RUNTIME_INIT_H */

Include/internal/pycore_runtime_structs.h

@@ -223,9 +223,6 @@ struct pyruntimestate {
     struct _pythread_runtime_state threads;
     struct _signals_runtime_state signals;
 
-    /* Used for the thread state bound to the current thread. */
-    Py_tss_t autoTSSkey;
-
     /* Used instead of PyThreadState.trash when there is not current tstate. */
     Py_tss_t trashTSSkey;
 

Lib/test/test_embed.py

@@ -1915,6 +1915,10 @@ def test_get_incomplete_frame(self):
         self.run_embedded_interpreter("test_get_incomplete_frame")
 
 
+    def test_gilstate_after_finalization(self):
+        self.run_embedded_interpreter("test_gilstate_after_finalization")
+
+
 class MiscTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_unicode_id_init(self):
         # bpo-42882: Test that _PyUnicode_FromId() works

Misc/NEWS.d/next/C_API/2025-04-14-07-41-28.gh-issue-131185.ZCjMHD.rst

@@ -0,0 +1,2 @@
+:c:func:`PyGILState_Ensure` no longer crashes when called after interpreter
+finalization.

Programs/_testembed.c

@@ -8,6 +8,7 @@
 #include <Python.h>
 #include "pycore_initconfig.h"    // _PyConfig_InitCompatConfig()
 #include "pycore_runtime.h"       // _PyRuntime
+#include "pycore_lock.h"          // PyEvent
 #include "pycore_pythread.h"      // PyThread_start_joinable_thread()
 #include "pycore_import.h"        // _PyImport_FrozenBootstrap
 #include <inttypes.h>
@@ -2312,6 +2313,32 @@ test_get_incomplete_frame(void)
     return result;
 }
 
+static void
+do_gilstate_ensure(void *event_ptr)
+{
+    PyEvent *event = (PyEvent *)event_ptr;
+    // Signal to the calling thread that we've started
+    _PyEvent_Notify(event);
+    PyGILState_Ensure(); // This should hang
+    assert(NULL);
+}
+
+static int
+test_gilstate_after_finalization(void)
+{
+    _testembed_initialize();
+    Py_Finalize();
+    PyThread_handle_t handle;
+    PyThread_ident_t ident;
+    PyEvent event = {0};
+    if (PyThread_start_joinable_thread(&do_gilstate_ensure, &event, &ident, &handle) < 0) {
+        return -1;
+    }
+    PyEvent_Wait(&event);
+    // We're now pretty confident that the thread went for
+    // PyGILState_Ensure(), but that means it got hung.
+    return PyThread_detach_thread(handle);
+}
 
 /* *********************************************************
  * List of test cases and the function that implements it.
@@ -2402,7 +2429,7 @@ static struct TestCase TestCases[] = {
     {"test_frozenmain", test_frozenmain},
 #endif
     {"test_get_incomplete_frame", test_get_incomplete_frame},
-
+    {"test_gilstate_after_finalization", test_gilstate_after_finalization},
     {NULL, NULL}
 };