diff options
| -rw-r--r-- | Doc/c-api/init.rst | 157 | ||||
| -rw-r--r-- | Doc/whatsnew/3.7.rst | 32 | ||||
| -rw-r--r-- | Include/internal/pystate.h | 2 | ||||
| -rw-r--r-- | Include/pythread.h | 72 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst | 4 | ||||
| -rw-r--r-- | Modules/_testcapimodule.c | 56 | ||||
| -rw-r--r-- | Modules/_tracemalloc.c | 25 | ||||
| -rw-r--r-- | Modules/faulthandler.c | 2 | ||||
| -rw-r--r-- | Modules/posixmodule.c | 3 | ||||
| -rw-r--r-- | PC/python3.def | 7 | ||||
| -rw-r--r-- | Python/pystate.c | 71 | ||||
| -rw-r--r-- | Python/thread.c | 60 | ||||
| -rw-r--r-- | Python/thread_nt.h | 82 | ||||
| -rw-r--r-- | Python/thread_pthread.h | 90 | ||||
| -rw-r--r-- | Python/traceback.c | 2 | ||||
| -rwxr-xr-x | configure | 69 | ||||
| -rw-r--r-- | configure.ac | 19 | ||||
| -rw-r--r-- | pyconfig.h.in | 6 |
18 files changed, 651 insertions, 108 deletions
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index c12e1c7fba3..7792058683d 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1192,3 +1192,160 @@ These functions are only intended to be used by advanced debugging tools. Return the next thread state object after *tstate* from the list of all such objects belonging to the same :c:type:`PyInterpreterState` object. + +.. _thread-local-storage: + +Thread Local Storage Support +============================ + +.. sectionauthor:: Masayuki Yamamoto <ma3yuki.8mamo10@gmail.com> + +The Python interpreter provides low-level support for thread-local storage +(TLS) which wraps the underlying native TLS implementation to support the +Python-level thread local storage API (:class:`threading.local`). The +CPython C level APIs are similar to those offered by pthreads and Windows: +use a thread key and functions to associate a :c:type:`void\*` value per +thread. + +The GIL does *not* need to be held when calling these functions; they supply +their own locking. + +Note that :file:`Python.h` does not include the declaration of the TLS APIs, +you need to include :file:`pythread.h` to use thread-local storage. + +.. note:: + None of these API functions handle memory management on behalf of the + :c:type:`void\*` values. You need to allocate and deallocate them yourself. + If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these + functions don't do refcount operations on them either. + +.. _thread-specific-storage-api: + +Thread Specific Storage (TSS) API +--------------------------------- + +TSS API is introduced to supersede the use of the existing TLS API within the +CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of +:c:type:`int` to represent thread keys. + +.. versionadded:: 3.7 + +.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) + + +.. c:type:: Py_tss_t + + This data structure represents the state of a thread key, the definition of + which may depend on the underlying TLS implementation, and it has an + internal field representing the key's initialization state. There are no + public members in this structure. + + When :ref:`Py_LIMITED_API <stable>` is not defined, static allocation of + this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. + + +.. c:macro:: Py_tss_NEEDS_INIT + + This macro expands to the default value for :c:type:`Py_tss_t` variables. + Note that this macro won't be defined with :ref:`Py_LIMITED_API <stable>`. + + +Dynamic Allocation +~~~~~~~~~~~~~~~~~~ + +Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules +built with :ref:`Py_LIMITED_API <stable>`, where static allocation of this type +is not possible due to its implementation being opaque at build time. + + +.. c:function:: Py_tss_t* PyThread_tss_alloc() + + Return a value which is the same state as a value initialized with + :c:macro:`Py_tss_NEEDS_INIT`, or *NULL* in the case of dynamic allocation + failure. + + +.. c:function:: void PyThread_tss_free(Py_tss_t *key) + + Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after + first calling :c:func:`PyThread_tss_delete` to ensure any associated + thread locals have been unassigned. This is a no-op if the *key* + argument is `NULL`. + + .. note:: + A freed key becomes a dangling pointer, you should reset the key to + `NULL`. + + +Methods +~~~~~~~ + +The parameter *key* of these functions must not be *NULL*. Moreover, the +behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are +undefined if the given :c:type:`Py_tss_t` has not been initialized by +:c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) + + Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized + by :c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_create(Py_tss_t *key) + + Return a zero value on successful initialization of a TSS key. The behavior + is undefined if the value pointed to by the *key* argument is not + initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called + repeatedly on the same key -- calling it on an already initialized key is a + no-op and immediately returns success. + + +.. c:function:: void PyThread_tss_delete(Py_tss_t *key) + + Destroy a TSS key to forget the values associated with the key across all + threads, and change the key's initialization state to uninitialized. A + destroyed key is able to be initialized again by + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key -- calling it on an already destroyed key is a no-op. + + +.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) + + Return a zero value to indicate successfully associating a :c:type:`void\*` + value with a TSS key in the current thread. Each thread has a distinct + mapping of the key to a :c:type:`void\*` value. + + +.. c:function:: void* PyThread_tss_get(Py_tss_t *key) + + Return the :c:type:`void\*` value associated with a TSS key in the current + thread. This returns *NULL* if no value is associated with the key in the + current thread. + + +.. _thread-local-storage-api: + +Thread Local Storage (TLS) API +------------------------------ + +.. deprecated:: 3.7 + This API is superseded by + :ref:`Thread Specific Storage (TSS) API <thread-specific-storage-api>`. + +.. note:: + This version of the API does not support platforms where the native TLS key + is defined in a way that cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +Due to the compatibility problem noted above, this version of the API should not +be used in new code. + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() + diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 3e8617ef2a1..ecdd2fe2713 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -127,6 +127,38 @@ built-in ``breakpoint()``. PEP written and implemented by Barry Warsaw +.. _whatsnew37-pep539: + +PEP 539: A New C-API for Thread-Local Storage in CPython +-------------------------------------------------------- + +While Python provides a C API for thread-local storage support; the existing +:ref:`Thread Local Storage (TLS) API <thread-local-storage-api>` has used +:c:type:`int` to represent TLS keys across all platforms. This has not +generally been a problem for officially-support platforms, but that is neither +POSIX-compliant, nor portable in any practical sense. + +:pep:`539` changes this by providing a new :ref:`Thread Specific Storage (TSS) +API <thread-specific-storage-api>` to CPython which supersedes use of the +existing TLS API within the CPython interpreter, while deprecating the existing +API. The TSS API uses a new type :c:type:`Py_tss_t` instead of :c:type:`int` +to represent TSS keys--an opaque type the definition of which may depend on +the underlying TLS implementation. Therefore, this will allow to build CPython +on platforms where the native TLS key is defined in a way that cannot be safely +cast to :c:type:`int`. + +Note that on platforms where the native TLS key is defined in a way that cannot +be safely cast to :c:type:`int`, all functions of the existing TLS API will be +no-op and immediately return failure. This indicates clearly that the old API +is not supported on platforms where it cannot be used reliably, and that no +effort will be made to add such support. + +.. seealso:: + + :pep:`539` -- A New C-API for Thread-Local Storage in CPython + PEP written by Erik M. Bray; implementation by Masayuki Yamamoto. + + Other Language Changes ====================== diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 20c5946b14f..210917bb325 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -26,7 +26,7 @@ struct _gilstate_runtime_state { */ /* TODO: Given interp_main, it may be possible to kill this ref */ PyInterpreterState *autoInterpreterState; - int autoTLSkey; + Py_tss_t autoTSSkey; }; /* hook for PyEval_GetFrame(), requested for Psyco */ diff --git a/Include/pythread.h b/Include/pythread.h index dbacb8bbbcd..d6674685f28 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -29,8 +29,8 @@ PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); PyAPI_FUNC(PyThread_type_lock) PyThread_allocate_lock(void); PyAPI_FUNC(void) PyThread_free_lock(PyThread_type_lock); PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); -#define WAIT_LOCK 1 -#define NOWAIT_LOCK 0 +#define WAIT_LOCK 1 +#define NOWAIT_LOCK 0 /* PY_TIMEOUT_T is the integral type used to specify timeouts when waiting on a lock (see PyThread_acquire_lock_timed() below). @@ -77,15 +77,69 @@ PyAPI_FUNC(int) PyThread_set_stacksize(size_t); PyAPI_FUNC(PyObject*) PyThread_GetInfo(void); #endif -/* Thread Local Storage (TLS) API */ -PyAPI_FUNC(int) PyThread_create_key(void); -PyAPI_FUNC(void) PyThread_delete_key(int); -PyAPI_FUNC(int) PyThread_set_key_value(int, void *); -PyAPI_FUNC(void *) PyThread_get_key_value(int); -PyAPI_FUNC(void) PyThread_delete_key_value(int key); + +/* Thread Local Storage (TLS) API + TLS API is DEPRECATED. Use Thread Specific Storage (TSS) API. + + The existing TLS API has used int to represent TLS keys across all + platforms, but it is not POSIX-compliant. Therefore, the new TSS API uses + opaque data type to represent TSS keys to be compatible (see PEP 539). +*/ +PyAPI_FUNC(int) PyThread_create_key(void) Py_DEPRECATED(3.7); +PyAPI_FUNC(void) PyThread_delete_key(int key) Py_DEPRECATED(3.7); +PyAPI_FUNC(int) PyThread_set_key_value(int key, void *value) Py_DEPRECATED(3.7); +PyAPI_FUNC(void *) PyThread_get_key_value(int key) Py_DEPRECATED(3.7); +PyAPI_FUNC(void) PyThread_delete_key_value(int key) Py_DEPRECATED(3.7); /* Cleanup after a fork */ -PyAPI_FUNC(void) PyThread_ReInitTLS(void); +PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); + + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 +/* New in 3.7 */ +/* Thread Specific Storage (TSS) API */ + +typedef struct _Py_tss_t Py_tss_t; /* opaque */ + +#ifndef Py_LIMITED_API +#if defined(_POSIX_THREADS) + /* Darwin needs pthread.h to know type name the pthread_key_t. */ +# include <pthread.h> +# define NATIVE_TSS_KEY_T pthread_key_t +#elif defined(NT_THREADS) + /* In Windows, native TSS key type is DWORD, + but hardcode the unsigned long to avoid errors for include directive. + */ +# define NATIVE_TSS_KEY_T unsigned long +#else +# error "Require native threads. See https://bugs.python.org/issue31370" +#endif + +/* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is + exposed to allow static allocation in the API clients. Even in this case, + you must handle TSS keys through API functions due to compatibility. +*/ +struct _Py_tss_t { + int _is_initialized; + NATIVE_TSS_KEY_T _key; +}; + +#undef NATIVE_TSS_KEY_T + +/* When static allocation, you must initialize with Py_tss_NEEDS_INIT. */ +#define Py_tss_NEEDS_INIT {0} +#endif /* !Py_LIMITED_API */ + +PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); +PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); + +/* The parameter key must not be NULL. */ +PyAPI_FUNC(int) PyThread_tss_is_created(Py_tss_t *key); +PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); +PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); +PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); +PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); +#endif /* New in 3.7 */ #ifdef __cplusplus } diff --git a/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst b/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst new file mode 100644 index 00000000000..d4bb19acc4b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst @@ -0,0 +1,4 @@ +Implement PEP 539 for Thread Specific Stroage (TSS) API: it is a new Thread +Local Storage (TLS) API to CPython which would supersede use of the existing +TLS API within the CPython interpreter, while deprecating the existing API. +PEP written by Erik M. Bray, patch by Masayuki Yamamoto. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1a296214739..b512c05a3fe 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4306,6 +4306,61 @@ py_w_stopcode(PyObject *self, PyObject *args) #endif +static PyObject * +test_pythread_tss_key_state(PyObject *self, PyObject *args) +{ + Py_tss_t tss_key = Py_tss_NEEDS_INIT; + if (PyThread_tss_is_created(&tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "TSS key not in an uninitialized state at " + "creation time"); + } + if (PyThread_tss_create(&tss_key) != 0) { + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed"); + return NULL; + } + if (!PyThread_tss_is_created(&tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_create succeeded, " + "but with TSS key in an uninitialized state"); + } + if (PyThread_tss_create(&tss_key) != 0) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_create unsuccessful with " + "an already initialized key"); + } +#define CHECK_TSS_API(expr) \ + (void)(expr); \ + if (!PyThread_tss_is_created(&tss_key)) { \ + return raiseTestError("test_pythread_tss_key_state", \ + "TSS key initialization state was not " \ + "preserved after calling " #expr); } + CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); + CHECK_TSS_API(PyThread_tss_get(&tss_key)); +#undef CHECK_TSS_API + PyThread_tss_delete(&tss_key); + if (PyThread_tss_is_created(&tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_delete called, but did not " + "set the key state to uninitialized"); + } + + Py_tss_t *ptr_key = PyThread_tss_alloc(); + if (ptr_key == NULL) { + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_alloc failed"); + return NULL; + } + if (PyThread_tss_is_created(ptr_key)) { + return raiseTestError("test_pythread_tss_key_state", + "TSS key not in an uninitialized state at " + "allocation time"); + } + PyThread_tss_free(ptr_key); + ptr_key = NULL; + Py_RETURN_NONE; +} + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4518,6 +4573,7 @@ static PyMethodDef TestMethods[] = { #ifdef W_STOPCODE {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif + {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 386f2f11ba7..af2a2fa4d52 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -167,14 +167,7 @@ tracemalloc_error(const char *format, ...) #if defined(TRACE_RAW_MALLOC) #define REENTRANT_THREADLOCAL -/* If your OS does not provide native thread local storage, you can implement - it manually using a lock. Functions of thread.c cannot be used because - they use PyMem_RawMalloc() which leads to a reentrant call. */ -#if !(defined(_POSIX_THREADS) || defined(NT_THREADS)) -# error "need native thread local storage (TLS)" -#endif - -static int tracemalloc_reentrant_key = -1; +static Py_tss_t tracemalloc_reentrant_key = Py_tss_NEEDS_INIT; /* Any non-NULL pointer can be used */ #define REENTRANT Py_True @@ -184,8 +177,8 @@ get_reentrant(void) { void *ptr; - assert(tracemalloc_reentrant_key != -1); - ptr = PyThread_get_key_value(tracemalloc_reentrant_key); + assert(PyThread_tss_is_created(&tracemalloc_reentrant_key)); + ptr = PyThread_tss_get(&tracemalloc_reentrant_key); if (ptr != NULL) { assert(ptr == REENTRANT); return 1; @@ -198,15 +191,15 @@ static void set_reentrant(int reentrant) { assert(reentrant == 0 || reentrant == 1); - assert(tracemalloc_reentrant_key != -1); + assert(PyThread_tss_is_created(&tracemalloc_reentrant_key)); if (reentrant) { assert(!get_reentrant()); - PyThread_set_key_value(tracemalloc_reentrant_key, REENTRANT); + PyThread_tss_set(&tracemalloc_reentrant_key, REENTRANT); } else { assert(get_reentrant()); - PyThread_set_key_value(tracemalloc_reentrant_key, NULL); + PyThread_tss_set(&tracemalloc_reentrant_key, NULL); } } @@ -975,8 +968,7 @@ tracemalloc_init(void) PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); #ifdef REENTRANT_THREADLOCAL - tracemalloc_reentrant_key = PyThread_create_key(); - if (tracemalloc_reentrant_key == -1) { + if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) { #ifdef MS_WINDOWS PyErr_SetFromWindowsErr(0); #else @@ -1061,8 +1053,7 @@ tracemalloc_deinit(void) #endif #ifdef REENTRANT_THREADLOCAL - PyThread_delete_key(tracemalloc_reentrant_key); - tracemalloc_reentrant_key = -1; + PyThread_tss_delete(&tracemalloc_reentrant_key); #endif Py_XDECREF(unknown_filename); diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 4f3d971bc52..c2c2c537b12 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -232,7 +232,7 @@ faulthandler_dump_traceback(int fd, int all_threads, PyThreadState_Get() doesn't give the state of the thread that caused the fault if the thread released the GIL, and so this function cannot be - used. Read the thread local storage (TLS) instead: call + used. Read the thread specific storage (TSS) instead: call PyGILState_GetThisThreadState(). */ tstate = PyGILState_GetThisThreadState(); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 13ff9166e71..5f30b202ec6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -454,9 +454,6 @@ PyOS_AfterFork_Parent(void) void PyOS_AfterFork_Child(void) { - /* PyThread_ReInitTLS() must be called early, to make sure that the TLS API - * can be called safely. */ - PyThread_ReInitTLS(); _PyGILState_Reinit(); PyEval_ReInitThreads(); _PyImport_ReInitLock(); diff --git a/PC/python3.def b/PC/python3.def index ad65294045f..1d089ec28b1 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -570,6 +570,13 @@ EXPORTS PyThreadState_New=python37.PyThreadState_New PyThreadState_SetAsyncExc=python37.PyThreadState_SetAsyncExc PyThreadState_Swap=python37.PyThreadState_Swap + PyThread_tss_alloc=python37.PyThread_tss_alloc + PyThread_tss_create=python37.PyThread_tss_create + PyThread_tss_delete=python37.PyThread_tss_delete + PyThread_tss_free=python37.PyThread_tss_free + PyThread_tss_get=python37.PyThread_tss_get + PyThread_tss_is_created=python37.PyThread_tss_is_created + PyThread_tss_set=python37.PyThread_tss_set PyTraceBack_Here=python37.PyTraceBack_Here PyTraceBack_Print=python37.PyTraceBack_Print PyTraceBack_Type=python37.PyTraceBack_Type DATA diff --git a/Python/pystate.c b/Python/pystate.c index 53c12361457..3feae346d44 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -46,7 +46,12 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) _PyEval_Initialize(&runtime->ceval); runtime->gilstate.check_enabled = 1; - runtime->gilstate.autoTLSkey = -1; + /* A TSS key must be initialized with Py_tss_NEEDS_INIT + in accordance with the specification. */ + { + Py_tss_t initial = Py_tss_NEEDS_INIT; + runtime->gilstate.autoTSSkey = initial; + } runtime->interpreters.mutex = PyThread_allocate_lock(); if (runtime->interpreters.mutex == NULL) @@ -485,9 +490,9 @@ PyThreadState_Delete(PyThreadState *tstate) if (tstate == GET_TSTATE()) Py_FatalError("PyThreadState_Delete: tstate is still current"); if (_PyRuntime.gilstate.autoInterpreterState && - PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == tstate) + PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == tstate) { - PyThread_delete_key_value(_PyRuntime.gilstate.autoTLSkey); + PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, NULL); } tstate_delete_common(tstate); } @@ -502,9 +507,9 @@ PyThreadState_DeleteCurrent() "PyThreadState_DeleteCurrent: no current tstate"); tstate_delete_common(tstate); if (_PyRuntime.gilstate.autoInterpreterState && - PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == tstate) + PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == tstate) { - PyThread_delete_key_value(_PyRuntime.gilstate.autoTLSkey); + PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, NULL); } SET_TSTATE(NULL); PyEval_ReleaseLock(); @@ -761,11 +766,11 @@ void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) { assert(i && t); /* must init with valid states */ - _PyRuntime.gilstate.autoTLSkey = PyThread_create_key(); - if (_PyRuntime.gilstate.autoTLSkey == -1) - Py_FatalError("Could not allocate TLS entry"); + if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) { + Py_FatalError("Could not allocate TSS entry"); + } _PyRuntime.gilstate.autoInterpreterState = i; - assert(PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == NULL); + assert(PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == NULL); assert(t->gilstate_counter == 0); _PyGILState_NoteThreadState(t); @@ -780,14 +785,13 @@ _PyGILState_GetInterpreterStateUnsafe(void) void _PyGILState_Fini(void) { - PyThread_delete_key(_PyRuntime.gilstate.autoTLSkey); - _PyRuntime.gilstate.autoTLSkey = -1; + PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey); _PyRuntime.gilstate.autoInterpreterState = NULL; } -/* Reset the TLS key - called by PyOS_AfterFork_Child(). +/* Reset the TSS key - called by PyOS_AfterFork_Child(). * This should not be necessary, but some - buggy - pthread implementations - * don't reset TLS upon fork(), see issue #10517. + * don't reset TSS upon fork(), see issue #10517. */ void _PyGILState_Reinit(void) @@ -796,15 +800,18 @@ _PyGILState_Reinit(void) if (_PyRuntime.interpreters.mutex == NULL) Py_FatalError("Can't initialize threads for interpreter"); PyThreadState *tstate = PyGILState_GetThisThreadState(); - PyThread_delete_key(_PyRuntime.gilstate.autoTLSkey); - if ((_PyRuntime.gilstate.autoTLSkey = PyThread_create_key()) == -1) - Py_FatalError("Could not allocate TLS entry"); + PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey); + if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) { + Py_FatalError("Could not allocate TSS entry"); + } /* If the thread had an associated auto thread state, reassociate it with * the new key. */ - if (tstate && PyThread_set_key_value(_PyRuntime.gilstate.autoTLSkey, - (void *)tstate) < 0) - Py_FatalError("Couldn't create autoTLSkey mapping"); + if (tstate && + PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, (void *)tstate) != 0) + { + Py_FatalError("Couldn't create autoTSSkey mapping"); + } } /* When a thread state is created for a thread by some mechanism other than @@ -815,13 +822,13 @@ _PyGILState_Reinit(void) static void _PyGILState_NoteThreadState(PyThreadState* tstate) { - /* If autoTLSkey isn't initialized, this must be the very first + /* If autoTSSkey isn't initialized, this must be the very first threadstate created in Py_Initialize(). Don't do anything for now (we'll be back here when _PyGILState_Init is called). */ if (!_PyRuntime.gilstate.autoInterpreterState) return; - /* Stick the thread state for this thread in thread local storage. + /* Stick the thread state for this thread in thread specific storage. The only situation where you can legitimately have more than one thread state for an OS level thread is when there are multiple @@ -833,12 +840,11 @@ _PyGILState_NoteThreadState(PyThreadState* tstate) The first thread state created for that given OS level thread will "win", which seems reasonable behaviour. */ - if (PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == NULL) { - if ((PyThread_set_key_value(_PyRuntime.gilstate.autoTLSkey, - (void *)tstate) - ) < 0) + if (PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == NULL) { + if ((PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, (void *)tstate) + ) != 0) { - Py_FatalError("Couldn't create autoTLSkey mapping"); + Py_FatalError("Couldn't create autoTSSkey mapping"); } } @@ -852,8 +858,7 @@ PyGILState_GetThisThreadState(void) { if (_PyRuntime.gilstate.autoInterpreterState == NULL) return NULL; - return (PyThreadState *)PyThread_get_key_value( - _PyRuntime.gilstate.autoTLSkey); + return (PyThreadState *)PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey); } int @@ -864,8 +869,9 @@ PyGILState_Check(void) if (!_PyGILState_check_enabled) return 1; - if (_PyRuntime.gilstate.autoTLSkey == -1) + if (!PyThread_tss_is_created(&_PyRuntime.gilstate.autoTSSkey)) { return 1; + } tstate = GET_TSTATE(); if (tstate == NULL) @@ -886,8 +892,7 @@ PyGILState_Ensure(void) */ /* Py_Initialize() hasn't been called! */ assert(_PyRuntime.gilstate.autoInterpreterState); - tcur = (PyThreadState *)PyThread_get_key_value( - _PyRuntime.gilstate.autoTLSkey); + tcur = (PyThreadState *)PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey); if (tcur == NULL) { /* At startup, Python has no concrete GIL. If PyGILState_Ensure() is called from a new thread for the first time, we need the create the @@ -919,8 +924,8 @@ PyGILState_Ensure(void) void PyGILState_Release(PyGILState_STATE oldstate) { - PyThreadState *tcur = (PyThreadState *)PyThread_get_key_value( - _PyRuntime.gilstate.autoTLSkey); + PyThreadState *tcur = (PyThreadState *)PyThread_tss_get( + &_PyRuntime.gilstate.autoTSSkey); if (tcur == NULL) Py_FatalError("auto-releasing thread-state, " "but no thread-state for this thread"); diff --git a/Python/thread.c b/Python/thread.c index f742d0521e1..7eac836dc16 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -84,7 +84,7 @@ PyThread_init_thread(void) # define PYTHREAD_NAME "nt" # include "thread_nt.h" #else -# error "Require native thread feature. See https://bugs.python.org/issue30832" +# error "Require native threads. See https://bugs.python.org/issue31370" #endif @@ -111,41 +111,37 @@ PyThread_set_stacksize(size_t size) } -/* ------------------------------------------------------------------------ -Per-thread data ("key") support. +/* Thread Specific Storage (TSS) API -Use PyThread_create_key() to create a new key. This is typically shared -across threads. - -Use PyThread_set_key_value(thekey, value) to associate void* value with -thekey in the current thread. Each thread has a distinct mapping of thekey -to a void* value. Caution: if the current thread already has a mapping -for thekey, value is ignored. - -Use PyThread_get_key_value(thekey) to retrieve the void* value associated -with thekey in the current thread. This returns NULL if no value is -associated with thekey in the current thread. - -Use PyThread_delete_key_value(thekey) to forget the current thread's associated -value for thekey. PyThread_delete_key(thekey) forgets the values associated -with thekey across *all* threads. - -While some of these functions have error-return values, none set any -Python exception. + Cross-platform components of TSS API implementation. +*/ -None of the functions does memory management on behalf of the void* values. -You need to allocate and deallocate them yourself. If the void* values -happen to be PyObject*, these functions don't do refcount operations on -them either. +Py_tss_t * +PyThread_tss_alloc(void) +{ + Py_tss_t *new_key = (Py_tss_t *)PyMem_RawMalloc(sizeof(Py_tss_t)); + if (new_key == NULL) { + return NULL; + } + new_key->_is_initialized = 0; + return new_key; +} -The GIL does not need to be held when calling these functions; they supply -their own locking. This isn't true of PyThread_create_key(), though (see -next paragraph). +void +PyThread_tss_free(Py_tss_t *key) +{ + if (key != NULL) { + PyThread_tss_delete(key); + PyMem_RawFree((void *)key); + } +} -There's a hidden assumption that PyThread_create_key() will be called before -any of the other functions are called. There's also a hidden assumption -that calls to PyThread_create_key() are serialized externally. ------------------------------------------------------------------------- */ +int +PyThread_tss_is_created(Py_tss_t *key) +{ + assert(key != NULL); + return key->_is_initialized; +} PyDoc_STRVAR(threadinfo__doc__, diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 2f3a71b86ad..bae8bcc3566 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -349,10 +349,15 @@ _pythread_nt_set_stacksize(size_t size) #define THREAD_SET_STACKSIZE(x) _pythread_nt_set_stacksize(x) +/* Thread Local Storage (TLS) API + + This API is DEPRECATED since Python 3.7. See PEP 539 for details. +*/ + int PyThread_create_key(void) { - DWORD result= TlsAlloc(); + DWORD result = TlsAlloc(); if (result == TLS_OUT_OF_INDEXES) return -1; return (int)result; @@ -367,12 +372,8 @@ PyThread_delete_key(int key) int PyThread_set_key_value(int key, void *value) { - BOOL ok; - - ok = TlsSetValue(key, value); - if (!ok) - return -1; - return 0; + BOOL ok = TlsSetValue(key, value); + return ok ? 0 : -1; } void * @@ -399,9 +400,74 @@ PyThread_delete_key_value(int key) TlsSetValue(key, NULL); } + /* reinitialization of TLS is not necessary after fork when using * the native TLS functions. And forking isn't supported on Windows either. */ void PyThread_ReInitTLS(void) -{} +{ +} + + +/* Thread Specific Storage (TSS) API + + Platform-specific components of TSS API implementation. +*/ + +int +PyThread_tss_create(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has been created, function is silently skipped. */ + if (key->_is_initialized) { + return 0; + } + + DWORD result = TlsAlloc(); + if (result == TLS_OUT_OF_INDEXES) { + return -1; + } + /* In Windows, platform-specific key type is DWORD. */ + key->_key = result; + key->_is_initialized = 1; + return 0; +} + +void +PyThread_tss_delete(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has not been created, function is silently skipped. */ + if (!key->_is_initialized) { + return; + } + + TlsFree(key->_key); + key->_key = TLS_OUT_OF_INDEXES; + key->_is_initialized = 0; +} + +int +PyThread_tss_set(Py_tss_t *key, void *value) +{ + assert(key != NULL); + BOOL ok = TlsSetValue(key->_key, value); + return ok ? 0 : -1; +} + +void * +PyThread_tss_get(Py_tss_t *key) +{ + assert(key != NULL); + /* because TSS is used in the Py_END_ALLOW_THREAD macro, + * it is necessary to preserve the windows error state, because + * it is assumed to be preserved across the call to the macro. + * Ideally, the macro should be fixed, but it is simpler to + * do it here. + */ + DWORD error = GetLastError(); + void *result = TlsGetValue(key->_key); + SetLastError(error); + return result; +} diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 2dcd107c2ce..c5b7f3256fa 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -589,9 +589,25 @@ _pythread_pthread_set_stacksize(size_t size) #define THREAD_SET_STACKSIZE(x) _pythread_pthread_set_stacksize(x) +/* Thread Local Storage (TLS) API + + This API is DEPRECATED since Python 3.7. See PEP 539 for details. +*/ + +/* Issue #25658: On platforms where native TLS key is defined in a way that + cannot be safely cast to int, PyThread_create_key returns immediately a + failure status and other TLS functions all are no-ops. This indicates + clearly that the old API is not supported on platforms where it cannot be + used reliably, and that no effort will be made to add such support. + + Note: PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT will be unnecessary after + removing this API. +*/ + int PyThread_create_key(void) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT pthread_key_t key; int fail = pthread_key_create(&key, NULL); if (fail) @@ -603,34 +619,102 @@ PyThread_create_key(void) return -1; } return (int)key; +#else + return -1; /* never return valid key value. */ +#endif } void PyThread_delete_key(int key) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT pthread_key_delete(key); +#endif } void PyThread_delete_key_value(int key) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT pthread_setspecific(key, NULL); +#endif } int PyThread_set_key_value(int key, void *value) { - int fail; - fail = pthread_setspecific(key, value); +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT + int fail = pthread_setspecific(key, value); return fail ? -1 : 0; +#else + return -1; +#endif } void * PyThread_get_key_value(int key) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT return pthread_getspecific(key); +#else + return NULL; +#endif } + void PyThread_ReInitTLS(void) -{} +{ +} + + +/* Thread Specific Storage (TSS) API + + Platform-specific components of TSS API implementation. +*/ + +int +PyThread_tss_create(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has been created, function is silently skipped. */ + if (key->_is_initialized) { + return 0; + } + + int fail = pthread_key_create(&(key->_key), NULL); + if (fail) { + return -1; + } + key->_is_initialized = 1; + return 0; +} + +void +PyThread_tss_delete(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has not been created, function is silently skipped. */ + if (!key->_is_initialized) { + return; + } + + pthread_key_delete(key->_key); + /* pthread has not provided the defined invalid value for the key. */ + key->_is_initialized = 0; +} + +int +PyThread_tss_set(Py_tss_t *key, void *value) +{ + assert(key != NULL); + int fail = pthread_setspecific(key->_key, value); + return fail ? -1 : 0; +} + +void * +PyThread_tss_get(Py_tss_t *key) +{ + assert(key != NULL); + return pthread_getspecific(key->_key); +} diff --git a/Python/traceback.c b/Python/traceback.c index ba979aad8fa..21b36b14719 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -760,7 +760,7 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState_Get() doesn't give the state of the thread that caused the fault if the thread released the GIL, and so this function - cannot be used. Read the thread local storage (TLS) instead: call + cannot be used. Read the thread specific storage (TSS) instead: call PyGILState_GetThisThreadState(). */ current_tstate = PyGILState_GetThisThreadState(); } diff --git a/configure b/configure index 2d1bed22476..741a834ab38 100755 --- a/configure +++ b/configure @@ -8884,6 +8884,75 @@ _ACEOF fi + +# Issue #25658: POSIX hasn't defined that pthread_key_t is compatible with int. +# This checking will be unnecessary after removing deprecated TLS API. +# The cast to long int works around a bug in the HP C Compiler +# version HP92453-01 B.11.11.23709.GP, which incorrectly rejects +# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# This bug is HP SR number 8606223364. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking size of pthread_key_t" >&5 +$as_echo_n "checking size of pthread_key_t... " >&6; } +if ${ac_cv_sizeof_pthread_key_t+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_key_t))" "ac_cv_sizeof_pthread_key_t" "#include <pthread.h> +"; then : + +else + if test "$ac_cv_type_pthread_key_t" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute sizeof (pthread_key_t) +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_sizeof_pthread_key_t=0 + fi +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pthread_key_t" >&5 +$as_echo "$ac_cv_sizeof_pthread_key_t" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define SIZEOF_PTHREAD_KEY_T $ac_cv_sizeof_pthread_key_t +_ACEOF + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether pthread_key_t is compatible with int" >&5 +$as_echo_n "checking whether pthread_key_t is compatible with int... " >&6; } +if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <pthread.h> +int +main () +{ +pthread_key_t k; k * 1; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_pthread_key_t_is_arithmetic_type=yes +else + ac_pthread_key_t_is_arithmetic_type=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pthread_key_t_is_arithmetic_type" >&5 +$as_echo "$ac_pthread_key_t_is_arithmetic_type" >&6; } + if test "$ac_pthread_key_t_is_arithmetic_type" = yes ; then + +$as_echo "#define PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT 1" >>confdefs.h + + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi CC="$ac_save_cc" diff --git a/configure.ac b/configure.ac index b562fe405a4..a3114be7387 100644 --- a/configure.ac +++ b/configure.ac @@ -2263,6 +2263,25 @@ if test "$have_pthread_t" = yes ; then #endif ]) fi + +# Issue #25658: POSIX hasn't defined that pthread_key_t is compatible with int. +# This checking will be unnecessary after removing deprecated TLS API. +AC_CHECK_SIZEOF(pthread_key_t, [], [[#include <pthread.h>]]) +AC_MSG_CHECKING(whether pthread_key_t is compatible with int) +if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[#include <pthread.h>]], [[pthread_key_t k; k * 1;]])], + [ac_pthread_key_t_is_arithmetic_type=yes], + [ac_pthread_key_t_is_arithmetic_type=no] + ) + AC_MSG_RESULT($ac_pthread_key_t_is_arithmetic_type) + if test "$ac_pthread_key_t_is_arithmetic_type" = yes ; then + AC_DEFINE(PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT, 1, + [Define if pthread_key_t is compatible with int.]) + fi +else + AC_MSG_RESULT(no) +fi CC="$ac_save_cc" AC_SUBST(OTHER_LIBTOOL_OPT) diff --git a/pyconfig.h.in b/pyconfig.h.in index 637341afee3..7f0c6b33a95 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1241,6 +1241,9 @@ /* Define if POSIX semaphores aren't enabled on your system */ #undef POSIX_SEMAPHORES_NOT_ENABLED +/* Define if pthread_key_t is compatible with int. */ +#undef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT + /* Defined if PTHREAD_SCOPE_SYSTEM supported. */ #undef PTHREAD_SYSTEM_SCHED_SUPPORTED @@ -1302,6 +1305,9 @@ /* The size of `pid_t', as computed by sizeof. */ #undef SIZEOF_PID_T +/* The size of `pthread_key_t', as computed by sizeof. */ +#undef SIZEOF_PTHREAD_KEY_T + /* The size of `pthread_t', as computed by sizeof. */ #undef SIZEOF_PTHREAD_T |