Merge pull request #1355 from davidhewitt/finalization

gil: tidy ups to finalization

CHANGELOG.md

@@ -7,8 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 ### Added
+- Add unsafe API `with_embedded_python_interpreter` to initalize a Python interpreter, execute a closure, and finalize the interpreter. [#1355](https://github.com/PyO3/pyo3/pull/1355)
 - Add `serde` feature to support `Serialize/Deserialize` for `Py<T>`. [#1366](https://github.com/PyO3/pyo3/pull/1366)
 
+### Changed
+- `prepare_freethreaded_python` will no longer register an `atexit` handler to call `Py_Finalize`. [#1355](https://github.com/PyO3/pyo3/pull/1355)
+
 ## [0.13.1] - 2021-01-10
 ### Added
 - Add support for `#[pyclass(dict)]` and `#[pyclass(weakref)]` with the `abi3` feature on Python 3.9 and up. [#1342](https://github.com/PyO3/pyo3/pull/1342)

guide/src/features.md

@@ -36,6 +36,8 @@ This feature changes [`Python::with_gil`](https://docs.rs/pyo3/latest/pyo3/struc
 
 This feature is not needed for extension modules, but for compatibility it is enabled by default until at least the PyO3 0.14 release.
 
+If you choose not to enable this feature, you should call `pyo3::prepare_freethreaded_python()` before attempting to call any other Python APIs.
+
 > This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml.
 
 ## Advanced Features

src/gil.rs

@@ -46,68 +46,136 @@ pub(crate) fn gil_is_acquired() -> bool {
 /// this function has no effect.
 ///
 /// # Availability
-///
 /// This function is only available when linking against Python distributions that contain a
 /// shared library.
 ///
 /// This function is not available on PyPy.
 ///
-/// # Panic
-/// If the Python interpreter is initialized but Python threading is not,
-/// a panic occurs.
-/// It is not possible to safely access the Python runtime unless the main
-/// thread (the thread which originally initialized Python) also initializes
-/// threading.
+/// # Panics
+/// - If the Python interpreter is initialized but Python threading is not,
+///   a panic occurs.
+///   It is not possible to safely access the Python runtime unless the main
+///   thread (the thread which originally initialized Python) also initializes
+///   threading.
+///
+/// # Example
+/// ```rust
+/// use pyo3::prelude::*;
+///
+/// # #[allow(clippy::needless_doctest_main)]
+/// fn main() {
+///     pyo3::prepare_freethreaded_python();
+///     Python::with_gil(|py| {
+///         py.run("print('Hello World')", None, None)
+///     });
+/// }
+/// ```
 #[cfg(all(Py_SHARED, not(PyPy)))]
 pub fn prepare_freethreaded_python() {
-    // Protect against race conditions when Python is not yet initialized
-    // and multiple threads concurrently call 'prepare_freethreaded_python()'.
-    // Note that we do not protect against concurrent initialization of the Python runtime
-    // by other users of the Python C API.
-    START.call_once(|| unsafe {
+    // Protect against race conditions when Python is not yet initialized and multiple threads
+    // concurrently call 'prepare_freethreaded_python()'. Note that we do not protect against
+    // concurrent initialization of the Python runtime by other users of the Python C API.
+    START.call_once_force(|_| unsafe {
+        // Use call_once_force because if initialization panics, it's okay to try again.
         if ffi::Py_IsInitialized() != 0 {
             // If Python is already initialized, we expect Python threading to also be initialized,
             // as we can't make the existing Python main thread acquire the GIL.
             assert_ne!(ffi::PyEval_ThreadsInitialized(), 0);
         } else {
-            // Initialize Python.
-            // We use Py_InitializeEx() with initsigs=0 to disable Python signal handling.
-            // Signal handling depends on the notion of a 'main thread', which doesn't exist in this case.
-            // Note that the 'main thread' notion in Python isn't documented properly;
-            // and running Python without one is not officially supported.
-
             ffi::Py_InitializeEx(0);
 
-            // Make sure Py_Finalize will be called before exiting.
-            extern "C" fn finalize() {
-                unsafe {
-                    if ffi::Py_IsInitialized() != 0 {
-                        ffi::PyGILState_Ensure();
-                        ffi::Py_Finalize();
-                    }
-                }
-            }
-            libc::atexit(finalize);
-
-            // > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
-            // > to call it yourself anymore.
+            // Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t
+            // have to call it yourself anymore.
             #[cfg(not(Py_3_7))]
             if ffi::PyEval_ThreadsInitialized() == 0 {
                 ffi::PyEval_InitThreads();
             }
 
-            // Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
-            // (it's not acquired in the other code paths)
-            // So immediately release the GIL:
-            let _thread_state = ffi::PyEval_SaveThread();
-            // Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
-            // and will be restored by PyGILState_Ensure.
+            // Release the GIL.
+            ffi::PyEval_SaveThread();
         }
     });
 }
 
-/// RAII type that represents the Global Interpreter Lock acquisition. To get hold of a value
-/// of this type, see [`Python::acquire_gil`](struct.Python.html#method.acquire_gil).
+/// Executes the provided closure with an embedded Python interpreter.
+///
+/// This function intializes the Python interpreter, executes the provided closure, and then
+/// finalizes the Python interpreter.
+///
+/// After execution all Python resources are cleaned up, and no further Python APIs can be called.
+/// Because many Python modules implemented in C do not support multiple Python interpreters in a
+/// single process, it is not safe to call this function more than once. (Many such modules will not
+/// initialize correctly on the second run.)
+///
+/// # Availability
+/// This function is only available when linking against Python distributions that contain a shared
+/// library.
+///
+/// This function is not available on PyPy.
+///
+/// # Panics
+/// - If the Python interpreter is already initalized before calling this function.
+///
+/// # Safety
+/// - This function should only ever be called once per process (usually as part of the `main`
+///   function). It is also not thread-safe.
+/// - No Python APIs can be used after this function has finished executing.
+/// - The return value of the closure must not contain any Python value, _including_ `PyResult`.
+///
+/// # Example
+/// ```rust
+/// use pyo3::prelude::*;
+///
+/// # #[allow(clippy::needless_doctest_main)]
+/// fn main() {
+///     unsafe {
+///         pyo3::with_embedded_python_interpreter(|py| {
+///             py.run("print('Hello World')", None, None)
+///         });
+///     }
+/// }
+/// ```
+#[cfg(all(Py_SHARED, not(PyPy)))]
+pub unsafe fn with_embedded_python_interpreter<F, R>(f: F) -> R
+where
+    F: for<'p> FnOnce(Python<'p>) -> R,
+{
+    assert_eq!(
+        ffi::Py_IsInitialized(),
+        0,
+        "called `with_embedded_python_interpreter` but a Python interpreter is already running."
+    );
+
+    ffi::Py_InitializeEx(0);
+
+    // Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have to
+    // call it yourself anymore.
+    #[cfg(not(Py_3_7))]
+    if ffi::PyEval_ThreadsInitialized() == 0 {
+        ffi::PyEval_InitThreads();
+    }
+
+    // Safe: the GIL is already held because of the Py_IntializeEx call.
+    let pool = GILPool::new();
+
+    // Import the threading module - this ensures that it will associate this thread as the "main"
+    // thread, which is important to avoid an `AssertionError` at finalization.
+    pool.python().import("threading").unwrap();
+
+    // Execute the closure.
+    let result = f(pool.python());
+
+    // Drop the pool before finalizing.
+    drop(pool);
+
+    // Finalize the Python interpreter.
+    ffi::Py_Finalize();
+
+    result
+}
+
+/// RAII type that represents the Global Interpreter Lock acquisition. To get hold of a value of
+/// this type, see [`Python::acquire_gil`](struct.Python.html#method.acquire_gil).
 ///
 /// # Example
 /// ```
@@ -133,13 +201,13 @@ impl GILGuard {
 
     /// PyO3 internal API for acquiring the GIL. The public API is Python::acquire_gil.
     ///
-    /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this
-    /// new `GILGuard` will also contain a `GILPool`.
+    /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this new
+    /// `GILGuard` will also contain a `GILPool`.
     pub(crate) fn acquire() -> GILGuard {
         // Maybe auto-initialize the GIL:
         //  - If auto-initialize feature set and supported, try to initalize the interpreter.
-        //  - If the auto-initialize feature is set but unsupported, emit hard errors only when
-        //    the extension-module feature is not activated - extension modules don't care about
+        //  - If the auto-initialize feature is set but unsupported, emit hard errors only when the
+        //    extension-module feature is not activated - extension modules don't care about
         //    auto-initialize so this avoids breaking existing builds.
         //  - Otherwise, just check the GIL is initialized.
         cfg_if::cfg_if! {
@@ -170,8 +238,9 @@ impl GILGuard {
                 // extension module feature enabled and PyPy or static linking
                 // OR auto-initialize feature not enabled
                 START.call_once_force(|_| unsafe {
-                    // Use call_once_force because if there is a panic because the interpreter is not
-                    // initialized, it's fine for the user to initialize the interpreter and retry.
+                    // Use call_once_force because if there is a panic because the interpreter is
+                    // not initialized, it's fine for the user to initialize the interpreter and
+                    // retry.
                     assert_ne!(
                         ffi::Py_IsInitialized(),
                         0,

src/lib.rs

@@ -152,7 +152,7 @@ pub use crate::conversion::{
 };
 pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
 #[cfg(all(Py_SHARED, not(PyPy)))]
-pub use crate::gil::prepare_freethreaded_python;
+pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter};
 pub use crate::gil::{GILGuard, GILPool};
 pub use crate::instance::{Py, PyNativeType, PyObject};
 pub use crate::pycell::{PyCell, PyRef, PyRefMut};