ffi: define some cpython/unicodeobject bindings

pyo3 doesn't currently define various Unicode bindings that allow the
retrieval of raw data from Python strings. Said bindings are a
prerequisite to possibly exposing this data in the Rust API (PyO3#1776).
Even if those high-level APIs never materialize, the FFI bindings are
necessary to enable consumers of the raw C API to utilize them.

This commit partially defines the FFI bindings as defined in
CPython's Include/cpython/unicodeobject.h file.

I used the latest CPython 3.9 Git commit for defining the order
of the symbols and the implementation of various inline preprocessor
macros. I tried to be as faithful as possible to the original
implementation, preserving intermediate `#define`s as inline functions.

The structs are a bit wonky and probably warrant the most review
scrutiny. I haven't tested this code thoroughly.

Missing symbols have been annotated with `skipped` and symbols currently
defined in `src/ffi/unicodeobject.rs` have been annotated with `move`.

src/ffi/cpython/mod.rs

@@ -22,6 +22,9 @@ pub(crate) mod pylifecycle;
 #[cfg(all(Py_3_8, not(PyPy)))]
 pub(crate) mod pystate;
 
+#[cfg(not(PyPy))]
+pub(crate) mod unicodeobject;
+
 pub use self::abstract_::*;
 #[cfg(not(PyPy))]
 pub use self::bytesobject::*;
@@ -42,3 +45,6 @@ pub use self::pylifecycle::*;
 
 #[cfg(all(Py_3_8, not(PyPy)))]
 pub use self::pystate::*;
+
+#[cfg(not(PyPy))]
+pub use self::unicodeobject::*;

src/ffi/cpython/unicodeobject.rs

@@ -0,0 +1,314 @@
+use crate::ffi::{PyObject, PyUnicode_Check, Py_UCS1, Py_UCS2, Py_UCS4, Py_hash_t, Py_ssize_t};
+use libc::wchar_t;
+use std::os::raw::{c_char, c_int, c_uint, c_void};
+
+// skipped Py_UNICODE_ISSPACE()
+// skipped Py_UNICODE_ISLOWER()
+// skipped Py_UNICODE_ISUPPER()
+// skipped Py_UNICODE_ISTITLE()
+// skipped Py_UNICODE_ISLINEBREAK
+// skipped Py_UNICODE_TOLOWER
+// skipped Py_UNICODE_TOUPPER
+// skipped Py_UNICODE_TOTITLE
+// skipped Py_UNICODE_ISDECIMAL
+// skipped Py_UNICODE_ISDIGIT
+// skipped Py_UNICODE_ISNUMERIC
+// skipped Py_UNICODE_ISPRINTABLE
+// skipped Py_UNICODE_TODECIMAL
+// skipped Py_UNICODE_TODIGIT
+// skipped Py_UNICODE_TONUMERIC
+// skipped Py_UNICODE_ISALPHA
+// skipped Py_UNICODE_ISALNUM
+// skipped Py_UNICODE_COPY
+// skipped Py_UNICODE_FILL
+// skipped Py_UNICODE_IS_SURROGATE
+// skipped Py_UNICODE_IS_HIGH_SURROGATE
+// skipped Py_UNICODE_IS_LOW_SURROGATE
+// skipped Py_UNICODE_JOIN_SURROGATES
+// skipped Py_UNICODE_HIGH_SURROGATE
+// skipped Py_UNICODE_LOW_SURROGATE
+
+// This type is defined inline in CPython source.
+#[repr(C)]
+pub struct PyASCIIObjectState {
+    // SSTATE_* constants.
+    pub interned: c_uint,
+    // PyUnicode_*_KIND constants.
+    pub kind: c_uint,
+    pub compact: c_uint,
+    pub ascii: c_uint,
+    pub ready: c_uint,
+    pub _a: c_uint,
+}
+
+#[repr(C)]
+pub struct PyASCIIObject {
+    pub ob_base: PyObject,
+    pub length: Py_ssize_t,
+    pub hash: Py_hash_t,
+    pub state: PyASCIIObjectState,
+    pub wstr: *mut wchar_t,
+}
+
+#[repr(C)]
+pub struct PyCompactUnicodeObject {
+    pub _base: PyASCIIObject,
+    pub utf8_length: Py_ssize_t,
+    pub utf8: *mut c_char,
+    pub wstr_length: Py_ssize_t,
+}
+
+#[repr(C)]
+pub union PyUnicodeObjectData {
+    any: *mut c_void,
+    latin1: *mut Py_UCS1,
+    ucs2: *mut Py_UCS2,
+    ucs4: *mut Py_UCS4,
+}
+
+#[repr(C)]
+pub struct PyUnicodeObject {
+    pub _base: PyCompactUnicodeObject,
+    pub data: PyUnicodeObjectData,
+}
+
+extern "C" {
+    pub fn _PyUnicode_CheckConsistency(op: *mut PyObject, check_content: c_int) -> c_int;
+}
+
+// skipped PyUnicode_GET_SIZE
+// skipped PyUnicode_GET_DATA_SIZE
+// skipped PyUnicode_AS_UNICODE
+// skipped PyUnicode_AS_DATA
+
+pub const SSTATE_NOT_INTERNED: c_uint = 0;
+pub const SSTATE_INTERNED_MORTAL: c_uint = 1;
+pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2;
+
+#[inline]
+pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint {
+    debug_assert!(PyUnicode_Check(op) != 0);
+    debug_assert!(PyUnicode_IS_READY(op) != 0);
+
+    (*(op as *mut PyASCIIObject)).state.ascii
+}
+
+#[inline]
+pub unsafe fn PyUnicode_IS_COMPACT(op: *mut PyObject) -> c_uint {
+    (*(op as *mut PyASCIIObject)).state.compact
+}
+
+#[inline]
+pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint {
+    if (*(op as *mut PyASCIIObject)).state.ascii != 0 && PyUnicode_IS_COMPACT(op) != 0 {
+        1
+    } else {
+        0
+    }
+}
+
+pub const PyUnicode_WCHAR_KIND: c_uint = 0;
+pub const PyUnicode_1BYTE_KIND: c_uint = 1;
+pub const PyUnicode_2BYTE_KIND: c_uint = 2;
+pub const PyUnicode_4BYTE_KIND: c_uint = 4;
+
+#[inline]
+pub unsafe fn PyUnicode_1BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS1 {
+    PyUnicode_DATA(op) as *mut Py_UCS1
+}
+
+#[inline]
+pub unsafe fn PyUnicode_2BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS2 {
+    PyUnicode_DATA(op) as *mut Py_UCS2
+}
+
+#[inline]
+pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 {
+    PyUnicode_DATA(op) as *mut Py_UCS4
+}
+
+#[inline]
+pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint {
+    debug_assert!(PyUnicode_Check(op) != 0);
+    debug_assert!(PyUnicode_IS_READY(op) != 0);
+
+    (*(op as *mut PyASCIIObject)).state.kind
+}
+
+#[inline]
+pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void {
+    if PyUnicode_IS_ASCII(op) != 0 {
+        (op as *mut PyASCIIObject).offset(1) as *mut c_void
+    } else {
+        (op as *mut PyCompactUnicodeObject).offset(1) as *mut c_void
+    }
+}
+
+#[inline]
+pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void {
+    debug_assert!(!(*(op as *mut PyUnicodeObject)).data.any.is_null());
+
+    (*(op as *mut PyUnicodeObject)).data.any
+}
+
+#[inline]
+pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void {
+    debug_assert!(PyUnicode_Check(op) != 0);
+
+    if PyUnicode_IS_COMPACT(op) != 0 {
+        _PyUnicode_COMPACT_DATA(op)
+    } else {
+        _PyUnicode_NONCOMPACT_DATA(op)
+    }
+}
+
+// skipped PyUnicode_WRITE
+// skipped PyUnicode_READ
+// skipped PyUnicode_READ_CHAR
+
+#[inline]
+pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t {
+    debug_assert!(PyUnicode_Check(op) != 0);
+    debug_assert!(PyUnicode_IS_READY(op) != 0);
+
+    (*(op as *mut PyASCIIObject)).length
+}
+
+#[inline]
+pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint {
+    (*(op as *mut PyASCIIObject)).state.ready
+}
+
+#[inline]
+pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int {
+    debug_assert!(PyUnicode_Check(op) != 0);
+
+    if PyUnicode_IS_READY(op) != 0 {
+        0
+    } else {
+        _PyUnicode_Ready(op)
+    }
+}
+
+// skipped PyUnicode_MAX_CHAR_VALUE
+// skipped _PyUnicode_get_wstr_length
+// skipped PyUnicode_WSTR_LENGTH
+
+extern "C" {
+    // move PyUnicode_New
+
+    pub fn _PyUnicode_Ready(unicode: *mut PyObject) -> c_int;
+
+    // skipped _PyUnicode_Copy
+    // move PyUnicode_CopyCharacters
+    // skipped _PyUnicode_FastCopyCharacters
+    // move PyUnicode_Fill
+    // skipped _PyUnicode_FastFill
+    // move PyUnicode_FromUnicode
+    // move PyUnicode_FromKindAndData
+    // skipped _PyUnicode_FromASCII
+    // skipped _PyUnicode_FindMaxChar
+    // move PyUnicode_AsUnicode
+    // skipped _PyUnicode_AsUnicode
+    // move PyUnicode_AsUnicodeAndSize
+    // skipped PyUnicode_GetMax
+}
+
+// skipped _PyUnicodeWriter
+// skipped _PyUnicodeWriter_Init
+// skipped _PyUnicodeWriter_Prepare
+// skipped _PyUnicodeWriter_PrepareInternal
+// skipped _PyUnicodeWriter_PrepareKind
+// skipped _PyUnicodeWriter_PrepareKindInternal
+// skipped _PyUnicodeWriter_WriteChar
+// skipped _PyUnicodeWriter_WriteStr
+// skipped _PyUnicodeWriter_WriteSubstring
+// skipped _PyUnicodeWriter_WriteASCIIString
+// skipped _PyUnicodeWriter_WriteLatin1String
+// skipped _PyUnicodeWriter_Finish
+// skipped _PyUnicodeWriter_Dealloc
+// skipped _PyUnicode_FormatAdvancedWriter
+
+// move PyUnicode_AsUTF8AndSize
+// skipped _PyUnicode_AsStringAndSize
+// move PyUnicode_AsUTF8
+// skipped _PyUnicode_AsString
+
+// move PyUnicode_Encode
+// move PyUnicode_EncodeUTF7
+// skipped _PyUnicode_EncodeUTF7
+
+// skipped _PyUnicode_AsUTF8String
+// move PyUnicode_EncodeUTF8
+
+// move PyUnicode_EncodeUTF32
+// skipped _PyUnicode_EncodeUTF32
+
+// move PyUnicode_EncodeUTF16
+// skipped _PyUnicode_EncodeUTF16
+
+// skipped _PyUnicode_DecodeUnicodeEscape
+// move PyUnicode_EncodeUnicodeEscape
+// move PyUnicode_EncodeRawUnicodeEscape
+
+// skipped _PyUnicode_AsLatin1String
+// move PyUnicode_EncodeLatin1
+
+// skipped _PyUnicode_AsASCIIString
+// move PyUnicode_EncodeASCII
+
+// move PyUnicode_EncodeCharmap
+// skipped _PyUnicode_EncodeCharmap
+// move PyUnicode_TranslateCharmap
+
+// skipped PyUnicode_EncodeMBCS
+
+// move PyUnicode_EncodeDecimal
+// move PyUnicode_TransformDecimalToASCII
+// skipped _PyUnicode_TransformDecimalAndSpaceToASCII
+
+// skipped _PyUnicode_JoinArray
+// skipped _PyUnicode_EqualToASCIIId
+// skipped _PyUnicode_EqualToASCIIString
+// skipped _PyUnicode_XStrip
+// skipped _PyUnicode_InsertThousandsGrouping
+
+// skipped _Py_ascii_whitespace
+
+// skipped _PyUnicode_IsLowercase
+// skipped _PyUnicode_IsUppercase
+// skipped _PyUnicode_IsTitlecase
+// skipped _PyUnicode_IsXidStart
+// skipped _PyUnicode_IsXidContinue
+// skipped _PyUnicode_IsWhitespace
+// skipped _PyUnicode_IsLinebreak
+// skipped _PyUnicode_ToLowercase
+// skipped _PyUnicode_ToUppercase
+// skipped _PyUnicode_ToTitlecase
+// skipped _PyUnicode_ToLowerFull
+// skipped _PyUnicode_ToTitleFull
+// skipped _PyUnicode_ToUpperFull
+// skipped _PyUnicode_ToFoldedFull
+// skipped _PyUnicode_IsCaseIgnorable
+// skipped _PyUnicode_IsCased
+// skipped _PyUnicode_ToDecimalDigit
+// skipped _PyUnicode_ToDigit
+// skipped _PyUnicode_ToNumeric
+// skipped _PyUnicode_IsDecimalDigit
+// skipped _PyUnicode_IsDigit
+// skipped _PyUnicode_IsNumeric
+// skipped _PyUnicode_IsPrintable
+// skipped _PyUnicode_IsAlpha
+// skipped Py_UNICODE_strlen
+// skipped Py_UNICODE_strcpy
+// skipped Py_UNICODE_strcat
+// skipped Py_UNICODE_strncpy
+// skipped Py_UNICODE_strcmp
+// skipped Py_UNICODE_strncmp
+// skipped Py_UNICODE_strchr
+// skipped Py_UNICODE_strrchr
+// skipped _PyUnicode_FormatLong
+// skipped PyUnicode_AsUnicodeCopy
+// skipped _PyUnicode_FromId
+// skipped _PyUnicode_EQ
+// skipped _PyUnicode_ScanIdentifier