gh-139353: Add Objects/unicode_codecs_utf.c file

Rename functions:

* ascii_decode() => _PyUnicode_DecodeASCII()
* backslashreplace() => _PyUnicode_backslashreplace()
* raise_encode_exception() => _PyUnicode_RaiseEncodeException()
* unicode_decode_call_errorhandler_writer() => _PyUnicode_DecodeCallErrorHandler()
* unicode_decode_utf8() => _PyUnicode_DecodeUTF8()
* unicode_encode_call_errorhandler() => _PyUnicode_EncodeCallErrorHandler()
* unicode_encode_utf8() => _PyUnicode_EncodeUTF8()
* xmlcharrefreplace() => _PyUnicode_xmlcharrefreplace()

Move static inline functions and macros to pycore_unicodeobject.h:

* _PyUnicode_CHECK()
* _PyUnicode_UTF8()
* PyUnicode_UTF8()
* PyUnicode_SET_UTF8()
* PyUnicode_UTF8_LENGTH()
* PyUnicode_SET_UTF8_LENGTH()

Author: vstinner

Include/internal/pycore_unicodeobject.h

@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_fileutils.h"     // _Py_error_handler
+#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELEASE()
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
 
 
@@ -32,6 +33,110 @@ extern PyObject* _PyUnicode_ResizeCompact(
     PyObject *unicode,
     Py_ssize_t length);
 extern PyObject* _PyUnicode_GetEmpty(void);
+extern int _PyUnicode_DecodeCallErrorHandler(
+    const char *errors,
+    PyObject **errorHandler,
+    const char *encoding,
+    const char *reason,
+    const char **input,
+    const char **inend,
+    Py_ssize_t *startinpos,
+    Py_ssize_t *endinpos,
+    PyObject **exceptionObject,
+    const char **inptr,
+    _PyUnicodeWriter *writer);
+extern char* _PyUnicode_backslashreplace(
+    PyBytesWriter *writer,
+    char *str,
+    PyObject *unicode,
+    Py_ssize_t collstart,
+    Py_ssize_t collend);
+extern char* _PyUnicode_xmlcharrefreplace(
+    PyBytesWriter *writer,
+    char *str,
+    PyObject *unicode,
+    Py_ssize_t collstart,
+    Py_ssize_t collend);
+extern PyObject* _PyUnicode_EncodeCallErrorHandler(
+    const char *errors,
+    PyObject **errorHandler,
+    const char *encoding,
+    const char *reason,
+    PyObject *unicode,
+    PyObject **exceptionObject,
+    Py_ssize_t startpos,
+    Py_ssize_t endpos,
+    Py_ssize_t *newpos);
+extern void _PyUnicode_RaiseEncodeException(
+    PyObject **exceptionObject,
+    const char *encoding,
+    PyObject *unicode,
+    Py_ssize_t startpos,
+    Py_ssize_t endpos,
+    const char *reason);
+extern Py_ssize_t _PyUnicode_DecodeASCII(
+    const char *start,
+    const char *end,
+    Py_UCS1 *dest);
+extern PyObject* _PyUnicode_EncodeUTF8(
+    PyObject *unicode,
+    _Py_error_handler error_handler,
+    const char *errors);
+extern PyObject* _PyUnicode_DecodeUTF8(
+    const char *s,
+    Py_ssize_t size,
+    _Py_error_handler error_handler,
+    const char *errors,
+    Py_ssize_t *consumed);
+
+// Export for '_json' shared extension
+PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
+    PyObject *op,
+    int check_content);
+
+
+#ifdef Py_DEBUG
+#  define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0)
+#else
+#  define _PyUnicode_CHECK(op) PyUnicode_Check(op)
+#endif
+
+static inline char* _PyUnicode_UTF8(PyObject *op)
+{
+    return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8);
+}
+
+static inline char* PyUnicode_UTF8(PyObject *op)
+{
+    assert(_PyUnicode_CHECK(op));
+    if (PyUnicode_IS_COMPACT_ASCII(op)) {
+        return ((char*)(_PyASCIIObject_CAST(op) + 1));
+    }
+    else {
+         return _PyUnicode_UTF8(op);
+    }
+}
+
+static inline Py_ssize_t PyUnicode_UTF8_LENGTH(PyObject *op)
+{
+    assert(_PyUnicode_CHECK(op));
+    if (PyUnicode_IS_COMPACT_ASCII(op)) {
+         return _PyASCIIObject_CAST(op)->length;
+    }
+    else {
+         return _PyCompactUnicodeObject_CAST(op)->utf8_length;
+    }
+}
+
+static inline void PyUnicode_SET_UTF8(PyObject *op, char *utf8)
+{
+    FT_ATOMIC_STORE_PTR_RELEASE(_PyCompactUnicodeObject_CAST(op)->utf8, utf8);
+}
+
+static inline void PyUnicode_SET_UTF8_LENGTH(PyObject *op, Py_ssize_t length)
+{
+    _PyCompactUnicodeObject_CAST(op)->utf8_length = length;
+}
 
 
 /* Generic helper macro to convert characters of different types.
@@ -116,11 +221,6 @@ _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch)
 
 /* --- Unicode API -------------------------------------------------------- */
 
-// Export for '_json' shared extension
-PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
-    PyObject *op,
-    int check_content);
-
 PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op);
 extern Py_ssize_t _PyUnicode_InternedSize(void);
 extern Py_ssize_t _PyUnicode_InternedSize_Immortal(void);

Makefile.pre.in

@@ -558,6 +558,7 @@ OBJECT_OBJS=	\
 		Objects/tupleobject.o \
 		Objects/typeobject.o \
 		Objects/typevarobject.o \
+		Objects/unicode_codecs_utf.o \
 		Objects/unicode_format.o \
 		Objects/unicode_formatter.o \
 		Objects/unicode_writer.o \

Objects/stringlib/codecs.h

@@ -352,8 +352,8 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
             case _Py_ERROR_BACKSLASHREPLACE:
                 /* subtract preallocated bytes */
                 writer->size -= max_char_size * (endpos - startpos);
-                p = backslashreplace(writer, p,
-                                     unicode, startpos, endpos);
+                p = _PyUnicode_backslashreplace(writer, p,
+                                                unicode, startpos, endpos);
                 if (p == NULL)
                     goto error;
                 i += (endpos - startpos - 1);
@@ -362,8 +362,8 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
             case _Py_ERROR_XMLCHARREFREPLACE:
                 /* subtract preallocated bytes */
                 writer->size -= max_char_size * (endpos - startpos);
-                p = xmlcharrefreplace(writer, p,
-                                      unicode, startpos, endpos);
+                p = _PyUnicode_xmlcharrefreplace(writer, p,
+                                                 unicode, startpos, endpos);
                 if (p == NULL)
                     goto error;
                 i += (endpos - startpos - 1);
@@ -384,7 +384,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
                 assert(startpos < endpos);
                 _Py_FALLTHROUGH;
             default:
-                rep = unicode_encode_call_errorhandler(
+                rep = _PyUnicode_EncodeCallErrorHandler(
                       errors, &error_handler_obj, "utf-8", "surrogates not allowed",
                       unicode, &exc, startpos, endpos, &newpos);
                 if (!rep)
@@ -415,9 +415,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
                 else {
                     /* rep is unicode */
                     if (!PyUnicode_IS_ASCII(rep)) {
-                        raise_encode_exception(&exc, "utf-8", unicode,
-                                               startpos, endpos,
-                                               "surrogates not allowed");
+                        _PyUnicode_RaiseEncodeException(
+                            &exc, "utf-8", unicode,
+                            startpos, endpos,
+                            "surrogates not allowed");
                         goto error;
                     }
 
@@ -452,7 +453,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
 #if STRINGLIB_SIZEOF_CHAR > 2
         else /* ch >= 0x10000 */
         {
-            assert(ch <= MAX_UNICODE);
+            assert(ch <= _Py_MAX_UNICODE);
             /* Encode UCS4 Unicode ordinals */
             *p++ = (char)(0xf0 | (ch >> 18));
             *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));