Add tests comparing Ryū output to the reference dtoa

Export _Py_dg_dtoa() and _Py_dg_freedtoa() via PyAPI_FUNC and add
_testinternalcapi.compare_ryu_dtoa(), which calls both backends on
the same double and reports any difference in the (digits, decpt,
sign) result.

The new test_ryu_matches_dtoa exercises ~13k values including all
powers of two and ten, IEEE-754 boundary values, the upstream Ryū
regression-test cases, and random bit patterns.

Add several exact-string assertions to test_short_repr that pin
behaviour discussed at https://discuss.python.org/t/2466: the
1/2**24 'shortest is not round-to-N' corner case; 1/2**25 and
6/2**25 ties-to-even cases; and the smallest subnormals 5e-324
and 1e-323 (one-digit shortest).

Add test_alternate_form_shortest covering format(x, '#') for
single-significant-digit values in exponential range, which had no
prior coverage.

Add Lib/test/test_float_property.py with hypothesis-based property
tests: float and complex repr() round-trip, repr() is the shortest
round-tripping decimal, and Ryū matches dtoa for arbitrary doubles.
Without hypothesis installed these run only the @example cases; the
test-hypothesis CI job runs them with shrinking and a persistent
failure database.

Author: gpshead

Include/internal/pycore_dtoa.h

@@ -25,9 +25,10 @@ extern "C" {
 #endif
 
 extern double _Py_dg_strtod(const char *str, char **ptr);
-extern char* _Py_dg_dtoa(double d, int mode, int ndigits,
-                         int *decpt, int *sign, char **rve);
-extern void _Py_dg_freedtoa(char *s);
+// Export for '_testinternalcapi' shared extension.
+PyAPI_FUNC(char*) _Py_dg_dtoa(double d, int mode, int ndigits,
+                              int *decpt, int *sign, char **rve);
+PyAPI_FUNC(void) _Py_dg_freedtoa(char *s);
 
 
 extern PyStatus _PyDtoa_Init(PyInterpreterState *interp);

Lib/test/test_float.py

@@ -858,6 +858,21 @@ def test_issue35560(self):
         self.assertEqual(format(-123.34, '00.10e'), '-1.2334000000e+02')
         self.assertEqual(format(-123.34, '00.10g'), '-123.34')
 
+    @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short',
+                         "applies only when using short float repr style")
+    def test_alternate_form_shortest(self):
+        # '#' with no type code goes through the shortest-repr conversion;
+        # for single-significant-digit values in exponential range it must
+        # still force the decimal point.
+        self.assertEqual(format(1e16, '#'), '1.e+16')
+        self.assertEqual(format(1e300, '#'), '1.e+300')
+        self.assertEqual(format(5e-324, '#'), '5.e-324')
+        self.assertEqual(format(-1e16, '+#'), '-1.e+16')
+        self.assertEqual(format(1e16, ''), '1e+16')
+        self.assertEqual(format(1.5, '#'), '1.5')
+        self.assertEqual(format(100000.0, '#'), '100000.0')
+
+
 class ReprTestCase(unittest.TestCase):
     def test_repr(self):
         with open(os.path.join(os.path.split(__file__)[0],
@@ -912,6 +927,22 @@ def test_short_repr(self):
             '2.86438000439698e+28',
             '8.89142905246179e+28',
             '3.08578087079232e+35',
+            # 1/2**24: a value whose shortest repr differs from
+            # "round x to N significant digits" for every N (rounding
+            # to 16 digits gives ...062 which doesn't round-trip;
+            # to 17 gives ...0625 which does, but ...063 is shorter
+            # and also round-trips).
+            '5.960464477539063e-08',
+            # Ties to even: both ...12 and ...13 round-trip to
+            # 1/2**25 and are exactly equidistant from it; the repr
+            # must end in the even digit.
+            '2.9802322387695312e-08',
+            '1.7881393432617188e-07',
+            # Smallest subnormal: '5e-324' (one digit) is shortest;
+            # '4.9e-324' is closer to the true value but longer.
+            '5e-324',
+            # Same one-vs-two-digit choice for the next subnormal up.
+            '1e-323',
             ]
 
         for s in test_strings:
@@ -922,6 +953,51 @@ def test_short_repr(self):
             self.assertEqual(repr(float(s)), str(float(s)))
             self.assertEqual(repr(float(negs)), str(float(negs)))
 
+    @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short',
+                         "applies only when using short float repr style")
+    @support.cpython_only
+    def test_ryu_matches_dtoa(self):
+        # Shortest float repr is computed by the Ryū algorithm; check
+        # that it produces the same digits, decimal-point position and
+        # sign as the reference dtoa implementation for a variety of
+        # values, including random bit patterns.
+        from test.support import import_helper
+        _testinternalcapi = import_helper.import_module('_testinternalcapi')
+        compare = _testinternalcapi.compare_ryu_dtoa
+
+        cases = [
+            0.0, -0.0, 1.0, -1.0, 0.1, 0.5, 1.5, 2.5,
+            5e-324,                    # smallest subnormal
+            2.2250738585072014e-308,   # smallest normal
+            1.7976931348623157e+308,   # largest finite
+            INF, -INF, NAN,
+            # Regression cases from the upstream Ryū test suite.
+            2.98023223876953125e-8,    # LotsOfTrailingZeros
+            -2.109808898695963e16, 4.940656e-318, 1.18575755e-316,
+            2.989102097996e-312, 9.0608011534336e15,
+            4.708356024711512e18, 9.409340012568248e18,
+            float.fromhex('0x1.0F0CF064DD592p+131'),  # LooksLikePow5
+            float.fromhex('0x1.0F0CF064DD592p+132'),
+            float.fromhex('0x1.0F0CF064DD592p+133'),
+            4.294967294, 4.294967295, 4.294967296, 4.294967297,
+        ]
+        # Powers of two include the cases where the shortest repr is
+        # not equal to "round x to N significant digits" for any N
+        # (e.g. 1/2**24, 2**89, 2**-44).
+        cases.extend(2.0 ** e for e in range(-1074, 1024))
+        cases.extend(float(f'1e{e}') for e in range(-323, 309))
+        rng = random.Random(96313)
+        for _ in range(10_000):
+            bits = rng.getrandbits(64)
+            cases.append(struct.unpack('<d', struct.pack('<Q', bits))[0])
+
+        for x in cases:
+            mismatch = compare(x)
+            if mismatch is not None and not math.isnan(x):
+                self.fail(f'ryu/dtoa mismatch for {x.hex()} ({x!r}): '
+                          f'dtoa={mismatch[0]} ryu={mismatch[1]}')
+
+
 @support.requires_IEEE_754
 class RoundTestCase(unittest.TestCase, FloatsAreIdenticalMixin):
 

Lib/test/test_float_property.py

@@ -0,0 +1,95 @@
+import math
+import sys
+import unittest
+from test import support
+from test.support.hypothesis_helper import hypothesis
+
+floats = hypothesis.strategies.floats
+given = hypothesis.given
+example = hypothesis.example
+
+
+@unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short',
+                     "applies only when using short float repr style")
+class ShortFloatReprProperties(unittest.TestCase):
+
+    @given(x=floats())
+    @example(x=0.0)
+    @example(x=-0.0)
+    @example(x=5e-324)
+    @example(x=sys.float_info.min)
+    @example(x=sys.float_info.max)
+    def test_repr_round_trip(self, x):
+        s = repr(x)
+        if math.isnan(x):
+            self.assertTrue(math.isnan(float(s)))
+        else:
+            self.assertEqual(float(s), x)
+            self.assertEqual(str(x), s)
+
+    @given(x=floats(allow_nan=False, allow_infinity=False))
+    @example(x=2.0**-24)
+    @example(x=1e-323)
+    @example(x=2.0**89)
+    @example(x=2.0**-44)
+    @example(x=1.0)
+    @example(x=0.0)
+    def test_repr_is_shortest(self, x):
+        # Steele & White criterion (1): no shorter decimal string round-trips.
+        s = repr(x)
+        mant = s.partition('e')[0]
+        digits = ''.join(c for c in mant if c.isdigit())
+        if '.' in mant:
+            digits = digits.rstrip('0')
+        digits = digits.lstrip('0') or '0'
+        n = len(digits)
+        if n > 1:
+            shorter = format(x, f'.{n - 1}g')
+            if float(shorter) == x:
+                self.fail(f'repr({x.hex()}) = {s!r} ({n} digits) '
+                          f'but {shorter!r} ({n - 1} digits) also '
+                          f'round-trips')
+
+    @support.cpython_only
+    @given(x=floats())
+    @example(x=2.0**-24)
+    @example(x=2.0**-25)
+    @example(x=5e-324)
+    @example(x=2.98023223876953125e-8)
+    @example(x=float.fromhex('0x1.0F0CF064DD592p+132'))
+    @example(x=4.940656e-318)
+    @example(x=9.0608011534336e15)
+    @example(x=4.708356024711512e18)
+    @example(x=-0.0)
+    def test_ryu_matches_dtoa(self, x):
+        # Ryu and the reference dtoa must produce identical
+        # (digits, decpt, sign) triples for the shortest representation.
+        from test.support import import_helper
+        _testinternalcapi = import_helper.import_module('_testinternalcapi')
+        mismatch = _testinternalcapi.compare_ryu_dtoa(x)
+        if mismatch is not None and not math.isnan(x):
+            self.fail(f'ryu/dtoa mismatch for {x.hex()} ({x!r}): '
+                      f'dtoa={mismatch[0]} ryu={mismatch[1]}')
+
+    @given(re=floats(), im=floats())
+    @example(re=-0.0, im=-0.0)
+    @example(re=0.0, im=-0.0)
+    @example(re=math.inf, im=-math.inf)
+    @example(re=1e308, im=1e-308)
+    @example(re=5e-324, im=5e-324)
+    def test_complex_repr_round_trip(self, re, im):
+        c = complex(re, im)
+        s = repr(c)
+        c2 = complex(s)
+        if math.isnan(re):
+            self.assertTrue(math.isnan(c2.real))
+        else:
+            self.assertEqual(c2.real, re)
+        if math.isnan(im):
+            self.assertTrue(math.isnan(c2.imag))
+        else:
+            self.assertEqual(c2.imag, im)
+
+
+if __name__ == '__main__':
+    unittest.main()

Modules/_testinternalcapi.c

@@ -19,6 +19,7 @@
 #include "pycore_compile.h"       // _PyCompile_CodeGen()
 #include "pycore_context.h"       // _PyContext_NewHamtForTests()
 #include "pycore_dict.h"          // PyDictValues
+#include "pycore_dtoa.h"          // _Py_dg_dtoa()
 #include "pycore_fileutils.h"     // _Py_normpath()
 #include "pycore_flowgraph.h"     // _PyCompile_OptimizeCfg()
 #include "pycore_frame.h"         // _PyInterpreterFrame
@@ -37,6 +38,7 @@
 #include "pycore_pylifecycle.h"   // _PyInterpreterConfig_InitFromDict()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_runtime_structs.h" // _PY_NSMALLPOSINTS
+#include "pycore_ryu.h"           // _Py_ryu_dtoa()
 #include "pycore_unicodeobject.h" // _PyUnicode_TransformDecimalAndSpaceToASCII()
 
 #include "clinic/_testinternalcapi.c.h"
@@ -2852,6 +2854,47 @@ _pyerr_setkeyerror(PyObject *self, PyObject *arg)
 }
 
 
+#if _PY_SHORT_FLOAT_REPR == 1
+static PyObject *
+compare_ryu_dtoa(PyObject *self, PyObject *arg)
+{
+    double d = PyFloat_AsDouble(arg);
+    if (d == -1.0 && PyErr_Occurred()) {
+        return NULL;
+    }
+
+    int g_decpt, g_sign;
+    char *g_end;
+    char *g_digits = _Py_dg_dtoa(d, 0, 0, &g_decpt, &g_sign, &g_end);
+    if (g_digits == NULL) {
+        return PyErr_NoMemory();
+    }
+
+    char r_buf[_Py_RYU_DTOA_BUFSIZE];
+    int r_decpt, r_sign;
+    int r_len = _Py_ryu_dtoa(d, r_buf, &r_decpt, &r_sign);
+
+    int g_len = (int)(g_end - g_digits);
+    int ok = (g_decpt == r_decpt
+              && g_sign == r_sign
+              && g_len == r_len
+              && memcmp(g_digits, r_buf, (size_t)r_len) == 0);
+
+    PyObject *result;
+    if (ok) {
+        result = Py_NewRef(Py_None);
+    }
+    else {
+        result = Py_BuildValue("((sii)(sii))",
+                               g_digits, g_decpt, g_sign,
+                               r_buf, r_decpt, r_sign);
+    }
+    _Py_dg_freedtoa(g_digits);
+    return result;
+}
+#endif
+
+
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
     {"get_eval_frame_stats", get_eval_frame_stats, METH_NOARGS, NULL},
@@ -2974,6 +3017,9 @@ static PyMethodDef module_functions[] = {
     {"test_threadstate_set_stack_protection",
      test_threadstate_set_stack_protection, METH_NOARGS},
     {"_pyerr_setkeyerror", _pyerr_setkeyerror, METH_O},
+#if _PY_SHORT_FLOAT_REPR == 1
+    {"compare_ryu_dtoa", compare_ryu_dtoa, METH_O},
+#endif
     {NULL, NULL} /* sentinel */
 };