Add support for t-string gettext extraction

Author: ThiefMaster

Lib/gettext.py

@@ -43,6 +43,7 @@
 #   you'll need to study the GNU gettext code to do this.
 
 
+import ast
 import operator
 import os
 import sys
@@ -655,3 +656,96 @@ def npgettext(context, msgid1, msgid2, n):
 # gettext.
 
 Catalog = translation
+
+
+# utils for t-string handling in gettext translation + pygettext extraction
+# TBD where they should go, and whether this should be a public API or internal,
+# especially the part about generating names from interpolations which is IMHO
+# beneficial to have in stdlib so any implementation can re-use it without
+# risking diverging behavior for the same expression between implementations
+
+class _NameTooComplexError(ValueError):
+    """
+    Raised when an expression is too complex to derive a format string name
+    from it, or the resulting name would not be valid in a format string.
+    """
+
+
+class _ExtractNamesVisitor(ast.NodeVisitor):
+    def __init__(self):
+        self._name_parts = []
+
+    @property
+    def name(self) -> str:
+        name = '__'.join(self._name_parts)
+        if not name.isidentifier():
+            raise _NameTooComplexError(
+                'Only expressions which can be converted to a format string '
+                'placeholder may be used in a gettext call; assign the '
+                'expression to a variable and use that instead'
+            )
+        return name
+
+    def generic_visit(self, node):
+        name = node.__class__.__name__
+        raise _NameTooComplexError(
+            f'Only simple expressions are supported, {name} is not allowed; '
+            'assign the expression to a variable and use that instead'
+        )
+
+    def visit_Attribute(self, node):
+        self.visit(node.value)
+        self._name_parts.append(node.attr)
+
+    def visit_Name(self, node):
+        self._name_parts.append(node.id)
+
+    def visit_Subscript(self, node):
+        self.visit(node.value)
+        if not isinstance(node.slice, ast.Constant):
+            raise _NameTooComplexError(
+                'Only constant value dict keys may be used in a gettext call; '
+                'assign the expression to a variable and use that instead'
+            )
+        self.visit(node.slice)
+
+    def visit_Constant(self, node):
+        self._name_parts.append(str(node.value))
+
+    def visit_Call(self, node):
+        self.visit(node.func)
+        if node.args:
+            raise _NameTooComplexError(
+                'Function calls with arguments are not supported in gettext '
+                'calls; assign the result to a variable and use that instead'
+            )
+
+
+def _template_node_to_format(node: ast.TemplateStr) -> str:
+    """Generate a format string from a template string AST node.
+
+    This fails with a :exc:`_NameTooComplexError` in case the expression is
+    not suitable for conversion.
+    """
+    parts = []
+    interpolation_format_names = {}
+    for child in node.values:
+        match child:
+            case ast.Constant(value):
+                parts.append(value.replace('{', '{{').replace('}', '}}'))
+            case ast.Interpolation(value):
+                visitor = _ExtractNamesVisitor()
+                visitor.visit(value)
+                name = visitor.name
+                expr = ast.unparse(value)
+                if (
+                    existing_expr := interpolation_format_names.get(name)
+                ) and existing_expr != expr:
+                    raise _NameTooComplexError(
+                        f'Interpolations of {existing_expr} and {expr} cannot '
+                        'be mixed in the same gettext call; assign one of '
+                        'them to a variable and use that instead'
+                    )
+                interpolation_format_names[name] = expr
+                parts.append(f'{{{name}}}')
+    return ''.join(parts)

Lib/test/test_tools/i18n_data/messages.pot

@@ -97,3 +97,59 @@ msgid_plural "worlds"
 msgstr[0] ""
 msgstr[1] ""
 
+#: messages.py:122 messages.py:123
+msgid "Hello World"
+msgstr ""
+
+#: messages.py:124
+msgid "Hello {name}"
+msgstr ""
+
+#: messages.py:125
+msgid "Hello {name__title}"
+msgstr ""
+
+#: messages.py:126 messages.py:127 messages.py:128
+msgid "Hello {user__name}"
+msgstr ""
+
+#: messages.py:129
+msgid "Hello {numbers__69}"
+msgstr ""
+
+#: messages.py:132
+msgid "Hello {{escaped braces}}"
+msgstr ""
+
+#: messages.py:133
+msgid "Hello {{{interpolated_braces}}} inside esacped braces"
+msgstr ""
+
+#: messages.py:134
+msgid "}}Even{{ more {{braces}}"
+msgstr ""
+
+#: messages.py:135
+msgid "}}Even{{ more {{{interpolated_braces}}}"
+msgstr ""
+
+#: messages.py:139
+msgid "Weird {meow__False}"
+msgstr ""
+
+#: messages.py:140
+msgid "Weird {meow__True}"
+msgstr ""
+
+#: messages.py:141
+msgid "Weird {meow__69j}"
+msgstr ""
+
+#: messages.py:142
+msgid "Weird {meow__Ellipsis}"
+msgstr ""
+
+#: messages.py:143
+msgid "Weird {meow__None}"
+msgstr ""
+

Lib/test/test_tools/i18n_data/messages.py

@@ -117,3 +117,42 @@ def _(x="don't extract me"):
 # f-strings
 f"Hello, {_('world')}!"
 f"Hello, {ngettext('world', 'worlds', 3)}!"
+
+# t-strings
+_(t'Hello World')
+_(t'Hello' t' World')
+_(t'Hello {name}')
+_(t'Hello {name.title()}')
+_(t'Hello {user.name}')
+_(t'Hello {user['name']}')
+_(t'Hello {user["name"]}')
+_(t'Hello {numbers[69]}')
+
+# t-strings - escaped braces
+_(t'Hello {{escaped braces}}')
+_(t'Hello {{{interpolated_braces}}} inside esacped braces')
+_(t'}}Even{{ more {{braces}}')
+_(t'}}Even{{ more {{{interpolated_braces}}}')
+
+# t-strings - slightly weird cases but simple enough to convert in a
+# straightforward manner
+_(t'Weird {meow[False]}')
+_(t'Weird {meow[True]}')
+_(t'Weird {meow[69j]}')
+_(t'Weird {meow[...]}')
+_(t'Weird {meow[None]}')
+
+# t-strings - invalid cases
+_(t'Invalid {t"nesting"}')  # nested tstrings are not allowed
+_(t'Invalid {meow[meow()]}')  # non-const subscript
+_(t'Invalid {meow[kitty]}')  # non-const subscript
+_(t'Invalid {meow[()]}')  # non-primitive subscript
+_(t'Invalid {meow(42)}')  # call with argument
+_(t'Invalid {meow["foo:r"]}')  # subscript that cannot be formatstringified
+_(t'Invalid {meow[3.14]}')  # subscript that cannot be formatstringified
+_(t'Invalid {meow[...]} {meow.Ellipsis}')  # same name for different expressions
+_(t'Invalid {meow.loudly} {meow["loudly"]}')  # same name for different expressions
+_(t'Invalid {meow.loudly} {meow.loudly()}')  # same name for different expressions
+_(t'Invalid {3.14}')  # format string is not a valid identifier
+_(t'Invalid {42}')  # format string is not a valid identifier
+_(t'Invalid {69j}')  # format string is not a valid identifier

Tools/i18n/pygettext.py

@@ -149,6 +149,7 @@
 import time
 import tokenize
 from dataclasses import dataclass, field
+from gettext import _NameTooComplexError, _template_node_to_format
 from io import BytesIO
 from operator import itemgetter
 
@@ -537,6 +538,12 @@ def _extract_message_with_spec(self, node, spec):
         msg_data = {}
         for arg_type, position in spec.items():
             arg = node.args[position]
+            if self._is_template_str(arg):
+                try:
+                    msg_data[arg_type] = _template_node_to_format(arg)
+                except _NameTooComplexError as exc:
+                    return str(exc)
+                continue
             if not self._is_string_const(arg):
                 return (f'Expected a string constant for argument '
                         f'{position + 1}, got {ast.unparse(arg)}')
@@ -626,6 +633,9 @@ def _get_func_name(self, node):
     def _is_string_const(self, node):
         return isinstance(node, ast.Constant) and isinstance(node.value, str)
 
+    def _is_template_str(self, node):
+        return isinstance(node, ast.TemplateStr)
+
 def write_pot_file(messages, options, fp):
     timestamp = time.strftime('%Y-%m-%d %H:%M%z')
     encoding = fp.encoding if fp.encoding else 'UTF-8'