Sending the message as JSON and encode the name using base64 to keep the sent length below 512

Author: rani-pinchuk

Lib/multiprocessing/resource_tracker.py

@@ -23,6 +23,8 @@
 import warnings
 from collections import deque
 
+import json
+
 from . import spawn
 from . import util
 
@@ -192,6 +194,17 @@ def _launch(self):
         finally:
             os.close(r)
 
+    def _make_probe_message(self):
+        """Return a JSON-encoded probe message."""
+        return (
+            json.dumps(
+                {"cmd": "PROBE", "rtype": "noop", "base64_name": ""},
+                ensure_ascii=True,
+                separators=(",", ":"),
+            )
+            + "\n"
+        ).encode("ascii")
+
     def _ensure_running_and_write(self, msg=None):
         with self._lock:
             if self._lock._recursion_count() > 1:
@@ -203,7 +216,7 @@ def _ensure_running_and_write(self, msg=None):
             if self._fd is not None:
                 # resource tracker was launched before, is it still running?
                 if msg is None:
-                    to_send = b'PROBE:0:noop\n'
+                    to_send = self._make_probe_message()
                 else:
                     to_send = msg
                 try:
@@ -230,7 +243,7 @@ def _check_alive(self):
         try:
             # We cannot use send here as it calls ensure_running, creating
             # a cycle.
-            os.write(self._fd, b'PROBE:0:noop\n')
+            os.write(self._fd, self._make_probe_message())
         except OSError:
             return False
         else:
@@ -249,17 +262,23 @@ def _write(self, msg):
         assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}"
 
     def _send(self, cmd, name, rtype):
-        # Encode shared_memory names as they are created by the user and may contain
-        # colons or newlines.
-        if rtype == "shared_memory":
-            b = name.encode('utf-8', 'surrogateescape')
-            name = base64.urlsafe_b64encode(b).decode('ascii')
-
-        msg = f"{cmd}:{name}:{rtype}\n".encode("ascii")
-        if len(msg) > 512:
-            # posix guarantees that writes to a pipe of less than PIPE_BUF
-            # bytes are atomic, and that PIPE_BUF >= 512
-            raise ValueError('msg too long')
+        # POSIX shm_open() and sem_open() require the name, including its leading slash,
+        # to be at most NAME_MAX bytes (255 on Linux)
+        # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char
+        # escape like \uDC80.
+        # As we want the overall message to be kept atomic and therefore smaller than 512,
+        # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name
+        # will not exceed 340 bytes.
+        b = name.encode('utf-8', 'surrogateescape')
+        if len(b) > 255:
+            raise ValueError('shared memory name too long (max 255 bytes)')
+        b64 = base64.urlsafe_b64encode(b).decode('ascii')
+
+        payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64}
+        msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii")
+
+        # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction.
+        assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)"
 
         self._ensure_running_and_write(msg)
 
@@ -290,14 +309,27 @@ def main(fd):
     try:
         # keep track of registered/unregistered resources
         with open(fd, 'rb') as f:
-            for line in f:
+            for raw in f:
                 try:
-                    cmd, enc_name, rtype = line.rstrip(b'\n').decode('ascii').split(':', 2)
-                    if rtype == "shared_memory":
-                        name = base64.urlsafe_b64decode(enc_name.encode('ascii')).decode('utf-8', 'surrogateescape')
-                    else:
-                        # Semaphore names are generated internally, so no encoding is needed.
-                        name = enc_name
+                    line = raw.rstrip(b'\n')
+                    try:
+                        obj = json.loads(line.decode('ascii'))
+                    except Exception as e:
+                        raise ValueError("malformed resource_tracker message: %r" % (line,)) from e
+
+                    cmd = obj.get("cmd")
+                    rtype = obj.get("rtype")
+                    b64  = obj.get("base64_name")
+
+                    if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str):
+                        raise ValueError("malformed resource_tracker fields: %r" % (obj,))
+
+                    enc = b64.encode('ascii')
+                    enc += b'=' * (-len(enc) % 4)  # normalize padding
+                    try:
+                        name = base64.urlsafe_b64decode(enc).decode('utf-8', 'surrogateescape')
+                    except ValueError as e:
+                        raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e
 
                     cleanup_func = _CLEANUP_FUNCS.get(rtype, None)
                     if cleanup_func is None: