gh-133403: Check generate_stdlib_module_names and check_extension_modules with mypy

Author: sobolevn

.github/workflows/mypy.yml

@@ -13,9 +13,11 @@ on:
       - "Lib/test/libregrtest/**"
       - "Lib/tomllib/**"
       - "Misc/mypy/**"
+      - "Tools/build/check_extension_modules.py"
       - "Tools/build/compute-changes.py"
       - "Tools/build/deepfreeze.py"
       - "Tools/build/generate_sbom.py"
+      - "Tools/build/generate_stdlib_module_names.py"
       - "Tools/build/generate-build-details.py"
       - "Tools/build/verify_ensurepip_wheels.py"
       - "Tools/build/update_file.py"

Tools/build/check_extension_modules.py

@@ -17,6 +17,9 @@
 
 See --help for more information
 """
+
+from __future__ import annotations
+
 import _imp
 import argparse
 import collections
@@ -29,13 +32,14 @@
 import sysconfig
 import warnings
 from collections.abc import Iterable
-from importlib._bootstrap import _load as bootstrap_load
+from importlib._bootstrap import _load as bootstrap_load  # type: ignore[attr-defined]
 from importlib.machinery import (
     BuiltinImporter,
     ExtensionFileLoader,
     ModuleSpec,
 )
 from importlib.util import spec_from_file_location, spec_from_loader
+from typing import NamedTuple
 
 SRC_DIR = pathlib.Path(__file__).parent.parent.parent
 
@@ -112,6 +116,7 @@
 )
 
 
+@enum.unique
 class ModuleState(enum.Enum):
     # Makefile state "yes"
     BUILTIN = "builtin"
@@ -123,21 +128,23 @@ class ModuleState(enum.Enum):
     # disabled by Setup / makesetup rule
     DISABLED_SETUP = "disabled_setup"
 
-    def __bool__(self):
+    def __bool__(self) -> bool:
         return self.value in {"builtin", "shared"}
 
 
-ModuleInfo = collections.namedtuple("ModuleInfo", "name state")
+class ModuleInfo(NamedTuple):
+    name: str
+    state: ModuleState
 
 
 class ModuleChecker:
     pybuilddir_txt = "pybuilddir.txt"
 
     setup_files = (
         # see end of configure.ac
-        "Modules/Setup.local",
-        "Modules/Setup.stdlib",
-        "Modules/Setup.bootstrap",
+        pathlib.Path("Modules/Setup.local"),
+        pathlib.Path("Modules/Setup.stdlib"),
+        pathlib.Path("Modules/Setup.bootstrap"),
         SRC_DIR / "Modules/Setup",
     )
 
@@ -149,15 +156,15 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False):
         self.builddir = self.get_builddir()
         self.modules = self.get_modules()
 
-        self.builtin_ok = []
-        self.shared_ok = []
-        self.failed_on_import = []
-        self.missing = []
-        self.disabled_configure = []
-        self.disabled_setup = []
-        self.notavailable = []
+        self.builtin_ok: list[ModuleInfo] = []
+        self.shared_ok: list[ModuleInfo] = []
+        self.failed_on_import: list[ModuleInfo] = []
+        self.missing: list[ModuleInfo] = []
+        self.disabled_configure: list[ModuleInfo] = []
+        self.disabled_setup: list[ModuleInfo] = []
+        self.notavailable: list[ModuleInfo] = []
 
-    def check(self):
+    def check(self) -> None:
         if not hasattr(_imp, 'create_dynamic'):
             logger.warning(
                 ('Dynamic extensions not supported '
@@ -189,10 +196,10 @@ def check(self):
                         assert modinfo.state == ModuleState.SHARED
                         self.shared_ok.append(modinfo)
 
-    def summary(self, *, verbose: bool = False):
+    def summary(self, *, verbose: bool = False) -> None:
         longest = max([len(e.name) for e in self.modules], default=0)
 
-        def print_three_column(modinfos: list[ModuleInfo]):
+        def print_three_column(modinfos: list[ModuleInfo]) -> None:
             names = [modinfo.name for modinfo in modinfos]
             names.sort(key=str.lower)
             # guarantee zip() doesn't drop anything
@@ -262,12 +269,12 @@ def print_three_column(modinfos: list[ModuleInfo]):
             f"{len(self.failed_on_import)} failed on import)"
         )
 
-    def check_strict_build(self):
+    def check_strict_build(self) -> None:
         """Fail if modules are missing and it's a strict build"""
         if self.strict_extensions_build and (self.failed_on_import or self.missing):
             raise RuntimeError("Failed to build some stdlib modules")
 
-    def list_module_names(self, *, all: bool = False) -> set:
+    def list_module_names(self, *, all: bool = False) -> set[str]:
         names = {modinfo.name for modinfo in self.modules}
         if all:
             names.update(WINDOWS_MODULES)
@@ -280,9 +287,9 @@ def get_builddir(self) -> pathlib.Path:
         except FileNotFoundError:
             logger.error("%s must be run from the top build directory", __file__)
             raise
-        builddir = pathlib.Path(builddir)
-        logger.debug("%s: %s", self.pybuilddir_txt, builddir)
-        return builddir
+        builddir_path = pathlib.Path(builddir)
+        logger.debug("%s: %s", self.pybuilddir_txt, builddir_path)
+        return builddir_path
 
     def get_modules(self) -> list[ModuleInfo]:
         """Get module info from sysconfig and Modules/Setup* files"""
@@ -367,7 +374,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
                     case ["*disabled*"]:
                         state = ModuleState.DISABLED
                     case ["*noconfig*"]:
-                        state = None
+                        continue
                     case [*items]:
                         if state == ModuleState.DISABLED:
                             # *disabled* can disable multiple modules per line
@@ -384,34 +391,41 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
     def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
         """Get ModuleSpec for builtin or extension module"""
         if modinfo.state == ModuleState.SHARED:
-            location = os.fspath(self.get_location(modinfo))
+            mod_location = self.get_location(modinfo)
+            assert mod_location is not None
+            location = os.fspath(mod_location)
             loader = ExtensionFileLoader(modinfo.name, location)
-            return spec_from_file_location(modinfo.name, location, loader=loader)
+            spec = spec_from_file_location(modinfo.name, location, loader=loader)
+            assert spec is not None
+            return spec
         elif modinfo.state == ModuleState.BUILTIN:
-            return spec_from_loader(modinfo.name, loader=BuiltinImporter)
+            spec = spec_from_loader(modinfo.name, loader=BuiltinImporter)
+            assert spec is not None
+            return spec
         else:
             raise ValueError(modinfo)
 
-    def get_location(self, modinfo: ModuleInfo) -> pathlib.Path:
+    def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None:
         """Get shared library location in build directory"""
         if modinfo.state == ModuleState.SHARED:
             return self.builddir / f"{modinfo.name}{self.ext_suffix}"
         else:
             return None
 
-    def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec):
+    def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None:
         """Check that the module file is present and not empty"""
-        if spec.loader is BuiltinImporter:
+        if spec.loader is BuiltinImporter:  # type: ignore[comparison-overlap]
             return
         try:
+            assert spec.origin is not None
             st = os.stat(spec.origin)
         except FileNotFoundError:
             logger.error("%s (%s) is missing", modinfo.name, spec.origin)
             raise
         if not st.st_size:
             raise ImportError(f"{spec.origin} is an empty file")
 
-    def check_module_import(self, modinfo: ModuleInfo):
+    def check_module_import(self, modinfo: ModuleInfo) -> None:
         """Attempt to import module and report errors"""
         spec = self.get_spec(modinfo)
         self._check_file(modinfo, spec)
@@ -430,7 +444,7 @@ def check_module_import(self, modinfo: ModuleInfo):
             logger.exception("Importing extension '%s' failed!", modinfo.name)
             raise
 
-    def check_module_cross(self, modinfo: ModuleInfo):
+    def check_module_cross(self, modinfo: ModuleInfo) -> None:
         """Sanity check for cross compiling"""
         spec = self.get_spec(modinfo)
         self._check_file(modinfo, spec)
@@ -443,6 +457,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None:
 
         failed_name = f"{modinfo.name}_failed{self.ext_suffix}"
         builddir_path = self.get_location(modinfo)
+        assert builddir_path is not None
         if builddir_path.is_symlink():
             symlink = builddir_path
             module_path = builddir_path.resolve().relative_to(os.getcwd())
@@ -466,7 +481,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None:
             logger.debug("Rename '%s' -> '%s'", module_path, failed_path)
 
 
-def main():
+def main() -> None:
     args = parser.parse_args()
     if args.debug:
         args.verbose = True

Tools/build/generate_stdlib_module_names.py

@@ -1,9 +1,12 @@
 # This script lists the names of standard library modules
 # to update Python/stdlib_module_names.h
+from __future__ import annotations
+
 import _imp
 import os.path
 import sys
 import sysconfig
+from typing import TextIO
 
 from check_extension_modules import ModuleChecker
 
@@ -48,12 +51,12 @@
 }
 
 # Built-in modules
-def list_builtin_modules(names):
+def list_builtin_modules(names: set[str]) -> None:
     names |= set(sys.builtin_module_names)
 
 
 # Pure Python modules (Lib/*.py)
-def list_python_modules(names):
+def list_python_modules(names: set[str]) -> None:
     for filename in os.listdir(STDLIB_PATH):
         if not filename.endswith(".py"):
             continue
@@ -62,7 +65,7 @@ def list_python_modules(names):
 
 
 # Packages in Lib/
-def list_packages(names):
+def list_packages(names: set[str]) -> None:
     for name in os.listdir(STDLIB_PATH):
         if name in IGNORE:
             continue
@@ -76,16 +79,16 @@ def list_packages(names):
 
 # Built-in and extension modules built by Modules/Setup*
 # includes Windows and macOS extensions.
-def list_modules_setup_extensions(names):
+def list_modules_setup_extensions(names: set[str]) -> None:
     checker = ModuleChecker()
     names.update(checker.list_module_names(all=True))
 
 
 # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
 # Use the "./Programs/_testembed list_frozen" command.
-def list_frozen(names):
+def list_frozen(names: set[str]) -> None:
     submodules = set()
-    for name in _imp._frozen_module_names():
+    for name in _imp._frozen_module_names():  # type: ignore[attr-defined]
         # To skip __hello__, __hello_alias__ and etc.
         if name.startswith('__'):
             continue
@@ -101,8 +104,8 @@ def list_frozen(names):
         raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')
 
 
-def list_modules():
-    names = set()
+def list_modules() -> set[str]:
+    names: set[str] = set()
 
     list_builtin_modules(names)
     list_modules_setup_extensions(names)
@@ -127,7 +130,7 @@ def list_modules():
     return names
 
 
-def write_modules(fp, names):
+def write_modules(fp: TextIO, names: set[str]) -> None:
     print(f"// Auto-generated by {SCRIPT_NAME}.",
           file=fp)
     print("// List used to create sys.stdlib_module_names.", file=fp)
@@ -138,7 +141,7 @@ def write_modules(fp, names):
     print("};", file=fp)
 
 
-def main():
+def main() -> None:
     if not sysconfig.is_python_build():
         print(f"ERROR: {sys.executable} is not a Python build",
               file=sys.stderr)

Tools/build/mypy.ini

@@ -3,10 +3,12 @@
 # Please, when adding new files here, also add them to:
 # .github/workflows/mypy.yml
 files =
+    Tools/build/check_extension_modules.py,
     Tools/build/compute-changes.py,
     Tools/build/deepfreeze.py,
     Tools/build/generate-build-details.py,
     Tools/build/generate_sbom.py,
+    Tools/build/generate_stdlib_module_names.py,
     Tools/build/verify_ensurepip_wheels.py,
     Tools/build/update_file.py,
     Tools/build/umarshal.py