Move worker downtime configuration to workers.py (GH-701)

We use `builder.canStartBuild` for scheduled downtime, which is per-worker configuration.
Move it out of `master.cfg` to the appropriate places:

-  config in `workers.py`
- implementation details in a new module, `worker_downtime.py`

This applies `ambv-bb-win11`'s maintenance window to the PR builds, for which it was left out in GH-589.

Author: encukou

master/custom/worker_downtime.py

@@ -0,0 +1,49 @@
+from datetime import datetime, timedelta
+
+
+def no_builds_between(start, end, *, day_of_week=None, tz=None):
+    """A function for "builder.canStartBuild" that implements planned downtime
+
+    Avoid a build to be started between start and end time and delay such
+    builds until end time.
+    """
+
+    start = datetime.strptime(start, "%H:%M").time()
+    end = datetime.strptime(end, "%H:%M").time()
+    def canStartBuild(builder, wfb, request):
+        now_dt = datetime.now(tz=tz)
+        if day_of_week is not None and now_dt.weekday() != day_of_week:
+            return True
+        now = now_dt.time()
+        if is_within_time_range(now, start, end):
+            delay = get_delay(now, end)
+            # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions
+            wfb.worker.quarantine_timeout = delay
+            wfb.worker.putInQuarantine()
+            # This does not take the worker out of quarantine, it only resets
+            # the timeout value to default (restarting the default
+            # exponential backoff)
+            wfb.worker.resetQuarantine()
+            return False
+        # Schedule the build now
+        return True
+    return canStartBuild
+
+
+def is_within_time_range(now, start, end):
+    if start <= end:
+        return start <= now <= end
+    else:
+        return now >= start or now <= end
+
+
+def get_delay(now, end):
+    today = datetime.today()
+    now = datetime.combine(today, now)
+    end = datetime.combine(today, end)
+
+    if now > end:
+        end += timedelta(days=1)
+
+    difference = end - now
+    return difference.total_seconds()

master/custom/workers.py

@@ -3,10 +3,13 @@
 # vim:set ts=8 sw=4 sts=4 et:
 
 from functools import partial
+from zoneinfo import ZoneInfo
+import calendar
 
 from buildbot.plugins import worker as _worker
 
 from custom.factories import MAIN_BRANCH_NAME
+from custom.worker_downtime import no_builds_between
 
 
 # By default, the buildmaster sends a simple, non-blocking message to each
@@ -32,6 +35,7 @@ def __init__(
         parallel_tests=None,
         timeout_factor=1,
         exclude_test_resources=None,
+        downtime=None
     ):
         self.name = name
         self.tags = tags or set()
@@ -41,6 +45,7 @@ def __init__(
         self.parallel_tests = parallel_tests
         self.timeout_factor = timeout_factor
         self.exclude_test_resources = exclude_test_resources or []
+        self.downtime = downtime
 
         worker_settings = settings.workers[name]
         owner = name.split("-")[0]
@@ -55,6 +60,14 @@ def __init__(
                                             notify_on_missing=emails,
                                             keepalive_interval=KEEPALIVE)
 
+# Some of Itamar's workers are reprovisioned every Wednesday at 9am PT.
+# Builds scheduled between 8am - 10am PT on Wednesdays will be delayed to
+# 10am PT.
+itamaro_downtime = no_builds_between(
+    "8:00", "10:00",
+    day_of_week=calendar.WEDNESDAY,
+    tz=ZoneInfo("America/Los_Angeles"),
+)
 
 def get_workers(settings):
     cpw = partial(CPythonWorker, settings)
@@ -160,6 +173,10 @@ def get_workers(settings):
             tags=['linux', 'unix', 'ubuntu', 'arm', 'arm64', 'aarch64', 'bigmem'],
             not_branches=['3.10', '3.11', '3.12', '3.13', '3.14'],
             parallel_tests=8,
+            # This worker runs pyperformance at 12am UTC.
+            # If a build is scheduled between 10pm UTC and 2am UTC,
+            # it will be delayed to 2am UTC.
+            downtime=no_builds_between("22:00", "2:00")
         ),
         cpw(
             name="cstratak-fedora-rawhide-s390x",
@@ -294,20 +311,28 @@ def get_workers(settings):
             tags=['windows', 'win11', 'amd64', 'x86-64', 'bigmem'],
             not_branches=['3.10', '3.11', '3.12', '3.13', '3.14'],
             parallel_tests=4,
+            # This worker restarts every day at 9am UTC to work around issues
+            # stemming from failing bigmem tests trashing disk space and
+            # fragmenting RAM.
+            # Builds scheduled between 07:20am - 9:20am UTC will be delayed
+            # to 9:20am UTC.
+            downtime=no_builds_between("7:20", "9:20")
         ),
         cpw(
             name="itamaro-centos-aws",
             tags=['linux', 'unix', 'rhel', 'amd64', 'x86-64'],
             not_branches=['3.10', '3.11', '3.12'],
             parallel_tests=10,
             parallel_builders=2,
+            downtime=itamaro_downtime,
         ),
         cpw(
             name="itamaro-win64-srv-22-aws",
             tags=['windows', 'win-srv-22', 'amd64', 'x86-64'],
             not_branches=['3.10', '3.11', '3.12'],
             parallel_tests=10,
             parallel_builders=2,
+            downtime=itamaro_downtime,
         ),
         cpw(
             name="itamaro-macos-intel-aws",

master/master.cfg

@@ -14,9 +14,8 @@ import os
 import subprocess
 import sys
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 from functools import partial
-from zoneinfo import ZoneInfo
 
 from buildbot.plugins import reporters, schedulers, util
 from buildbot import locks
@@ -194,57 +193,13 @@ def is_important_change(change):
     return any(is_important_file(filename) for filename in change.files)
 
 
-def is_within_time_range(now, start, end):
-    if start <= end:
-        return start <= now <= end
-    else:
-        return now >= start or now <= end
-
-
-def get_delay(now, end):
-    today = datetime.today()
-    now = datetime.combine(today, now)
-    end = datetime.combine(today, end)
-
-    if now > end:
-        end += timedelta(days=1)
-
-    difference = end - now
-    return difference.total_seconds()
-
-
-# Avoid a build to be started between start and end time and delay such build
-# at end time
-def no_builds_between(start, end, *, day_of_week=None, tz=None):
-    start = datetime.strptime(start, "%H:%M").time()
-    end = datetime.strptime(end, "%H:%M").time()
-    def canStartBuild(builder, wfb, request):
-        now_dt = datetime.now(tz=tz)
-        if day_of_week is not None and now_dt.weekday() != day_of_week:
-            return True
-        now = now_dt.time()
-        if is_within_time_range(now, start, end):
-            delay = get_delay(now, end)
-            # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions
-            wfb.worker.quarantine_timeout = delay
-            wfb.worker.putInQuarantine()
-            # This does not take the worker out of quarantine, it only resets
-            # the timeout value to default (restarting the default
-            # exponential backoff)
-            wfb.worker.resetQuarantine()
-            return False
-        # Schedule the build now
-        return True
-    return canStartBuild
-
-
 github_status_builders = []
 release_status_builders = []
 mail_status_builders = []
 
 # Regular builders
 
-for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches):
+for git_url, branchname, git_branch in git_branches:
     buildernames = []
     refleakbuildernames = []
     for name, worker, buildfactory, stability, tier in BUILDERS:
@@ -308,24 +263,8 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches):
             locks=[cpulock.access("counting")],
         )
 
-        # This worker runs pyperformance at 12am UTC. If a build is scheduled between
-        # 10pm UTC and 2am UTC, it will be delayed to 2am UTC.
-        if worker.name == "diegorusso-aarch64-bigmem":
-            builder.canStartBuild = no_builds_between("22:00", "2:00")
-
-        # This worker restarts every day at 9am UTC to work around issues stemming from
-        # failing bigmem tests trashing disk space and fragmenting RAM. Builds scheduled
-        # between 07:20am - 9:20am UTC will be delayed to 9:20am UTC.
-        if worker.name == "ambv-bb-win11":
-            builder.canStartBuild = no_builds_between("7:20", "9:20")
-
-        # These workers are reprovisioned every Wednesday at 9am PT. Builds
-        # scheduled between 8am - 10am PT on Wednesdays will be delayed to
-        # 10am PT.
-        if worker.name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"):
-            builder.canStartBuild = no_builds_between(
-                "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles")
-            )
+        if worker.downtime:
+            builder.canStartBuild = worker.downtime
 
         c["builders"].append(builder)
 
@@ -397,18 +336,8 @@ for name, worker, buildfactory, stability, tier in BUILDERS:
         locks=[cpulock.access("counting")],
     )
 
-    # This worker runs pyperformance at 12am. If a build is scheduled between
-    # 10pm and 2am, it will be delayed at 2am.
-    if worker.name == "diegorusso-aarch64-bigmem":
-        builder.canStartBuild = no_builds_between("22:00", "2:00")
-
-    # These workers are reprovisioned every Wednesday at 9am PT. Builds
-    # scheduled between 8am - 10am PT on Wednesdays will be delayed to
-    # 10am PT.
-    if worker.name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"):
-        builder.canStartBuild = no_builds_between(
-            "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles")
-        )
+    if worker.downtime:
+        builder.canStartBuild = worker.downtime
 
     c["builders"].append(builder)