Core: Add QUnit.config.testFilter to programmatically filter tests

Ref #1814.
Closes #1815.

Author: gbudjeakp

docs/api/config/testFilter.md

@@ -0,0 +1,96 @@
+---
+layout: page-api
+title: QUnit.config.testFilter
+excerpt: Programmatically filter which tests to run.
+groups:
+  - config
+version_added: "2.25.0"
+---
+
+Programmatically filter which tests to run.
+
+<table>
+<tr>
+  <th>type</th>
+  <td markdown="span">`function` or `null`</td>
+</tr>
+<tr>
+  <th>default</th>
+  <td markdown="span">`null`</td>
+</tr>
+</table>
+
+The `testFilter` property allows you to implement custom logic for filtering which tests to run at runtime. This is useful for advanced scenarios such as:
+
+* Quarantining flaky tests in CI environments
+* Distributing tests across parallel workers
+* Loading filter criteria from external sources (APIs, files, etc.)
+
+The callback receives a `testInfo` object and must return a boolean:
+* Return `true` to run the test
+* Return `false` to skip the test
+
+If the callback throws an error, the error is logged as a warning and the test is skipped.
+
+## Parameters
+
+### `testInfo` (object)
+
+| Property | Type | Description |
+|----------|------|-------------|
+| `testId` | string | Internal hash identifier (used by "Rerun" links)
+| `testName` | string | Name of the test
+| `module` | string | Name of the parent module
+| `skip` | boolean | Whether test was already marked to skip
+
+## See also
+
+* [QUnit.config.filter](./filter.md)
+* [QUnit.config.module](./module.md)
+* [test.if()](../QUnit/test.if.md)
+
+## Examples
+
+### Quarantine flaky tests
+
+Use an external quarantine list to skip unstable tests in CI without modifying test code.
+
+```js
+const quarantineList = ['flaky network test', 'timing-dependent test'];
+
+QUnit.config.testFilter = function (testInfo) {
+  if (process.env.CI === 'true') {
+    const isQuarantined = quarantineList.some(function (pattern) {
+      return testInfo.testName.indexOf(pattern) !== -1;
+    });
+    if (isQuarantined) {
+      console.log('[QUARANTINE] Skipping: ' + testInfo.testName);
+      return false;
+    }
+  }
+  return true;
+};
+```
+
+### Parallel test sharding
+
+Distribute tests across multiple workers using deterministic hash-based assignment.
+
+```js
+const WORKER_ID = parseInt(process.env.WORKER_ID, 10);
+const TOTAL_WORKERS = parseInt(process.env.TOTAL_WORKERS, 10);
+
+QUnit.config.testFilter = function (testInfo) {
+  let hash = 0;
+  for (let i = 0; i < testInfo.testId.length; i++) {
+    const char = testInfo.testId.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash;
+  }
+  hash = Math.abs(hash);
+
+  // Assign test to worker
+  const assignedWorker = hash % TOTAL_WORKERS;
+  return assignedWorker === WORKER_ID;
+};
+```

src/core/config.js

@@ -25,6 +25,8 @@ const config = {
   // Select by pattern or case-insensitive substring match against "moduleName: testName"
   filter: '',
 
+  testFilter: null,
+
   fixture: undefined,
 
   // HTML Reporter: Hide results of passed tests.

src/core/test.js

@@ -810,15 +810,34 @@ Test.prototype = {
     }
 
     const filter = config.filter;
-    if (!filter) {
-      return true;
+    if (filter) {
+      const regexFilter = /^(!?)\/([\w\W]*)\/(i?$)/.exec(filter);
+      const fullName = (this.module.name + ': ' + this.testName);
+      if (regexFilter) {
+        if (!this.regexFilter(!!regexFilter[1], regexFilter[2], regexFilter[3], fullName)) {
+          return false;
+        }
+      } else if (!this.stringFilter(filter, fullName)) {
+        return false;
+      }
+    }
+
+    if (typeof config.testFilter === 'function') {
+      const testInfo = {
+        testId: this.testId,
+        testName: this.testName,
+        module: this.module.name,
+        skip: !!this.skip
+      };
+      try {
+        return !!config.testFilter(testInfo);
+      } catch (error) {
+        Logger.warn('Error in QUnit.config.testFilter callback: ', error);
+        return false;
+      }
     }
 
-    const regexFilter = /^(!?)\/([\w\W]*)\/(i?$)/.exec(filter);
-    const fullName = (this.module.name + ': ' + this.testName);
-    return regexFilter
-      ? this.regexFilter(!!regexFilter[1], regexFilter[2], regexFilter[3], fullName)
-      : this.stringFilter(filter, fullName);
+    return true;
   },
 
   regexFilter: function (exclude, pattern, flags, fullName) {

test/cli/fixtures/config-testFilter.js

@@ -0,0 +1,25 @@
+const quarantineList = ['flaky test', 'broken test'];
+
+QUnit.config.testFilter = function (testInfo) {
+  return !quarantineList.some(function (pattern) {
+    return testInfo.testName.indexOf(pattern) !== -1;
+  });
+};
+
+QUnit.module('testFilter demo');
+
+QUnit.test('stable test', function (assert) {
+  assert.true(true, 'this test should run');
+});
+
+QUnit.test('flaky test', function (assert) {
+  assert.true(false, 'this test should be filtered out');
+});
+
+QUnit.test('broken test', function (assert) {
+  assert.true(false, 'this test should also be filtered out');
+});
+
+QUnit.test('another stable test', function (assert) {
+  assert.true(true, 'this test should also run');
+});

test/cli/fixtures/config-testFilter.tap.txt

@@ -0,0 +1,11 @@
+# name: config.testFilter
+# command: ["qunit", "config-testFilter.js"]
+
+TAP version 13
+ok 1 testFilter demo > stable test
+ok 2 testFilter demo > another stable test
+1..2
+# pass 2
+# skip 0
+# todo 0
+# fail 0