Skip to content

Commit e979b6e

Browse files
pwang347pwang347
committed
update
1 parent 41b7641 commit e979b6e

File tree

3 files changed

+78
-72
lines changed

3 files changed

+78
-72
lines changed

‎.github/workflows/chat-perf.yml‎

Lines changed: 4 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ jobs:
324324
- name: Run memory leak check
325325
id: leak
326326
run: |
327-
LEAK_ARGS="--verbose"
327+
LEAK_ARGS="--verbose --ci"
328328
if [[ -n "$TEST_COMMIT" ]]; then
329329
LEAK_ARGS="$LEAK_ARGS --build $TEST_COMMIT"
330330
fi
@@ -345,6 +345,7 @@ jobs:
345345
path: |
346346
leak-output.log
347347
.chat-simulation-data/chat-simulation-leak-results.json
348+
.chat-simulation-data/ci-summary-leak.md
348349
retention-days: 30
349350

350351
# ── Report: collect results, write summary, fail on regression ──────
@@ -390,40 +391,9 @@ jobs:
390391
echo "⚠️ No summary files generated. Check perf-output.log artifacts." >> "$GITHUB_STEP_SUMMARY"
391392
fi
392393
393-
if [[ -f leak-results/.chat-simulation-data/chat-simulation-leak-results.json ]]; then
394-
echo "" >> "$GITHUB_STEP_SUMMARY"
395-
echo "## Memory Leak Check" >> "$GITHUB_STEP_SUMMARY"
394+
if [[ -f leak-results/.chat-simulation-data/ci-summary-leak.md ]]; then
396395
echo "" >> "$GITHUB_STEP_SUMMARY"
397-
398-
node -e "
399-
const r = JSON.parse(require('fs').readFileSync('leak-results/.chat-simulation-data/chat-simulation-leak-results.json', 'utf-8'));
400-
const threshold = r.leakThresholdMB || 10;
401-
const leaked = r.totalResidualMB > threshold;
402-
const verdict = leaked ? '❌ **LEAK DETECTED**' : '✅ **No leak detected**';
403-
const lines = [];
404-
lines.push('| | |');
405-
lines.push('|---|---|');
406-
lines.push('| **Verdict** | ' + verdict + ' |');
407-
lines.push('| **Threshold** | ' + threshold + ' MB |');
408-
lines.push('| **Iterations** | ' + (r.iterationCount || r.iterations.length) + ' (+ 1 warmup) |');
409-
lines.push('| **Scenarios per iteration** | ' + (r.scenarioCount || '—') + ' |');
410-
lines.push('');
411-
lines.push('| Phase | Heap (MB) | DOM Nodes |');
412-
lines.push('|-------|----------:|----------:|');
413-
lines.push('| Baseline (post-warmup) | ' + r.baseline.heapMB + ' | ' + r.baseline.domNodes + ' |');
414-
for (let i = 0; i < r.iterations.length; i++) {
415-
const it = r.iterations[i];
416-
const sign = it.deltaHeapMB > 0 ? '+' : '';
417-
const domSign = it.deltaDomNodes > 0 ? '+' : '';
418-
lines.push('| Iteration ' + (i + 1) + ' | ' + it.afterHeapMB + ' (' + sign + it.deltaHeapMB + ') | ' + it.afterDomNodes + ' (' + domSign + it.deltaDomNodes + ') |');
419-
}
420-
lines.push('| **Final** | **' + r.final.heapMB + '** | **' + r.final.domNodes + '** |');
421-
lines.push('');
422-
const sign = r.totalResidualMB > 0 ? '+' : '';
423-
const domSign = r.totalResidualNodes > 0 ? '+' : '';
424-
lines.push('**Total residual growth:** ' + sign + r.totalResidualMB + ' MB heap, ' + domSign + r.totalResidualNodes + ' DOM nodes');
425-
console.log(lines.join('\n'));
426-
" >> "$GITHUB_STEP_SUMMARY"
396+
cat leak-results/.chat-simulation-data/ci-summary-leak.md >> "$GITHUB_STEP_SUMMARY"
427397
fi
428398
429399
- name: Zip diagnostic outputs

‎scripts/chat-simulation/test-chat-mem-leaks.js‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ function parseArgs() {
5555
iterations: CONFIG.iterations ?? 3,
5656
messages: CONFIG.messages ?? 5,
5757
verbose: false,
58+
ci: false,
5859
/** @type {string | undefined} */
5960
build: undefined,
6061
leakThresholdMB: CONFIG.leakThresholdMB ?? 5,
@@ -64,6 +65,7 @@ function parseArgs() {
6465
case '--iterations': opts.iterations = parseInt(args[++i], 10); break;
6566
case '--messages': case '-n': opts.messages = parseInt(args[++i], 10); break;
6667
case '--verbose': opts.verbose = true; break;
68+
case '--ci': opts.ci = true; break;
6769
case '--build': case '-b': opts.build = args[++i]; break;
6870
case '--threshold': opts.leakThresholdMB = parseFloat(args[++i]); break;
6971
case '--help': case '-h':
@@ -73,6 +75,7 @@ function parseArgs() {
7375
'Options:',
7476
' --iterations <n> Number of open→work→reset cycles (default: 3)',
7577
' --messages <n> Messages to send per iteration (default: 5)',
78+
' --ci CI mode: write Markdown summary to ci-summary.md',
7679
' --build <path|ver> Path to VS Code build or version to download',
7780
' --threshold <MB> Max total residual heap growth in MB (default: 5)',
7881
' --verbose Print per-step details',
@@ -413,8 +416,51 @@ async function main() {
413416
console.log(`[chat-simulation] No leak detected (${result.totalResidualMB}MB residual < ${opts.leakThresholdMB}MB threshold)`);
414417
}
415418

419+
if (opts.ci) {
420+
const summary = generateLeakCISummary(result, opts);
421+
const summaryPath = path.join(DATA_DIR, 'ci-summary-leak.md');
422+
fs.writeFileSync(summaryPath, summary);
423+
console.log(`[chat-simulation] CI summary written to ${summaryPath}`);
424+
}
425+
416426
await mockServer.close();
417427
process.exit(leaked ? 1 : 0);
418428
}
419429

430+
/**
431+
* Generate a Markdown summary for CI, matching the perf script pattern.
432+
* @param {{ baseline: { heapMB: number, domNodes: number }, final: { heapMB: number, domNodes: number }, totalResidualMB: number, totalResidualNodes: number, iterations: { beforeHeapMB: number, afterHeapMB: number, deltaHeapMB: number, beforeDomNodes: number, afterDomNodes: number, deltaDomNodes: number }[] }} result
433+
* @param {{ leakThresholdMB: number, iterations: number }} opts
434+
*/
435+
function generateLeakCISummary(result, opts) {
436+
const leaked = result.totalResidualMB > opts.leakThresholdMB;
437+
const verdict = leaked ? '\u274C **LEAK DETECTED**' : '\u2705 **No leak detected**';
438+
const lines = [];
439+
lines.push('## Memory Leak Check');
440+
lines.push('');
441+
lines.push('| | |');
442+
lines.push('|---|---|');
443+
lines.push(`| **Verdict** | ${verdict} |`);
444+
lines.push(`| **Threshold** | ${opts.leakThresholdMB} MB |`);
445+
lines.push(`| **Iterations** | ${opts.iterations} (+ 1 warmup) |`);
446+
lines.push(`| **Scenarios per iteration** | ${getScenarioIds().length} |`);
447+
lines.push('');
448+
lines.push('| Phase | Heap (MB) | DOM Nodes |');
449+
lines.push('|-------|----------:|----------:|');
450+
lines.push(`| Baseline (post-warmup) | ${result.baseline.heapMB} | ${result.baseline.domNodes} |`);
451+
for (let i = 0; i < result.iterations.length; i++) {
452+
const it = result.iterations[i];
453+
const sign = it.deltaHeapMB > 0 ? '+' : '';
454+
const domSign = it.deltaDomNodes > 0 ? '+' : '';
455+
lines.push(`| Iteration ${i + 1} | ${it.afterHeapMB} (${sign}${it.deltaHeapMB}) | ${it.afterDomNodes} (${domSign}${it.deltaDomNodes}) |`);
456+
}
457+
lines.push(`| **Final** | **${result.final.heapMB}** | **${result.final.domNodes}** |`);
458+
lines.push('');
459+
const sign = result.totalResidualMB > 0 ? '+' : '';
460+
const domSign = result.totalResidualNodes > 0 ? '+' : '';
461+
lines.push(`**Total residual growth:** ${sign}${result.totalResidualMB} MB heap, ${domSign}${result.totalResidualNodes} DOM nodes`);
462+
lines.push('');
463+
return lines.join('\n');
464+
}
465+
420466
main().catch(err => { console.error(err); process.exit(1); });

‎scripts/chat-simulation/test-chat-perf-regression.js‎

Lines changed: 28 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ async function runOnce(electronPath, scenario, mockServer, verbose, runIndex, ru
255255
let extHostInspector = null;
256256
/** @type {{ usedSize: number, totalSize: number } | null} */
257257
let extHostHeapBefore = null;
258-
/** @type {Omit<RunMetrics, 'majorGCs' | 'minorGCs' | 'gcDurationMs' | 'longTaskCount' | 'timeToUIUpdated' | 'timeToFirstToken' | 'timeToComplete' | 'instructionCollectionTime' | 'agentInvokeTime' | 'hasInternalMarks' | 'internalFirstToken'> | null} */
258+
/** @type {Omit<RunMetrics, 'majorGCs' | 'minorGCs' | 'gcDurationMs' | 'longTaskCount' | 'longAnimationFrameCount' | 'longAnimationFrameTotalMs' | 'timeToUIUpdated' | 'timeToFirstToken' | 'timeToComplete' | 'instructionCollectionTime' | 'agentInvokeTime' | 'hasInternalMarks' | 'internalFirstToken'> | null} */
259259
let partialMetrics = null;
260260
// Timing vars hoisted for access in post-close trace parsing
261261
let submitTime = 0;
@@ -365,26 +365,6 @@ async function runOnce(electronPath, scenario, mockServer, verbose, runIndex, ru
365365
await cdp.send('Profiler.enable');
366366
await cdp.send('Profiler.start');
367367

368-
// Install a PerformanceObserver for Long Animation Frames (LoAF)
369-
// to capture frame-level jank that longTaskCount alone misses.
370-
await window.evaluate(() => {
371-
// @ts-ignore
372-
globalThis._chatLoAFEntries = [];
373-
try {
374-
// @ts-ignore
375-
globalThis._chatLoAFObserver = new PerformanceObserver((list) => {
376-
for (const entry of list.getEntries()) {
377-
// @ts-ignore
378-
globalThis._chatLoAFEntries.push({ duration: entry.duration, startTime: entry.startTime });
379-
}
380-
});
381-
// @ts-ignore
382-
globalThis._chatLoAFObserver.observe({ type: 'long-animation-frame', buffered: false });
383-
} catch {
384-
// long-animation-frame not supported in this build — metrics will be 0
385-
}
386-
});
387-
388368
// Submit
389369
const completionsBefore = mockServer.completionCount();
390370
submitTime = Date.now();
@@ -505,21 +485,6 @@ async function runOnce(electronPath, scenario, mockServer, verbose, runIndex, ru
505485
console.log(` [debug] Client-side timing: firstResponse=${firstResponseTime - submitTime}ms, complete=${responseCompleteTime - submitTime}ms`);
506486
}
507487

508-
// Collect Long Animation Frame entries and tear down the observer
509-
const loafData = await window.evaluate(() => {
510-
// @ts-ignore
511-
if (globalThis._chatLoAFObserver) { globalThis._chatLoAFObserver.disconnect(); }
512-
// @ts-ignore
513-
const entries = globalThis._chatLoAFEntries ?? [];
514-
// @ts-ignore
515-
delete globalThis._chatLoAFEntries;
516-
// @ts-ignore
517-
delete globalThis._chatLoAFObserver;
518-
const count = entries.length;
519-
const totalMs = entries.reduce((/** @type {number} */ sum, /** @type {any} */ e) => sum + e.duration, 0);
520-
return { count, totalMs };
521-
});
522-
523488
const heapAfter = /** @type {any} */ (await cdp.send('Runtime.getHeapUsage'));
524489
const metricsAfter = await cdp.send('Performance.getMetrics');
525490

@@ -617,8 +582,6 @@ async function runOnce(electronPath, scenario, mockServer, verbose, runIndex, ru
617582
layoutCount: getMetric(metricsAfter, 'LayoutCount') - getMetric(metricsBefore, 'LayoutCount'),
618583
recalcStyleCount: getMetric(metricsAfter, 'RecalcStyleCount') - getMetric(metricsBefore, 'RecalcStyleCount'),
619584
forcedReflowCount: getMetric(metricsAfter, 'ForcedStyleRecalcs') - getMetric(metricsBefore, 'ForcedStyleRecalcs'),
620-
longAnimationFrameCount: loafData.count,
621-
longAnimationFrameTotalMs: Math.round(loafData.totalMs * 100) / 100,
622585
frameCount: getMetric(metricsAfter, 'FrameCount') - getMetric(metricsBefore, 'FrameCount'),
623586
compositeLayers: getMetric(metricsAfter, 'CompositeLayers') - getMetric(metricsBefore, 'CompositeLayers'),
624587
paintCount: getMetric(metricsAfter, 'PaintCount') - getMetric(metricsBefore, 'PaintCount'),
@@ -693,6 +656,30 @@ async function runOnce(electronPath, scenario, mockServer, verbose, runIndex, ru
693656
if (event.name === 'RunTask' && event.dur && event.dur > 50_000) { longTaskCount++; }
694657
}
695658

659+
// Parse Long Animation Frame (LoAF) events from devtools.timeline trace.
660+
// AnimationFrame events use async flow pairs (ph:'s' start, ph:'f' finish)
661+
// with matching ids. Compute duration from each s→f pair.
662+
let longAnimationFrameCount = 0;
663+
let longAnimationFrameTotalMs = 0;
664+
{
665+
/** @type {Map<number, number>} */
666+
const frameStarts = new Map();
667+
for (const event of traceEvents) {
668+
if (event.cat === 'devtools.timeline' && event.name === 'AnimationFrame') {
669+
if (event.ph === 's') {
670+
frameStarts.set(event.id, event.ts);
671+
} else if (event.ph === 'f' && frameStarts.has(event.id)) {
672+
const durationMs = (event.ts - frameStarts.get(event.id)) / 1000;
673+
frameStarts.delete(event.id);
674+
if (durationMs > 50) {
675+
longAnimationFrameCount++;
676+
longAnimationFrameTotalMs += durationMs;
677+
}
678+
}
679+
}
680+
}
681+
}
682+
696683
return {
697684
...partialMetrics,
698685
timeToUIUpdated, timeToFirstToken, timeToComplete, instructionCollectionTime, agentInvokeTime,
@@ -701,6 +688,8 @@ async function runOnce(electronPath, scenario, mockServer, verbose, runIndex, ru
701688
majorGCs, minorGCs,
702689
gcDurationMs: Math.round(gcDurationMs * 100) / 100,
703690
longTaskCount,
691+
longAnimationFrameCount,
692+
longAnimationFrameTotalMs: Math.round(longAnimationFrameTotalMs * 100) / 100,
704693
};
705694
}
706695

@@ -970,6 +959,7 @@ function generateCISummary(jsonReport, baseline, opts) {
970959
else if (v.verdict === 'improved') { verdictDisplay = '\u2B06\uFE0F improved'; }
971960
else if (v.verdict === 'ok') { verdictDisplay = '\u2705 ok'; }
972961
else if (v.verdict === 'noise') { verdictDisplay = '\uD83C\uDF2B\uFE0F noise'; }
962+
else if (v.verdict === 'info') { verdictDisplay = '\u2139\uFE0F'; }
973963
lines.push(`| ${v.metric} | ${v.basStr} | ${v.curStr} | ${pct} | ${v.pValue} | ${verdictDisplay} |`);
974964
}
975965
lines.push('');

0 commit comments

Comments
 (0)