benchmark.yml

name: Benchmark Generator

on:
  workflow_dispatch:

permissions:
  contents: read
  actions: write

jobs:
  benchmark:
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-java@v5
        with:
          distribution: 'temurin'
          java-version: '25'

      - uses: jbangdev/setup-jbang@main

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'

      - name: Run benchmark
        shell: bash
        run: |
          JAR="html-generators/generate.jar"
          AOT="html-generators/generate.aot"
          STEADY_RUNS=5

          snippet_count=$(find content -name '*.json' | wc -l | tr -d ' ')
          java_ver=$(java -version 2>&1 | head -1 | sed 's/.*"\(.*\)".*/\1/')
          os_name="${{ matrix.os }}"

          # Timing helper using bash TIMEFORMAT
          measure() {
            TIMEFORMAT='%R'
            local t
            t=$( { time "$@" > /dev/null 2>&1; } 2>&1 )
            echo "$t"
          }

          avg_runs() {
            local n="$1"; shift
            local sum=0
            for ((i = 1; i <= n; i++)); do
              local t
              t=$(measure "$@")
              sum=$(awk "BEGIN {print $sum + $t}")
            done
            awk "BEGIN {printf \"%.2f\", $sum / $n}"
          }

          echo "Running benchmark on $os_name (Java $java_ver, $snippet_count snippets)..."

          # --- Phase 1: Training / build cost ---
          rm -f "$JAR" "$AOT"
          find html-generators -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true

          PY_TRAIN=$(measure python3 html-generators/generate.py)
          JBANG_EXPORT=$(measure jbang export fatjar --force --output "$JAR" html-generators/generate.java)
          AOT_TRAIN=$(measure java -XX:AOTCacheOutput="$AOT" -jar "$JAR")

          # --- Phase 2: Steady-state execution ---
          PY_STEADY=$(avg_runs $STEADY_RUNS python3 html-generators/generate.py)
          JBANG_STEADY=$(avg_runs $STEADY_RUNS jbang html-generators/generate.java)
          JAR_STEADY=$(avg_runs $STEADY_RUNS java -jar "$JAR")
          AOT_STEADY=$(avg_runs $STEADY_RUNS java -XX:AOTCache="$AOT" -jar "$JAR")

          # Write to GitHub Actions Job Summary
          {
            echo "## Benchmark Results — \`$os_name\`"
            echo ""
            echo "Java $java_ver · $snippet_count snippets"
            echo ""
            echo "### Phase 1: Training / Build Cost (one-time)"
            echo ""
            echo "| Step | Time | What it does |"
            echo "|------|------|-------------|"
            echo "| Python first run | ${PY_TRAIN}s | Interprets source, creates \`__pycache__\` bytecode |"
            echo "| JBang export | ${JBANG_EXPORT}s | Compiles source + bundles dependencies into fat JAR |"
            echo "| AOT training run | ${AOT_TRAIN}s | Runs JAR once to record class loading, produces \`.aot\` cache |"
            echo ""
            echo "### Phase 2: Steady-State Execution (avg of $STEADY_RUNS runs)"
            echo ""
            echo "| Method | Avg Time |"
            echo "|--------|---------|"
            echo "| **Fat JAR + AOT** | **${AOT_STEADY}s** |"
            echo "| **Fat JAR** | ${JAR_STEADY}s |"
            echo "| **JBang** | ${JBANG_STEADY}s |"
            echo "| **Python** | ${PY_STEADY}s |"
          } >> "$GITHUB_STEP_SUMMARY"

  # ---------------------------------------------------------------------------
  # Phase 3: CI cold start — runs on a completely fresh runner.
  # JAR and AOT are built once then passed via artifact, simulating
  # the actions cache restore that happens in the deploy workflow.
  # Python and JBang start with zero caches, just like real CI.
  # ---------------------------------------------------------------------------
  build-jar:
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-java@v5
        with:
          distribution: 'temurin'
          java-version: '25'

      - uses: jbangdev/setup-jbang@main

      - name: Build fat JAR and AOT cache
        run: |
          jbang export fatjar --force --output html-generators/generate.jar html-generators/generate.java
          java -XX:AOTCacheOutput=html-generators/generate.aot -jar html-generators/generate.jar

      - name: Upload JAR and AOT
        uses: actions/upload-artifact@v4
        with:
          name: generator-${{ matrix.os }}
          path: |
            html-generators/generate.jar
            html-generators/generate.aot

  ci-cold-start:
    needs: build-jar
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-java@v5
        with:
          distribution: 'temurin'
          java-version: '25'

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'

      - name: Download JAR and AOT
        uses: actions/download-artifact@v4
        with:
          name: generator-${{ matrix.os }}
          path: html-generators

      - name: CI cold-start benchmark
        shell: bash
        run: |
          os_name="${{ matrix.os }}"
          java_ver=$(java -version 2>&1 | head -1 | sed 's/.*"\(.*\)".*/\1/')
          snippet_count=$(find content -name '*.json' | wc -l | tr -d ' ')

          measure() {
            TIMEFORMAT='%R'
            local t
            t=$( { time "$@" > /dev/null 2>&1; } 2>&1 )
            echo "$t"
          }

          # Everything is cold: no __pycache__, no JBang cache, fresh JVM
          PY_CI=$(measure python3 html-generators/generate.py)
          JAR_CI=$(measure java -jar html-generators/generate.jar)
          AOT_CI=$(measure java -XX:AOTCache=html-generators/generate.aot -jar html-generators/generate.jar)

          {
            echo "## CI Cold Start — \`$os_name\`"
            echo ""
            echo "Java $java_ver · $snippet_count snippets"
            echo ""
            echo "Fresh runner, no caches. JAR and AOT restored from artifact"
            echo "(simulates actions cache restore in deploy workflow)."
            echo ""
            echo "| Method | Time |"
            echo "|--------|------|"
            echo "| **Fat JAR + AOT** | **${AOT_CI}s** |"
            echo "| **Fat JAR** | ${JAR_CI}s |"
            echo "| **Python** | ${PY_CI}s |"
          } >> "$GITHUB_STEP_SUMMARY"