e2e-autotest.yml

name: E2E AutoTest

on:
  schedule:
    # Every weekday (Mon–Fri) at 13:00 Shanghai time (05:00 UTC)
    - cron: '0 5 * * 1-5'
  workflow_dispatch:
    inputs:
      test_plan:
        description: "Test plan to run (leave empty for all)"
        required: false
        default: ""
        type: string
      vsix_urls:
        description: "VSIX URLs or GitHub release tag URLs, comma-separated (e.g. https://github.com/redhat-developer/vscode-java/releases/tag/v1.54.0,https://host/vscode-java-debug-0.58.0.vsix). Release tag URLs auto-resolve to the platform-specific VSIX for the runner OS."
        required: false
        default: ""
        type: string
      pre_release:
        description: "Install pre-release versions of marketplace extensions"
        required: false
        default: true
        type: boolean

jobs:
  # ── Job 1: Discover test plans ──────────────────────────
  discover:
    if: ${{ inputs.test_plan == '' }}
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.scan.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Scan test plans
        id: scan
        run: |
          plans=$(ls test-plans/*.yaml | xargs -I{} basename {} .yaml | grep -v java-fresh-import | jq -R . | jq -sc .)
          echo "matrix=$plans" >> "$GITHUB_OUTPUT"
          echo "Found plans: $plans"

  # ── Job 2: Run each test plan in parallel ───────────────
  e2e-test:
    if: ${{ inputs.test_plan == '' }}
    needs: discover
    runs-on: windows-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        plan: ${{ fromJson(needs.discover.outputs.matrix) }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Clone vscode-java (test projects)
        run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java

      - name: Clone eclipse.jdt.ls (Gradle test projects)
        run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Setup Java 25 (for java25 test plans)
        if: contains(matrix.plan, 'java25')
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 25-ea

      - name: Create JDK 25 path
        if: contains(matrix.plan, 'java25')
        shell: pwsh
        run: |
          New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME

      - name: Setup Java 21
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 21

      - name: Install autotest CLI
        run: npm install -g @vscjava/vscode-autotest

      - name: Download VSIX files
        if: ${{ inputs.vsix_urls != '' }}
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Path vsix -Force | Out-Null
          $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" }

          # Map runner OS/arch to vscode-java platform identifiers
          $platformMap = @{ "Windows" = "win32"; "Linux" = "linux"; "macOS" = "darwin" }
          $archMap = @{ "X64" = "x64"; "ARM64" = "arm64" }
          $platform = $platformMap["${{ runner.os }}"]
          $arch = $archMap["${{ runner.arch }}"]
          $platformId = "$platform-$arch"
          Write-Host "Runner platform: $platformId (${{ runner.os }}/${{ runner.arch }})"

          $resolvedUrls = @()
          foreach ($url in $urls) {
            if ($url -match '^https://github\.com/([^/]+)/([^/]+)/releases/tag/(.+)$') {
              $owner = $Matches[1]; $repo = $Matches[2]; $tag = $Matches[3]
              Write-Host "Resolving GitHub release: $owner/$repo@$tag for platform $platformId"
              $apiUrl = "https://api.github.com/repos/$owner/$repo/releases/tags/$tag"
              $release = Invoke-RestMethod -Uri $apiUrl -Headers @{ Accept = "application/vnd.github.v3+json" } -UseBasicParsing
              $platformAsset = $release.assets | Where-Object { $_.name -like "*-$platformId-*" -and $_.name -like "*.vsix" } | Select-Object -First 1
              if ($platformAsset) {
                Write-Host "  Found platform-specific VSIX: $($platformAsset.name)"
                $resolvedUrls += $platformAsset.browser_download_url
              } else {
                $universalAsset = $release.assets | Where-Object { $_.name -notmatch '-(darwin|linux|win32)-' -and $_.name -like "*.vsix" } | Select-Object -First 1
                if ($universalAsset) {
                  Write-Host "  No platform-specific VSIX found, using universal: $($universalAsset.name)"
                  $resolvedUrls += $universalAsset.browser_download_url
                } else {
                  Write-Host "::warning::No matching VSIX found in release $owner/$repo@$tag for platform $platformId"
                }
              }
            } else {
              $resolvedUrls += $url
            }
          }

          foreach ($url in $resolvedUrls) {
            $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0])
            Write-Host "Downloading: $url → vsix/$fileName"
            Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing
          }
          Write-Host "Downloaded VSIX files:"
          Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { Write-Host "  $($_.Name) ($([math]::Round($_.Length/1MB, 1)) MB)" }

      - name: Run ${{ matrix.plan }}
        shell: pwsh
        env:
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
        run: |
          $autotestArgs = @("run", "test-plans/${{ matrix.plan }}.yaml")
          if (Test-Path vsix) {
            $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join ","
            if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) }
          }
          if ("${{ inputs.pre_release }}" -ne "false") { $autotestArgs += "--pre-release" }
          Write-Host "Running: autotest $($autotestArgs -join ' ')"
          & autotest @autotestArgs

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: results-${{ matrix.plan }}
          path: test-results/
          retention-days: 30

  # ── Job 2b: Run a single test plan (when specified) ─────
  e2e-single:
    if: ${{ inputs.test_plan != '' }}
    runs-on: windows-latest
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Clone vscode-java
        run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java

      - name: Clone eclipse.jdt.ls
        run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Setup Java 25
        if: contains(inputs.test_plan, 'java25')
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 25-ea

      - name: Create JDK 25 path
        if: contains(inputs.test_plan, 'java25')
        shell: pwsh
        run: |
          New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME

      - name: Setup Java 21
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 21

      - name: Install autotest CLI
        run: npm install -g @vscjava/vscode-autotest

      - name: Download VSIX files
        if: ${{ inputs.vsix_urls != '' }}
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Path vsix -Force | Out-Null
          $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" }

          # Map runner OS/arch to vscode-java platform identifiers
          $platformMap = @{ "Windows" = "win32"; "Linux" = "linux"; "macOS" = "darwin" }
          $archMap = @{ "X64" = "x64"; "ARM64" = "arm64" }
          $platform = $platformMap["${{ runner.os }}"]
          $arch = $archMap["${{ runner.arch }}"]
          $platformId = "$platform-$arch"
          Write-Host "Runner platform: $platformId (${{ runner.os }}/${{ runner.arch }})"

          $resolvedUrls = @()
          foreach ($url in $urls) {
            if ($url -match '^https://github\.com/([^/]+)/([^/]+)/releases/tag/(.+)$') {
              $owner = $Matches[1]; $repo = $Matches[2]; $tag = $Matches[3]
              Write-Host "Resolving GitHub release: $owner/$repo@$tag for platform $platformId"
              $apiUrl = "https://api.github.com/repos/$owner/$repo/releases/tags/$tag"
              $release = Invoke-RestMethod -Uri $apiUrl -Headers @{ Accept = "application/vnd.github.v3+json" } -UseBasicParsing
              $platformAsset = $release.assets | Where-Object { $_.name -like "*-$platformId-*" -and $_.name -like "*.vsix" } | Select-Object -First 1
              if ($platformAsset) {
                Write-Host "  Found platform-specific VSIX: $($platformAsset.name)"
                $resolvedUrls += $platformAsset.browser_download_url
              } else {
                $universalAsset = $release.assets | Where-Object { $_.name -notmatch '-(darwin|linux|win32)-' -and $_.name -like "*.vsix" } | Select-Object -First 1
                if ($universalAsset) {
                  Write-Host "  No platform-specific VSIX found, using universal: $($universalAsset.name)"
                  $resolvedUrls += $universalAsset.browser_download_url
                } else {
                  Write-Host "::warning::No matching VSIX found in release $owner/$repo@$tag for platform $platformId"
                }
              }
            } else {
              $resolvedUrls += $url
            }
          }

          foreach ($url in $resolvedUrls) {
            $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0])
            Write-Host "Downloading: $url → vsix/$fileName"
            Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing
          }
          Write-Host "Downloaded VSIX files:"
          Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { Write-Host "  $($_.Name) ($([math]::Round($_.Length/1MB, 1)) MB)" }

      - name: Run ${{ inputs.test_plan }}
        shell: pwsh
        env:
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
        run: |
          $autotestArgs = @("run", "test-plans/${{ inputs.test_plan }}")
          if (Test-Path vsix) {
            $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join ","
            if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) }
          }
          if ("${{ inputs.pre_release }}" -ne "false") { $autotestArgs += "--pre-release" }
          Write-Host "Running: autotest $($autotestArgs -join ' ')"
          & autotest @autotestArgs

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-test-results
          path: test-results/
          retention-days: 30

  # ── Job 3: Aggregate analysis ───────────────────────────
  analyze:
    if: ${{ always() && inputs.test_plan == '' }}
    needs: e2e-test
    runs-on: ubuntu-latest

    steps:
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install autotest CLI
        run: npm install -g @vscjava/vscode-autotest

      - name: Download all results
        uses: actions/download-artifact@v4
        with:
          pattern: results-*
          path: all-results
          merge-multiple: false

      - name: Organize results
        run: |
          mkdir -p test-results
          for dir in all-results/results-*/; do
            find "$dir" -name "results.json" -exec dirname {} \; | while read d; do
              name=$(basename "$d")
              cp -r "$d" "test-results/$name"
            done
          done
          echo "Organized results:"
          ls test-results/

      - name: Analyze results
        env:
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
        run: autotest analyze test-results --output test-results

      - name: Write Job Summary
        if: always()
        run: |
          if [ -f test-results/summary.md ]; then
            cat test-results/summary.md >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload aggregate results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-aggregate-summary
          path: test-results/summary.md
          retention-days: 30