Migrate CI to protected runners and JFrog PyPI proxy #241
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code Coverage | |
| permissions: | |
| contents: read | |
| id-token: write | |
| on: [pull_request, workflow_dispatch] | |
| jobs: | |
| test-with-coverage: | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
| DATABRICKS_CATALOG: peco | |
| DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup JFrog | |
| uses: ./.github/actions/setup-jfrog | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libkrb5-dev | |
| - name: Install Poetry | |
| run: | | |
| pip install poetry==2.2.1 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project true | |
| poetry config installer.parallel true | |
| - name: Configure Poetry for JFrog | |
| run: | | |
| poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple | |
| poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" | |
| poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple | |
| poetry lock | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| - name: Install Kerberos system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libkrb5-dev | |
| - name: Install library | |
| run: poetry install --no-interaction --all-extras | |
| - name: Run parallel tests with coverage | |
| continue-on-error: false | |
| run: | | |
| poetry run pytest tests/unit tests/e2e \ | |
| -m "not serial" \ | |
| -n auto \ | |
| --cov=src \ | |
| --cov-report=xml \ | |
| --cov-report=term \ | |
| -v | |
| - name: Run telemetry tests with coverage (isolated) | |
| continue-on-error: false | |
| run: | | |
| poetry run pytest tests/e2e/test_concurrent_telemetry.py \ | |
| --cov=src \ | |
| --cov-append \ | |
| --cov-report=xml \ | |
| --cov-report=term \ | |
| -v | |
| - name: Check for coverage override | |
| id: override | |
| env: | |
| PR_BODY: ${{ github.event.pull_request.body }} | |
| run: | | |
| OVERRIDE_COMMENT=$(echo "$PR_BODY" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "") | |
| if [ -n "$OVERRIDE_COMMENT" ]; then | |
| echo "override=true" >> $GITHUB_OUTPUT | |
| REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/') | |
| echo "reason=$REASON" >> $GITHUB_OUTPUT | |
| echo "Coverage override found in PR description: $REASON" | |
| else | |
| echo "override=false" >> $GITHUB_OUTPUT | |
| echo "No coverage override found" | |
| fi | |
| - name: Check coverage percentage | |
| if: steps.override.outputs.override == 'false' | |
| run: | | |
| COVERAGE_FILE="coverage.xml" | |
| if [ ! -f "$COVERAGE_FILE" ]; then | |
| echo "ERROR: Coverage file not found at $COVERAGE_FILE" | |
| exit 1 | |
| fi | |
| if ! command -v xmllint &> /dev/null; then | |
| sudo apt-get update && sudo apt-get install -y libxml2-utils | |
| fi | |
| COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE") | |
| TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE") | |
| PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))") | |
| echo "Branch Coverage: $PERCENTAGE%" | |
| echo "Required Coverage: 85%" | |
| python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)" | |
| if [ $? -eq 1 ]; then | |
| echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%" | |
| exit 1 | |
| else | |
| echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%" | |
| fi | |
| - name: Coverage enforcement summary | |
| env: | |
| OVERRIDE: ${{ steps.override.outputs.override }} | |
| REASON: ${{ steps.override.outputs.reason }} | |
| run: | | |
| if [ "$OVERRIDE" == "true" ]; then | |
| echo "Coverage checks bypassed: $REASON" | |
| echo "Please ensure this override is justified and temporary" | |
| else | |
| echo "Coverage checks enforced - minimum 85% required" | |
| fi |