From 2bf65b91eb56125ec6cac59061d76a78f7816e53 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 09:45:20 +0000 Subject: [PATCH 1/8] Migrate CI to databricks-protected runners and route PyPI through JFrog Protected runners are required for Databricks OSS repos. Add a setup-jfrog composite action (OIDC-based, matching databricks-odbc) that sets PIP_INDEX_URL so all pip/poetry installs go through the JFrog PyPI proxy. Every workflow now runs on the databricks-protected-runner-group with id-token: write for the OIDC exchange. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/setup-jfrog/action.yml | 32 +++++++++++++++++++++++ .github/workflows/code-coverage.yml | 7 ++++- .github/workflows/code-quality-checks.yml | 25 +++++++++++++++--- .github/workflows/daily-telemetry-e2e.yml | 10 +++++-- .github/workflows/dco-check.yml | 4 ++- .github/workflows/integration.yml | 13 +++++++-- 6 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 .github/actions/setup-jfrog/action.yml diff --git a/.github/actions/setup-jfrog/action.yml b/.github/actions/setup-jfrog/action.yml new file mode 100644 index 000000000..97ae146ba --- /dev/null +++ b/.github/actions/setup-jfrog/action.yml @@ -0,0 +1,32 @@ +name: Setup JFrog OIDC +description: Obtain a JFrog access token via GitHub OIDC and configure pip to use JFrog PyPI proxy + +runs: + using: composite + steps: + - name: Get JFrog OIDC token + shell: bash + run: | + set -euo pipefail + ID_TOKEN=$(curl -sLS \ + -H "User-Agent: actions/oidc-client" \ + -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"') + echo "::add-mask::${ID_TOKEN}" + ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \ + "https://databricks.jfrog.io/access/api/v1/oidc/token" \ + -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"') + echo "::add-mask::${ACCESS_TOKEN}" + if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then + echo "FAIL: Could not extract JFrog access token" + exit 1 + fi + echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" + echo "JFrog OIDC token obtained successfully" + + - name: Configure pip + shell: bash + run: | + set -euo pipefail + echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" + echo "pip configured to use JFrog registry" diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 9250d0ca5..616d0b317 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -2,12 +2,15 @@ name: Code Coverage permissions: contents: read + id-token: write on: [pull_request, workflow_dispatch] jobs: test-with-coverage: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest environment: azure-prod env: DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} @@ -23,6 +26,8 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 13a889c56..9e6afc54b 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -4,10 +4,13 @@ on: [pull_request] permissions: contents: read + id-token: write jobs: run-unit-tests: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] @@ -27,6 +30,8 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -92,7 +97,9 @@ jobs: - name: Run tests run: poetry run python -m pytest tests/unit run-unit-tests-with-arrow: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] @@ -111,6 +118,8 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 @@ -179,7 +188,9 @@ jobs: - name: Run tests run: poetry run python -m pytest tests/unit check-linting: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] @@ -189,6 +200,8 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -232,7 +245,9 @@ jobs: run: poetry run black --check src check-types: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] @@ -242,6 +257,8 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index c3d0da5df..4cc28b78d 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -14,10 +14,13 @@ on: permissions: contents: read + id-token: write jobs: telemetry-e2e-tests: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest environment: azure-prod env: @@ -33,7 +36,10 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 diff --git a/.github/workflows/dco-check.yml b/.github/workflows/dco-check.yml index b504ad84b..fdcf1b3bb 100644 --- a/.github/workflows/dco-check.yml +++ b/.github/workflows/dco-check.yml @@ -10,7 +10,9 @@ permissions: jobs: dco-check: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest name: Check DCO Sign-off steps: - name: Checkout diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 49dedfd91..950605b49 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,10 +8,13 @@ on: permissions: contents: read + id-token: write jobs: run-non-telemetry-tests: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest environment: azure-prod env: DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} @@ -25,6 +28,8 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -71,7 +76,9 @@ jobs: -n auto run-telemetry-tests: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest needs: run-non-telemetry-tests # Run after non-telemetry tests complete environment: azure-prod env: @@ -83,6 +90,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 From 96f33ce9c19f8bd78dc65df75e94ad97ca872a60 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 09:51:39 +0000 Subject: [PATCH 2/8] Add Poetry JFrog source configuration to all workflows The previous commit only set PIP_INDEX_URL, but Poetry uses its own resolver and needs explicit source configuration. Add a "Configure Poetry for JFrog" step after poetry install in every job that sets up the JFrog repository and credentials, then adds it as the primary source for the project. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/setup-jfrog/action.yml | 3 ++- .github/workflows/code-coverage.yml | 5 +++++ .github/workflows/code-quality-checks.yml | 20 ++++++++++++++++++++ .github/workflows/daily-telemetry-e2e.yml | 5 +++++ .github/workflows/integration.yml | 10 ++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-jfrog/action.yml b/.github/actions/setup-jfrog/action.yml index 97ae146ba..d42931ea1 100644 --- a/.github/actions/setup-jfrog/action.yml +++ b/.github/actions/setup-jfrog/action.yml @@ -1,5 +1,5 @@ name: Setup JFrog OIDC -description: Obtain a JFrog access token via GitHub OIDC and configure pip to use JFrog PyPI proxy +description: Obtain a JFrog access token via GitHub OIDC and configure pip and poetry to use JFrog PyPI proxy runs: using: composite @@ -30,3 +30,4 @@ runs: set -euo pipefail echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" echo "pip configured to use JFrog registry" + diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 616d0b317..6abbee1fb 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -50,6 +50,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 9e6afc54b..3f44a6c7c 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -47,6 +47,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple #---------------------------------------------- # load cached venv if cache exists @@ -135,6 +140,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple #---------------------------------------------- # load cached venv if cache exists @@ -217,6 +227,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple #---------------------------------------------- # load cached venv if cache exists @@ -274,6 +289,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index 4cc28b78d..c23c67e24 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -56,6 +56,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 950605b49..8279a5eb5 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -45,6 +45,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple #---------------------------------------------- # load cached venv if cache exists @@ -107,6 +112,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 From c0bbbcfabea9da611316f92b3c1a6e8a3a972e73 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 09:56:28 +0000 Subject: [PATCH 3/8] Fix step ordering: move JFrog setup after poetry install The snok/install-poetry action uses pip internally to install poetry. When PIP_INDEX_URL was set before this step, the installer tried to route through JFrog and failed with an SSL error. Move the JFrog OIDC token + PIP_INDEX_URL + poetry source configuration to run after Install Poetry but before poetry install. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-coverage.yml | 20 +++++++------ .github/workflows/code-quality-checks.yml | 36 +++++++++++++++-------- .github/workflows/daily-telemetry-e2e.yml | 23 ++++++++------- .github/workflows/integration.yml | 18 ++++++++---- 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 6abbee1fb..17134096e 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -26,8 +26,6 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -50,6 +48,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -107,7 +110,7 @@ jobs: --cov-report=xml \ --cov-report=term \ -v - + #---------------------------------------------- # check for coverage override #---------------------------------------------- @@ -137,19 +140,19 @@ jobs: echo "ERROR: Coverage file not found at $COVERAGE_FILE" exit 1 fi - + # Install xmllint if not available if ! command -v xmllint &> /dev/null; then sudo apt-get update && sudo apt-get install -y libxml2-utils fi - + COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE") TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE") PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))") - + echo "Branch Coverage: $PERCENTAGE%" echo "Required Coverage: 85%" - + # Use Python to compare the coverage with 85 python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)" if [ $? -eq 1 ]; then @@ -158,7 +161,7 @@ jobs: else echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%" fi - + #---------------------------------------------- # coverage enforcement summary #---------------------------------------------- @@ -173,4 +176,3 @@ jobs: else echo "✅ Coverage checks enforced - minimum 85% required" fi - diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 3f44a6c7c..4d9293b18 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -21,17 +21,15 @@ jobs: dependency-version: "min" - python-version: "3.13" dependency-version: "min" - + name: "Unit Tests (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)" - + steps: #---------------------------------------------- # check-out repo and set-up python #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -47,6 +45,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -114,17 +117,15 @@ jobs: dependency-version: "min" - python-version: "3.13" dependency-version: "min" - + name: "Unit Tests + PyArrow (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)" - + steps: #---------------------------------------------- # check-out repo and set-up python #---------------------------------------------- - name: Check out repository uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 @@ -140,6 +141,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -210,8 +216,6 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -227,6 +231,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -272,8 +281,6 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -289,6 +296,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index c23c67e24..ae0951171 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -3,7 +3,7 @@ name: Daily Telemetry E2E Tests on: schedule: - cron: '0 0 * * 0' # Run every Sunday at midnight UTC - + workflow_dispatch: # Allow manual triggering inputs: test_pattern: @@ -22,14 +22,14 @@ jobs: group: databricks-protected-runner-group labels: linux-ubuntu-latest environment: azure-prod - + env: DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_CATALOG: peco DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} - + steps: #---------------------------------------------- # check-out repo and set-up python @@ -37,15 +37,12 @@ jobs: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - + #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- @@ -56,6 +53,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -71,7 +73,7 @@ jobs: with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - + #---------------------------------------------- # install dependencies if cache does not exist #---------------------------------------------- @@ -81,7 +83,7 @@ jobs: sudo apt-get install -y libkrb5-dev - name: Install dependencies run: poetry install --no-interaction --all-extras - + #---------------------------------------------- # run telemetry E2E tests #---------------------------------------------- @@ -90,7 +92,7 @@ jobs: TEST_PATTERN="${{ github.event.inputs.test_pattern || 'tests/e2e/test_telemetry_e2e.py' }}" echo "Running tests: $TEST_PATTERN" poetry run python -m pytest $TEST_PATTERN -v -s - + #---------------------------------------------- # upload test results on failure #---------------------------------------------- @@ -103,4 +105,3 @@ jobs: .pytest_cache/ tests-unsafe.log retention-days: 7 - diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8279a5eb5..84b926ce1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,7 +1,7 @@ name: Integration Tests on: - push: + push: branches: - main pull_request: @@ -28,8 +28,6 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -45,6 +43,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -95,8 +98,6 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -112,6 +113,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -130,4 +136,4 @@ jobs: # Run test_concurrent_telemetry.py in isolation with complete process separation # Use --dist=loadgroup to respect @pytest.mark.xdist_group markers poetry run python -m pytest tests/e2e/test_concurrent_telemetry.py \ - -n auto --dist=loadgroup -v \ No newline at end of file + -n auto --dist=loadgroup -v From abca166798854f1da1ef093f375a65fd735b7094 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:01:19 +0000 Subject: [PATCH 4/8] Replace snok/install-poetry with pip install through JFrog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hardened runners block direct access to install.python-poetry.org, causing snok/install-poetry to fail with SSL errors. Replace it with `pip install poetry==2.2.1` which routes through the JFrog PyPI proxy. New step ordering: checkout → setup-python → Setup JFrog (OIDC + PIP_INDEX_URL) → pip install poetry → Configure Poetry for JFrog → poetry install. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-coverage.yml | 21 +++---- .github/workflows/code-quality-checks.yml | 76 +++++++++++------------ .github/workflows/daily-telemetry-e2e.yml | 20 +++--- .github/workflows/integration.yml | 39 ++++++------ 4 files changed, 74 insertions(+), 82 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 17134096e..98793d272 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -32,6 +32,11 @@ jobs: with: python-version: "3.10" #---------------------------------------------- + # ----- configure JFrog PyPI proxy ----- + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + #---------------------------------------------- # ----- install system dependencies ----- #---------------------------------------------- - name: Install system dependencies @@ -42,17 +47,11 @@ jobs: # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 4d9293b18..16842e953 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -36,20 +36,19 @@ jobs: with: python-version: ${{ matrix.python-version }} #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # ----- configure JFrog PyPI proxy ----- #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -132,20 +131,19 @@ jobs: with: python-version: ${{ matrix.python-version }} #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # ----- configure JFrog PyPI proxy ----- #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -222,20 +220,19 @@ jobs: with: python-version: ${{ matrix.python-version }} #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # ----- configure JFrog PyPI proxy ----- #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -287,20 +284,19 @@ jobs: with: python-version: ${{ matrix.python-version }} #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # ----- configure JFrog PyPI proxy ----- #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index ae0951171..9bc2b20ef 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -42,22 +42,20 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true #---------------------------------------------- # ----- configure JFrog PyPI proxy ----- #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 84b926ce1..69ccf4fa6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -34,20 +34,19 @@ jobs: with: python-version: "3.10" #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # ----- configure JFrog PyPI proxy ----- #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple @@ -103,21 +102,21 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true #---------------------------------------------- # ----- configure JFrog PyPI proxy ----- #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libkrb5-dev + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true - name: Configure Poetry for JFrog run: | poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple From 7a5740d41c0a7d498769efd2cef77eaa1991e464 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:05:55 +0000 Subject: [PATCH 5/8] Add poetry lock --no-update after source add to fix lock mismatch poetry source add modifies pyproject.toml, which makes poetry refuse to install from the existing lock file. Running poetry lock --no-update regenerates the lock file metadata without changing dependency versions. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-coverage.yml | 1 + .github/workflows/code-quality-checks.yml | 4 ++++ .github/workflows/daily-telemetry-e2e.yml | 1 + .github/workflows/integration.yml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 98793d272..e0c65ffb9 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -57,6 +57,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 16842e953..75817e2dc 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -54,6 +54,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update #---------------------------------------------- # load cached venv if cache exists @@ -149,6 +150,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update #---------------------------------------------- # load cached venv if cache exists @@ -238,6 +240,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update #---------------------------------------------- # load cached venv if cache exists @@ -302,6 +305,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index 9bc2b20ef..9495f6cb8 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -61,6 +61,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 69ccf4fa6..157707f96 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -52,6 +52,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update #---------------------------------------------- # load cached venv if cache exists @@ -122,6 +123,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 From 2067a8a259d0322a309e5fd8d071a076532f2690 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:11:14 +0000 Subject: [PATCH 6/8] Fix poetry lock flag and YAML indentation Poetry 2.x doesn't have --no-update flag, use poetry lock instead. Also fix indentation of poetry lock in the arrow test job. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-coverage.yml | 2 +- .github/workflows/code-quality-checks.yml | 8 ++++---- .github/workflows/daily-telemetry-e2e.yml | 2 +- .github/workflows/integration.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index e0c65ffb9..1c9d53f16 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -57,7 +57,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 75817e2dc..2775559c1 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -54,7 +54,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock #---------------------------------------------- # load cached venv if cache exists @@ -150,7 +150,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock #---------------------------------------------- # load cached venv if cache exists @@ -240,7 +240,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock #---------------------------------------------- # load cached venv if cache exists @@ -305,7 +305,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index 9495f6cb8..86fa82d49 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -61,7 +61,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 157707f96..a3a31bae6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -52,7 +52,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock #---------------------------------------------- # load cached venv if cache exists @@ -123,7 +123,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry lock - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 From 002b492ce88a6a50ac76f814f1214b46cfaaf14d Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:24:20 +0000 Subject: [PATCH 7/8] Move JFrog setup before setup-python, matching sqlalchemy pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow the proven pattern from databricks/databricks-sqlalchemy#59: checkout → Setup JFrog → setup-python → pip install poetry → poetry source add + poetry lock → poetry install. The hardened runners block pypi.org at the network level, so JFrog must be configured before actions/setup-python (which upgrades pip). Also simplified workflows by removing verbose section comments. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/setup-jfrog/action.yml | 3 +- .github/workflows/code-coverage.yml | 55 +----- .github/workflows/code-quality-checks.yml | 221 ++++++---------------- .github/workflows/daily-telemetry-e2e.yml | 38 +--- .github/workflows/integration.yml | 35 +--- 5 files changed, 77 insertions(+), 275 deletions(-) diff --git a/.github/actions/setup-jfrog/action.yml b/.github/actions/setup-jfrog/action.yml index d42931ea1..97ae146ba 100644 --- a/.github/actions/setup-jfrog/action.yml +++ b/.github/actions/setup-jfrog/action.yml @@ -1,5 +1,5 @@ name: Setup JFrog OIDC -description: Obtain a JFrog access token via GitHub OIDC and configure pip and poetry to use JFrog PyPI proxy +description: Obtain a JFrog access token via GitHub OIDC and configure pip to use JFrog PyPI proxy runs: using: composite @@ -30,4 +30,3 @@ runs: set -euo pipefail echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" echo "pip configured to use JFrog registry" - diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 1c9d53f16..9ed03596f 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -19,33 +19,21 @@ jobs: DATABRICKS_CATALOG: peco DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install system dependencies ----- - #---------------------------------------------- - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 @@ -58,34 +46,21 @@ jobs: poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry lock - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - name: Install Kerberos system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - name: Install library run: poetry install --no-interaction --all-extras - #---------------------------------------------- - # run parallel tests with coverage - #---------------------------------------------- - name: Run parallel tests with coverage continue-on-error: false run: | @@ -96,24 +71,15 @@ jobs: --cov-report=xml \ --cov-report=term \ -v - - #---------------------------------------------- - # run telemetry tests with coverage (isolated) - #---------------------------------------------- - name: Run telemetry tests with coverage (isolated) continue-on-error: false run: | - # Run test_concurrent_telemetry.py separately for isolation poetry run pytest tests/e2e/test_concurrent_telemetry.py \ --cov=src \ --cov-append \ --cov-report=xml \ --cov-report=term \ -v - - #---------------------------------------------- - # check for coverage override - #---------------------------------------------- - name: Check for coverage override id: override env: @@ -129,9 +95,6 @@ jobs: echo "override=false" >> $GITHUB_OUTPUT echo "No coverage override found" fi - #---------------------------------------------- - # check coverage percentage - #---------------------------------------------- - name: Check coverage percentage if: steps.override.outputs.override == 'false' run: | @@ -140,20 +103,14 @@ jobs: echo "ERROR: Coverage file not found at $COVERAGE_FILE" exit 1 fi - - # Install xmllint if not available if ! command -v xmllint &> /dev/null; then sudo apt-get update && sudo apt-get install -y libxml2-utils fi - COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE") TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE") PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))") - echo "Branch Coverage: $PERCENTAGE%" echo "Required Coverage: 85%" - - # Use Python to compare the coverage with 85 python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)" if [ $? -eq 1 ]; then echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%" @@ -161,18 +118,14 @@ jobs: else echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%" fi - - #---------------------------------------------- - # coverage enforcement summary - #---------------------------------------------- - name: Coverage enforcement summary env: OVERRIDE: ${{ steps.override.outputs.override }} REASON: ${{ steps.override.outputs.reason }} run: | if [ "$OVERRIDE" == "true" ]; then - echo "⚠️ Coverage checks bypassed: $REASON" + echo "Coverage checks bypassed: $REASON" echo "Please ensure this override is justified and temporary" else - echo "✅ Coverage checks enforced - minimum 85% required" + echo "Coverage checks enforced - minimum 85% required" fi diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 2775559c1..6d9d3add5 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -25,24 +25,15 @@ jobs: name: "Unit Tests (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)" steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 @@ -55,55 +46,36 @@ jobs: poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry lock - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - name: Install library run: poetry install --no-interaction - #---------------------------------------------- - # override with custom dependency versions - #---------------------------------------------- - name: Install Python tools for custom versions if: matrix.dependency-version != 'default' run: poetry run pip install toml packaging - - name: Generate requirements file if: matrix.dependency-version != 'default' run: | poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}.txt echo "Generated requirements for ${{ matrix.dependency-version }} versions:" cat requirements-${{ matrix.dependency-version }}.txt - - name: Override with custom dependency versions if: matrix.dependency-version != 'default' run: poetry run pip install -r requirements-${{ matrix.dependency-version }}.txt - - #---------------------------------------------- - # run test suite - #---------------------------------------------- - name: Show installed versions run: | echo "=== Dependency Version: ${{ matrix.dependency-version }} ===" poetry run pip list - - name: Run tests run: poetry run python -m pytest tests/unit + run-unit-tests-with-arrow: runs-on: group: databricks-protected-runner-group @@ -121,88 +93,61 @@ jobs: name: "Unit Tests + PyArrow (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)" steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Check out repository - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 - with: - python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv-pyarrow - key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install Kerberos system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - - name: Install library - run: poetry install --no-interaction --all-extras - #---------------------------------------------- - # override with custom dependency versions - #---------------------------------------------- - - name: Install Python tools for custom versions - if: matrix.dependency-version != 'default' - run: poetry run pip install toml packaging - - - name: Generate requirements file with pyarrow - if: matrix.dependency-version != 'default' - run: | - poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}-arrow.txt - echo "Generated requirements for ${{ matrix.dependency-version }} versions with PyArrow:" - cat requirements-${{ matrix.dependency-version }}-arrow.txt - - - name: Override with custom dependency versions - if: matrix.dependency-version != 'default' - run: poetry run pip install -r requirements-${{ matrix.dependency-version }}-arrow.txt - #---------------------------------------------- - # run test suite - #---------------------------------------------- - - name: Show installed versions - run: | - echo "=== Dependency Version: ${{ matrix.dependency-version }} with PyArrow ===" - poetry run pip list + - name: Check out repository + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + - name: Set up python ${{ matrix.python-version }} + id: setup-python + uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true + - name: Configure Poetry for JFrog + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .venv-pyarrow + key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + - name: Install Kerberos system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libkrb5-dev + - name: Install library + run: poetry install --no-interaction --all-extras + - name: Install Python tools for custom versions + if: matrix.dependency-version != 'default' + run: poetry run pip install toml packaging + - name: Generate requirements file with pyarrow + if: matrix.dependency-version != 'default' + run: | + poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}-arrow.txt + echo "Generated requirements for ${{ matrix.dependency-version }} versions with PyArrow:" + cat requirements-${{ matrix.dependency-version }}-arrow.txt + - name: Override with custom dependency versions + if: matrix.dependency-version != 'default' + run: poetry run pip install -r requirements-${{ matrix.dependency-version }}-arrow.txt + - name: Show installed versions + run: | + echo "=== Dependency Version: ${{ matrix.dependency-version }} with PyArrow ===" + poetry run pip list + - name: Run tests + run: poetry run python -m pytest tests/unit - - name: Run tests - run: poetry run python -m pytest tests/unit check-linting: runs-on: group: databricks-protected-runner-group @@ -211,24 +156,15 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 @@ -241,30 +177,17 @@ jobs: poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry lock - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - name: Install library run: poetry install --no-interaction - #---------------------------------------------- - # black the code - #---------------------------------------------- - name: Black run: poetry run black --check src @@ -276,24 +199,15 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 @@ -306,30 +220,17 @@ jobs: poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry lock - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - name: Install library run: poetry install --no-interaction - #---------------------------------------------- - # mypy the code - #---------------------------------------------- - name: Mypy run: | mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index 86fa82d49..ff962ebb5 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -31,25 +31,19 @@ jobs: DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- + - name: Install Kerberos system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libkrb5-dev - name: Install Poetry run: | pip install poetry==2.2.1 @@ -62,39 +56,19 @@ jobs: poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry lock - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install Kerberos system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - name: Install dependencies run: poetry install --no-interaction --all-extras - - #---------------------------------------------- - # run telemetry E2E tests - #---------------------------------------------- - name: Run telemetry E2E tests run: | TEST_PATTERN="${{ github.event.inputs.test_pattern || 'tests/e2e/test_telemetry_e2e.py' }}" echo "Running tests: $TEST_PATTERN" poetry run python -m pytest $TEST_PATTERN -v -s - - #---------------------------------------------- - # upload test results on failure - #---------------------------------------------- - name: Upload test results on failure if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a3a31bae6..5c0364d6b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,24 +23,15 @@ jobs: DATABRICKS_CATALOG: peco DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 @@ -53,31 +44,20 @@ jobs: poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry lock - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install Kerberos system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - name: Install dependencies run: poetry install --no-interaction --all-extras - #---------------------------------------------- - # run test suite - #---------------------------------------------- - name: Run non-telemetry e2e tests run: | - # Exclude all telemetry tests - they run in separate job for isolation poetry run python -m pytest tests/e2e \ --ignore=tests/e2e/test_telemetry_e2e.py \ --ignore=tests/e2e/test_concurrent_telemetry.py \ @@ -87,7 +67,7 @@ jobs: runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest - needs: run-non-telemetry-tests # Run after non-telemetry tests complete + needs: run-non-telemetry-tests environment: azure-prod env: DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} @@ -98,16 +78,13 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - #---------------------------------------------- - # ----- configure JFrog PyPI proxy ----- - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Install system dependencies run: | sudo apt-get update @@ -134,7 +111,5 @@ jobs: run: poetry install --no-interaction --all-extras - name: Run telemetry tests in isolation run: | - # Run test_concurrent_telemetry.py in isolation with complete process separation - # Use --dist=loadgroup to respect @pytest.mark.xdist_group markers poetry run python -m pytest tests/e2e/test_concurrent_telemetry.py \ -n auto --dist=loadgroup -v From 23489a616766385ea7687fa03d28c46d8e42d3c6 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:32:59 +0000 Subject: [PATCH 8/8] Extract setup-poetry composite action to remove duplication Create .github/actions/setup-poetry that bundles JFrog setup, setup-python, poetry install via pip, JFrog source config, cache, and dependency install into a single reusable action with inputs for python-version, install-args, cache-path, and cache-suffix. All workflows now call setup-poetry instead of repeating these steps, matching the pattern from databricks/databricks-sqlalchemy#59. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/setup-poetry/action.yml | 63 ++++++++++ .github/workflows/code-coverage.yml | 37 +----- .github/workflows/code-quality-checks.yml | 133 +++------------------- .github/workflows/daily-telemetry-e2e.yml | 30 +---- .github/workflows/integration.yml | 62 ++-------- 5 files changed, 96 insertions(+), 229 deletions(-) create mode 100644 .github/actions/setup-poetry/action.yml diff --git a/.github/actions/setup-poetry/action.yml b/.github/actions/setup-poetry/action.yml new file mode 100644 index 000000000..f7e15b1c0 --- /dev/null +++ b/.github/actions/setup-poetry/action.yml @@ -0,0 +1,63 @@ +name: Setup Poetry with JFrog +description: Install Poetry, configure JFrog as primary PyPI source, and install project dependencies + +inputs: + python-version: + description: Python version to set up + required: true + install-args: + description: Extra arguments for poetry install (e.g. --all-extras) + required: false + default: "" + cache-path: + description: Path to the virtualenv for caching (e.g. .venv or .venv-pyarrow) + required: false + default: ".venv" + cache-suffix: + description: Extra suffix for the cache key to avoid collisions across job variants + required: false + default: "" + +runs: + using: composite + steps: + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Set up python ${{ inputs.python-version }} + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Install Poetry + shell: bash + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true + + - name: Configure Poetry JFrog source + shell: bash + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ inputs.cache-path }} + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ inputs.cache-suffix }}${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + shell: bash + run: poetry install --no-interaction --no-root + + - name: Install library + shell: bash + run: poetry install --no-interaction ${{ inputs.install-args }} diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 9ed03596f..c188c5b3a 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -23,44 +23,15 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.10" - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - name: Install Kerberos system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - - name: Install library - run: poetry install --no-interaction --all-extras + python-version: "3.10" + install-args: "--all-extras" - name: Run parallel tests with coverage continue-on-error: false run: | diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 6d9d3add5..ecc238263 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -15,7 +15,6 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] dependency-version: ["default", "min"] - # Optimize matrix - test min/max on subset of Python versions exclude: - python-version: "3.12" dependency-version: "min" @@ -27,36 +26,11 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - name: Install library - run: poetry install --no-interaction + cache-suffix: "${{ matrix.dependency-version }}-" - name: Install Python tools for custom versions if: matrix.dependency-version != 'default' run: poetry run pip install toml packaging @@ -94,41 +68,18 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv-pyarrow - key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Install Kerberos system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - - name: Install library - run: poetry install --no-interaction --all-extras + - name: Setup Poetry + uses: ./.github/actions/setup-poetry + with: + python-version: ${{ matrix.python-version }} + install-args: "--all-extras" + cache-path: ".venv-pyarrow" + cache-suffix: "pyarrow-${{ matrix.dependency-version }}-" - name: Install Python tools for custom versions if: matrix.dependency-version != 'default' run: poetry run pip install toml packaging @@ -158,36 +109,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - name: Install library - run: poetry install --no-interaction - name: Black run: poetry run black --check src @@ -201,37 +126,11 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - name: Install library - run: poetry install --no-interaction - name: Mypy run: | - mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 + mkdir .mypy_cache poetry run mypy --install-types --non-interactive src diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index ff962ebb5..b6f78726c 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -33,37 +33,15 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.10" - name: Install Kerberos system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - run: poetry install --no-interaction --all-extras + python-version: "3.10" + install-args: "--all-extras" - name: Run telemetry E2E tests run: | TEST_PATTERN="${{ github.event.inputs.test_pattern || 'tests/e2e/test_telemetry_e2e.py' }}" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 5c0364d6b..6c0cc7059 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -25,37 +25,15 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.10" - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - name: Install Kerberos system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - - name: Install dependencies - run: poetry install --no-interaction --all-extras + - name: Setup Poetry + uses: ./.github/actions/setup-poetry + with: + python-version: "3.10" + install-args: "--all-extras" - name: Run non-telemetry e2e tests run: | poetry run python -m pytest tests/e2e \ @@ -78,37 +56,15 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.10" - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y libkrb5-dev - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry for JFrog - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - run: poetry install --no-interaction --all-extras + python-version: "3.10" + install-args: "--all-extras" - name: Run telemetry tests in isolation run: | poetry run python -m pytest tests/e2e/test_concurrent_telemetry.py \