Skip to content

Commit c3e7920

Browse files
msrathore-dbmsrathore-db
committed
Merge remote-tracking branch 'origin/main' into spog-fix
2 parents efcea13 + 9031863 commit c3e7920

17 files changed

Lines changed: 473 additions & 677 deletions

File tree

‎.github/actions/setup-jfrog/action.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Setup JFrog OIDC
2+
description: Obtain a JFrog access token via GitHub OIDC and configure pip to use JFrog PyPI proxy
3+
4+
runs:
5+
using: composite
6+
steps:
7+
- name: Get JFrog OIDC token
8+
shell: bash
9+
run: |
10+
set -euo pipefail
11+
ID_TOKEN=$(curl -sLS \
12+
-H "User-Agent: actions/oidc-client" \
13+
-H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
14+
"${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"')
15+
echo "::add-mask::${ID_TOKEN}"
16+
ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \
17+
"https://databricks.jfrog.io/access/api/v1/oidc/token" \
18+
-d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"')
19+
echo "::add-mask::${ACCESS_TOKEN}"
20+
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
21+
echo "FAIL: Could not extract JFrog access token"
22+
exit 1
23+
fi
24+
echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV"
25+
echo "JFrog OIDC token obtained successfully"
26+
27+
- name: Configure pip
28+
shell: bash
29+
run: |
30+
set -euo pipefail
31+
echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV"
32+
echo "pip configured to use JFrog registry"

‎.github/actions/setup-poetry/action.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Setup Poetry with JFrog
2+
description: Install Poetry, configure JFrog as primary PyPI source, and install project dependencies
3+
4+
inputs:
5+
python-version:
6+
description: Python version to set up
7+
required: true
8+
install-args:
9+
description: Extra arguments for poetry install (e.g. --all-extras)
10+
required: false
11+
default: ""
12+
cache-path:
13+
description: Path to the virtualenv for caching (e.g. .venv or .venv-pyarrow)
14+
required: false
15+
default: ".venv"
16+
cache-suffix:
17+
description: Extra suffix for the cache key to avoid collisions across job variants
18+
required: false
19+
default: ""
20+
21+
runs:
22+
using: composite
23+
steps:
24+
- name: Setup JFrog
25+
uses: ./.github/actions/setup-jfrog
26+
27+
- name: Set up python ${{ inputs.python-version }}
28+
id: setup-python
29+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
30+
with:
31+
python-version: ${{ inputs.python-version }}
32+
33+
- name: Install Poetry
34+
shell: bash
35+
run: |
36+
pip install poetry==2.2.1
37+
poetry config virtualenvs.create true
38+
poetry config virtualenvs.in-project true
39+
poetry config installer.parallel true
40+
41+
- name: Configure Poetry JFrog source
42+
shell: bash
43+
run: |
44+
poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
45+
poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}"
46+
poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
47+
poetry lock
48+
49+
- name: Load cached venv
50+
id: cached-poetry-dependencies
51+
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
52+
with:
53+
path: ${{ inputs.cache-path }}
54+
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ inputs.cache-suffix }}${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
55+
56+
- name: Install dependencies
57+
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
58+
shell: bash
59+
run: poetry install --no-interaction --no-root
60+
61+
- name: Install library
62+
shell: bash
63+
run: poetry install --no-interaction ${{ inputs.install-args }}

‎.github/workflows/code-coverage.yml‎

Lines changed: 20 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
1-
name: Code Coverage
1+
name: E2E Tests and Code Coverage
22

33
permissions:
44
contents: read
5+
id-token: write
56

6-
on: [pull_request, workflow_dispatch]
7+
on:
8+
push:
9+
branches:
10+
- main
11+
pull_request:
12+
workflow_dispatch:
713

814
jobs:
915
test-with-coverage:
10-
runs-on: ubuntu-latest
16+
runs-on:
17+
group: databricks-protected-runner-group
18+
labels: linux-ubuntu-latest
1119
environment: azure-prod
1220
env:
1321
DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
@@ -16,91 +24,29 @@ jobs:
1624
DATABRICKS_CATALOG: peco
1725
DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
1826
steps:
19-
#----------------------------------------------
20-
# check-out repo and set-up python
21-
#----------------------------------------------
2227
- name: Check out repository
2328
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
2429
with:
2530
fetch-depth: 0
26-
- name: Set up python
27-
id: setup-python
28-
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
29-
with:
30-
python-version: "3.10"
31-
#----------------------------------------------
32-
# ----- install system dependencies -----
33-
#----------------------------------------------
3431
- name: Install system dependencies
3532
run: |
3633
sudo apt-get update
3734
sudo apt-get install -y libkrb5-dev
38-
#----------------------------------------------
39-
# ----- install & configure poetry -----
40-
#----------------------------------------------
41-
- name: Install Poetry
42-
uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
43-
with:
44-
version: "2.2.1"
45-
virtualenvs-create: true
46-
virtualenvs-in-project: true
47-
installer-parallel: true
48-
49-
#----------------------------------------------
50-
# load cached venv if cache exists
51-
#----------------------------------------------
52-
- name: Load cached venv
53-
id: cached-poetry-dependencies
54-
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
35+
- name: Setup Poetry
36+
uses: ./.github/actions/setup-poetry
5537
with:
56-
path: .venv
57-
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
58-
#----------------------------------------------
59-
# install dependencies if cache does not exist
60-
#----------------------------------------------
61-
- name: Install dependencies
62-
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
63-
run: poetry install --no-interaction --no-root
64-
#----------------------------------------------
65-
# install your root project, if required
66-
#----------------------------------------------
67-
- name: Install Kerberos system dependencies
68-
run: |
69-
sudo apt-get update
70-
sudo apt-get install -y libkrb5-dev
71-
- name: Install library
72-
run: poetry install --no-interaction --all-extras
73-
#----------------------------------------------
74-
# run parallel tests with coverage
75-
#----------------------------------------------
76-
- name: Run parallel tests with coverage
38+
python-version: "3.10"
39+
install-args: "--all-extras"
40+
- name: Run all tests with coverage
7741
continue-on-error: false
7842
run: |
7943
poetry run pytest tests/unit tests/e2e \
80-
-m "not serial" \
81-
-n auto \
44+
-n 4 \
45+
--dist=loadgroup \
8246
--cov=src \
8347
--cov-report=xml \
8448
--cov-report=term \
8549
-v
86-
87-
#----------------------------------------------
88-
# run telemetry tests with coverage (isolated)
89-
#----------------------------------------------
90-
- name: Run telemetry tests with coverage (isolated)
91-
continue-on-error: false
92-
run: |
93-
# Run test_concurrent_telemetry.py separately for isolation
94-
poetry run pytest tests/e2e/test_concurrent_telemetry.py \
95-
--cov=src \
96-
--cov-append \
97-
--cov-report=xml \
98-
--cov-report=term \
99-
-v
100-
101-
#----------------------------------------------
102-
# check for coverage override
103-
#----------------------------------------------
10450
- name: Check for coverage override
10551
id: override
10652
env:
@@ -116,9 +62,6 @@ jobs:
11662
echo "override=false" >> $GITHUB_OUTPUT
11763
echo "No coverage override found"
11864
fi
119-
#----------------------------------------------
120-
# check coverage percentage
121-
#----------------------------------------------
12265
- name: Check coverage percentage
12366
if: steps.override.outputs.override == 'false'
12467
run: |
@@ -127,40 +70,29 @@ jobs:
12770
echo "ERROR: Coverage file not found at $COVERAGE_FILE"
12871
exit 1
12972
fi
130-
131-
# Install xmllint if not available
13273
if ! command -v xmllint &> /dev/null; then
13374
sudo apt-get update && sudo apt-get install -y libxml2-utils
13475
fi
135-
13676
COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
13777
TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")
13878
PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")
139-
14079
echo "Branch Coverage: $PERCENTAGE%"
14180
echo "Required Coverage: 85%"
142-
143-
# Use Python to compare the coverage with 85
14481
python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)"
14582
if [ $? -eq 1 ]; then
14683
echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%"
14784
exit 1
14885
else
14986
echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%"
15087
fi
151-
152-
#----------------------------------------------
153-
# coverage enforcement summary
154-
#----------------------------------------------
15588
- name: Coverage enforcement summary
15689
env:
15790
OVERRIDE: ${{ steps.override.outputs.override }}
15891
REASON: ${{ steps.override.outputs.reason }}
15992
run: |
16093
if [ "$OVERRIDE" == "true" ]; then
161-
echo "⚠️ Coverage checks bypassed: $REASON"
94+
echo "Coverage checks bypassed: $REASON"
16295
echo "Please ensure this override is justified and temporary"
16396
else
164-
echo "✅ Coverage checks enforced - minimum 85% required"
97+
echo "Coverage checks enforced - minimum 85% required"
16598
fi
166-

0 commit comments

Comments
 (0)