databricks-sql-python/.github/workflows/integration.yml at 744214a8168bdda8871fc4039532f943bf06388b · databricks/databricks-sql-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
name: Integration Tests

on:
  push:
    branches:
      - main
  pull_request:

permissions:
  contents: read
  id-token: write

jobs:
  run-non-telemetry-tests:
    runs-on:
      group: databricks-protected-runner-group
      labels: linux-ubuntu-latest
    environment: azure-prod
    env:
      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
      DATABRICKS_CATALOG: peco
      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
    steps:
      #----------------------------------------------
      #       check-out repo and set-up python
      #----------------------------------------------
      - name: Check out repository
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.10"
      #----------------------------------------------
      #  -----  configure JFrog PyPI proxy  -----
      #----------------------------------------------
      - name: Setup JFrog
        uses: ./.github/actions/setup-jfrog
      #----------------------------------------------
      #  -----  install & configure poetry  -----
      #----------------------------------------------
      - name: Install Poetry
        run: |
          pip install poetry==2.2.1
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true
          poetry config installer.parallel true
      - name: Configure Poetry for JFrog
        run: |
          poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
          poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}"
          poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
          poetry lock --no-update

      #----------------------------------------------
      #       load cached venv if cache exists
      #----------------------------------------------
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
      #----------------------------------------------
      # install dependencies if cache does not exist
      #----------------------------------------------
      - name: Install Kerberos system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libkrb5-dev
      - name: Install dependencies
        run: poetry install --no-interaction --all-extras
      #----------------------------------------------
      #              run test suite
      #----------------------------------------------
      - name: Run non-telemetry e2e tests
        run: |
          # Exclude all telemetry tests - they run in separate job for isolation
          poetry run python -m pytest tests/e2e \
            --ignore=tests/e2e/test_telemetry_e2e.py \
            --ignore=tests/e2e/test_concurrent_telemetry.py \
            -n auto

  run-telemetry-tests:
    runs-on:
      group: databricks-protected-runner-group
      labels: linux-ubuntu-latest
    needs: run-non-telemetry-tests  # Run after non-telemetry tests complete
    environment: azure-prod
    env:
      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
      DATABRICKS_CATALOG: peco
      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
    steps:
      - name: Check out repository
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.10"
      #----------------------------------------------
      #  -----  configure JFrog PyPI proxy  -----
      #----------------------------------------------
      - name: Setup JFrog
        uses: ./.github/actions/setup-jfrog
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libkrb5-dev
      - name: Install Poetry
        run: |
          pip install poetry==2.2.1
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true
          poetry config installer.parallel true
      - name: Configure Poetry for JFrog
        run: |
          poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
          poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}"
          poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
          poetry lock --no-update
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
      - name: Install dependencies
        run: poetry install --no-interaction --all-extras
      - name: Run telemetry tests in isolation
        run: |
          # Run test_concurrent_telemetry.py in isolation with complete process separation
          # Use --dist=loadgroup to respect @pytest.mark.xdist_group markers
          poetry run python -m pytest tests/e2e/test_concurrent_telemetry.py \
            -n auto --dist=loadgroup -v