Feature/ted9 47 enhance the e forms eligibility checking component

.github/workflows/deploy-infrastructure.yml

@@ -0,0 +1,61 @@
+name: Deploy Infrastructure
+
+on:
+  workflow_call:
+    secrets:
+      VAULT_TOKEN:
+        required: true
+      VAULT_ADDR:
+        required: true
+
+env:
+  VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
+  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+  TRAEFIK_DATA_PATH: /opt/tedsws/traefik
+  STACK_PATH: src/infra/ted-sws-stack
+
+jobs:
+  deploy:
+    name: Deploy Infrastructure
+    runs-on: tedsws-staging
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup environment
+        run: |
+          # Create .env.staging from Vault (passwords only, everything else hardcoded)
+          make staging-unified-dotenv
+
+          # Runtime-specific values (not in Vault)
+          echo "AIRFLOW_UID=$(id -u)" >> ${STACK_PATH}/.env.staging
+          echo "AIRFLOW_IMAGE=tedsws/airflow:staging" >> ${STACK_PATH}/.env.staging
+
+          # Setup deployment directory (source/config only — data uses named volumes)
+          sudo mkdir -p /opt/tedsws ${TRAEFIK_DATA_PATH}/letsencrypt
+          sudo chown -R $USER:$USER /opt/tedsws
+          mkdir -p /opt/tedsws/src /opt/tedsws/logs /opt/tedsws/plugins /opt/tedsws/test
+
+          # Copy source files
+          rsync -a --delete src/ /opt/tedsws/src/
+
+      - name: Download libraries
+        run: |
+          make init-libraries
+          rsync -a --delete libraries/ /opt/tedsws/libraries/
+
+      - name: Build Airflow image
+        run: |
+          cp requirements.txt ${STACK_PATH}/airflow/
+          docker build -t tedsws/airflow:staging ${STACK_PATH}/airflow/
+
+      - name: Deploy stack
+        run: |
+          docker compose \
+            -f ${STACK_PATH}/docker-compose.yml \
+            -f ${STACK_PATH}/docker-compose.staging.yml \
+            --env-file ${STACK_PATH}/.env.common \
+            --env-file ${STACK_PATH}/.env.staging \
+            up -d --force-recreate

.github/workflows/deploy-pipeline.yml

@@ -0,0 +1,43 @@
+name: Deploy Pipeline Code
+
+on:
+  workflow_call:
+    inputs:
+      restart_worker:
+        description: 'Restart Airflow worker (needed if ted_sws code changed)'
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  deploy:
+    name: Deploy DAGs and Code
+    runs-on: tedsws-staging
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Sync code to deployment directory
+        run: |
+          echo "Syncing DAGs and ted_sws to /opt/tedsws..."
+          rsync -av --delete src/dags/ /opt/tedsws/src/dags/
+          rsync -av --delete src/ted_sws/ /opt/tedsws/src/ted_sws/
+          echo "Sync complete."
+
+      - name: Restart Airflow worker
+        if: ${{ inputs.restart_worker }}
+        run: |
+          echo "Restarting Airflow worker to pick up ted_sws changes..."
+          docker restart airflow-worker
+          echo "Worker restarted."
+
+      - name: Verify deployment
+        run: |
+          echo "=== Deployed files ==="
+          echo "DAGs:"
+          ls -la /opt/tedsws/src/dags/ | head -10
+          echo ""
+          echo "ted_sws modules:"
+          ls -la /opt/tedsws/src/ted_sws/ | head -10

.github/workflows/deploy.yml

@@ -0,0 +1,119 @@
+name: Deploy TED-SWS Stack
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'src/infra/**'
+      - 'src/dags/**'
+      - 'src/ted_sws/**'
+      - 'requirements.txt'
+      - 'Makefile'
+      - '.github/workflows/deploy*.yml'
+
+concurrency:
+  group: deploy-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    name: Detect Changes
+    runs-on: tedsws-staging
+    timeout-minutes: 5
+    outputs:
+      stack: ${{ steps.filter.outputs.stack }}
+      pipeline: ${{ steps.filter.outputs.pipeline }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          base: ${{ github.ref }}
+          filters: |
+            stack:
+              - 'src/infra/**'
+              - 'requirements.txt'
+              - 'Makefile'
+              - '.github/workflows/deploy-infrastructure.yml'
+            pipeline:
+              - 'src/dags/**'
+              - 'src/ted_sws/**'
+              - '.github/workflows/deploy-pipeline.yml'
+
+  stack:
+    name: Deploy TED-SWS Stack
+    needs: changes
+    if: ${{ needs.changes.outputs.stack == 'true' }}
+    uses: ./.github/workflows/deploy-infrastructure.yml
+    secrets:
+      VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
+      VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+
+  tests:
+    name: Run Tests
+    needs: [changes, stack]
+    if: ${{ always() && needs.changes.outputs.pipeline == 'true' && (needs.stack.result == 'success' || needs.stack.result == 'skipped') }}
+    uses: ./.github/workflows/tests.yml
+    with:
+      full_tests: true
+    secrets:
+      VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
+      VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+  pipeline:
+    name: Deploy TED-SWS Pipeline
+    needs: [changes, stack, tests]
+    if: ${{ always() && needs.changes.outputs.pipeline == 'true' && needs.tests.result == 'success' }}
+    uses: ./.github/workflows/deploy-pipeline.yml
+    with:
+      restart_worker: true
+
+  health-check:
+    name: Health Check
+    needs: [stack, pipeline]
+    if: always()
+    runs-on: tedsws-staging
+    timeout-minutes: 5
+
+    steps:
+      - name: Check all services
+        run: |
+          echo "Waiting for services to stabilize..."
+          sleep 10
+
+          failed=0
+
+          echo "=== Checking Traefik ==="
+          docker ps | grep -q "traefik" && echo "✓ Traefik" || { echo "✗ Traefik"; failed=1; }
+
+          echo "=== Checking Data Stores ==="
+          docker ps | grep -q "mongodb" && echo "✓ MongoDB" || { echo "✗ MongoDB"; failed=1; }
+          docker ps | grep -q "fuseki" && echo "✓ Fuseki" || { echo "✗ Fuseki"; failed=1; }
+          docker ps | grep -q "minio" && echo "✓ MinIO" || { echo "✗ MinIO"; failed=1; }
+          docker ps | grep -q "sftp" && echo "✓ SFTP" || { echo "✗ SFTP"; failed=1; }
+
+          echo "=== Checking Airflow ==="
+          docker ps | grep -q "airflow-webserver" && echo "✓ Airflow webserver" || { echo "✗ Airflow webserver"; failed=1; }
+          docker ps | grep -q "airflow-scheduler" && echo "✓ Airflow scheduler" || { echo "✗ Airflow scheduler"; failed=1; }
+          docker ps | grep -q "airflow-worker" && echo "✓ Airflow worker" || { echo "✗ Airflow worker"; failed=1; }
+
+          echo "=== Checking APIs ==="
+          docker ps | grep -q "metabase" && echo "✓ Metabase" || { echo "✗ Metabase"; failed=1; }
+
+          echo ""
+          echo "=== All containers ==="
+          docker ps --format "table {{.Names}}\t{{.Status}}"
+
+          exit $failed
+
+      - name: Cleanup old Docker resources
+        if: always()
+        run: |
+          echo "=== Cleaning up old Docker resources ==="
+          docker system prune -f --filter "until=24h"
+          echo ""
+          echo "=== Disk usage after cleanup ==="
+          df -h /
+          docker system df

.github/workflows/tests.yml

@@ -0,0 +1,153 @@
+name: Tests
+
+on:
+  push:
+    branches: [feature/**, hotfix/**, release/**]
+  pull_request:
+  workflow_call:
+    inputs:
+      full_tests:
+        description: 'Run full test suite (including integration tests)'
+        type: boolean
+        default: true
+    secrets:
+      VAULT_TOKEN:
+        required: true
+      VAULT_ADDR:
+        required: true
+      SONAR_TOKEN:
+        required: false
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
+  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+
+jobs:
+  test:
+    name: ${{ (github.event_name == 'pull_request' || inputs.full_tests) && 'Full Tests' || 'Unit Tests' }}
+    runs-on: ${{ (github.event_name == 'pull_request' || inputs.full_tests) && 'tedsws-staging' || 'ubuntu-latest' }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Set up Java
+        if: github.event_name == 'pull_request' || inputs.full_tests
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+
+      - name: Install Vault
+        if: github.event_name == 'pull_request' || inputs.full_tests
+        run: |
+          if ! command -v vault &> /dev/null; then
+            wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+            echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+            sudo apt-get update && sudo apt-get install -y vault
+          else
+            echo "Vault already installed: $(vault --version)"
+          fi
+
+      - name: Install system dependencies
+        run: |
+          if ! dpkg -s libssl-dev &> /dev/null; then
+            sudo apt-get update && sudo apt-get install -y libssl-dev libcurl4-openssl-dev
+          else
+            echo "System dependencies already installed"
+          fi
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade setuptools pip wheel
+          make install
+          make install-dev
+
+      - name: Resolve Docker container hostnames
+        if: github.event_name == 'pull_request' || inputs.full_tests
+        run: |
+          # Add /etc/hosts entries for all containers on the unified stack network
+          for entry in $(docker network inspect tedsws-internal -f '{{range .Containers}}{{.Name}}:{{.IPv4Address}} {{end}}'); do
+            name="${entry%%:*}"
+            ip="${entry#*:}"
+            ip="${ip%/*}"
+            echo "$ip $name" | sudo tee -a /etc/hosts
+          done
+          echo "=== /etc/hosts entries ==="
+          grep -vE "^#|^$|localhost" /etc/hosts | tail -20
+          echo "=== Connectivity check (Docker services) ==="
+          FAIL=0
+          curl -sf --max-time 5 http://fuseki:3030 > /dev/null && echo "OK: fuseki:3030" || { echo "FAIL: fuseki:3030"; FAIL=1; }
+          nc -z -w5 minio 9000 && echo "OK: minio:9000" || { echo "FAIL: minio:9000"; FAIL=1; }
+          nc -z -w5 mongodb 27017 && echo "OK: mongodb:27017" || { echo "FAIL: mongodb:27017"; FAIL=1; }
+          nc -z -w5 sftp 22 && echo "OK: sftp:22" || { echo "FAIL: sftp:22"; FAIL=1; }
+          if [ "$FAIL" -eq 1 ]; then
+            echo "::error::Required Docker services are not reachable"
+            exit 1
+          fi
+
+      - name: Create env file
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ inputs.full_tests }}" == "true" ]; then
+            # Merge app config (.env.common) + Vault secrets (.env.staging)
+            make staging-unified-dotenv
+            cat src/infra/ted-sws-stack/.env.common src/infra/ted-sws-stack/.env.staging > .env
+            # Use host-local libraries (downloaded by make init-libraries)
+            echo "RML_MAPPER_PATH=$(pwd)/libraries/.rmlmapper/rmlmapper.jar" >> .env
+            echo "XML_PROCESSOR_PATH=$(pwd)/libraries/.saxon/saxon-he-10.9.jar" >> .env
+            echo "LIMES_ALIGNMENT_PATH=$(pwd)/libraries/.limes/limes.jar" >> .env
+          else
+            echo "VAULT_TOKEN=${{ secrets.VAULT_TOKEN }}" >> .env
+            echo "VAULT_ADDR=${{ secrets.VAULT_ADDR }}" >> .env
+          fi
+
+      - name: Preflight check (external endpoints)
+        if: github.event_name == 'pull_request' || inputs.full_tests
+        run: |
+          set -a && source .env && set +a
+          FAIL=0
+          curl -so /dev/null --max-time 5 "$TED_API_URL" && echo "OK: TED API ($TED_API_URL)" || { echo "FAIL: TED API ($TED_API_URL)"; FAIL=1; }
+          curl -so /dev/null --max-time 5 "$ALLEGRO_HOST" && echo "OK: AllegroGraph ($ALLEGRO_HOST)" || { echo "FAIL: AllegroGraph ($ALLEGRO_HOST)"; FAIL=1; }
+          curl -so /dev/null --max-time 5 "http://fuseki:3030/test_limes" && echo "OK: Fuseki test_limes" || { echo "FAIL: Fuseki test_limes dataset"; FAIL=1; }
+          if [ "$FAIL" -eq 1 ]; then
+            echo "::error::Required external endpoints are not reachable"
+            exit 1
+          fi
+
+      - name: Get Java tools (Saxon, Limes, RML mapper)
+        if: github.event_name == 'pull_request' || inputs.full_tests
+        run: make init-libraries
+
+      # Using python -m tox instead of make targets to avoid conflict with
+      # outdated system tox on self-hosted runner (/home/lps/.local/bin/tox)
+      - name: Run tests
+        run: ${{ (github.event_name == 'pull_request' || inputs.full_tests) && 'python -m tox' || 'python -m tox -e unit' }}
+
+      - name: SonarCloud Scan
+        uses: SonarSource/sonarqube-scan-action@v6
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+      - name: Cleanup
+        if: always()
+        run: |
+          echo "=== Cleaning up test artifacts ==="
+          rm -rf .tox .pytest_cache __pycache__ .coverage coverage.xml
+          find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+          find . -type f -name "*.pyc" -delete 2>/dev/null || true
+          # Remove Docker container host entries added during this run
+          docker network inspect tedsws-internal -f '{{range .Containers}}{{.Name}}{{"\n"}}{{end}}' 2>/dev/null | \
+            xargs -I{} sudo sed -i '/\b{}\b/d' /etc/hosts || true
+          echo "=== Disk usage ==="
+          df -h /