initial commit

Author: gildesmarais

.dockerignore

@@ -0,0 +1,22 @@
+.git
+.gitignore
+AGENTS.md
+
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+.venv/
+venv/
+env/
+
+.DS_Store
+*.log
+*.tmp
+*.swp
+
+README.md

.github/dependabot.yml

@@ -0,0 +1,25 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 1
+    groups:
+      monthly-python:
+        patterns:
+          - "*"
+    commit-message:
+      prefix: "deps"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 1
+    groups:
+      monthly-actions:
+        patterns:
+          - "*"
+    commit-message:
+      prefix: "deps"

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - "**"
+
+permissions:
+  contents: read
+
+jobs:
+  unit-tests:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Execute unit test suite
+        run: python -m unittest discover -s tests -p "test_*.py" -v
+
+  smoke-test:
+    name: Run docker smoke test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Execute smoke test
+        run: make smoke

.github/workflows/docker-publish.yml

@@ -0,0 +1,53 @@
+name: Docker Publish
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+env:
+  IMAGE_NAME: html2rss/botasaurus-scrape-api
+
+jobs:
+  docker-publish:
+    name: Build and publish Docker image
+    if: >-
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'push' &&
+      github.event.workflow_run.head_branch == 'main'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository at triggering commit
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push multi-arch image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            ${{ env.IMAGE_NAME }}:latest
+            ${{ env.IMAGE_NAME }}:${{ github.event.workflow_run.head_sha }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.event.workflow_run.head_sha }}
+            org.opencontainers.image.title=botasaurus-scrape-api
+            org.opencontainers.image.description=Botasaurus scrape API image

.gitignore

@@ -0,0 +1,3 @@
+.ruff_cache/
+__pycache__/
+*.pyc

AGENTS.md

@@ -0,0 +1,52 @@
+# AGENTS.md
+
+## Core Rules
+
+- Docker-first and Docker-only unless user asks otherwise.
+- Keep repo focused: stable Botasaurus scrape API wrapper, not generic framework.
+
+## Contract (Do Not Break)
+
+- Endpoints: `GET /health`, `POST /scrape`.
+- Stable legacy `/scrape` fields: `url`, `final_url`, `status_code`, `headers`, `html`, `error`, `metadata_error`.
+- Additive diagnostics fields (current contract): `request_id`, `attempts`, `strategy_used`, `render_ms`, `blocked_detected`, `challenge_detected`, `error_category`.
+- Request options (current contract): `navigation_mode`, `max_retries`, `wait_for_selector`, `wait_timeout_seconds`, `block_images`.
+- Error codes:
+  - `400` validation/resolution failure
+  - `403` SSRF guardrail block
+  - `422` request schema validation
+  - `502` scrape execution failure
+  - `504` timeout
+
+## Runtime + Browser Constraints
+
+- `POST /scrape` is async API over sync browser work (threadpool).
+- Each scrape request must use isolated runtime state:
+  - request-scoped runtime dir `/tmp/scrape/<request_id>`
+  - request-scoped browser profile
+  - no cache/profile/driver reuse across requests
+- Cleanup is mandatory in `finally`:
+  - close browser driver
+  - delete request runtime dir
+  - remove in-memory active request id
+- Keep request-id collision/invariant guard (`_active_request_ids`) intact.
+- `driver.requests.get` metadata is best-effort; metadata failure must not fail HTML success.
+- Keep strategy engine behavior:
+  - `auto` mode attempt order: `google_get` -> `google_get_bypass` -> `get`
+  - do not alter retry semantics without docs/tests update
+- Multi-arch image required:
+  - all architectures: Chromium install
+  - keep `/usr/bin/google-chrome` symlink to Chromium for compatibility
+- If browser install logic changes, re-verify binary path and Botasaurus startup.
+
+## Safety
+
+- Keep SSRF guardrails: localhost/domain checks and blocked IP classes (loopback/private/link-local/multicast/reserved/unspecified).
+- Do not weaken URL validation without explicit request plus docs/tests updates.
+
+## Done Criteria
+
+- Run `make smoke` before finish.
+- `make smoke` must cover build, boot, `/health`, `/scrape` happy path, strategy override, retry path, isolation check, localhost guardrail.
+- If API contract, Docker behavior, or error semantics changed, update README in same change.
+- Keep commits scoped (infra vs API vs docs).

Dockerfile

@@ -0,0 +1,47 @@
+# Use official Python runtime pinned by digest
+FROM python:3.12-slim-bookworm@sha256:d97792894a6a4162cae14da44542a83c75e56c77a27b92d58f3f83b7bc961292 AS builder
+
+WORKDIR /build
+
+# Build wheels in an isolated stage (botasaurus dependency is sourced from git).
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /build/requirements.txt
+RUN pip wheel --no-cache-dir --wheel-dir /build/wheels -r /build/requirements.txt
+
+FROM python:3.12-slim-bookworm@sha256:d97792894a6a4162cae14da44542a83c75e56c77a27b92d58f3f83b7bc961292
+
+WORKDIR /app
+
+# Install minimal browser/runtime dependencies for Botasaurus.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        chromium \
+        xvfb \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/requirements.txt
+COPY --from=builder /build/wheels /wheels
+RUN grep -v '^botasaurus @ git+' /app/requirements.txt > /app/requirements.runtime.txt \
+    && echo 'botasaurus' >> /app/requirements.runtime.txt \
+    && pip install --no-cache-dir --no-index --find-links /wheels -r /app/requirements.runtime.txt \
+    && rm -f /app/requirements.runtime.txt \
+    && rm -rf /wheels
+
+# Copy application code
+COPY . /app
+
+# Keep both paths available; Botasaurus integrations often look for google-chrome.
+ENV CHROME_BIN=/usr/bin/chromium
+RUN ln -sf /usr/bin/chromium /usr/bin/google-chrome
+
+# Run as unprivileged user
+RUN useradd --create-home --shell /usr/sbin/nologin appuser \
+    && chown -R appuser:appuser /app
+USER appuser
+
+EXPOSE 4010
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "4010"]

Makefile

@@ -0,0 +1,22 @@
+.PHONY: build serve health scrape-example smoke
+
+IMAGE ?= botasaurus-api
+PORT ?= 4010
+BASE_URL ?= http://localhost:$(PORT)
+
+build:
+	docker build -t $(IMAGE) .
+
+serve: build
+	docker run --rm -p $(PORT):4010 $(IMAGE)
+
+health:
+	curl -s $(BASE_URL)/health
+
+scrape-example:
+	curl -s -X POST $(BASE_URL)/scrape \
+		-H 'Content-Type: application/json' \
+		-d '{"url":"https://example.com"}'
+
+smoke:
+	./scripts/smoke.sh