feat: set baseline scrape defaults for reliability and integration (#1)

* Set scrape baseline to block images by default

* Align scrape and wait timeout defaults with html2rss pipeline

Author: gildesmarais

README.md

@@ -8,7 +8,7 @@ Docker-only FastAPI service that uses [Botasaurus](https://github.com/omkarcloud
   - `GET /health`
   - `POST /scrape`
 - Intended usage: run and test through Docker only.
-- Runtime boundary: async FastAPI handler delegates sync browser work to a bounded threadpool (`SCRAPE_MAX_WORKERS`), with a per-request timeout (`SCRAPE_TIMEOUT_SECONDS`).
+- Runtime boundary: async FastAPI handler delegates sync browser work to a bounded threadpool (`SCRAPE_MAX_WORKERS`, default `4`), with a per-request timeout (`SCRAPE_TIMEOUT_SECONDS`, default `25`).
 - On-demand isolation-first runtime: every scrape request runs with an ephemeral browser profile and request-scoped runtime dir, then gets fully cleaned up.
 
 ## Prerequisites
@@ -105,7 +105,7 @@ Request body (full options):
   "max_retries": 2,
   "wait_for_selector": "h1",
   "wait_timeout_seconds": 15,
-  "block_images": false,
+  "block_images": true,
   "block_images_and_css": false,
   "wait_for_complete_page_load": true,
   "user_agent": "Mozilla/5.0 ...",
@@ -125,8 +125,8 @@ Request options (contract):
   - `google_get_bypass`: only `google_get(bypass_cloudflare=true)`
 - `max_retries`: `0..3`, default `2` (attempts = `1 + max_retries`, with `auto` capped by 3 strategy steps).
 - `wait_for_selector`: if set, response waits for selector before capture.
-- `wait_timeout_seconds`: selector wait timeout (capped by service timeout).
-- `block_images`: pass image blocking to driver.
+- `wait_timeout_seconds`: selector wait timeout (default `15`, capped by service timeout).
+- `block_images`: pass image blocking to driver. Default `true`.
 
 Currently accepted passthrough options (implemented, not part of stable request-options contract):
 
@@ -255,7 +255,7 @@ curl -s -X POST http://localhost:4010/scrape \
     "url":"https://truthsocial.com/@realDonaldTrump",
     "navigation_mode":"auto",
     "max_retries":2,
-    "wait_timeout_seconds":20,
+    "wait_timeout_seconds":15,
     "headless":false
   }'
 ```

app/main.py

@@ -18,7 +18,8 @@
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field, HttpUrl, field_validator
 
-DEFAULT_SCRAPE_TIMEOUT_SECONDS = int(os.getenv("SCRAPE_TIMEOUT_SECONDS", "60"))
+DEFAULT_SCRAPE_TIMEOUT_SECONDS = int(os.getenv("SCRAPE_TIMEOUT_SECONDS", "25"))
+DEFAULT_WAIT_TIMEOUT_SECONDS = min(15, DEFAULT_SCRAPE_TIMEOUT_SECONDS)
 _MAX_WORKERS = int(os.getenv("SCRAPE_MAX_WORKERS", "4"))
 _RUNTIME_ROOT = Path("/tmp/scrape")
 
@@ -60,11 +61,11 @@ class ScrapeRequest(BaseModel):
     max_retries: int = Field(default=2, ge=0, le=3)
     wait_for_selector: Optional[str] = None
     wait_timeout_seconds: int = Field(
-        default=DEFAULT_SCRAPE_TIMEOUT_SECONDS,
+        default=DEFAULT_WAIT_TIMEOUT_SECONDS,
         ge=1,
         le=DEFAULT_SCRAPE_TIMEOUT_SECONDS,
     )
-    block_images: bool = False
+    block_images: bool = True
     block_images_and_css: bool = False
     wait_for_complete_page_load: bool = True
     user_agent: Optional[str] = None

tests/test_api_contract.py

@@ -59,7 +59,8 @@ def test_request_defaults(self):
         payload = main.ScrapeRequest(url="https://example.com")
         self.assertEqual(payload.navigation_mode, "auto")
         self.assertEqual(payload.max_retries, 2)
-        self.assertFalse(payload.block_images)
+        self.assertEqual(payload.wait_timeout_seconds, 15)
+        self.assertTrue(payload.block_images)
         self.assertFalse(payload.block_images_and_css)
         self.assertTrue(payload.wait_for_complete_page_load)
         self.assertIsNone(payload.user_agent)