Add User-Agent header and remove HTTP adapter for HTTPS-only requests

Goziee-git · Goziee-git · commit 70d191fb8293 · 2025-10-22T04:20:12.000+01:00
diff --git a/scripts/1-fetch/arxiv_fetch.py b/scripts/1-fetch/arxiv_fetch.py
@@ -199,7 +199,7 @@ def get_requests_session():
         status_forcelist=[408, 429, 500, 502, 503, 504]
     )
     session = requests.Session()
-    session.mount("http://", HTTPAdapter(max_retries=retry_strategy))
+    session.headers.update({"User-Agent": shared.USER_AGENT})
     session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
     return session
 

Original file line number	Diff line number	Diff line change
`@@ -199,7 +199,7 @@ def get_requests_session():`
`199`	`199`	`status_forcelist=[408, 429, 500, 502, 503, 504]`
`200`	`200`	`)`
`201`	`201`	`session = requests.Session()`
`202`		`- session.mount("http://", HTTPAdapter(max_retries=retry_strategy))`
	`202`	`+ session.headers.update({"User-Agent": shared.USER_AGENT})`
`203`	`203`	`session.mount("https://", HTTPAdapter(max_retries=retry_strategy))`
`204`	`204`	`return session`
`205`	`205`