Limit database cleanup to once per hour

Author: philippesaade-wmde

main.py

@@ -3,6 +3,8 @@
 from fastapi import BackgroundTasks
 import traceback
 import requests
+import time
+import os
 
 from src.Normalizer import TTLNormalizer, JSONNormalizer
 from src.WikidataLabel import WikidataLabel, LazyLabelFactory
@@ -27,6 +29,9 @@
     allow_headers=["*"],
 )
 
+LABEL_CLEANUP_INTERVAL_SECONDS = int(os.environ.get("LABEL_CLEANUP_INTERVAL_SECONDS", 3600))
+_last_label_cleanup = 0.0
+
 @app.on_event("startup")
 async def startup():
     WikidataLabel.initialize_database()
@@ -169,7 +174,11 @@ async def get_textified_wd(
 
             return_data[qid] = results
 
-        background_tasks.add_task(WikidataLabel.delete_old_labels)
+        global _last_label_cleanup
+        if time.time() - _last_label_cleanup > LABEL_CLEANUP_INTERVAL_SECONDS:
+            background_tasks.add_task(WikidataLabel.delete_old_labels)
+            _last_label_cleanup = time.time()
+
         return return_data
 
     except HTTPException:

src/WikidataLabel.py

@@ -2,8 +2,8 @@
 from sqlalchemy.dialects.mysql import JSON
 from sqlalchemy.orm import sessionmaker, declarative_base
 
+from .utils import get_wikidata_json_by_ids
 from datetime import datetime, timedelta
-import requests
 import os
 import json
 
@@ -253,38 +253,7 @@ def _get_labels_wdapi(ids):
         Returns:
         - dict: A dictionary mapping each ID to its labels.
         """
-        entities_data = {}
-
-        if isinstance(ids, str):
-            ids = ids.split('|')
-        ids = list(set(ids)) # Ensure unique IDs
-
-        # Wikidata API has a limit on the number of IDs per request, typically 50 for wbgetentities.
-        for chunk_idx in range(0, len(ids), 50):
-
-            ids_chunk = ids[chunk_idx:chunk_idx+50]
-            ids_chunk = "|".join(ids_chunk)
-            params = {
-                'action': 'wbgetentities',
-                'ids': ids_chunk,
-                'props': 'labels',
-                'format': 'json',
-                'origin': '*',
-            }
-            headers = {
-                'User-Agent': 'Wikidata Textifier (embedding@wikimedia.de)'
-            }
-
-            response = requests.get(
-                "https://www.wikidata.org/w/api.php?",
-                params=params,
-                headers=headers,
-                timeout=REQUEST_TIMEOUT_SECONDS,
-            )
-            response.raise_for_status()
-            chunk_data = response.json().get("entities", {})
-            entities_data = entities_data | chunk_data
-
+        entities_data = get_wikidata_json_by_ids(ids, props="labels")
         entities_data = WikidataLabel._compress_labels(entities_data)
         return entities_data
 

src/utils.py

@@ -1,10 +1,17 @@
 import requests
+from requests.adapters import HTTPAdapter
+
 import json
 import html
 import os
 
 REQUEST_TIMEOUT_SECONDS = float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "15"))
 
+SESSION = requests.Session()
+adapter = HTTPAdapter(pool_connections=20, pool_maxsize=20)
+SESSION.mount("http://", adapter)
+SESSION.mount("https://", adapter)
+
 def get_wikidata_ttl_by_id(
         id,
         lang='en',
@@ -25,7 +32,7 @@ def get_wikidata_ttl_by_id(
         'User-Agent': 'Wikidata Textifier (embeddings@wikimedia.de)'
     }
 
-    response = requests.get(
+    response = SESSION.get(
         f"https://www.wikidata.org/wiki/Special:EntityData/{id}.ttl",
         params=params,
         headers=headers,
@@ -52,7 +59,7 @@ def get_wikidata_json_by_ids(
 
     if isinstance(ids, str):
         ids = ids.split('|')
-    ids = list(set(ids)) # Ensure unique IDs
+    ids = list(dict.fromkeys(ids))  # Ensure unique IDs
 
     entities_data = {}
 
@@ -72,7 +79,7 @@ def get_wikidata_json_by_ids(
             'User-Agent': 'Wikidata Textifier (embeddings@wikimedia.de)'
         }
 
-        response = requests.get(
+        response = SESSION.get(
             "https://www.wikidata.org/w/api.php?",
             params=params,
             headers=headers,
@@ -116,7 +123,7 @@ def wikidata_time_to_text(value: dict, lang: str = "en"):
         },
     }
 
-    r = requests.post(WIKIBASE_API, data={
+    r = SESSION.post(WIKIBASE_API, data={
         "action": "wbformatvalue",
         "format": "json",
         "uselang": lang,
@@ -148,7 +155,7 @@ def wikidata_geolocation_to_text(value: dict, lang: str = "en"):
         },
     }
 
-    r = requests.post(WIKIBASE_API, data={
+    r = SESSION.post(WIKIBASE_API, data={
         "action": "wbformatvalue",
         "format": "json",
         "uselang": lang,