Adapting text format in multiple languages

Author: philippesaade-wmde

docker-compose.yml

@@ -69,6 +69,10 @@ services:
       wikibase:
         condition: service_healthy
     environment:
+      LABEL_UNLIMITED: ${LABEL_UNLIMITED}
+      LABEL_TTL_DAYS: ${LABEL_TTL_DAYS}
+      LABEL_MAX_ROWS: ${LABEL_MAX_ROWS}
+
       WIKIBASE_HOST: wikibase
       DB_HOST: db
       DB_NAME: ${DB_NAME_LABEL}

main.py

@@ -2,6 +2,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi import BackgroundTasks
 import traceback
+import requests
 
 from src.Normalizer import TTLNormalizer, JSONNormalizer
 from src.WikidataLabel import WikidataLabel
@@ -26,6 +27,10 @@
     allow_headers=["*"],
 )
 
+@app.on_event("startup")
+async def startup():
+    WikidataLabel.initialize_database()
+
 @app.get(
     "/",
     responses={
@@ -58,6 +63,7 @@ async def get_textified_wd(
     external_ids: bool = True,
     references: bool = False,
     all_ranks: bool = False,
+    qualifiers: bool = True,
     fallback_lang: str = 'en'
 ):
     """
@@ -71,17 +77,13 @@ async def get_textified_wd(
         external_ids (bool): If True, includes external IDs in the response.
         all_ranks (bool): If True, includes statements of all ranks (preferred, normal, deprecated).
         references (bool): If True, includes references in the response. (only available in JSON format)
+        qualifiers (bool): If True, includes qualifiers in the response.
         fallback_lang (str): The fallback language code if the preferred language is not available.
 
     Returns:
         list: A list of dictionaries containing QIDs and the similarity scores.
     """
     try:
-
-        if not id:
-            response = "ID is missing"
-            return HTTPException(status_code=422, detail=response)
-
         filter_pids = []
         if pid:
             filter_pids = [p.strip() for p in pid.split(',')]
@@ -94,7 +96,7 @@ async def get_textified_wd(
             entity_data = utils.get_wikidata_ttl_by_id(qids[0], lang=lang)
             if not entity_data:
                 response = "ID not found"
-                return HTTPException(status_code=404, detail=response)
+                raise HTTPException(status_code=404, detail=response)
 
             entity_data = TTLNormalizer(
                 entity_id=qids[0],
@@ -109,15 +111,16 @@ async def get_textified_wd(
                     external_ids=external_ids,
                     all_ranks=all_ranks,
                     references=references,
-                    filter_pids=filter_pids
+                    filter_pids=filter_pids,
+                    qualifiers=qualifiers,
                 )
             }
         else:
             # JSON is used with Action API for bulk retrieval
             entity_data = utils.get_wikidata_json_by_ids(qids)
             if not entity_data:
                 response = "IDs not found"
-                return HTTPException(status_code=404, detail=response)
+                raise HTTPException(status_code=404, detail=response)
 
             entity_data = {
                 qid: JSONNormalizer(
@@ -135,7 +138,8 @@ async def get_textified_wd(
                     external_ids=external_ids,
                     all_ranks=all_ranks,
                     references=references,
-                    filter_pids=filter_pids
+                    filter_pids=filter_pids,
+                    qualifiers=qualifiers
                 ) if entity else None
                 for qid, entity in entity_data.items()
             }
@@ -147,7 +151,7 @@ async def get_textified_wd(
                 continue
 
             if format == 'text':
-                results = str(entity)
+                results = entity.to_text(lang)
             elif format == 'triplet':
                 results = entity.to_triplet()
             else:
@@ -158,6 +162,10 @@ async def get_textified_wd(
         background_tasks.add_task(WikidataLabel.delete_old_labels)
         return return_data
 
-    except Exception as e:
+    except HTTPException:
+        raise
+    except requests.RequestException:
+        raise HTTPException(status_code=502, detail="Upstream service unavailable")
+    except Exception:
         traceback.print_exc()
         raise HTTPException(status_code=500, detail="Internal Server Error")

requirements.txt

@@ -1,12 +1,12 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml --output-file requirements.txt --no-deps
-babel==2.17.0
-    # via wikidatatextifier (pyproject.toml)
 fastapi==0.116.1
     # via wikidatatextifier (pyproject.toml)
 gunicorn==23.0.0
     # via wikidatatextifier (pyproject.toml)
-python-dateutil==2.9.0.post0
+pymysql==1.1.2
+    # via wikidatatextifier (pyproject.toml)
+rdflib==7.6.0
     # via wikidatatextifier (pyproject.toml)
 requests==2.32.4
     # via wikidatatextifier (pyproject.toml)

src/Normalizer/JSONNormalizer.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 from typing import Any, Dict, List, Optional
+import requests
 
 from ..WikidataLabel import WikidataLabel, LazyLabelFactory
-from ..WikidataTextifier import (
+from ..Textifier.WikidataTextifier import (
     WikidataClaim,
     WikidataClaimValue,
     WikidataCoordinates,
@@ -45,6 +46,7 @@ def normalize(
         external_ids: bool = True,
         references: bool = False,
         all_ranks: bool = False,
+        qualifiers: bool = True,
         filter_pids: List[str] = [],
     ) -> WikidataEntity:
         e = self.entity_json
@@ -91,6 +93,7 @@ def normalize(
                 external_ids=external_ids,
                 include_references=references,
                 all_ranks=all_ranks,
+                qualifiers=qualifiers
             )
             if claim_obj and claim_obj.values:
                 claims_out.append(claim_obj)
@@ -116,6 +119,7 @@ def _build_claim(
         external_ids: bool,
         include_references: bool,
         all_ranks: bool,
+        qualifiers: bool,
     ) -> Optional[WikidataClaim]:
         datatype = self._claim_datatype_from_statements(statements) or "string"
         if (not external_ids) and datatype == "external-id":
@@ -141,6 +145,7 @@ def _build_claim(
                 statement=st,
                 datatype=datatype,
                 include_references=include_references,
+                qualifiers=qualifiers
             )
             if cv is not None:
                 values.append(cv)
@@ -187,6 +192,7 @@ def _build_claim_value(
         statement: Dict[str, Any],
         datatype: str,
         include_references: bool,
+        qualifiers: bool,
     ) -> Optional[WikidataClaimValue]:
         mainsnak = statement.get("mainsnak", statement)
         if not isinstance(mainsnak, dict):
@@ -206,7 +212,9 @@ def _build_claim_value(
         datavalue = mainsnak.get("datavalue")
         value_obj = self._to_value_object(datatype, datavalue)
 
-        qualifiers_obj = self._parse_qualifiers(statement.get("qualifiers", {}) or {})
+        qualifiers_obj: List[WikidataClaim] = []
+        if qualifiers:
+            qualifiers_obj = self._parse_qualifiers(statement.get("qualifiers", {}) or {})
         references_obj: List[List[WikidataClaim]] = []
         if include_references:
             references_obj = self._parse_references(statement.get("references", []) or [])
@@ -336,7 +344,7 @@ def _to_value_object(
                     dv_val,
                     self.lang,
                 )
-            except (ValueError, TypeError) as e:
+            except (ValueError, TypeError, KeyError, requests.RequestException) as e:
                 if self.debug:
                     print(f"Warning: Failed to parse time value {time_val}: {e}")
                 return None
@@ -376,7 +384,7 @@ def _to_value_object(
 
             try:
                 string_val = wikidata_geolocation_to_text(dv_val, self.lang)
-            except (ValueError, TypeError) as e:
+            except (ValueError, TypeError, KeyError, requests.RequestException) as e:
                 if self.debug:
                     print(f"Warning: Failed to parse coordinates ({lat}, {lon}): {e}")
                 return None

src/Normalizer/TTLNormalizer.py

@@ -2,12 +2,13 @@
 
 from collections import defaultdict
 from typing import Any, DefaultDict, Dict, List, Optional, Set
+import requests
 
 from rdflib import Graph, Literal, Namespace, URIRef
 from rdflib.namespace import RDF, RDFS
 
 from ..WikidataLabel import WikidataLabel, LazyLabelFactory
-from ..WikidataTextifier import (
+from ..Textifier.WikidataTextifier import (
     WikidataClaim,
     WikidataClaimValue,
     WikidataCoordinates,
@@ -80,6 +81,7 @@ def normalize(
         external_ids: bool = True,
         references: bool = False,
         all_ranks: bool = False,
+        qualifiers: bool = True,
         filter_pids: List[str] = []
     ) -> WikidataEntity:
         # Preload labels found inside TTL so LazyLabelFactory can avoid lookups.
@@ -107,6 +109,7 @@ def normalize(
             external_ids=external_ids,
             include_references=references,
             all_ranks=all_ranks,
+            qualifiers=qualifiers,
             filter_pids=filter_pids
         )
 
@@ -124,6 +127,7 @@ def normalize(
                 pid=pid,
                 statements=statements,
                 include_references=references,
+                qualifiers=qualifiers,
             )
             for pid, statements in claims_dict.items()
             if statements
@@ -162,6 +166,7 @@ def _claims_for_subject(
         external_ids: bool,
         include_references: bool,
         all_ranks: bool,
+        qualifiers: bool,
         filter_pids: List[str] = []
     ) -> Dict[str, List[Dict[str, Any]]]:
         """Return mapping: pid -> list of statement dicts."""
@@ -195,7 +200,7 @@ def _claims_for_subject(
             is_special = self._is_special_main_value(obj, pid)
             main = None if is_special else self._main_value(obj, pid, datatype)
 
-            qualifiers = self._qualifiers(obj)
+            qualifiers_data = self._qualifiers(obj) if qualifiers else {}
             refs = self._references(obj) if include_references else []
 
             out[pid].append(
@@ -204,7 +209,7 @@ def _claims_for_subject(
                     "datatype": datatype,
                     "rank": rank,
                     "main": main,
-                    "qualifiers": qualifiers if qualifiers else {},
+                    "qualifiers": qualifiers_data if qualifiers_data else {},
                     "references": refs if refs else [],
                     "is_special_value": is_special,
                 }
@@ -273,6 +278,7 @@ def _build_claim_object(
         pid: str,
         statements: List[Dict[str, Any]],
         include_references: bool,
+        qualifiers: bool = True,
     ) -> WikidataClaim:
         prop_ent = WikidataEntity(
             id=pid,
@@ -291,15 +297,17 @@ def _build_claim_object(
                 print(f"{pid}: {st.get('main')} (special: {st.get('is_special_value', False)})")
 
             value_obj = self._to_value_object(st["datatype"], st.get("main"))
-
-            qualifiers_obj: List[WikidataClaim] = [
-                self._build_snak_claim(
-                    pid=qpid,
-                    datatype=self._prop_datatype(qpid),
-                    snaks=qsnaks,
-                )
-                for qpid, qsnaks in (st.get("qualifiers") or {}).items()
-            ]
+            qualifiers_obj: List[WikidataClaim] = []
+
+            if qualifiers:
+                qualifiers_obj = [
+                    self._build_snak_claim(
+                        pid=qpid,
+                        datatype=self._prop_datatype(qpid),
+                        snaks=qsnaks,
+                    )
+                    for qpid, qsnaks in (st.get("qualifiers") or {}).items()
+                ]
 
             refs_obj: List[List[WikidataClaim]] = []
             if include_references:
@@ -409,7 +417,7 @@ def _to_value_object(self, datatype: str, parsed: Any) -> Any:
                     parsed,
                     self.lang,
                 )
-            except (ValueError, TypeError) as e:
+            except (ValueError, TypeError, KeyError, requests.RequestException) as e:
                 if self.debug:
                     print(f"Warning: Failed to parse time value {time_val}: {e}")
                 return None
@@ -436,7 +444,7 @@ def _to_value_object(self, datatype: str, parsed: Any) -> Any:
 
             try:
                 string_val = wikidata_geolocation_to_text(parsed, self.lang)
-            except (ValueError, TypeError) as e:
+            except (ValueError, TypeError, KeyError, requests.RequestException) as e:
                 if self.debug:
                     print(f"Warning: Failed to parse coordinates ({lat}, {lon}): {e}")
                 return None