External ID filtering

Author: philippesaade-wmde

README.md

@@ -0,0 +1,29 @@
+# Wikidata Textifier
+
+**Wikidata Textifier** is an API that transforms Wikidata items into compact JSON formats or textual representations for use in LLMs and GenAI applications. It resolves missing labels of properties and claim values by querying the Wikidata Action API, making it efficient and suitable for AI pipelines.
+
+🔗 Live API: [https://wd-textify.toolforge.org/](https://wd-textify.toolforge.org/)
+---
+
+## Functionalities
+
+- **Textifies** any Wikidata item into a readable or JSON format suitable for LLMs.
+- **Resolves all labels**, including those missing when querying the Wikidata API.
+- **Caches labels** for 90 days to boost performance and reduce API load.
+- **Avoids SPARQL** and uses the Wikidata Action API for better efficiency and compatibility.
+- **Hosted on Toolforge**: [https://wd-textify.toolforge.org/](https://wd-textify.toolforge.org/)
+
+---
+
+## API Usage
+
+### `GET /`
+
+#### Query Parameters
+
+| Name           | Type    | Required | Description                                                                 |
+|----------------|---------|----------|-----------------------------------------------------------------------------|
+| `id`           | string  | Yes      | Wikidata item ID (e.g., `Q42`)                                              |
+| `lang`         | string  | No       | Language code for labels (default: `en`)                                   |
+| `json`         | bool    | No       | If `true`, returns JSON. If `false`, returns text representation (default: `false`) |
+| `external_ids` | bool    | No       | Whether to include external IDs in the output (default: `true`)            |

main.py

@@ -51,13 +51,16 @@ async def property_query_route(
     id: str = Query(..., examples="Q42"),
     lang: str = 'en',
     json: bool = False,
+    external_ids: bool = True
 ):
     """
     Retrieve a Wikidata item with all labels or textual representations for an LLM.
 
     Args:
         id (str): The Wikidata item ID (e.g., "Q42").
         json (bool): If True, returns the item in JSON format.
+        lang (str): The language code for labels (default is 'en').
+        external_ids (bool): If True, includes external IDs in the response.
 
     Returns:
         list: A list of dictionaries containing QIDs and the similarity scores.
@@ -67,7 +70,11 @@ async def property_query_route(
         return HTTPException(status_code=422, detail=response)
 
     try:
-        entity = WikidataEntity.from_id(id, lang=lang)
+        entity = WikidataEntity.from_id(
+            id,
+            lang=lang,
+            external_ids=external_ids
+        )
 
         if not entity:
             response = "Item not found"

src/WikidataLabel.py

@@ -2,6 +2,9 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.types import TypeDecorator
+
+from .utils import get_lang_val
+
 from datetime import datetime, timedelta
 import json
 import requests
@@ -164,7 +167,6 @@ def get_bulk_labels(ids):
         # Fallback when labels are missing from the database
         missing_ids = set(ids) - set(labels.keys())
         if missing_ids:
-            print(f"Missing IDs: {missing_ids}")
             missing_labels = WikidataLabel._get_labels_wdapi(missing_ids)
             labels.update(missing_labels)
 
@@ -267,10 +269,8 @@ def resolve_all(self):
 
     def get_label(self, qid: str) -> str:
         label_dict = self._resolved_labels.get(qid, {})
-        label = label_dict.get(self.lang) or label_dict.get('mul') or ''
-        if isinstance(label, dict):
-            return label.get('value', '')
-        return label or ''
+        label = get_lang_val(label_dict, lang=self.lang)
+        return label
 
     def set_lang(self, lang: str):
         self.lang = lang

src/WikidataTextifier.py

@@ -253,7 +253,11 @@ class WikidataClaimValue:
     @classmethod
     def from_raw(cls, claim, value, qualifiers, lazylabel):
         if value.get('value') is None:
-            parsed_value = None
+            return cls(
+                claim=claim,
+                value=None,
+                qualifiers=[]
+            )
         elif value.get('type') == 'wikibase-entityid':
             parsed_value = WikidataEntity(
                 id=value['value']['id'],
@@ -306,14 +310,20 @@ def from_raw(cls, claim, value, qualifiers, lazylabel):
         )
 
     def __str__(self):
+        if not self:
+            return ''
+
         string = str(self.value)
-        attributes = [str(q) for q in self.qualifiers if str(q) != '']
+        attributes = [str(q) for q in self.qualifiers if q]
         if len(attributes) > 0:
             string += f" ({', '.join(attributes)})"
         return string
 
+    def __bool__(self):
+        return (self.value is not None) and str(self.value) != ''
+
     def to_json(self):
-        if self.value is None:
+        if not self:
             return None
 
         value = self.value.to_json()
@@ -324,10 +334,15 @@ def to_json(self):
                 'label': value['label']
             }
 
+        qualifiers = [q.to_json() for q in self.qualifiers if q]
+        if len(qualifiers) == 0:
+            return {
+                "value": value
+            }
+
         return {
             "value": value,
-            "qualifiers": [q.to_json() for q in self.qualifiers \
-                           if q is not None]
+            "qualifiers": [q.to_json() for q in self.qualifiers if q]
         }
 
 
@@ -339,7 +354,7 @@ class WikidataClaim:
     datatype: str
 
     @classmethod
-    def from_raw(cls, subject, property, claim, lazylabel):
+    def from_raw(cls, subject, property, claim, lazylabel, external_ids=True):
         if not claim:
             return cls(
                 subject=subject,
@@ -348,6 +363,16 @@ def from_raw(cls, subject, property, claim, lazylabel):
                 datatype='empty'
             )
 
+        datatype = claim[0].get('mainsnak', claim[0])\
+                            .get('datatype', {})
+        if not external_ids and datatype == 'external-id':
+            return cls(
+                subject=subject,
+                property=property,
+                values=[],
+                datatype=datatype
+            )
+
         rank_preferred_found = False
         for i in range(len(claim)):
             claim[i]['datavalue'] = claim[i].get('mainsnak', claim[i])\
@@ -369,8 +394,6 @@ def from_raw(cls, subject, property, claim, lazylabel):
             claim[i]['include'] = rank_normal_condition or \
                                   rank_preferred_condition
 
-        datatype = claim[0].get('mainsnak', claim[0])\
-                            .get('datatype', {})
         values = [
             WikidataClaimValue.from_raw(
                 claim=None,
@@ -394,28 +417,30 @@ def from_raw(cls, subject, property, claim, lazylabel):
 
 
     def __str__(self):
-        if str(self.property.label) == '':
+        if not self:
             return ''
 
-        if len(self.values) == 0:
-            values = "no value"
-        else:
-            values = [str(val) for val in self.values if str(val) != '']
-            values = ", ".join(values)
-
-        if values == '':
+        if not str(self.property.label):
             return ''
 
+        values = [str(v) for v in self.values if v]
+        values = ", ".join(values)
+
         return f"{str(self.property.label)}: {values}"
 
+    def __bool__(self):
+        return (self.property is not None) and \
+                str(self.property) != '' and \
+                    (len(self.values) > 0) and \
+                        any(bool(val) for val in self.values)
+
     def to_json(self):
         property = self.property.to_json()
         return {
             "PID": property['QID'],
             "property_label": property['label'],
             "datatype": self.datatype,
-            "values": [v.to_json() for v in self.values \
-                        if v is not None]
+            "values": [v.to_json() for v in self.values if v]
         }
 
 
@@ -429,7 +454,7 @@ class WikidataEntity:
     claims: list[WikidataClaim]
 
     @classmethod
-    def from_id(cls, id: str, lang: str = 'en'):
+    def from_id(cls, id: str, lang: str = 'en', external_ids: bool = True):
         entity_dict = get_wikidata_entities_by_ids(id)
         if id not in entity_dict:
             raise ValueError(f"ID not found.")
@@ -459,7 +484,8 @@ def from_id(cls, id: str, lang: str = 'en'):
                     claims=[]
                 ),
                 claim=claim,
-                lazylabel=lazylabel
+                lazylabel=lazylabel,
+                external_ids=external_ids
             ) for pid, claim in entity_dict.get('claims', {}).items()
         ]
 
@@ -492,7 +518,7 @@ def __str__(self):
         if self.aliases:
             string += f", also known as {', '.join(map(str, self.aliases))}"
 
-        attributes = [str(c) for c in self.claims if str(c) != '']
+        attributes = [str(c) for c in self.claims if c]
         if len(attributes) > 0:
             attributes = "\n- ".join(attributes)
             string += f". Attributes:\n- {attributes}"
@@ -501,12 +527,16 @@ def __str__(self):
 
         return string
 
+    def __bool__(self):
+        return (self.id is not None) and \
+               (self.label is not None) and \
+               (str(self.label) != '')
+
     def to_json(self):
         return {
             'QID': self.id,
             'label': str(self.label),
             'description': self.description,
             'aliases': self.aliases,
-            'claims': [c.to_json() for c in self.claims \
-                       if c is not None]
+            'claims': [c.to_json() for c in self.claims if c]
         }