Fix bug with missing items

Author: philippesaade-wmde

src/WikidataLabel.py

@@ -247,7 +247,7 @@ def _get_labels_wdapi(ids):
                 'origin': '*',
             }
             headers = {
-                'User-Agent': 'Wikidata Textifier'
+                'User-Agent': 'Wikidata Textifier (embedding@wikimedia.de)'
             }
 
             response = requests.get(
@@ -263,22 +263,25 @@ def _get_labels_wdapi(ids):
         return entities_data
 
     @staticmethod
-    def _compress_labels(labels):
+    def _compress_labels(data):
         """
         Compress labels by extracting the 'value' field from each label.
 
         Parameters:
-        - labels (dict): A dictionary of labels from Wikidata API.
+        - data (dict): A dictionary of labels from Wikidata API.
 
         Returns:
         - dict: A new dictionary with labels compressed to their 'value' field.
         """
         new_labels = {}
-        for qid, labels in labels.items():
-            new_labels[qid] = {
-                lang: label.get('value') \
-                    for lang,label in labels['labels'].items()
-            }
+        for qid, labels in data.items():
+            if 'labels' in labels:
+                new_labels[qid] = {
+                    lang: label.get('value') \
+                        for lang, label in labels['labels'].items()
+                }
+            else:
+                new_labels[qid] = {}
         return new_labels
 
     @staticmethod

src/WikidataTextifier.py

@@ -288,8 +288,9 @@ def to_json(self):
         value = self.value.to_json()
         if isinstance(self.value, WikidataEntity):
             ID_name = "QID" if self.claim.datatype == 'wikibase-item' else "PID"
+            entity_id = value.get('QID') or value.get('PID')
             value = {
-                ID_name: value['QID'],
+                ID_name: entity_id,
                 'label': str(value['label'])
             }
 
@@ -322,9 +323,10 @@ def to_triplet(self):
         if self.rank == 'deprecated':
             string += " [deprecated]"
 
-        qualifiers = [q.to_triplet() for q in self.qualifiers if q]
+        qualifiers = [q.to_triplet(as_qualifier=True) \
+                      for q in self.qualifiers if q]
         if len(qualifiers) > 0:
-            string += f" | {' | '.join(qualifiers)})"
+            string += f" | {' | '.join(qualifiers)}"
         return string
 
 
@@ -427,24 +429,31 @@ def __bool__(self):
 
     def to_json(self):
         property = self.property.to_json()
+        property_id = property.get('PID') or property.get('QID')
         return {
-            "PID": property['QID'],
+            "PID": property_id,
             "property_label": property['label'],
             "datatype": self.datatype,
             "values": [v.to_json() for v in self.values if v]
         }
 
-    def to_triplet(self):
+    def to_triplet(self, as_qualifier=False):
         if not self:
             return ''
 
         label = f"{str(self.property.label)} ({self.property.id})"
         values = [v.to_triplet() for v in self.values if v]
 
         if len(values) > 0:
-            values = [f"{label}: {v}" for v in values]
-            values = "\n".join(values)
-            return values
+            if as_qualifier:
+                # For qualifiers: join multiple values with comma on same line
+                values_str = ", ".join(values)
+                return f"{label}: {values_str}"
+            else:
+                # For main claims: each value gets its own line
+                values = [f"{label}: {v}" for v in values]
+                values = "\n".join(values)
+                return values
 
         return ''
 
@@ -519,7 +528,7 @@ def from_id(cls, id: str,
         return entity
 
     def __str__(self):
-        label_str = str(self.label)
+        label_str = str(self.label) if self.label else '<missing>'
         string = label_str
 
         if self.description:
@@ -542,16 +551,18 @@ def __bool__(self):
                (str(self.label) != '')
 
     def to_json(self):
+        id_key = 'PID' if self.id.startswith('P') else 'QID'
+
         return {
-            'QID': self.id,
-            'label': str(self.label),
+            id_key: self.id,
+            'label': str(self.label) if self.label else None,
             'description': self.description,
             'aliases': self.aliases,
             'claims': [c.to_json() for c in self.claims if c]
         }
 
     def to_triplet(self):
-        label = f"{str(self.label)} ({self.id})"
+        label = f"{str(self.label) if self.label else '<missing>'} ({self.id})"
         attributes = []
         if self.description:
             attributes.append(f"description: {self.description}")