Add description and aliases in triplet form

Author: philippesaade-wmde

Dockerfile

@@ -13,4 +13,4 @@ RUN uv sync
 COPY --chmod=755 . .
 
 # Container start script
-CMD ["uv", "run", "gunicorn", "main:app", "-k", "uvicorn.workers.UvicornWorker", "-w", "4", "-b", "0.0.0.0:8000"]
+CMD ["uv", "run", "gunicorn", "main:app", "-k", "uvicorn.workers.UvicornWorker", "-w", "4", "-b", "0.0.0.0:5000"]

docker-compose.yml

@@ -5,4 +5,4 @@ services:
       - ./data:/workspace/data
     container_name: wdtextifier
     ports:
-      - "8000:8000"
+      - "5000:5000"

main.py

@@ -1,8 +1,10 @@
 from fastapi import FastAPI, Header, HTTPException, Query, Request
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi import BackgroundTasks
 import traceback
 
 from src.WikidataTextifier import WikidataEntity
+from src.WikidataLabel import WikidataLabel
 
 # Start Fastapi app
 app = FastAPI(
@@ -46,8 +48,8 @@
         },
     },
 )
-async def property_query_route(
-    request: Request,
+async def get_labels(
+    request: Request, background_tasks: BackgroundTasks,
     id: str = Query(..., examples="Q42"),
     pid: str = Query(None, examples="P31,P279"),
     lang: str = 'en',
@@ -103,6 +105,7 @@ async def property_query_route(
             response = "Invalid format specified"
             return HTTPException(status_code=422, detail=response)
 
+        background_tasks.add_task(WikidataLabel.delete_old_labels)
         return results
     except Exception as e:
         traceback.print_exc()

src/WikidataLabel.py

@@ -175,6 +175,48 @@ def get_bulk_labels(ids):
 
         return labels
 
+    @staticmethod
+    def delete_old_labels():
+        """
+        Delete labels older than 90 days.
+        If the database exceeds 10 million rows, delete the oldest rows until it is below the threshold.
+        """
+        with Session() as session:
+            try:
+                # Step 1: Delete labels older than 90 days
+                date_limit = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%d %H:%M:%S')
+                session.execute(
+                    text("DELETE FROM labels WHERE date_added < :date_limit"),
+                    {"date_limit": date_limit}
+                )
+                session.commit()
+
+                # Step 2: Check total count
+                total_count = session.execute(text("SELECT COUNT(*) FROM labels")).scalar()
+                max_rows = 10_000_000
+
+                if total_count > max_rows:
+                    # Calculate how many rows to delete
+                    rows_to_delete = total_count - max_rows
+
+                    # Delete oldest rows by date_added
+                    session.execute(text(f"""
+                        DELETE FROM labels
+                        WHERE id IN (
+                            SELECT id FROM labels
+                            ORDER BY date_added ASC
+                            LIMIT :rows_to_delete
+                        )
+                    """), {"rows_to_delete": rows_to_delete})
+
+                    session.commit()
+
+                return True
+            except Exception as e:
+                session.rollback()
+                print(f"Error while deleting old labels: {e}")
+                return False
+
     @staticmethod
     def _get_labels_wdapi(ids):
         """

src/WikidataTextifier.py

@@ -552,7 +552,12 @@ def to_json(self):
 
     def to_triplet(self):
         label = f"{str(self.label)} ({self.id})"
-        attributes = [c.to_triplet() for c in self.claims if c]
+        attributes = []
+        if self.description:
+            attributes.append(f"description: {self.description}")
+        if self.aliases:
+            attributes.append(f"aliases: {', '.join(map(str, self.aliases))}")
+        attributes = [*attributes, *[c.to_triplet() for c in self.claims if c]]
 
         if len(attributes) > 0:
             attributes = "\n".join(attributes).split("\n")