upgraded libraries and tests fix WIP (pydantic 2)

Author: Dragos0000

pyproject.toml

@@ -0,0 +1,46 @@
+[project]
+name = "ted-rdf-conversion-pipeline"
+version = "0.1.0"
+requires-python = ">=3.9"
+dependencies = [
+    "agraph-python>=104.2.0",
+    "allure-pytest-bdd>=2.14.0",
+    "apache-airflow==2.10.5",
+    "certifi==2025.1.31",
+    "colorama==0.4.6",
+    "coverage>=7.8.0",
+    "decorator==5.1.1",
+    "deepdiff>=8.4.2",
+    "elasticsearch-dbapi>=0.2.11",
+    "fastapi>=0.115.12",
+    "hvac>=2.3.0",
+    "jinja2~=3.1.5",
+    "json2html>=1.3.0",
+    "minio>=7.2.15",
+    "mongomock>=4.3.0",
+    "openpyxl>=3.1.5",
+    "ordered-set~=4.1.0",
+    "pandas==2.1.4",
+    "paramiko>=3.5.1",
+    "pendulum==3.0.0",
+    "pycurl>=7.45.6",
+    "pydantic==2.10.6",
+    "pymongo>=4.12.0",
+    "pyshacl>=0.30.1",
+    "pytest>=8.3.5",
+    "pytest-bdd>=8.1.0",
+    "pytest-cov>=6.1.1",
+    "pytest-subtests>=0.14.1",
+    "python-dateutil==2.9.0.post0",
+    "python-dotenv>=1.1.0",
+    "rdflib>=7.1.4",
+    "requests==2.32.3",
+    "saxonche>=12.5.0",
+    "semantic-version>=2.10.0",
+    "shortuuid>=1.0.13",
+    "sparqlwrapper>=2.0.0",
+    "tox>=4.25.0",
+    "tox-pytest-summary>=0.1.2",
+    "urllib3==2.3.0",
+    "uvicorn[standard]>=0.34.0",
+]

requirements.dev.txt

@@ -1,12 +1,12 @@
 # Development libraries
-coverage~=6.3.1
-pytest~=7.0.0
-pytest-bdd~=5.0.0
-pytest-cov~=3.0.0
-pytest-subtests~=0.6.0
-tox~=3.24.5
-tox-pytest-summary~=0.1.2
-mongomock==4.1.2
-uvicorn[standard]==0.24.0
-allure-pytest-bdd==2.10.0
-pycurl~=7.45.2
+coverage>=7.8.0
+pytest>=8.3.5
+pytest-bdd>=8.1.0
+pytest-cov>=6.1.1
+pytest-subtests>=0.14.1
+tox>=4.25.0
+tox-pytest-summary>=0.1.2
+mongomock>=4.3.0
+uvicorn[standard]>=0.34.0
+allure-pytest-bdd>=2.14.0
+pycurl>=7.45.6

requirements.txt

@@ -1,30 +1,30 @@
-# Project dependecies
-pydantic~=1.9.0
-requests~=2.28.2
-deepdiff~=5.7.0
-jinja2~=3.1.2
-python-dotenv~=0.21.0
-pymongo~=4.0.1
-apache-airflow~=2.5.1
-hvac==0.11.2
-SPARQLWrapper==1.8.5
-pandas==1.5.2
-openpyxl==3.0.9
-colorama~=0.4
-fastapi~=0.77
-python-dateutil~=2.8.2
-rdflib~=6.1.1
-pyshacl~=0.19.0
-agraph-python==101.0.10
-decorator~=5.1.1
-urllib3[secure]
-semantic-version==2.10.0
-paramiko~=3.0.0
-ordered-set~=4.0.2
-json2html~=1.3.0
-minio~=7.1.1
-certifi~=2022.12.7
-shortuuid~=1.0.11
-pendulum~=2.1.2
-saxonche~=12.4
-elasticsearch-dbapi==0.2.11
+# Project dependencies
+pydantic==2.10.6
+requests==2.32.3
+deepdiff>=8.4.2
+jinja2~=3.1.5
+python-dotenv>=1.1.0
+pymongo>=4.12.0
+apache-airflow==2.10.5
+hvac>=2.3.0
+sparqlwrapper>=2.0.0
+pandas==2.1.4
+openpyxl>=3.1.5
+colorama==0.4.6
+fastapi>=0.115.12
+python-dateutil==2.9.0.post0
+rdflib>=7.1.4
+pyshacl>=0.30.1
+agraph-python>=104.2.0
+decorator==5.1.1
+urllib3==2.3.0
+semantic-version>=2.10.0
+paramiko>=3.5.1
+ordered-set~=4.1.0
+json2html>=1.3.0
+minio>=7.2.15
+certifi==2025.1.31
+shortuuid>=1.0.13
+pendulum==3.0.0
+saxonche>=12.5.0
+elasticsearch-dbapi>=0.2.11

ted_sws/core/model/__init__.py

@@ -22,24 +22,24 @@ class PropertyBaseModel(BaseModel):
     https://github.com/samuelcolvin/pydantic/issues/935
     is solved
     """
-
-    @classmethod
-    def get_properties(cls):
-        return [
-            prop for prop in dir(cls)
-            if isinstance(getattr(cls, prop), property) and prop not in ("__values__", "fields")
-        ]
-
-    def dict(self, *args, **kwargs) -> 'DictStrAny':
-        self.__dict__.update({prop: getattr(self, prop) for prop in self.get_properties()})
-
-        return super().dict(*args, **kwargs)
+    #TODO : remove this code, because this implementation is not necessary anymore in pydantic 2
+    # @classmethod
+    # def get_properties(cls):
+    #     return [
+    #         prop for prop in dir(cls)
+    #         if isinstance(getattr(cls, prop), property) and prop not in ("__values__", "fields")
+    #     ]
+    #TODO : remove this code, because this implementation is not necessary anymore in pydantic 2
+    # def dict(self, *args, **kwargs) -> 'DictStrAny':
+    #     self.__dict__.update({prop: getattr(self, prop) for prop in self.get_properties()})
+    #
+    #     return super().dict(*args, **kwargs)
 
     def __eq__(self, other) -> bool:
         if not isinstance(other, self.__class__) or not other:
             return False
             # raise ValueError(f"Must compare objects of the same class {self.__class__}")
-        difference = DeepDiff(self.dict(), other.dict())
+        difference = DeepDiff(self.model_dump(), other.model_dump())
         return not difference
 
     def __ne__(self, other):

ted_sws/core/model/manifestation.py

@@ -68,7 +68,7 @@ class XPATHCoverageSummaryResult(PropertyBaseModel):
 
 
 class XPATHCoverageSummaryReport(PropertyBaseModel):
-    mapping_suite_identifier: Optional[str]
+    mapping_suite_identifier: Optional[str] = None
     validation_result: Optional[XPATHCoverageSummaryResult] = XPATHCoverageSummaryResult()
 
 
@@ -85,8 +85,8 @@ class SPARQLSummaryCountReport(PropertyBaseModel):
 
 
 class SPARQLSummaryResult(PropertyBaseModel):
-    test_suite_identifier: Optional[str]
-    mapping_suite_identifier: Optional[str]
+    test_suite_identifier: Optional[str] = None
+    mapping_suite_identifier: Optional[str] = None
     aggregate: Optional[SPARQLSummaryCountReport] = SPARQLSummaryCountReport()
 
 
@@ -106,8 +106,8 @@ class SHACLSummaryResultSeverityReport(PropertyBaseModel):
 
 
 class SHACLSummaryResult(PropertyBaseModel):
-    test_suite_identifier: Optional[str]
-    mapping_suite_identifier: Optional[str]
+    test_suite_identifier: Optional[str] = None
+    mapping_suite_identifier: Optional[str] = None
     result_severity: Optional[SHACLSummaryResultSeverityReport] = SHACLSummaryResultSeverityReport()
 
 
@@ -140,11 +140,11 @@ class XPATHCoverageValidationAssertion(PropertyBaseModel):
     """
 
     """
-    form_field: Optional[str]
-    xpath: Optional[str]
-    count: Optional[int]
-    notice_hit: Optional[Dict[str, int]]
-    query_result: Optional[bool]
+    form_field: Optional[str] = None
+    xpath: Optional[str] = None
+    count: Optional[int] = None
+    notice_hit: Optional[Dict[str, int]] = None
+    query_result: Optional[bool] = None
 
 
 class XPATHCoverageValidationResultBase(PropertyBaseModel):
@@ -159,22 +159,22 @@ class XPATHCoverageValidationResult(XPATHCoverageValidationResultBase):
     """
     XPATHCoverageValidationResult for Notice
     """
-    notices: Optional[List[ReportNoticeData]]
+    notices: Optional[List[ReportNoticeData]] = None
 
 
 class XPATHCoverageValidationReport(XMLValidationManifestation):
     """
     This is the model structure for Notice(s) XPATHs Coverage Report
     """
 
-    validation_result: Optional[XPATHCoverageValidationResult]
+    validation_result: Optional[XPATHCoverageValidationResult] = None
 
 
 class XMLManifestation(Manifestation):
     """
         Original XML Notice manifestation as published on the TED website.
     """
-    xpath_coverage_validation: XPATHCoverageValidationReport = None
+    xpath_coverage_validation: Optional[XPATHCoverageValidationReport] = None
 
     def add_validation(self, validation: Union[XPATHCoverageValidationReport]):
         if type(validation) == XPATHCoverageValidationReport:
@@ -200,15 +200,15 @@ class RDFValidationManifestation(ValidationManifestation):
 
     """
     mapping_suite_identifier: str
-    test_suite_identifier: Optional[str]
+    test_suite_identifier: Optional[str] = None
 
 
 class SPARQLQuery(PropertyBaseModel):
     """
     Stores SPARQL query details
     """
-    title: Optional[str]
-    description: Optional[str]
+    title: Optional[str] = None
+    description: Optional[str] = None
     xpath: Optional[List[str]] = []
     query: str
 
@@ -218,13 +218,13 @@ class SPARQLQueryResult(PropertyBaseModel):
     Stores SPARQL query execution result
     """
     query: SPARQLQuery
-    result: Optional[SPARQLQueryRefinedResultType]
-    query_result: Optional[str]
+    result: Optional[SPARQLQueryRefinedResultType] = None
+    query_result: Optional[str] = None
     fields_covered: Optional[bool] = True
     missing_fields: Optional[List[str]] = []
-    error: Optional[str]
-    message: Optional[str]
-    identifier: Optional[str]
+    error: Optional[str] = None
+    message: Optional[str] = None
+    identifier: Optional[str] = None
 
     class Config:
         use_enum_values = True
@@ -242,10 +242,10 @@ class QueriedSHACLShapeValidationResult(PropertyBaseModel):
     Queried SHACL Validation Report which contains the following variables
     ?focusNode ?message ?resultPath ?resultSeverity ?sourceConstraintComponent ?sourceShape ?value
     """
-    conforms: Optional[str]
-    results_dict: Optional[dict]
-    error: Optional[str]
-    identifier: Optional[str]
+    conforms: Optional[str] = None
+    results_dict: Optional[dict] = None
+    error: Optional[str] = None
+    identifier: Optional[str] = None
 
 
 class SHACLTestSuiteValidationReport(RDFValidationManifestation):
@@ -256,7 +256,7 @@ class SHACLTestSuiteValidationReport(RDFValidationManifestation):
 
 
 class EntityDeduplicationReport(Manifestation):
-    object_data: Optional[str]
+    object_data: Optional[str] = None
     number_of_duplicates: int
     number_of_cets: int
     uries: List[str]
@@ -266,10 +266,10 @@ class RDFManifestation(Manifestation):
     """
         Transformed manifestation in RDF format
     """
-    mapping_suite_id = "unknown_mapping_suite_id"
+    mapping_suite_id: str = "unknown_mapping_suite_id"
     shacl_validations: List[SHACLTestSuiteValidationReport] = []
     sparql_validations: List[SPARQLTestSuiteValidationReport] = []
-    deduplication_report: Optional[EntityDeduplicationReport]
+    deduplication_report: Optional[EntityDeduplicationReport] = None
 
     def validation_exists(self, validation, validations):
         """