Merge branch 'main' into feature/TED-607

Author: CaptainOfHacks

dags/worker_single_notice_process_orchestrator.py

@@ -11,6 +11,21 @@
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.data_sampler.services.notice_xml_indexer import index_notice
+from ted_sws.data_sampler.services.notice_xml_indexer import index_notice_by_id
+from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice_by_id
+
+from airflow.decorators import dag
+from airflow.operators.python import get_current_context, BranchPythonOperator, PythonOperator
+
+from dags import DEFAULT_DAG_ARGUMENTS
+from ted_sws.notice_metadata_processor.services.notice_eligibility import notice_eligibility_checker_by_id
+from ted_sws.notice_packager.services.notice_packager import create_notice_package
+from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapper
+from ted_sws.notice_transformer.services.notice_transformer import transform_notice_by_id
+from ted_sws.notice_validator.services.shacl_test_suite_runner import validate_notice_by_id_with_shacl_suite
+from ted_sws.notice_validator.services.sparql_test_suite_runner import validate_notice_by_id_with_sparql_suite
+from ted_sws.notice_validator.services.xpath_coverage_runner import validate_xpath_coverage_notice_by_id
+from ted_sws.event_manager.adapters.event_logger import EventLogger
 from ted_sws.event_manager.adapters.event_log_decorator import event_log
 from ted_sws.event_manager.adapters.event_logger import EventLogger
 from ted_sws.event_manager.model.event_message import NoticeEventMessage, EventMessageProcessType, EventMessageMetadata, \
@@ -190,12 +205,23 @@ def _validate_transformed_rdf_manifestation(**context_args):
         event_message: NoticeEventMessage = NoticeEventMessage()
         event_message.start_record()
 
-        notice = pull_dag_upstream(NOTICE_OBJECT)
-        mapping_suite = pull_dag_upstream(MAPPING_SUITE_OBJECT)
-        validate_notice_with_sparql_suite(notice=notice, mapping_suite_package=mapping_suite)
-        validate_notice_with_shacl_suite(notice=notice, mapping_suite_package=mapping_suite)
-        notice_id = notice.ted_id
-        push_dag_downstream(NOTICE_OBJECT, notice)
+        notice_id = pull_dag_upstream(NOTICE_ID)
+        mapping_suite_id = pull_dag_upstream(MAPPING_SUITE_ID)
+        mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
+        notice_repository = NoticeRepository(mongodb_client=mongodb_client)
+        mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
+        validate_notice_by_id_with_sparql_suite(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
+                                                notice_repository=notice_repository,
+                                                mapping_suite_repository=mapping_suite_repository)
+        validate_notice_by_id_with_shacl_suite(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
+                                               notice_repository=notice_repository,
+                                               mapping_suite_repository=mapping_suite_repository)
+        validate_xpath_coverage_notice_by_id(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
+                                             mapping_suite_repository=mapping_suite_repository,
+                                             mongodb_client=mongodb_client)
+        push_dag_downstream(NOTICE_ID, notice_id)
+        push_dag_downstream(MAPPING_SUITE_ID, mapping_suite_id)
+
         context = get_current_context()
 
         handle_event_message_metadata_dag_context(event_message, context)

requirements.txt

@@ -20,4 +20,4 @@ pyshacl~=0.19.0
 agraph-python==101.0.10
 decorator~=5.1.1
 urllib3[secure]
-semantic-version==2.10.0
+semantic-version==2.10.0

ted_sws/core/model/manifestation.py

@@ -8,7 +8,7 @@
 """ """
 from datetime import datetime
 from enum import Enum
-from typing import List, Union, Optional
+from typing import List, Union, Optional, Dict
 
 from pydantic import Field
 
@@ -42,10 +42,66 @@ def __str__(self):
         return f"/{str(content)[:STR_LEN]}" + ("..." if len(content) > STR_LEN else "") + "/"
 
 
+class ValidationManifestation(Manifestation):
+    """
+        The validation report
+    """
+    created: str = datetime.now().isoformat()
+
+
+class XMLValidationManifestation(ValidationManifestation):
+    """
+
+    """
+    mapping_suite_identifier: str
+
+
+class XPATHCoverageValidationAssertion(PropertyBaseModel):
+    """
+
+    """
+    title: Optional[str]
+    xpath: Optional[str]
+    count: Optional[int]
+    notice_hit: Optional[Dict[str, int]]
+    query_result: Optional[bool]
+
+
+class XPATHCoverageValidationResult(PropertyBaseModel):
+    """
+
+    """
+    notice_id: Optional[List[str]] = []
+    xpath_assertions: Optional[List[XPATHCoverageValidationAssertion]] = []
+    xpath_covered: Optional[List[str]] = []
+    xpath_not_covered: Optional[List[str]] = []
+    xpath_extra: Optional[List[str]] = []
+    coverage: Optional[float]
+    conceptual_coverage: Optional[float]
+
+
+class XPATHCoverageValidationReport(XMLValidationManifestation):
+    """
+    This is the model structure for Notice(s) XPATHs Coverage Report
+    """
+
+    validation_result: Optional[XPATHCoverageValidationResult]
+
+
 class XMLManifestation(Manifestation):
     """
         Original XML Notice manifestation as published on the TED website.
     """
+    xpath_coverage_validation: XPATHCoverageValidationReport = None
+
+    def add_validation(self, validation: Union[XPATHCoverageValidationReport]):
+        if type(validation) == XPATHCoverageValidationReport:
+            self.xpath_coverage_validation: XPATHCoverageValidationReport = validation
+
+    def is_validated(self) -> bool:
+        if self.xpath_coverage_validation:
+            return True
+        return False
 
 
 class METSManifestation(Manifestation):
@@ -54,13 +110,13 @@ class METSManifestation(Manifestation):
     """
 
 
-class RDFValidationManifestation(Manifestation):
+class RDFValidationManifestation(ValidationManifestation):
     """
-        The validation report
+        The RDF validation report
+
     """
-    created: str = datetime.now().isoformat()
-    test_suite_identifier: str
     mapping_suite_identifier: str
+    test_suite_identifier: Optional[str]
 
 
 class SPARQLQuery(PropertyBaseModel):
@@ -128,3 +184,67 @@ def is_validated(self) -> bool:
         if len(self.shacl_validations) and len(self.sparql_validations):
             return True
         return False
+
+
+class XPATHCoverageSummaryResult(PropertyBaseModel):
+    xpath_covered: Optional[int] = 0
+    xpath_not_covered: Optional[int] = 0
+
+
+class XPATHCoverageSummaryReport(PropertyBaseModel):
+    mapping_suite_identifier: Optional[str]
+    validation_result: Optional[XPATHCoverageSummaryResult] = XPATHCoverageSummaryResult()
+
+
+class XMLManifestationValidationSummaryReport(PropertyBaseModel):
+    xpath_coverage_summary: Optional[XPATHCoverageSummaryReport] = XPATHCoverageSummaryReport()
+
+
+class SPARQLSummaryCountReport(PropertyBaseModel):
+    success: Optional[int] = 0
+    fail: Optional[int] = 0
+    error: Optional[int] = 0
+
+
+class SPARQLSummaryResult(PropertyBaseModel):
+    test_suite_identifier: Optional[str]
+    mapping_suite_identifier: Optional[str]
+    aggregate: Optional[SPARQLSummaryCountReport] = SPARQLSummaryCountReport()
+
+
+class SPARQLSummaryReport(PropertyBaseModel):
+    validation_results: Optional[List[SPARQLSummaryResult]] = []
+    aggregate: Optional[SPARQLSummaryCountReport] = SPARQLSummaryCountReport()
+
+
+class SHACLSummarySeverityCountReport(PropertyBaseModel):
+    info: Optional[int] = 0
+    warning: Optional[int] = 0
+    violation: Optional[int] = 0
+
+
+class SHACLSummaryResultSeverityReport(PropertyBaseModel):
+    aggregate: Optional[SHACLSummarySeverityCountReport] = SHACLSummarySeverityCountReport()
+
+
+class SHACLSummaryResult(PropertyBaseModel):
+    test_suite_identifier: Optional[str]
+    mapping_suite_identifier: Optional[str]
+    result_severity: Optional[SHACLSummaryResultSeverityReport] = SHACLSummaryResultSeverityReport()
+
+
+class SHACLSummaryReport(PropertyBaseModel):
+    validation_results: Optional[List[SHACLSummaryResult]] = []
+    result_severity: Optional[SHACLSummaryResultSeverityReport] = SHACLSummaryResultSeverityReport()
+
+
+class RDFManifestationValidationSummaryReport(PropertyBaseModel):
+    sparql_summary: Optional[SPARQLSummaryReport] = SPARQLSummaryReport()
+    shacl_summary: Optional[SHACLSummaryReport] = SHACLSummaryReport()
+
+
+class ValidationSummaryReport(ValidationManifestation):
+    xml_manifestation: Optional[XMLManifestationValidationSummaryReport] = XMLManifestationValidationSummaryReport()
+    rdf_manifestation: Optional[RDFManifestationValidationSummaryReport] = RDFManifestationValidationSummaryReport()
+    distilled_rdf_manifestation: Optional[
+        RDFManifestationValidationSummaryReport] = RDFManifestationValidationSummaryReport()

ted_sws/core/model/notice.py

@@ -22,7 +22,8 @@
 
 from ted_sws.core.model import PropertyBaseModel
 from ted_sws.core.model.manifestation import METSManifestation, RDFManifestation, XMLManifestation, \
-    RDFValidationManifestation, SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport
+    RDFValidationManifestation, SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport, \
+    XPATHCoverageValidationReport, XMLValidationManifestation, ValidationSummaryReport
 from ted_sws.core.model.metadata import TEDMetadata, NormalisedMetadata, XMLMetadata
 
 
@@ -69,7 +70,7 @@ def __str__(self):
 
 #  possible downstream transitions
 NOTICE_STATUS_DOWNSTREAM_TRANSITION = {NoticeStatus.RAW: [NoticeStatus.INDEXED],
-                                       NoticeStatus.INDEXED : [NoticeStatus.NORMALISED_METADATA],
+                                       NoticeStatus.INDEXED: [NoticeStatus.NORMALISED_METADATA],
                                        NoticeStatus.NORMALISED_METADATA: [NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION,
                                                                           NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION],
                                        NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION: [
@@ -162,6 +163,7 @@ class Notice(WorkExpression):
     _rdf_manifestation: Optional[RDFManifestation] = None
     _mets_manifestation: Optional[METSManifestation] = None
     xml_metadata: Optional[XMLMetadata] = None
+    validation_summary: Optional[ValidationSummaryReport] = None
 
     @property
     def preprocessed_xml_manifestation(self) -> XMLManifestation:
@@ -201,9 +203,13 @@ def get_distilled_rdf_validation(self) -> Optional[List[RDFValidationManifestati
             result.append(shacl_validation)
         for sparql_validation in self.distilled_rdf_manifestation.sparql_validations:
             result.append(sparql_validation)
-
         return result
 
+    def get_xml_validation(self) -> Optional[List[XMLValidationManifestation]]:
+        result = []
+        if self.xml_manifestation.xpath_coverage_validation:
+            result.append(self.xml_manifestation.xpath_coverage_validation)
+        return result
 
     def set_xml_metadata(self, xml_metadata: XMLMetadata):
         """
@@ -269,13 +275,14 @@ def _check_status_is_validated(self) -> bool:
 
         :return:
         """
-        if self._rdf_manifestation and self._distilled_rdf_manifestation:
-            if self._rdf_manifestation.is_validated() and self._distilled_rdf_manifestation.is_validated():
+        if self._rdf_manifestation and self._distilled_rdf_manifestation and self.xml_manifestation:
+            if self._rdf_manifestation.is_validated() and self._distilled_rdf_manifestation.is_validated() \
+                    and self.xml_manifestation.is_validated():
                 return True
         return False
 
-    def set_rdf_validation(self,
-                           rdf_validation: Union[SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
+    def set_rdf_validation(self, rdf_validation: Union[SPARQLTestSuiteValidationReport,
+                                                       SHACLTestSuiteValidationReport]):
         """
             Add an RDF validation result to the notice.
             If METS package data are available, erase them and reset the state.
@@ -290,8 +297,8 @@ def set_rdf_validation(self,
         if self._check_status_is_validated():
             self.update_status_to(NoticeStatus.VALIDATED)
 
-    def set_distilled_rdf_validation(self, rdf_validation: Union[
-        SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
+    def set_distilled_rdf_validation(self, rdf_validation: Union[SPARQLTestSuiteValidationReport,
+                                                                 SHACLTestSuiteValidationReport]):
         """
 
         :param rdf_validation:
@@ -305,6 +312,16 @@ def set_distilled_rdf_validation(self, rdf_validation: Union[
         if self._check_status_is_validated():
             self.update_status_to(NoticeStatus.VALIDATED)
 
+    def set_xml_validation(self, xml_validation: Union[XPATHCoverageValidationReport]):
+        """
+            Add an XML validation result to the notice.
+        :param xml_validation:
+        :return:
+        """
+        self.xml_manifestation.add_validation(validation=xml_validation)
+        if self._check_status_is_validated():
+            self.update_status_to(NoticeStatus.VALIDATED)
+
     def set_mets_manifestation(self, mets_manifestation: METSManifestation):
         """
             Add a METS package manifestation to the notice.

ted_sws/core/model/transform.py

@@ -65,6 +65,23 @@ class TransformationTestData(MappingSuiteComponent):
     test_data: List[FileResource]
 
 
+class ConceptualMappingXPATH(MappingSuiteComponent):
+    xpath: str
+    name: str
+
+
+class ConceptualMappingMetadata(MappingSuiteComponent):
+    base_xpath: Optional[str]
+
+
+class ConceptualMapping(MappingSuiteComponent):
+    """
+
+    """
+    xpaths: List[ConceptualMappingXPATH] = []
+    metadata: Optional[ConceptualMappingMetadata]
+
+
 class MappingSuite(MappingSuiteComponent):
     """
 
@@ -81,3 +98,4 @@ class MappingSuite(MappingSuiteComponent):
     shacl_test_suites: List[SHACLTestSuite]
     sparql_test_suites: List[SPARQLTestSuite]
     transformation_test_data: TransformationTestData
+    conceptual_mapping: Optional[ConceptualMapping]

ted_sws/data_manager/adapters/mapping_suite_repository.py

@@ -8,8 +8,10 @@
 
 from ted_sws import config
 from ted_sws.core.model.transform import MappingSuite, FileResource, TransformationRuleSet, SHACLTestSuite, \
-    SPARQLTestSuite, MetadataConstraints, TransformationTestData
+    SPARQLTestSuite, MetadataConstraints, TransformationTestData, ConceptualMapping
 from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC
+from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import CONCEPTUAL_MAPPINGS_FILE_NAME, \
+    mapping_suite_read_conceptual_mapping
 
 METADATA_FILE_NAME = "metadata.json"
 TRANSFORM_PACKAGE_NAME = "transformation"
@@ -141,6 +143,12 @@ def _read_sparql_test_suites(self, package_path: pathlib.Path) -> List[SPARQLTes
                                 sparql_tests=self._read_file_resources(path=sparql_test_suite_path))
                 for sparql_test_suite_path in sparql_test_suite_paths]
 
+    @classmethod
+    def _read_conceptual_mapping(cls, package_path: pathlib.Path) -> ConceptualMapping:
+        return mapping_suite_read_conceptual_mapping(
+            package_path / TRANSFORM_PACKAGE_NAME / CONCEPTUAL_MAPPINGS_FILE_NAME
+        )
+
     def _write_package_metadata(self, mapping_suite: MappingSuite):
         """
             This method creates the metadata of a package based on the metadata in the mapping_suite.
@@ -297,6 +305,7 @@ def _read_mapping_suite_package(self, mapping_suite_identifier: str) -> Optional
             package_metadata["shacl_test_suites"] = self._read_shacl_test_suites(package_path)
             package_metadata["sparql_test_suites"] = self._read_sparql_test_suites(package_path)
             package_metadata["transformation_test_data"] = self._read_test_data_package(package_path)
+            package_metadata["conceptual_mapping"] = self._read_conceptual_mapping(package_path)
             return MappingSuite(**package_metadata)
         return None
 

ted_sws/data_sampler/services/notice_xml_indexer.py

@@ -175,7 +175,9 @@ def get_unique_xpaths_covered_by_notices(notice_ids: List[str], mongodb_client:
     results = notice_repository.collection.aggregate([{"$match": {"ted_id": {"$in": notice_ids}}}], allowDiskUse=True)
     unique_xpaths = set()
     for result in results:
-        unique_xpaths.update(result["xml_metadata"]["unique_xpaths"])
+        xml_metadata = result["xml_metadata"]
+        if xml_metadata is not None:
+            unique_xpaths.update(result["xml_metadata"]["unique_xpaths"])
     return list(unique_xpaths)
 
 

ted_sws/mapping_suite_processor/__init__.py

@@ -1 +1,4 @@
 CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME = "Metadata"
+CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME = "Rules"
+RULES_E_FORM_BT_NAME = 'eForm BT Name (O)'
+RULES_FIELD_XPATH = 'Field XPath (M)'

ted_sws/mapping_suite_processor/entrypoints/cli/__init__.py

@@ -1,3 +1,5 @@
 from ted_sws.data_manager.adapters.mapping_suite_repository import TRANSFORM_PACKAGE_NAME
+from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import CONCEPTUAL_MAPPINGS_FILE_NAME
 
-CONCEPTUAL_MAPPINGS_FILE = '{mappings_path}/{mapping_suite_id}/' + TRANSFORM_PACKAGE_NAME + '/conceptual_mappings.xlsx'
+CONCEPTUAL_MAPPINGS_FILE_TEMPLATE = '{mappings_path}/{mapping_suite_id}/' + TRANSFORM_PACKAGE_NAME + '/' \
+                           + CONCEPTUAL_MAPPINGS_FILE_NAME