Merge branch 'main' into feature/TED-405

Author: Dragos0000

ted_sws/core/model/manifestation.py

@@ -6,10 +6,11 @@
 # Email: costezki.eugen@gmail.com 
 
 """ """
+from datetime import datetime
 from enum import Enum
-from typing import List
+from typing import List, Union, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 from ted_sws.core.model import PropertyBaseModel
 
@@ -57,10 +58,73 @@ class RDFValidationManifestation(Manifestation):
     """
         The validation report
     """
+    created: str = datetime.now().isoformat()
+    test_suite_identifier: str
+    mapping_suite_identifier: str
+
+
+class SPARQLQuery(PropertyBaseModel):
+    """
+    Stores SPARQL query details
+    """
+    title: Optional[str]
+    description: Optional[str]
+    query: str
+
+
+class SPARQLQueryResult(PropertyBaseModel):
+    """
+    Stores SPARQL query execution result
+    """
+    query: SPARQLQuery
+    result: Optional[str]
+    error: Optional[str]
+    identifier: Optional[str]
+
+
+class SPARQLTestSuiteValidationReport(RDFValidationManifestation):
+    """
+    Stores execution results for a SPARQL test suite
+    """
+    validation_results: Union[List[SPARQLQueryResult], str]
+
+
+class QueriedSHACLShapeValidationResult(PropertyBaseModel):
+    """
+    Queried SHACL Validation Report which contains the following variables
+    ?focusNode ?message ?resultPath ?resultSeverity ?sourceConstraintComponent ?sourceShape ?value
+    """
+    conforms: Optional[str]
+    results_dict: Optional[dict]
+    error: Optional[str]
+    identifier: Optional[str]
+
+
+class SHACLTestSuiteValidationReport(RDFValidationManifestation):
+    """
+    This is validation report for a SHACL test suite that contains json and html representation
+    """
+    validation_results: Union[QueriedSHACLShapeValidationResult, str]
 
 
 class RDFManifestation(Manifestation):
     """
         Transformed manifestation in RDF format
     """
-    validation: List[RDFValidationManifestation] = []
+    shacl_validations: List[SHACLTestSuiteValidationReport] = []
+    sparql_validations: List[SPARQLTestSuiteValidationReport] = []
+
+    def add_validation(self, validation: Union[SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
+        if type(validation) == SHACLTestSuiteValidationReport:
+            shacl_validation: SHACLTestSuiteValidationReport = validation
+            if shacl_validation not in self.shacl_validations:
+                self.shacl_validations.append(shacl_validation)
+        elif type(validation) == SPARQLTestSuiteValidationReport:
+            sparql_validation: SPARQLTestSuiteValidationReport = validation
+            if sparql_validation not in self.sparql_validations:
+                self.sparql_validations.append(sparql_validation)
+
+    def is_validated(self) -> bool:
+        if len(self.shacl_validations) and len(self.sparql_validations):
+            return True
+        return False

ted_sws/core/model/notice.py

@@ -16,13 +16,13 @@
 from datetime import datetime
 from enum import IntEnum
 from functools import total_ordering
-from typing import Optional, List
+from typing import Optional, List, Union
 
 from pydantic import Field
 
 from ted_sws.core.model import PropertyBaseModel
 from ted_sws.core.model.manifestation import METSManifestation, RDFManifestation, XMLManifestation, \
-    RDFValidationManifestation
+    RDFValidationManifestation, SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport
 from ted_sws.core.model.metadata import TEDMetadata, NormalisedMetadata, XMLMetadata
 
 
@@ -184,13 +184,23 @@ def mets_manifestation(self) -> METSManifestation:
     def get_rdf_validation(self) -> Optional[List[RDFValidationManifestation]]:
         if not self.rdf_manifestation:
             return None
-
-        return self.rdf_manifestation.validation
+        result = []
+        for shacl_validation in self.rdf_manifestation.shacl_validations:
+            result.append(shacl_validation)
+        for sparql_validation in self.rdf_manifestation.sparql_validations:
+            result.append(sparql_validation)
+        return result
 
     def get_distilled_rdf_validation(self) -> Optional[List[RDFValidationManifestation]]:
         if not self.distilled_rdf_manifestation:
             return None
-        return self.distilled_rdf_manifestation.validation
+        result = []
+        for shacl_validation in self.distilled_rdf_manifestation.shacl_validations:
+            result.append(shacl_validation)
+        for sparql_validation in self.distilled_rdf_manifestation.sparql_validations:
+            result.append(sparql_validation)
+
+        return result
 
     def set_preprocessed_xml_manifestation(self, preprocessed_xml_manifestation: XMLManifestation):
         """
@@ -237,25 +247,23 @@ def set_rdf_manifestation(self, rdf_manifestation: RDFManifestation):
         if self.rdf_manifestation == rdf_manifestation:
             return
         self._rdf_manifestation = rdf_manifestation
-        if not rdf_manifestation.validation:
+        if (not rdf_manifestation.sparql_validations) and (not rdf_manifestation.shacl_validations):
             self.update_status_to(NoticeStatus.TRANSFORMED)
         else:
             self.update_status_to(NoticeStatus.VALIDATED)
 
     def _check_status_is_validated(self) -> bool:
+        """
 
-        def _get_number_of_validation_types(validations: List[RDFValidationManifestation]):
-            unique_validation_types = set([type(validation).__name__ for validation in validations])
-            return len(unique_validation_types)
-
+        :return:
+        """
         if self._rdf_manifestation and self._distilled_rdf_manifestation:
-            if self._rdf_manifestation.validation and self._distilled_rdf_manifestation.validation:
-                if (_get_number_of_validation_types(self._rdf_manifestation.validation) > 1) and (
-                        _get_number_of_validation_types(self._distilled_rdf_manifestation.validation) > 1):
-                    return True
+            if self._rdf_manifestation.is_validated() and self._distilled_rdf_manifestation.is_validated():
+                return True
         return False
 
-    def set_rdf_validation(self, rdf_validation: RDFValidationManifestation):
+    def set_rdf_validation(self,
+                           rdf_validation: Union[SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
         """
             Add an RDF validation result to the notice.
             If METS package data are available, erase them and reset the state.
@@ -265,23 +273,23 @@ def set_rdf_validation(self, rdf_validation: RDFValidationManifestation):
         if not self.rdf_manifestation:
             raise ValueError("Cannot set the RDF validation of a non-existent RDF manifestation")
 
-        for validation in self._rdf_manifestation.validation:
-            if validation == rdf_validation:
-                return
+        self._rdf_manifestation.add_validation(validation=rdf_validation)
 
-        self._rdf_manifestation.validation.append(rdf_validation)
         if self._check_status_is_validated():
             self.update_status_to(NoticeStatus.VALIDATED)
 
-    def set_distilled_rdf_validation(self, rdf_validation: RDFValidationManifestation):
-        if not self.distilled_rdf_manifestation:
+    def set_distilled_rdf_validation(self, rdf_validation: Union[
+        SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
+        """
+
+        :param rdf_validation:
+        :return:
+        """
+        if not self._distilled_rdf_manifestation:
             raise ValueError("Cannot set the RDF validation of a non-existent RDF manifestation")
 
-        for validation in self._distilled_rdf_manifestation.validation:
-            if validation == rdf_validation:
-                return
+        self._distilled_rdf_manifestation.add_validation(validation=rdf_validation)
 
-        self._distilled_rdf_manifestation.validation.append(rdf_validation)
         if self._check_status_is_validated():
             self.update_status_to(NoticeStatus.VALIDATED)
 

ted_sws/core/model/validate.py

@@ -1,8 +0,0 @@
-#!/usr/bin/python3
-
-# validation.py
-# Date:  29/01/2022
-# Author: Eugeniu Costetchi
-# Email: costezki.eugen@gmail.com 
-
-""" """

ted_sws/data_manager/adapters/notice_repository.py

@@ -78,11 +78,17 @@ def write_large_field(large_field: Manifestation):
         write_large_field(notice.distilled_rdf_manifestation)
         write_large_field(notice.preprocessed_xml_manifestation)
         if notice.rdf_manifestation:
-            for validation_report in notice.rdf_manifestation.validation:
+            for validation_report in notice.rdf_manifestation.shacl_validations:
+                write_large_field(validation_report)
+
+            for validation_report in notice.rdf_manifestation.sparql_validations:
                 write_large_field(validation_report)
 
         if notice.distilled_rdf_manifestation:
-            for validation_report in notice.distilled_rdf_manifestation.validation:
+            for validation_report in notice.distilled_rdf_manifestation.shacl_validations:
+                write_large_field(validation_report)
+
+            for validation_report in notice.distilled_rdf_manifestation.sparql_validations:
                 write_large_field(validation_report)
 
         return notice
@@ -104,10 +110,16 @@ def load_large_field(large_field: Manifestation):
         load_large_field(large_field=notice.distilled_rdf_manifestation)
         load_large_field(large_field=notice.preprocessed_xml_manifestation)
         if notice.rdf_manifestation:
-            for validation_report in notice.rdf_manifestation.validation:
+            for validation_report in notice.rdf_manifestation.shacl_validations:
                 load_large_field(validation_report)
+            for validation_report in notice.rdf_manifestation.sparql_validations:
+                load_large_field(validation_report)
+
         if notice.distilled_rdf_manifestation:
-            for validation_report in notice.distilled_rdf_manifestation.validation:
+            for validation_report in notice.distilled_rdf_manifestation.shacl_validations:
+                load_large_field(validation_report)
+
+            for validation_report in notice.distilled_rdf_manifestation.sparql_validations:
                 load_large_field(validation_report)
 
         return notice

ted_sws/mapping_suite_processor/services/conceptual_mapping_processor.py

@@ -56,6 +56,7 @@ def mapping_suite_processor_zip_package(mapping_suite_package_path: pathlib.Path
 def mapping_suite_processor_expand_package(mapping_suite_package_path: pathlib.Path):
     """
         This function reads data from conceptual_mappings.xlsx and expand provided package.
+                Note: we don't use this at  the moment
     :param mapping_suite_package_path:
     :return:
     """
@@ -149,7 +150,6 @@ def mapping_suite_processor_from_github_expand_and_load_package_in_mongo_db(mapp
         mapping_suite_package_downloader.download(mapping_suite_package_name=mapping_suite_package_name,
                                                   output_mapping_suite_package_path=tmp_dir_path)
         mapping_suite_package_path = tmp_dir_path / mapping_suite_package_name
-        mapping_suite_processor_expand_package(mapping_suite_package_path=mapping_suite_package_path)
         mapping_suite_processor_load_package_in_mongo_db(mapping_suite_package_path=mapping_suite_package_path,
                                                          mongodb_client=mongodb_client,
                                                          load_test_data=load_test_data

ted_sws/notice_validator/entrypoints/cli/cmd_shacl_runner.py

@@ -5,11 +5,10 @@
 import click
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH, DEFAULT_OUTPUT_PATH
-from ted_sws.core.model.manifestation import RDFManifestation
+from ted_sws.core.model.manifestation import RDFManifestation, SHACLTestSuiteValidationReport
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
-from ted_sws.notice_validator.model.shacl_test_suite import SHACLSuiteValidationReport
-from ted_sws.notice_validator.services.shacl_test_suite_runner import SHACLTestSuiteRunner, SHACLReportBuilder
+from ted_sws.notice_validator.services.shacl_test_suite_runner import SHACLTestSuiteRunner, generate_shacl_report
 
 DEFAULT_RDF_FOLDER = '{mappings_path}/{mapping_suite_id}/' + DEFAULT_OUTPUT_PATH
 DEFAULT_TEST_SUITE_REPORT_FOLDER = "test_suite_report"
@@ -57,8 +56,7 @@ def validate(self, rdf_file, base_report_path):
                                                         shacl_test_suite=shacl_test_suite,
                                                         mapping_suite=self.mapping_suite).execute_test_suite()
 
-            report_builder = SHACLReportBuilder(shacl_test_suite_execution=test_suite_execution)
-            report: SHACLSuiteValidationReport = report_builder.generate_report()
+            report: SHACLTestSuiteValidationReport = generate_shacl_report(shacl_test_suite_execution=test_suite_execution)
 
             suite_id = shacl_test_suite.identifier
             data = report.object_data

ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py

@@ -6,7 +6,7 @@
 import click
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH, DEFAULT_OUTPUT_PATH
-from ted_sws.core.model.manifestation import RDFManifestation, RDFValidationManifestation
+from ted_sws.core.model.manifestation import RDFManifestation
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
 from ted_sws.notice_validator.services.sparql_test_suite_runner import SPARQLTestSuiteRunner, SPARQLReportBuilder