main merge

Author: Dragos0000

ted_sws/core/model/manifestation.py

@@ -6,10 +6,11 @@
 # Email: costezki.eugen@gmail.com 
 
 """ """
+from datetime import datetime
 from enum import Enum
-from typing import List
+from typing import List, Union, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 from ted_sws.core.model import PropertyBaseModel
 
@@ -57,10 +58,73 @@ class RDFValidationManifestation(Manifestation):
     """
         The validation report
     """
+    created: str = datetime.now().isoformat()
+    test_suite_identifier: str
+    mapping_suite_identifier: str
+
+
+class SPARQLQuery(PropertyBaseModel):
+    """
+    Stores SPARQL query details
+    """
+    title: Optional[str]
+    description: Optional[str]
+    query: str
+
+
+class SPARQLQueryResult(PropertyBaseModel):
+    """
+    Stores SPARQL query execution result
+    """
+    query: SPARQLQuery
+    result: Optional[str]
+    error: Optional[str]
+    identifier: Optional[str]
+
+
+class SPARQLTestSuiteValidationReport(RDFValidationManifestation):
+    """
+    Stores execution results for a SPARQL test suite
+    """
+    validation_results: Union[List[SPARQLQueryResult], str]
+
+
+class QueriedSHACLShapeValidationResult(PropertyBaseModel):
+    """
+    Queried SHACL Validation Report which contains the following variables
+    ?focusNode ?message ?resultPath ?resultSeverity ?sourceConstraintComponent ?sourceShape ?value
+    """
+    conforms: Optional[str]
+    results_dict: Optional[dict]
+    error: Optional[str]
+    identifier: Optional[str]
+
+
+class SHACLTestSuiteValidationReport(RDFValidationManifestation):
+    """
+    This is validation report for a SHACL test suite that contains json and html representation
+    """
+    validation_results: Union[QueriedSHACLShapeValidationResult, str]
 
 
 class RDFManifestation(Manifestation):
     """
         Transformed manifestation in RDF format
     """
-    validation: List[RDFValidationManifestation] = []
+    shacl_validations: List[SHACLTestSuiteValidationReport] = []
+    sparql_validations: List[SPARQLTestSuiteValidationReport] = []
+
+    def add_validation(self, validation: Union[SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
+        if type(validation) == SHACLTestSuiteValidationReport:
+            shacl_validation: SHACLTestSuiteValidationReport = validation
+            if shacl_validation not in self.shacl_validations:
+                self.shacl_validations.append(shacl_validation)
+        elif type(validation) == SPARQLTestSuiteValidationReport:
+            sparql_validation: SPARQLTestSuiteValidationReport = validation
+            if sparql_validation not in self.sparql_validations:
+                self.sparql_validations.append(sparql_validation)
+
+    def is_validated(self) -> bool:
+        if len(self.shacl_validations) and len(self.sparql_validations):
+            return True
+        return False

ted_sws/core/model/notice.py

@@ -16,13 +16,13 @@
 from datetime import datetime
 from enum import IntEnum
 from functools import total_ordering
-from typing import Optional, List
+from typing import Optional, List, Union
 
 from pydantic import Field
 
 from ted_sws.core.model import PropertyBaseModel
 from ted_sws.core.model.manifestation import METSManifestation, RDFManifestation, XMLManifestation, \
-    RDFValidationManifestation
+    RDFValidationManifestation, SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport
 from ted_sws.core.model.metadata import TEDMetadata, NormalisedMetadata, XMLMetadata
 
 
@@ -184,13 +184,23 @@ def mets_manifestation(self) -> METSManifestation:
     def get_rdf_validation(self) -> Optional[List[RDFValidationManifestation]]:
         if not self.rdf_manifestation:
             return None
-
-        return self.rdf_manifestation.validation
+        result = []
+        for shacl_validation in self.rdf_manifestation.shacl_validations:
+            result.append(shacl_validation)
+        for sparql_validation in self.rdf_manifestation.sparql_validations:
+            result.append(sparql_validation)
+        return result
 
     def get_distilled_rdf_validation(self) -> Optional[List[RDFValidationManifestation]]:
         if not self.distilled_rdf_manifestation:
             return None
-        return self.distilled_rdf_manifestation.validation
+        result = []
+        for shacl_validation in self.distilled_rdf_manifestation.shacl_validations:
+            result.append(shacl_validation)
+        for sparql_validation in self.distilled_rdf_manifestation.sparql_validations:
+            result.append(sparql_validation)
+
+        return result
 
     def set_preprocessed_xml_manifestation(self, preprocessed_xml_manifestation: XMLManifestation):
         """
@@ -237,25 +247,23 @@ def set_rdf_manifestation(self, rdf_manifestation: RDFManifestation):
         if self.rdf_manifestation == rdf_manifestation:
             return
         self._rdf_manifestation = rdf_manifestation
-        if not rdf_manifestation.validation:
+        if (not rdf_manifestation.sparql_validations) and (not rdf_manifestation.shacl_validations):
             self.update_status_to(NoticeStatus.TRANSFORMED)
         else:
             self.update_status_to(NoticeStatus.VALIDATED)
 
     def _check_status_is_validated(self) -> bool:
+        """
 
-        def _get_number_of_validation_types(validations: List[RDFValidationManifestation]):
-            unique_validation_types = set([type(validation).__name__ for validation in validations])
-            return len(unique_validation_types)
-
+        :return:
+        """
         if self._rdf_manifestation and self._distilled_rdf_manifestation:
-            if self._rdf_manifestation.validation and self._distilled_rdf_manifestation.validation:
-                if (_get_number_of_validation_types(self._rdf_manifestation.validation) > 1) and (
-                        _get_number_of_validation_types(self._distilled_rdf_manifestation.validation) > 1):
-                    return True
+            if self._rdf_manifestation.is_validated() and self._distilled_rdf_manifestation.is_validated():
+                return True
         return False
 
-    def set_rdf_validation(self, rdf_validation: RDFValidationManifestation):
+    def set_rdf_validation(self,
+                           rdf_validation: Union[SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
         """
             Add an RDF validation result to the notice.
             If METS package data are available, erase them and reset the state.
@@ -265,23 +273,23 @@ def set_rdf_validation(self, rdf_validation: RDFValidationManifestation):
         if not self.rdf_manifestation:
             raise ValueError("Cannot set the RDF validation of a non-existent RDF manifestation")
 
-        for validation in self._rdf_manifestation.validation:
-            if validation == rdf_validation:
-                return
+        self._rdf_manifestation.add_validation(validation=rdf_validation)
 
-        self._rdf_manifestation.validation.append(rdf_validation)
         if self._check_status_is_validated():
             self.update_status_to(NoticeStatus.VALIDATED)
 
-    def set_distilled_rdf_validation(self, rdf_validation: RDFValidationManifestation):
-        if not self.distilled_rdf_manifestation:
+    def set_distilled_rdf_validation(self, rdf_validation: Union[
+        SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport]):
+        """
+
+        :param rdf_validation:
+        :return:
+        """
+        if not self._distilled_rdf_manifestation:
             raise ValueError("Cannot set the RDF validation of a non-existent RDF manifestation")
 
-        for validation in self._distilled_rdf_manifestation.validation:
-            if validation == rdf_validation:
-                return
+        self._distilled_rdf_manifestation.add_validation(validation=rdf_validation)
 
-        self._distilled_rdf_manifestation.validation.append(rdf_validation)
         if self._check_status_is_validated():
             self.update_status_to(NoticeStatus.VALIDATED)
 

ted_sws/core/model/validate.py

@@ -1,8 +0,0 @@
-#!/usr/bin/python3
-
-# validation.py
-# Date:  29/01/2022
-# Author: Eugeniu Costetchi
-# Email: costezki.eugen@gmail.com 
-
-""" """

ted_sws/data_manager/adapters/notice_repository.py

@@ -76,11 +76,17 @@ def write_large_field(large_field: Manifestation):
         write_large_field(notice.distilled_rdf_manifestation)
         write_large_field(notice.preprocessed_xml_manifestation)
         if notice.rdf_manifestation:
-            for validation_report in notice.rdf_manifestation.validation:
+            for validation_report in notice.rdf_manifestation.shacl_validations:
+                write_large_field(validation_report)
+
+            for validation_report in notice.rdf_manifestation.sparql_validations:
                 write_large_field(validation_report)
 
         if notice.distilled_rdf_manifestation:
-            for validation_report in notice.distilled_rdf_manifestation.validation:
+            for validation_report in notice.distilled_rdf_manifestation.shacl_validations:
+                write_large_field(validation_report)
+
+            for validation_report in notice.distilled_rdf_manifestation.sparql_validations:
                 write_large_field(validation_report)
 
         return notice
@@ -102,10 +108,16 @@ def load_large_field(large_field: Manifestation):
         load_large_field(large_field=notice.distilled_rdf_manifestation)
         load_large_field(large_field=notice.preprocessed_xml_manifestation)
         if notice.rdf_manifestation:
-            for validation_report in notice.rdf_manifestation.validation:
+            for validation_report in notice.rdf_manifestation.shacl_validations:
                 load_large_field(validation_report)
+            for validation_report in notice.rdf_manifestation.sparql_validations:
+                load_large_field(validation_report)
+
         if notice.distilled_rdf_manifestation:
-            for validation_report in notice.distilled_rdf_manifestation.validation:
+            for validation_report in notice.distilled_rdf_manifestation.shacl_validations:
+                load_large_field(validation_report)
+
+            for validation_report in notice.distilled_rdf_manifestation.sparql_validations:
                 load_large_field(validation_report)
 
         return notice

ted_sws/notice_validator/entrypoints/cli/cmd_shacl_runner.py

@@ -5,11 +5,10 @@
 import click
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH, DEFAULT_OUTPUT_PATH
-from ted_sws.core.model.manifestation import RDFManifestation
+from ted_sws.core.model.manifestation import RDFManifestation, SHACLTestSuiteValidationReport
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
-from ted_sws.notice_validator.model.shacl_test_suite import SHACLSuiteValidationReport
-from ted_sws.notice_validator.services.shacl_test_suite_runner import SHACLTestSuiteRunner, SHACLReportBuilder
+from ted_sws.notice_validator.services.shacl_test_suite_runner import SHACLTestSuiteRunner, generate_shacl_report
 
 DEFAULT_RDF_FOLDER = '{mappings_path}/{mapping_suite_id}/' + DEFAULT_OUTPUT_PATH
 DEFAULT_TEST_SUITE_REPORT_FOLDER = "test_suite_report"
@@ -57,8 +56,7 @@ def validate(self, rdf_file, base_report_path):
                                                         shacl_test_suite=shacl_test_suite,
                                                         mapping_suite=self.mapping_suite).execute_test_suite()
 
-            report_builder = SHACLReportBuilder(shacl_test_suite_execution=test_suite_execution)
-            report: SHACLSuiteValidationReport = report_builder.generate_report()
+            report: SHACLTestSuiteValidationReport = generate_shacl_report(shacl_test_suite_execution=test_suite_execution)
 
             suite_id = shacl_test_suite.identifier
             data = report.object_data

ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py

@@ -6,7 +6,7 @@
 import click
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH, DEFAULT_OUTPUT_PATH
-from ted_sws.core.model.manifestation import RDFManifestation, RDFValidationManifestation
+from ted_sws.core.model.manifestation import RDFManifestation
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
 from ted_sws.notice_validator.services.sparql_test_suite_runner import SPARQLTestSuiteRunner, SPARQLReportBuilder

ted_sws/notice_validator/model/__init__.py

@@ -29,7 +29,7 @@
 <h2>Report details: </h2>
 <ul>
     <li>Date created: {{ created }}</li>
-    <li>SHACL test suite identifier: {{ shacl_test_suite_identifier }}</li>
+    <li>SHACL test suite identifier: {{ test_suite_identifier }}</li>
     <li>Mapping suite identifier: {{ mapping_suite_identifier }}</li>
 </ul>
 <hr>
@@ -44,18 +44,18 @@
     </thead>
     <tbody>
     <tr>
-        <td>{{ validation_result.identifier }}</td>
-        <td{% if validation_result.conforms == "True" %}
-            class="success"{% endif %}>{{ validation_result.conforms }}</td>
-        <td>{{ validation_result.error }}</td>
+        <td>{{ validation_results.identifier }}</td>
+        <td{% if validation_results.conforms == "True" %}
+            class="success"{% endif %}>{{ validation_results.conforms }}</td>
+        <td>{{ validation_results.error }}</td>
     </tr>
     </tbody>
 </table>
 
-{% if validation_result.results_dict %}
+{% if validation_results.results_dict %}
     <hr>
     <h2>Extended results</h2>
-    <h1>{{ validation_result.identifier }}</h1>
+    <h1>{{ validation_results.identifier }}</h1>
     <table class="display">
      <thead class="center aligned">
         <tr>
@@ -67,7 +67,7 @@
         </tr>
         </thead>
         <tbody>
-        {% for biding in validation_result.results_dict.results.bindings %}
+        {% for biding in validation_results.results_dict.results.bindings %}
             <tr>
                 <td>{{ biding.focusNode.value }}</td>
                 <td>{{ biding.resultPath.value }}</td>