CLI-Tool for Validation Summary (File based)

Author: Kolea Plesco

ted_sws/core/model/manifestation.py

@@ -60,6 +60,8 @@ class XPATHCoverageValidationAssertion(PropertyBaseModel):
     """
 
     """
+    standard_form_field_id: Optional[str]
+    eform_bt_id: Optional[str]
     title: Optional[str]
     xpath: Optional[str]
     count: Optional[int]

ted_sws/core/model/transform.py

@@ -67,7 +67,9 @@ class TransformationTestData(MappingSuiteComponent):
 
 class ConceptualMappingXPATH(MappingSuiteComponent):
     xpath: str
-    name: str
+    name: Optional[str]
+    standard_form_field_id: Optional[str]
+    eform_bt_id: Optional[str]
 
 
 class ConceptualMappingMetadata(MappingSuiteComponent):

ted_sws/mapping_suite_processor/__init__.py

@@ -1,4 +1,6 @@
 CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME = "Metadata"
 CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME = "Rules"
 RULES_E_FORM_BT_NAME = 'eForm BT Name (O)'
-RULES_FIELD_XPATH = 'Field XPath (M)'
+RULES_FIELD_XPATH = 'Field XPath (M)'
+RULES_SF_FIELD_ID = 'Standard Form Field ID (M)'
+RULES_E_FORM_BT_ID = 'eForm BT-ID (O)'

ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_sparql_queries.py

@@ -5,14 +5,11 @@
 import pandas as pd
 
 from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME, \
-    CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME
+    CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME, RULES_SF_FIELD_ID, RULES_E_FORM_BT_ID
 from ted_sws.notice_validator import BASE_XPATH_FIELD
 from ted_sws.resources.prefixes import PREFIXES_DEFINITIONS
 
-
-RULES_SF_FIELD_ID = 'Standard Form Field ID (M)'
 RULES_SF_FIELD_NAME = 'Standard Form Field Name (M)'
-RULES_E_FORM_BT_ID = 'eForm BT-ID (O)'
 RULES_CLASS_PATH = 'Class path (M)'
 RULES_PROPERTY_PATH = 'Property path (M)'
 

ted_sws/mapping_suite_processor/services/conceptual_mapping_reader.py

@@ -6,7 +6,7 @@
 
 from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingXPATH, ConceptualMappingMetadata
 from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME, \
-    CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME
+    CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME, RULES_SF_FIELD_ID, RULES_E_FORM_BT_ID
 from ted_sws.notice_validator import BASE_XPATH_FIELD
 
 CONCEPTUAL_MAPPINGS_FILE_NAME = "conceptual_mappings.xlsx"
@@ -46,6 +46,8 @@ def mapping_suite_read_conceptual_mapping(conceptual_mappings_file_path: pathlib
         rules_df = pd.read_excel(excel_file, sheet_name=CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, header=1)
         df_xpaths = rules_df[RULES_FIELD_XPATH].tolist()
         df_bt_names = rules_df[RULES_E_FORM_BT_NAME].tolist()
+        df_eform_bt_ids = rules_df[RULES_E_FORM_BT_ID].tolist()
+        df_sform_field_ids = rules_df[RULES_SF_FIELD_ID].tolist()
         processed_xpaths = set()
         for idx, xpath_row in enumerate(df_xpaths):
             if xpath_row is not np.nan:
@@ -54,8 +56,16 @@ def mapping_suite_read_conceptual_mapping(conceptual_mappings_file_path: pathlib
                     if xpath:
                         xpath = base_xpath + "/" + xpath
                         if xpath not in processed_xpaths:
-                            xpath_name = df_bt_names[idx]
-                            cm_xpath = ConceptualMappingXPATH(xpath=xpath, name=xpath_name)
+                            xpath_name = df_bt_names[idx] if df_bt_names[idx] is not np.nan else None
+                            eform_bt_id = df_eform_bt_ids[idx] if df_eform_bt_ids[idx] is not np.nan else None
+                            sform_field_id = df_sform_field_ids[idx] if df_sform_field_ids[idx] is not np.nan else None
+
+                            cm_xpath: ConceptualMappingXPATH = ConceptualMappingXPATH(
+                                xpath=xpath,
+                                name=xpath_name,
+                                standard_form_field_id=sform_field_id,
+                                eform_bt_id=eform_bt_id
+                            )
                             conceptual_mapping_xpaths.append(cm_xpath)
                             processed_xpaths.add(xpath)
 

ted_sws/notice_validator/adapters/validation_summary_runner.py

@@ -22,63 +22,66 @@ def __init__(self, notices: List[Notice]):
 class RDFManifestationValidationSummaryRunner(ManifestationValidationSummaryRunner):
     @classmethod
     def notice_sparql_summary(cls, notice: Notice, report: RDFManifestationValidationSummaryReport):
-        report_count: SPARQLSummaryCountReport = report.sparql_summary.aggregate
-        result_counts: List[SPARQLSummaryResult] = report.sparql_summary.validation_results
-        sparql_reports: List[SPARQLTestSuiteValidationReport] = notice.rdf_manifestation.sparql_validations
-        if sparql_reports:
-            for sparql_report in sparql_reports:
-                validation_results: List[SPARQLQueryResult] = sparql_report.validation_results
-                result_validation: SPARQLSummaryResult = SPARQLSummaryResult()
-                result_validation.mapping_suite_identifier = sparql_report.mapping_suite_identifier
-                result_validation.test_suite_identifier = sparql_report.test_suite_identifier
-                result_count: SPARQLSummaryCountReport = result_validation.aggregate
-                if validation_results:
-                    for validation in validation_results:
-                        if validation.result == 'True':
-                            report_count.success += 1
-                            result_count.success += 1
-                        else:
-                            report_count.fail += 1
-                            result_count.fail += 1
-                        if validation.error:
-                            report_count.error += 1
-                            result_count.error += 1
-
-                result_counts.append(result_validation)
+        manifestation = cls._manifestation(notice)
+        if manifestation:
+            report_count: SPARQLSummaryCountReport = report.sparql_summary.aggregate
+            result_counts: List[SPARQLSummaryResult] = report.sparql_summary.validation_results
+            sparql_reports: List[SPARQLTestSuiteValidationReport] = manifestation.sparql_validations
+            if sparql_reports:
+                for sparql_report in sparql_reports:
+                    validation_results: List[SPARQLQueryResult] = sparql_report.validation_results
+                    result_validation: SPARQLSummaryResult = SPARQLSummaryResult()
+                    result_validation.mapping_suite_identifier = sparql_report.mapping_suite_identifier
+                    result_validation.test_suite_identifier = sparql_report.test_suite_identifier
+                    result_count: SPARQLSummaryCountReport = result_validation.aggregate
+                    if validation_results:
+                        for validation in validation_results:
+                            if validation.result == 'True':
+                                report_count.success += 1
+                                result_count.success += 1
+                            else:
+                                report_count.fail += 1
+                                result_count.fail += 1
+                            if validation.error:
+                                report_count.error += 1
+                                result_count.error += 1
+
+                    result_counts.append(result_validation)
 
     @classmethod
     def _manifestation(cls, notice: Notice) -> RDFManifestation:
         return notice.rdf_manifestation
 
     @classmethod
     def notice_shacl_summary(cls, notice: Notice, report: RDFManifestationValidationSummaryReport):
-        report_count: SHACLSummarySeverityCountReport = report.shacl_summary.result_severity.aggregate
-        result_counts: List[SHACLSummaryResult] = report.shacl_summary.validation_results
         manifestation = cls._manifestation(notice)
-        shacl_reports: List[SHACLTestSuiteValidationReport] = manifestation.shacl_validations
-        if shacl_reports:
-            for shacl_report in shacl_reports:
-                validation_results = shacl_report.validation_results
-                result_validation: SHACLSummaryResult = SHACLSummaryResult()
-                result_validation.mapping_suite_identifier = shacl_report.mapping_suite_identifier
-                result_validation.test_suite_identifier = shacl_report.test_suite_identifier
-                result_count: SHACLSummarySeverityCountReport = result_validation.result_severity.aggregate
-                if validation_results:
-                    bindings = validation_results.results_dict['results']['bindings']
-                    for binding in bindings:
-                        result_severity = binding['resultSeverity']
-                        if result_severity:
-                            if result_severity['value'].endswith("#Violation"):
-                                report_count.violation += 1
-                                result_count.violation += 1
-                            elif result_severity['value'].endswith("#Info"):
-                                report_count.info += 1
-                                result_count.info += 1
-                            elif result_severity['value'].endswith("#Warning"):
-                                report_count.warning += 1
-                                result_count.warning += 1
-
-                result_counts.append(result_validation)
+        if manifestation:
+            report_count: SHACLSummarySeverityCountReport = report.shacl_summary.result_severity.aggregate
+            result_counts: List[SHACLSummaryResult] = report.shacl_summary.validation_results
+            shacl_reports: List[SHACLTestSuiteValidationReport] = manifestation.shacl_validations
+            if shacl_reports:
+                for shacl_report in shacl_reports:
+                    validation_results = shacl_report.validation_results
+                    result_validation: SHACLSummaryResult = SHACLSummaryResult()
+                    result_validation.mapping_suite_identifier = shacl_report.mapping_suite_identifier
+                    result_validation.test_suite_identifier = shacl_report.test_suite_identifier
+                    result_count: SHACLSummarySeverityCountReport = result_validation.result_severity.aggregate
+                    if validation_results:
+                        bindings = validation_results.results_dict['results']['bindings']
+                        for binding in bindings:
+                            result_severity = binding['resultSeverity']
+                            if result_severity:
+                                if result_severity['value'].endswith("#Violation"):
+                                    report_count.violation += 1
+                                    result_count.violation += 1
+                                elif result_severity['value'].endswith("#Info"):
+                                    report_count.info += 1
+                                    result_count.info += 1
+                                elif result_severity['value'].endswith("#Warning"):
+                                    report_count.warning += 1
+                                    result_count.warning += 1
+
+                    result_counts.append(result_validation)
 
     def validation_summary(self) -> RDFManifestationValidationSummaryReport:
         notices = self.notices

ted_sws/notice_validator/adapters/xpath_coverage_runner.py

@@ -8,12 +8,10 @@
 from ted_sws.core.model.manifestation import XPATHCoverageValidationReport, XPATHCoverageValidationAssertion, \
     XPATHCoverageValidationResult
 from ted_sws.core.model.notice import Notice
-from ted_sws.core.model.transform import ConceptualMapping
-from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, get_unique_xpaths_covered_by_notices
-from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import mapping_suite_read_metadata, \
-    mapping_suite_read_conceptual_mapping
-from ted_sws.notice_validator import BASE_XPATH_FIELD
+from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingXPATH
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
+from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, get_unique_xpaths_covered_by_notices
+from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import mapping_suite_read_conceptual_mapping
 
 TEMPLATES = Environment(loader=PackageLoader("ted_sws.notice_validator.resources", "templates"))
 XPATH_COVERAGE_REPORT_TEMPLATE = "xpath_coverage_report.jinja2"
@@ -28,7 +26,7 @@ class CoverageRunner:
     """
 
     conceptual_xpaths: Set[str] = set()
-    conceptual_xpath_names: Dict[str, str] = {}
+    conceptual_xpath_data: Dict[str, ConceptualMappingXPATH] = {}
     mongodb_client: MongoClient
     base_xpath: str
     mapping_suite_id: str
@@ -51,7 +49,7 @@ def __init__(self, mapping_suite_id: str, conceptual_mappings_file_path: PATH_TY
 
         for cm_xpath in conceptual_mapping.xpaths:
             self.conceptual_xpaths.add(cm_xpath.xpath)
-            self.conceptual_xpath_names[cm_xpath.xpath] = cm_xpath.name
+            self.conceptual_xpath_data[cm_xpath.xpath] = cm_xpath
 
         self.base_xpath = conceptual_mapping.metadata.base_xpath
 
@@ -68,8 +66,11 @@ def xpath_assertions(self, notice_xpaths: XPATH_TYPE,
         xpath_assertions = []
         for xpath in self.conceptual_xpaths:
             xpath_assertion = XPATHCoverageValidationAssertion()
-            title = self.conceptual_xpath_names[xpath]
+            xpath_data = self.conceptual_xpath_data[xpath]
+            title = xpath_data.name
             xpath_assertion.title = title if title is not np.nan else ''
+            xpath_assertion.standard_form_field_id = xpath_data.standard_form_field_id
+            xpath_assertion.eform_bt_id = xpath_data.eform_bt_id
             xpath_assertion.xpath = xpath
             xpath_assertion.count = xpaths_list.count(xpath)
             xpath_assertion.notice_hit = self.find_notice_by_xpath(notice_xpaths, xpath)

ted_sws/notice_validator/entrypoints/cli/cmd_shacl_runner.py

@@ -4,14 +4,18 @@
 
 import click
 
+from typing import List
+
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
 from ted_sws.core.model.manifestation import RDFManifestation, SHACLTestSuiteValidationReport
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
 from ted_sws.notice_validator.entrypoints.cli import DEFAULT_RDF_FOLDER, DEFAULT_TEST_SUITE_REPORT_FOLDER
 from ted_sws.notice_validator.services.shacl_test_suite_runner import SHACLTestSuiteRunner, generate_shacl_report
+import json
 
 HTML_REPORT = "shacl_{id}.html"
+JSON_VALIDATIONS_REPORT = "shacl_validations.json"
 CMD_NAME = "CMD_SHACL_RUNNER"
 
 """
@@ -39,7 +43,9 @@ def __init__(
         self.mapping_suite = mapping_suite_repository.get(reference=self.mapping_suite_id)
 
     def save_report(self, report_path, report_name, report_id, content):
-        with open(report_path / report_name.format(id=report_id), "w+") as f:
+        if report_id is not None:
+            report_name = report_name.format(id=report_id)
+        with open(report_path / report_name, "w+") as f:
             f.write(content)
 
     def validate(self, rdf_file, base_report_path):
@@ -49,6 +55,7 @@ def validate(self, rdf_file, base_report_path):
         report_path = base_report_path / DEFAULT_TEST_SUITE_REPORT_FOLDER
         report_path.mkdir(parents=True, exist_ok=True)
 
+        shacl_validations: List[SHACLTestSuiteValidationReport] = []
         shacl_test_suites = self.mapping_suite.shacl_test_suites
         for shacl_test_suite in shacl_test_suites:
             test_suite_execution = SHACLTestSuiteRunner(rdf_manifestation=rdf_manifestation,
@@ -59,8 +66,12 @@ def validate(self, rdf_file, base_report_path):
                 shacl_test_suite_execution=test_suite_execution)
 
             suite_id = shacl_test_suite.identifier
-            data = report.object_data
-            self.save_report(report_path, HTML_REPORT, suite_id, data)
+            self.save_report(report_path, HTML_REPORT, suite_id, report.object_data)
+            report.object_data = "SHACLTestSuiteValidationReport"
+            shacl_validations.append(report)
+
+        self.save_report(report_path, JSON_VALIDATIONS_REPORT, None,
+                         json.dumps(shacl_validations, default=lambda o: o.dict(), sort_keys=True, indent=4))
 
     def run_cmd(self):
         error = None

ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python3
 
+import json
 from pathlib import Path
+from typing import List
 
 import click
 
@@ -9,9 +11,10 @@
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
 from ted_sws.notice_validator.entrypoints.cli import DEFAULT_RDF_FOLDER, DEFAULT_TEST_SUITE_REPORT_FOLDER
-from ted_sws.notice_validator.services.sparql_test_suite_runner import SPARQLTestSuiteRunner, SPARQLReportBuilder
+from ted_sws.notice_validator.services.sparql_test_suite_runner import SPARQLTestSuiteRunner, SPARQLReportBuilder, \
+    SPARQLTestSuiteValidationReport
 
-JSON_REPORT = "sparql_{id}.json"
+JSON_VALIDATIONS_REPORT = "sparql_validations.json"
 HTML_REPORT = "sparql_{id}.html"
 CMD_NAME = "CMD_SPARQL_RUNNER"
 
@@ -41,7 +44,9 @@ def __init__(
 
     @classmethod
     def save_report(cls, report_path, report_name, report_id, content):
-        with open(report_path / report_name.format(id=report_id), "w+") as f:
+        if report_id is not None:
+            report_name = report_name.format(id=report_id)
+        with open(report_path / report_name, "w+") as f:
             f.write(content)
 
     def validate(self, rdf_file, base_report_path):
@@ -51,18 +56,23 @@ def validate(self, rdf_file, base_report_path):
         report_path = base_report_path / DEFAULT_TEST_SUITE_REPORT_FOLDER
         report_path.mkdir(parents=True, exist_ok=True)
 
+        sparql_validations: List[SPARQLTestSuiteValidationReport] = []
         sparql_test_suites = self.mapping_suite.sparql_test_suites
         for sparql_test_suite in sparql_test_suites:
             test_suite_execution = SPARQLTestSuiteRunner(rdf_manifestation=rdf_manifestation,
                                                          sparql_test_suite=sparql_test_suite,
                                                          mapping_suite=self.mapping_suite).execute_test_suite()
 
             report_builder = SPARQLReportBuilder(sparql_test_suite_execution=test_suite_execution)
-            html_report = report_builder.generate_report()
+            report: SPARQLTestSuiteValidationReport = report_builder.generate_report()
 
             suite_id = sparql_test_suite.identifier
-            html_data = html_report.object_data
-            self.save_report(report_path, HTML_REPORT, suite_id, html_data)
+            self.save_report(report_path, HTML_REPORT, suite_id, report.object_data)
+            report.object_data = "SPARQLTestSuiteValidationReport"
+            sparql_validations.append(report)
+
+        self.save_report(report_path, JSON_VALIDATIONS_REPORT, None,
+                         json.dumps(sparql_validations, default=lambda o: o.dict(), sort_keys=True, indent=4))
 
     def run_cmd(self):
         error = None