Removed object_data from validation reports

Author: Kolea Plesco

ted_sws/data_manager/adapters/notice_repository.py

@@ -185,19 +185,6 @@ def write_large_field(large_field: Manifestation):
         write_large_field(notice.mets_manifestation)
         write_large_field(notice.distilled_rdf_manifestation)
         write_large_field(notice.preprocessed_xml_manifestation)
-        if notice.rdf_manifestation:
-            for validation_report in notice.rdf_manifestation.shacl_validations:
-                write_large_field(validation_report)
-
-            for validation_report in notice.rdf_manifestation.sparql_validations:
-                write_large_field(validation_report)
-
-        if notice.distilled_rdf_manifestation:
-            for validation_report in notice.distilled_rdf_manifestation.shacl_validations:
-                write_large_field(validation_report)
-
-            for validation_report in notice.distilled_rdf_manifestation.sparql_validations:
-                write_large_field(validation_report)
 
         return notice, linked_file_ids, new_linked_file_ids
 

ted_sws/notice_validator/entrypoints/cli/cmd_shacl_runner.py

@@ -35,6 +35,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.notice_ids = self._init_list_input_opts(notice_ids)
         self.mappings_path = mappings_path
@@ -65,7 +66,7 @@ def validate(self, rdf_file, base_report_path, notice_ids: List[str] = None):
                                                         mapping_suite=self.mapping_suite).execute_test_suite()
 
             report: SHACLTestSuiteValidationReport = generate_shacl_report(
-                shacl_test_suite_execution=test_suite_execution, notice_ids=notice_ids, with_html=True)
+                shacl_test_suite_execution=test_suite_execution, notice_ids=notice_ids, with_html=self.with_html)
 
             suite_id = shacl_test_suite.identifier
             self.save_report(report_path, HTML_REPORT, suite_id, report.object_data)

ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py

@@ -39,6 +39,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.notice_ids = self._init_list_input_opts(notice_ids)
         self.mappings_path = mappings_path
@@ -75,7 +76,7 @@ def validate(self, rdf_file, xpath_report, base_report_path, notice_ids: List[st
                                                          mapping_suite=self.mapping_suite).execute_test_suite()
 
             report_builder = SPARQLReportBuilder(sparql_test_suite_execution=test_suite_execution,
-                                                 notice_ids=notice_ids, with_html=True)
+                                                 notice_ids=notice_ids, with_html=self.with_html)
             report: SPARQLTestSuiteValidationReport = report_builder.generate_report()
 
             suite_id = sparql_test_suite.identifier

ted_sws/notice_validator/entrypoints/cli/cmd_validation_summary_runner.py

@@ -44,6 +44,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.mappings_path = mappings_path
         self.notice_aggregate = notice_aggregate
@@ -121,7 +122,7 @@ def _read_output_notice_ids(self) -> List[str]:
     def _generate_report(self, notices: List[Notice], label: str, output_path: Path):
         self.log("Generating validation summary report for {label} ... ".format(label=label))
 
-        report: ValidationSummaryReport = generate_validation_summary_report_notices(notices, with_html=True)
+        report: ValidationSummaryReport = generate_validation_summary_report_notices(notices, with_html=self.with_html)
 
         self.save_html_report(output_path, report.object_data)
         del report.object_data

ted_sws/notice_validator/entrypoints/cli/cmd_xpath_coverage_runner.py

@@ -43,6 +43,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.notice_ids = self._init_list_input_opts(notice_ids)
         self.mappings_path = mappings_path
@@ -82,7 +83,8 @@ def coverage_report(self, notices: List[Notice], output_path: Path, label: str):
                                               self.conceptual_mappings_file_path,
                                               self.coverage_runner)
         self.save_json_report(Path(str(output_path) + ".json"), xpath_coverage_json_report(report))
-        self.save_html_report(Path(str(output_path) + ".html"), xpath_coverage_html_report(report))
+        if self.with_html:
+            self.save_html_report(Path(str(output_path) + ".html"), xpath_coverage_html_report(report))
 
     def run_cmd(self):
         super().run_cmd()

ted_sws/notice_validator/services/shacl_test_suite_runner.py

@@ -63,6 +63,7 @@ def generate_shacl_report(shacl_test_suite_execution: SHACLTestSuiteValidationRe
                           notice_ids: List[str] = None, with_html: bool = False) -> SHACLTestSuiteValidationReport:
     """
         This function generate html report after SHACL test execution.
+    :param with_html: generate HTML report
     :param notice_ids:
     :param shacl_test_suite_execution:
     :return:
@@ -76,16 +77,17 @@ def generate_shacl_report(shacl_test_suite_execution: SHACLTestSuiteValidationRe
 
 
 def validate_notice_with_shacl_suite(notice: Union[Notice, List[Notice]], mapping_suite_package: MappingSuite,
-                                     execute_full_validation: bool = True):
+                                     execute_full_validation: bool = True, with_html: bool = False):
     """
     Validates a notice with a shacl test suites
+    :param with_html: generate HTML report
     :param notice:
     :param mapping_suite_package:
     :param execute_full_validation:
     :return:
     """
 
-    def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
+    def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
             -> List[SHACLTestSuiteValidationReport]:
         reports = []
         shacl_test_suites = mapping_suite_package.shacl_test_suites
@@ -94,26 +96,30 @@ def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
                                                         shacl_test_suite=shacl_test_suite,
                                                         mapping_suite=mapping_suite_package).execute_test_suite()
             reports.append(generate_shacl_report(shacl_test_suite_execution=test_suite_execution,
-                                                 notice_ids=[notice_item.ted_id]))
+                                                 notice_ids=[notice_item.ted_id], with_html=with_html))
 
         return sorted(reports, key=lambda x: x.test_suite_identifier)
 
     notices = notice if isinstance(notice, List) else [notice]
 
     for notice in notices:
         if execute_full_validation:
-            for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
+            for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
+                                           with_html=with_html):
                 notice.set_rdf_validation(rdf_validation=report)
 
-        for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
+        for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
+                                       with_html=with_html):
             notice.set_distilled_rdf_validation(rdf_validation=report)
 
 
 def validate_notice_by_id_with_shacl_suite(notice_id: str, mapping_suite_identifier: str,
                                            notice_repository: NoticeRepositoryABC,
-                                           mapping_suite_repository: MappingSuiteRepositoryABC):
+                                           mapping_suite_repository: MappingSuiteRepositoryABC,
+                                           with_html: bool = False):
     """
     Validates a notice by id with a shacl test suites
+    :param with_html: generate HTML report
     :param notice_id:
     :param mapping_suite_identifier:
     :param notice_repository:
@@ -127,5 +133,5 @@ def validate_notice_by_id_with_shacl_suite(notice_id: str, mapping_suite_identif
     mapping_suite_package = mapping_suite_repository.get(reference=mapping_suite_identifier)
     if mapping_suite_package is None:
         raise ValueError(f'Mapping suite package, with {mapping_suite_identifier} id, was not found')
-    validate_notice_with_shacl_suite(notice=notice, mapping_suite_package=mapping_suite_package)
+    validate_notice_with_shacl_suite(notice=notice, mapping_suite_package=mapping_suite_package, with_html=with_html)
     notice_repository.update(notice=notice)

ted_sws/notice_validator/services/sparql_test_suite_runner.py

@@ -128,6 +128,11 @@ class SPARQLReportBuilder:
 
     def __init__(self, sparql_test_suite_execution: SPARQLTestSuiteValidationReport, notice_ids: List[str] = None,
                  with_html: bool = False):
+        """
+        :param sparql_test_suite_execution:
+        :param notice_ids:
+        :param with_html:
+        """
         self.sparql_test_suite_execution = sparql_test_suite_execution
         self.notice_ids = notice_ids
         self.with_html = with_html
@@ -142,16 +147,17 @@ def generate_report(self) -> SPARQLTestSuiteValidationReport:
 
 
 def validate_notice_with_sparql_suite(notice: Union[Notice, List[Notice]], mapping_suite_package: MappingSuite,
-                                      execute_full_validation: bool = True):
+                                      execute_full_validation: bool = True, with_html: bool = False):
     """
     Validates a notice with a sparql test suites
+    :param with_html: generate HTML report
     :param notice:
     :param mapping_suite_package:
     :param execute_full_validation:
     :return:
     """
 
-    def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
+    def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
             -> List[SPARQLTestSuiteValidationReport]:
         reports = []
         sparql_test_suites = mapping_suite_package.sparql_test_suites
@@ -162,26 +168,30 @@ def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation)
                                                          mapping_suite=mapping_suite_package
                                                          ).execute_test_suite()
             report_builder = SPARQLReportBuilder(sparql_test_suite_execution=test_suite_execution,
-                                                 notice_ids=[notice_item.ted_id])
+                                                 notice_ids=[notice_item.ted_id], with_html=with_html)
             reports.append(report_builder.generate_report())
         return sorted(reports, key=lambda x: x.test_suite_identifier)
 
     notices = notice if isinstance(notice, List) else [notice]
 
     for notice in notices:
         if execute_full_validation:
-            for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
+            for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
+                                            with_html=with_html):
                 notice.set_rdf_validation(rdf_validation=report)
 
-        for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
+        for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
+                                        with_html=with_html):
             notice.set_distilled_rdf_validation(rdf_validation=report)
 
 
 def validate_notice_by_id_with_sparql_suite(notice_id: str, mapping_suite_identifier: str,
                                             notice_repository: NoticeRepositoryABC,
-                                            mapping_suite_repository: MappingSuiteRepositoryABC):
+                                            mapping_suite_repository: MappingSuiteRepositoryABC,
+                                            with_html: bool = False):
     """
     Validates a notice by id with a sparql test suites
+    :param with_html: generate HTML report
     :param notice_id:
     :param mapping_suite_identifier:
     :param notice_repository:
@@ -195,7 +205,7 @@ def validate_notice_by_id_with_sparql_suite(notice_id: str, mapping_suite_identi
     mapping_suite_package = mapping_suite_repository.get(reference=mapping_suite_identifier)
     if mapping_suite_package is None:
         raise ValueError(f'Mapping suite package, with {mapping_suite_identifier} id, was not found')
-    validate_notice_with_sparql_suite(notice=notice, mapping_suite_package=mapping_suite_package)
+    validate_notice_with_sparql_suite(notice=notice, mapping_suite_package=mapping_suite_package, with_html=with_html)
     notice_repository.update(notice=notice)
 
 

ted_sws/notice_validator/services/xpath_coverage_runner.py

@@ -15,6 +15,11 @@ class XPATHCoverageReportBuilder:
     """
 
     def __init__(self, xpath_coverage_report: XPATHCoverageValidationReport, with_html: bool = False):
+        """
+
+        :param xpath_coverage_report:
+        :param with_html: generate HTML report
+        """
         self.xpath_coverage_report = xpath_coverage_report
         self.with_html = with_html
 
@@ -45,6 +50,14 @@ def xpath_coverage_html_report(report: XPATHCoverageValidationReport) -> str:
 
 def validate_xpath_coverage_notice(notice: Notice, mapping_suite: MappingSuite, mongodb_client: MongoClient,
                                    with_html: bool = False):
+    """
+
+    :param notice:
+    :param mapping_suite:
+    :param mongodb_client:
+    :param with_html: generate HTML report
+    :return:
+    """
     xpath_coverage_report = coverage_notice_xpath_report(notices=[notice],
                                                          mapping_suite_id=mapping_suite.get_mongodb_id(),
                                                          mongodb_client=mongodb_client)
@@ -55,6 +68,15 @@ def validate_xpath_coverage_notice(notice: Notice, mapping_suite: MappingSuite,
 def validate_xpath_coverage_notice_by_id(notice_id: str, mapping_suite_identifier: str,
                                          mapping_suite_repository: MappingSuiteRepositoryABC,
                                          mongodb_client: MongoClient, with_html: bool = False):
+    """
+
+    :param notice_id:
+    :param mapping_suite_identifier:
+    :param mapping_suite_repository:
+    :param mongodb_client:
+    :param with_html: generate HTML report
+    :return:
+    """
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     notice = notice_repository.get(reference=notice_id)
     if notice is None:

tests/unit/notice_validator/test_xpath_coverage_runner.py

@@ -11,8 +11,7 @@
 
 def test_xpath_coverage_runner(fake_notice_F03, fake_conceptual_mappings_F03_path,
                                fake_mapping_suite_F03_id, fake_mapping_suite_F03_id_with_version, mongodb_client,
-                               fake_mapping_suite_F03_path,
-                               invalid_mapping_suite_id):
+                               fake_mapping_suite_F03_path, invalid_mapping_suite_id):
     report = coverage_notice_xpath_report([fake_notice_F03], fake_mapping_suite_F03_id,
                                           fake_conceptual_mappings_F03_path, None)
     json_report = xpath_coverage_json_report(report)
@@ -51,7 +50,8 @@ def test_validate_xpath_coverage_notice_by_id(fake_notice_id, fake_mapping_suite
             notice_id=fake_notice_id,
             mapping_suite_identifier=fake_mapping_suite_F03_id,
             mapping_suite_repository=mapping_suite_repository_db,
-            mongodb_client=mongodb_client)
+            mongodb_client=mongodb_client,
+            with_html=True)
 
     notice_repository.add(fake_notice_F03)