Merge pull request #362 from OP-TED/feature/TED-923

Feature/ted 923

Author: CaptainOfHacks

ted_sws/data_manager/adapters/notice_repository.py

@@ -187,19 +187,6 @@ def write_large_field(large_field: Manifestation):
         write_large_field(notice.mets_manifestation)
         write_large_field(notice.distilled_rdf_manifestation)
         write_large_field(notice.preprocessed_xml_manifestation)
-        if notice.rdf_manifestation:
-            for validation_report in notice.rdf_manifestation.shacl_validations:
-                write_large_field(validation_report)
-
-            for validation_report in notice.rdf_manifestation.sparql_validations:
-                write_large_field(validation_report)
-
-        if notice.distilled_rdf_manifestation:
-            for validation_report in notice.distilled_rdf_manifestation.shacl_validations:
-                write_large_field(validation_report)
-
-            for validation_report in notice.distilled_rdf_manifestation.sparql_validations:
-                write_large_field(validation_report)
 
         return notice, linked_file_ids, new_linked_file_ids
 
@@ -219,18 +206,6 @@ def load_large_field(large_field: Manifestation):
         load_large_field(large_field=notice.mets_manifestation)
         load_large_field(large_field=notice.distilled_rdf_manifestation)
         load_large_field(large_field=notice.preprocessed_xml_manifestation)
-        if notice.rdf_manifestation:
-            for validation_report in notice.rdf_manifestation.shacl_validations:
-                load_large_field(validation_report)
-            for validation_report in notice.rdf_manifestation.sparql_validations:
-                load_large_field(validation_report)
-
-        if notice.distilled_rdf_manifestation:
-            for validation_report in notice.distilled_rdf_manifestation.shacl_validations:
-                load_large_field(validation_report)
-
-            for validation_report in notice.distilled_rdf_manifestation.sparql_validations:
-                load_large_field(validation_report)
 
         return notice
 

ted_sws/notice_validator/entrypoints/cli/cmd_shacl_runner.py

@@ -35,6 +35,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.notice_ids = self._init_list_input_opts(notice_ids)
         self.mappings_path = mappings_path
@@ -65,7 +66,7 @@ def validate(self, rdf_file, base_report_path, notice_ids: List[str] = None):
                                                         mapping_suite=self.mapping_suite).execute_test_suite()
 
             report: SHACLTestSuiteValidationReport = generate_shacl_report(
-                shacl_test_suite_execution=test_suite_execution, notice_ids=notice_ids)
+                shacl_test_suite_execution=test_suite_execution, notice_ids=notice_ids, with_html=self.with_html)
 
             suite_id = shacl_test_suite.identifier
             self.save_report(report_path, HTML_REPORT, suite_id, report.object_data)

ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py

@@ -39,6 +39,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.notice_ids = self._init_list_input_opts(notice_ids)
         self.mappings_path = mappings_path
@@ -75,7 +76,7 @@ def validate(self, rdf_file, xpath_report, base_report_path, notice_ids: List[st
                                                          mapping_suite=self.mapping_suite).execute_test_suite()
 
             report_builder = SPARQLReportBuilder(sparql_test_suite_execution=test_suite_execution,
-                                                 notice_ids=notice_ids)
+                                                 notice_ids=notice_ids, with_html=self.with_html)
             report: SPARQLTestSuiteValidationReport = report_builder.generate_report()
 
             suite_id = sparql_test_suite.identifier

ted_sws/notice_validator/entrypoints/cli/cmd_validation_summary_runner.py

@@ -44,6 +44,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.mappings_path = mappings_path
         self.notice_aggregate = notice_aggregate
@@ -121,7 +122,7 @@ def _read_output_notice_ids(self) -> List[str]:
     def _generate_report(self, notices: List[Notice], label: str, output_path: Path):
         self.log("Generating validation summary report for {label} ... ".format(label=label))
 
-        report: ValidationSummaryReport = generate_validation_summary_report_notices(notices)
+        report: ValidationSummaryReport = generate_validation_summary_report_notices(notices, with_html=self.with_html)
 
         self.save_html_report(output_path, report.object_data)
         del report.object_data

ted_sws/notice_validator/entrypoints/cli/cmd_xpath_coverage_runner.py

@@ -43,6 +43,7 @@ def __init__(
             mappings_path
     ):
         super().__init__(name=CMD_NAME)
+        self.with_html = True
         self.mapping_suite_id = mapping_suite_id
         self.notice_ids = self._init_list_input_opts(notice_ids)
         self.mappings_path = mappings_path
@@ -82,7 +83,8 @@ def coverage_report(self, notices: List[Notice], output_path: Path, label: str):
                                               self.conceptual_mappings_file_path,
                                               self.coverage_runner)
         self.save_json_report(Path(str(output_path) + ".json"), xpath_coverage_json_report(report))
-        self.save_html_report(Path(str(output_path) + ".html"), xpath_coverage_html_report(report))
+        if self.with_html:
+            self.save_html_report(Path(str(output_path) + ".html"), xpath_coverage_html_report(report))
 
     def run_cmd(self):
         super().run_cmd()

ted_sws/notice_validator/services/shacl_test_suite_runner.py

@@ -60,31 +60,34 @@ def execute_test_suite(self) -> SHACLTestSuiteValidationReport:
 
 
 def generate_shacl_report(shacl_test_suite_execution: SHACLTestSuiteValidationReport,
-                          notice_ids: List[str] = None) -> SHACLTestSuiteValidationReport:
+                          notice_ids: List[str] = None, with_html: bool = False) -> SHACLTestSuiteValidationReport:
     """
         This function generate html report after SHACL test execution.
+    :param with_html: generate HTML report
     :param notice_ids:
     :param shacl_test_suite_execution:
     :return:
     """
-    template_data: dict = shacl_test_suite_execution.dict()
-    template_data[NOTICE_IDS_FIELD] = notice_ids
-    html_report = TEMPLATES.get_template(SHACL_TEST_SUITE_EXECUTION_HTML_REPORT_TEMPLATE).render(template_data)
-    shacl_test_suite_execution.object_data = html_report
+    if with_html:
+        template_data: dict = shacl_test_suite_execution.dict()
+        template_data[NOTICE_IDS_FIELD] = notice_ids
+        html_report = TEMPLATES.get_template(SHACL_TEST_SUITE_EXECUTION_HTML_REPORT_TEMPLATE).render(template_data)
+        shacl_test_suite_execution.object_data = html_report
     return shacl_test_suite_execution
 
 
 def validate_notice_with_shacl_suite(notice: Union[Notice, List[Notice]], mapping_suite_package: MappingSuite,
-                                     execute_full_validation: bool = True):
+                                     execute_full_validation: bool = True, with_html: bool = False):
     """
     Validates a notice with a shacl test suites
+    :param with_html: generate HTML report
     :param notice:
     :param mapping_suite_package:
     :param execute_full_validation:
     :return:
     """
 
-    def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
+    def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
             -> List[SHACLTestSuiteValidationReport]:
         reports = []
         shacl_test_suites = mapping_suite_package.shacl_test_suites
@@ -93,26 +96,30 @@ def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
                                                         shacl_test_suite=shacl_test_suite,
                                                         mapping_suite=mapping_suite_package).execute_test_suite()
             reports.append(generate_shacl_report(shacl_test_suite_execution=test_suite_execution,
-                                                 notice_ids=[notice_item.ted_id]))
+                                                 notice_ids=[notice_item.ted_id], with_html=with_html))
 
         return sorted(reports, key=lambda x: x.test_suite_identifier)
 
     notices = notice if isinstance(notice, List) else [notice]
 
     for notice in notices:
         if execute_full_validation:
-            for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
+            for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
+                                           with_html=with_html):
                 notice.set_rdf_validation(rdf_validation=report)
 
-        for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
+        for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
+                                       with_html=with_html):
             notice.set_distilled_rdf_validation(rdf_validation=report)
 
 
 def validate_notice_by_id_with_shacl_suite(notice_id: str, mapping_suite_identifier: str,
                                            notice_repository: NoticeRepositoryABC,
-                                           mapping_suite_repository: MappingSuiteRepositoryABC):
+                                           mapping_suite_repository: MappingSuiteRepositoryABC,
+                                           with_html: bool = False):
     """
     Validates a notice by id with a shacl test suites
+    :param with_html: generate HTML report
     :param notice_id:
     :param mapping_suite_identifier:
     :param notice_repository:
@@ -126,5 +133,5 @@ def validate_notice_by_id_with_shacl_suite(notice_id: str, mapping_suite_identif
     mapping_suite_package = mapping_suite_repository.get(reference=mapping_suite_identifier)
     if mapping_suite_package is None:
         raise ValueError(f'Mapping suite package, with {mapping_suite_identifier} id, was not found')
-    validate_notice_with_shacl_suite(notice=notice, mapping_suite_package=mapping_suite_package)
+    validate_notice_with_shacl_suite(notice=notice, mapping_suite_package=mapping_suite_package, with_html=with_html)
     notice_repository.update(notice=notice)

ted_sws/notice_validator/services/sparql_test_suite_runner.py

@@ -126,29 +126,38 @@ class SPARQLReportBuilder:
         Given a SPARQLQueryResult, generates JSON and HTML reports.
     """
 
-    def __init__(self, sparql_test_suite_execution: SPARQLTestSuiteValidationReport, notice_ids: List[str] = None):
+    def __init__(self, sparql_test_suite_execution: SPARQLTestSuiteValidationReport, notice_ids: List[str] = None,
+                 with_html: bool = False):
+        """
+        :param sparql_test_suite_execution:
+        :param notice_ids:
+        :param with_html: generate HTML report
+        """
         self.sparql_test_suite_execution = sparql_test_suite_execution
         self.notice_ids = notice_ids
+        self.with_html = with_html
 
     def generate_report(self) -> SPARQLTestSuiteValidationReport:
-        template_data: dict = self.sparql_test_suite_execution.dict()
-        template_data[NOTICE_IDS_FIELD] = self.notice_ids
-        html_report = TEMPLATES.get_template(SPARQL_TEST_SUITE_EXECUTION_HTML_REPORT_TEMPLATE).render(template_data)
-        self.sparql_test_suite_execution.object_data = html_report
+        if self.with_html:
+            template_data: dict = self.sparql_test_suite_execution.dict()
+            template_data[NOTICE_IDS_FIELD] = self.notice_ids
+            html_report = TEMPLATES.get_template(SPARQL_TEST_SUITE_EXECUTION_HTML_REPORT_TEMPLATE).render(template_data)
+            self.sparql_test_suite_execution.object_data = html_report
         return self.sparql_test_suite_execution
 
 
 def validate_notice_with_sparql_suite(notice: Union[Notice, List[Notice]], mapping_suite_package: MappingSuite,
-                                      execute_full_validation: bool = True):
+                                      execute_full_validation: bool = True, with_html: bool = False):
     """
     Validates a notice with a sparql test suites
+    :param with_html: generate HTML report
     :param notice:
     :param mapping_suite_package:
     :param execute_full_validation:
     :return:
     """
 
-    def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
+    def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
             -> List[SPARQLTestSuiteValidationReport]:
         reports = []
         sparql_test_suites = mapping_suite_package.sparql_test_suites
@@ -159,26 +168,30 @@ def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation)
                                                          mapping_suite=mapping_suite_package
                                                          ).execute_test_suite()
             report_builder = SPARQLReportBuilder(sparql_test_suite_execution=test_suite_execution,
-                                                 notice_ids=[notice_item.ted_id])
+                                                 notice_ids=[notice_item.ted_id], with_html=with_html)
             reports.append(report_builder.generate_report())
         return sorted(reports, key=lambda x: x.test_suite_identifier)
 
     notices = notice if isinstance(notice, List) else [notice]
 
     for notice in notices:
         if execute_full_validation:
-            for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
+            for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
+                                            with_html=with_html):
                 notice.set_rdf_validation(rdf_validation=report)
 
-        for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
+        for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
+                                        with_html=with_html):
             notice.set_distilled_rdf_validation(rdf_validation=report)
 
 
 def validate_notice_by_id_with_sparql_suite(notice_id: str, mapping_suite_identifier: str,
                                             notice_repository: NoticeRepositoryABC,
-                                            mapping_suite_repository: MappingSuiteRepositoryABC):
+                                            mapping_suite_repository: MappingSuiteRepositoryABC,
+                                            with_html: bool = False):
     """
     Validates a notice by id with a sparql test suites
+    :param with_html: generate HTML report
     :param notice_id:
     :param mapping_suite_identifier:
     :param notice_repository:
@@ -192,7 +205,7 @@ def validate_notice_by_id_with_sparql_suite(notice_id: str, mapping_suite_identi
     mapping_suite_package = mapping_suite_repository.get(reference=mapping_suite_identifier)
     if mapping_suite_package is None:
         raise ValueError(f'Mapping suite package, with {mapping_suite_identifier} id, was not found')
-    validate_notice_with_sparql_suite(notice=notice, mapping_suite_package=mapping_suite_package)
+    validate_notice_with_sparql_suite(notice=notice, mapping_suite_package=mapping_suite_package, with_html=with_html)
     notice_repository.update(notice=notice)
 
 

ted_sws/notice_validator/services/validation_summary_runner.py

@@ -12,29 +12,34 @@ class ValidationSummaryReportBuilder:
 
     report: ValidationSummaryReport
 
-    def __init__(self, report: ValidationSummaryReport):
+    def __init__(self, report: ValidationSummaryReport, with_html: bool = False):
         self.report = report
+        self.with_html = with_html
 
     def generate_report(self) -> ValidationSummaryReport:
-        html_report = ValidationSummaryRunner.html_report(self.report)
-        self.report.object_data = html_report
+        if self.with_html:
+            html_report = ValidationSummaryRunner.html_report(self.report)
+            self.report.object_data = html_report
         return self.report
 
 
-def generate_validation_summary_report_notices(notices: List[Notice]) -> ValidationSummaryReport:
+def generate_validation_summary_report_notices(notices: List[Notice],
+                                               with_html: bool = False) -> ValidationSummaryReport:
     validation_summary_report = ValidationSummaryRunner()
-    report_builder = ValidationSummaryReportBuilder(validation_summary_report.validation_summary(notices))
+    report_builder = ValidationSummaryReportBuilder(validation_summary_report.validation_summary(notices),
+                                                    with_html=with_html)
     return report_builder.generate_report()
 
 
-def validation_summary_report_notice(notice: Notice):
-    notice.validation_summary = generate_validation_summary_report_notices([notice])
+def validation_summary_report_notice(notice: Notice, with_html: bool = False):
+    notice.validation_summary = generate_validation_summary_report_notices([notice], with_html=with_html)
 
 
-def validation_summary_report_notice_by_id(notice_id: str, notice_repository: NoticeRepository):
+def validation_summary_report_notice_by_id(notice_id: str, notice_repository: NoticeRepository,
+                                           with_html: bool = False):
     notice = notice_repository.get(reference=notice_id)
     if notice is None:
         raise ValueError(f'Notice, with {notice_id} id, was not found')
 
-    validation_summary_report_notice(notice=notice)
+    validation_summary_report_notice(notice=notice, with_html=with_html)
     notice_repository.update(notice=notice)