Skip to content

Commit c5fb63a

Browse files
kaleanychkaleanych
authored
Merge pull request #214 from OP-TED/feature/TED-543
Feature/ted 543
2 parents a6c0bce + 66e298c commit c5fb63a

32 files changed

Lines changed: 21486 additions & 100 deletions

‎setup.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ def open_local(paths, mode="r", encoding="utf8"):
7878
"sparql_runner = ted_sws.notice_validator.entrypoints.cli.cmd_sparql_runner:main",
7979
"shacl_runner = ted_sws.notice_validator.entrypoints.cli.cmd_shacl_runner:main",
8080
"xpath_coverage_runner = ted_sws.notice_validator.entrypoints.cli.cmd_xpath_coverage_runner:main",
81+
"validation_summary_runner = ted_sws.notice_validator.entrypoints.cli.cmd_validation_summary_runner:main",
8182
"rml_report_generator = ted_sws.rml_to_html.entrypoints.cli.cmd_rml_report_generator:main",
8283
"mapping_suite_processor = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_mapping_suite_processor:main",
8384
"metadata_generator = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_metadata_generator:main",

‎ted_sws/core/model/manifestation.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ class XPATHCoverageValidationAssertion(PropertyBaseModel):
6060
"""
6161
6262
"""
63+
standard_form_field_id: Optional[str]
64+
eform_bt_id: Optional[str]
6365
title: Optional[str]
6466
xpath: Optional[str]
6567
count: Optional[int]

‎ted_sws/core/model/transform.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,9 @@ class TransformationTestData(MappingSuiteComponent):
6767

6868
class ConceptualMappingXPATH(MappingSuiteComponent):
6969
xpath: str
70-
name: str
70+
name: Optional[str]
71+
standard_form_field_id: Optional[str]
72+
eform_bt_id: Optional[str]
7173

7274

7375
class ConceptualMappingMetadata(MappingSuiteComponent):

‎ted_sws/mapping_suite_processor/__init__.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME = "Metadata"
22
CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME = "Rules"
33
RULES_E_FORM_BT_NAME = 'eForm BT Name (O)'
4-
RULES_FIELD_XPATH = 'Field XPath (M)'
4+
RULES_FIELD_XPATH = 'Field XPath (M)'
5+
RULES_SF_FIELD_ID = 'Standard Form Field ID (M)'
6+
RULES_E_FORM_BT_ID = 'eForm BT-ID (O)'

‎ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_sparql_queries.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,11 @@
55
import pandas as pd
66

77
from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME, \
8-
CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME
8+
CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME, RULES_SF_FIELD_ID, RULES_E_FORM_BT_ID
99
from ted_sws.notice_validator import BASE_XPATH_FIELD
1010
from ted_sws.resources.prefixes import PREFIXES_DEFINITIONS
1111

12-
13-
RULES_SF_FIELD_ID = 'Standard Form Field ID (M)'
1412
RULES_SF_FIELD_NAME = 'Standard Form Field Name (M)'
15-
RULES_E_FORM_BT_ID = 'eForm BT-ID (O)'
1613
RULES_CLASS_PATH = 'Class path (M)'
1714
RULES_PROPERTY_PATH = 'Property path (M)'
1815

‎ted_sws/mapping_suite_processor/services/conceptual_mapping_reader.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingXPATH, ConceptualMappingMetadata
88
from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME, \
9-
CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME
9+
CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_E_FORM_BT_NAME, RULES_SF_FIELD_ID, RULES_E_FORM_BT_ID
1010
from ted_sws.notice_validator import BASE_XPATH_FIELD
1111

1212
CONCEPTUAL_MAPPINGS_FILE_NAME = "conceptual_mappings.xlsx"
@@ -46,6 +46,8 @@ def mapping_suite_read_conceptual_mapping(conceptual_mappings_file_path: pathlib
4646
rules_df = pd.read_excel(excel_file, sheet_name=CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, header=1)
4747
df_xpaths = rules_df[RULES_FIELD_XPATH].tolist()
4848
df_bt_names = rules_df[RULES_E_FORM_BT_NAME].tolist()
49+
df_eform_bt_ids = rules_df[RULES_E_FORM_BT_ID].tolist()
50+
df_sform_field_ids = rules_df[RULES_SF_FIELD_ID].tolist()
4951
processed_xpaths = set()
5052
for idx, xpath_row in enumerate(df_xpaths):
5153
if xpath_row is not np.nan:
@@ -54,8 +56,16 @@ def mapping_suite_read_conceptual_mapping(conceptual_mappings_file_path: pathlib
5456
if xpath:
5557
xpath = base_xpath + "/" + xpath
5658
if xpath not in processed_xpaths:
57-
xpath_name = df_bt_names[idx]
58-
cm_xpath = ConceptualMappingXPATH(xpath=xpath, name=xpath_name)
59+
xpath_name = df_bt_names[idx] if df_bt_names[idx] is not np.nan else None
60+
eform_bt_id = df_eform_bt_ids[idx] if df_eform_bt_ids[idx] is not np.nan else None
61+
sform_field_id = df_sform_field_ids[idx] if df_sform_field_ids[idx] is not np.nan else None
62+
63+
cm_xpath: ConceptualMappingXPATH = ConceptualMappingXPATH(
64+
xpath=xpath,
65+
name=xpath_name,
66+
standard_form_field_id=sform_field_id,
67+
eform_bt_id=eform_bt_id
68+
)
5969
conceptual_mapping_xpaths.append(cm_xpath)
6070
processed_xpaths.add(xpath)
6171

‎ted_sws/notice_validator/adapters/validation_summary_runner.py‎

Lines changed: 52 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -22,63 +22,66 @@ def __init__(self, notices: List[Notice]):
2222
class RDFManifestationValidationSummaryRunner(ManifestationValidationSummaryRunner):
2323
@classmethod
2424
def notice_sparql_summary(cls, notice: Notice, report: RDFManifestationValidationSummaryReport):
25-
report_count: SPARQLSummaryCountReport = report.sparql_summary.aggregate
26-
result_counts: List[SPARQLSummaryResult] = report.sparql_summary.validation_results
27-
sparql_reports: List[SPARQLTestSuiteValidationReport] = notice.rdf_manifestation.sparql_validations
28-
if sparql_reports:
29-
for sparql_report in sparql_reports:
30-
validation_results: List[SPARQLQueryResult] = sparql_report.validation_results
31-
result_validation: SPARQLSummaryResult = SPARQLSummaryResult()
32-
result_validation.mapping_suite_identifier = sparql_report.mapping_suite_identifier
33-
result_validation.test_suite_identifier = sparql_report.test_suite_identifier
34-
result_count: SPARQLSummaryCountReport = result_validation.aggregate
35-
if validation_results:
36-
for validation in validation_results:
37-
if validation.result == 'True':
38-
report_count.success += 1
39-
result_count.success += 1
40-
else:
41-
report_count.fail += 1
42-
result_count.fail += 1
43-
if validation.error:
44-
report_count.error += 1
45-
result_count.error += 1
46-
47-
result_counts.append(result_validation)
25+
manifestation = cls._manifestation(notice)
26+
if manifestation:
27+
report_count: SPARQLSummaryCountReport = report.sparql_summary.aggregate
28+
result_counts: List[SPARQLSummaryResult] = report.sparql_summary.validation_results
29+
sparql_reports: List[SPARQLTestSuiteValidationReport] = manifestation.sparql_validations
30+
if sparql_reports:
31+
for sparql_report in sparql_reports:
32+
validation_results: List[SPARQLQueryResult] = sparql_report.validation_results
33+
result_validation: SPARQLSummaryResult = SPARQLSummaryResult()
34+
result_validation.mapping_suite_identifier = sparql_report.mapping_suite_identifier
35+
result_validation.test_suite_identifier = sparql_report.test_suite_identifier
36+
result_count: SPARQLSummaryCountReport = result_validation.aggregate
37+
if validation_results:
38+
for validation in validation_results:
39+
if validation.result == 'True':
40+
report_count.success += 1
41+
result_count.success += 1
42+
else:
43+
report_count.fail += 1
44+
result_count.fail += 1
45+
if validation.error:
46+
report_count.error += 1
47+
result_count.error += 1
48+
49+
result_counts.append(result_validation)
4850

4951
@classmethod
5052
def _manifestation(cls, notice: Notice) -> RDFManifestation:
5153
return notice.rdf_manifestation
5254

5355
@classmethod
5456
def notice_shacl_summary(cls, notice: Notice, report: RDFManifestationValidationSummaryReport):
55-
report_count: SHACLSummarySeverityCountReport = report.shacl_summary.result_severity.aggregate
56-
result_counts: List[SHACLSummaryResult] = report.shacl_summary.validation_results
5757
manifestation = cls._manifestation(notice)
58-
shacl_reports: List[SHACLTestSuiteValidationReport] = manifestation.shacl_validations
59-
if shacl_reports:
60-
for shacl_report in shacl_reports:
61-
validation_results = shacl_report.validation_results
62-
result_validation: SHACLSummaryResult = SHACLSummaryResult()
63-
result_validation.mapping_suite_identifier = shacl_report.mapping_suite_identifier
64-
result_validation.test_suite_identifier = shacl_report.test_suite_identifier
65-
result_count: SHACLSummarySeverityCountReport = result_validation.result_severity.aggregate
66-
if validation_results:
67-
bindings = validation_results.results_dict['results']['bindings']
68-
for binding in bindings:
69-
result_severity = binding['resultSeverity']
70-
if result_severity:
71-
if result_severity['value'].endswith("#Violation"):
72-
report_count.violation += 1
73-
result_count.violation += 1
74-
elif result_severity['value'].endswith("#Info"):
75-
report_count.info += 1
76-
result_count.info += 1
77-
elif result_severity['value'].endswith("#Warning"):
78-
report_count.warning += 1
79-
result_count.warning += 1
80-
81-
result_counts.append(result_validation)
58+
if manifestation:
59+
report_count: SHACLSummarySeverityCountReport = report.shacl_summary.result_severity.aggregate
60+
result_counts: List[SHACLSummaryResult] = report.shacl_summary.validation_results
61+
shacl_reports: List[SHACLTestSuiteValidationReport] = manifestation.shacl_validations
62+
if shacl_reports:
63+
for shacl_report in shacl_reports:
64+
validation_results = shacl_report.validation_results
65+
result_validation: SHACLSummaryResult = SHACLSummaryResult()
66+
result_validation.mapping_suite_identifier = shacl_report.mapping_suite_identifier
67+
result_validation.test_suite_identifier = shacl_report.test_suite_identifier
68+
result_count: SHACLSummarySeverityCountReport = result_validation.result_severity.aggregate
69+
if validation_results:
70+
bindings = validation_results.results_dict['results']['bindings']
71+
for binding in bindings:
72+
result_severity = binding['resultSeverity']
73+
if result_severity:
74+
if result_severity['value'].endswith("#Violation"):
75+
report_count.violation += 1
76+
result_count.violation += 1
77+
elif result_severity['value'].endswith("#Info"):
78+
report_count.info += 1
79+
result_count.info += 1
80+
elif result_severity['value'].endswith("#Warning"):
81+
report_count.warning += 1
82+
result_count.warning += 1
83+
84+
result_counts.append(result_validation)
8285

8386
def validation_summary(self) -> RDFManifestationValidationSummaryReport:
8487
notices = self.notices

‎ted_sws/notice_validator/adapters/xpath_coverage_runner.py‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,10 @@
88
from ted_sws.core.model.manifestation import XPATHCoverageValidationReport, XPATHCoverageValidationAssertion, \
99
XPATHCoverageValidationResult
1010
from ted_sws.core.model.notice import Notice
11-
from ted_sws.core.model.transform import ConceptualMapping
12-
from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, get_unique_xpaths_covered_by_notices
13-
from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import mapping_suite_read_metadata, \
14-
mapping_suite_read_conceptual_mapping
15-
from ted_sws.notice_validator import BASE_XPATH_FIELD
11+
from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingXPATH
1612
from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
13+
from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, get_unique_xpaths_covered_by_notices
14+
from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import mapping_suite_read_conceptual_mapping
1715

1816
TEMPLATES = Environment(loader=PackageLoader("ted_sws.notice_validator.resources", "templates"))
1917
XPATH_COVERAGE_REPORT_TEMPLATE = "xpath_coverage_report.jinja2"
@@ -28,7 +26,7 @@ class CoverageRunner:
2826
"""
2927

3028
conceptual_xpaths: Set[str] = set()
31-
conceptual_xpath_names: Dict[str, str] = {}
29+
conceptual_xpath_data: Dict[str, ConceptualMappingXPATH] = {}
3230
mongodb_client: MongoClient
3331
base_xpath: str
3432
mapping_suite_id: str
@@ -51,7 +49,7 @@ def __init__(self, mapping_suite_id: str, conceptual_mappings_file_path: PATH_TY
5149

5250
for cm_xpath in conceptual_mapping.xpaths:
5351
self.conceptual_xpaths.add(cm_xpath.xpath)
54-
self.conceptual_xpath_names[cm_xpath.xpath] = cm_xpath.name
52+
self.conceptual_xpath_data[cm_xpath.xpath] = cm_xpath
5553

5654
self.base_xpath = conceptual_mapping.metadata.base_xpath
5755

@@ -68,8 +66,11 @@ def xpath_assertions(self, notice_xpaths: XPATH_TYPE,
6866
xpath_assertions = []
6967
for xpath in self.conceptual_xpaths:
7068
xpath_assertion = XPATHCoverageValidationAssertion()
71-
title = self.conceptual_xpath_names[xpath]
69+
xpath_data = self.conceptual_xpath_data[xpath]
70+
title = xpath_data.name
7271
xpath_assertion.title = title if title is not np.nan else ''
72+
xpath_assertion.standard_form_field_id = xpath_data.standard_form_field_id
73+
xpath_assertion.eform_bt_id = xpath_data.eform_bt_id
7374
xpath_assertion.xpath = xpath
7475
xpath_assertion.count = xpaths_list.count(xpath)
7576
xpath_assertion.notice_hit = self.find_notice_by_xpath(notice_xpaths, xpath)

‎ted_sws/notice_validator/entrypoints/cli/cmd_shacl_runner.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,18 @@
44

55
import click
66

7+
from typing import List
8+
79
from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
810
from ted_sws.core.model.manifestation import RDFManifestation, SHACLTestSuiteValidationReport
911
from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
1012
from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
1113
from ted_sws.notice_validator.entrypoints.cli import DEFAULT_RDF_FOLDER, DEFAULT_TEST_SUITE_REPORT_FOLDER
1214
from ted_sws.notice_validator.services.shacl_test_suite_runner import SHACLTestSuiteRunner, generate_shacl_report
15+
import json
1316

1417
HTML_REPORT = "shacl_{id}.html"
18+
JSON_VALIDATIONS_REPORT = "shacl_validations.json"
1519
CMD_NAME = "CMD_SHACL_RUNNER"
1620

1721
"""
@@ -39,7 +43,9 @@ def __init__(
3943
self.mapping_suite = mapping_suite_repository.get(reference=self.mapping_suite_id)
4044

4145
def save_report(self, report_path, report_name, report_id, content):
42-
with open(report_path / report_name.format(id=report_id), "w+") as f:
46+
if report_id is not None:
47+
report_name = report_name.format(id=report_id)
48+
with open(report_path / report_name, "w+") as f:
4349
f.write(content)
4450

4551
def validate(self, rdf_file, base_report_path):
@@ -49,6 +55,7 @@ def validate(self, rdf_file, base_report_path):
4955
report_path = base_report_path / DEFAULT_TEST_SUITE_REPORT_FOLDER
5056
report_path.mkdir(parents=True, exist_ok=True)
5157

58+
shacl_validations: List[SHACLTestSuiteValidationReport] = []
5259
shacl_test_suites = self.mapping_suite.shacl_test_suites
5360
for shacl_test_suite in shacl_test_suites:
5461
test_suite_execution = SHACLTestSuiteRunner(rdf_manifestation=rdf_manifestation,
@@ -59,8 +66,12 @@ def validate(self, rdf_file, base_report_path):
5966
shacl_test_suite_execution=test_suite_execution)
6067

6168
suite_id = shacl_test_suite.identifier
62-
data = report.object_data
63-
self.save_report(report_path, HTML_REPORT, suite_id, data)
69+
self.save_report(report_path, HTML_REPORT, suite_id, report.object_data)
70+
report.object_data = "SHACLTestSuiteValidationReport"
71+
shacl_validations.append(report)
72+
73+
self.save_report(report_path, JSON_VALIDATIONS_REPORT, None,
74+
json.dumps(shacl_validations, default=lambda o: o.dict(), sort_keys=True, indent=4))
6475

6576
def run_cmd(self):
6677
error = None

‎ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/usr/bin/python3
22

3+
import json
34
from pathlib import Path
5+
from typing import List
46

57
import click
68

@@ -9,9 +11,10 @@
911
from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
1012
from ted_sws.event_manager.adapters.logger import LOG_INFO_TEXT
1113
from ted_sws.notice_validator.entrypoints.cli import DEFAULT_RDF_FOLDER, DEFAULT_TEST_SUITE_REPORT_FOLDER
12-
from ted_sws.notice_validator.services.sparql_test_suite_runner import SPARQLTestSuiteRunner, SPARQLReportBuilder
14+
from ted_sws.notice_validator.services.sparql_test_suite_runner import SPARQLTestSuiteRunner, SPARQLReportBuilder, \
15+
SPARQLTestSuiteValidationReport
1316

14-
JSON_REPORT = "sparql_{id}.json"
17+
JSON_VALIDATIONS_REPORT = "sparql_validations.json"
1518
HTML_REPORT = "sparql_{id}.html"
1619
CMD_NAME = "CMD_SPARQL_RUNNER"
1720

@@ -41,7 +44,9 @@ def __init__(
4144

4245
@classmethod
4346
def save_report(cls, report_path, report_name, report_id, content):
44-
with open(report_path / report_name.format(id=report_id), "w+") as f:
47+
if report_id is not None:
48+
report_name = report_name.format(id=report_id)
49+
with open(report_path / report_name, "w+") as f:
4550
f.write(content)
4651

4752
def validate(self, rdf_file, base_report_path):
@@ -51,18 +56,23 @@ def validate(self, rdf_file, base_report_path):
5156
report_path = base_report_path / DEFAULT_TEST_SUITE_REPORT_FOLDER
5257
report_path.mkdir(parents=True, exist_ok=True)
5358

59+
sparql_validations: List[SPARQLTestSuiteValidationReport] = []
5460
sparql_test_suites = self.mapping_suite.sparql_test_suites
5561
for sparql_test_suite in sparql_test_suites:
5662
test_suite_execution = SPARQLTestSuiteRunner(rdf_manifestation=rdf_manifestation,
5763
sparql_test_suite=sparql_test_suite,
5864
mapping_suite=self.mapping_suite).execute_test_suite()
5965

6066
report_builder = SPARQLReportBuilder(sparql_test_suite_execution=test_suite_execution)
61-
html_report = report_builder.generate_report()
67+
report: SPARQLTestSuiteValidationReport = report_builder.generate_report()
6268

6369
suite_id = sparql_test_suite.identifier
64-
html_data = html_report.object_data
65-
self.save_report(report_path, HTML_REPORT, suite_id, html_data)
70+
self.save_report(report_path, HTML_REPORT, suite_id, report.object_data)
71+
report.object_data = "SPARQLTestSuiteValidationReport"
72+
sparql_validations.append(report)
73+
74+
self.save_report(report_path, JSON_VALIDATIONS_REPORT, None,
75+
json.dumps(sparql_validations, default=lambda o: o.dict(), sort_keys=True, indent=4))
6676

6777
def run_cmd(self):
6878
error = None

0 commit comments

Comments
 (0)