Merge branch 'main' into feature/TED-1043

Author: CaptainOfHacks

README.md

@@ -1,4 +1,4 @@
-# TED Semantic Web Services
+# TED-RDF Conversion Pipeline
 
 [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=meaningfy-ws_ted-sws&metric=ncloc)](https://sonarcloud.io/summary/new_code?id=meaningfy-ws_ted-sws)
 [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=meaningfy-ws_ted-sws&metric=bugs)](https://sonarcloud.io/summary/new_code?id=meaningfy-ws_ted-sws)
@@ -18,10 +18,10 @@
 
 ## Documentation
 
-If you contribute to this projector are simple user please refer to the following project documentation:
+This repository is part of the TED Semantic Web Services, aka TED-SWS System. If you would like to contribute to this project, or you are a simple user, please refer to the following project documentation pages:
 
-* [Project documentation ](https://docs.ted.europa.eu/ted-rdf-conversion-pipeline/ted-sws/index.html)
-* [GitHub pages with the enterprise architecture (in development)](https://docs.ted.europa.eu/ted-rdf-conversion-pipeline/ted-sws/_attachments/ted-sws-architecture/index.html)
+* [Project documentation ](https://docs.ted.europa.eu/rdf-conversion/index.html)
+* [GitHub pages with the enterprise architecture (in development)](https://docs.ted.europa.eu/rdf-conversion/_attachments/ted-sws-architecture/index.html)
 * [Enterprise architecture model file (in development)](https://drive.google.com/file/d/1YB2dPYe9E9bAR2peVraQaUANS-hXetms/view?usp=sharing)
 
 
@@ -30,8 +30,8 @@ Two sets of installation instruction are foreseen: one for the software engineer
 The software engineers need these instructions to deploy on a local or a single machine testing server.
 The semantic engineers need a set of CLI tools that help develop, validate and publish mapping suites.
 
-* [Demo & Testing installation instructions for software engineers](https://docs.ted.europa.eu/ted-rdf-conversion-pipeline/ted-sws/demo_installation.html)
-* [Installation & Usage instructions for the Mapping Suite CLI toolchain](https://docs.ted.europa.eu/ted-rdf-conversion-pipeline/ted-sws/mapping_suite_cli_toolchain.html)
+* [Demo & Testing installation instructions for software engineers](https://docs.ted.europa.eu/rdf-conversion/demo_installation.html)
+* [Installation & Usage instructions for the Mapping Suite CLI toolchain](https://docs.ted.europa.eu/rdf-conversion/mapping_suite_cli_toolchain.html)
 
 ## Contributing
 

docs/antora/modules/ROOT/nav.adoc

@@ -1,4 +1,5 @@
 
 * xref:index.adoc[Home]
 * link:{attachmentsdir}/ted-sws-architecture/index.html[Preliminary Project Architecture^]
-* xref:mapping_suite_cli_toolchain.adoc[Mapping Suite CLI Toolchain]
+* xref:mapping_suite_cli_toolchain.adoc[Mapping Suite CLI Toolchain]
+* xref:demo_installation.adoc[Instructions for Software Engineers]

docs/antora/modules/ROOT/pages/index.adoc

@@ -1,4 +1,8 @@
-= TED-SWS Project documentation
+= TED-RDF Conversion Pipeline Documentation
+
+The TED-RDF Conversion Pipeline, which is part of the TED Semantic Web Services, aka TED-SWS system, provides tools an infrastructure to convert TED notices available in XML format into RDF. This conversion pipeline is designed to work with the https://docs.ted.europa.eu/rdf-mapping/index.html[TED-RDF Mappings].
+
+== Quick references for users
 
 * xref:mapping_suite_cli_toolchain.adoc[Installation and usage instructions for the Mapping Suite CLI toolchain]
 * link:{attachmentsdir}/ted-sws-architecture/index.html[Preliminary project architecture (in progress)^]

docs/antora/modules/ROOT/pages/mapping_suite_cli_toolchain.adoc

@@ -1,6 +1,6 @@
 = Mapping Suite CLI toolchain
 
-This set of instructions is prepared fo semantic engineers who intend to work with mapping suites.
+This set of instructions is prepared fo semantic engineers who intend to work with mapping suites already available or being developed in the https://docs.ted.europa.eu/rdf-mapping/index.html[TED-RDF Mappings] repository.
 
 == Installation instructions for semantic engineers
 

ted_sws/core/model/transform.py

@@ -147,15 +147,37 @@ class ConceptualMappingRMLModule(MappingSuiteComponent):
     file_name: Optional[str]
 
 
+class ConceptualMappingRemark(MappingSuiteComponent):
+    """
+
+    """
+    standard_form_field_id: Optional[str]
+    standard_form_field_name: Optional[str]
+    field_xpath: Optional[List[str]]
+
+
+class ConceptualMappingControlList(MappingSuiteComponent):
+    """
+
+    """
+    field_value: Optional[str]
+    mapping_reference: Optional[str]
+    super_type: Optional[str]
+    xml_path_fragment: Optional[str]
+
+
 class ConceptualMapping(MappingSuiteComponent):
     """
 
     """
     xpaths: List[ConceptualMappingXPATH] = []
     metadata: Optional[ConceptualMappingMetadata]
     rules: List[ConceptualMappingRule] = []
+    mapping_remarks: List[ConceptualMappingRemark] = []
     resources: List[ConceptualMappingResource] = []
     rml_modules: List[ConceptualMappingRMLModule] = []
+    cl1_roles: List[ConceptualMappingControlList] = []
+    cl2_organisations: List[ConceptualMappingControlList] = []
 
 
 class MappingSuite(MappingSuiteComponent):

ted_sws/mapping_suite_processor/__init__.py

@@ -1,11 +1,19 @@
 CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME = "Metadata"
 CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME = "Rules"
+CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME = "Mapping Remarks"
 CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME = "Resources"
 CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME = "RML_Modules"
+CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME = "CL1 Controlled List of Roles"
+CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME = "CL2 Controlled List for Organis"
 
 RULES_E_FORM_BT_NAME = 'eForm BT Name (Provisional/Indicative) (O)'
 RULES_FIELD_XPATH = 'Field XPath (M)'
 RULES_FIELD_XPATH_CONDITION = 'Field XPath condition (M)'
 RULES_SF_FIELD_ID = 'Standard Form Field ID (M)'
 RULES_SF_FIELD_NAME = 'Standard Form Field Name (M)'
 RULES_E_FORM_BT_ID = 'eForm BT-ID (Provisional/Indicative) (O)'
+
+CL_FIELD_VALUE = 'Field Value (in XML)'
+CL_MAPPING_REFERENCE = 'Mapping Reference (in ePO)'
+CL_SUPERTYPE = 'SuperType'
+CL_XML_PATH_FRAGMENT = 'XML PATH Fragment'

ted_sws/mapping_suite_processor/entrypoints/cli/cmd_conceptual_mapping_differ.py

@@ -4,19 +4,19 @@
 from pathlib import Path
 
 import click
-from ted_sws.event_manager.adapters.log import LOG_WARN_TEXT
+from pydantic.utils import deep_update
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
+from ted_sws.core.model.transform import ConceptualMappingDiff
+from ted_sws.event_manager.adapters.log import LOG_WARN_TEXT
 from ted_sws.mapping_suite_processor.entrypoints.cli import CONCEPTUAL_MAPPINGS_FILE_TEMPLATE
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_differ import \
     mapping_suite_diff_files_conceptual_mappings, mapping_suite_diff_repo_conceptual_mappings, \
-    generate_conceptual_mappings_diff_html_report, transform_conceptual_mappings_diff_data
-from ted_sws.core.model.transform import ConceptualMappingDiff
+    generate_conceptual_mappings_diff_html_report, generate_conceptual_mappings_diff_filename
 
 CMD_NAME = "CMD_CONCEPTUAL_MAPPING_DIFFER"
 
 DEFAULT_REPORT_OUTPUT_FOLDER = "."
-DEFAULT_REPORT_FILE_NAME = "conceptual_mappings_diff"
 
 """
 USAGE:
@@ -51,16 +51,27 @@ def __init__(
         self.output_folder = output_folder
 
     def _report(self, diff, files: list):
-        data = diff['data']
-        report_file_file_name_json = Path(self.output_folder) / (DEFAULT_REPORT_FILE_NAME + ".json")
+        diff['metadata'] = deep_update(diff['metadata'], {
+            "branches": self.branches,
+            "mapping_suite_ids": self.mapping_suite_ids,
+            "files": files
+        })
+        diff_object = ConceptualMappingDiff(**diff)
+        filename_base = generate_conceptual_mappings_diff_filename(diff_object)
+        output_path: Path = Path(self.output_folder)
+
+        report_file_file_name_json = output_path / (filename_base + ".json")
         with open(report_file_file_name_json, 'w+') as report_file:
-            report_file.write(json.dumps(data, indent=2))
-        report_file_file_name_html = Path(self.output_folder) / (DEFAULT_REPORT_FILE_NAME + ".html")
+            report_file.write(json.dumps(diff, indent=2))
+
+        report_file_file_name_html = output_path / (filename_base + ".html")
         with open(report_file_file_name_html, 'w+') as report_file:
-            report_file.write(
-                generate_conceptual_mappings_diff_html_report(
-                    ConceptualMappingDiff(**diff))
-            )
+            report_file.write(generate_conceptual_mappings_diff_html_report(diff_object))
+
+        self.log(
+            f"Generated {LOG_WARN_TEXT.format(report_file_file_name_json)}" +
+            f" and {LOG_WARN_TEXT.format(report_file_file_name_html)} report files" +
+            f" in \"{LOG_WARN_TEXT.format(self.output_folder)}\" folder.")
 
     @classmethod
     def _conceptual_mappings_file_path(cls, mappings_path, mapping_suite_id):

ted_sws/mapping_suite_processor/services/conceptual_mapping_differ.py

@@ -21,14 +21,19 @@
 GITHUB_CONCEPTUAL_MAPPINGS_PATH = "{GITHUB_BASE}/raw/{GIT_BRANCH}/mappings/{MAPPING_SUITE_ID}/" + \
                                   MS_TRANSFORM_FOLDER_NAME + "/" + MS_CONCEPTUAL_MAPPING_FILE_NAME
 
+DEFAULT_REPORT_FILE_NAME = "cm_diff"
+
 
 class ConceptualMappingDiffDataTransformer:
     data: dict
     tabs: dict = {
         "metadata": {},
         "rules": {},
+        "mapping_remarks": {},
         "resources": {},
-        "rml_modules": {}
+        "rml_modules": {},
+        "cl1_roles": {},
+        "cl2_organisations": {}
     }
     labels: dict
 
@@ -45,8 +50,11 @@ def init_labels(cls):
             "tabs": {
                 "metadata": "Metadata",
                 "rules": "Rules",
+                "mapping_remarks": "Remarks",
                 "resources": "Resources",
-                "rml_modules": "RML Modules"
+                "rml_modules": "RML Modules",
+                "cl1_roles": "CL1 Roles",
+                "cl2_organisations": "CL2 Organisations"
             },
             "actions": {
                 "set_item_added": "Set Added",
@@ -81,7 +89,11 @@ def init_labels(cls):
                 "fragment_fingerprint": "Fragment Fingerprint",
                 "file_name": "File name",
                 "old_value": "Old value",
-                "new_value": "New value"
+                "new_value": "New value",
+                "field_value": "Field Value (in XML)",
+                "mapping_reference": "Mapping Reference (in ePO)",
+                "super_type": "SuperType",
+                "xml_path_fragment": "XML PATH Fragment"
             }
         }
 
@@ -227,3 +239,19 @@ def generate_conceptual_mappings_diff_html_report(diff: ConceptualMappingDiff):
     )
     html_report = TEMPLATES.get_template(CONCEPTUAL_MAPPINGS_DIFF_HTML_REPORT_TEMPLATE).render(diff)
     return html_report
+
+
+def generate_conceptual_mappings_diff_filename(diff: ConceptualMappingDiff, prefix: str = DEFAULT_REPORT_FILE_NAME,
+                                               ext: str = None) -> str:
+    filename: str = prefix
+    cm1_metadata: dict = diff.metadata.metadata[0]
+    if cm1_metadata:
+        filename += f"_{cm1_metadata['identifier']}_v{cm1_metadata['mapping_version']}"
+    cm2_metadata: dict = diff.metadata.metadata[1]
+    if cm2_metadata:
+        if cm1_metadata:
+            filename += "_vs"
+        filename += f"_{cm2_metadata['identifier']}_v{cm2_metadata['mapping_version']}"
+    if ext:
+        filename += ext
+    return filename

ted_sws/mapping_suite_processor/services/conceptual_mapping_reader.py

@@ -5,11 +5,14 @@
 import pandas as pd
 
 from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingXPATH, ConceptualMappingMetadata, \
-    ConceptualMappingResource, ConceptualMappingMetadataConstraints, ConceptualMappingRule, ConceptualMappingRMLModule
+    ConceptualMappingResource, ConceptualMappingMetadataConstraints, ConceptualMappingRule, ConceptualMappingRMLModule, \
+    ConceptualMappingRemark, ConceptualMappingControlList
 from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME, \
     CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_SF_FIELD_ID, RULES_SF_FIELD_NAME, \
     CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME, CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME, RULES_E_FORM_BT_ID, \
-    RULES_E_FORM_BT_NAME, RULES_FIELD_XPATH_CONDITION
+    RULES_E_FORM_BT_NAME, RULES_FIELD_XPATH_CONDITION, CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME, \
+    CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME, CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME, CL_MAPPING_REFERENCE, \
+    CL_SUPERTYPE, CL_FIELD_VALUE, CL_XML_PATH_FRAGMENT
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_files_injection import FILE_NAME_KEY
 from ted_sws.notice_validator import BASE_XPATH_FIELD
 
@@ -124,6 +127,25 @@ def _read_conceptual_mapping_rules(df: pd.DataFrame) -> List[ConceptualMappingRu
     return rules
 
 
+def _read_conceptual_mapping_remarks(df: pd.DataFrame) -> List[ConceptualMappingRemark]:
+    """
+
+    :param df:
+    :return:
+    """
+
+    remarks_df = df[0:].copy()
+    remarks = []
+    remark: ConceptualMappingRemark
+    for idx, row in remarks_df.iterrows():
+        remark = ConceptualMappingRemark()
+        remark.standard_form_field_id = _read_pd_value(row[RULES_SF_FIELD_ID])
+        remark.standard_form_field_name = _read_pd_value(row[RULES_SF_FIELD_NAME])
+        remark.field_xpath = _read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH])
+        remarks.append(remark)
+    return remarks
+
+
 def _read_conceptual_mapping_resources(df: pd.DataFrame) -> List[ConceptualMappingResource]:
     """
 
@@ -156,6 +178,28 @@ def _read_conceptual_mapping_rml_modules(df: pd.DataFrame) -> List[ConceptualMap
     return rml_modules
 
 
+def _read_conceptual_mapping_control_list(df: pd.DataFrame) -> List[ConceptualMappingControlList]:
+    """
+
+    :param df:
+    :return:
+    """
+
+    df.columns = df.iloc[0]
+    control_list_df = df[1:].copy()
+
+    control_list = []
+    item: ConceptualMappingControlList
+    for idx, row in control_list_df.iterrows():
+        item = ConceptualMappingControlList()
+        item.field_value = _read_pd_value(row[CL_FIELD_VALUE])
+        item.mapping_reference = _read_pd_value(row[CL_MAPPING_REFERENCE])
+        item.super_type = _read_pd_value(row[CL_SUPERTYPE])
+        item.xml_path_fragment = _read_pd_value(row[CL_XML_PATH_FRAGMENT])
+        control_list.append(item)
+    return control_list
+
+
 def _read_conceptual_mapping_xpaths(rules_df: pd.DataFrame, base_xpath: str) -> List[ConceptualMappingXPATH]:
     """
 
@@ -209,9 +253,14 @@ def mapping_suite_read_conceptual_mapping(conceptual_mappings_file_path: Path) -
         metadata = _read_conceptual_mapping_metadata(dfs[CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME])
         conceptual_mapping.metadata = metadata
         conceptual_mapping.rules = _read_conceptual_mapping_rules(dfs[CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME])
+        conceptual_mapping.mapping_remarks = _read_conceptual_mapping_remarks(dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME])
         conceptual_mapping.resources = _read_conceptual_mapping_resources(dfs[CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME])
         conceptual_mapping.rml_modules = _read_conceptual_mapping_rml_modules(
             dfs[CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME])
+        conceptual_mapping.cl1_roles = _read_conceptual_mapping_control_list(
+            dfs[CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME])
+        conceptual_mapping.cl2_organisations = _read_conceptual_mapping_control_list(
+            dfs[CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME])
         conceptual_mapping.xpaths = _read_conceptual_mapping_xpaths(
             rules_df=dfs[CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME][1:].copy(),
             base_xpath=metadata.base_xpath

tests/e2e/mapping_suite_processor/conceptual_mapping_differ/test_conceptual_mapping_differ.py

@@ -1,8 +1,9 @@
-from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingMetadata
+from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingMetadata, ConceptualMappingDiff
 from ted_sws.data_manager.adapters.mapping_suite_repository import MS_TRANSFORM_FOLDER_NAME, \
     MS_CONCEPTUAL_MAPPING_FILE_NAME
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_differ import mapping_suite_diff_conceptual_mappings, \
-    mapping_suite_diff_files_conceptual_mappings, mapping_suite_diff_repo_conceptual_mappings
+    mapping_suite_diff_files_conceptual_mappings, mapping_suite_diff_repo_conceptual_mappings, \
+    generate_conceptual_mappings_diff_filename
 
 
 def test_mapping_suite_diff_conceptual_mappings():
@@ -53,3 +54,14 @@ def test_mapping_suite_diff_repo_conceptual_mappings(github_mapping_suite_id, pa
         mapping_suite_id=[github_mapping_suite_id, "package_F03_test"]
     )
     assert diff['data']['original']
+
+
+def test_generate_conceptual_mappings_diff_filename(package_folder_path, package_F03_folder_path):
+    """"""
+    filepath1 = package_folder_path / MS_TRANSFORM_FOLDER_NAME / MS_CONCEPTUAL_MAPPING_FILE_NAME
+    filepath2 = package_F03_folder_path / MS_TRANSFORM_FOLDER_NAME / MS_CONCEPTUAL_MAPPING_FILE_NAME
+
+    diff = mapping_suite_diff_files_conceptual_mappings([filepath1, filepath2])
+    filename = generate_conceptual_mappings_diff_filename(ConceptualMappingDiff(**diff), "conceptual_mappings_diff",
+                                                          ".json")
+    assert filename == "conceptual_mappings_diff_F03_v0.0.1_vs_package_F03_v2.2.0.json"