Merge pull request #413 from OP-TED/feature/TED-1053

Feature/ted 1053

Author: kaleanych

ted_sws/core/model/transform.py

@@ -147,15 +147,37 @@ class ConceptualMappingRMLModule(MappingSuiteComponent):
     file_name: Optional[str]
 
 
+class ConceptualMappingRemark(MappingSuiteComponent):
+    """
+
+    """
+    standard_form_field_id: Optional[str]
+    standard_form_field_name: Optional[str]
+    field_xpath: Optional[List[str]]
+
+
+class ConceptualMappingControlList(MappingSuiteComponent):
+    """
+
+    """
+    field_value: Optional[str]
+    mapping_reference: Optional[str]
+    super_type: Optional[str]
+    xml_path_fragment: Optional[str]
+
+
 class ConceptualMapping(MappingSuiteComponent):
     """
 
     """
     xpaths: List[ConceptualMappingXPATH] = []
     metadata: Optional[ConceptualMappingMetadata]
     rules: List[ConceptualMappingRule] = []
+    mapping_remarks: List[ConceptualMappingRemark] = []
     resources: List[ConceptualMappingResource] = []
     rml_modules: List[ConceptualMappingRMLModule] = []
+    cl1_roles: List[ConceptualMappingControlList] = []
+    cl2_organisations: List[ConceptualMappingControlList] = []
 
 
 class MappingSuite(MappingSuiteComponent):

ted_sws/mapping_suite_processor/__init__.py

@@ -1,11 +1,19 @@
 CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME = "Metadata"
 CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME = "Rules"
+CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME = "Mapping Remarks"
 CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME = "Resources"
 CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME = "RML_Modules"
+CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME = "CL1 Controlled List of Roles"
+CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME = "CL2 Controlled List for Organis"
 
 RULES_E_FORM_BT_NAME = 'eForm BT Name (Provisional/Indicative) (O)'
 RULES_FIELD_XPATH = 'Field XPath (M)'
 RULES_FIELD_XPATH_CONDITION = 'Field XPath condition (M)'
 RULES_SF_FIELD_ID = 'Standard Form Field ID (M)'
 RULES_SF_FIELD_NAME = 'Standard Form Field Name (M)'
 RULES_E_FORM_BT_ID = 'eForm BT-ID (Provisional/Indicative) (O)'
+
+CL_FIELD_VALUE = 'Field Value (in XML)'
+CL_MAPPING_REFERENCE = 'Mapping Reference (in ePO)'
+CL_SUPERTYPE = 'SuperType'
+CL_XML_PATH_FRAGMENT = 'XML PATH Fragment'

ted_sws/mapping_suite_processor/entrypoints/cli/cmd_conceptual_mapping_differ.py

@@ -4,14 +4,15 @@
 from pathlib import Path
 
 import click
-from ted_sws.event_manager.adapters.log import LOG_WARN_TEXT
+from pydantic.utils import deep_update
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
+from ted_sws.core.model.transform import ConceptualMappingDiff
+from ted_sws.event_manager.adapters.log import LOG_WARN_TEXT
 from ted_sws.mapping_suite_processor.entrypoints.cli import CONCEPTUAL_MAPPINGS_FILE_TEMPLATE
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_differ import \
     mapping_suite_diff_files_conceptual_mappings, mapping_suite_diff_repo_conceptual_mappings, \
-    generate_conceptual_mappings_diff_html_report, transform_conceptual_mappings_diff_data
-from ted_sws.core.model.transform import ConceptualMappingDiff
+    generate_conceptual_mappings_diff_html_report
 
 CMD_NAME = "CMD_CONCEPTUAL_MAPPING_DIFFER"
 
@@ -51,15 +52,20 @@ def __init__(
         self.output_folder = output_folder
 
     def _report(self, diff, files: list):
-        data = diff['data']
+        diff['metadata'] = deep_update(diff['metadata'], {
+            "branches": self.branches,
+            "mapping_suite_ids": self.mapping_suite_ids,
+            "files": files
+        })
         report_file_file_name_json = Path(self.output_folder) / (DEFAULT_REPORT_FILE_NAME + ".json")
         with open(report_file_file_name_json, 'w+') as report_file:
-            report_file.write(json.dumps(data, indent=2))
+            report_file.write(json.dumps(diff, indent=2))
         report_file_file_name_html = Path(self.output_folder) / (DEFAULT_REPORT_FILE_NAME + ".html")
         with open(report_file_file_name_html, 'w+') as report_file:
             report_file.write(
                 generate_conceptual_mappings_diff_html_report(
-                    ConceptualMappingDiff(**diff))
+                    ConceptualMappingDiff(**diff)
+                )
             )
 
     @classmethod

ted_sws/mapping_suite_processor/services/conceptual_mapping_differ.py

@@ -27,8 +27,11 @@ class ConceptualMappingDiffDataTransformer:
     tabs: dict = {
         "metadata": {},
         "rules": {},
+        "mapping_remarks": {},
         "resources": {},
-        "rml_modules": {}
+        "rml_modules": {},
+        "cl1_roles": {},
+        "cl2_organisations": {}
     }
     labels: dict
 
@@ -45,8 +48,11 @@ def init_labels(cls):
             "tabs": {
                 "metadata": "Metadata",
                 "rules": "Rules",
+                "mapping_remarks": "Remarks",
                 "resources": "Resources",
-                "rml_modules": "RML Modules"
+                "rml_modules": "RML Modules",
+                "cl1_roles": "CL1 Roles",
+                "cl2_organisations": "CL2 Organisations"
             },
             "actions": {
                 "set_item_added": "Set Added",
@@ -81,7 +87,11 @@ def init_labels(cls):
                 "fragment_fingerprint": "Fragment Fingerprint",
                 "file_name": "File name",
                 "old_value": "Old value",
-                "new_value": "New value"
+                "new_value": "New value",
+                "field_value": "Field Value (in XML)",
+                "mapping_reference": "Mapping Reference (in ePO)",
+                "super_type": "SuperType",
+                "xml_path_fragment": "XML PATH Fragment"
             }
         }
 

ted_sws/mapping_suite_processor/services/conceptual_mapping_reader.py

@@ -5,11 +5,14 @@
 import pandas as pd
 
 from ted_sws.core.model.transform import ConceptualMapping, ConceptualMappingXPATH, ConceptualMappingMetadata, \
-    ConceptualMappingResource, ConceptualMappingMetadataConstraints, ConceptualMappingRule, ConceptualMappingRMLModule
+    ConceptualMappingResource, ConceptualMappingMetadataConstraints, ConceptualMappingRule, ConceptualMappingRMLModule, \
+    ConceptualMappingRemark, ConceptualMappingControlList
 from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME, \
     CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME, RULES_FIELD_XPATH, RULES_SF_FIELD_ID, RULES_SF_FIELD_NAME, \
     CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME, CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME, RULES_E_FORM_BT_ID, \
-    RULES_E_FORM_BT_NAME, RULES_FIELD_XPATH_CONDITION
+    RULES_E_FORM_BT_NAME, RULES_FIELD_XPATH_CONDITION, CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME, \
+    CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME, CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME, CL_MAPPING_REFERENCE, \
+    CL_SUPERTYPE, CL_FIELD_VALUE, CL_XML_PATH_FRAGMENT
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_files_injection import FILE_NAME_KEY
 from ted_sws.notice_validator import BASE_XPATH_FIELD
 
@@ -124,6 +127,25 @@ def _read_conceptual_mapping_rules(df: pd.DataFrame) -> List[ConceptualMappingRu
     return rules
 
 
+def _read_conceptual_mapping_remarks(df: pd.DataFrame) -> List[ConceptualMappingRemark]:
+    """
+
+    :param df:
+    :return:
+    """
+
+    remarks_df = df[0:].copy()
+    remarks = []
+    remark: ConceptualMappingRemark
+    for idx, row in remarks_df.iterrows():
+        remark = ConceptualMappingRemark()
+        remark.standard_form_field_id = _read_pd_value(row[RULES_SF_FIELD_ID])
+        remark.standard_form_field_name = _read_pd_value(row[RULES_SF_FIELD_NAME])
+        remark.field_xpath = _read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH])
+        remarks.append(remark)
+    return remarks
+
+
 def _read_conceptual_mapping_resources(df: pd.DataFrame) -> List[ConceptualMappingResource]:
     """
 
@@ -156,6 +178,28 @@ def _read_conceptual_mapping_rml_modules(df: pd.DataFrame) -> List[ConceptualMap
     return rml_modules
 
 
+def _read_conceptual_mapping_control_list(df: pd.DataFrame) -> List[ConceptualMappingControlList]:
+    """
+
+    :param df:
+    :return:
+    """
+
+    df.columns = df.iloc[0]
+    control_list_df = df[1:].copy()
+
+    control_list = []
+    item: ConceptualMappingControlList
+    for idx, row in control_list_df.iterrows():
+        item = ConceptualMappingControlList()
+        item.field_value = _read_pd_value(row[CL_FIELD_VALUE])
+        item.mapping_reference = _read_pd_value(row[CL_MAPPING_REFERENCE])
+        item.super_type = _read_pd_value(row[CL_SUPERTYPE])
+        item.xml_path_fragment = _read_pd_value(row[CL_XML_PATH_FRAGMENT])
+        control_list.append(item)
+    return control_list
+
+
 def _read_conceptual_mapping_xpaths(rules_df: pd.DataFrame, base_xpath: str) -> List[ConceptualMappingXPATH]:
     """
 
@@ -209,9 +253,14 @@ def mapping_suite_read_conceptual_mapping(conceptual_mappings_file_path: Path) -
         metadata = _read_conceptual_mapping_metadata(dfs[CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME])
         conceptual_mapping.metadata = metadata
         conceptual_mapping.rules = _read_conceptual_mapping_rules(dfs[CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME])
+        conceptual_mapping.mapping_remarks = _read_conceptual_mapping_remarks(dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME])
         conceptual_mapping.resources = _read_conceptual_mapping_resources(dfs[CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME])
         conceptual_mapping.rml_modules = _read_conceptual_mapping_rml_modules(
             dfs[CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME])
+        conceptual_mapping.cl1_roles = _read_conceptual_mapping_control_list(
+            dfs[CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME])
+        conceptual_mapping.cl2_organisations = _read_conceptual_mapping_control_list(
+            dfs[CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME])
         conceptual_mapping.xpaths = _read_conceptual_mapping_xpaths(
             rules_df=dfs[CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME][1:].copy(),
             base_xpath=metadata.base_xpath