Merge branch 'main' into feature/TED-1066

Author: CaptainOfHacks

ted_sws/core/model/transform.py

@@ -72,6 +72,7 @@ class ConceptualMappingXPATH(MappingSuiteComponent):
 
 class ConceptualMappingDiff(MappingSuiteComponent):
     """"""
+    created_at: str = datetime.now().isoformat()
     metadata: Optional[dict]
     data: Optional[dict]
 

ted_sws/mapping_suite_processor/entrypoints/cli/cmd_conceptual_mapping_differ.py

@@ -44,13 +44,14 @@ def __init__(
             output_folder
     ):
         super().__init__(name=CMD_NAME)
-        self.mapping_suite_id = self._init_list_input_opts_split(mapping_suite_id)
-        self.file = self._init_list_input_opts_split(file)
-        self.branch = self._init_list_input_opts_split(branch)
+        self.mapping_suite_ids = self._init_list_input_opts_split(mapping_suite_id)
+        self.files = self._init_list_input_opts_split(file)
+        self.branches = self._init_list_input_opts_split(branch)
         self.mappings_path = mappings_path
         self.output_folder = output_folder
 
-    def _report(self, data, files: list):
+    def _report(self, diff, files: list):
+        data = diff['data']
         report_file_file_name_json = Path(self.output_folder) / (DEFAULT_REPORT_FILE_NAME + ".json")
         with open(report_file_file_name_json, 'w+') as report_file:
             report_file.write(json.dumps(data, indent=2))
@@ -60,9 +61,11 @@ def _report(self, data, files: list):
                 generate_conceptual_mappings_diff_html_report(
                     ConceptualMappingDiff(
                         metadata={
-                            "branches": self.branch,
-                            "mapping_suite_ids": self.mapping_suite_id,
-                            "files": files
+                            "branches": self.branches,
+                            "mapping_suite_ids": self.mapping_suite_ids,
+                            "files": files,
+                            "defaults": diff['metadata']['defaults'],
+                            "metadata": diff['metadata']['metadata']
                         },
                         data=data
                     ))
@@ -81,12 +84,12 @@ def _mappings_path(self) -> Path:
         return mappings_path
 
     def _display_input(self):
-        if self.branch:
-            self.log(LOG_WARN_TEXT.format("GIT Branches: ") + str(self.branch))
-        if self.mapping_suite_id:
-            self.log(LOG_WARN_TEXT.format("MappingSuites: ") + str(self.mapping_suite_id))
-        if self.file:
-            self.log(LOG_WARN_TEXT.format("Files: ") + str(self.file))
+        if self.branches:
+            self.log(LOG_WARN_TEXT.format("GIT Branches: ") + str(self.branches))
+        if self.mapping_suite_ids:
+            self.log(LOG_WARN_TEXT.format("MappingSuites: ") + str(self.mapping_suite_ids))
+        if self.files:
+            self.log(LOG_WARN_TEXT.format("Files: ") + str(self.files))
 
     def run_cmd(self):
         self._display_input()
@@ -95,49 +98,51 @@ def run_cmd(self):
         filepath1 = None
         filepath2 = None
 
-        file_len = len(self.file)
-        mapping_suite_id_len = len(self.mapping_suite_id)
-        branch_len = len(self.branch)
+        file_len = len(self.files)
+        mapping_suite_id_len = len(self.mapping_suite_ids)
+        branch_len = len(self.branches)
 
-        if not self.branch:
-            if self.file and file_len == 2:
-                filepath1 = self.file[0]
+        if not self.branches:
+            if self.files and file_len == 2:
+                filepath1 = self.files[0]
                 assert Path(filepath1).is_file()
-                filepath2 = self.file[1]
+                filepath2 = self.files[1]
                 assert Path(filepath2).is_file()
-            elif self.mapping_suite_id:
+            elif self.mapping_suite_ids:
                 mappings_path = self._mappings_path()
                 if mapping_suite_id_len == 2:
-                    filepath1 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[0])
+                    filepath1 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_ids[0])
                     assert Path(filepath1).is_file()
-                    filepath2 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[1])
+                    filepath2 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_ids[1])
                     assert Path(filepath2).is_file()
                 elif mapping_suite_id_len == 1 and file_len == 1:
-                    filepath1 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[0])
+                    filepath1 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_ids[0])
                     assert Path(filepath1).is_file()
-                    filepath2 = self.file[0]
+                    filepath2 = str(Path(self.files[0]).resolve())
                     assert Path(filepath2).is_file()
 
         error = None
         if filepath1 and filepath2:
             diff = mapping_suite_diff_files_conceptual_mappings([Path(filepath1), Path(filepath2)])
-        elif self.branch:
+        elif self.branches:
             assert mapping_suite_id_len > 0
-            if branch_len == 1 and mapping_suite_id_len == 1 and not self.file:
+            if branch_len == 1 and mapping_suite_id_len == 1 and not self.files:
                 mappings_path = self._mappings_path()
-                filepath2 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_id[0])
+                filepath2 = self._conceptual_mappings_file_path(mappings_path, self.mapping_suite_ids[0])
             else:
-                filepath2 = (self.file[0] if file_len == 1 else None)
+                filepath2 = (self.files[0] if file_len == 1 else None)
 
             diff = mapping_suite_diff_repo_conceptual_mappings(
-                branch_or_tag_name=self.branch,
-                mapping_suite_id=self.mapping_suite_id,
+                branch_or_tag_name=self.branches,
+                mapping_suite_id=self.mapping_suite_ids,
                 filepath=Path(filepath2) if filepath2 else None
             )
         else:
             error = Exception("Cannot do a diff with provided input!")
 
-        self._report(data=diff, files=[filepath1, filepath2])
+        if not error:
+            self._report(diff=diff, files=[filepath1, filepath2])
+
         self.run_cmd_result(error)
 
 

ted_sws/mapping_suite_processor/resources/templates/conceptual_mappings_diff_report.jinja2

@@ -20,20 +20,85 @@
     table td {
         width: auto
     }
+
+    table.heading td,
+    table.heading th {
+        border: 1px solid #ddd;
+    }
+    table.heading th {
+        background-color: #eee;
+        text-align: left;
+    }
+    table.heading thead td {
+        text-align: left;
+        font-weight: bold;
+        width: 50%;
+    }
+    table.heading tbody th {
+        white-space: nowrap;
+    }
     </style>
   </head>
   <body>
 	<h1>Conceptual Mappings Diff HTML report </h1>
 	<hr>
-    <h3>
-    {{ metadata | replace('\n', '<br>') | replace(' ', '&nbsp;') }}
-    </h3>
+    <ul>
+        <li>Created at: {{ created_at }}</li>
+    </ul>
+	<hr>
+    <h2>
+        Conceptual Mappings
+    </h2>
+    <table class="heading">
+        <thead>
+            <tr>
+                <th></th>
+                <td>Conceptual Mapping 1</td>
+                <td>Conceptual Mapping 2</td>
+            </tr>
+        </thead>
+        <tbody>
+            {% set branch1 = metadata['branches'][0] or metadata['defaults']['branch'] or "local" %}
+            {% set branch2 = metadata['branches'][1] or metadata['defaults']['branch'] or "local" %}
+            <tr>
+                <th>Branch</th>
+                <td>{{ branch1 }}</td>
+                <td>{{ branch2 }}</td>
+            </tr>
+            {% set mapping_suite1 = metadata['mapping_suite_ids'][0] or metadata['mapping_suite_ids'][1] %}
+            {% set mapping_suite2 = metadata['mapping_suite_ids'][1] or metadata['mapping_suite_ids'][0] %}
+            <tr>
+                <th>Mapping Suite</th>
+                <td>{{ mapping_suite1 }}</td>
+                <td>{{ mapping_suite2 }}</td>
+            </tr>
+            {% set file1 = metadata['files'][0] or metadata['defaults']['conceptual_mapping'] %}
+            {% set file2 = metadata['files'][1] or metadata['defaults']['conceptual_mapping'] %}
+            <tr>
+                <th>Conceptual Mapping File</th>
+                <td>{{ file1 }}</td>
+                <td>{{ file2 }}</td>
+            </tr>
+            <tr>
+                <th>Mapping Version</th>
+                <td>{{ metadata['metadata'][0]['mapping_version'] }}</td>
+                <td>{{ metadata['metadata'][1]['mapping_version'] }}</td>
+            </tr>
+        </tbody>
+    </table>
     <hr>
     <h2>Report details: </h2>
-
-    {{ data }}
+    {% if data %}
+        {{ data }}
+    {% else %}
+        <h3>NO DIFF</h3>
+    {% endif %}
     <script>
-        document.querySelector('body>table').classList.add('dataTable');
+        const tables = document.querySelectorAll('body>table');
+        console.log(tables);
+        for (const table of tables) {
+          table.classList.add('dataTable');
+        }
     </script>
   </body>
 </html>

ted_sws/mapping_suite_processor/services/conceptual_mapping_differ.py

@@ -28,7 +28,19 @@ def mapping_suite_diff_conceptual_mappings(mappings: List[ConceptualMapping]) ->
     :return:
     """
     assert mappings and len(mappings) == 2
-    return DeepDiff(mappings[0].dict(), mappings[1].dict(), ignore_order=True)
+    diff: ConceptualMappingDiff = ConceptualMappingDiff()
+    diff.metadata = {
+        "defaults": {
+            "branch": "local",
+            "conceptual_mapping": MS_TRANSFORM_FOLDER_NAME + "/" + MS_CONCEPTUAL_MAPPING_FILE_NAME
+        },
+        "metadata": [
+            mappings[0].metadata.dict(),
+            mappings[1].metadata.dict()
+        ]
+    }
+    diff.data = DeepDiff(mappings[0].dict(), mappings[1].dict(), ignore_order=True)
+    return diff.dict()
 
 
 def mapping_suite_diff_files_conceptual_mappings(filepaths: List[Path]) -> dict:
@@ -103,7 +115,8 @@ def mapping_suite_diff_repo_conceptual_mappings(branch_or_tag_name: List[str], m
 
 def generate_conceptual_mappings_diff_html_report(diff: ConceptualMappingDiff):
     html_report = TEMPLATES.get_template(CONCEPTUAL_MAPPINGS_DIFF_HTML_REPORT_TEMPLATE).render({
-        "metadata": json.dumps(diff.metadata, indent=2),
+        "metadata": diff.metadata,
+        "created_at": diff.created_at,
         "data": json2html.convert(
             json=diff.data,
             table_attributes='class="display" border="1"',

ted_sws/master_data_registry/resources/__init__.py

@@ -3,4 +3,5 @@
 MASTER_DATA_REGISTRY_RESOURCES_PATH = pathlib.Path(__file__).parent.resolve()
 
 TRIPLES_BY_CET_URI_SPARQL_QUERY_TEMPLATE_PATH = MASTER_DATA_REGISTRY_RESOURCES_PATH / "sparql_query_templates/get_by_cet_uri.rq"
+PROCEDURE_SUBJECTS_SPARQL_QUERY_TEMPLATE_PATH = MASTER_DATA_REGISTRY_RESOURCES_PATH / "sparql_query_templates/get_procedure_uris.rq"
 RDF_FRAGMENT_BY_URI_SPARQL_QUERY_TEMPLATE_PATH = MASTER_DATA_REGISTRY_RESOURCES_PATH / "sparql_query_templates/get_2_dependency_levels_for_a_uri_as_root.rq"

ted_sws/master_data_registry/resources/sparql_query_templates/get_procedure_uris.rq

@@ -0,0 +1,8 @@
+prefix epo:<http://data.europa.eu/a4g/ontology#>
+
+SELECT DISTINCT ?s
+WHERE {
+  ?s a epo:Procedure.
+  ?o ?po ?s.
+  filter not exists {?o epo:refersToPreviousProcedure ?s}
+}

ted_sws/master_data_registry/services/entity_deduplication.py

@@ -17,7 +17,7 @@
 from ted_sws.event_manager.services.log import log_error, log_notice_error
 from ted_sws.master_data_registry.services.rdf_fragment_processor import get_rdf_fragments_by_cet_uri_from_notices, \
     merge_rdf_fragments_into_graph, write_rdf_fragments_in_triple_store, RDF_FRAGMENT_FROM_NOTICE_PROPERTY, \
-    get_subjects_by_cet_uri, get_rdf_fragment_by_cet_uri_from_notice
+    get_procedure_subjects, get_rdf_fragment_by_root_uri_from_notice
 
 MDR_TEMPORARY_FUSEKI_DATASET_NAME = "tmp_mdr_dataset"
 MDR_FUSEKI_DATASET_NAME = "mdr_dataset"
@@ -258,7 +258,7 @@ def deduplicate_procedure_entities(notices: List[Notice], procedure_cet_uri: str
         if parent_notice and parent_notice.rdf_manifestation and parent_notice.rdf_manifestation.object_data:
             rdf_content = parent_notice.rdf_manifestation.object_data
             sparql_endpoint = SPARQLStringEndpoint(rdf_content=rdf_content)
-            result_uris = get_subjects_by_cet_uri(sparql_endpoint=sparql_endpoint, cet_uri=procedure_cet_uri)
+            result_uris = get_procedure_subjects(sparql_endpoint=sparql_endpoint)
             result_uris_len = len(result_uris)
             if result_uris_len != 1:
                 notice_normalised_metadata = parent_notice.normalised_metadata
@@ -269,11 +269,12 @@ def deduplicate_procedure_entities(notices: List[Notice], procedure_cet_uri: str
                     notice_status=parent_notice.status,
                     notice_eforms_subtype=notice_normalised_metadata.eforms_subtype if notice_normalised_metadata else None)
             else:
+                result_uri = result_uris[0]
                 parent_procedure_uri = rdflib.URIRef(result_uris[0])
                 parent_uries[parent_notice_id] = parent_procedure_uri
-                parent_procedure_rdf_fragments = get_rdf_fragment_by_cet_uri_from_notice(notice=parent_notice,
-                                                                                         cet_uri=procedure_cet_uri)
-                parent_new_cet = {parent_procedure_uri: parent_procedure_rdf_fragments[0]}
+                parent_procedure_rdf_fragment = get_rdf_fragment_by_root_uri_from_notice(notice=parent_notice,
+                                                                                         root_uri=result_uri)
+                parent_new_cet = {parent_procedure_uri: parent_procedure_rdf_fragment}
                 register_new_cets_in_mdr(new_canonical_entities=parent_new_cet, triple_store=triple_store,
                                          mdr_dataset_name=mdr_dataset_name)
 
@@ -282,7 +283,7 @@ def deduplicate_procedure_entities(notices: List[Notice], procedure_cet_uri: str
         for child_notice in notice_families[parent_uri_key]:
             rdf_content = child_notice.rdf_manifestation.object_data
             sparql_endpoint = SPARQLStringEndpoint(rdf_content=rdf_content)
-            result_uris = get_subjects_by_cet_uri(sparql_endpoint=sparql_endpoint, cet_uri=procedure_cet_uri)
+            result_uris = get_procedure_subjects(sparql_endpoint=sparql_endpoint)
             result_uris_len = len(result_uris)
             if result_uris_len != 1:
                 notice_normalised_metadata = child_notice.normalised_metadata

ted_sws/master_data_registry/services/rdf_fragment_processor.py

@@ -10,15 +10,15 @@
 """
 import pathlib
 from string import Template
-from typing import List, Tuple
+from typing import List, Tuple, Optional
 
 import rdflib
 
 from ted_sws.core.model.notice import Notice
 from ted_sws.data_manager.adapters.sparql_endpoint import SPARQLStringEndpoint
 from ted_sws.data_manager.adapters.triple_store import TripleStoreABC
 from ted_sws.master_data_registry.resources import RDF_FRAGMENT_BY_URI_SPARQL_QUERY_TEMPLATE_PATH, \
-    TRIPLES_BY_CET_URI_SPARQL_QUERY_TEMPLATE_PATH
+    TRIPLES_BY_CET_URI_SPARQL_QUERY_TEMPLATE_PATH, PROCEDURE_SUBJECTS_SPARQL_QUERY_TEMPLATE_PATH
 
 RDFTriple = Tuple[rdflib.term.Node, rdflib.term.Node, rdflib.term.Node]
 
@@ -39,6 +39,17 @@ def get_subjects_by_cet_uri(sparql_endpoint: SPARQLStringEndpoint, cet_uri: str)
     return query_table_result["s"].to_list()
 
 
+def get_procedure_subjects(sparql_endpoint: SPARQLStringEndpoint) -> List[str]:
+    """
+        This function return a list of procedure subjects.
+    :param sparql_endpoint:
+    :return:
+    """
+    sparql_query = PROCEDURE_SUBJECTS_SPARQL_QUERY_TEMPLATE_PATH.read_text(encoding="utf-8")
+    query_table_result = sparql_endpoint.with_query(sparql_query=sparql_query).fetch_tabular()
+    return query_table_result["s"].to_list()
+
+
 def get_rdf_fragment_by_root_uri(sparql_endpoint: SPARQLStringEndpoint, root_uri: str,
                                  inject_triples: List[RDFTriple] = None) -> rdflib.Graph:
     """
@@ -92,6 +103,25 @@ def get_rdf_fragments_by_cet_uri_from_file(rdf_file_path: pathlib.Path, cet_uri:
                                                    rdf_content_format=rdf_file_content_format)
 
 
+def get_rdf_fragment_by_root_uri_from_notice(notice: Notice, root_uri: str) -> Optional[rdflib.Graph]:
+    """
+         This function extracts from a Notice RDF content a RDFFragment dependent on a root URI.
+    :param notice:
+    :param root_uri:
+    :return:
+    """
+    sparql_endpoint = SPARQLStringEndpoint(rdf_content=notice.rdf_manifestation.object_data,
+                                          rdf_content_format=DEFAULT_RDF_FILE_FORMAT)
+    rdf_fragment = get_rdf_fragment_by_root_uri(sparql_endpoint=sparql_endpoint, root_uri=root_uri,
+                                                inject_triples=[(rdflib.URIRef(root_uri),
+                                                                 RDF_FRAGMENT_FROM_NOTICE_PROPERTY,
+                                                                 rdflib.Literal(notice.ted_id))
+                                                                ]
+                                                )
+    return rdf_fragment
+
+
+
 def get_rdf_fragment_by_cet_uri_from_notice(notice: Notice, cet_uri: str) -> List[rdflib.Graph]:
     """
         This function extracts from a Notice RDF content a list of RDFFragments dependent on a CET URI.

tests/e2e/mapping_suite_processor/conceptual_mapping_differ/test_cmd_conceptual_mapping_differ.py

@@ -48,6 +48,7 @@ def test_cmd_conceptual_mapping_differ(caplog, cli_runner, fake_test_mapping_sui
         response = cli_runner.invoke(cli_main,
                                      ["--opt-mappings-folder", temp_mapping_suite_path,
                                       "--opt-output-folder", temp_folder])
+
         assert "FAILED" in response.output
         assert "Cannot do a diff" in response.output