Merge branch 'main' into feature/TED-1046

Author: CaptainOfHacks

docs/antora/modules/ROOT/attachments/FATs/2022-11-15-TED-SWS-FAT-complete.html

@@ -10,17 +10,15 @@ xref:demo_installation.adoc[Installation instructions for development and testin
 
 xref:attachment$/aws-infra-docs/TED-SWS-AWS-Infrastructure-architecture-overview-v0.9.pdf[TED-SWS AWS Infrastructure architecture overview v0.9]
 
-xref:attachment$/aws-infra-docs/TED-SWS-AWS-Installation-manual-v0.9.pdf[TED-SWS AWS Installation manual v0.9]
-
-Developers who need to know how the logger works can find here a xref:event_manager.adoc[little explanation].
+xref:attachment$/aws-infra-docs/TED-SWS Installation manual v2.0.2.pdf[TED-SWS AWS Installation manual v2.0.2]
 
 == Project roadmap
 
 |===
 |Reference | Description | Estimated delivery | Factory Acceptance Test | Release date | Release
 
 | Phase 1 | The first phase places high priority on the deployment into the OP AWS Cloud environment.| August 2022 | xref:attachment$/FATs/2022-08-29-report/index.html[2022-08-29 report] | 29 August 2022 | link:https://github.com/OP-TED/ted-rdf-conversion-pipeline/releases/tag/0.0.9-beta[0.0.9-beta]
-| Phase 2 | Provided that the deployment in the acceptance environment is successful, the delivery of Phase 2 aims to provide the first production version of the TED SWS system. | Nov 2022 | --- | --- | ---
+| Phase 2 | Provided that the deployment in the acceptance environment is successful, the delivery of Phase 2 aims to provide the first production version of the TED SWS system. | Nov 2022 | xref:attachment$/FATs/2022-11-22-TED-SWS-FAT-complete.html[2022-11-22 report] | 20 Nov 2022 | https://github.com/OP-TED/ted-rdf-conversion-pipeline/releases/tag/1.0.0-beta[1.0.0-beta]
 | Phase 3 | This phase delivers the documentation and components and improvements that could not be covered in the previous phases. | Feb 2023 | --- | --- | ---
 
 |===

docs/antora/modules/ROOT/attachments/FATs/2022-11-22-TED-SWS-FAT-complete.html

@@ -150,7 +150,6 @@ class MappingSuite(MappingSuiteComponent):
     title: str = "no_title"
     version: str = "0.1.1"
     ontology_version: str = "0.0.1"
-    xsd_version: str = "no_xsd_version"
     git_latest_commit_hash: str = "no_hash"
     mapping_suite_hash_digest: str = "no_hash"
     metadata_constraints: MetadataConstraints

docs/antora/modules/ROOT/attachments/aws-infra-docs/TED-SWS Installation manual v2.0.2.pdf

@@ -30,4 +30,4 @@ def create_batch_collection_materialised_view(mongo_client: MongoClient):
         {
             "$out": NOTICE_PROCESS_BATCH_COLLECTION_NAME
         }
-    ])
+    ], allowDiskUse=True)

docs/antora/modules/ROOT/pages/index.adoc

@@ -40,7 +40,7 @@ def create_notice_collection_materialised_view(mongo_client: MongoClient):
         {
             "$out": NOTICES_MATERIALISED_VIEW_NAME
         }
-    ])
+    ], allowDiskUse=True)
     materialised_view = database[NOTICES_MATERIALISED_VIEW_NAME]
     materialised_view.create_index([("created_at", DESCENDING)])
     materialised_view.create_index([("publication_date", DESCENDING)])
@@ -80,4 +80,4 @@ def create_notice_kpi_collection(mongo_client: MongoClient):
         {
             "$out": NOTICE_KPI_COLLECTION_NAME
         }
-    ])
+    ], allowDiskUse=True)

ted_sws/core/model/transform.py

@@ -1,7 +1,7 @@
 import pathlib
 import tempfile
 from io import StringIO
-from typing import List, Set, Tuple, Dict
+from typing import List, Tuple, Dict
 import rdflib
 from pymongo import MongoClient
 from rdflib import RDF, URIRef, OWL
@@ -12,16 +12,17 @@
 from ted_sws.core.model.notice import Notice
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.data_manager.adapters.sparql_endpoint import SPARQLStringEndpoint
-from ted_sws.data_manager.adapters.triple_store import FusekiAdapter, TripleStoreABC, FusekiException, \
+from ted_sws.data_manager.adapters.triple_store import FusekiAdapter, TripleStoreABC, \
     FUSEKI_REPOSITORY_ALREADY_EXIST_ERROR_MSG
-from ted_sws.event_manager.services.log import log_error
+from ted_sws.event_manager.services.log import log_error, log_notice_error
 from ted_sws.master_data_registry.services.rdf_fragment_processor import get_rdf_fragments_by_cet_uri_from_notices, \
     merge_rdf_fragments_into_graph, write_rdf_fragments_in_triple_store, RDF_FRAGMENT_FROM_NOTICE_PROPERTY, \
-    get_subjects_by_cet_uri
+    get_subjects_by_cet_uri, get_rdf_fragment_by_cet_uri_from_notice
 
 MDR_TEMPORARY_FUSEKI_DATASET_NAME = "tmp_mdr_dataset"
 MDR_FUSEKI_DATASET_NAME = "mdr_dataset"
 MDR_CANONICAL_CET_PROPERTY = rdflib.term.URIRef("http://www.meaningfy.ws/mdr#isCanonicalEntity")
+DEDUPLICATE_PROCEDURE_ENTITIES_DOMAIN_ACTION = "deduplicate_procedure_entities"
 
 
 def generate_mdr_alignment_links(merged_rdf_fragments: rdflib.Graph, cet_uri: str,
@@ -225,12 +226,14 @@ def deduplicate_entities_by_cet_uri(notices: List[Notice], cet_uri: str,
                                        alignment_graph=cet_alignment_links, inject_reflexive_links=True)
 
 
-def deduplicate_procedure_entities(notices: List[Notice], procedure_cet_uri: str, mongodb_client: MongoClient):
+def deduplicate_procedure_entities(notices: List[Notice], procedure_cet_uri: str, mongodb_client: MongoClient,
+                                   mdr_dataset_name: str = MDR_FUSEKI_DATASET_NAME):
     """
          This function deduplicate procedure entities for each notice from batch of notices.
     :param notices:
     :param procedure_cet_uri:
     :param mongodb_client:
+    :param mdr_dataset_name:
     :return:
     """
     notice_families = defaultdict(list)
@@ -242,26 +245,57 @@ def deduplicate_procedure_entities(notices: List[Notice], procedure_cet_uri: str
 
     parent_uries = {}
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
+    triple_store = FusekiAdapter()
+    if mdr_dataset_name not in triple_store.list_repositories():
+        try:
+            triple_store.create_repository(repository_name=mdr_dataset_name)
+        except Exception as exception:
+            if str(exception) != FUSEKI_REPOSITORY_ALREADY_EXIST_ERROR_MSG:
+                log_error(message=str(exception))
+
     for parent_notice_id in notice_families.keys():
         parent_notice = notice_repository.get(reference=parent_notice_id)
         if parent_notice and parent_notice.rdf_manifestation and parent_notice.rdf_manifestation.object_data:
             rdf_content = parent_notice.rdf_manifestation.object_data
             sparql_endpoint = SPARQLStringEndpoint(rdf_content=rdf_content)
             result_uris = get_subjects_by_cet_uri(sparql_endpoint=sparql_endpoint, cet_uri=procedure_cet_uri)
-            assert len(result_uris) == 1
-            parent_procedure_uri = rdflib.URIRef(result_uris[0])
-            parent_uries[parent_notice_id] = parent_procedure_uri
+            result_uris_len = len(result_uris)
+            if result_uris_len != 1:
+                notice_normalised_metadata = parent_notice.normalised_metadata
+                log_notice_error(
+                    message=f"Parent notice with notice_id=[{parent_notice.ted_id}] have {result_uris_len} Procedure CETs!",
+                    notice_id=parent_notice.ted_id, domain_action=DEDUPLICATE_PROCEDURE_ENTITIES_DOMAIN_ACTION,
+                    notice_form_number=notice_normalised_metadata.form_number if notice_normalised_metadata else None,
+                    notice_status=parent_notice.status,
+                    notice_eforms_subtype=notice_normalised_metadata.eforms_subtype if notice_normalised_metadata else None)
+            else:
+                parent_procedure_uri = rdflib.URIRef(result_uris[0])
+                parent_uries[parent_notice_id] = parent_procedure_uri
+                parent_procedure_rdf_fragments = get_rdf_fragment_by_cet_uri_from_notice(notice=parent_notice,
+                                                                                         cet_uri=procedure_cet_uri)
+                parent_new_cet = {parent_procedure_uri: parent_procedure_rdf_fragments[0]}
+                register_new_cets_in_mdr(new_canonical_entities=parent_new_cet, triple_store=triple_store,
+                                         mdr_dataset_name=mdr_dataset_name)
 
     for parent_uri_key in parent_uries.keys():
         parent_uri = parent_uries[parent_uri_key]
         for child_notice in notice_families[parent_uri_key]:
             rdf_content = child_notice.rdf_manifestation.object_data
             sparql_endpoint = SPARQLStringEndpoint(rdf_content=rdf_content)
             result_uris = get_subjects_by_cet_uri(sparql_endpoint=sparql_endpoint, cet_uri=procedure_cet_uri)
-            assert len(result_uris) == 1
-            child_procedure_uri = rdflib.URIRef(result_uris[0])
-            inject_links = rdflib.Graph()
-            inject_links.add((child_procedure_uri, OWL.sameAs, parent_uri))
-            child_notice.distilled_rdf_manifestation.object_data = '\n'.join(
-                [child_notice.distilled_rdf_manifestation.object_data,
-                 str(inject_links.serialize(format="nt"))])
+            result_uris_len = len(result_uris)
+            if result_uris_len != 1:
+                notice_normalised_metadata = child_notice.normalised_metadata
+                log_notice_error(
+                    message=f"Child notice with notice_id=[{child_notice.ted_id}] have {result_uris_len} Procedure CETs!",
+                    notice_id=child_notice.ted_id, domain_action=DEDUPLICATE_PROCEDURE_ENTITIES_DOMAIN_ACTION,
+                    notice_form_number=notice_normalised_metadata.form_number if notice_normalised_metadata else None,
+                    notice_status=child_notice.status,
+                    notice_eforms_subtype=notice_normalised_metadata.eforms_subtype if notice_normalised_metadata else None)
+            else:
+                child_procedure_uri = rdflib.URIRef(result_uris[0])
+                inject_links = rdflib.Graph()
+                inject_links.add((child_procedure_uri, OWL.sameAs, parent_uri))
+                child_notice.distilled_rdf_manifestation.object_data = '\n'.join(
+                    [child_notice.distilled_rdf_manifestation.object_data,
+                     str(inject_links.serialize(format="nt"))])

ted_sws/data_manager/services/create_batch_collection_materialised_view.py

@@ -24,7 +24,7 @@
 		"locn": "http://www.w3.org/ns/locn#",
 		"legal": "https://www.w3.org/ns/legal#",
 
-		"core-voc": "http://data.europa.eu/m8g/",
+		"cv": "http://data.europa.eu/m8g/",
 		"cccev": "http://data.europa.eu/m8g/",
 		"cpov": "http://data.europa.eu/m8g/",
 		"geosparql": "http://www.opengis.net/ont/geosparql#",
@@ -51,10 +51,10 @@
 
 	"prefix_selections" : {
 		"sparql_generator" : ["xml", "xsd", "rdf", "rdfs", "owl", "skos", "dc", "dct", "foaf", "epo", "epd", "locn", "legal"],
-		"sparql_generator_ext" : ["time", "vann", "cc", "org", "core-voc", "cccev", "cpov", "geosparql", "dul"],
+		"sparql_generator_ext" : ["time", "vann", "cc", "org", "cv", "cccev", "cpov", "geosparql", "dul"],
 		"yarrrml_rules" : ["epo", "epd", "locn", "xml", "xsd", "rdf", "rdfs", "dct"],
 		"rml_rules" : ["rr", "rml", "ql", "epo", "epd", "tedm", "locn", "xml", "xsd", "rdf", "rdfs", "owl", "dct", "cc", "cccev", "cpov", "skos", "time", "vann", "legal"],
-		"epo_ontology" : ["epo", "epd", "locn", "xml", "xsd", "rdf", "rdfs", "owl", "skos", "dc", "dct", "foaf", "core-voc", "cccev", "cpov", "geosparql", "dul", "cc", "time", "vann", "legal"],
+		"epo_ontology" : ["epo", "epd", "locn", "xml", "xsd", "rdf", "rdfs", "owl", "skos", "dc", "dct", "foaf", "cv", "cccev", "cpov", "geosparql", "dul", "cc", "time", "vann", "legal"],
 		"currently_unused" : ["epor", "epos", "grel", "xf", "map", "sd", "ht", "v"]
 	}
 }

ted_sws/data_manager/services/create_notice_collection_materialised_view.py

@@ -58,7 +58,6 @@ def the_notice_rdf_manifestation_publication_is_executed(publish_eligible_notice
     publish_result = publish_notice_rdf_into_s3(notice=publish_eligible_notice, s3_publisher=s3_publisher,
                                                 bucket_name=s3_bucket_name)
     assert publish_result
-    publish_eligible_notice.update_status_to(new_status=NoticeStatus.PUBLISHED)
     return publish_eligible_notice
 
 
@@ -80,5 +79,5 @@ def the_rdf_manifestation_is_available_in_a_s3_bucket(s3_publisher, s3_bucket_na
 @then('the notice status is PUBLISHED')
 def the_notice_status_is_published(published_notice: Notice, s3_publisher, s3_bucket_name):
     """the notice status is PUBLISHED."""
-    assert published_notice.status == NoticeStatus.PUBLISHED
+    assert published_notice.status == NoticeStatus.ELIGIBLE_FOR_PUBLISHING