bug fixes

CaptainOfHacks · CaptainOfHacks · commit b3cd5d1ee7bf · 2023-02-06T21:11:18.000+02:00
diff --git a/ted_sws/core/model/notice.py b/ted_sws/core/model/notice.py
@@ -203,7 +203,7 @@ def original_metadata(self) -> Optional[TEDMetadata]:
     @property
     def xml_manifestation(self) -> XMLManifestation:
         if self._xml_manifestation is None:
-            self.load_lazy_field(property_field=Notice.xml_metadata)
+            self.load_lazy_field(property_field=Notice.xml_manifestation)
         return self._xml_manifestation
 
     def set_original_metadata(self, ted_metadata: TEDMetadata):
diff --git a/ted_sws/data_manager/adapters/manifestation_repository.py b/ted_sws/data_manager/adapters/manifestation_repository.py
@@ -13,6 +13,7 @@
 FILE_STORAGE_COLLECTION_NAME = "fs.files"
 MANIFESTATION_ID = "manifestation_id"
 OBJECT_DATA_KEY = "object_data"
+AGGREGATE_REFERENCE_ID = "ted_id"
 
 
 class BaseManifestationRepository(ManifestationRepositoryABC):
@@ -25,6 +26,7 @@ def __init__(self, mongodb_client: MongoClient, database_name: str = None):
         db = mongodb_client[self._database_name]
         self.file_storage = gridfs.GridFS(db)  # TODO: Investigate how it works in multiple processes in parallel.
         self.collection = db[self._collection_name]
+        self.collection.create_index([(AGGREGATE_REFERENCE_ID, ASCENDING)])
         self.file_storage_collection = db[FILE_STORAGE_COLLECTION_NAME]
         self.file_storage_collection.create_index([(MANIFESTATION_ID,
                                                     ASCENDING)])  # TODO: index creation may bring race condition error.
@@ -56,6 +58,8 @@ def _update_manifestation(self, reference: str, manifestation: Manifestation, up
         """
         if manifestation is not None:
             manifestation_dict = manifestation.dict()
+            manifestation_dict[AGGREGATE_REFERENCE_ID] = reference
+            reference = self._build_reference(base_reference=reference)
             manifestation_dict[MONGODB_COLLECTION_ID] = reference
             old_linked_manifestation_file = self.file_storage.find_one({MANIFESTATION_ID: reference})
             manifestation_dict[OBJECT_DATA_KEY] = self._put_file_content_in_grid_fs(file_reference=reference,
@@ -66,10 +70,12 @@ def _update_manifestation(self, reference: str, manifestation: Manifestation, up
                 self.file_storage.delete(file_id=old_linked_manifestation_file._id)
 
     def _get_manifestation_dict(self, reference: str) -> Optional[dict]:
+        reference = self._build_reference(base_reference=reference)
         result_dict = self.collection.find_one({MONGODB_COLLECTION_ID: reference})
         if result_dict:
             result_dict[OBJECT_DATA_KEY] = self._get_file_content_from_grid_fs(file_id=result_dict[OBJECT_DATA_KEY])
             del result_dict[MONGODB_COLLECTION_ID]
+            del result_dict[AGGREGATE_REFERENCE_ID]
         return result_dict
 
     @abc.abstractmethod
@@ -95,7 +101,6 @@ def add(self, reference: str, manifestation: Manifestation):
         :param manifestation:
         :return:
         """
-        reference = self._build_reference(base_reference=reference)
         self._update_manifestation(reference=reference, manifestation=manifestation, upsert=True)
 
     def update(self, reference: str, manifestation: Manifestation):
@@ -105,8 +110,6 @@ def update(self, reference: str, manifestation: Manifestation):
         :param manifestation:
         :return:
         """
-        reference = self._build_reference(base_reference=reference)
-        reference = f"{reference}_rdf"
         self._update_manifestation(reference=reference, manifestation=manifestation)
 
     def get(self, reference: str) -> Optional[Manifestation]:
@@ -115,7 +118,6 @@ def get(self, reference: str) -> Optional[Manifestation]:
         :param reference:
         :return:
         """
-        reference = self._build_reference(base_reference=reference)
         result_dict = self._get_manifestation_dict(reference=reference)
         if result_dict is not None:
             return self._build_manifestation_from_dict(manifestation_dict=result_dict)
diff --git a/ted_sws/data_manager/adapters/metadata_repository.py b/ted_sws/data_manager/adapters/metadata_repository.py
@@ -1,13 +1,14 @@
 import abc
 from typing import Optional
 
-from pymongo import MongoClient
+from pymongo import MongoClient, ASCENDING
 
 from ted_sws import config
 from ted_sws.core.model.metadata import Metadata, NormalisedMetadata, TEDMetadata, XMLMetadata
 from ted_sws.data_manager.adapters.repository_abc import RepositoryABC
 
 MONGODB_COLLECTION_ID = "_id"
+AGGREGATE_REFERENCE_ID = "ted_id"
 
 
 class BaseMetadataRepository(RepositoryABC):
@@ -22,6 +23,7 @@ def __init__(self, mongodb_client: MongoClient, database_name: str = None):
         self.mongodb_client = mongodb_client
         db = mongodb_client[self._database_name]
         self.collection = db[self._collection_name]
+        self.collection.create_index([(AGGREGATE_REFERENCE_ID, ASCENDING)])
 
     def _update_metadata(self, reference: str, metadata: Metadata, upsert: bool = False):
         """
@@ -33,6 +35,7 @@ def _update_metadata(self, reference: str, metadata: Metadata, upsert: bool = Fa
         """
         if metadata is not None:
             metadata_dict = metadata.dict()
+            metadata_dict[AGGREGATE_REFERENCE_ID] = reference
             reference = self._build_reference(base_reference=reference)
             metadata_dict[MONGODB_COLLECTION_ID] = reference
             self.collection.update_one({MONGODB_COLLECTION_ID: reference}, {"$set": metadata_dict}, upsert=upsert)
@@ -47,6 +50,7 @@ def _get_metadata_dict(self, reference: str) -> Optional[dict]:
         result_dict = self.collection.find_one({MONGODB_COLLECTION_ID: reference})
         if result_dict:
             del result_dict[MONGODB_COLLECTION_ID]
+            del result_dict[AGGREGATE_REFERENCE_ID]
         return result_dict
 
     @abc.abstractmethod
diff --git a/ted_sws/data_manager/adapters/notice_repository.py b/ted_sws/data_manager/adapters/notice_repository.py
@@ -38,6 +38,7 @@
 NOTICE_TED_METADATA = "original_metadata"
 NOTICE_XML_MANIFESTATION = "xml_manifestation"
 NOTICE_XML_METADATA = "xml_metadata"
+VALIDATION_SUMMARY = "validation_summary"
 
 METADATA_PUBLICATION_DATE = "publication_date"
 METADATA_DOCUMENT_SENT_DATE = "document_sent_date"
@@ -46,11 +47,13 @@
 NOTICE_NORMALISED_METADATA_PRIVATE_KEY = "_normalised_metadata"
 NOTICE_XML_METADATA_PRIVATE_KEY = "_xml_metadata"
 NOTICE_XML_MANIFESTATION_PRIVATE_KEY = "_xml_manifestation"
+NOTICE_PREPROCESSED_XML_MANIFESTATION_KEY = "_preprocessed_xml_manifestation"
 NOTICE_RDF_MANIFESTATION_PRIVATE_KEY = "_rdf_manifestation"
 NOTICE_DISTILLED_RDF_MANIFESTATION_PRIVATE_KEY = "_distilled_rdf_manifestation"
 NOTICE_METS_MANIFESTATION_PRIVATE_KEY = "_mets_manifestation"
 
 
+
 class NoticeRepositoryInFileSystem(NoticeRepositoryABC):
     """
        This repository is intended for storing Notice objects as JSON files in file system.
@@ -182,6 +185,10 @@ def _mapping_lazy_fields(self):
                                   self.xml_metadata_repository),
             Notice.xml_manifestation: (NOTICE_XML_MANIFESTATION_PRIVATE_KEY,
                                        self.xml_manifestation_repository),
+            #@Note: preprocessed_xml_manifestation at the moment is same as xml_manifestation
+            # in this case is used same repository, in future need to create another repository
+            Notice.preprocessed_xml_manifestation: (NOTICE_PREPROCESSED_XML_MANIFESTATION_KEY,
+                                                    self.xml_manifestation_repository),
             Notice.rdf_manifestation: (NOTICE_RDF_MANIFESTATION_PRIVATE_KEY,
                                        self.rdf_manifestation_repository),
             Notice.distilled_rdf_manifestation: (NOTICE_DISTILLED_RDF_MANIFESTATION_PRIVATE_KEY,
@@ -234,7 +241,8 @@ def _create_dict_from_notice(notice: Notice) -> dict:
         :return:
         """
 
-        notice_dict = notice.dict(include={NOTICE_TED_ID: True, NOTICE_STATUS: True, NOTICE_CREATED_AT: True})
+        notice_dict = notice.dict(include={NOTICE_TED_ID: True, NOTICE_STATUS: True,
+                                           NOTICE_CREATED_AT: True, VALIDATION_SUMMARY: True})
         notice_dict[MONGODB_COLLECTION_ID] = notice_dict[NOTICE_TED_ID]
         notice_dict[NOTICE_STATUS] = str(notice_dict[NOTICE_STATUS])
         notice_dict[NOTICE_CREATED_AT] = datetime.fromisoformat(notice_dict[NOTICE_CREATED_AT])
diff --git a/ted_sws/data_sampler/services/notice_xml_indexer.py b/ted_sws/data_sampler/services/notice_xml_indexer.py
@@ -114,7 +114,7 @@ def get_unique_xpaths_from_notice_repository(mongodb_client: MongoClient) -> Lis
     :return:
     """
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
-    return notice_repository.collection.distinct("xml_metadata.unique_xpaths")
+    return notice_repository.xml_metadata_repository.collection.distinct("unique_xpaths")
 
 
 def get_unique_notice_id_from_notice_repository(mongodb_client: MongoClient) -> List[str]:
@@ -138,14 +138,14 @@ def get_minimal_set_of_xpaths_for_coverage_notices(notice_ids: List[str], mongod
     unique_notice_ids = notice_ids.copy()
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     while len(unique_notice_ids):
-        tmp_result = list(notice_repository.collection.aggregate([
-            {"$unwind": "$xml_metadata.unique_xpaths"},
+        tmp_result = list(notice_repository.xml_metadata_repository.collection.aggregate([
+            {"$unwind": "$unique_xpaths"},
             {"$match": {
-                "xml_metadata.unique_xpaths": {"$nin": minimal_set_of_xpaths},
+                "unique_xpaths": {"$nin": minimal_set_of_xpaths},
                 "ted_id": {"$in": unique_notice_ids}
             }
             },
-            {"$group": {"_id": "$xml_metadata.unique_xpaths", "count": {"$sum": 1},
+            {"$group": {"_id": "$unique_xpaths", "count": {"$sum": 1},
                         "notice_ids": {"$push": "$ted_id"}}},
             {"$sort": {"count": -1}},
             {"$limit": 1}
@@ -171,17 +171,17 @@ def get_minimal_set_of_notices_for_coverage_xpaths(notice_ids: List[str], mongod
     search_notices = notice_ids.copy()
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     while len(unique_xpaths):
-        tmp_result = list(notice_repository.collection.aggregate([
+        tmp_result = list(notice_repository.xml_metadata_repository.collection.aggregate([
             {"$match": {
-                "_id": {"$in": search_notices}
+                "ted_id": {"$in": search_notices}
             }
             },
-            {"$unwind": "$xml_metadata.unique_xpaths"},
+            {"$unwind": "$unique_xpaths"},
             {"$match": {
-                "xml_metadata.unique_xpaths": {"$in": unique_xpaths},
+                "unique_xpaths": {"$in": unique_xpaths},
             }
             },
-            {"$group": {"_id": "$ted_id", "count": {"$sum": 1}, "xpaths": {"$addToSet": "$xml_metadata.unique_xpaths"}
+            {"$group": {"_id": "$ted_id", "count": {"$sum": 1}, "xpaths": {"$addToSet": "$unique_xpaths"}
                         }},
             {"$sort": {"count": -1}},
             {"$limit": 1}
@@ -204,8 +204,8 @@ def get_unique_notices_id_covered_by_xpaths(xpaths: List[str], mongodb_client: M
     :return:
     """
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
-    results = list(notice_repository.collection.aggregate([
-        {"$match": {"xml_metadata.unique_xpaths": {"$in": xpaths}}},
+    results = list(notice_repository.xml_metadata_repository.collection.aggregate([
+        {"$match": {"unique_xpaths": {"$in": xpaths}}},
         {
             "$group": {"_id": None,
                        "ted_ids": {"$push": "$ted_id"}
@@ -223,12 +223,11 @@ def get_unique_xpaths_covered_by_notices(notice_ids: List[str], mongodb_client:
     :return:
     """
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
-    results = notice_repository.collection.aggregate([{"$match": {"ted_id": {"$in": notice_ids}}}], allowDiskUse=True)
+    results = notice_repository.xml_metadata_repository.collection.aggregate([{"$match": {"ted_id": {"$in": notice_ids}}}], allowDiskUse=True)
     unique_xpaths = set()
     for result in results:
-        xml_metadata = result["xml_metadata"]
-        if xml_metadata is not None:
-            unique_xpaths.update(result["xml_metadata"]["unique_xpaths"])
+        if result["unique_xpaths"] is not None:
+            unique_xpaths.update(result["unique_xpaths"])
     return list(unique_xpaths)
 
 
diff --git a/tests/unit/notice_validator/test_shacl_test_suite_runner.py b/tests/unit/notice_validator/test_shacl_test_suite_runner.py
@@ -66,6 +66,7 @@ def test_validate_notice_by_id_with_shacl_suite(notice_with_distilled_status, rd
     mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=path_to_file_system_repository)
     notice_repository.add(notice)
 
+    assert len(notice.get_rdf_validation()) == 0
     validate_notice_by_id_with_shacl_suite(notice_id="408313-2020",
                                            mapping_suite_repository=mapping_suite_repository,
                                            notice_repository=notice_repository,
