fix bugs and add improvements

Author: CaptainOfHacks

ted_sws/core/model/lazy_object.py

@@ -13,6 +13,15 @@ def load_lazy_field(self, source_object: Any, property_field: property) -> Any:
         :return:
         """
 
+    @abc.abstractmethod
+    def remove_lazy_field(self, source_object: Any, property_field: property):
+        """
+
+        :param source_object:
+        :param property_field:
+        :return:
+        """
+
 
 class LazyObjectABC(abc.ABC):
 
@@ -41,3 +50,13 @@ def load_lazy_field(self, property_field: property):
             return self.get_lazy_object_fields_loader().load_lazy_field(source_object=self,
                                                                         property_field=property_field)
         return None
+
+    def remove_lazy_field(self, property_field: property):
+        """
+
+        :param property_field:
+        :return:
+        """
+        if self.get_lazy_object_fields_loader():
+            self.get_lazy_object_fields_loader().remove_lazy_field(source_object=self,
+                                                                   property_field=property_field)

ted_sws/core/model/notice.py

@@ -338,8 +338,8 @@ def _check_status_is_validated(self) -> bool:
 
         :return:
         """
-        if self._rdf_manifestation and self._distilled_rdf_manifestation and self.xml_manifestation:
-            if self._distilled_rdf_manifestation.is_validated() and self.xml_manifestation.is_validated():
+        if self.rdf_manifestation and self.distilled_rdf_manifestation and self.xml_manifestation:
+            if self.distilled_rdf_manifestation.is_validated() and self.xml_manifestation.is_validated():
                 return True
         return False
 
@@ -354,7 +354,7 @@ def set_rdf_validation(self, rdf_validation: Union[SPARQLTestSuiteValidationRepo
         if not self.rdf_manifestation:
             raise ValueError("Cannot set the RDF validation of a non-existent RDF manifestation")
 
-        self._rdf_manifestation.add_validation(validation=rdf_validation)
+        self.rdf_manifestation.add_validation(validation=rdf_validation)
 
     def set_distilled_rdf_validation(self, rdf_validation: Union[SPARQLTestSuiteValidationReport,
                                                                  SHACLTestSuiteValidationReport]):
@@ -363,10 +363,10 @@ def set_distilled_rdf_validation(self, rdf_validation: Union[SPARQLTestSuiteVali
         :param rdf_validation:
         :return:
         """
-        if not self._distilled_rdf_manifestation:
+        if not self.distilled_rdf_manifestation:
             raise ValueError("Cannot set the RDF validation of a non-existent RDF manifestation")
 
-        self._distilled_rdf_manifestation.add_validation(validation=rdf_validation)
+        self.distilled_rdf_manifestation.add_validation(validation=rdf_validation)
 
         if self._check_status_is_validated():
             self.update_status_to(NoticeStatus.VALIDATED)
@@ -479,13 +479,17 @@ def update_status_to(self, new_status: NoticeStatus):
                 raise UnsupportedStatusTransition(
                     f"Unsupported transition from state {self._status} to state {new_status}.")
         elif self._status > new_status:
-            # TODO: implement delete actions
             self._status = new_status
+            if new_status < NoticeStatus.INDEXED:
+                self.remove_lazy_field(Notice.xml_metadata)
             if new_status < NoticeStatus.NORMALISED_METADATA:
-                self._normalised_metadata = None
+                self.remove_lazy_field(Notice.normalised_metadata)
+                #TODO: preprocessed_xml_manifestation is the same as xml_manifestation
+                # if delete preprocessed xml manifestation will delete xml_manifestation
+                # in future remove _preprocessed_xml_manifestation field from model
                 self._preprocessed_xml_manifestation = None
             if new_status < NoticeStatus.TRANSFORMED:
-                self._rdf_manifestation = None
-                self._distilled_rdf_manifestation = None
+                self.remove_lazy_field(Notice.rdf_manifestation)
+                self.remove_lazy_field(Notice.distilled_rdf_manifestation)
             if new_status < NoticeStatus.PACKAGED:
-                self._mets_manifestation = None
+                self.remove_lazy_field(Notice.mets_manifestation)

ted_sws/data_manager/adapters/manifestation_repository.py

@@ -14,10 +14,12 @@
 MANIFESTATION_ID = "manifestation_id"
 OBJECT_DATA_KEY = "object_data"
 AGGREGATE_REFERENCE_ID = "ted_id"
+MANIFESTATION_TYPE_ID = "manifestation_type"
 
 
 class BaseManifestationRepository(ManifestationRepositoryABC):
     _collection_name = "notice_manifestations"
+    _manifestation_type = "unknown"
 
     def __init__(self, mongodb_client: MongoClient, database_name: str = None):
         database_name = database_name if database_name else config.MONGO_DB_AGGREGATES_DATABASE_NAME
@@ -59,6 +61,7 @@ def _update_manifestation(self, reference: str, manifestation: Manifestation, up
         if manifestation is not None:
             manifestation_dict = manifestation.dict()
             manifestation_dict[AGGREGATE_REFERENCE_ID] = reference
+            manifestation_dict[MANIFESTATION_TYPE_ID] = self._manifestation_type
             reference = self._build_reference(base_reference=reference)
             manifestation_dict[MONGODB_COLLECTION_ID] = reference
             old_linked_manifestation_file = self.file_storage.find_one({MANIFESTATION_ID: reference})
@@ -76,15 +79,16 @@ def _get_manifestation_dict(self, reference: str) -> Optional[dict]:
             result_dict[OBJECT_DATA_KEY] = self._get_file_content_from_grid_fs(file_id=result_dict[OBJECT_DATA_KEY])
             del result_dict[MONGODB_COLLECTION_ID]
             del result_dict[AGGREGATE_REFERENCE_ID]
+            del result_dict[MANIFESTATION_TYPE_ID]
         return result_dict
 
-    @abc.abstractmethod
     def _build_reference(self, base_reference: str) -> str:
         """
 
         :param base_reference:
         :return:
         """
+        return f"{base_reference}_{self._manifestation_type}"
 
     @abc.abstractmethod
     def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
@@ -123,34 +127,36 @@ def get(self, reference: str) -> Optional[Manifestation]:
             return self._build_manifestation_from_dict(manifestation_dict=result_dict)
         return None
 
+    def remove(self, reference: str):
+        """
+            This method remove a manifestation based on an identification reference.
+        :param reference:
+        :return:
+        """
+        reference = self._build_reference(reference)
+        self.collection.delete_one({MONGODB_COLLECTION_ID: reference})
 
-class RDFManifestationRepository(BaseManifestationRepository):
 
-    def _build_reference(self, base_reference: str) -> str:
-        return f"{base_reference}_rdf"
+class RDFManifestationRepository(BaseManifestationRepository):
+    _manifestation_type: str = "rdf"
 
     def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
         return RDFManifestation(**manifestation_dict)
 
 
 class DistilledRDFManifestationRepository(RDFManifestationRepository):
-    def _build_reference(self, base_reference: str) -> str:
-        return f"{base_reference}_distilled_rdf"
+    _manifestation_type: str = "distilled_rdf"
 
 
 class XMLManifestationRepository(BaseManifestationRepository):
-
-    def _build_reference(self, base_reference: str) -> str:
-        return f"{base_reference}_xml"
+    _manifestation_type: str = "xml"
 
     def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
         return XMLManifestation(**manifestation_dict)
 
 
 class METSManifestationRepository(BaseManifestationRepository):
-
-    def _build_reference(self, base_reference: str) -> str:
-        return f"{base_reference}_mets"
+    _manifestation_type: str = "mets"
 
     def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
         return METSManifestation(**manifestation_dict)

ted_sws/data_manager/adapters/metadata_repository.py

@@ -5,17 +5,19 @@
 
 from ted_sws import config
 from ted_sws.core.model.metadata import Metadata, NormalisedMetadata, TEDMetadata, XMLMetadata
-from ted_sws.data_manager.adapters.repository_abc import RepositoryABC
+from ted_sws.data_manager.adapters.repository_abc import MetadataRepositoryABC
 
 MONGODB_COLLECTION_ID = "_id"
 AGGREGATE_REFERENCE_ID = "ted_id"
+METADATA_TYPE_ID = "metadata_type"
 
 
-class BaseMetadataRepository(RepositoryABC):
+class BaseMetadataRepository(MetadataRepositoryABC, abc.ABC):
     """
        This repository is intended for storing Metadata objects.
     """
-    _collection_name = "notice_metadata"
+    _collection_name: str = "notice_metadata"
+    _metadata_type: str = "unknown"
 
     def __init__(self, mongodb_client: MongoClient, database_name: str = None):
         database_name = database_name if database_name else config.MONGO_DB_AGGREGATES_DATABASE_NAME
@@ -24,6 +26,7 @@ def __init__(self, mongodb_client: MongoClient, database_name: str = None):
         db = mongodb_client[self._database_name]
         self.collection = db[self._collection_name]
         self.collection.create_index([(AGGREGATE_REFERENCE_ID, ASCENDING)])
+        self.collection.create_index([(METADATA_TYPE_ID, ASCENDING)])
 
     def _update_metadata(self, reference: str, metadata: Metadata, upsert: bool = False):
         """
@@ -36,6 +39,7 @@ def _update_metadata(self, reference: str, metadata: Metadata, upsert: bool = Fa
         if metadata is not None:
             metadata_dict = metadata.dict()
             metadata_dict[AGGREGATE_REFERENCE_ID] = reference
+            metadata_dict[METADATA_TYPE_ID] = self._metadata_type
             reference = self._build_reference(base_reference=reference)
             metadata_dict[MONGODB_COLLECTION_ID] = reference
             self.collection.update_one({MONGODB_COLLECTION_ID: reference}, {"$set": metadata_dict}, upsert=upsert)
@@ -51,21 +55,30 @@ def _get_metadata_dict(self, reference: str) -> Optional[dict]:
         if result_dict:
             del result_dict[MONGODB_COLLECTION_ID]
             del result_dict[AGGREGATE_REFERENCE_ID]
+            del result_dict[METADATA_TYPE_ID]
         return result_dict
 
-    @abc.abstractmethod
     def _build_reference(self, base_reference: str) -> str:
         """
 
         :param base_reference:
         :return:
         """
+        return f"{base_reference}_{self._metadata_type}"
 
 
-class NormalisedMetadataRepository(BaseMetadataRepository):
+    def remove(self, reference: str):
+        """
+            This method remove a metadata based on an identification reference.
+        :param reference:
+        :return:
+        """
+        reference = self._build_reference(reference)
+        self.collection.delete_one({MONGODB_COLLECTION_ID: reference})
 
-    def _build_reference(self, base_reference: str) -> str:
-        return f"{base_reference}_normalised"
+
+class NormalisedMetadataRepository(BaseMetadataRepository):
+    _metadata_type: str = "normalised"
 
     def add(self, reference: str, metadata: NormalisedMetadata):
         """
@@ -100,9 +113,7 @@ def get(self, reference: str) -> Optional[NormalisedMetadata]:
 
 
 class TEDMetadataRepository(BaseMetadataRepository):
-
-    def _build_reference(self, base_reference: str) -> str:
-        return f"{base_reference}_ted"
+    _metadata_type: str = "ted"
 
     def add(self, reference: str, metadata: TEDMetadata):
         """
@@ -136,8 +147,7 @@ def get(self, reference: str) -> Optional[TEDMetadata]:
 
 
 class XMLMetadataRepository(BaseMetadataRepository):
-    def _build_reference(self, base_reference: str) -> str:
-        return f"{base_reference}_xml"
+    _metadata_type: str = "xml"
 
     def add(self, reference: str, metadata: XMLMetadata):
         """

ted_sws/data_manager/adapters/notice_repository.py

@@ -53,7 +53,6 @@
 NOTICE_METS_MANIFESTATION_PRIVATE_KEY = "_mets_manifestation"
 
 
-
 class NoticeRepositoryInFileSystem(NoticeRepositoryABC):
     """
        This repository is intended for storing Notice objects as JSON files in file system.
@@ -185,7 +184,7 @@ def _mapping_lazy_fields(self):
                                   self.xml_metadata_repository),
             Notice.xml_manifestation: (NOTICE_XML_MANIFESTATION_PRIVATE_KEY,
                                        self.xml_manifestation_repository),
-            #@Note: preprocessed_xml_manifestation at the moment is same as xml_manifestation
+            # @Note: preprocessed_xml_manifestation at the moment is same as xml_manifestation
             # in this case is used same repository, in future need to create another repository
             Notice.preprocessed_xml_manifestation: (NOTICE_PREPROCESSED_XML_MANIFESTATION_KEY,
                                                     self.xml_manifestation_repository),
@@ -209,6 +208,12 @@ def load_lazy_field(self, source_object: Notice, property_field: property) -> An
         notice_field_data = field_repository.get(source_object.ted_id)
         setattr(source_object, notice_field, notice_field_data)
 
+    def remove_lazy_field(self, source_object: Any, property_field: property):
+        mapping_lazy_fields = self._mapping_lazy_fields()
+        notice_field, field_repository = mapping_lazy_fields[property_field]
+        field_repository.remove(source_object.ted_id)
+        setattr(source_object, notice_field, None)
+
     def _write_lazy_fields(self, notice: Notice):
         mapping_lazy_fields = self._mapping_lazy_fields()
         for notice_field, repository in mapping_lazy_fields.values():

ted_sws/data_manager/adapters/repository_abc.py

@@ -45,6 +45,13 @@ def get(self, reference: str) -> Optional[Metadata]:
         :return: Metadata
         """
 
+    @abc.abstractmethod
+    def remove(self, reference: str):
+        """
+            This method remove a metadata based on an identification reference.
+        :param reference:
+        :return:
+        """
 
 class ManifestationRepositoryABC(RepositoryABC):
     """
@@ -77,6 +84,13 @@ def get(self, reference: str) -> Optional[Manifestation]:
         :return: Manifestation
         """
 
+    @abc.abstractmethod
+    def remove(self, reference: str):
+        """
+            This method remove a manifestation based on an identification reference.
+        :param reference:
+        :return:
+        """
 
 class NoticeRepositoryABC(RepositoryABC):
     """

ted_sws/data_sampler/services/notice_selectors.py

@@ -20,7 +20,7 @@ def get_notice_ids_by_normalised_metadata_field_value(field_name: str, field_val
     :return:
     """
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
-    match_filter = {f"normalised_metadata.{field_name}": field_value, "xml_metadata": {"$ne": None}}
+    match_filter = {f"normalised_metadata.{field_name}": field_value}
     if notice_filter:
         match_filter.update(notice_filter)
     notice_ids = list(notice_repository.collection.aggregate([

ted_sws/data_sampler/services/notice_xml_indexer.py

@@ -139,6 +139,7 @@ def get_minimal_set_of_xpaths_for_coverage_notices(notice_ids: List[str], mongod
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     while len(unique_notice_ids):
         tmp_result = list(notice_repository.xml_metadata_repository.collection.aggregate([
+            {"$match": {"metadata_type": {"$eq": "xml"}}},
             {"$unwind": "$unique_xpaths"},
             {"$match": {
                 "unique_xpaths": {"$nin": minimal_set_of_xpaths},
@@ -173,7 +174,8 @@ def get_minimal_set_of_notices_for_coverage_xpaths(notice_ids: List[str], mongod
     while len(unique_xpaths):
         tmp_result = list(notice_repository.xml_metadata_repository.collection.aggregate([
             {"$match": {
-                "ted_id": {"$in": search_notices}
+                "ted_id": {"$in": search_notices},
+                "metadata_type": {"$eq": "xml"}
             }
             },
             {"$unwind": "$unique_xpaths"},
@@ -205,7 +207,10 @@ def get_unique_notices_id_covered_by_xpaths(xpaths: List[str], mongodb_client: M
     """
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     results = list(notice_repository.xml_metadata_repository.collection.aggregate([
-        {"$match": {"unique_xpaths": {"$in": xpaths}}},
+        {"$match": {"unique_xpaths": {"$in": xpaths},
+                    "metadata_type": {"$eq": "xml"}
+                    }
+         },
         {
             "$group": {"_id": None,
                        "ted_ids": {"$push": "$ted_id"}
@@ -223,7 +228,10 @@ def get_unique_xpaths_covered_by_notices(notice_ids: List[str], mongodb_client:
     :return:
     """
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
-    results = notice_repository.xml_metadata_repository.collection.aggregate([{"$match": {"ted_id": {"$in": notice_ids}}}], allowDiskUse=True)
+    results = notice_repository.xml_metadata_repository.collection.aggregate([{"$match": {"ted_id": {"$in": notice_ids},
+                                                                                          "metadata_type": {"$eq":"xml"}
+                                                                                          }
+                                                                               }], allowDiskUse=True)
     unique_xpaths = set()
     for result in results:
         if result["unique_xpaths"] is not None: