refactor data model, add lazy fields, refactor repositories

Author: CaptainOfHacks

ted_sws/core/model/lazy_object.py

@@ -0,0 +1,43 @@
+import abc
+from typing import Optional, Any
+
+
+class LazyObjectFieldsLoaderABC(abc.ABC):
+
+    @abc.abstractmethod
+    def load_lazy_field(self, source_object: Any, property_field: property) -> Any:
+        """
+
+        :param source_object:
+        :param property_field:
+        :return:
+        """
+
+
+class LazyObject:
+    _lazy_object_fields_loader: LazyObjectFieldsLoaderABC = None
+
+    def set_lazy_object_fields_loader(self, lazy_object_fields_loader: LazyObjectFieldsLoaderABC):
+        """
+
+        :param lazy_object_fields_loader:
+        :return:
+        """
+        self._lazy_object_fields_loader = lazy_object_fields_loader
+
+    def get_lazy_object_fields_loader(self) -> Optional[LazyObjectFieldsLoaderABC]:
+        """
+
+        :return:
+        """
+        return self._lazy_object_fields_loader
+
+    def load_lazy_field(self, property_field: property):
+        """
+
+        :param property_field:
+        :return:
+        """
+        if self._lazy_object_fields_loader:
+            return self._lazy_object_fields_loader.load_lazy_field(source_object=self, property_field=property_field)
+        return None

ted_sws/core/model/metadata.py

@@ -82,6 +82,31 @@ class NormalisedMetadata(Metadata):
     xsd_version: str
 
 
+class NormalisedMetadataView(Metadata):
+    title: str
+    long_title: str
+    notice_publication_number: str
+    publication_date: str
+    ojs_issue_number: str
+    ojs_type: str
+    city_of_buyer: Optional[str]
+    name_of_buyer: Optional[str]
+    original_language: Optional[str]
+    country_of_buyer: Optional[str]
+    eu_institution: Optional[bool]
+    document_sent_date: Optional[str]
+    deadline_for_submission: Optional[str]
+    notice_type: str
+    form_type: str
+    place_of_performance: Optional[List[str]]
+    extracted_legal_basis_directive: Optional[str]
+    legal_basis_directive: str
+    form_number: str
+    eforms_subtype: str
+    xsd_version: str
+
+
+
 class TEDMetadata(Metadata):
     """
         Stores notice original metadata

ted_sws/core/model/notice.py

@@ -21,6 +21,7 @@
 from pydantic import Field
 
 from ted_sws.core.model import PropertyBaseModel
+from ted_sws.core.model.lazy_object import LazyObject
 from ted_sws.core.model.manifestation import METSManifestation, RDFManifestation, XMLManifestation, \
     RDFValidationManifestation, SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport, \
     XPATHCoverageValidationReport, XMLValidationManifestation, ValidationSummaryReport
@@ -142,7 +143,7 @@ def update_status_to(self, new_status):
         """
 
 
-class Notice(WorkExpression):
+class Notice(WorkExpression, LazyObject):
     """
         A TED notice in any of its forms across the TED-SWS pipeline. This class is conceptualised as a merger of Work
         and Expression in the FRBR class hierarchy and is connected to some of its Manifestations.
@@ -161,36 +162,70 @@ class Notice(WorkExpression):
         The original XML manifestation of the notice as downloaded from the TED website.
 
     """
-    _status: NoticeStatus = NoticeStatus.RAW  # PrivateAttr(default=NoticeStatus.RAW)
+    _status: NoticeStatus = NoticeStatus.RAW
     ted_id: str = Field(..., allow_mutation=False)
-    original_metadata: Optional[TEDMetadata] = None
-    xml_manifestation: XMLManifestation = Field(..., allow_mutation=False)
+    _original_metadata: Optional[TEDMetadata] = None
+    _xml_manifestation: Optional[XMLManifestation] = None
     _normalised_metadata: Optional[NormalisedMetadata] = None
     _preprocessed_xml_manifestation: Optional[XMLManifestation] = None
     _distilled_rdf_manifestation: Optional[RDFManifestation] = None
     _rdf_manifestation: Optional[RDFManifestation] = None
     _mets_manifestation: Optional[METSManifestation] = None
-    xml_metadata: Optional[XMLMetadata] = None
+    _xml_metadata: Optional[XMLMetadata] = None
     validation_summary: Optional[ValidationSummaryReport] = None
 
+    @property
+    def original_metadata(self) -> Optional[TEDMetadata]:
+        if self._original_metadata is None:
+            self.load_lazy_field(property_field=Notice.original_metadata)
+        return self._original_metadata
+
+    @property
+    def xml_manifestation(self) -> XMLManifestation:
+        if self._xml_manifestation is None:
+            self.load_lazy_field(property_field=Notice.xml_metadata)
+        return self._xml_manifestation
+
+    def set_original_metadata(self, ted_metadata: TEDMetadata):
+        self._original_metadata = ted_metadata
+
+    def set_xml_manifestation(self, xml_manifestation: XMLManifestation):
+        self._xml_manifestation = xml_manifestation
+
+    @property
+    def xml_metadata(self) -> XMLMetadata:
+        if self._xml_metadata is None:
+            self.load_lazy_field(property_field=Notice.xml_metadata)
+        return self._xml_metadata
+
     @property
     def preprocessed_xml_manifestation(self) -> XMLManifestation:
+        if self._preprocessed_xml_manifestation is None:
+            self.load_lazy_field(property_field=Notice.preprocessed_xml_manifestation)
         return self._preprocessed_xml_manifestation
 
     @property
     def distilled_rdf_manifestation(self) -> RDFManifestation:
+        if self._distilled_rdf_manifestation is None:
+            self.load_lazy_field(property_field=Notice.distilled_rdf_manifestation)
         return self._distilled_rdf_manifestation
 
     @property
     def normalised_metadata(self) -> NormalisedMetadata:
+        if self._normalised_metadata is None:
+            self.load_lazy_field(property_field=Notice.normalised_metadata)
         return self._normalised_metadata
 
     @property
     def rdf_manifestation(self) -> RDFManifestation:
+        if self._rdf_manifestation is None:
+            self.load_lazy_field(property_field=Notice.rdf_manifestation)
         return self._rdf_manifestation
 
     @property
     def mets_manifestation(self) -> METSManifestation:
+        if self._mets_manifestation is None:
+            self.load_lazy_field(property_field=Notice.mets_manifestation)
         return self._mets_manifestation
 
     def get_rdf_validation(self) -> Optional[List[RDFValidationManifestation]]:
@@ -225,7 +260,7 @@ def set_xml_metadata(self, xml_metadata: XMLMetadata):
         :param xml_metadata:
         :return:
         """
-        self.xml_metadata = xml_metadata
+        self._xml_metadata = xml_metadata
         self.update_status_to(NoticeStatus.INDEXED)
 
     def set_preprocessed_xml_manifestation(self, preprocessed_xml_manifestation: XMLManifestation):

ted_sws/data_manager/adapters/manifestation_repository.py

@@ -0,0 +1,154 @@
+import abc
+from typing import Optional
+
+import gridfs
+from bson import ObjectId
+from pymongo import MongoClient, ASCENDING
+
+from ted_sws import config
+from ted_sws.core.model.manifestation import Manifestation, RDFManifestation, XMLManifestation, METSManifestation
+from ted_sws.data_manager.adapters.repository_abc import ManifestationRepositoryABC
+
+MONGODB_COLLECTION_ID = "_id"
+FILE_STORAGE_COLLECTION_NAME = "fs.files"
+MANIFESTATION_ID = "manifestation_id"
+OBJECT_DATA_KEY = "object_data"
+
+
+class BaseManifestationRepository(ManifestationRepositoryABC):
+    _collection_name = "notice_manifestations"
+
+    def __init__(self, mongodb_client: MongoClient, database_name: str = None):
+        database_name = database_name if database_name else config.MONGO_DB_AGGREGATES_DATABASE_NAME
+        self._database_name = database_name
+        self.mongodb_client = mongodb_client
+        db = mongodb_client[self._database_name]
+        self.file_storage = gridfs.GridFS(db)  # TODO: Investigate how it works in multiple processes in parallel.
+        self.collection = db[self._collection_name]
+        self.file_storage_collection = db[FILE_STORAGE_COLLECTION_NAME]
+        self.file_storage_collection.create_index([(MANIFESTATION_ID,
+                                                    ASCENDING)])  # TODO: index creation may bring race condition error.
+
+    def _get_file_content_from_grid_fs(self, file_id: str) -> str:
+        """
+            This method load file_content from GridFS by field_id.
+        :param file_id:
+        :return:
+        """
+        return self.file_storage.get(file_id=ObjectId(file_id)).read().decode("utf-8")
+
+    def _put_file_content_in_grid_fs(self, file_reference: str, file_content: str) -> ObjectId:
+        """
+            This method store file_content in GridFS and set notice_id as file metadata.
+        :param file_reference:
+        :param file_content:
+        :return:
+        """
+        return self.file_storage.put(data=file_content.encode("utf-8"), file_reference=file_reference)
+
+    def _update_manifestation(self, reference: str, manifestation: Manifestation, upsert: bool = False):
+        """
+
+        :param reference:
+        :param manifestation:
+        :param upsert:
+        :return:
+        """
+        if manifestation is not None:
+            manifestation_dict = manifestation.dict()
+            manifestation_dict[MONGODB_COLLECTION_ID] = reference
+            old_linked_manifestation_file = self.file_storage.find_one({MANIFESTATION_ID: reference})
+            manifestation_dict[OBJECT_DATA_KEY] = self._put_file_content_in_grid_fs(file_reference=reference,
+                                                                                    file_content=manifestation_dict[
+                                                                                        OBJECT_DATA_KEY])
+            self.collection.update_one({MONGODB_COLLECTION_ID: reference}, {"$set": manifestation_dict}, upsert=upsert)
+            if old_linked_manifestation_file is not None:
+                self.file_storage.delete(file_id=old_linked_manifestation_file._id)
+
+    def _get_manifestation_dict(self, reference: str) -> Optional[dict]:
+        result_dict = self.collection.find_one({MONGODB_COLLECTION_ID: reference})
+        if result_dict:
+            result_dict[OBJECT_DATA_KEY] = self._get_file_content_from_grid_fs(file_id=result_dict[OBJECT_DATA_KEY])
+            del result_dict[MONGODB_COLLECTION_ID]
+        return result_dict
+
+    @abc.abstractmethod
+    def _build_reference(self, base_reference: str) -> str:
+        """
+
+        :param base_reference:
+        :return:
+        """
+
+    @abc.abstractmethod
+    def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
+        """
+
+        :param manifestation_dict:
+        :return:
+        """
+
+    def add(self, reference: str, manifestation: Manifestation):
+        """
+
+        :param reference:
+        :param manifestation:
+        :return:
+        """
+        reference = self._build_reference(base_reference=reference)
+        self._update_manifestation(reference=reference, manifestation=manifestation, upsert=True)
+
+    def update(self, reference: str, manifestation: Manifestation):
+        """
+
+        :param reference:
+        :param manifestation:
+        :return:
+        """
+        reference = self._build_reference(base_reference=reference)
+        reference = f"{reference}_rdf"
+        self._update_manifestation(reference=reference, manifestation=manifestation)
+
+    def get(self, reference: str) -> Optional[Manifestation]:
+        """
+
+        :param reference:
+        :return:
+        """
+        reference = self._build_reference(base_reference=reference)
+        result_dict = self._get_manifestation_dict(reference=reference)
+        if result_dict is not None:
+            return self._build_manifestation_from_dict(manifestation_dict=result_dict)
+        return None
+
+
+class RDFManifestationRepository(BaseManifestationRepository):
+
+    def _build_reference(self, base_reference: str) -> str:
+        return f"{base_reference}_rdf"
+
+    def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
+        return RDFManifestation(**manifestation_dict)
+
+
+class DistilledRDFManifestationRepository(RDFManifestationRepository):
+    def _build_reference(self, base_reference: str) -> str:
+        return f"{base_reference}_distilled_rdf"
+
+
+class XMLManifestationRepository(BaseManifestationRepository):
+
+    def _build_reference(self, base_reference: str) -> str:
+        return f"{base_reference}_xml"
+
+    def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
+        return XMLManifestation(**manifestation_dict)
+
+
+class METSManifestationRepository(BaseManifestationRepository):
+
+    def _build_reference(self, base_reference: str) -> str:
+        return f"{base_reference}_mets"
+
+    def _build_manifestation_from_dict(self, manifestation_dict: dict) -> Manifestation:
+        return METSManifestation(**manifestation_dict)