feat: migrate metadata resource files loading to read from MS Config

Author: Kolea PLESCO

src/ted_sws/core/model/notice.py

@@ -192,6 +192,7 @@ class Notice(LazyWorkExpression):
     _mets_manifestation: Optional[METSManifestation] = None
     _xml_metadata: Optional[XMLMetadata] = None
     validation_summary: Optional[ValidationSummaryReport] = None
+    mapping_package_identifier: Optional[str] = None
 
     @computed_field
     @property

src/ted_sws/core/model/transform.py

@@ -160,6 +160,7 @@ class MappingPackage(MappingPackageComponent, MappingPackageV3):
     version: str = Field(default="0.1.1")
     ontology_version: str = Field(default="0.0.1")
     git_latest_commit_hash: str = Field(default="")
+    mapping_suite_identifier: str = Field(default="")
     mapping_suite_hash_digest: str = Field(default="")
     mapping_type: Optional[MappingPackageType] = Field( default=MappingPackageType.STANDARD_FORMS)
     metadata_constraints: Optional[MetadataConstraints] = Field(default=None)

src/ted_sws/data_manager/adapters/notice_repository.py

@@ -18,6 +18,7 @@
     XMLMetadataRepository
 from src.ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC
 from src.ted_sws.notice_metadata_processor.services.metadata_normalizer import create_normalised_metadata_view
+from src.ted_sws.notice_validator.adapters.validation_summary_runner import MAPPING_PACKAGE_IDENTIFIER
 
 logger = logging.getLogger(__name__)
 
@@ -244,7 +245,8 @@ def _create_dict_from_notice(notice: Notice) -> dict:
         """
 
         notice_dict = notice.model_dump(include={NOTICE_TED_ID: True, NOTICE_STATUS: True,
-                                           NOTICE_CREATED_AT: True, VALIDATION_SUMMARY: True})
+                                           NOTICE_CREATED_AT: True, VALIDATION_SUMMARY: True,
+                                                 MAPPING_PACKAGE_IDENTIFIER: True})
         notice_dict[MONGODB_COLLECTION_ID] = notice_dict[NOTICE_TED_ID]
         notice_dict[NOTICE_STATUS] = str(notice_dict[NOTICE_STATUS])
         notice_dict[NOTICE_CREATED_AT] = datetime.fromisoformat(notice_dict[NOTICE_CREATED_AT])

src/ted_sws/mapping_suite_processor/adapters/github_ms_project_downloader.py

@@ -72,19 +72,16 @@ def get_git_head_hash(git_repository_path: pathlib.Path) -> str:
                 shell=True,
                 stdout=subprocess.PIPE)
             git_head_hash = result.stdout.decode(encoding="utf-8")
-            return git_head_hash
+            return (git_head_hash or "").strip()
 
         with tempfile.TemporaryDirectory() as tmp_dir:
             temp_dir_path = pathlib.Path(tmp_dir)
             bash_script = f"cd {temp_dir_path} && git clone --depth 1 --branch {self.branch_or_tag_name} {self.github_repository_url}"
             result = subprocess.run(bash_script, shell=True,
                                     capture_output=True, text=True)
-            log_technical_info(
-                message=f"Downloaded stdout '{result.stdout}'")
-            log_technical_info(
-                message=f"Downloaded stderr '{result.stderr}'")
-            git_last_commit_hash = get_git_head_hash(
-                git_repository_path=temp_dir_path / self.repository_name)
+            log_technical_info(message=f"Downloaded stdout '{result.stdout}'")
+            log_technical_info(message=f"Downloaded stderr '{result.stderr}'")
+            git_last_commit_hash = get_git_head_hash(git_repository_path=temp_dir_path / self.repository_name)
             downloaded_tmp_project_path = temp_dir_path / self.repository_name
             shutil.copytree(downloaded_tmp_project_path, output_project_path, dirs_exist_ok=True)
 

src/ted_sws/mapping_suite_processor/services/mapping_package_processor.py

@@ -2,6 +2,7 @@
 import tempfile
 from typing import List
 
+from mapping_suite_sdk.mapping_suite.models import MappingSuite
 from pymongo import MongoClient
 
 from src.ted_sws import config
@@ -36,10 +37,11 @@
 
 
 def mapping_package_processor_load_package_in_mongo_db(
-    package: MappingPackage,
-    mongodb_client: MongoClient,
-    load_test_data: bool = False,
-    git_last_commit_hash: str = None
+        package: MappingPackage,
+        mongodb_client: MongoClient,
+        load_test_data: bool = False,
+        git_last_commit_hash: str = None,
+        mapping_suite: MappingSuite = None
 ) -> List[str]:
     """Load a mapping package to MongoDB.
 
@@ -50,6 +52,7 @@ def mapping_package_processor_load_package_in_mongo_db(
         mongodb_client: MongoDB client instance
         load_test_data: Whether to load test data as notices
         git_last_commit_hash: Optional git commit hash to store
+        mapping_suite: Optional mapping suite that the package belongs to
 
     Returns:
         List of notice IDs that were loaded (if load_test_data=True)
@@ -60,6 +63,9 @@ def mapping_package_processor_load_package_in_mongo_db(
     # Update git hash if provided and field exists
     if git_last_commit_hash is not None:
         package.git_latest_commit_hash = git_last_commit_hash
+
+    if mapping_suite:
+        package.mapping_suite_identifier = get_mapping_suite_identifier(mapping_suite)
     result_notice_ids = []
 
     # Load test data if requested
@@ -68,7 +74,7 @@ def mapping_package_processor_load_package_in_mongo_db(
         notice_repository = NoticeRepository(mongodb_client=mongodb_client)
         for test_data in tests_data:
             notice_id = test_data.file_name.split(".")[0]
-            notice = Notice(ted_id=notice_id)
+            notice = Notice(ted_id=notice_id, mapping_package_identifier=package.identifier)
             notice.set_xml_manifestation(XMLManifestation(object_data=test_data.file_content))
             notice_repository.add(notice=notice)
             result_notice_ids.append(notice_id)
@@ -113,6 +119,7 @@ def load_mapping_suite_and_packages_from_github_to_mongo_db(mongodb_client: Mong
         git_last_commit_hash = mapping_package_downloader.download(output_project_path=tmp_dir_path)
 
         # load project config if available
+        mapping_suite = None
         if ms_config_file_path.is_file():
             log_technical_info(message=f"Mapping suite config found at '{ms_config_file_path}'")
             mapping_suite = load_mapping_suite_from_folder(mapping_suite_folder_path=tmp_dir_path)
@@ -131,7 +138,7 @@ def load_mapping_suite_and_packages_from_github_to_mongo_db(mongodb_client: Mong
 
         # continue loading mapping packages
         mapping_package_paths = [
-            mappings_dir_path / mapping_package_name ] if mapping_package_name else list(mappings_dir_path.iterdir())
+            mappings_dir_path / mapping_package_name] if mapping_package_name else list(mappings_dir_path.iterdir())
         result_notice_ids = []
         for mapping_package_path in mapping_package_paths:
             detected_version = detect_mapping_package_version(mapping_package_path)
@@ -159,7 +166,8 @@ def load_mapping_suite_and_packages_from_github_to_mongo_db(mongodb_client: Mong
                     package=mapping_package,
                     mongodb_client=mongodb_client,
                     load_test_data=load_test_data,
-                    git_last_commit_hash=git_last_commit_hash
+                    git_last_commit_hash=git_last_commit_hash,
+                    mapping_suite=mapping_suite
                 ))
                 log_mapping_package_info(
                     message=f"Mapping package with id={mapping_package.id} loaded with success in MongoDB!",
@@ -178,3 +186,10 @@ def _convert_to_mapping_package(mssdk_package) -> MappingPackage:
     """Convert MSSDK package to extended MappingPackage."""
     data = mssdk_package.model_dump(exclude={'test_results'})
     return MappingPackage(**data)
+
+
+def get_mapping_suite_identifier(mapping_suite):
+    ms_config = getattr(mapping_suite, "mapping_suite_config", None)
+    metadata = getattr(ms_config, "mapping_suite_metadata", None) if ms_config else None
+    identifier = getattr(metadata, "mapping_suite_identifier", None) if metadata else None
+    return identifier

src/ted_sws/notice_metadata_processor/adapters/notice_metadata_normaliser.py

@@ -1,11 +1,15 @@
 import abc
+import html
+import re
 from datetime import datetime
 from typing import Dict, Tuple, List
-import re
+
 import pandas as pd
-import html
+from pymongo import MongoClient
 
 from src.ted_sws.core.model.metadata import NormalisedMetadata, LanguageTaggedString, NoticeSource
+from src.ted_sws.core.model.notice import Notice
+from src.ted_sws.event_manager.services.log import log_notice_info
 from src.ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata
 from src.ted_sws.notice_metadata_processor.services.metadata_constraints import filter_df_by_variables
 from src.ted_sws.resources.mapping_files_registry import MappingFilesRegistry
@@ -40,7 +44,6 @@
 EFORM_SDK_VERSION_KEY = "eform_sdk_version"
 NOTICE_SOURCE_KEY = "notice_source"
 ENGLISH_LANGUAGE_TAG = "EN"
-mapping_registry = MappingFilesRegistry()
 
 
 def get_html_compatible_string(input_string: LanguageTaggedString) -> LanguageTaggedString:
@@ -86,6 +89,8 @@ def normalise_metadata(self, extracted_metadata: ExtractedMetadata) -> Normalise
 
 
 class DefaultNoticeMetadataNormaliser(NoticeMetadataNormaliserABC):
+    def __init__(self, notice: Notice, mongodb_client: MongoClient = None):
+        self.mapping_registry = MappingFilesRegistry(notice=notice, mongodb_client=mongodb_client)
 
     @classmethod
     def normalise_legal_basis_value(cls, value: str) -> str:
@@ -207,15 +212,15 @@ def normalise_metadata(self, extracted_metadata: ExtractedMetadata) -> Normalise
             Generate the normalised metadata
         :return:
         """
-        countries_map = mapping_registry.countries
-        form_type_map = mapping_registry.form_type
-        languages_map = mapping_registry.languages
-        legal_basis_map = mapping_registry.legal_basis
-        notice_type_map = mapping_registry.notice_type
-        nuts_map = mapping_registry.nuts
-        standard_forms_map = mapping_registry.sf_notice_df
-        eforms_map = mapping_registry.ef_notice_df
-        filter_map = mapping_registry.filter_map_df
+        countries_map = self.mapping_registry.countries
+        form_type_map = self.mapping_registry.form_type
+        languages_map = self.mapping_registry.languages
+        legal_basis_map = self.mapping_registry.legal_basis
+        notice_type_map = self.mapping_registry.notice_type
+        nuts_map = self.mapping_registry.nuts
+        standard_forms_map = self.mapping_registry.sf_notice_df
+        eforms_map = self.mapping_registry.ef_notice_df
+        filter_map = self.mapping_registry.filter_map_df
         form_type, notice_type, legal_basis, eforms_subtype = self.get_form_type_and_notice_type(
             sf_map=standard_forms_map, ef_map=eforms_map, filter_map=filter_map,
             extracted_notice_type=extracted_metadata.extracted_notice_type,
@@ -273,6 +278,8 @@ class EformsNoticeMetadataNormaliser(NoticeMetadataNormaliserABC):
     """
      Metadata normaliser for eForms
     """
+    def __init__(self, notice: Notice, mongodb_client: MongoClient = None):
+        self.mapping_registry = MappingFilesRegistry(notice=notice, mongodb_client=mongodb_client)
 
     @classmethod
     def iso_date_format(cls, _date: str, with_none=False):
@@ -283,17 +290,17 @@ def iso_date_format(cls, _date: str, with_none=False):
             return datetime.fromisoformat(_date).isoformat()
         return None
 
-    @classmethod
-    def get_form_type_notice_type_and_legal_basis(cls, extracted_notice_subtype: str) -> Tuple:
+    def get_form_type_notice_type_and_legal_basis(self, extracted_notice_subtype: str) -> Tuple:
         """
          Get the values for form type, notice type and legal basis from the eForm mapping files
         """
-        ef_map: pd.DataFrame = mapping_registry.ef_notice_df
+        ef_map: pd.DataFrame = self.mapping_registry.ef_notice_df
         try:
             filtered_df = ef_map.query(f"{E_FORMS_SUBTYPE_KEY}=='{extracted_notice_subtype}'").to_dict(orient='records')[0]
         except:
             raise Exception(
                 f'No mapping available for {extracted_notice_subtype} notice subtype. Please check that the field exists in the XML content if the notice subtype is not specified in this message')
+
         try:
             form_type = filtered_df[FORM_TYPE_KEY]
             notice_type = filtered_df[E_FORM_NOTICE_TYPE_COLUMN]
@@ -312,11 +319,11 @@ def normalise_metadata(self, extracted_metadata: ExtractedMetadata) -> Normalise
         :return:
         """
         extracted_metadata = extracted_metadata
-        form_type_map = mapping_registry.form_type
-        languages_map = mapping_registry.languages
-        legal_basis_map = mapping_registry.legal_basis
-        notice_type_map = mapping_registry.notice_type
-        nuts_map = mapping_registry.nuts
+        form_type_map = self.mapping_registry.form_type
+        languages_map = self.mapping_registry.languages
+        legal_basis_map = self.mapping_registry.legal_basis
+        notice_type_map = self.mapping_registry.notice_type
+        nuts_map = self.mapping_registry.nuts
         form_type, notice_type, legal_basis = self.get_form_type_notice_type_and_legal_basis(
             extracted_notice_subtype=extracted_metadata.extracted_notice_subtype)
         metadata = {
@@ -354,3 +361,4 @@ def normalise_metadata(self, extracted_metadata: ExtractedMetadata) -> Normalise
         }
 
         return NormalisedMetadata(**metadata)
+

src/ted_sws/notice_metadata_processor/services/metadata_normalizer.py

@@ -1,6 +1,8 @@
 from typing import Optional
 import xml.etree.ElementTree as ET
 
+from pymongo import MongoClient
+
 from src.ted_sws.core.model.manifestation import XMLManifestation
 from src.ted_sws.core.model.metadata import NormalisedMetadata, NormalisedMetadataView
 from src.ted_sws.core.model.notice import Notice
@@ -32,14 +34,17 @@ def find_metadata_extractor_based_on_xml_manifestation(
 
 
 def find_metadata_normaliser_based_on_xml_manifestation(
-        xml_manifestation: XMLManifestation) -> NoticeMetadataNormaliserABC:
+        notice: Notice,
+        xml_manifestation: XMLManifestation,
+        mongodb_client: MongoClient = None
+) -> NoticeMetadataNormaliserABC:
     """
         Find the correct extractor based on the XML Manifestation
     """
     if check_if_xml_manifestation_is_eform(xml_manifestation):
-        return EformsNoticeMetadataNormaliser()
+        return EformsNoticeMetadataNormaliser(notice=notice, mongodb_client=mongodb_client)
     else:
-        return DefaultNoticeMetadataNormaliser()
+        return DefaultNoticeMetadataNormaliser(notice=notice, mongodb_client=mongodb_client)
 
 
 def extract_notice_metadata(metadata_extractor: NoticeMetadataExtractorABC) -> ExtractedMetadata:
@@ -57,32 +62,36 @@ def normalise_notice_metadata(extracted_metadata: ExtractedMetadata,
     return metadata_normaliser.normalise_metadata(extracted_metadata)
 
 
-def extract_and_normalise_notice_metadata(xml_manifestation: XMLManifestation) -> NormalisedMetadata:
+def extract_and_normalise_notice_metadata(notice: Notice, xml_manifestation: XMLManifestation, mongodb_client: MongoClient = None) -> NormalisedMetadata:
     """
         Extract and normalise metadata using the correct extractor and normaliser type
     """
     metadata_extractor = find_metadata_extractor_based_on_xml_manifestation(xml_manifestation)
     extracted_metadata = extract_notice_metadata(metadata_extractor)
-    metadata_normaliser = find_metadata_normaliser_based_on_xml_manifestation(xml_manifestation)
+    metadata_normaliser = find_metadata_normaliser_based_on_xml_manifestation(
+        notice=notice,
+        xml_manifestation=xml_manifestation,
+        mongodb_client=mongodb_client
+    )
     normalised_metadata = normalise_notice_metadata(extracted_metadata, metadata_normaliser)
     return normalised_metadata
 
 
-def extract_and_normalise_notice_metadata_from_notice(notice: Notice) -> NormalisedMetadata:
+def extract_and_normalise_notice_metadata_from_notice(notice: Notice, mongodb_client: MongoClient = None) -> NormalisedMetadata:
     """
         Extract and normalise metadata using the correct extractor and normaliser type
     """
     xml_manifestation = notice.xml_manifestation
-    return extract_and_normalise_notice_metadata(xml_manifestation)
+    return extract_and_normalise_notice_metadata(notice=notice, xml_manifestation=xml_manifestation, mongodb_client=mongodb_client)
 
 
-def normalise_notice(notice: Notice) -> Notice:
+def normalise_notice(notice: Notice, mongodb_client: MongoClient = None) -> Notice:
     """
         Given a notice object, normalise metadata and return the updated object
     :param notice:
     :return:
     """
-    normalised_metadata = extract_and_normalise_notice_metadata_from_notice(notice=notice)
+    normalised_metadata = extract_and_normalise_notice_metadata_from_notice(notice=notice, mongodb_client=mongodb_client)
     notice.set_normalised_metadata(normalised_metadata)
     return notice
 

src/ted_sws/resources/__init__.py

@@ -1,33 +1,5 @@
-import json
 import pathlib
 
-import pandas as pd
-
-try:
-    import importlib.resources as pkg_resources
-except ImportError:
-    # Try backported to PY<37 `importlib_resources`.
-    import importlib_resources as pkg_resources
-
-import src.ted_sws.resources.mapping_files
-
-
-def get_mapping_json_file(mapping_file_name: str) -> dict:
-    """
-        get a predefined index mapping by reference to file name
-    """
-    with pkg_resources.path(mapping_files, mapping_file_name) as path:
-        return json.loads(path.read_bytes())
-
-
-def get_mapping_csv_file(mapping_file_name: str) -> pd.DataFrame:
-    """
-        get content of a csv file in pandas dataframe format
-    """
-    with pkg_resources.path(mapping_files, mapping_file_name) as path:
-        return pd.read_csv(path).fillna("")
-
-
 RESOURCES_PATH = pathlib.Path(__file__).parent.resolve()
 
 PREFIXES_PATH = RESOURCES_PATH / 'prefixes'