fix: circular bug - decouple MappingFilesRegistry from notice, package

There was a hidden circular dependency in the metadata resource
migration to MS Config via MSSDK.

The previous design required a notice with `mapping_package_identifier`
to load resources, but this created a circular dependency: normalisation
needs resources, yet eligibility checking (which returns a package
identifier but does not set one on the notice) needs normalised
metadata.

Initial assumptions may have been anchored on the resources being
project-specific. However, this is problematic as not all projects may
be updated with the mapping suite configuration. Therefore, resource
files (country.json, languages.json, etc.) can be interpreted to be
global for now during the transition period.

Once all currently known production projects are updated with the
configuration, a more dynamic method to select the mapping suite can be
implemented, for e.g. via the `document_probing` conditions specified in
the config, which defines what XPaths must and must not be available to
be compatible with the project.

Changes:
- MappingFilesRegistry now loads resources from any available MappingSuite
- Removed notice parameter from DefaultNoticeMetadataNormaliser and
  EformsNoticeMetadataNormaliser constructors
- Updated find_metadata_normaliser_based_on_xml_manifestation() and
  extract_and_normalise_notice_metadata() to not require notice
- Added MappingSuiteConfigError for when no MappingSuite is available
- Updated all test fixtures to use the new API
- Remove all traces and dependence on a Notice
  mapping_package_identifier

TODO: The mapping suite must be made mandatory and be fetched from a
default known project with the configuration if not given.

Author: schivmeister

src/ted_sws/core/model/notice.py

@@ -192,7 +192,6 @@ class Notice(LazyWorkExpression):
     _mets_manifestation: Optional[METSManifestation] = None
     _xml_metadata: Optional[XMLMetadata] = None
     validation_summary: Optional[ValidationSummaryReport] = None
-    mapping_package_identifier: Optional[str] = None
 
     @computed_field
     @property

src/ted_sws/mapping_suite_processor/services/mapping_package_processor.py

@@ -74,7 +74,7 @@ def mapping_package_processor_load_package_in_mongo_db(
         notice_repository = NoticeRepository(mongodb_client=mongodb_client)
         for test_data in tests_data:
             notice_id = test_data.file_name.split(".")[0]
-            notice = Notice(ted_id=notice_id, mapping_package_identifier=package.identifier)
+            notice = Notice(ted_id=notice_id)
             notice.set_xml_manifestation(XMLManifestation(object_data=test_data.file_content))
             notice_repository.add(notice=notice)
             result_notice_ids.append(notice_id)

src/ted_sws/notice_metadata_processor/adapters/notice_metadata_normaliser.py

@@ -8,7 +8,6 @@
 from pymongo import MongoClient
 
 from src.ted_sws.core.model.metadata import NormalisedMetadata, LanguageTaggedString, NoticeSource
-from src.ted_sws.core.model.notice import Notice
 from src.ted_sws.event_manager.services.log import log_notice_info
 from src.ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata
 from src.ted_sws.notice_metadata_processor.services.metadata_constraints import filter_df_by_variables
@@ -89,8 +88,8 @@ def normalise_metadata(self, extracted_metadata: ExtractedMetadata) -> Normalise
 
 
 class DefaultNoticeMetadataNormaliser(NoticeMetadataNormaliserABC):
-    def __init__(self, notice: Notice, mongodb_client: MongoClient = None):
-        self.mapping_registry = MappingFilesRegistry(notice=notice, mongodb_client=mongodb_client)
+    def __init__(self, mongodb_client: MongoClient = None):
+        self.mapping_registry = MappingFilesRegistry(mongodb_client=mongodb_client)
 
     @classmethod
     def normalise_legal_basis_value(cls, value: str) -> str:
@@ -278,8 +277,8 @@ class EformsNoticeMetadataNormaliser(NoticeMetadataNormaliserABC):
     """
      Metadata normaliser for eForms
     """
-    def __init__(self, notice: Notice, mongodb_client: MongoClient = None):
-        self.mapping_registry = MappingFilesRegistry(notice=notice, mongodb_client=mongodb_client)
+    def __init__(self, mongodb_client: MongoClient = None):
+        self.mapping_registry = MappingFilesRegistry(mongodb_client=mongodb_client)
 
     @classmethod
     def iso_date_format(cls, _date: str, with_none=False):

src/ted_sws/notice_metadata_processor/services/metadata_normalizer.py

@@ -34,17 +34,16 @@ def find_metadata_extractor_based_on_xml_manifestation(
 
 
 def find_metadata_normaliser_based_on_xml_manifestation(
-        notice: Notice,
         xml_manifestation: XMLManifestation,
         mongodb_client: MongoClient = None
 ) -> NoticeMetadataNormaliserABC:
     """
         Find the correct extractor based on the XML Manifestation
     """
     if check_if_xml_manifestation_is_eform(xml_manifestation):
-        return EformsNoticeMetadataNormaliser(notice=notice, mongodb_client=mongodb_client)
+        return EformsNoticeMetadataNormaliser(mongodb_client=mongodb_client)
     else:
-        return DefaultNoticeMetadataNormaliser(notice=notice, mongodb_client=mongodb_client)
+        return DefaultNoticeMetadataNormaliser(mongodb_client=mongodb_client)
 
 
 def extract_notice_metadata(metadata_extractor: NoticeMetadataExtractorABC) -> ExtractedMetadata:
@@ -62,14 +61,13 @@ def normalise_notice_metadata(extracted_metadata: ExtractedMetadata,
     return metadata_normaliser.normalise_metadata(extracted_metadata)
 
 
-def extract_and_normalise_notice_metadata(notice: Notice, xml_manifestation: XMLManifestation, mongodb_client: MongoClient = None) -> NormalisedMetadata:
+def extract_and_normalise_notice_metadata(xml_manifestation: XMLManifestation, mongodb_client: MongoClient = None) -> NormalisedMetadata:
     """
         Extract and normalise metadata using the correct extractor and normaliser type
     """
     metadata_extractor = find_metadata_extractor_based_on_xml_manifestation(xml_manifestation)
     extracted_metadata = extract_notice_metadata(metadata_extractor)
     metadata_normaliser = find_metadata_normaliser_based_on_xml_manifestation(
-        notice=notice,
         xml_manifestation=xml_manifestation,
         mongodb_client=mongodb_client
     )
@@ -82,7 +80,7 @@ def extract_and_normalise_notice_metadata_from_notice(notice: Notice, mongodb_cl
         Extract and normalise metadata using the correct extractor and normaliser type
     """
     xml_manifestation = notice.xml_manifestation
-    return extract_and_normalise_notice_metadata(notice=notice, xml_manifestation=xml_manifestation, mongodb_client=mongodb_client)
+    return extract_and_normalise_notice_metadata(xml_manifestation=xml_manifestation, mongodb_client=mongodb_client)
 
 
 def normalise_notice(notice: Notice, mongodb_client: MongoClient = None) -> Notice:

src/ted_sws/resources/mapping_files_registry.py

@@ -4,8 +4,6 @@
 from pymongo import MongoClient
 
 from src.ted_sws import config
-from src.ted_sws.core.model.notice import Notice
-from src.ted_sws.data_manager.adapters.mapping_package_repository import MappingPackageRepositoryMongoDB
 from src.ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
 
 COUNTRIES_MAPPING_FILE = "country.json"
@@ -21,18 +19,31 @@
 CSV_EXT = ".csv"
 
 
+class MappingSuiteConfigError(Exception):
+    """Raised when no MappingSuite is found in the database."""
+    pass
+
+
 class MappingFilesRegistry:
     """
-     Registry of mapping files. This will return the specific file content
+     Registry of mapping files. This will return the specific file content.
+
+     Resource files (country.json, languages.json, etc.) are global and identical
+     across all mapping packages, so we can load them from any available MappingSuite.
     """
 
-    def __init__(self, notice: Notice, mongodb_client: MongoClient = None):
+    def __init__(self, mongodb_client: MongoClient = None):
         if not mongodb_client:
             mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
-        mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=mongodb_client)
-        mapping_package = mapping_package_repository.get(notice.mapping_package_identifier)
         mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
-        self.mapping_suite = mapping_suite_repository.get(mapping_package.mapping_suite_identifier)
+        # Get any available MappingSuite - resources are global/identical across all suites
+        all_suites = mapping_suite_repository.list()
+        if not all_suites:
+            raise MappingSuiteConfigError(
+                "No MappingSuite found in the database. Please ensure at least one "
+                "mapping suite is loaded before attempting to normalise notices."
+            )
+        self.mapping_suite = all_suites[0]
 
     @staticmethod
     def extract_filename_from_path(path: str) -> str:

test/conftest.py

@@ -86,7 +86,6 @@ def raw_notice(ted_document_search, notice_repository, notice_id) -> Notice:
 @pytest.fixture
 def indexed_notice(raw_notice) -> Notice:
     raw_notice.set_xml_metadata(XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
-    raw_notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return raw_notice
 
 
@@ -139,7 +138,6 @@ def notice_2018():
     notice = Notice(ted_id=ted_id)
     notice.set_xml_manifestation(xml_manifestation)
     notice.set_original_metadata(original_metadata)
-    notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return notice
 
 
@@ -156,7 +154,6 @@ def notice_2020():
     notice.set_xml_metadata(XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
     notice.set_xml_manifestation(xml_manifestation)
     notice.set_original_metadata(original_metadata)
-    notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return notice
 
 
@@ -300,7 +297,6 @@ def notice_2021():
     notice = Notice(ted_id=ted_id)
     notice.set_xml_manifestation(xml_manifestation)
     notice.set_original_metadata(original_metadata)
-    notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return notice
 
 
@@ -335,7 +331,6 @@ def eform_notice_622690():
     notice = Notice(ted_id=ted_id)
     notice.set_xml_manifestation(xml_manifestation)
     notice.set_original_metadata(original_metadata)
-    notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return notice
 
 

test/e2e/data_manager/test_mongodb_client.py

@@ -139,7 +139,7 @@ def test_mongo_db_query_2():
     ])
 
 
-def test_create_matview_for_notices(fake_mongodb_client, load_mapping_suite_and_package_fake, mapping_package):
+def test_create_matview_for_notices(fake_mongodb_client, load_mapping_suite_and_package_fake):
     notice_id = "696661-2022"
     ted_api_query = {"query": f"ND={notice_id}"}
     mongodb_client = fake_mongodb_client
@@ -152,7 +152,6 @@ def test_create_matview_for_notices(fake_mongodb_client, load_mapping_suite_and_
     notice_event.caller_name = "execute"
     notice_event.start_record()
     notice = notice_repository.get(reference=notice_id)
-    notice.mapping_package_identifier = mapping_package.id
     indexed_notice = index_notice(notice=notice)
     normalised_notice = normalise_notice(notice=indexed_notice, mongodb_client=mongodb_client)
     notice = normalised_notice

test/features/conftest.py

@@ -44,47 +44,48 @@ def notice_repository(mongodb_client):
 
 @pytest.fixture
 def load_mapping_suite_and_package(mongodb_client, mapping_suite, mapping_package):
-    """Load mapping suite and package into MongoDB for tests that need mapping resources."""
+    """Load mapping suite and package into MongoDB for tests that need mapping resources.
+
+    Note: Only the mapping_suite is strictly required for normalisation (resources are global),
+    but we also load mapping_package for other tests that may need it.
+    """
     mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
     mapping_suite_repository.add(mapping_suite=mapping_suite)
     mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=mongodb_client)
     mapping_package_repository.add(mapping_package=mapping_package)
 
 
 @pytest.fixture
-def f03_notice_2020(notice_repository, ted_api_end_point, load_mapping_suite_and_package, mapping_package):
+def f03_notice_2020(notice_repository, ted_api_end_point, load_mapping_suite_and_package):
     notice_search_query = {"query": "ND=408313-2020"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
     notice = notice_repository.get(reference="408313-2020")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
-    notice.mapping_package_identifier = mapping_package.id
     return notice
 
 @pytest.fixture
-def eForm_notice_2023(notice_repository, ted_api_end_point, load_mapping_suite_and_package, mapping_package):
+def eForm_notice_2023(notice_repository, ted_api_end_point, load_mapping_suite_and_package):
     notice_search_query = {"query": "ND=17554-2024"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
     notice = notice_repository.get(reference="17554-2024")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
-    notice.mapping_package_identifier = mapping_package.id
     return notice
 
 @pytest.fixture
-def f18_notice_2022(notice_repository, ted_api_end_point, load_mapping_suite_and_package, mapping_package):
+def f18_notice_2022(notice_repository, ted_api_end_point, load_mapping_suite_and_package):
     notice_search_query = {"query": "ND=67623-2022"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
     notice = notice_repository.get(reference="67623-2022")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
-    notice.mapping_package_identifier = mapping_package.id
     return notice
 
 

test/features/notice_metadata_processor/conftest.py

@@ -33,17 +33,15 @@ def notice_eligibility_repository_path():
 
 
 @pytest.fixture
-def normalised_notice(notice_2020, load_mapping_suite_and_package, mapping_package, mongodb_client):
+def normalised_notice(notice_2020, load_mapping_suite_and_package, mongodb_client):
     notice = notice_2020.copy()
-    notice.mapping_package_identifier = mapping_package.id
     normalise_notice(notice=notice, mongodb_client=mongodb_client)
     return notice
 
 
 @pytest.fixture
-def normalised_eForm_notice(indexed_eform_notice_622690, load_mapping_suite_and_package, mapping_package, mongodb_client):
+def normalised_eForm_notice(indexed_eform_notice_622690, load_mapping_suite_and_package, mongodb_client):
     notice = indexed_eform_notice_622690.copy()
-    notice.mapping_package_identifier = mapping_package.id
     normalise_notice(notice=notice, mongodb_client=mongodb_client)
     return notice
 
@@ -68,12 +66,10 @@ def sample_ef_html_unsafe_notice_path() -> pathlib.Path:
 @pytest.fixture
 def sample_indexed_ef_html_unsafe_notice(
         sample_ef_html_unsafe_notice_path: pathlib.Path,
-        load_mapping_suite_and_package,
-        mapping_package) -> Notice:
+        load_mapping_suite_and_package) -> Notice:
     notice: Notice = Notice(ted_id=sample_ef_html_unsafe_notice_path.name)
     notice.set_xml_manifestation(
         XMLManifestation(object_data=sample_ef_html_unsafe_notice_path.read_text()))
-    notice.mapping_package_identifier = mapping_package.id
 
     return index_notice(notice)
 
@@ -86,12 +82,10 @@ def sample_sf_html_unsafe_notice_path() -> pathlib.Path:
 @pytest.fixture
 def sample_indexed_sf_html_unsafe_notice(
         sample_sf_html_unsafe_notice_path: pathlib.Path,
-        load_mapping_suite_and_package,
-        mapping_package) -> Notice:
+        load_mapping_suite_and_package) -> Notice:
     notice: Notice = Notice(ted_id=sample_sf_html_unsafe_notice_path.name)
     notice.set_xml_manifestation(
         XMLManifestation(object_data=sample_sf_html_unsafe_notice_path.read_text()))
-    notice.mapping_package_identifier = mapping_package.id
 
     return index_notice(notice)
 

test/unit/notice_metadata_processor/conftest.py

@@ -40,7 +40,6 @@ def sample_indexed_ef_html_unsafe_notice(
     notice: Notice = Notice(ted_id=sample_ef_html_unsafe_notice_path.name)
     notice.set_xml_manifestation(
         XMLManifestation(object_data=sample_ef_html_unsafe_notice_path.read_text()))
-    notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return index_notice(notice)
 
 
@@ -55,7 +54,6 @@ def sample_indexed_sf_html_unsafe_notice(
     notice: Notice = Notice(ted_id=sample_sf_html_unsafe_notice_path.name)
     notice.set_xml_manifestation(
         XMLManifestation(object_data=sample_sf_html_unsafe_notice_path.read_text()))
-    notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return index_notice(notice)