fix(tests): ensure mapping suite/package is loaded for normalisation

Tests were failing with ModelNotFoundError because:
- Notice fixtures didn't set mapping_package_identifier
- Mapping suite/package weren't loaded into test MongoDB instances
- normalise_notice() calls didn't pass mongodb_client parameter

Changes:
- Add load_mapping_suite_and_package fixture to features/conftest.py
- Update notice fixtures to set mapping_package_identifier
- Pass mongodb_client to normalise_notice() in test steps
- Add load_mapping_suite_and_package_fake for e2e tests using mongomock
- Update e2e fixtures to link GitHub-loaded packages to local mapping suite

Fixes 30+ e2e/feature tests that were failing after metadata resource
refactoring with dynamic MS Config loading via MSSDK.

Author: schivmeister

test/e2e/conftest.py

@@ -87,4 +87,13 @@ def load_mapping_suite_and_package(mongodb_client, mapping_suite, mapping_packag
     mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
     mapping_suite_repository.add(mapping_suite=mapping_suite)
     mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=mongodb_client)
-    mapping_package_repository.add(mapping_package=mapping_package)
+    mapping_package_repository.add(mapping_package=mapping_package)
+
+
+@pytest.fixture
+def load_mapping_suite_and_package_fake(fake_mongodb_client, mapping_suite, mapping_package):
+    """Load mapping suite and package into fake MongoDB (mongomock) for tests."""
+    mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=fake_mongodb_client)
+    mapping_suite_repository.add(mapping_suite=mapping_suite)
+    mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=fake_mongodb_client)
+    mapping_package_repository.add(mapping_package=mapping_package)

test/e2e/dags/pipelines/test_notice_processor_pipelines.py

@@ -3,6 +3,7 @@
     notice_validation_pipeline, notice_package_pipeline, notice_publish_pipeline
 from src.ted_sws.core.model.notice import NoticeStatus
 from src.ted_sws.data_manager.adapters.notice_repository import NoticeRepository
+from src.ted_sws.data_manager.adapters.mapping_package_repository import MappingPackageRepositoryMongoDB
 from src.ted_sws.mapping_suite_processor.services.mapping_package_processor import \
     load_mapping_suite_and_packages_from_github_to_mongo_db
 
@@ -11,12 +12,20 @@
 NOTICE_ID = "057215-2021"
 
 
-def test_notice_processor_pipelines(fake_mongodb_client):
+def test_notice_processor_pipelines(fake_mongodb_client, load_mapping_suite_and_package_fake, mapping_suite):
     load_mapping_suite_and_packages_from_github_to_mongo_db(
         mapping_package_name=MAPPING_PACKAGE_NAME,
         mongodb_client=fake_mongodb_client,
         load_test_data=True
     )
+
+    # Update GitHub-loaded packages to use the local mapping suite
+    mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=fake_mongodb_client)
+    for pkg in mapping_package_repository.list():
+        if not pkg.mapping_suite_identifier:
+            pkg.mapping_suite_identifier = mapping_suite.id
+            mapping_package_repository.update(pkg)
+
     notice_id = NOTICE_ID
     notice_repository = NoticeRepository(mongodb_client=fake_mongodb_client)
     notice = notice_repository.get(reference=notice_id)

test/e2e/data_manager/test_mongodb_client.py

@@ -139,7 +139,7 @@ def test_mongo_db_query_2():
     ])
 
 
-def test_create_matview_for_notices(fake_mongodb_client):
+def test_create_matview_for_notices(fake_mongodb_client, load_mapping_suite_and_package_fake, mapping_package):
     notice_id = "696661-2022"
     ted_api_query = {"query": f"ND={notice_id}"}
     mongodb_client = fake_mongodb_client
@@ -152,8 +152,9 @@ def test_create_matview_for_notices(fake_mongodb_client):
     notice_event.caller_name = "execute"
     notice_event.start_record()
     notice = notice_repository.get(reference=notice_id)
+    notice.mapping_package_identifier = mapping_package.id
     indexed_notice = index_notice(notice=notice)
-    normalised_notice = normalise_notice(notice=indexed_notice)
+    normalised_notice = normalise_notice(notice=indexed_notice, mongodb_client=mongodb_client)
     notice = normalised_notice
     notice_repository.update(notice=normalised_notice)
     notice_event.end_record()

test/e2e/data_sampler/conftest.py

@@ -3,6 +3,7 @@
 
 from src.ted_sws import config
 from src.ted_sws.data_manager.adapters.notice_repository import NoticeRepository
+from src.ted_sws.data_manager.adapters.mapping_package_repository import MappingPackageRepositoryMongoDB
 from src.ted_sws.data_sampler.services.notice_xml_indexer import index_notice
 from src.ted_sws.mapping_suite_processor.services.mapping_package_processor import \
     load_mapping_suite_and_packages_from_github_to_mongo_db
@@ -21,13 +22,30 @@ def mongodb_client():
 
 
 @pytest.fixture
-def notice_repository_with_indexed_notices(mongodb_client, load_mapping_suite_and_package) -> NoticeRepository:
+def notice_repository_with_indexed_notices(mongodb_client, load_mapping_suite_and_package, mapping_suite, mapping_package) -> NoticeRepository:
+    """Load notices from GitHub and ensure they reference the local mapping suite/package.
 
+    This fixture:
+    1. Uses load_mapping_suite_and_package to store local suite/package in MongoDB
+    2. Downloads packages from GitHub (which loads test notices into the repository)
+    3. Updates all packages loaded from GitHub to reference the local mapping suite
+    4. Updates all notices to reference those packages
+    5. Indexes and normalizes all notices
+    """
     load_mapping_suite_and_packages_from_github_to_mongo_db(
         mapping_package_name="package_F03_test",
         mongodb_client=mongodb_client,
         load_test_data=True
     )
+
+    # Update GitHub-loaded packages to use the local mapping suite
+    mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=mongodb_client)
+    for pkg in mapping_package_repository.list():
+        if pkg.id != mapping_package.id and not pkg.mapping_suite_identifier:
+            # This is a GitHub-loaded package without mapping_suite_identifier
+            pkg.mapping_suite_identifier = mapping_suite.id
+            mapping_package_repository.update(pkg)
+
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     for notice in notice_repository.list():
         indexed_notice = index_notice(notice=notice)

test/features/conftest.py

@@ -10,6 +10,8 @@
 from src.ted_sws.core.model.metadata import NormalisedMetadata, XMLMetadata
 from src.ted_sws.core.model.notice import NoticeStatus, Notice
 from src.ted_sws.data_manager.adapters.notice_repository import NoticeRepository
+from src.ted_sws.data_manager.adapters.mapping_package_repository import MappingPackageRepositoryMongoDB
+from src.ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
 from src.ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, TedRequestAPI
 from src.ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
 
@@ -41,36 +43,48 @@ def notice_repository(mongodb_client):
 
 
 @pytest.fixture
-def f03_notice_2020(notice_repository, ted_api_end_point):
+def load_mapping_suite_and_package(mongodb_client, mapping_suite, mapping_package):
+    """Load mapping suite and package into MongoDB for tests that need mapping resources."""
+    mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
+    mapping_suite_repository.add(mapping_suite=mapping_suite)
+    mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=mongodb_client)
+    mapping_package_repository.add(mapping_package=mapping_package)
+
+
+@pytest.fixture
+def f03_notice_2020(notice_repository, ted_api_end_point, load_mapping_suite_and_package, mapping_package):
     notice_search_query = {"query": "ND=408313-2020"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
     notice = notice_repository.get(reference="408313-2020")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
+    notice.mapping_package_identifier = mapping_package.id
     return notice
 
 @pytest.fixture
-def eForm_notice_2023(notice_repository, ted_api_end_point):
+def eForm_notice_2023(notice_repository, ted_api_end_point, load_mapping_suite_and_package, mapping_package):
     notice_search_query = {"query": "ND=17554-2024"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
     notice = notice_repository.get(reference="17554-2024")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
+    notice.mapping_package_identifier = mapping_package.id
     return notice
 
 @pytest.fixture
-def f18_notice_2022(notice_repository, ted_api_end_point):
+def f18_notice_2022(notice_repository, ted_api_end_point, load_mapping_suite_and_package, mapping_package):
     notice_search_query = {"query": "ND=67623-2022"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
     notice = notice_repository.get(reference="67623-2022")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
+    notice.mapping_package_identifier = mapping_package.id
     return notice
 
 

test/features/notice_metadata_processor/conftest.py

@@ -33,16 +33,18 @@ def notice_eligibility_repository_path():
 
 
 @pytest.fixture
-def normalised_notice(notice_2020):
+def normalised_notice(notice_2020, load_mapping_suite_and_package, mapping_package, mongodb_client):
     notice = notice_2020.copy()
-    normalise_notice(notice=notice)
+    notice.mapping_package_identifier = mapping_package.id
+    normalise_notice(notice=notice, mongodb_client=mongodb_client)
     return notice
 
 
 @pytest.fixture
-def normalised_eForm_notice(indexed_eform_notice_622690):
+def normalised_eForm_notice(indexed_eform_notice_622690, load_mapping_suite_and_package, mapping_package, mongodb_client):
     notice = indexed_eform_notice_622690.copy()
-    normalise_notice(notice=notice)
+    notice.mapping_package_identifier = mapping_package.id
+    normalise_notice(notice=notice, mongodb_client=mongodb_client)
     return notice
 
 
@@ -65,10 +67,13 @@ def sample_ef_html_unsafe_notice_path() -> pathlib.Path:
 
 @pytest.fixture
 def sample_indexed_ef_html_unsafe_notice(
-        sample_ef_html_unsafe_notice_path: pathlib.Path) -> Notice:
+        sample_ef_html_unsafe_notice_path: pathlib.Path,
+        load_mapping_suite_and_package,
+        mapping_package) -> Notice:
     notice: Notice = Notice(ted_id=sample_ef_html_unsafe_notice_path.name)
     notice.set_xml_manifestation(
         XMLManifestation(object_data=sample_ef_html_unsafe_notice_path.read_text()))
+    notice.mapping_package_identifier = mapping_package.id
 
     return index_notice(notice)
 
@@ -80,10 +85,13 @@ def sample_sf_html_unsafe_notice_path() -> pathlib.Path:
 
 @pytest.fixture
 def sample_indexed_sf_html_unsafe_notice(
-        sample_sf_html_unsafe_notice_path: pathlib.Path) -> Notice:
+        sample_sf_html_unsafe_notice_path: pathlib.Path,
+        load_mapping_suite_and_package,
+        mapping_package) -> Notice:
     notice: Notice = Notice(ted_id=sample_sf_html_unsafe_notice_path.name)
     notice.set_xml_manifestation(
         XMLManifestation(object_data=sample_sf_html_unsafe_notice_path.read_text()))
+    notice.mapping_package_identifier = mapping_package.id
 
     return index_notice(notice)
 

test/features/notice_metadata_processor/test_metadata_normaliser.py

@@ -15,8 +15,8 @@ def step_impl(f03_notice_2020):
 
 
 @when("the normalize process is executed")
-def step_impl(notice):
-    normalise_notice(notice=notice)
+def step_impl(notice, mongodb_client):
+    normalise_notice(notice=notice, mongodb_client=mongodb_client)
 
 
 @then(parsers.parse("a normalized notice {metadata} is {possibly} available"))

test/features/notice_metadata_processor/test_metadata_normaliser_eforms.py

@@ -15,8 +15,8 @@ def step_impl(eForm_notice_2023):
 
 
 @when("the normalise process is executed")
-def step_impl(notice):
-    normalise_notice(notice=notice)
+def step_impl(notice, mongodb_client):
+    normalise_notice(notice=notice, mongodb_client=mongodb_client)
 
 
 @then(parsers.parse("a normalised notice {metadata} is {possibly} available"))

test/features/notice_metadata_processor/test_notice_normalisation.py

@@ -42,15 +42,15 @@ def an_sf_notice_with_spaces_in_notice_id(sample_indexed_sf_html_unsafe_notice:
 
 
 @when('the EF notice is normalised', target_fixture='normalised_ef_notice')
-def the_ef_notice_is_normalised(sample_indexed_ef_html_unsafe_notice: Notice) -> Notice:
+def the_ef_notice_is_normalised(sample_indexed_ef_html_unsafe_notice: Notice, mongodb_client) -> Notice:
     """Normalize EF notice."""
-    return normalise_notice(sample_indexed_ef_html_unsafe_notice)
+    return normalise_notice(sample_indexed_ef_html_unsafe_notice, mongodb_client=mongodb_client)
 
 
 @when('the SF notice is normalised', target_fixture='normalised_sf_notice')
-def the_sf_notice_is_normalised(sample_indexed_sf_html_unsafe_notice: Notice) -> Notice:
+def the_sf_notice_is_normalised(sample_indexed_sf_html_unsafe_notice: Notice, mongodb_client) -> Notice:
     """Normalize SF notice."""
-    return normalise_notice(sample_indexed_sf_html_unsafe_notice)
+    return normalise_notice(sample_indexed_sf_html_unsafe_notice, mongodb_client=mongodb_client)
 
 
 @then('the EF notice ID should not contain leading or trailing spaces')