fix: SF normalization with MS Config, MSSDK CSV-Pandas compatibility

- pass MongoDB client to normalise_notice function
- reparse MSSDK CSV list object w/ Pandas to reinterpret numbers
- update tests

Author: Kolea PLESCO

src/ted_sws/notice_metadata_processor/services/metadata_normalizer.py

@@ -97,7 +97,7 @@ def normalise_notice(notice: Notice, mongodb_client: MongoClient = None) -> Noti
     return notice
 
 
-def normalise_notice_by_id(notice_id: str, notice_repository: NoticeRepositoryABC) -> Notice:
+def normalise_notice_by_id(notice_id: str, notice_repository: NoticeRepositoryABC, mongodb_client: MongoClient = None) -> Notice:
     """
         Given a notice id, find the notice in the database, normalise its metadata, and store the updated state.
     :param notice_id:
@@ -108,7 +108,7 @@ def normalise_notice_by_id(notice_id: str, notice_repository: NoticeRepositoryAB
     if notice is None:
         raise ValueError('Notice, with "%s" notice_id, was not found' % notice_id)
 
-    return normalise_notice(notice)
+    return normalise_notice(notice, mongodb_client=mongodb_client)
 
 
 def create_normalised_metadata_view(normalised_metadata: NormalisedMetadata) -> Optional[NormalisedMetadataView]:

src/ted_sws/resources/mapping_files_registry.py

@@ -55,7 +55,7 @@ def get_suite_resource_content(self, filename: str):
         resource_content = resource['object'] if resource else None
 
         if self.extract_filename_ext(filename) == CSV_EXT:
-            return pd.DataFrame(resource_content).fillna("")
+            return pd.DataFrame(resource_content).apply(pd.to_numeric, errors="ignore").fillna("")
 
         return resource_content
 

test/conftest.py

@@ -156,6 +156,7 @@ def notice_2020():
     notice.set_xml_metadata(XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
     notice.set_xml_manifestation(xml_manifestation)
     notice.set_original_metadata(original_metadata)
+    notice.mapping_package_identifier = "test_package_eforms_sdk1.8"
     return notice
 
 

test/e2e/conftest.py

@@ -4,6 +4,8 @@
 from pymongo import MongoClient
 
 from src.ted_sws import config
+from src.ted_sws.data_manager.adapters.mapping_package_repository import MappingPackageRepositoryMongoDB
+from src.ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
 from src.ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from src.ted_sws.data_manager.adapters.triple_store import AllegroGraphTripleStore, FusekiAdapter
 
@@ -78,3 +80,11 @@ def path_to_file_system_repository():
 @pytest.fixture
 def fake_notice_repository(fake_mongodb_client):
     return NoticeRepository(mongodb_client=fake_mongodb_client)
+
+
+@pytest.fixture
+def load_mapping_suite_and_package(mongodb_client, mapping_suite, mapping_package):
+    mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
+    mapping_suite_repository.add(mapping_suite=mapping_suite)
+    mapping_package_repository = MappingPackageRepositoryMongoDB(mongodb_client=mongodb_client)
+    mapping_package_repository.add(mapping_package=mapping_package)

test/e2e/data_sampler/conftest.py

@@ -21,7 +21,7 @@ def mongodb_client():
 
 
 @pytest.fixture
-def notice_repository_with_indexed_notices(mongodb_client) -> NoticeRepository:
+def notice_repository_with_indexed_notices(mongodb_client, load_mapping_suite_and_package) -> NoticeRepository:
 
     load_mapping_suite_and_packages_from_github_to_mongo_db(
         mapping_package_name="package_F03_test",
@@ -31,6 +31,6 @@ def notice_repository_with_indexed_notices(mongodb_client) -> NoticeRepository:
     notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     for notice in notice_repository.list():
         indexed_notice = index_notice(notice=notice)
-        normalised_notice = normalise_notice(notice=indexed_notice)
+        normalised_notice = normalise_notice(notice=indexed_notice, mongodb_client=mongodb_client)
         notice_repository.update(notice=normalised_notice)
     return notice_repository

test/unit/notice_metadata_processor/test_eligibility.py

@@ -6,37 +6,48 @@
 from src.ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice
 from src.ted_sws.notice_metadata_processor.services.notice_eligibility import check_package, \
     notice_eligibility_checker, notice_eligibility_checker_by_id, format_version_with_zero_patch, is_date_in_range
+from test.unit.notice_metadata_processor import load_mapping_suite_and_package
 
 
-def test_non_eligibility_by_notice(notice_eligibility_repository_path, indexed_notice):
-    mapping_package_repository = MappingPackageRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
-    normalise_notice(notice=indexed_notice)
+def test_non_eligibility_by_notice(notice_eligibility_repository_path, indexed_notice,
+                                   mongodb_client, load_mapping_suite_and_package):
+    mapping_package_repository = MappingPackageRepositoryInFileSystem(
+        repository_path=notice_eligibility_repository_path)
+    normalise_notice(notice=indexed_notice, mongodb_client=mongodb_client)
     notice_eligibility_checker(notice=indexed_notice, mapping_package_repository=mapping_package_repository)
     assert indexed_notice.status == NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION
 
 
-def test_eforms_eligibility_by_notice(notice_eligibility_repository_path, indexed_eform_notice_622690):
-    mapping_package_repository = MappingPackageRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
-    normalise_notice(notice=indexed_eform_notice_622690)
-    notice_eligibility_checker(notice=indexed_eform_notice_622690, mapping_package_repository=mapping_package_repository)
+def test_eforms_eligibility_by_notice(notice_eligibility_repository_path, indexed_eform_notice_622690,
+                                      mongodb_client, load_mapping_suite_and_package):
+    mapping_package_repository = MappingPackageRepositoryInFileSystem(
+        repository_path=notice_eligibility_repository_path)
+    normalise_notice(notice=indexed_eform_notice_622690, mongodb_client=mongodb_client)
+    notice_eligibility_checker(notice=indexed_eform_notice_622690,
+                               mapping_package_repository=mapping_package_repository)
     assert indexed_eform_notice_622690.status == NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
 
 
-def test_eligibility_by_notice(notice_eligibility_repository_path, notice_2020):
-    mapping_package_repository = MappingPackageRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
-    normalise_notice(notice=notice_2020)
-    notice_checker = notice_eligibility_checker(notice=notice_2020, mapping_package_repository=mapping_package_repository)
+def test_eligibility_by_notice(notice_eligibility_repository_path, notice_2020,
+                               mongodb_client, load_mapping_suite_and_package):
+    mapping_package_repository = MappingPackageRepositoryInFileSystem(
+        repository_path=notice_eligibility_repository_path)
+    normalise_notice(notice=notice_2020, mongodb_client=mongodb_client)
+    notice_checker = notice_eligibility_checker(notice=notice_2020,
+                                                mapping_package_repository=mapping_package_repository)
     notice_id, mapping_package_identifier = notice_checker
     assert notice_id == "408313-2020"
     assert mapping_package_identifier == "test_package2"
     assert notice_2020.status == NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
 
 
-def test_eligibility_by_notice_id(notice_eligibility_repository_path, notice_2020, notice_repository):
-    normalise_notice(notice=notice_2020)
+def test_eligibility_by_notice_id(notice_eligibility_repository_path, notice_2020, notice_repository,
+                                  mongodb_client, load_mapping_suite_and_package):
+    normalise_notice(notice=notice_2020, mongodb_client=mongodb_client)
     notice_repository.add(notice_2020)
-    mapping_package_repository = MappingPackageRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
-    notice_checker =     notice_eligibility_checker_by_id(notice_id="408313-2020",
+    mapping_package_repository = MappingPackageRepositoryInFileSystem(
+        repository_path=notice_eligibility_repository_path)
+    notice_checker = notice_eligibility_checker_by_id(notice_id="408313-2020",
                                                       mapping_package_repository=mapping_package_repository,
                                                       notice_repository=notice_repository)
     notice_id, mapping_package_identifier = notice_checker
@@ -47,8 +58,9 @@ def test_eligibility_by_notice_id(notice_eligibility_repository_path, notice_202
 
 
 def test_check_mapping_package(notice_eligibility_repository_path, normalised_metadata_object,
-                             eform_normalised_metadata_object):
-    mapping_package_repository = MappingPackageRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
+                               eform_normalised_metadata_object):
+    mapping_package_repository = MappingPackageRepositoryInFileSystem(
+        repository_path=notice_eligibility_repository_path)
     is_valid = check_package(mapping_package=mapping_package_repository.get("test_package"),
                              notice_metadata=normalised_metadata_object)
 

test/unit/notice_metadata_processor/test_mapping_files_registry.py

@@ -1,17 +1,36 @@
 import pandas as pd
 
 from src.ted_sws.resources.mapping_files_registry import MappingFilesRegistry
+from test.unit.notice_metadata_processor import load_mapping_suite_and_package
 
 
-def test_mapping_file_registry():
-    json_resource_files = [MappingFilesRegistry().countries, MappingFilesRegistry().notice_type,
-                           MappingFilesRegistry().languages,
-                           MappingFilesRegistry().legal_basis]
+def test_mapping_file_registry(indexed_notice, mongodb_client, load_mapping_suite_and_package):
+    json_resource_files = [
+        MappingFilesRegistry(
+            notice=indexed_notice, mongodb_client=mongodb_client
+        ).countries,
+        MappingFilesRegistry(
+            notice=indexed_notice, mongodb_client=mongodb_client
+        ).notice_type,
+        MappingFilesRegistry(
+            notice=indexed_notice, mongodb_client=mongodb_client
+        ).languages,
+        MappingFilesRegistry(
+            notice=indexed_notice, mongodb_client=mongodb_client
+        ).legal_basis
+    ]
     for file_content in json_resource_files:
         assert isinstance(file_content, dict)
         assert "results" in file_content.keys()
 
-    csv_resource_files = [MappingFilesRegistry().sf_notice_df, MappingFilesRegistry().ef_notice_df]
+    csv_resource_files = [
+        MappingFilesRegistry(
+            notice=indexed_notice, mongodb_client=mongodb_client
+        ).sf_notice_df,
+        MappingFilesRegistry(
+            notice=indexed_notice, mongodb_client=mongodb_client
+        ).ef_notice_df
+    ]
 
     for file_content in csv_resource_files:
         assert isinstance(file_content, pd.DataFrame)

test/unit/notice_metadata_processor/test_metadata_normaliser.py

@@ -2,11 +2,11 @@
 from xml.etree.ElementTree import ParseError
 
 import pytest
+
 from src.ted_sws.core.model.manifestation import XMLManifestation
 from src.ted_sws.core.model.metadata import NormalisedMetadata, LanguageTaggedString
 from src.ted_sws.core.model.notice import NoticeStatus, Notice
-from src.ted_sws.data_manager.adapters.mapping_package_repository import MappingPackageRepositoryMongoDB
-from src.ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
+from src.ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from src.ted_sws.notice_metadata_processor.adapters.notice_metadata_extractor import \
     DefaultNoticeMetadataExtractor, EformsNoticeMetadataExtractor
 from src.ted_sws.notice_metadata_processor.adapters.notice_metadata_normaliser import \
@@ -26,17 +26,19 @@ def html_str(content: str) -> str:
     return f"""<?xml version="1.0" encoding="UTF-8"?> <body>{content}</body>"""
 
 
-def test_metadata_normaliser_by_notice(indexed_notice):
-    notice = normalise_notice(indexed_notice)
+def test_metadata_normaliser_by_notice(indexed_notice, mongodb_client, load_mapping_suite_and_package):
+    notice = normalise_notice(indexed_notice, mongodb_client=mongodb_client)
     assert notice.normalised_metadata
     assert notice.normalised_metadata.title
     assert isinstance(notice.normalised_metadata.eforms_subtype, str)
     assert notice.status == NoticeStatus.NORMALISED_METADATA
 
 
-def test_metadata_normaliser_by_notice_id(notice_id, notice_repository, notice_2020):
+def test_metadata_normaliser_by_notice_id(notice_id, notice_2020, mongodb_client, load_mapping_suite_and_package):
+    notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     notice_repository.add(notice_2020)
-    notice = normalise_notice_by_id(notice_id=notice_2020.ted_id, notice_repository=notice_repository)
+    notice = normalise_notice_by_id(notice_id=notice_2020.ted_id, notice_repository=notice_repository,
+                                    mongodb_client=mongodb_client)
     assert notice.normalised_metadata
     assert notice.normalised_metadata.title
     assert notice.status == NoticeStatus.NORMALISED_METADATA
@@ -48,9 +50,9 @@ def test_metadata_normaliser_by_wrong_notice_id(notice_repository):
         normalise_notice_by_id(notice_id=notice_id, notice_repository=notice_repository)
 
 
-def test_metadata_normaliser(indexed_notice):
+def test_metadata_normaliser(indexed_notice, mongodb_client, load_mapping_suite_and_package):
     notice = indexed_notice
-    normalise_notice(notice=notice)
+    normalise_notice(notice=notice, mongodb_client=mongodb_client)
 
     assert notice.normalised_metadata
     assert notice.normalised_metadata.title
@@ -213,8 +215,9 @@ def test_find_metadata_extractor_based_on_xml_manifestation(eform_notice_622690,
         DefaultNoticeMetadataExtractor)
 
 
-def test_find_metadata_normaliser_based_on_xml_manifestation(eform_notice_622690, notice_2018, mongodb_client,
-                                                             load_mapping_suite_and_package):
+def test_find_metadata_normaliser_based_on_xml_manifestation(
+        eform_notice_622690, notice_2018, mongodb_client, load_mapping_suite_and_package
+):
     assert isinstance(
         find_metadata_normaliser_based_on_xml_manifestation(
             notice=eform_notice_622690,

test/unit/notice_packager/conftest.py

@@ -16,9 +16,11 @@
     DefaultNoticeMetadataExtractor
 from src.ted_sws.notice_metadata_processor.adapters.notice_metadata_normaliser import \
     DefaultNoticeMetadataNormaliser
-from src.ted_sws.notice_packager.model.metadata import PackagerMetadata, NoticeMetadata, WorkMetadata, ExpressionMetadata, \
+from src.ted_sws.notice_packager.model.metadata import PackagerMetadata, NoticeMetadata, WorkMetadata, \
+    ExpressionMetadata, \
     ManifestationMetadata
 from test import TEST_DATA_PATH
+from test.unit.notice_metadata_processor import load_mapping_suite_and_package
 
 
 # template_metadata START
@@ -75,10 +77,14 @@ def template_sample_manifestation(template_sample_metadata) -> ManifestationMeta
 # notice_metadata START
 
 @pytest.fixture
-def notice_sample_metadata(notice_2018) -> NormalisedMetadata:
-    normalised_metadata = DefaultNoticeMetadataNormaliser().normalise_metadata(
+def notice_sample_metadata(notice_2018, mongodb_client, load_mapping_suite_and_package) -> NormalisedMetadata:
+    normalised_metadata = DefaultNoticeMetadataNormaliser(
+        notice=notice_2018, mongodb_client=mongodb_client
+    ).normalise_metadata(
         extracted_metadata=DefaultNoticeMetadataExtractor(
-            xml_manifestation=notice_2018.xml_manifestation).extract_metadata())
+            xml_manifestation=notice_2018.xml_manifestation
+        ).extract_metadata()
+    )
 
     return normalised_metadata
 
@@ -119,4 +125,4 @@ def notice_id():
 @pytest.fixture
 def work_id_predicate():
     """Returns the URI predicate for the CDM work identifier."""
-    return "http://publications.europa.eu/ontology/cdm#work_id"
+    return "http://publications.europa.eu/ontology/cdm#work_id"

test/unit/notice_packager/test_metadata_transformer.py

@@ -47,4 +47,3 @@ def test_publication_notice_uri(notice_id, notice_sample_metadata):
 def test_publication_work_identifier(notice_id, notice_sample_metadata):
     work_id = publication_work_identifier(notice_id, notice_sample_metadata)
     assert work_id == "2018_S_22_196390"
-