Merge pull request #517 from OP-TED/feature/TED4-84

update MappingSuite to support eForms

Author: CaptainOfHacks

ted_sws/core/model/transform.py

@@ -8,6 +8,7 @@
 """ """
 import abc
 from datetime import datetime
+from enum import Enum
 from typing import List, Optional
 
 from ted_sws.core.model import PropertyBaseModel
@@ -188,6 +189,14 @@ class ConceptualMapping(MappingSuiteComponent):
     cl2_organisations: List[ConceptualMappingControlList] = []
 
 
+class MappingSuiteType(str, Enum):
+    STANDARD_FORMS = "standard_forms"
+    ELECTRONIC_FORMS = "eforms"
+
+    def __str__(self):
+        return self.value
+
+
 class MappingSuite(MappingSuiteComponent):
     """
 
@@ -199,6 +208,7 @@ class MappingSuite(MappingSuiteComponent):
     ontology_version: str = "0.0.1"
     git_latest_commit_hash: str = "no_hash"
     mapping_suite_hash_digest: str = "no_hash"
+    mapping_type: Optional[MappingSuiteType] = MappingSuiteType.STANDARD_FORMS
     metadata_constraints: MetadataConstraints
     transformation_rule_set: TransformationRuleSet
     shacl_test_suites: List[SHACLTestSuite]

ted_sws/notice_metadata_processor/services/notice_eligibility.py

@@ -5,13 +5,13 @@
 
 from ted_sws.core.model.metadata import NormalisedMetadata, NoticeSource
 from ted_sws.core.model.notice import Notice
-from ted_sws.core.model.transform import MappingSuite
+from ted_sws.core.model.transform import MappingSuite, MappingSuiteType
 from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC, NoticeRepositoryABC
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_metadata import START_DATE_KEY, END_DATE_KEY, \
     MIN_XSD_VERSION_KEY, MAX_XSD_VERSION_KEY, E_FORMS_SUBTYPE_KEY, EFORMS_SDK_VERSIONS_KEY
 
 
-def format_version_with_zero_patch(version_string:str) -> semantic_version.Version:
+def format_version_with_zero_patch(version_string: str) -> semantic_version.Version:
     """
     This will take a string version (1.7 or 1.7.6) and will transform it to a semantic version with 0 as patch
     1.7 -> 1.7.0
@@ -35,6 +35,25 @@ def is_date_in_range(publication_date, constraint_start_date_value, constraint_e
     return start_date <= publication_date <= end_date
 
 
+def is_version_in_range(notice_metadata: NormalisedMetadata, mapping_suite: MappingSuite) -> bool:
+    constraints = mapping_suite.metadata_constraints.constraints
+    if mapping_suite.mapping_type == MappingSuiteType.ELECTRONIC_FORMS and notice_metadata.notice_source == NoticeSource.ELECTRONIC_FORM:
+        notice_xsd_version = notice_metadata.eform_sdk_version
+        # eform sdk version value in metadata example: eforms-sdk-1.7 or  eforms-sdk-1.7.9
+        # we need to extract only the version i.e 1.7 or 1.7.9
+        eforms_sdk_version = notice_xsd_version.rsplit('-', 1)[1]
+        constraint_version_range = [format_version_with_zero_patch(version) for version in
+                                    constraints[EFORMS_SDK_VERSIONS_KEY]]
+        return format_version_with_zero_patch(eforms_sdk_version) in constraint_version_range
+    elif mapping_suite.mapping_type == MappingSuiteType.STANDARD_FORMS and notice_metadata.notice_source == NoticeSource.STANDARD_FORM:
+        notice_xsd_version = notice_metadata.xsd_version
+        constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0]
+        constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0]
+        return constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version
+
+    return False
+
+
 def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetadata):
     """
     Check if mapping suite is valid for notice
@@ -47,20 +66,7 @@ def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetada
 
     eform_subtype = notice_metadata.eforms_subtype
     notice_publication_date = datetime.datetime.fromisoformat(notice_metadata.publication_date)
-
-    if notice_metadata.notice_source == NoticeSource.ELECTRONIC_FORM:
-        notice_xsd_version = notice_metadata.eform_sdk_version
-        # eform sdk version value in metadata example: eforms-sdk-1.7 or  eforms-sdk-1.7.9
-        # we need to extract only the version i.e 1.7 or 1.7.9
-        eforms_sdk_version = notice_xsd_version.rsplit('-', 1)[1]
-        constraint_version_range = [format_version_with_zero_patch(version) for version in
-                                    constraints[EFORMS_SDK_VERSIONS_KEY]]
-        in_version_range = format_version_with_zero_patch(eforms_sdk_version) in constraint_version_range
-    else:
-        notice_xsd_version = notice_metadata.xsd_version
-        constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0]
-        constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0]
-        in_version_range = constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version
+    in_version_range = is_version_in_range(notice_metadata=notice_metadata, mapping_suite=mapping_suite)
 
     in_date_range = is_date_in_range(publication_date=notice_publication_date,
                                      constraint_start_date_value=constraints[START_DATE_KEY],

tests/conftest.py

@@ -212,6 +212,7 @@ def normalised_metadata_object():
 
     return NormalisedMetadata(**data)
 
+
 @pytest.fixture
 def eform_normalised_metadata_object():
     data = {
@@ -242,6 +243,7 @@ def eform_normalised_metadata_object():
 
     return NormalisedMetadata(**data)
 
+
 @pytest.fixture
 @mongomock.patch(servers=(('server.example.com', 27017),))
 def mongodb_client():
@@ -302,4 +304,10 @@ def eform_notice_622690():
     notice = Notice(ted_id=ted_id)
     notice.set_xml_manifestation(xml_manifestation)
     notice.set_original_metadata(original_metadata)
-    return notice
+    return notice
+
+
+@pytest.fixture
+def indexed_eform_notice_622690(eform_notice_622690):
+    eform_notice_622690.set_xml_metadata(XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
+    return eform_notice_622690

tests/test_data/notice_transformer/test_repository/test_package4/metadata.json

@@ -4,10 +4,9 @@
   "description": "This is the conceptual mapping for bla bla bla",
   "mapping_version": "3.0.0-alpha.1",
   "ontology_version": "4.0.0",
+  "mapping_type": "eforms",
   "metadata_constraints": {
     "constraints": {
-      "min_xsd_version": ["R2.0.9.S04.E01"],
-      "max_xsd_version": ["R2.0.9.S04.E01"],
       "eforms_subtype": [
         "16",
         "10",

tests/unit/data_manager/conftest.py

@@ -16,6 +16,15 @@ def file_system_repository_path():
     return TEST_DATA_PATH / "notice_transformer" / "test_file_system_repository"
 
 
+@pytest.fixture
+def file_system_repository_with_packages_path():
+    return TEST_DATA_PATH / "notice_transformer" / "test_repository"
+
+
+@pytest.fixture
+def epo_mapping_suite_package_name():
+    return "test_package4"
+
 @pytest.fixture
 def fake_mapping_suite():
     metadata_constrains = MetadataConstraints(constraints=dict())

tests/unit/data_manager/test_mapping_suite_repository.py

@@ -38,6 +38,22 @@ def test_mapping_suite_repository_mongodb_update_invalid_id(mongodb_client, fake
     mongodb_client.drop_database(aggregates_database_name)
 
 
+def test_epo_mapping_suite_repository_in_file_system(file_system_repository_with_packages_path,
+                                                     epo_mapping_suite_package_name):
+    assert file_system_repository_with_packages_path.exists()
+    mapping_suite_repository = MappingSuiteRepositoryInFileSystem(
+        repository_path=file_system_repository_with_packages_path)
+    result_mapping_suite = mapping_suite_repository.get(reference=epo_mapping_suite_package_name)
+    assert result_mapping_suite
+    assert result_mapping_suite.identifier == "package_EF16"
+    assert result_mapping_suite.title == "Package EF16 v1.2"
+    assert result_mapping_suite.mapping_type == "eforms"
+    assert result_mapping_suite.metadata_constraints.constraints
+    constraints = result_mapping_suite.metadata_constraints.constraints
+    assert constraints["eforms_subtype"]
+    assert constraints["eforms_sdk_versions"]
+
+
 def test_mapping_suite_repository_in_file_system(file_system_repository_path, fake_mapping_suite):
     mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=file_system_repository_path)
     mapping_suite_repository.clear_repository()

tests/unit/notice_metadata_processor/conftest.py

@@ -11,4 +11,3 @@ def notice_eligibility_repository_path():
 @pytest.fixture
 def file_system_repository_path():
     return TEST_DATA_PATH / "notice_transformer" / "mapping_suite_processor_repository"
-

tests/unit/notice_metadata_processor/test_eligibility.py

@@ -9,15 +9,20 @@
     notice_eligibility_checker, notice_eligibility_checker_by_id, format_version_with_zero_patch, is_date_in_range
 
 
-# TODO remove min and max xsd version when the mapping loader is refactored tests/test_data/notice_transformer/test_repository/test_package4/metadata.json
-# TODO Add test for eform using mappinSuiteRepository
 def test_non_eligibility_by_notice(notice_eligibility_repository_path, indexed_notice):
     mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
     normalise_notice(notice=indexed_notice)
     notice_eligibility_checker(notice=indexed_notice, mapping_suite_repository=mapping_suite_repository)
     assert indexed_notice.status == NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION
 
 
+def test_eforms_eligibility_by_notice(notice_eligibility_repository_path, indexed_eform_notice_622690):
+    mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
+    normalise_notice(notice=indexed_eform_notice_622690)
+    notice_eligibility_checker(notice=indexed_eform_notice_622690, mapping_suite_repository=mapping_suite_repository)
+    assert indexed_eform_notice_622690.status == NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
+
+
 def test_eligibility_by_notice(notice_eligibility_repository_path, notice_2020):
     mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
     normalise_notice(notice=notice_2020)
@@ -42,7 +47,8 @@ def test_eligibility_by_notice_id(notice_eligibility_repository_path, notice_202
     assert notice_2020.status == NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
 
 
-def test_check_mapping_suite(notice_eligibility_repository_path, normalised_metadata_object, eform_normalised_metadata_object):
+def test_check_mapping_suite(notice_eligibility_repository_path, normalised_metadata_object,
+                             eform_normalised_metadata_object):
     mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path)
     is_valid = check_package(mapping_suite=mapping_suite_repository.get("test_package"),
                              notice_metadata=normalised_metadata_object)