Merge pull request #516 from OP-TED/feature/TED4-85

added test package and refactored logic for eligibility checker

Author: Dragos0000

ted_sws/core/model/metadata.py

@@ -6,6 +6,7 @@
 # Email: costezki.eugen@gmail.com 
 
 """ """
+from enum import Enum
 from typing import List, Optional
 
 from pydantic import Field
@@ -55,6 +56,17 @@ class EncodedValue(NamedTuple):
     value: str = None
 
 
+class NoticeSource(str, Enum):
+    """
+    Holds source of notice
+    """
+    STANDARD_FORM = "standard_forms"
+    ELECTRONIC_FORM = "eforms"
+
+    def __str__(self):
+        return self.value
+
+
 class NormalisedMetadata(Metadata):
     """
         Stores notice normalised metadata
@@ -79,9 +91,10 @@ class NormalisedMetadata(Metadata):
     legal_basis_directive: str
     form_number: str
     eforms_subtype: str
-    xsd_version: str
+    xsd_version: Optional[str]
     published_in_cellar_counter: int = Field(default=0)
-    is_eform: Optional[bool] = False
+    notice_source: Optional[NoticeSource] = NoticeSource.STANDARD_FORM
+    eform_sdk_version: Optional[str]
 
 
 class NormalisedMetadataView(Metadata):
@@ -105,8 +118,10 @@ class NormalisedMetadataView(Metadata):
     legal_basis_directive: str
     form_number: str
     eforms_subtype: str
-    xsd_version: str
+    xsd_version: Optional[str]
     published_in_cellar_counter: int = Field(default=0)
+    notice_source: Optional[NoticeSource] = NoticeSource.STANDARD_FORM
+    eform_sdk_version: Optional[str]
 
 
 

ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_metadata.py

@@ -18,6 +18,7 @@
 END_DATE_KEY = "end_date"
 MIN_XSD_VERSION_KEY = "min_xsd_version"
 MAX_XSD_VERSION_KEY = "max_xsd_version"
+EFORMS_SDK_VERSIONS_KEY = "eforms_sdk_versions"
 TITLE_KEY = "title"
 CREATED_KEY = "created_at"
 IDENTIFIER_KEY = "identifier"

ted_sws/notice_metadata_processor/adapters/notice_metadata_normaliser.py

@@ -4,7 +4,7 @@
 import re
 import pandas as pd
 
-from ted_sws.core.model.metadata import NormalisedMetadata, LanguageTaggedString
+from ted_sws.core.model.metadata import NormalisedMetadata, LanguageTaggedString, NoticeSource
 from ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata
 from ted_sws.notice_metadata_processor.services.metadata_constraints import filter_df_by_variables
 from ted_sws.resources.mapping_files_registry import MappingFilesRegistry
@@ -36,7 +36,8 @@
 DEADLINE_DATE_KEY = "deadline_for_submission"
 NOTICE_TYPE_KEY = "notice_type"
 XSD_VERSION_KEY = "xsd_version"
-IS_EFORM_KEY = "is_eform"
+EFORM_SDK_VERSION_KEY = "eform_sdk_version"
+NOTICE_SOURCE_KEY = "notice_source"
 ENGLISH_LANGUAGE_TAG = "EN"
 mapping_registry = MappingFilesRegistry()
 
@@ -257,7 +258,7 @@ def normalise_metadata(self, extracted_metadata: ExtractedMetadata) -> Normalise
             LEGAL_BASIS_DIRECTIVE_KEY: get_map_value(mapping=legal_basis_map, value=legal_basis),
             E_FORMS_SUBTYPE_KEY: str(eforms_subtype),
             XSD_VERSION_KEY: extracted_metadata.xml_schema_version,
-            IS_EFORM_KEY: False
+            NOTICE_SOURCE_KEY: NoticeSource.STANDARD_FORM
         }
 
         return NormalisedMetadata(**metadata)
@@ -339,8 +340,8 @@ def normalise_metadata(self, extracted_metadata: ExtractedMetadata) -> Normalise
             LEGAL_BASIS_DIRECTIVE_KEY: get_map_value(mapping=legal_basis_map,
                                                      value=legal_basis),
             E_FORMS_SUBTYPE_KEY: extracted_metadata.extracted_notice_subtype,
-            XSD_VERSION_KEY: extracted_metadata.xml_schema_version,
-            IS_EFORM_KEY: True
+            EFORM_SDK_VERSION_KEY: extracted_metadata.xml_schema_version,
+            NOTICE_SOURCE_KEY: NoticeSource.ELECTRONIC_FORM
         }
 
         return NormalisedMetadata(**metadata)

ted_sws/notice_metadata_processor/services/notice_eligibility.py

@@ -3,12 +3,36 @@
 
 import semantic_version
 
-from ted_sws.core.model.metadata import NormalisedMetadata
+from ted_sws.core.model.metadata import NormalisedMetadata, NoticeSource
 from ted_sws.core.model.notice import Notice
 from ted_sws.core.model.transform import MappingSuite
 from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC, NoticeRepositoryABC
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_metadata import START_DATE_KEY, END_DATE_KEY, \
-    MIN_XSD_VERSION_KEY, MAX_XSD_VERSION_KEY, E_FORMS_SUBTYPE_KEY
+    MIN_XSD_VERSION_KEY, MAX_XSD_VERSION_KEY, E_FORMS_SUBTYPE_KEY, EFORMS_SDK_VERSIONS_KEY
+
+
+def format_version_with_zero_patch(version_string:str) -> semantic_version.Version:
+    """
+    This will take a string version (1.7 or 1.7.6) and will transform it to a semantic version with 0 as patch
+    1.7 -> 1.7.0
+    1.7.6 -> 1.7.0
+    """
+    parsed_version = semantic_version.Version.coerce(version_string)
+    return semantic_version.Version(major=parsed_version.major, minor=parsed_version.minor, patch=0)
+
+
+def is_date_in_range(publication_date, constraint_start_date_value, constraint_end_date_value) -> bool:
+    """
+    This will return True or False if publication_date is in range looking at the start and end date constraints in the
+    metadata of a mapping suite
+    """
+    if not constraint_start_date_value and not constraint_end_date_value:
+        return True
+
+    start_date = datetime.datetime.fromisoformat(constraint_start_date_value[0])
+    end_date = datetime.datetime.fromisoformat(
+        constraint_end_date_value[0] if constraint_end_date_value else datetime.datetime.now().isoformat())
+    return start_date <= publication_date <= end_date
 
 
 def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetadata):
@@ -18,25 +42,34 @@ def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetada
     :param mapping_suite:
     :return:
     """
+
     constraints = mapping_suite.metadata_constraints.constraints
 
     eform_subtype = notice_metadata.eforms_subtype
     notice_publication_date = datetime.datetime.fromisoformat(notice_metadata.publication_date)
-    notice_xsd_version = notice_metadata.xsd_version
 
-    end_date = constraints[END_DATE_KEY][0] if constraints[END_DATE_KEY] else datetime.datetime.now().isoformat()
-    constraint_start_date = datetime.datetime.fromisoformat(constraints[START_DATE_KEY][0])
-    constraint_end_date = datetime.datetime.fromisoformat(end_date)
-    constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0]
-    constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0]
+    if notice_metadata.notice_source == NoticeSource.ELECTRONIC_FORM:
+        notice_xsd_version = notice_metadata.eform_sdk_version
+        # eform sdk version value in metadata example: eforms-sdk-1.7 or  eforms-sdk-1.7.9
+        # we need to extract only the version i.e 1.7 or 1.7.9
+        eforms_sdk_version = notice_xsd_version.rsplit('-', 1)[1]
+        constraint_version_range = [format_version_with_zero_patch(version) for version in
+                                    constraints[EFORMS_SDK_VERSIONS_KEY]]
+        in_version_range = format_version_with_zero_patch(eforms_sdk_version) in constraint_version_range
+    else:
+        notice_xsd_version = notice_metadata.xsd_version
+        constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0]
+        constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0]
+        in_version_range = constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version
+
+    in_date_range = is_date_in_range(publication_date=notice_publication_date,
+                                     constraint_start_date_value=constraints[START_DATE_KEY],
+                                     constraint_end_date_value=constraints[END_DATE_KEY])
     eform_subtype_constraint_values = [str(eforms_subtype_value) for eforms_subtype_value in
                                        constraints[E_FORMS_SUBTYPE_KEY]]
-
-    in_date_range = constraint_start_date <= notice_publication_date <= constraint_end_date
-    in_version_range = constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version
     covered_eform_type = eform_subtype in eform_subtype_constraint_values
 
-    return True if in_date_range and in_version_range and covered_eform_type else False
+    return in_date_range and in_version_range and covered_eform_type
 
 
 def notice_eligibility_checker(notice: Notice, mapping_suite_repository: MappingSuiteRepositoryABC) -> Tuple:

tests/conftest.py

@@ -18,7 +18,7 @@
     NOTICE_NUMBER_KEY, PUBLICATION_DATE_KEY, OJS_NUMBER_KEY, OJS_TYPE_KEY, BUYER_CITY_KEY, BUYER_NAME_KEY, LANGUAGE_KEY, \
     BUYER_COUNTRY_KEY, EU_INSTITUTION_KEY, SENT_DATE_KEY, DEADLINE_DATE_KEY, NOTICE_TYPE_KEY, FORM_TYPE_KEY, \
     PLACE_OF_PERFORMANCE_KEY, EXTRACTED_LEGAL_BASIS_KEY, FORM_NUMBER_KEY, LEGAL_BASIS_DIRECTIVE_KEY, \
-    E_FORMS_SUBTYPE_KEY, XSD_VERSION_KEY
+    E_FORMS_SUBTYPE_KEY, XSD_VERSION_KEY, EFORM_SDK_VERSION_KEY, NOTICE_SOURCE_KEY
 
 from tests import TEST_DATA_PATH
 from tests.fakes.fake_repository import FakeNoticeRepository
@@ -212,6 +212,35 @@ def normalised_metadata_object():
 
     return NormalisedMetadata(**data)
 
+@pytest.fixture
+def eform_normalised_metadata_object():
+    data = {
+        TITLE_KEY: [LanguageTaggedString(text="Eteläisen Salon liikuntapaikkojen hoidon hankinta", language="FIN")],
+        LONG_TITLE_KEY: [
+            LanguageTaggedString(text="FIN :: Eteläisen Salon liikuntapaikkojen hoidon hankinta", language="FIN")],
+        NOTICE_NUMBER_KEY: "00622690-2023",
+        PUBLICATION_DATE_KEY: "2023-10-13T00:00:00",
+        OJS_NUMBER_KEY: "198/2023",
+        OJS_TYPE_KEY: "S",
+        BUYER_CITY_KEY: None,
+        BUYER_NAME_KEY: None,
+        LANGUAGE_KEY: None,
+        BUYER_COUNTRY_KEY: None,
+        EU_INSTITUTION_KEY: None,
+        SENT_DATE_KEY: "2023-10-12T00:00:00",
+        DEADLINE_DATE_KEY: None,
+        NOTICE_TYPE_KEY: "http://publications.europa.eu/resource/authority/notice-type/cn-standard",
+        FORM_TYPE_KEY: "http://publications.europa.eu/resource/authority/form-type/competition",
+        PLACE_OF_PERFORMANCE_KEY: ["http://data.europa.eu/nuts/code/FI1C1"],
+        EXTRACTED_LEGAL_BASIS_KEY: "http://publications.europa.eu/resource/authority/legal-basis/32014L0024",
+        LEGAL_BASIS_DIRECTIVE_KEY: "http://publications.europa.eu/resource/authority/legal-basis/32014L0024",
+        FORM_NUMBER_KEY: "",
+        E_FORMS_SUBTYPE_KEY: "16",
+        EFORM_SDK_VERSION_KEY: "eforms-sdk-1.7",
+        NOTICE_SOURCE_KEY: "eforms"
+    }
+
+    return NormalisedMetadata(**data)
 
 @pytest.fixture
 @mongomock.patch(servers=(('server.example.com', 27017),))

tests/test_data/notice_transformer/test_repository/test_package4/metadata.json

@@ -0,0 +1,34 @@
+{
+  "identifier": "package_EF16",
+  "title": "Package EF16 v1.2",
+  "description": "This is the conceptual mapping for bla bla bla",
+  "mapping_version": "3.0.0-alpha.1",
+  "ontology_version": "4.0.0",
+  "metadata_constraints": {
+    "constraints": {
+      "min_xsd_version": ["R2.0.9.S04.E01"],
+      "max_xsd_version": ["R2.0.9.S04.E01"],
+      "eforms_subtype": [
+        "16",
+        "10",
+        "11",
+        "12",
+        "13",
+        "X1",
+        "T1"
+      ],
+      "start_date": null,
+      "end_date": null,
+      "eforms_sdk_versions": [
+        "1.3",
+        "1.4",
+        "1.5",
+        "1.6",
+        "1.7",
+        "1.8",
+        "1.9",
+        "1.10"
+      ]
+    }
+  }
+}