Merge remote-tracking branch 'origin/main' into feature/TED-585

Author: Kolea Plesco

dags/selector_retransform_process_orchestrator.py

@@ -8,17 +8,18 @@
 from ted_sws.core.model.notice import NoticeStatus
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.event_manager.adapters.event_log_decorator import event_log
+from ted_sws.event_manager.adapters.event_logger import EventLogger
 from ted_sws.event_manager.model.event_message import TechnicalEventMessage, EventMessageMetadata, \
-    EventMessageProcessType
+    EventMessageProcessType, EventMessage
+from ted_sws.event_manager.services.logger_from_context import get_logger_from_dag_context
 
 DAG_NAME = "selector_re_transform_process_orchestrator"
 
-RE_TRANSFORM_TARGET_NOTICE_STATES = [NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION, NoticeStatus.NORMALISED_METADATA,
-                                     NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION,
+RE_TRANSFORM_TARGET_NOTICE_STATES = [NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION,
                                      NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION,
                                      NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION, NoticeStatus.TRANSFORMED,
-                                     NoticeStatus.DISTILLED,
-                                     NoticeStatus.VALIDATED, NoticeStatus.INELIGIBLE_FOR_PACKAGING
+                                     NoticeStatus.DISTILLED, NoticeStatus.VALIDATED,
+                                     NoticeStatus.INELIGIBLE_FOR_PACKAGING
                                      ]
 
 
@@ -33,13 +34,15 @@ def selector_re_transform_process_orchestrator():
             process_type=EventMessageProcessType.DAG, process_name=DAG_NAME
         ))
     )
-    def select_notices_for_re_transform_and_reset_status():
+    def select_notices_for_re_transform_and_reset_status(**context_args):
+        event_logger: EventLogger = get_logger_from_dag_context(context_args)
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
         notice_repository = NoticeRepository(mongodb_client=mongodb_client)
         for target_notice_state in RE_TRANSFORM_TARGET_NOTICE_STATES:
+            event_logger.info(event_message=EventMessage(message=f"select notices with status : {target_notice_state}"))
             notices = notice_repository.get_notice_by_status(notice_status=target_notice_state)
             for notice in notices:
-                notice.update_status_to(new_status=NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION)
+                notice.update_status_to(new_status=NoticeStatus.NORMALISED_METADATA)
                 notice_repository.update(notice=notice)
 
     @task
@@ -53,7 +56,7 @@ def trigger_worker_for_transform_branch():
         context = get_current_context()
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
         notice_repository = NoticeRepository(mongodb_client=mongodb_client)
-        notices = notice_repository.get_notice_by_status(notice_status=NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION)
+        notices = notice_repository.get_notice_by_status(notice_status=NoticeStatus.NORMALISED_METADATA)
         for notice in notices:
             TriggerDagRunOperator(
                 task_id=f'trigger_worker_dag_{notice.ted_id}',

dags/worker_single_notice_process_orchestrator.py

@@ -453,9 +453,9 @@ def _check_notice_state_before_notice_successfully_processed():
     state_skip_table = {
         NoticeStatus.RAW: "index_notice_xml_content",
         NoticeStatus.INDEXED: "index_notice_xml_content",
-        NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION: "check_eligibility_for_transformation",
-        NoticeStatus.ELIGIBLE_FOR_PACKAGING: "generate_mets_package",
-        NoticeStatus.ELIGIBLE_FOR_PUBLISHING: "publish_notice_in_cellar",
+        NoticeStatus.NORMALISED_METADATA: "check_eligibility_for_transformation",
+        NoticeStatus.ELIGIBLE_FOR_PACKAGING: "check_notice_state_before_generate_mets_package",
+        NoticeStatus.ELIGIBLE_FOR_PUBLISHING: "check_notice_state_before_publish_notice_in_cellar",
     }
 
     def _get_task_run():

requirements.txt

@@ -19,4 +19,5 @@ rdflib~=6.1.1
 pyshacl~=0.19.0
 agraph-python==101.0.10
 decorator~=5.1.1
-urllib3[secure]
+urllib3[secure]
+semantic-version==2.10.0

ted_sws/notice_metadata_processor/services/notice_eligibility.py

@@ -1,6 +1,8 @@
 import datetime
 from typing import Tuple
 
+import semantic_version
+
 from ted_sws.core.model.metadata import NormalisedMetadata
 from ted_sws.core.model.notice import Notice
 from ted_sws.core.model.transform import MappingSuite
@@ -35,17 +37,6 @@ def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetada
     return True if in_date_range and in_version_range and covered_eform_type else False
 
 
-def transform_version_string_into_int(version_string: str) -> int:
-    """
-    Transforming a version string into a number. (example_version = "1.2.3")
-    :param version_string:
-    :return:
-    """
-    version_numbers = [int(x) for x in version_string.split(".")]
-    assert len(version_numbers) == 3
-    return ((version_numbers[0] * 100) + version_numbers[1]) * 100 + version_numbers[2]
-
-
 def notice_eligibility_checker(notice: Notice, mapping_suite_repository: MappingSuiteRepositoryABC) -> Tuple:
     """
     Check if notice in eligible for transformation
@@ -60,11 +51,13 @@ def notice_eligibility_checker(notice: Notice, mapping_suite_repository: Mapping
             possible_mapping_suites.append(mapping_suite)
 
     if possible_mapping_suites:
-        best_version = max([transform_version_string_into_int(version_string=mapping_suite.version) for mapping_suite in
-                            possible_mapping_suites])
-        mapping_suite_identifier = next((mapping_suite.identifier for mapping_suite in possible_mapping_suites if
-                                         transform_version_string_into_int(
-                                             version_string=mapping_suite.version) == best_version), None)
+        best_version = possible_mapping_suites[0].version
+        mapping_suite_identifier = possible_mapping_suites[0].identifier
+        for mapping_suite in possible_mapping_suites[1:]:
+            if semantic_version.Version(mapping_suite.version) > semantic_version.Version(best_version):
+                best_version = mapping_suite.version
+                mapping_suite_identifier = mapping_suite.identifier
+
         notice.set_is_eligible_for_transformation(eligibility=True)
         return notice.ted_id, mapping_suite_identifier
     else:

tests/unit/notice_metadata_processor/test_eligibility.py

@@ -2,7 +2,7 @@
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.notice_metadata_processor.services.metadata_normalizer import MetadataNormaliser
 from ted_sws.notice_metadata_processor.services.notice_eligibility import check_package, \
-    notice_eligibility_checker, notice_eligibility_checker_by_id, transform_version_string_into_int
+    notice_eligibility_checker, notice_eligibility_checker_by_id
 
 
 def test_non_eligibility_by_notice(notice_eligibility_repository_path, indexed_notice):
@@ -47,10 +47,3 @@ def test_check_mapping_suite(notice_eligibility_repository_path, normalised_meta
     is_valid = check_package(mapping_suite=mapping_suite_repository.get("test_package"),
                              notice_metadata=normalised_metadata_object)
     assert not is_valid
-
-
-def test_transform_version_string_into_int():
-    funky_version_string = "5.7.6"
-    funky_version_int = transform_version_string_into_int(version_string=funky_version_string)
-    assert isinstance(funky_version_int, int)
-    assert funky_version_int == 50706