Merge branch 'main' into feature/TED-593

Author: CaptainOfHacks

Makefile

@@ -93,7 +93,7 @@ create-env-airflow:
 
 build-airflow: guard-ENVIRONMENT create-env-airflow build-externals
 	@ echo -e "$(BUILD_PRINT) Build Airflow services $(END_BUILD_PRINT)"
-	@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} build --no-cache --force-rm
+	@ docker build -t meaningfy/airflow ./infra/airflow/
 	@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} up -d --force-recreate
 
 start-airflow: build-externals

dags/selector_daily_fetch_orchestrator.py

@@ -1,4 +1,4 @@
-import datetime
+from datetime import datetime, timedelta
 from typing import List
 
 from airflow.decorators import dag, task
@@ -40,13 +40,15 @@ def fetch_notice_from_ted():
         if WILD_CARD_PARAM in dag_params.keys():
             current_datetime_wildcard = dag_params[WILD_CARD_PARAM]
         else:
-            current_datetime_wildcard = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y%m%d*")
+            current_datetime_wildcard = (datetime.now() - timedelta(days=1)).strftime("%Y%m%d*")
+        notice_publication_date = datetime.strptime(current_datetime_wildcard, "%Y%m%d*").date()
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
         notice_ids = NoticeFetcher(notice_repository=NoticeRepository(mongodb_client=mongodb_client),
                                    ted_api_adapter=TedAPIAdapter(
                                        request_api=TedRequestAPI())).fetch_notices_by_date_wild_card(
             wildcard_date=current_datetime_wildcard)
-        create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
+        create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client,
+                                                        notice_publication_date=notice_publication_date)
         return notice_ids
 
     @task

dags/worker_single_notice_process_orchestrator.py

@@ -75,7 +75,11 @@ def _normalise_notice_metadata(**context_args):
 
         notice: Notice = pull_dag_upstream(NOTICE_OBJECT)
         normalised_notice = normalise_notice(notice=notice)
-        push_dag_downstream(NOTICE_OBJECT, normalised_notice)
+
+        mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
+        notice_repository = NoticeRepository(mongodb_client=mongodb_client)
+        notice_repository.update(notice=normalised_notice)
+        push_dag_downstream(NOTICE_ID, notice.ted_id)
 
         context = get_current_context()
 
@@ -92,14 +96,14 @@ def _check_eligibility_for_transformation(**context_args):
         event_message: NoticeEventMessage = NoticeEventMessage()
         event_message.start_record()
 
-        notice = pull_dag_upstream(NOTICE_OBJECT)
+        notice_id = pull_dag_upstream(NOTICE_ID)
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
         notice_repository = NoticeRepository(mongodb_client=mongodb_client)
+        notice = notice_repository.get(reference=notice_id)
         mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
         result = notice_eligibility_checker(notice=notice, mapping_suite_repository=mapping_suite_repository)
         notice_repository.update(notice=notice)
         mapping_suite_id = None
-        notice_id = notice.ted_id
         if result:
             notice_id, mapping_suite_id = result
             push_dag_downstream(MAPPING_SUITE_ID, mapping_suite_id)
@@ -441,6 +445,7 @@ def _check_notice_state_before_notice_successfully_processed():
     state_skip_table = {
         NoticeStatus.RAW: "index_notice_xml_content",
         NoticeStatus.INDEXED: "index_notice_xml_content",
+        NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION: "check_eligibility_for_transformation",
         NoticeStatus.NORMALISED_METADATA: "check_eligibility_for_transformation",
         NoticeStatus.ELIGIBLE_FOR_PACKAGING: "check_notice_state_before_generate_mets_package",
         NoticeStatus.ELIGIBLE_FOR_PUBLISHING: "check_notice_state_before_publish_notice_in_cellar",

infra/airflow/docker-compose.yaml

@@ -45,7 +45,8 @@ x-airflow-common:
   # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
   # and uncomment the "build" line below, Then run `docker-compose build` to build the images.
   #image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.2-python3.8}
-  build: .
+  #build: .
+  image: meaningfy/airflow:latest
   env_file:
     - ../../.env
   environment:

setup.py

@@ -78,6 +78,7 @@ def open_local(paths, mode="r", encoding="utf8"):
             "sparql_runner = ted_sws.notice_validator.entrypoints.cli.cmd_sparql_runner:main",
             "shacl_runner = ted_sws.notice_validator.entrypoints.cli.cmd_shacl_runner:main",
             "xpath_coverage_runner = ted_sws.notice_validator.entrypoints.cli.cmd_xpath_coverage_runner:main",
+            "validation_summary_runner = ted_sws.notice_validator.entrypoints.cli.cmd_validation_summary_runner:main",
             "rml_report_generator = ted_sws.rml_to_html.entrypoints.cli.cmd_rml_report_generator:main",
             "mapping_suite_processor = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_mapping_suite_processor:main",
             "metadata_generator = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_metadata_generator:main",

ted_sws/core/model/manifestation.py

@@ -60,6 +60,8 @@ class XPATHCoverageValidationAssertion(PropertyBaseModel):
     """
 
     """
+    standard_form_field_id: Optional[str]
+    eform_bt_id: Optional[str]
     title: Optional[str]
     xpath: Optional[str]
     count: Optional[int]

ted_sws/core/model/supra_notice.py

@@ -7,7 +7,7 @@
 
 """ This module implements aggregates over groups of notices and the appropriate business needs, on those groups """
 import abc
-from datetime import datetime, time
+from datetime import datetime, date
 from typing import List, Optional
 
 from ted_sws.core.model import PropertyBaseModel
@@ -45,5 +45,7 @@ class DailySupraNotice(SupraNotice):
     """
         This is an aggregate over the notices published in TED in a specific day.
     """
-    notice_publication_day: datetime = datetime.combine(datetime.today(), time())
+    notice_publication_date: date
     validation_report: Optional[SupraNoticeValidationReport]
+
+

ted_sws/core/model/transform.py

@@ -67,7 +67,9 @@ class TransformationTestData(MappingSuiteComponent):
 
 class ConceptualMappingXPATH(MappingSuiteComponent):
     xpath: str
-    name: str
+    name: Optional[str]
+    standard_form_field_id: Optional[str]
+    eform_bt_id: Optional[str]
 
 
 class ConceptualMappingMetadata(MappingSuiteComponent):

ted_sws/data_manager/adapters/supra_notice_repository.py

@@ -1,3 +1,4 @@
+from datetime import datetime, time
 from typing import Optional, Iterator
 
 from pymongo import MongoClient, ASCENDING
@@ -6,7 +7,7 @@
 from ted_sws.core.model.supra_notice import DailySupraNotice
 from ted_sws.data_manager.adapters.repository_abc import DailySupraNoticeRepositoryABC
 
-DAILY_SUPRA_NOTICE_ID = "notice_publication_day"
+DAILY_SUPRA_NOTICE_ID = "notice_publication_date"
 
 
 class DailySupraNoticeRepository(DailySupraNoticeRepositoryABC):
@@ -26,6 +27,8 @@ def __init__(self, mongodb_client: MongoClient, database_name: str = _database_n
 
     def _update_daily_supra_notice(self, daily_supra_notice: DailySupraNotice, upsert: bool = False):
         daily_supra_notice_dict = daily_supra_notice.dict()
+        daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID] = datetime.combine(
+            daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID], time())
         self.collection.update_one({DAILY_SUPRA_NOTICE_ID: daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID]},
                                    {"$set": daily_supra_notice_dict}, upsert=upsert)
 
@@ -43,19 +46,18 @@ def update(self, daily_supra_notice: DailySupraNotice):
         :param daily_supra_notice:
         :return:
         """
-        daily_supra_notice_exist = self.collection.find_one(
-            {DAILY_SUPRA_NOTICE_ID: daily_supra_notice.notice_publication_day})
-        if daily_supra_notice_exist is not None:
-            self._update_daily_supra_notice(daily_supra_notice=daily_supra_notice)
+        self._update_daily_supra_notice(daily_supra_notice=daily_supra_notice)
 
     def get(self, reference) -> Optional[DailySupraNotice]:
         """
             This method allows a daily_supra_notice to be obtained based on an identification reference.
         :param reference:
         :return: DailySupraNotice
         """
+        reference = datetime.combine(reference, time())
         result_dict = self.collection.find_one({DAILY_SUPRA_NOTICE_ID: reference})
         if result_dict is not None:
+            result_dict[DAILY_SUPRA_NOTICE_ID] = result_dict[DAILY_SUPRA_NOTICE_ID].date()
             daily_supra_notice = DailySupraNotice.parse_obj(result_dict)
             return daily_supra_notice
         return None

ted_sws/mapping_suite_processor/__init__.py

@@ -1,4 +1,6 @@
 CONCEPTUAL_MAPPINGS_METADATA_SHEET_NAME = "Metadata"
 CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME = "Rules"
 RULES_E_FORM_BT_NAME = 'eForm BT Name (O)'
-RULES_FIELD_XPATH = 'Field XPath (M)'
+RULES_FIELD_XPATH = 'Field XPath (M)'
+RULES_SF_FIELD_ID = 'Standard Form Field ID (M)'
+RULES_E_FORM_BT_ID = 'eForm BT-ID (O)'