merge with main

Author: Kolea Plesco

dags/old_worker_single_notice_process_orchestrator.py

@@ -57,6 +57,7 @@ x-airflow-common:
     AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
     AIRFLOW__CORE__FERNET_KEY: ''
     AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
+    AIRFLOW__CORE__ENABLE_XCOM_PICKLING: "true"
     AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
     AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
     _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}

dags/worker_single_notice_process_orchestrator.py

@@ -99,7 +99,9 @@ def add_validation(self, validation: Union[XPATHCoverageValidationReport]):
             self.xpath_coverage_validation: XPATHCoverageValidationReport = validation
 
     def is_validated(self) -> bool:
+        print("K :: ", self.xpath_coverage_validation)
         if self.xpath_coverage_validation:
+            print("K :: VALIDATED")
             return True
         return False
 

infra/airflow/docker-compose.yaml

@@ -4,8 +4,8 @@
 import subprocess
 import tempfile
 
-GITHUB_TED_SWS_ARTEFACTS_REPOSITORY_NAME = "ted-sws-artefacts"
-GITHUB_TED_SWS_ARTEFACTS_MAPPINGS_PATH = "ted-sws-artefacts/mappings"
+GITHUB_TED_SWS_ARTEFACTS_REPOSITORY_NAME = "ted-rdf-mapping"
+GITHUB_TED_SWS_ARTEFACTS_MAPPINGS_PATH = f"{GITHUB_TED_SWS_ARTEFACTS_REPOSITORY_NAME}/mappings"
 
 
 class MappingSuitePackageDownloaderABC(abc.ABC):
@@ -48,6 +48,7 @@ def get_git_head_hash(git_repository_path: pathlib.Path) -> str:
                 This function return hash for last commit with git.
             :return:
             """
+            git_repository_path.mkdir(exist_ok=True, parents=True)
             result = subprocess.run(f'cd {git_repository_path} && git rev-parse origin/main', shell=True,
                                     stdout=subprocess.PIPE)
             git_head_hash = result.stdout.decode(encoding="utf-8")

ted_sws/core/model/manifestation.py

@@ -6,6 +6,7 @@
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC
 from ted_sws.notice_validator.adapters.xpath_coverage_runner import CoverageRunner, PATH_TYPE
+from pymongo import MongoClient
 
 
 class XPATHCoverageReportBuilder:
@@ -25,9 +26,8 @@ def generate_report(self) -> XPATHCoverageValidationReport:
 def coverage_notice_xpath_report(notices: List[Notice], mapping_suite_id,
                                  conceptual_mappings_file_path: PATH_TYPE = None,
                                  coverage_runner: CoverageRunner = None, xslt_transformer=None,
-                                 notice_repository: NoticeRepository = None) -> XPATHCoverageValidationReport:
+                                 mongodb_client: MongoClient = None) -> XPATHCoverageValidationReport:
     if not coverage_runner:
-        mongodb_client = notice_repository.mongodb_client if notice_repository else None
         coverage_runner = CoverageRunner(mapping_suite_id, conceptual_mappings_file_path, xslt_transformer,
                                          mongodb_client)
     report: XPATHCoverageValidationReport = coverage_runner.coverage_notice_xpath(notices, mapping_suite_id)
@@ -42,23 +42,24 @@ def xpath_coverage_html_report(report: XPATHCoverageValidationReport) -> str:
     return CoverageRunner.html_report(report)
 
 
-def validate_xpath_coverage_notice(notice: Notice, mapping_suite: MappingSuite, notice_repository: NoticeRepository):
+def validate_xpath_coverage_notice(notice: Notice, mapping_suite: MappingSuite, mongodb_client: MongoClient):
     xpath_coverage_report = coverage_notice_xpath_report(notices=[notice],
                                                          mapping_suite_id=mapping_suite.identifier,
-                                                         notice_repository=notice_repository)
+                                                         mongodb_client=mongodb_client)
     report_builder = XPATHCoverageReportBuilder(xpath_coverage_report=xpath_coverage_report)
     notice.set_xml_validation(xml_validation=report_builder.generate_report())
 
 
 def validate_xpath_coverage_notice_by_id(notice_id: str, mapping_suite_identifier: str,
                                          mapping_suite_repository: MappingSuiteRepositoryABC,
-                                         notice_repository: NoticeRepository):
+                                         mongodb_client: MongoClient):
+    notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     notice = notice_repository.get(reference=notice_id)
     if notice is None:
         raise ValueError(f'Notice, with {notice_id} id, was not found')
 
     mapping_suite = mapping_suite_repository.get(reference=mapping_suite_identifier)
     if mapping_suite is None:
         raise ValueError(f'Mapping suite, with {mapping_suite_identifier} id, was not found')
-    validate_xpath_coverage_notice(notice=notice, mapping_suite=mapping_suite, notice_repository=notice_repository)
+    validate_xpath_coverage_notice(notice=notice, mapping_suite=mapping_suite, mongodb_client=mongodb_client)
     notice_repository.update(notice=notice)

ted_sws/mapping_suite_processor/adapters/github_package_downloader.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python3
+
+# test_notice_transmisions.py
+# Date:  08.08.2022
+# Author: Stratulat Ștefan
+# Email: stefan.stratulat1997@gmail.com
+import pickle
+
+from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
+from deepdiff import DeepDiff
+
+
+def test_notice_is_convertible_to_dict(notice_2021):
+    notice = notice_2021
+    notice_dict = NoticeRepository._create_dict_from_notice(notice)
+    result_notice = NoticeRepository._create_notice_from_repository_result(notice_dict)
+    assert DeepDiff(notice.dict(), result_notice.dict(), ignore_order=True) == {}
+
+
+def test_notice_is_convertible_to_pickle(notice_2021):
+    notice = notice_2021
+    notice_dump = pickle.dumps(notice)
+    result_notice = pickle.loads(notice_dump)
+    assert DeepDiff(notice.dict(), result_notice.dict(), ignore_order=True) == {}

ted_sws/notice_validator/services/xpath_coverage_runner.py

@@ -10,7 +10,7 @@
     CHECK_NOTICE_STATE_BEFORE_TRANSFORM_TASK_ID, CHECK_NOTICE_STATE_BEFORE_GENERATE_METS_PACKAGE_TASK_ID, \
     INDEX_NOTICE_XML_CONTENT_TASK_ID
 
-DAG_ID = "worker_single_notice_process_orchestrator"
+DAG_ID = "old_worker_single_notice_process_orchestrator"
 
 
 def execute_dag_step(dag, task_id: str, dag_config: dict, xcom_push_data: dict = None):

tests/e2e/dags/test_notice_transmisions.py

@@ -12,7 +12,6 @@
 def test_xpath_coverage_runner(fake_notice_F03, fake_conceptual_mappings_F03_path, fake_xslt_transformer,
                                fake_mapping_suite_F03_id, mongodb_client, fake_mapping_suite_F03_path,
                                invalid_mapping_suite_id):
-    notice_repository = NoticeRepository(mongodb_client=mongodb_client)
     report = coverage_notice_xpath_report([fake_notice_F03], fake_mapping_suite_F03_id,
                                           fake_conceptual_mappings_F03_path, None, fake_xslt_transformer)
     json_report = xpath_coverage_json_report(report)
@@ -31,14 +30,14 @@ def test_xpath_coverage_runner(fake_notice_F03, fake_conceptual_mappings_F03_pat
 
     report = coverage_notice_xpath_report([fake_notice_F03], fake_mapping_suite_F03_id,
                                           fake_conceptual_mappings_F03_path, None, fake_xslt_transformer,
-                                          notice_repository)
+                                          mongodb_client)
     json_report = xpath_coverage_json_report(report)
     assert isinstance(json_report, dict)
 
     with pytest.raises(ValueError):
         coverage_notice_xpath_report([fake_notice_F03], invalid_mapping_suite_id,
                                      None, None, fake_xslt_transformer,
-                                     notice_repository)
+                                     mongodb_client)
 
 
 def test_validate_xpath_coverage_notice_by_id(fake_notice_id, fake_mapping_suite_F03_id,
@@ -54,7 +53,7 @@ def test_validate_xpath_coverage_notice_by_id(fake_notice_id, fake_mapping_suite
             notice_id=fake_notice_id,
             mapping_suite_identifier=fake_mapping_suite_F03_id,
             mapping_suite_repository=mapping_suite_repository_db,
-            notice_repository=notice_repository)
+            mongodb_client=mongodb_client)
 
     notice_repository.add(fake_notice_F03)
 
@@ -63,12 +62,12 @@ def test_validate_xpath_coverage_notice_by_id(fake_notice_id, fake_mapping_suite
             notice_id=fake_notice_id,
             mapping_suite_identifier=fake_mapping_suite_F03_id,
             mapping_suite_repository=mapping_suite_repository_db,
-            notice_repository=notice_repository)
+            mongodb_client=mongodb_client)
 
     mapping_suite_repository_db.add(mapping_suite)
 
     validate_xpath_coverage_notice_by_id(
         notice_id=fake_notice_id,
         mapping_suite_identifier=fake_mapping_suite_F03_id,
         mapping_suite_repository=mapping_suite_repository_db,
-        notice_repository=notice_repository)
+        mongodb_client=mongodb_client)