Merge pull request #210 from OP-TED/feature/TED-562

Feature/ted 562

Author: kaleanych

dags/old_worker_single_notice_process_orchestrator.py

@@ -57,6 +57,7 @@ x-airflow-common:
     AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
     AIRFLOW__CORE__FERNET_KEY: ''
     AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
+    AIRFLOW__CORE__ENABLE_XCOM_PICKLING: "true"
     AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
     AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
     _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}

dags/worker_single_notice_process_orchestrator.py

@@ -8,9 +8,10 @@ services:
     environment:
       - SFTP_USER=${SFTP_USER}
       - SFTP_PASSWORD=${SFTP_PASSWORD}
+      - SFTP_PORT=${SFTP_PORT}
     command: ${SFTP_USER}:${SFTP_PASSWORD}:::upload
     ports:
-      - ${SFTP_PORT:-2235}:22
+      - ${SFTP_PORT:-2235}:${SFTP_PORT}
     networks:
       - sftp-net
       - proxy-net

infra/airflow/docker-compose.yaml

@@ -21,3 +21,5 @@ agraph-python==101.0.10
 decorator~=5.1.1
 urllib3[secure]
 semantic-version==2.10.0
+pysftp
+

infra/sftp/docker-compose.yml

@@ -168,8 +168,31 @@ def FUSEKI_ADMIN_PASSWORD(self) -> str:
     def FUSEKI_ADMIN_HOST(self) -> str:
         return EnvConfigResolver().config_resolve()
 
+class SFTPConfig:
+    @property
+    def SFTP_HOST(self) -> str:
+        return EnvConfigResolver().config_resolve()
+
+    @property
+    def SFTP_PORT(self) -> int:
+        v = EnvConfigResolver().config_resolve()
+        return int(v) if v is not None else 22
+
+    @property
+    def SFTP_USER(self) -> str:
+        return EnvConfigResolver().config_resolve()
+
+    @property
+    def SFTP_PASSWORD(self)->str:
+        return EnvConfigResolver().config_resolve()
+
+    @property
+    def SFTP_PATH(self)->str:
+        return EnvConfigResolver().config_resolve()
+
+
 class TedConfigResolver(MongoDBConfig, RMLMapperConfig, XMLProcessorConfig, ELKConfig, LoggingConfig,
-                        GitHubArtefacts, API, AllegroConfig, TedAPIConfig, FusekiConfig):
+                        GitHubArtefacts, API, AllegroConfig, TedAPIConfig, SFTPConfig, FusekiConfig):
     """
         This class resolve the secrets of the ted-sws project.
     """

requirements.txt

@@ -4,8 +4,8 @@
 import subprocess
 import tempfile
 
-GITHUB_TED_SWS_ARTEFACTS_REPOSITORY_NAME = "ted-sws-artefacts"
-GITHUB_TED_SWS_ARTEFACTS_MAPPINGS_PATH = "ted-sws-artefacts/mappings"
+GITHUB_TED_SWS_ARTEFACTS_REPOSITORY_NAME = "ted-rdf-mapping"
+GITHUB_TED_SWS_ARTEFACTS_MAPPINGS_PATH = f"{GITHUB_TED_SWS_ARTEFACTS_REPOSITORY_NAME}/mappings"
 
 
 class MappingSuitePackageDownloaderABC(abc.ABC):
@@ -48,6 +48,7 @@ def get_git_head_hash(git_repository_path: pathlib.Path) -> str:
                 This function return hash for last commit with git.
             :return:
             """
+            git_repository_path.mkdir(exist_ok=True, parents=True)
             result = subprocess.run(f'cd {git_repository_path} && git rev-parse origin/main', shell=True,
                                     stdout=subprocess.PIPE)
             git_head_hash = result.stdout.decode(encoding="utf-8")

ted_sws/__init__.py

@@ -67,9 +67,8 @@ def mapping_suite_processor_from_github_expand_and_load_package_in_mongo_db(mapp
     :param load_test_data:
     :return:
     """
-    default_github_repository_url = "https://github.com/meaningfy-ws/ted-sws-artefacts.git"
     mapping_suite_package_downloader = GitHubMappingSuitePackageDownloader(
-        github_repository_url=config.GITHUB_TED_SWS_ARTEFACTS_URL or default_github_repository_url)
+        github_repository_url=config.GITHUB_TED_SWS_ARTEFACTS_URL)
     with tempfile.TemporaryDirectory() as tmp_dir:
         tmp_dir_path = pathlib.Path(tmp_dir)
         git_last_commit_hash = mapping_suite_package_downloader.download(