Merge remote-tracking branch 'origin/main' into feature/TED-1057

Author: Kolea Plesco

dags/fetch_notices_by_query.py

@@ -21,7 +21,7 @@
 
 
 @dag(default_args=DEFAULT_DAG_ARGUMENTS,
-     catchup=False,
+     schedule_interval=None,
      tags=['fetch'])
 def fetch_notices_by_query():
     @task

dags/operators/DagBatchPipelineOperator.py

@@ -8,7 +8,6 @@
     smart_xcom_push
 from dags.pipelines.pipeline_protocols import NoticePipelineCallable
 from ted_sws import config
-from ted_sws.core.model.notice import NoticeStatus
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.event_manager.model.event_message import EventMessage, NoticeEventMessage
 from ted_sws.event_manager.services.log import log_notice_error
@@ -18,7 +17,7 @@
 START_WITH_STEP_NAME_KEY = "start_with_step_name"
 EXECUTE_ONLY_ONE_STEP_KEY = "execute_only_one_step"
 DEFAULT_NUBER_OF_CELERY_WORKERS = 144  # TODO: revise this config
-NOTICE_PROCESS_WORKFLOW_DAG_NAME = "notice_process_workflow"
+NOTICE_PROCESSING_PIPELINE_DAG_NAME = "notice_processing_pipeline"
 DEFAULT_START_WITH_TASK_ID = "notice_normalisation_pipeline"
 DEFAULT_PIPELINE_NAME_FOR_LOGS = "unknown_pipeline_name"
 
@@ -136,7 +135,7 @@ def execute(self, context: Any):
             for notice_batch in chunks(notice_ids, chunk_size=batch_size):
                 TriggerDagRunOperator(
                     task_id=f'trigger_worker_dag_{uuid4().hex}',
-                    trigger_dag_id=NOTICE_PROCESS_WORKFLOW_DAG_NAME,
+                    trigger_dag_id=NOTICE_PROCESSING_PIPELINE_DAG_NAME,
                     conf={
                         NOTICE_IDS_KEY: list(notice_batch),
                         START_WITH_STEP_NAME_KEY: self.start_with_step_name,

setup.py

@@ -24,7 +24,6 @@
 kwargs["extras_require"] = {
 }
 
-
 def find_version(filename):
     _version_re = re.compile(r'__version__ = "(.*)"')
     for line in open(filename):
@@ -43,7 +42,7 @@ def open_local(paths, mode="r", encoding="utf8"):
 
 version = find_version("ted_sws/__init__.py")
 
-packages = find_packages(exclude=("examples*", "test*"))
+packages = find_packages(exclude=("examples*", "tests*", "dags*", "notebooks*", "infra*", "docs*"))
 
 setup(
     name="ted_sws",

ted_sws/mapping_suite_processor/services/conceptual_mapping_processor.py

@@ -20,7 +20,7 @@
 CONCEPTUAL_MAPPINGS_ASSERTIONS = "cm_assertions"
 SHACL_SHAPE_INJECTION_FOLDER = "ap_data_shape"
 SHACL_SHAPE_RESOURCES_FOLDER = "shacl_shapes"
-SHACL_SHAPE_FILE_NAME = "ePO_shacl_shapes.rdf"
+SHACL_SHAPE_FILE_NAME = "ePO_shacl_shapes.ttl"
 MAPPING_FILES_RESOURCES_FOLDER = "mapping_files"
 SPARQL_QUERIES_RESOURCES_FOLDER = "queries"
 SPARQL_QUERIES_INJECTION_FOLDER = "business_queries"