Merge branch 'main' into feature/TED-392

Author: CaptainOfHacks

.github/workflows/unit-tests-hermes.yml

@@ -27,8 +27,8 @@ jobs:
       - name: Get Saxon
         run: make init-saxon
 
-      - name: Start staging infra
-        run: make start-project-services
+#      - name: Start staging infra
+#        run: make start-project-services
 
       - name: Run unit tests
         run: make test-all

dags/__init__.py

@@ -9,7 +9,7 @@
     "email_on_retry": False,
     "retries": 0,
     "retry_delay": timedelta(minutes=3600),
-    "max_active_runs": 128,
-    "concurrency": 128,
+    "max_active_runs": 15,
+    "concurrency": 15,
     "execution_timeout": timedelta(days=10),
 }

dags/fetch_notices_per_day_worker.py

@@ -1,4 +1,6 @@
 import datetime
+import time
+from random import randint
 
 from airflow.operators.trigger_dagrun import TriggerDagRunOperator
 
@@ -38,16 +40,18 @@ def fetch_notices_and_trigger_index_and_normalise_notice_worker():
             while restart_dag_operator:
                 restart_dag_operator = False
                 try:
+                    time.sleep(randint(10, 500) / 1000)
                     TriggerDagRunOperator(
                         task_id=f'trigger_index_and_normalise_notice_worker_dag_{notice_id}',
                         trigger_dag_id="index_and_normalise_notice_worker",
                         trigger_run_id=notice_id,
-                        execution_date=datetime.datetime.now().replace(tzinfo=datetime.timezone.utc),
                         conf={NOTICE_ID: notice_id}
                     ).execute(context=context)
-                except:
+                except Exception as e:
+
                     restart_dag_operator = True
                     print("trigger dag operator restarted !!!")
+                    print("EXCEPTION message: ", e)
 
     fetch_notices_and_trigger_index_and_normalise_notice_worker()