WIP

Author: CaptainOfHacks

infra/airflow-cluster/docker-compose-worker.yaml

@@ -55,9 +55,9 @@ x-airflow-common:
     AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 256
     AIRFLOW__CORE__NON_POOLED_TASK_SLOT_COUNT: 256
     AIRFLOW__SCHEDULER__PARSING_PROCESSES: 24
-    AIRFLOW__CELERY__WORKER_CONCURRENCY: 32
-    AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE: 256
-    AIRFLOW__CORE__SQL_ALCHEMY_MAX_OVERFLOW: 512
+    AIRFLOW__CELERY__WORKER_CONCURRENCY: 64
+    AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE: 512
+    AIRFLOW__CORE__SQL_ALCHEMY_MAX_OVERFLOW: 1024
     AIRFLOW__CORE__SQL_ALCHEMY_CONN: "postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_USER}@postgres.${SUBDOMAIN}${DOMAIN}/airflow"
     AIRFLOW__CELERY__RESULT_BACKEND: "db+postgresql://${POSTGRES_USER}:${POSTGRES_USER}@postgres.${SUBDOMAIN}${DOMAIN}/airflow"
     AIRFLOW__CELERY__BROKER_URL: "redis://:${REDIS_PASSWORD}@redis.${SUBDOMAIN}${DOMAIN}:6379/0"

infra/airflow-cluster/docker-compose.yaml

@@ -54,10 +54,12 @@ x-airflow-common:
     AIRFLOW__CORE__PARALLELISM: 256
     AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 256
     AIRFLOW__CORE__NON_POOLED_TASK_SLOT_COUNT: 256
-    AIRFLOW__SCHEDULER__PARSING_PROCESSES: 12
-    AIRFLOW__CELERY__WORKER_CONCURRENCY: 12
-    AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE: 256
-    AIRFLOW__CORE__SQL_ALCHEMY_MAX_OVERFLOW: 512
+    AIRFLOW__SCHEDULER__PARSING_PROCESSES: 8
+    AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC: 1
+    AIRFLOW__SCHEDULER__MAX_DAGRUNS_PER_LOOP_TO_SCHEDULE: 128
+    AIRFLOW__CELERY__WORKER_CONCURRENCY: 8
+    AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE: 512
+    AIRFLOW__CORE__SQL_ALCHEMY_MAX_OVERFLOW: 1024
     AIRFLOW__CORE__SQL_ALCHEMY_CONN: "postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_USER}@postgres/airflow"
     AIRFLOW__CELERY__RESULT_BACKEND: "db+postgresql://${POSTGRES_USER}:${POSTGRES_USER}@postgres/airflow"
     AIRFLOW__CELERY__BROKER_URL: "redis://:${REDIS_PASSWORD}@redis:6379/0"