wip

Author: CaptainOfHacks

Makefile

@@ -287,6 +287,7 @@ prod-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
 	@ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env
 	@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/prod >> .env
 	@ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env
+	@ echo AIRFLOW_CELERY_WORKER_CONCURRENCY=32 >> .env
 	@ vault kv get -format="json" ted-prod/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 	@ vault kv get -format="json" ted-prod/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 	@ vault kv get -format="json" ted-prod/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env

infra/airflow-cluster/docker-compose-worker.yaml

@@ -55,7 +55,7 @@ x-airflow-common:
     AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 256
     AIRFLOW__CORE__NON_POOLED_TASK_SLOT_COUNT: 256
     AIRFLOW__SCHEDULER__PARSING_PROCESSES: 24
-    AIRFLOW__CELERY__WORKER_CONCURRENCY: 64
+    AIRFLOW__CELERY__WORKER_CONCURRENCY: ${AIRFLOW_CELERY_WORKER_CONCURRENCY}
     AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE: 512
     AIRFLOW__CORE__SQL_ALCHEMY_MAX_OVERFLOW: 1024
     AIRFLOW__CORE__SQL_ALCHEMY_CONN: "postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_USER}@postgres.${SUBDOMAIN}${DOMAIN}/airflow"