Skip to content

Commit a34a5fb

Browse files
schivmeisterschivmeister
authored
Merge pull request #42 from meaningfy-ws/feature/prepare-SRV-for-FAT-testing
feat(infra): add testing/SRV environment to unified ted-sws-stack
2 parents de984c8 + 3f2f5a5 commit a34a5fb

4 files changed

Lines changed: 237 additions & 10 deletions

File tree

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,7 @@ src/infra/airflow/src
129129
src/infra/airflow/libraries
130130
!src/infra/ted-sws-stack/.env.local
131131
!src/infra/ted-sws-stack/.env.common
132+
133+
# Airflow runtime directories (created manually on servers)
134+
/logs/
135+
/plugins/

‎Makefile‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,22 @@ start-local-stack-nodata: init-libraries
292292
@echo "$(BUILD_PRINT)Starting TED-SWS local stack (no persistent data) $(END_BUILD_PRINT)"
293293
@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.local.yml -f $(STACK_PATH)/docker-compose.local-nodata.yml --env-file $(STACK_PATH)/.env.local up -d $(SERVICES)
294294

295+
start-staging-stack:
296+
@echo "$(BUILD_PRINT)Starting TED-SWS staging stack $(END_BUILD_PRINT)"
297+
@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.staging.yml --env-file $(STACK_PATH)/.env.staging up -d $(SERVICES)
298+
299+
stop-staging-stack:
300+
@echo "$(BUILD_PRINT)Stopping TED-SWS staging stack $(END_BUILD_PRINT)"
301+
@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.staging.yml --env-file $(STACK_PATH)/.env.staging down
302+
303+
start-testing-stack:
304+
@echo "$(BUILD_PRINT)Starting TED-SWS testing stack (SRV) $(END_BUILD_PRINT)"
305+
@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.testing.yml --env-file $(STACK_PATH)/.env.testing up -d $(SERVICES)
306+
307+
stop-testing-stack:
308+
@echo "$(BUILD_PRINT)Stopping TED-SWS testing stack (SRV) $(END_BUILD_PRINT)"
309+
@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.testing.yml --env-file $(STACK_PATH)/.env.testing down
310+
295311
#-----------------------------------------------------------------------------
296312
# VAULT SERVICES
297313
#-----------------------------------------------------------------------------
@@ -330,6 +346,26 @@ staging-unified-dotenv: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
330346
echo "AIRFLOW__CELERY__WORKER_CONCURRENCY=16"; \
331347
} > $(STACK_PATH)/.env.staging
332348

349+
# Get secrets in dotenv format (unified stack - testing/SRV environment)
350+
testing-unified-dotenv: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
351+
@ echo -e "$(BUILD_PRINT)Creating unified stack .env.testing from Vault $(END_BUILD_PRINT)"
352+
@ VAULT_JSON=$$(vault kv get -format="json" ted-staging/ted-sws-deployment-secrets) && \
353+
MONGO_PW=$$(echo "$$VAULT_JSON" | jq -r '.data.data.MONGO_ROOT_PASSWORD') && \
354+
MINIO_PW=$$(echo "$$VAULT_JSON" | jq -r '.data.data.MINIO_ROOT_PASSWORD') && \
355+
{ \
356+
echo "$$VAULT_JSON" | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\""; \
357+
echo "MONGO_DB_AUTH_URL=mongodb://admin:$$MONGO_PW@mongodb:27017/"; \
358+
echo "S3_PUBLISH_PASSWORD=$$MINIO_PW"; \
359+
echo "ENVIRONMENT=testing"; \
360+
echo "SUBDOMAIN=tedsws-testing."; \
361+
echo "DOMAIN=meaningfy.ws"; \
362+
echo "AIRFLOW_INFRA_FOLDER=/home/lps/work/ted-rdf-conversion-pipeline"; \
363+
echo "AIRFLOW__CORE__PARALLELISM=32"; \
364+
echo "AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG=16"; \
365+
echo "AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=16"; \
366+
echo "AIRFLOW__CELERY__WORKER_CONCURRENCY=16"; \
367+
} > $(STACK_PATH)/.env.testing
368+
333369
# Get secrets in dotenv format (old - pulls everything from multiple Vault paths)
334370
staging-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
335371
@ echo -e "$(BUILD_PRINT)Creating .env.staging file $(END_BUILD_PRINT)"

‎src/infra/ted-sws-stack/README.md‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,17 +34,19 @@ labels.
3434
## Quick Start
3535

3636
```bash
37-
# Start all services
37+
# Local development
3838
make start-local-stack
39-
40-
# Start specific services only
41-
make start-local-stack SERVICES="mongodb fuseki minio"
42-
43-
# Stop the stack
39+
make start-local-stack SERVICES="mongodb fuseki minio" # specific services only
4440
make stop-local-stack
41+
make cleanup-local-stack # remove all data, volumes, images, build cache
4542

46-
# Stop and remove all data, volumes, images, and build cache
47-
make cleanup-local-stack
43+
# Staging (Hetzner VM) -- requires Vault access
44+
make staging-unified-dotenv # generate .env.staging from Vault (first time / secret rotation)
45+
make start-staging-stack
46+
47+
# Testing (SRV) -- requires Vault access
48+
make testing-unified-dotenv # generate .env.testing from Vault (first time / secret rotation)
49+
make start-testing-stack
4850

4951
# View logs
5052
docker compose -p ted-sws-stack logs -f
@@ -57,10 +59,12 @@ docker compose -p ted-sws-stack logs -f
5759
| `docker-compose.yml` | Base config -- service structure, `${VAR}` for passwords |
5860
| `docker-compose.local.yml` | Local overrides -- build, volumes, ports, Traefik labels |
5961
| `docker-compose.staging.yml` | Staging overrides -- host paths, TLS, labels |
62+
| `docker-compose.testing.yml` | Testing/SRV overrides -- host paths under `/home/lps/` |
6063
| `docker-compose.local-nodata.yml` | Ephemeral mode -- no persistent volumes |
6164
| `.env.common` | Shared app configs -- hostnames, ports, tool paths (committed) |
6265
| `.env.local` | Local passwords and connection strings (committed, safe defaults) |
6366
| `.env.staging` | Staging passwords from Vault (generated by `make staging-unified-dotenv`) |
67+
| `.env.testing` | Testing/SRV passwords from Vault (generated by `make testing-unified-dotenv`) |
6468
| `airflow/Dockerfile` | Airflow image with celery provider |
6569
| `airflow/requirements.txt` | Python dependencies for Airflow |
6670

@@ -77,16 +81,24 @@ env_file:
7781
- .env.local # local passwords + connection strings
7882
```
7983
80-
**Staging:**
84+
**Staging (Hetzner VM):**
8185
8286
```yaml
8387
env_file:
8488
- .env.common # same shared configs
8589
- .env.staging # Vault passwords + staging-specific overrides
8690
```
8791
92+
**Testing (SRV):**
93+
94+
```yaml
95+
env_file:
96+
- .env.common # same shared configs
97+
- .env.testing # Vault passwords + SRV-specific overrides (higher concurrency)
98+
```
99+
88100
The `.env.common` file is shared across all environments. Only passwords and
89-
environment-specific values differ between `.env.local` and `.env.staging`.
101+
environment-specific values differ between `.env.local`, `.env.staging`, and `.env.testing`.
90102

91103
No Vault access is needed for local development.
92104

‎src/infra/ted-sws-stack/docker-compose.testing.yml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# =============================================================================
2+
# TED-SWS Stack - Testing Environment (SRV)
3+
# Usage: docker compose -f docker-compose.yml -f docker-compose.testing.yml --env-file .env.testing up -d
4+
# =============================================================================
5+
6+
x-airflow-build: &airflow-build
7+
context: ./airflow
8+
dockerfile: Dockerfile
9+
10+
x-testing-volumes: &testing-volumes
11+
- /home/lps/work/ted-rdf-conversion-pipeline/src:/opt/airflow/src
12+
- /home/lps/work/ted-rdf-conversion-pipeline/test:/opt/airflow/test
13+
- /home/lps/work/ted-rdf-conversion-pipeline/logs:/opt/airflow/logs
14+
- /home/lps/work/ted-rdf-conversion-pipeline/plugins:/opt/airflow/plugins
15+
- /home/lps/work/ted-rdf-conversion-pipeline/libraries/.rmlmapper:/home/airflow/.rmlmapper
16+
- /home/lps/work/ted-rdf-conversion-pipeline/libraries/.saxon:/home/airflow/.saxon
17+
- /home/lps/work/ted-rdf-conversion-pipeline/libraries/.limes:/home/airflow/.limes
18+
19+
services:
20+
traefik:
21+
profiles:
22+
- disabled
23+
entrypoint: ["true"]
24+
25+
minio:
26+
volumes:
27+
- minio-data:/data
28+
labels:
29+
- "traefik.enable=true"
30+
- "traefik.docker.network=proxy-net"
31+
- "traefik.http.routers.testing-minio.entrypoints=web"
32+
- "traefik.http.routers.testing-minio.rule=Host(`minio.${SUBDOMAIN}${DOMAIN}`)"
33+
- "traefik.http.routers.testing-minio.middlewares=redirect@file"
34+
- "traefik.http.routers.testing-minio-secured.entrypoints=web-secured"
35+
- "traefik.http.routers.testing-minio-secured.rule=Host(`minio.${SUBDOMAIN}${DOMAIN}`)"
36+
- "traefik.http.routers.testing-minio-secured.tls.certresolver=mytlschallenge"
37+
- "traefik.http.services.testing-minio-secured.loadbalancer.server.port=9001"
38+
39+
mongodb:
40+
volumes:
41+
- mongodb-data:/data/db
42+
43+
mongo-express:
44+
labels:
45+
- "traefik.enable=true"
46+
- "traefik.docker.network=proxy-net"
47+
- "traefik.http.routers.testing-mongo-express.entrypoints=web"
48+
- "traefik.http.routers.testing-mongo-express.rule=Host(`mongo.${SUBDOMAIN}${DOMAIN}`)"
49+
- "traefik.http.routers.testing-mongo-express.middlewares=redirect@file"
50+
- "traefik.http.routers.testing-mongo-express-secured.entrypoints=web-secured"
51+
- "traefik.http.routers.testing-mongo-express-secured.rule=Host(`mongo.${SUBDOMAIN}${DOMAIN}`)"
52+
- "traefik.http.routers.testing-mongo-express-secured.tls.certresolver=mytlschallenge"
53+
54+
fuseki:
55+
volumes:
56+
- fuseki-data:/fuseki-base/databases
57+
- fuseki-config:/fuseki-base/configuration
58+
labels:
59+
- "traefik.enable=true"
60+
- "traefik.docker.network=proxy-net"
61+
- "traefik.http.routers.testing-fuseki.entrypoints=web"
62+
- "traefik.http.routers.testing-fuseki.rule=Host(`fuseki.${SUBDOMAIN}${DOMAIN}`)"
63+
- "traefik.http.routers.testing-fuseki.middlewares=redirect@file"
64+
- "traefik.http.routers.testing-fuseki-secured.entrypoints=web-secured"
65+
- "traefik.http.routers.testing-fuseki-secured.rule=Host(`fuseki.${SUBDOMAIN}${DOMAIN}`)"
66+
- "traefik.http.routers.testing-fuseki-secured.tls.certresolver=mytlschallenge"
67+
- "traefik.http.services.testing-fuseki-secured.loadbalancer.server.port=3030"
68+
69+
metabase-postgres:
70+
image: postgres:14-alpine
71+
container_name: metabase-postgres
72+
restart: unless-stopped
73+
environment:
74+
POSTGRES_DB: metabase
75+
POSTGRES_USER: metabase
76+
POSTGRES_PASSWORD: ${METABASE_POSTGRES_PASSWORD}
77+
healthcheck:
78+
test: ["CMD", "pg_isready", "-U", "metabase"]
79+
interval: 5s
80+
retries: 5
81+
volumes:
82+
- metabase-postgres-data:/var/lib/postgresql/data
83+
networks:
84+
- internal
85+
86+
metabase:
87+
environment:
88+
MB_DB_TYPE: postgres
89+
MB_DB_DBNAME: metabase
90+
MB_DB_PORT: 5432
91+
MB_DB_USER: metabase
92+
MB_DB_PASS: ${METABASE_POSTGRES_PASSWORD}
93+
MB_DB_HOST: metabase-postgres
94+
depends_on:
95+
metabase-postgres:
96+
condition: service_healthy
97+
labels:
98+
- "traefik.enable=true"
99+
- "traefik.docker.network=proxy-net"
100+
- "traefik.http.routers.testing-metabase.entrypoints=web"
101+
- "traefik.http.routers.testing-metabase.rule=Host(`metabase.${SUBDOMAIN}${DOMAIN}`)"
102+
- "traefik.http.routers.testing-metabase.middlewares=redirect@file"
103+
- "traefik.http.routers.testing-metabase-secured.entrypoints=web-secured"
104+
- "traefik.http.routers.testing-metabase-secured.rule=Host(`metabase.${SUBDOMAIN}${DOMAIN}`)"
105+
- "traefik.http.routers.testing-metabase-secured.tls.certresolver=mytlschallenge"
106+
- "traefik.http.services.testing-metabase-secured.loadbalancer.server.port=3000"
107+
108+
sftp:
109+
ports:
110+
- "2235:22"
111+
volumes:
112+
- sftp-data:/home
113+
114+
airflow-postgres:
115+
volumes:
116+
- airflow-postgres-data:/var/lib/postgresql/data
117+
118+
airflow-init:
119+
build: *airflow-build
120+
volumes: *testing-volumes
121+
env_file:
122+
- .env.common
123+
- .env.testing
124+
125+
airflow-webserver:
126+
build: *airflow-build
127+
volumes: *testing-volumes
128+
env_file:
129+
- .env.common
130+
- .env.testing
131+
labels:
132+
- "traefik.enable=true"
133+
- "traefik.docker.network=proxy-net"
134+
- "traefik.http.routers.testing-airflow.entrypoints=web"
135+
- "traefik.http.routers.testing-airflow.rule=Host(`airflow.${SUBDOMAIN}${DOMAIN}`)"
136+
- "traefik.http.routers.testing-airflow.middlewares=redirect@file"
137+
- "traefik.http.routers.testing-airflow-secured.entrypoints=web-secured"
138+
- "traefik.http.routers.testing-airflow-secured.rule=Host(`airflow.${SUBDOMAIN}${DOMAIN}`)"
139+
- "traefik.http.routers.testing-airflow-secured.tls.certresolver=mytlschallenge"
140+
- "traefik.http.services.testing-airflow-secured.loadbalancer.server.port=8080"
141+
142+
airflow-scheduler:
143+
build: *airflow-build
144+
volumes: *testing-volumes
145+
env_file:
146+
- .env.common
147+
- .env.testing
148+
149+
airflow-worker:
150+
build: *airflow-build
151+
volumes: *testing-volumes
152+
env_file:
153+
- .env.common
154+
- .env.testing
155+
156+
airflow-triggerer:
157+
build: *airflow-build
158+
volumes: *testing-volumes
159+
env_file:
160+
- .env.common
161+
- .env.testing
162+
163+
volumes:
164+
minio-data:
165+
mongodb-data:
166+
fuseki-data:
167+
fuseki-config:
168+
sftp-data:
169+
airflow-postgres-data:
170+
metabase-postgres-data:
171+
172+
networks:
173+
proxy-net:
174+
name: proxy-net
175+
external: true

0 commit comments

Comments
 (0)