Skip to content

Commit 1ad3b7f

Browse files
Dragos0000Dragos0000
committed
aws scripts
1 parent 82cf577 commit 1ad3b7f

11 files changed

Lines changed: 120 additions & 98 deletions

‎infra/aws/create-clusters.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
source .env
2+
export $(cat .env | xargs)
3+
CLUSTERS=(mongo digest-api airflow fuseki metabase mongo-express)
4+
5+
create_cluster_config_and_profiles() {
6+
ecs-cli configure --cluster $1-cluster --default-launch-type EC2 --config-name $1-cluster --region $REGION
7+
ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name $1-cluster-profile
8+
}
9+
10+
for CLUSTER in "${CLUSTERS[@]}"
11+
do
12+
create_cluster_config_and_profiles $CLUSTER
13+
done
14+
15+
create_cluster() {
16+
echo "VPC ID is $VPC_ID"
17+
echo "Creating cluster for $1"
18+
ecs-cli up --force --instance-role ecsInstanceRole --extra-user-data botocore.sh --keypair tedSWS --size 1 --instance-type $1 --cluster-config $2 --ecs-profile $3 --vpc $VPC_ID --subnets $SUBNET_1_ID,$SUBNET_2_ID,$SUBNET_3_ID --security-group $SECURITY_GROUP
19+
}
20+
21+
create_cluster $MONGO_INSTANCE_TYPE mongo-cluster mongo-cluster-profile
22+
create_cluster $DIGEST_API_INSTANCE_TYPE digest-api-cluster digest-api-cluster-profile
23+
create_cluster $AIRFLOW_INSTANCE_TYPE airflow-cluster airflow-cluster-profile
24+
create_cluster $FUSEKI_INSTANCE_TYPE fuseki-cluster fuseki-cluster-profile
25+
create_cluster $METABASE_INSTANCE_TYPE metabase-cluster metabase-cluster-profile
26+
create_cluster $MONGO_EXPRESS_INSTANCE_TYPE mongo-express-cluster mongo-express-cluster-profile

‎infra/aws/create-efs.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
EFS_VOLUMES=(metabase_postgres_db fuseki_data ted_sws logs dags airflow_postgres_db mongo_db)
2+
source .env
3+
export $(cat .env | xargs)
4+
5+
6+
create_volume(){
7+
echo "Creating volume $1"
8+
aws efs create-file-system --performance-mode generalPurpose --encrypted --tags Key=Name,Value=$1 > $1-file-system.json
9+
FILE_SYSTEM_ID=`jq -r ".FileSystemId" $1-file-system.json`
10+
FILE_SYSTEM_ARN=`jq -r ".FileSystemArn" $1-file-system.json`
11+
EFS_VOLUME_NAME=$1
12+
echo ${EFS_VOLUME_NAME^^}_VOLUME_ID=$FILE_SYSTEM_ID >> .env
13+
rm $1-file-system.json
14+
echo "Volume system id = $FILE_SYSTEM_ID"
15+
sleep 5
16+
aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_1_ID
17+
aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_2_ID
18+
aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_3_ID
19+
echo "Volume $1 successfully created "
20+
}
21+
22+
23+
echo "Creating volumes ..."
24+
for EFS_VOLUME in "${EFS_VOLUMES[@]}"
25+
do
26+
create_volume $EFS_VOLUME
27+
done

‎infra/aws/create-services.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
source .env
2+
export $(cat .env | xargs)
3+
4+
SERVICES_WITHOUT_DNS=(airflow metabase mongo-express)
5+
SERVICES_WITH_DNS_CREATION=(mongo digest-api fuseki)
6+
7+
create_service_with_service_discovery(){
8+
ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service create --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster --private-dns-namespace ted_sws --vpc $VPC_ID --dns-type A --enable-service-discovery
9+
}
10+
11+
create_service_without_service_discovery(){
12+
ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service create --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster
13+
}
14+
15+
16+
for SERVICE in "${SERVICES_WITH_DNS_CREATION[@]}"
17+
do
18+
echo "Starting $SERVICE service"
19+
create_service_with_service_discovery $SERVICE
20+
done
21+
22+
for SERVICE in "${SERVICES_WITHOUT_DNS[@]}"
23+
do
24+
echo "Starting $SERVICE service"
25+
create_service_without_service_discovery $SERVICE
26+
done

‎infra/aws/digest-api.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ services:
66
driver: awslogs
77
options:
88
awslogs-group: /ecs/digest-api-cli-task
9-
awslogs-region: eu-west-3
9+
awslogs-region: ${REGION}
1010
awslogs-stream-prefix: ecs
1111
networks:
1212
ecs-local-network: null

‎infra/aws/fuseki.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ services:
99
driver: awslogs
1010
options:
1111
awslogs-group: /ecs/fuseki-cli-task
12-
awslogs-region: eu-west-3
12+
awslogs-region: ${REGION}
1313
awslogs-stream-prefix: ecs
1414
networks:
1515
ecs-local-network: null

‎infra/aws/metabase.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ services:
1414
driver: awslogs
1515
options:
1616
awslogs-group: /ecs/metabase-cli-task
17-
awslogs-region: eu-west-3
17+
awslogs-region: ${REGION}
1818
awslogs-stream-prefix: ecs
1919
networks:
2020
ecs-local-network: null
@@ -33,7 +33,7 @@ services:
3333
driver: awslogs
3434
options:
3535
awslogs-group: /ecs/metabase-cli-task
36-
awslogs-region: eu-west-3
36+
awslogs-region: ${REGION}
3737
awslogs-stream-prefix: ecs
3838
networks:
3939
ecs-local-network: null

‎infra/aws/mongo-express.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ services:
1313
driver: awslogs
1414
options:
1515
awslogs-group: /ecs/mongo-express-cli-task
16-
awslogs-region: eu-west-3
16+
awslogs-region: ${REGION}
1717
awslogs-stream-prefix: ecs
1818
networks:
1919
ecs-local-network: null

‎infra/aws/mongo.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ services:
99
driver: awslogs
1010
options:
1111
awslogs-group: /ecs/mongo-cli-task
12-
awslogs-region: eu-west-3
12+
awslogs-region: ${REGION}
1313
awslogs-stream-prefix: ecs
1414
networks:
1515
ecs-local-network: null

‎infra/aws/start-infra.sh‎

Lines changed: 0 additions & 89 deletions
This file was deleted.

‎infra/aws/start-services.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
source .env
3+
export $(cat .env | xargs)
4+
SERVICES_WITH_DNS_CREATION=(mongo digest-api fuseki)
5+
SERVICES_WITHOUT_DNS=(airflow metabase mongo-express)
6+
7+
start_service_with_service_discovery(){
8+
# ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster --private-dns-namespace ted_sws --vpc $VPC_ID --dns-type A --enable-service-discovery --create-log-groups
9+
ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster --private-dns-namespace ted_sws --vpc $VPC_ID --dns-type A --enable-service-discovery
10+
}
11+
12+
start_service_without_service_discovery(){
13+
# ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster --create-log-groups
14+
ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster
15+
}
16+
17+
18+
for SERVICE in "${SERVICES_WITH_DNS_CREATION[@]}"
19+
do
20+
echo "Starting $SERVICE service"
21+
start_service_with_service_discovery $SERVICE
22+
done
23+
24+
for SERVICE in "${SERVICES_WITHOUT_DNS[@]}"
25+
do
26+
echo "Starting $SERVICE service"
27+
start_service_without_service_discovery $SERVICE
28+
done

0 commit comments

Comments
 (0)