|
1 | | -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" |
2 | | -unzip awscliv2.zip |
3 | | -sudo ./aws/install |
| 1 | +#!/bin/bash |
| 2 | +#curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" |
| 3 | +#unzip awscliv2.zip |
| 4 | +#sudo ./aws/install |
| 5 | +AWS_SECRETS_MANAGER_NAME=ted_sws_2 |
| 6 | +SERVICES=(mongo digest-api airflow fuseki metabase mongo-express) |
| 7 | + |
| 8 | +EFS_VOLUMES=(metabase_postgres_db fuseki_data ted_sws logs dags airflow_postgres_db mongo_db) |
| 9 | +echo "Creating env file ..." |
| 10 | +aws secretsmanager create-secret --secret-string file://s.json --name $AWS_SECRETS_MANAGER_NAME |
| 11 | +aws secretsmanager get-secret-value --secret-id $AWS_SECRETS_MANAGER_NAME | jq -r '.SecretString' | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' > .env |
| 12 | +source .env |
| 13 | +export $(cat .env | xargs) |
| 14 | + |
| 15 | + |
| 16 | +create_volume(){ |
| 17 | + echo "Creating volume $1" |
| 18 | + aws efs create-file-system --performance-mode generalPurpose --encrypted --tags Key=Name,Value=$1 > $1-file-system.json |
| 19 | + FILE_SYSTEM_ID=`jq -r ".FileSystemId" $1-file-system.json` |
| 20 | + FILE_SYSTEM_ARN=`jq -r ".FileSystemArn" $1-file-system.json` |
| 21 | + EFS_VOLUME_NAME=$1 |
| 22 | + echo ${EFS_VOLUME_NAME^^}_VOLUME_ID=$FILE_SYSTEM_ID >> .env |
| 23 | + rm $1-file-system.json |
| 24 | + echo "Volume system id = $FILE_SYSTEM_ID" |
| 25 | + sleep 5 |
| 26 | + aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_1_ID |
| 27 | + aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_2_ID |
| 28 | + aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_3_ID |
| 29 | +} |
| 30 | + |
| 31 | + |
| 32 | +echo "Creating volumes ..." |
| 33 | +for EFS_VOLUME in "${EFS_VOLUMES[@]}" |
| 34 | +do |
| 35 | + create_volume $EFS_VOLUME |
| 36 | +done |
| 37 | + |
| 38 | +source .env |
| 39 | +export $(cat .env | xargs) |
| 40 | + |
| 41 | +create_cluster_config_and_profiles() { |
| 42 | + ecs-cli configure --cluster $1-cluster --default-launch-type EC2 --config-name $1-cluster --region eu-west-3 |
| 43 | + ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name $1-cluster-profile |
| 44 | +} |
| 45 | + |
| 46 | +for SERVICE in "${SERVICES[@]}" |
| 47 | +do |
| 48 | + create_cluster_config_and_profiles $SERVICE |
| 49 | +done |
| 50 | + |
| 51 | +create_cluster() { |
| 52 | + ecs-cli up --force --keypair amazon --capability-iam --size 1 --instance-type $1 --cluster-config $2 --ecs-profile $3 --vpc $VPC_ID --subnets $SUBNET_1_ID,$SUBNET_2_ID,$SUBNET_3_ID --security-group $SECURITY_GROUP |
| 53 | +} |
| 54 | + |
| 55 | +create_cluster $MONGO_INSTANCE_TYPE mongo-cluster mongo-cluster-profile |
| 56 | +create_cluster $DIGEST_API_INSTANCE_TYPE digest-api-cluster digest-api-cluster-profile |
| 57 | +create_cluster $AIRFLOW_INSTANCE_TYPE airflow-cluster airflow-cluster-profile |
| 58 | +create_cluster $FUSEKI_INSTANCE_TYPE fuseki-cluster fuseki-cluster-profile |
| 59 | +create_cluster $METABASE_INSTANCE_TYPE metabase-cluster metabase-cluster-profile |
| 60 | +create_cluster $MONGO_EXPRESS_INSTANCE_TYPE mongo-express-cluster mongo-express-cluster-profile |
| 61 | + |
| 62 | + |
| 63 | + |
| 64 | +start_service_with_service_discovery(){ |
| 65 | + ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region eu-west-3 --ecs-profile $1-cluster-profile --cluster-config $1-cluster --private-dns-namespace ted_sws --vpc $VPC_ID --dns-type A --enable-service-discovery --create-log-groups |
| 66 | +} |
| 67 | + |
| 68 | +start_service_without_service_discovery(){ |
| 69 | + ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region eu-west-3 --ecs-profile $1-cluster-profile --cluster-config $1-cluster --create-log-groups |
| 70 | +} |
| 71 | + |
| 72 | + |
| 73 | +SERVICES_WITH_DNS_CREATION=(mongo digest-api fuseki) |
| 74 | +SERVICES_WITHOUT_DNS=(airflow metabase mongo-express) |
| 75 | + |
| 76 | +for SERVICE in "${SERVICES_WITH_DNS_CREATION[@]}" |
| 77 | +do |
| 78 | + echo "Starting $SERVICE service" |
| 79 | + start_service_with_service_discovery $SERVICE |
| 80 | +done |
| 81 | + |
| 82 | +for SERVICE in "${SERVICES_WITHOUT_DNS[@]}" |
| 83 | +do |
| 84 | + echo "Starting $SERVICE service" |
| 85 | + start_service_without_service_discovery $SERVICE |
| 86 | +done |
4 | 87 |
|
5 | 88 |
|
6 | | -SUBNET_1_ID=subnet-0633b255adc732d3f |
7 | | -SUBNET_2_ID=subnet-0088e8035626e7ab5 |
8 | | -SUBNET_3_ID=subnet-04ad482119216e427 |
9 | | -VPC_ID=vpc-093d2f650f90a4dac |
10 | | -SECURITY_GROUP=sg-03850090ad599ebc6 |
11 | | -ACCESS_KEY={$ACCESS_KEY} |
12 | | -SECRET_KEY={$SECRET_KEY} |
13 | | - |
14 | | - |
15 | | -#aws ec2 describe-subnets --filters "Name=vpc-id,Values=vpc-093d2f650f90a4dac" |
16 | | -#aws ec2 describe-vpcs --filters Name=tag:Name,Values=Meaningfy |
17 | | - |
18 | | - |
19 | | -#login |
20 | | - |
21 | | - |
22 | | -#efs |
23 | | -aws efs create-file-system --performance-mode generalPurpose --tags Key=Name,Value=mongo-db > create-file-system.json |
24 | | -FILE_SYSTEM_ID=`jq -r ".FileSystemId" create-file-system.json` |
25 | | -FILE_SYSTEM_ARN=`jq -r ".FileSystemArn" create-file-system.json` |
26 | | - |
27 | | -#find how to get file-system id and subnets ids |
28 | | -aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_1_ID |
29 | | -aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_2_ID |
30 | | -aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_3_ID |
31 | | - |
32 | | - |
33 | | -#Cluster with ec2 instance and using ecs-cli |
34 | | - |
35 | | -ecs-cli configure --cluster mongo-cluster --default-launch-type EC2 --config-name mongo-cluster --region eu-west-3 |
36 | | -ecs-cli configure --cluster airflow-cluster --default-launch-type EC2 --config-name airflow-cluster --region eu-west-3 |
37 | | -ecs-cli configure --cluster fuseki-cluster --default-launch-type EC2 --config-name fuseki-cluster --region eu-west-3 |
38 | | -ecs-cli configure --cluster metabase-cluster --default-launch-type EC2 --config-name metabase-cluster --region eu-west-3 |
39 | | -ecs-cli configure --cluster digest-api-cluster --default-launch-type EC2 --config-name digest-api-cluster --region eu-west-3 |
40 | | - |
41 | | -ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name mongo-cluster-profile |
42 | | -ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name airflow-cluster-profile |
43 | | -ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name fuseki-cluster-profile |
44 | | -ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name metabase-cluster-profile |
45 | | - |
46 | | -ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name cli-cluster-profile |
47 | | - |
48 | | -ecs-cli up --force --keypair amazon --capability-iam --size 1 --instance-type t2.medium --cluster-config mongo-cluster --ecs-profile mongo-cluster-profile --vpc $VPC_ID --subnets $SUBNET_1_ID,$SUBNET_2_ID,$SUBNET_3_ID --security-group $SECURITY_GROUP |
49 | | -ecs-cli up --force --keypair amazon --capability-iam --size 1 --instance-type t2.medium --cluster-config fuseki-cluster --ecs-profile fuseki-cluster-profile --vpc $VPC_ID --subnets $SUBNET_1_ID,$SUBNET_2_ID,$SUBNET_3_ID --security-group $SECURITY_GROUP |
50 | | -ecs-cli up --force --keypair amazon --capability-iam --size 1 --instance-type t3.medium --cluster-config airflow-cluster --ecs-profile airflow-cluster-profile --vpc $VPC_ID --subnets $SUBNET_1_ID,$SUBNET_2_ID,$SUBNET_3_ID --security-group $SECURITY_GROUP |
51 | | -ecs-cli up --force --keypair amazon --capability-iam --size 1 --instance-type t2.medium --cluster-config metabase-cluster --ecs-profile metabase-cluster-profile --vpc $VPC_ID --subnets $SUBNET_1_ID,$SUBNET_2_ID,$SUBNET_3_ID --security-group $SECURITY_GROUP |
52 | | - |
53 | | -ecs-cli up --force --keypair amazon --capability-iam --size 1 --instance-type t2.micro --cluster-config digest-api-cluster --ecs-profile cli-cluster-profile --vpc $VPC_ID --subnets $SUBNET_1_ID,$SUBNET_2_ID,$SUBNET_3_ID --security-group $SECURITY_GROUP |
54 | | - |
55 | | - |
56 | | -ecs-cli compose --project-name mongo-service --file mongo.yml --ecs-params mongo-ecs-params.yml --debug service up --region eu-west-3 --ecs-profile mongo-cluster-profile --cluster-config mongo-cluster --create-log-groups |
57 | | - |
58 | | -ecs-cli compose --project-name airflow-service --file airflow.yml --ecs-params airflow-ecs-param.yml --debug service up --region eu-west-3 --ecs-profile airflow-cluster-profile --cluster-config airflow-cluster --create-log-groups |
59 | | - |
60 | | -ecs-cli compose --project-name fuseki-service --file fuseki.yml --ecs-params fuseki-ecs-params.yml --debug service up --region eu-west-3 --ecs-profile fuseki-cluster-profile --cluster-config fuseki-cluster --create-log-groups |
61 | | - |
62 | | -ecs-cli compose --project-name metabase-service --file fuseki.yml --ecs-params metabase-ecs-params.yml --debug service up --region eu-west-3 --ecs-profile metabase-cluster-profile --cluster-config metabase-cluster --create-log-groups |
63 | | - |
|
0 commit comments