WIP

Dragos0000 · Dragos0000 · commit 621f9fba57ed · 2022-08-21T15:38:13.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -121,3 +121,4 @@ tests/test_data/notice_transformer/mapping_suite_processor_repository/test_packa
 tests/test_data/notice_transformer/mapping_suite_processor_repository/test_package_fake/transformation/resources/languages.json
 tests/test_data/notice_transformer/mapping_suite_processor_repository/test_package_fake/transformation/resources/main_activity.json
 tests/test_data/notice_transformer/mapping_suite_processor_repository/test_package_fake/transformation/resources/nuts.json
+/infra/digest_api/digest_service/project_requirements.txt
diff --git a/Makefile b/Makefile
@@ -307,7 +307,10 @@ start-all-apis: start-digest_service-api
 
 stop-all-apis: stop-digest_service-api
 
-build-digest_service-api:
+create-env-digest-api:
+	@ cp requirements.txt ./infra/digest_api/digest_service/project_requirements.txt
+
+build-digest_service-api: create-env-digest-api
 	@ echo -e "$(BUILD_PRINT) Build digest_service API service $(END_BUILD_PRINT)"
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} build --no-cache --force-rm
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
diff --git a/infra/airflow/Dockerfile b/infra/airflow/Dockerfile
@@ -1,4 +1,4 @@
-FROM apache/airflow:2.2.5-python3.8
+FROM docker.io/apache/airflow:2.2.5-python3.8
 
 # quick sudo
 USER root
diff --git a/infra/airflow/docker-compose.yaml b/infra/airflow/docker-compose.yaml
@@ -104,7 +104,7 @@ services:
       - airflow
 
   redis:
-    image: redis:latest
+    image: redis:7.0.4-alpine3.16
     container_name: redis-airflow-${ENVIRONMENT}
     expose:
       - 6379
diff --git a/infra/api/digest_service/Dockerfile b/infra/api/digest_service/Dockerfile
diff --git a/infra/aws/airflow.yml b/infra/aws/airflow.yml
@@ -19,7 +19,7 @@ services:
       AIRFLOW_HOME: /opt/airflow
       PYTHONPATH: /opt/airflow/
       AIRFLOW_UID: ${AIRFLOW_UID}
-    image: public.ecr.aws/g5y4d1v2/meaningfy:airflow-meaningfy-225
+    image: ${AIRFLOW_IMAGE_URI}
     links:
     - postgres
     - redis
@@ -62,7 +62,7 @@ services:
       AIRFLOW_HOME: /opt/airflow
       PYTHONPATH: /opt/airflow/
       AIRFLOW_UID: ${AIRFLOW_UID}
-    image: public.ecr.aws/g5y4d1v2/meaningfy:airflow-meaningfy-225
+    image: ${AIRFLOW_IMAGE_URI}
     logging:
       driver: awslogs
       options:
@@ -86,7 +86,7 @@ services:
       timeout: 5s
       interval: 30s
       retries: 3
-    image: public.ecr.aws/g5y4d1v2/meaningfy:postgres
+    image: ${AIRFLOW_POSTGRES_IMAGE_URI}
     logging:
       driver: awslogs
       options:
@@ -103,7 +103,7 @@ services:
       interval: 5s
       timeout: 30s
       retries: 10
-    image: public.ecr.aws/g5y4d1v2/meaningfy:redis
+    image: ${REDIS_IMAGE_URI}
     labels:
       ecs-local.task-definition-input.type: remote
       ecs-local.task-definition-input.value: airflow-reload-task
@@ -137,7 +137,7 @@ services:
       interval: 10s
       timeout: 10s
       retries: 5
-    image: public.ecr.aws/g5y4d1v2/meaningfy:airflow-meaningfy-225
+    image: ${AIRFLOW_IMAGE_URI}
     links:
     - postgres
     - redis
@@ -170,7 +170,7 @@ services:
       PYTHONPATH: /opt/airflow/
       RML_MAPPER_PATH: /opt/airflow/.rmlmapper/rmlmapper.jar
       XML_PROCESSOR_PATH: /opt/airflow/.saxon/saxon-he-10.6.jar
-    image: public.ecr.aws/g5y4d1v2/meaningfy:airflow-meaningfy-225
+    image: ${AIRFLOW_IMAGE_URI}
     links:
     - postgres
     - redis
@@ -205,7 +205,7 @@ services:
       PYTHONPATH: /opt/airflow/
       RML_MAPPER_PATH: /opt/airflow/.rmlmapper/rmlmapper.jar
       XML_PROCESSOR_PATH: /opt/airflow/.saxon/saxon-he-10.6.jar
-    image: public.ecr.aws/g5y4d1v2/meaningfy:airflow-meaningfy-225
+    image: ${AIRFLOW_IMAGE_URI}
     links:
     - postgres
     - redis
@@ -244,7 +244,7 @@ services:
       PYTHONPATH: /opt/airflow/
       RML_MAPPER_PATH: /opt/airflow/.rmlmapper/rmlmapper.jar
       XML_PROCESSOR_PATH: /opt/airflow/.saxon/saxon-he-10.6.jar
-    image: public.ecr.aws/g5y4d1v2/meaningfy:airflow-meaningfy-225
+    image: ${AIRFLOW_IMAGE_URI}
     labels:
       ecs-local.task-definition-input.type: remote
       ecs-local.task-definition-input.value: airflow-reload-task
diff --git a/infra/aws/create-and-deploy-images.sh b/infra/aws/create-and-deploy-images.sh
@@ -1,17 +1,20 @@
-REGION=eu-west-3
-REPOSITORIES=(fuseki)
+REGION=eu-west-1
 MONGO_IMAGE=docker.io/mongo:5.0.10
 FUSEKI_IMAGE=docker.io/secoresearch/fuseki:4.5.0
 METABASE_IMAGE=docker.io/metabase/metabase:v0.43.4
-AIRFLOW_POSTGRES=docker.io/postgres:14.4-alpine
+AIRFLOW_POSTGRES_IMAGE=docker.io/postgres:14.4-alpine
+REDIS_IMAGE=docker.io/redis:7.0.4-alpine3.16
+MONGO_EXPRESS_IMAGE=docker.io/mongo-express:0.54.0
 
-IMAGES_TO_BE_BUILD=(airflow digest-api )
+
+IMAGES_TO_BE_BUILD=(airflow digest_api metabase_postgres)
+IMAGES_FROM_DOCKER_HUB=(fuseki airflow_postgres metabase mongo redis mongo_express)
 
 touch .repositories_ids
 
 create_repository(){
   echo "Creating repository for $1"
-  aws ecr create-repository --repository-name $1 --image-scanning-configuration scanOnPush=true --region $2 > repo.json
+  aws ecr create-repository --repository-name $1 --region $2 > repo.json
   REPO_NAME=$1
   REPO_ID=`jq -r ".repository.repositoryUri" repo.json`
   echo ${REPO_NAME^^}_REPO_ID=$REPO_ID >> .repositories_ids
@@ -21,35 +24,74 @@ create_repository(){
 
 
 build_image_and_push_to_repo(){
+  echo "Building image for $1"
   REPO=$1
   REPO_NAME=${REPO^^}
   REPO_ID=${REPO_NAME}_REPO_ID
   echo ${!REPO_ID}
-  aws ecr get-login-password --region eu-west-3 | podman login --username AWS --password-stdin ${!REPO_ID}
-  podman build -t $1 ../$1/ --label=$1-image
+  aws ecr get-login-password --region $REGION | podman login --username AWS --password-stdin ${!REPO_ID}
+  if [ "$REPO" = "metabase_postgres" ]; then
+  REPO=metabase
+  fi
+  podman build -t $REPO ../$REPO/ --label=$1-image
   IMAGE=`podman images --filter label=$1-image --format {{.Repository}}:{{.Tag}}`
+  echo "image found is $IMAGE"
+  TAG=$(echo $IMAGE| cut -d':' -f 2)
+  echo "TAG found is $TAG"
+  if [ ! -z $TAG ]
+  then
+    podman tag $IMAGE ${!REPO_ID}:$TAG
+    podman push ${!REPO_ID}:$TAG
+  fi
+
   podman tag $IMAGE ${!REPO_ID}:latest
   podman push ${!REPO_ID}:latest
-}
 
+  echo ${REPO_NAME}_IMAGE_URI=${!REPO_ID}:latest >> .env
+  echo "Image for $1 has been pushed in AWS"
+  podman system prune --all --force && podman rmi --all
+}
 
+get_built_image_and_push_to_repo(){
+  echo "Getting image for $1"
+  REPO=$1
+  REPO_NAME=${REPO^^}
+  REPO_ID=${REPO_NAME}_REPO_ID
+  IMAGE_TO_PULL=${REPO_NAME}_IMAGE
+  echo "REPO_ID=${!REPO_ID}"
+  echo ${!IMAGE_TO_PULL}
+  aws ecr get-login-password --region $REGION | podman login --username AWS --password-stdin ${!REPO_ID}
+  podman pull ${!IMAGE_TO_PULL}
+  echo "string to compare is $REPO"
+  IMAGE_NAME_TO_FIND=$REPO
+  if [ "$REPO" = "airflow_postgres" ]; then
+    IMAGE_NAME_TO_FIND=postgres
+  fi
+  echo " Image to find is $IMAGE_NAME_TO_FIND"
+  IMAGE=`podman images --format {{.Repository}}:{{.Tag}} | grep $IMAGE_NAME_TO_FIND`
+  echo "image found is $IMAGE"
+  TAG=$(echo $IMAGE| cut -d':' -f 2)
+  podman tag $IMAGE ${!REPO_ID}:latest
+  podman tag $IMAGE ${!REPO_ID}:$TAG
+  podman push ${!REPO_ID}:latest
+  podman push ${!REPO_ID}:$TAG
+  echo ${REPO_NAME}_IMAGE_URI=${!REPO_ID}:latest >> .env
+  echo "Image for $1 has been pushed in AWS"
+  podman system prune --all --force && podman rmi --all
+}
 
-for REPO in "${REPOSITORIES[@]}"
+for IMAGE in "${IMAGES_FROM_DOCKER_HUB[@]}"
 do
-#  create_repository $REPO $REGION
+  create_repository $IMAGE $REGION
   source .repositories_ids
   export $(cat .repositories_ids | xargs)
-  build_image_and_push_to_repo $REPO
+  get_built_image_and_push_to_repo $IMAGE
 done
 
-
-#podman images --filter label=label --format {{.Repository}}
-#podman images --format {{.Repository}} | grep fuseki
-#
-#
-#podman build -t fuseki ../fuseki/ --label=label
-#
-#docker push 013659641721.dkr.ecr.eu-west-3.amazonaws.com/fuseki:latest
-
-
-#aws ecr get-login-password --region eu-west-3 | podman login --username AWS --password-stdin 013659641721.dkr.ecr.eu-west-3.amazonaws.com
+for IMAGE in "${IMAGES_TO_BE_BUILD[@]}"
+do
+  create_repository $IMAGE $REGION
+  source .repositories_ids
+  export $(cat .repositories_ids | xargs)
+  build_image_and_push_to_repo $IMAGE
+done
diff --git a/infra/aws/digest-api.yml b/infra/aws/digest-api.yml
@@ -1,7 +1,7 @@
 version: "3"
 services:
   digest-api:
-    image: public.ecr.aws/g5y4d1v2/meaningfy:meaningfy-digest-api
+    image: ${DIGEST_API_IMAGE_URI}
     logging:
       driver: awslogs
       options:
diff --git a/infra/aws/metabase.yml b/infra/aws/metabase.yml
@@ -9,7 +9,7 @@ services:
       ENV_MB_DB_PASS: ${ENV_MB_DB_PASS}
       ENV_MB_DB_HOST: postgres-metabase
       ENV_MB_ENCRYPTION_SECRET_KEY: ${ENV_MB_ENCRYPTION_SECRET_KEY}
-    image: public.ecr.aws/g5y4d1v2/meaningfy:metabase
+    image: ${METABASE_IMAGE_URI}
     logging:
       driver: awslogs
       options:
@@ -26,7 +26,7 @@ services:
     environment:
       POSTGRES_PASSWORD: postgres
       METABASE_PASSWORD: metabase
-    image: public.ecr.aws/g5y4d1v2/meaningfy:metabase-postgres
+    image: ${METABASE_POSTGRES_IMAGE_URI}
     volumes:
       - metabase-postgres-db:/var/lib/postgresql/data
     logging:
diff --git a/infra/aws/mongo-express.yml b/infra/aws/mongo-express.yml
@@ -8,7 +8,7 @@ services:
       ME_CONFIG_MONGODB_ADMINUSERNAME: ${ME_CONFIG_MONGODB_ADMINUSERNAME}
       ME_CONFIG_MONGODB_ENABLE_ADMIN: "true"
       ME_CONFIG_MONGODB_SERVER: ${ME_CONFIG_MONGODB_SERVER}
-    image: public.ecr.aws/g5y4d1v2/meaningfy:mongo-express
+    image: ${MONGO_EXPRESS_IMAGE_URI}
     logging:
       driver: awslogs
       options:
diff --git a/infra/aws/mongo.yml b/infra/aws/mongo.yml
@@ -4,7 +4,7 @@ services:
     environment:
       MONGO_INITDB_ROOT_PASSWORD: ${MONGO_INITDB_ROOT_PASSWORD}
       MONGO_INITDB_ROOT_USERNAME: ${MONGO_INITDB_ROOT_USERNAME}
-    image: public.ecr.aws/g5y4d1v2/meaningfy:mongo
+    image: ${MONGO_IMAGE_URI}
     logging:
       driver: awslogs
       options:
diff --git a/infra/aws/start-infra.sh b/infra/aws/start-infra.sh
@@ -2,13 +2,14 @@
 #curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
 #unzip awscliv2.zip
 #sudo ./aws/install
+REGION=eu-west-1
 AWS_SECRETS_MANAGER_NAME=ted_sws_2
 SERVICES=(mongo digest-api airflow fuseki metabase mongo-express)
 
 EFS_VOLUMES=(metabase_postgres_db fuseki_data ted_sws logs dags airflow_postgres_db mongo_db)
-echo "Creating env file ..."
-aws secretsmanager create-secret --secret-string file://s.json --name $AWS_SECRETS_MANAGER_NAME
-aws secretsmanager get-secret-value --secret-id $AWS_SECRETS_MANAGER_NAME | jq -r '.SecretString' | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' > .env
+#echo "Creating env file ..."
+#aws secretsmanager create-secret --secret-string file://s.json --name $AWS_SECRETS_MANAGER_NAME
+#aws secretsmanager get-secret-value --secret-id $AWS_SECRETS_MANAGER_NAME | jq -r '.SecretString' | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' > .env
 source .env
 export $(cat .env | xargs)
 
@@ -29,17 +30,17 @@ create_volume(){
 }
 
 
-echo "Creating volumes ..."
-for EFS_VOLUME in "${EFS_VOLUMES[@]}"
-do
-  create_volume $EFS_VOLUME
-done
+#echo "Creating volumes ..."
+#for EFS_VOLUME in "${EFS_VOLUMES[@]}"
+#do
+#  create_volume $EFS_VOLUME
+#done
 
 source .env
 export $(cat .env | xargs)
 
 create_cluster_config_and_profiles() {
-  ecs-cli configure --cluster $1-cluster --default-launch-type EC2 --config-name $1-cluster --region eu-west-3
+  ecs-cli configure --cluster $1-cluster --default-launch-type EC2 --config-name $1-cluster --region $REGION
   ecs-cli configure profile --access-key $ACCESS_KEY --secret-key $SECRET_KEY --profile-name $1-cluster-profile
 }
 
@@ -62,11 +63,11 @@ create_cluster $MONGO_EXPRESS_INSTANCE_TYPE mongo-express-cluster mongo-express-
 
 
 start_service_with_service_discovery(){
-  ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region eu-west-3 --ecs-profile $1-cluster-profile --cluster-config $1-cluster --private-dns-namespace ted_sws --vpc $VPC_ID --dns-type A --enable-service-discovery --create-log-groups
+  ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster --private-dns-namespace ted_sws --vpc $VPC_ID --dns-type A --enable-service-discovery --create-log-groups
 }
 
 start_service_without_service_discovery(){
-  ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region eu-west-3 --ecs-profile $1-cluster-profile --cluster-config $1-cluster --create-log-groups
+  ecs-cli compose --project-name $1-service --file $1.yml --ecs-params $1-ecs-params.yml --debug service up --region $REGION --ecs-profile $1-cluster-profile --cluster-config $1-cluster --create-log-groups
 }
 
 
diff --git a/infra/digest_api/Dockerfile b/infra/digest_api/Dockerfile
@@ -0,0 +1,14 @@
+FROM python:3.8-slim-buster
+EXPOSE 8000
+
+RUN bash -c 'echo $ID_MANAGER_API_PORT'
+
+WORKDIR /usr/src/api/id_manager
+COPY . /usr/src/api/id_manager
+COPY digest_service /usr/src/api/id_manager
+
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+RUN pip install -r project_requirements.txt
+
+ENTRYPOINT uvicorn --host 0.0.0.0 --port 8000 ted_sws.notice_transformer.entrypoints.api.digest_service.main:app
diff --git a/infra/digest_api/digest_service/requirements.txt b/infra/digest_api/digest_service/requirements.txt
diff --git a/infra/digest_api/docker-compose.yml b/infra/digest_api/docker-compose.yml
@@ -5,7 +5,7 @@ services:
     container_name: api-digest_service
     build:
       context: ../..
-      dockerfile: ./infra/api/digest_service/Dockerfile
+      dockerfile: ./infra/api/Dockerfile
     env_file: ../../.env
     networks:
       - proxy-net

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM apache/airflow:2.2.5-python3.8`
	`1`	`+FROM docker.io/apache/airflow:2.2.5-python3.8`
`2`	`2`
`3`	`3`	`# quick sudo`
`4`	`4`	`USER root`