Merge branch 'main' into feature/TED-545

Author: CaptainOfHacks

.github/workflows/unit-tests.yml

@@ -23,6 +23,7 @@ jobs:
       run: |
         python -m pip install --upgrade setuptools pip wheel
         make install
+        make install-dev
     - name: Make envfile
       uses: SpicyPizza/create-envfile@v1
       with:

Makefile

@@ -37,7 +37,7 @@ test-unit:
 
 test-features:
 	@ echo -e "$(BUILD_PRINT)Gherkin Features Testing ...$(END_BUILD_PRINT)"
-	@ tox -e features
+# 	@ tox -e features
 
 test-e2e:
 	@ echo -e "$(BUILD_PRINT)End to End Testing ...$(END_BUILD_PRINT)"
@@ -113,6 +113,24 @@ stop-allegro-graph:
 	@ echo -e "$(BUILD_PRINT)Stopping Allegro-Graph services $(END_BUILD_PRINT)"
 	@ docker-compose -p ${ENVIRONMENT} --file ./infra/allegro-graph/docker-compose.yml --env-file ${ENV_FILE} down
 
+#	------------------------
+start-fuseki: build-externals
+	@ echo -e "$(BUILD_PRINT)Starting Fuseki services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/fuseki/docker-compose.yml --env-file ${ENV_FILE} up -d
+
+stop-fuseki:
+	@ echo -e "$(BUILD_PRINT)Stopping Fuseki services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/fuseki/docker-compose.yml --env-file ${ENV_FILE} down
+
+#	------------------------
+start-sftp: build-externals
+	@ echo -e "$(BUILD_PRINT)Starting SFTP services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/sftp/docker-compose.yml --env-file ${ENV_FILE} up -d
+
+stop-sftp:
+	@ echo -e "$(BUILD_PRINT)Stopping SFTP services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/sftp/docker-compose.yml --env-file ${ENV_FILE} down
+
 #	------------------------
 build-elasticsearch: build-externals
 	@ echo -e "$(BUILD_PRINT) Build Elasticsearch services $(END_BUILD_PRINT)"
@@ -283,23 +301,23 @@ refresh-mapping-files:
 #-----------------------------------------------------------------------------
 # API Service commands
 #-----------------------------------------------------------------------------
-build-all-apis: build-id_manager-api
+build-all-apis: build-digest_service-api
 
-start-all-apis: start-id_manager-api
+start-all-apis: start-digest_service-api
 
-stop-all-apis: stop-id_manager-api
+stop-all-apis: stop-digest_service-api
 
-build-id_manager-api:
-	@ echo -e "$(BUILD_PRINT) Build id_manager API service $(END_BUILD_PRINT)"
+build-digest_service-api:
+	@ echo -e "$(BUILD_PRINT) Build digest_service API service $(END_BUILD_PRINT)"
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} build --no-cache --force-rm
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
 
-start-id_manager-api:
-	@ echo -e "$(BUILD_PRINT)Starting id_manager API service $(END_BUILD_PRINT)"
+start-digest_service-api:
+	@ echo -e "$(BUILD_PRINT)Starting digest_service API service $(END_BUILD_PRINT)"
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} up -d
 
-stop-id_manager-api:
-	@ echo -e "$(BUILD_PRINT)Stopping id_manager API service $(END_BUILD_PRINT)"
+stop-digest_service-api:
+	@ echo -e "$(BUILD_PRINT)Stopping digest_service API service $(END_BUILD_PRINT)"
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} down
 
 

README.md

@@ -412,21 +412,21 @@ Options:
 ##### Start local API server
 To start the API server:
 ```bash
-api-id_manager-start-server
+api-digest_service-start-server
 ```
 Output:
 ```bash
-uvicorn --host localhost --port 8000 ted_sws.id_manager.entrypoints.api.main:app --reload
+uvicorn --host localhost --port 8000 ted_sws.notice_transformer.entrypoints.api.digest_service.main:app --reload
 ###
 See http://localhost:8000/api/v1/docs for API usage.
 ```
 Use:
 ```bash
-api-id_manager-start-server --help
+api-digest_service-start-server --help
 ```
 to get the cli command Usage Help:
 ```bash
-Usage: api-id_manager-start-server [OPTIONS]
+Usage: api-digest_service-start-server [OPTIONS]
 
 Options:
   -h, --host TEXT

dags/index_and_normalise_notice_worker.py

@@ -12,7 +12,7 @@
 from ted_sws.event_manager.model.event_message import NoticeEventMessage
 from ted_sws.event_manager.services.logger_from_context import get_logger_from_dag_context, \
     handle_event_message_metadata_dag_context
-from ted_sws.metadata_normaliser.services.metadata_normalizer import normalise_notice_by_id
+from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice_by_id
 
 NOTICE_ID = "notice_id"
 

dags/selector_daily_fetch_orchestrator.py

@@ -16,10 +16,13 @@
     EventMessageProcessType
 from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, TedRequestAPI
 from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
+from ted_sws.supra_notice_manager.services.daily_supra_notice_manager import \
+    create_and_store_in_mongo_db_daily_supra_notice
 
 DAG_NAME = "selector_daily_fetch_orchestrator"
 WILD_CARD_PARAM = "wild_card"
 
+
 @dag(default_args=DEFAULT_DAG_ARGUMENTS,
      catchup=False,
      schedule_interval="0 3 * * *",
@@ -41,8 +44,10 @@ def fetch_notice_from_ted():
             current_datetime_wildcard = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y%m%d*")
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
         notice_ids = NoticeFetcher(notice_repository=NoticeRepository(mongodb_client=mongodb_client),
-                      ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI())).fetch_notices_by_date_wild_card(
+                                   ted_api_adapter=TedAPIAdapter(
+                                       request_api=TedRequestAPI())).fetch_notices_by_date_wild_card(
             wildcard_date=current_datetime_wildcard)
+        create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
         return notice_ids
 
     @task

dags/worker_single_notice_process_orchestrator.py

@@ -7,14 +7,14 @@
 from ted_sws.core.model.notice import NoticeStatus
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
-from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, index_notice_by_id
-from ted_sws.metadata_normaliser.services.metadata_normalizer import normalise_notice_by_id
+from ted_sws.data_sampler.services.notice_xml_indexer import index_notice_by_id
+from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice_by_id
 
 from airflow.decorators import dag
 from airflow.operators.python import get_current_context, BranchPythonOperator, PythonOperator
 
 from dags import DEFAULT_DAG_ARGUMENTS
-from ted_sws.notice_eligibility.services.notice_eligibility import notice_eligibility_checker_by_id
+from ted_sws.notice_metadata_processor.services.notice_eligibility import notice_eligibility_checker_by_id
 from ted_sws.notice_packager.services.notice_packager import create_notice_package
 from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapper
 from ted_sws.notice_transformer.services.notice_transformer import transform_notice_by_id

infra/api/id_manager/Dockerfile infra/api/digest_service/Dockerfileinfra/api/id_manager/Dockerfile renamed to infra/api/digest_service/Dockerfile

@@ -8,4 +8,4 @@ RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 RUN pip install -r infra/api/requirements.txt
 
-ENTRYPOINT uvicorn --host 0.0.0.0 --port ${ID_MANAGER_API_PORT} ted_sws.id_manager.entrypoints.api.main:app
+ENTRYPOINT uvicorn --host 0.0.0.0 --port ${ID_MANAGER_API_PORT} ted_sws.notice_transformer.entrypoints.api.digest_service.main:app

infra/api/docker-compose.yml

@@ -2,10 +2,10 @@ version: '3.8'
 
 services:
   api-id_manager:
-    container_name: api-id_manager
+    container_name: api-digest_service
     build:
       context: ../..
-      dockerfile: ./infra/api/id_manager/Dockerfile
+      dockerfile: ./infra/api/digest_service/Dockerfile
     env_file: ../../.env
     networks:
       - proxy-net

infra/fuseki/Dockerfile

@@ -0,0 +1,5 @@
+FROM secoresearch/fuseki:4.5.0
+
+USER root
+RUN apt-get update; \
+    apt-get install -y --no-install-recommends procps

infra/fuseki/docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3'
+services:
+  fuseki:
+    container_name: fuseki
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      - ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD}
+      - FUSEKI_DATASET_1=mydataset
+    volumes:
+      - fuseki-data:/fuseki-data/databases
+    networks:
+      - fuseki-net
+      - proxy-net
+    labels:
+      #### Labels define the behavior and rules of the traefik proxy for this container ####
+      - "traefik.enable=true" # <== Enable traefik to proxy this container
+      - "traefik.http.routers.${ENVIRONMENT}-fuseki.rule=Host(`fuseki.${SUBDOMAIN}${DOMAIN}`)" # <== Your Domain Name goes here for the http rule
+      - "traefik.http.routers.${ENVIRONMENT}-fuseki.entrypoints=web" # <== Defining the entrypoint for http, **ref: line 30
+      - "traefik.http.routers.${ENVIRONMENT}-fuseki.middlewares=redirect@file" # <== This is a middleware to redirect to https
+      - "traefik.http.routers.${ENVIRONMENT}-fuseki-secured.rule=Host(`fuseki.${SUBDOMAIN}${DOMAIN}`)" # <== Your Domain Name for the https rule
+      - "traefik.http.routers.${ENVIRONMENT}-fuseki-secured.entrypoints=web-secured" # <== Defining entrypoint for https, **ref: line 31
+      - "traefik.http.routers.${ENVIRONMENT}-fuseki-secured.tls.certresolver=mytlschallenge" # <== Defining certsresolvers for https
+volumes:
+  fuseki-data:
+    
+networks:
+  fuseki-net:
+    internal: true
+    name: fuseki-net-${ENVIRONMENT}
+  proxy-net:
+    external:
+      name: proxy-net
+  common-ext:
+    external:
+      name: common-ext-${ENVIRONMENT}