Merge pull request #196 from meaningfy-ws/feature/TED-480

Feature/ted 480

Author: costezki

.github/workflows/unit-tests.yml

@@ -23,6 +23,7 @@ jobs:
       run: |
         python -m pip install --upgrade setuptools pip wheel
         make install
+        make install-dev
     - name: Make envfile
       uses: SpicyPizza/create-envfile@v1
       with:

Makefile

@@ -19,15 +19,15 @@ XML_PROCESSOR_PATH = ${PROJECT_PATH}/.saxon/saxon-he-10.6.jar
 #-----------------------------------------------------------------------------
 # Dev commands
 #-----------------------------------------------------------------------------
-install: install-dev
+install:
 	@ echo -e "$(BUILD_PRINT)Installing the requirements$(END_BUILD_PRINT)"
 	@ pip install --upgrade pip
-	@ pip install -r requirements.txt
+	@ pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.5/constraints-no-providers-3.8.txt"
 
 install-dev:
 	@ echo -e "$(BUILD_PRINT)Installing the dev requirements$(END_BUILD_PRINT)"
 	@ pip install --upgrade pip
-	@ pip install -r requirements.dev.txt
+	@ pip install --no-cache-dir -r requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.5/constraints-no-providers-3.8.txt"
 
 test: test-unit
 
@@ -37,7 +37,7 @@ test-unit:
 
 test-features:
 	@ echo -e "$(BUILD_PRINT)Gherkin Features Testing ...$(END_BUILD_PRINT)"
-	@ tox -e features
+# 	@ tox -e features
 
 test-e2e:
 	@ echo -e "$(BUILD_PRINT)End to End Testing ...$(END_BUILD_PRINT)"
@@ -301,23 +301,23 @@ refresh-mapping-files:
 #-----------------------------------------------------------------------------
 # API Service commands
 #-----------------------------------------------------------------------------
-build-all-apis: build-id_manager-api
+build-all-apis: build-digest_service-api
 
-start-all-apis: start-id_manager-api
+start-all-apis: start-digest_service-api
 
-stop-all-apis: stop-id_manager-api
+stop-all-apis: stop-digest_service-api
 
-build-id_manager-api:
-	@ echo -e "$(BUILD_PRINT) Build id_manager API service $(END_BUILD_PRINT)"
+build-digest_service-api:
+	@ echo -e "$(BUILD_PRINT) Build digest_service API service $(END_BUILD_PRINT)"
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} build --no-cache --force-rm
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
 
-start-id_manager-api:
-	@ echo -e "$(BUILD_PRINT)Starting id_manager API service $(END_BUILD_PRINT)"
+start-digest_service-api:
+	@ echo -e "$(BUILD_PRINT)Starting digest_service API service $(END_BUILD_PRINT)"
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} up -d
 
-stop-id_manager-api:
-	@ echo -e "$(BUILD_PRINT)Stopping id_manager API service $(END_BUILD_PRINT)"
+stop-digest_service-api:
+	@ echo -e "$(BUILD_PRINT)Stopping digest_service API service $(END_BUILD_PRINT)"
 	@ docker-compose -p common --file infra/api/docker-compose.yml --env-file ${ENV_FILE} down
 
 

README.md

@@ -16,7 +16,8 @@
 ## Table of contents
 
 - [Developer documentation](#developer-documentation)
-- [Installation](#installation)
+- [Installation for Software Engineers](#installation-for-software-engineers)
+- [Installation for Semantic Engineers](#installation-for-semantic-engineers)
 - [Usage](#usage)
 - [Contributing](#contributing)
 - [Licence](#licence)
@@ -31,9 +32,68 @@ If you contribute to this project please refer to the following project document
 * [Enterprise architecture model file (in development)](https://drive.google.com/file/d/1YB2dPYe9E9bAR2peVraQaUANS-hXetms/view?usp=sharing)
 * [Meaningfy google Drive of the project (restricted)](https://drive.google.com/drive/folders/1wfWYDAtcaJrYTuB14VzTixr1mJUkCHYl?usp=sharing)
 
+Try to follow as close as possible the Clean Code Architecture (onion architecture), develop by writing tests first i.e. do Test Driven Development (TDD) and think of the Domain first, i.e. think Domain Driven Design (DDD).
+
 <hr>
 
-## Installation
+## Installation for software engineers
+
+
+Make sure you have installed `Docker` and `docker-compose`.
+
+Create the local `venv` and activate it.
+```shell
+pip install virtualenv
+virtualenv venv
+source venv/bin/activate
+```
+
+Install the python dependencies:
+
+```shell
+make install
+make istall-dev
+```
+
+Prepare the Airflow environment
+```shell
+make create-env-airflow
+```
+
+Set up the local infra 
+```shell
+make build-externals
+make build-airflow
+make init-rml-mapper
+make init-saxon
+```
+
+Start local infra services 
+```shell
+make start-traefik
+make start-mongo
+make start-metabase
+make start-allegro-graph
+```
+
+Start Airflow
+```shell
+make start-airflow
+```
+
+## Running tests
+
+```shell
+make install-dev
+```
+
+```shell
+make test-unit
+make test-feature
+make test-e2e
+```
+
+## Installation for semantic engineers
 ### Installation of ted-sws package within external projects using terminal
 #### 1. Using the package manager pip
 1. Go to the root folder of the project
@@ -352,21 +412,21 @@ Options:
 ##### Start local API server
 To start the API server:
 ```bash
-api-id_manager-start-server
+api-digest_service-start-server
 ```
 Output:
 ```bash
-uvicorn --host localhost --port 8000 ted_sws.id_manager.entrypoints.api.main:app --reload
+uvicorn --host localhost --port 8000 ted_sws.notice_transformer.entrypoints.api.digest_service.main:app --reload
 ###
 See http://localhost:8000/api/v1/docs for API usage.
 ```
 Use:
 ```bash
-api-id_manager-start-server --help
+api-digest_service-start-server --help
 ```
 to get the cli command Usage Help:
 ```bash
-Usage: api-id_manager-start-server [OPTIONS]
+Usage: api-digest_service-start-server [OPTIONS]
 
 Options:
   -h, --host TEXT

dags/index_and_normalise_notice_worker.py

@@ -12,7 +12,7 @@
 from ted_sws.event_manager.model.event_message import NoticeEventMessage
 from ted_sws.event_manager.services.logger_from_context import get_logger_from_dag_context, \
     handle_event_message_metadata_dag_context
-from ted_sws.metadata_normaliser.services.metadata_normalizer import normalise_notice_by_id
+from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice_by_id
 
 NOTICE_ID = "notice_id"
 

dags/worker_single_notice_process_orchestrator.py

@@ -7,14 +7,14 @@
 from ted_sws.core.model.notice import NoticeStatus
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
-from ted_sws.data_sampler.services.notice_xml_indexer import index_notice, index_notice_by_id
-from ted_sws.metadata_normaliser.services.metadata_normalizer import normalise_notice_by_id
+from ted_sws.data_sampler.services.notice_xml_indexer import index_notice_by_id
+from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice_by_id
 
 from airflow.decorators import dag
 from airflow.operators.python import get_current_context, BranchPythonOperator, PythonOperator
 
 from dags import DEFAULT_DAG_ARGUMENTS
-from ted_sws.notice_eligibility.services.notice_eligibility import notice_eligibility_checker_by_id
+from ted_sws.notice_metadata_processor.services.notice_eligibility import notice_eligibility_checker_by_id
 from ted_sws.notice_packager.services.notice_packager import create_notice_package
 from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapper
 from ted_sws.notice_transformer.services.notice_transformer import transform_notice_by_id

infra/api/id_manager/Dockerfile infra/api/digest_service/Dockerfileinfra/api/id_manager/Dockerfile renamed to infra/api/digest_service/Dockerfile

@@ -8,4 +8,4 @@ RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 RUN pip install -r infra/api/requirements.txt
 
-ENTRYPOINT uvicorn --host 0.0.0.0 --port ${ID_MANAGER_API_PORT} ted_sws.id_manager.entrypoints.api.main:app
+ENTRYPOINT uvicorn --host 0.0.0.0 --port ${ID_MANAGER_API_PORT} ted_sws.notice_transformer.entrypoints.api.digest_service.main:app

infra/api/docker-compose.yml

@@ -2,10 +2,10 @@ version: '3.8'
 
 services:
   api-id_manager:
-    container_name: api-id_manager
+    container_name: api-digest_service
     build:
       context: ../..
-      dockerfile: ./infra/api/id_manager/Dockerfile
+      dockerfile: ./infra/api/digest_service/Dockerfile
     env_file: ../../.env
     networks:
       - proxy-net

setup.py

@@ -89,7 +89,7 @@ def open_local(paths, mode="r", encoding="utf8"):
             "normalisation_resource_generator = ted_sws.data_manager.entrypoints.cli.cmd_generate_mapping_resources:main",
             "triple_store_loader = ted_sws.mapping_suite_processor.entrypoints.cli.cmd_triple_store_loader:main",
             "bulk_packager = ted_sws.notice_packager.entrypoints.cli.cmd_bulk_packager:main",
-            "api-id_manager-start-server = ted_sws.id_manager.entrypoints.api.server:api_server_start"
+            "api-digest_service-start-server = ted_sws.notice_transformer.entrypoints.api.digest_service.server:api_server_start"
         ],
     },
     include_package_data=True,

ted_sws/__init__.py

@@ -146,8 +146,14 @@ def ID_MANAGER_API_PORT(self) -> int:
         return int(v) if v else 8000
 
 
+class TedAPIConfig:
+    @property
+    def TED_API_URL(self) -> str:
+        return EnvConfigResolver().config_resolve()
+
+
 class TedConfigResolver(MongoDBConfig, RMLMapperConfig, XMLProcessorConfig, ELKConfig, LoggingConfig,
-                        GitHubArtefacts, API, AllegroConfig):
+                        GitHubArtefacts, API, AllegroConfig, TedAPIConfig):
     """
         This class resolve the secrets of the ted-sws project.
     """

ted_sws/core/adapters/vault_secrets_store.py

@@ -1,9 +1,10 @@
-from abc import ABC, abstractmethod
-from typing import List
-import hvac
 import json
 import os
+from abc import ABC, abstractmethod
+from typing import List
+
 import dotenv
+import hvac
 
 dotenv.load_dotenv(verbose=True, override=os.environ.get('IS_PRIME_ENV') != 'true')