Merge pull request #223 from OP-TED/feature/TED-593

Feature/ted 593

Author: CaptainOfHacks

Makefile

@@ -37,7 +37,7 @@ test-unit:
 
 test-features:
 	@ echo -e "$(BUILD_PRINT)Gherkin Features Testing ...$(END_BUILD_PRINT)"
-# 	@ tox -e features
+	@ tox -e features
 
 test-e2e:
 	@ echo -e "$(BUILD_PRINT)End to End Testing ...$(END_BUILD_PRINT)"

ted_sws/notice_transformer/services/notice_transformer.py

@@ -1,4 +1,3 @@
-import abc
 import tempfile
 from pathlib import Path
 

tests/features/__init__.py

@@ -1,19 +0,0 @@
-#!/usr/bin/python3
-
-# __init__.py
-# Date:  03/02/2022
-# Author: Eugeniu Costetchi
-# Email: costezki.eugen@gmail.com
-
-""" """
-
-
-def str2bool(value: str) -> bool:
-    """
-        Parse a string value and cast it into its boolean value
-    :param value:
-    :return:
-    """
-    if value in ["y", "yes", "t", "true", "on", "1"]: return True
-    if value in ["n", "no", "f", "false", "off", "0"]: return False
-    raise ValueError("boolean value unrecognised")

tests/features/conftest.py

@@ -1,60 +1,103 @@
+import mongomock
+import pymongo
 import pytest
-from pymongo import MongoClient
 
 from ted_sws import config
-from ted_sws.core.model.metadata import XMLMetadata
-from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
+from ted_sws.core.model.manifestation import METSManifestation, RDFManifestation, SHACLTestSuiteValidationReport, \
+    SPARQLTestSuiteValidationReport
+from ted_sws.core.model.metadata import NormalisedMetadata, XMLMetadata
+from ted_sws.core.model.notice import NoticeStatus, Notice
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
-
 from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, TedRequestAPI
 from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
-from tests import TEST_DATA_PATH
 
 NOTICE_STORAGE_FEATURES_TEST_DB = "features_test_db_for_notice"
 
 
 @pytest.fixture
-def api_end_point():
+def mongodb_end_point():
+    return config.MONGO_DB_AUTH_URL
+
+
+@pytest.fixture(scope="function")
+@mongomock.patch(servers=(('server.example.com', 27017),))
+def mongodb_client():
+    mongo_client = pymongo.MongoClient('server.example.com')
+    for database_name in mongo_client.list_database_names():
+        mongo_client.drop_database(database_name)
+    return mongo_client
+
+
+@pytest.fixture
+def ted_api_end_point():
     return config.TED_API_URL
 
 
 @pytest.fixture
-def notice_storage():
-    url = config.MONGO_DB_AUTH_URL
-    mongodb_client = MongoClient(url)
-    mongodb_client.drop_database(NOTICE_STORAGE_FEATURES_TEST_DB)
+def notice_repository(mongodb_client):
     return NoticeRepository(mongodb_client=mongodb_client, database_name=NOTICE_STORAGE_FEATURES_TEST_DB)
 
 
 @pytest.fixture
-def f03_notice_2020(notice_storage, api_end_point):
+def f03_notice_2020(notice_repository, ted_api_end_point):
     notice_search_query = {"q": "ND=[408313-2020]"}
-    NoticeFetcher(notice_repository=notice_storage,
+    NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
-                                                ted_api_url=api_end_point)).fetch_notices_by_query(
+                                                ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
-    notice = notice_storage.get(reference="408313-2020")
+    notice = notice_repository.get(reference="408313-2020")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
     return notice
 
 
 @pytest.fixture
-def f18_notice_2022(notice_storage, api_end_point):
+def f18_notice_2022(notice_repository, ted_api_end_point):
     notice_search_query = {"q": "ND=[067623-2022]"}
-    NoticeFetcher(notice_repository=notice_storage,
+    NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
-                                                ted_api_url=api_end_point)).fetch_notices_by_query(
+                                                ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
-    notice = notice_storage.get(reference="067623-2022")
+    notice = notice_repository.get(reference="067623-2022")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
     return notice
 
 
 @pytest.fixture
-def file_system_repository_path():
-    return TEST_DATA_PATH / "notice_transformer" / "test_repository"
+def notice_id(notice_2020):
+    return notice_2020.ted_id
 
 
 @pytest.fixture
-def mapping_suite_repository_in_file_system(file_system_repository_path):
-    return MappingSuiteRepositoryInFileSystem(repository_path=file_system_repository_path)
+def fetched_notice_data(notice_2020):
+    ted_id = notice_2020.ted_id
+    original_metadata = notice_2020.original_metadata
+    xml_manifestation = notice_2020.xml_manifestation
+    return ted_id, original_metadata, xml_manifestation
+
+
+@pytest.fixture(scope="function")
+def publicly_available_notice(fetched_notice_data, normalised_metadata_dict) -> Notice:
+    ted_id, original_metadata, xml_manifestation = fetched_notice_data
+    sparql_validation = SPARQLTestSuiteValidationReport(object_data="This is validation report!",
+                                                        test_suite_identifier="sparql_test_id",
+                                                        mapping_suite_identifier="mapping_suite_id",
+                                                        validation_results=[])
+    shacl_validation = SHACLTestSuiteValidationReport(object_data="This is validation report!",
+                                                      test_suite_identifier="shacl_test_id",
+                                                      mapping_suite_identifier="mapping_suite_id",
+                                                      validation_results=[])
+    notice = Notice(ted_id=ted_id, original_metadata=original_metadata,
+                    xml_manifestation=xml_manifestation)
+    notice._rdf_manifestation = RDFManifestation(object_data="RDF manifestation content",
+                                                 shacl_validations=[shacl_validation],
+                                                 sparql_validations=[sparql_validation]
+                                                 )
+    notice._distilled_rdf_manifestation = RDFManifestation(object_data="RDF manifestation content",
+                                                           shacl_validations=[shacl_validation],
+                                                           sparql_validations=[sparql_validation]
+                                                           )
+    notice._mets_manifestation = METSManifestation(object_data="METS manifestation content")
+    notice._normalised_metadata = NormalisedMetadata(**normalised_metadata_dict)
+    notice._preprocessed_xml_manifestation = xml_manifestation
+    notice._status = NoticeStatus.PUBLICLY_AVAILABLE
+    return notice

tests/features/model/__init__.py

@@ -1,8 +0,0 @@
-#!/usr/bin/python3
-
-# __init__.py
-# Date:  03/02/2022
-# Author: Eugeniu Costetchi
-# Email: costezki.eugen@gmail.com
-
-""" """

tests/features/model/conftest.py

@@ -1,12 +1,3 @@
-#!/usr/bin/python3
-
-# conftest.py
-# Date:  29/01/2022
-# Author: Eugeniu Costetchi
-# Email: costezki.eugen@gmail.com 
-
-""" """
-
 import pytest
 
 from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation, \
@@ -64,3 +55,6 @@ def transformation_eligible_notice(indexed_notice, normalised_metadata_dict) ->
     indexed_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION)
     indexed_notice.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION)
     return indexed_notice
+
+
+

tests/features/model/test_notice_operations.py

@@ -4,7 +4,17 @@
 from ted_sws.core.model.manifestation import RDFManifestation, RDFValidationManifestation, METSManifestation
 from ted_sws.core.model.metadata import NormalisedMetadata
 from ted_sws.core.model.notice import NoticeStatus
-from tests.features import str2bool
+
+
+def str2bool(value: str) -> bool:
+    """
+        Parse a string value and cast it into its boolean value
+    :param value:
+    :return:
+    """
+    if value in ["y", "yes", "t", "true", "on", "1"]: return True
+    if value in ["n", "no", "f", "false", "off", "0"]: return False
+    raise ValueError("boolean value unrecognised")
 
 
 @scenario("test_notice_operations.feature", "add normalised metadata")

tests/features/notice_fetcher/__init__.py

@@ -1,8 +0,0 @@
-#!/usr/bin/python3
-
-# __init__.py
-# Date:  03/02/2022
-# Author: Eugeniu Costetchi
-# Email: costezki.eugen@gmail.com
-
-""" """

tests/features/notice_fetcher/conftest.py

@@ -1,18 +1,43 @@
+from datetime import date
+
 import pytest
 
+from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, TedRequestAPI
+from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
+from tests.fakes.fake_ted_api import FakeTedApiAdapter
+
 
 @pytest.fixture
-def notice_identifier():
+def fetch_notice_id():
     return "067623-2022"
 
 
 @pytest.fixture
-def notice_search_query():
-    return {"q": "ND=[067623-2022]"}
+def fetch_start_date():
+    return date(2020, 2, 3)
+
+
+@pytest.fixture
+def fetch_date(fetch_start_date):
+    return fetch_start_date
+
+
+@pytest.fixture
+def fetch_end_date():
+    return date(2020, 2, 3)
+
+
+@pytest.fixture
+def fetch_wildcard_date():
+    return "20200203*"
 
 
 @pytest.fixture
-def notice_incorrect_search_query():
-    return {"q": "ND=067623-20224856"}
+def fetch_query(fetch_wildcard_date):
+    return {"q": f"PD=[{fetch_wildcard_date}]"}
 
 
+@pytest.fixture
+def notice_fetcher(notice_repository, ted_api_end_point):
+    return NoticeFetcher(notice_repository=notice_repository,
+                         ted_api_adapter=FakeTedApiAdapter())

tests/features/notice_fetcher/test_notice_fetcher.feature

@@ -1,21 +1,47 @@
-# Created by dude at 25/01/2022
+# Created by Stefan at 16.08.2022
 Feature: Notice fetcher
   The system is able to fetch selected TED-XML notices together with their metadata
 
-  Scenario: Fetch a TED notice
-    Given a TED REST API download endpoint
-    And correct download API parameters
-    When call to the API is made
-    Then a notice and notice metadata is received from the API
-    And the notice and notice metadata are stored
+  Scenario: Fetch a notice by id, from Ted
+    Given a notice_id
+    And knowing the TED API endpoint
+    And knowing database endpoint
+    When notice fetching by id is executed
+    Then fetched notice is available in database
+    And fetched notice have raw status
+    And fetched notice have xml_manifestation
+    And fetched notice have original_metadata
 
-  Scenario: Fail to fetch a TED notice
-    Given a TED REST API download endpoint
-    And incorrect download API parameters
-    When the call to the API is made
-    And no notice or metadata is returned
-    Then an error message is received indicating the problem
+  Scenario: Fetch notices by query, from Ted
+    Given a query
+    And knowing the TED API endpoint
+    And knowing database endpoint
+    When notices fetching by query is executed
+    Then a list of fetched notice_ids is returned
+    And foreach returned notice_id exist in database a notice with RAW status
+    And foreach returned notice_id exist in database a notice with xml_manifestation
+    And foreach returned notice_id exist in database a notice with original_metadata
 
+  Scenario: Fetch notices by date range, from Ted
+    Given a start_date
+    And a end_date
+    And knowing the TED API endpoint
+    And knowing database endpoint
+    When notices fetching by date range is executed
+    Then a list of fetched notice_ids is returned
+    And foreach returned notice_id exist in database a notice with RAW status
+    And foreach returned notice_id exist in database a notice with xml_manifestation
+    And foreach returned notice_id exist in database a notice with original_metadata
+
+  Scenario: Fetch notices by date wild card, from Ted
+    Given a wildcard_date
+    And knowing the TED API endpoint
+    And knowing database endpoint
+    When notices fetching by date wild card is executed
+    Then a list of fetched notice_ids is returned
+    And foreach returned notice_id exist in database a notice with RAW status
+    And foreach returned notice_id exist in database a notice with xml_manifestation
+    And foreach returned notice_id exist in database a notice with original_metadata