SupraNotice Validator

Author: Kolea Plesco

dags/selector_daily_fetch_orchestrator.py

@@ -10,7 +10,6 @@
 from ted_sws import config
 from ted_sws.core.model.notice import NoticeStatus
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
-from ted_sws.data_sampler.services.notice_xml_indexer import index_notice
 from ted_sws.event_manager.adapters.event_log_decorator import event_log
 from ted_sws.event_manager.model.event_message import TechnicalEventMessage, EventMessageMetadata, \
     EventMessageProcessType

ted_sws/core/model/supra_notice.py

@@ -7,11 +7,8 @@
 
 """ This module implements aggregates over groups of notices and the appropriate business needs, on those groups """
 import abc
-import json
-from datetime import datetime, date, time
-from typing import List, Any, Optional
-
-from pydantic import Field
+from datetime import datetime, time
+from typing import List, Optional
 
 from ted_sws.core.model import PropertyBaseModel
 from ted_sws.core.model.manifestation import Manifestation

ted_sws/data_manager/adapters/supra_notice_repository.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from typing import Optional, Iterator
 
 from pymongo import MongoClient, ASCENDING

ted_sws/notice_fetcher/adapters/ted_api_abc.py

@@ -17,7 +17,6 @@ def __call__(self, api_url: str, api_query: dict) -> dict:
         """
 
 
-
 class TedAPIAdapterABC(abc.ABC):
     @abc.abstractmethod
     def get_by_id(self, document_id: str) -> dict:

ted_sws/supra_notice_manager/services/supra_notice_validator.py

@@ -0,0 +1,38 @@
+from datetime import datetime, date, time
+from typing import Union
+
+from pymongo import MongoClient
+
+from ted_sws.core.model.supra_notice import SupraNoticeValidationReport, DailySupraNotice
+from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository
+from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, RequestAPI
+
+day_type = Union[datetime, date]
+
+
+def validate_and_update_daily_supra_notice(notice_publication_day: day_type, mongodb_client: MongoClient,
+                                           request_api: RequestAPI = None):
+    if isinstance(notice_publication_day, date):
+        notice_publication_day = datetime.combine(notice_publication_day, time())
+
+    repo = DailySupraNoticeRepository(mongodb_client=mongodb_client)
+    supra_notice: DailySupraNotice = repo.get(reference=notice_publication_day)
+
+    if not supra_notice:
+        raise ValueError("SupraNotice not found in Database!")
+
+    fetched_notice_ids_list = supra_notice.notice_ids or []
+    fetched_notice_ids = set(fetched_notice_ids_list)
+
+    ted_api_adapter: TedAPIAdapter = TedAPIAdapter(request_api=request_api)
+    documents = ted_api_adapter.get_by_wildcard_date(wildcard_date=notice_publication_day.strftime("%Y%m%d*"))
+    api_notice_ids_list = [document["ND"] for document in documents] if documents and len(documents) else []
+    api_notice_ids = set(api_notice_ids_list)
+
+    validation_report = SupraNoticeValidationReport(object_data="")
+    missing_notice_ids = api_notice_ids - fetched_notice_ids
+    if len(missing_notice_ids):
+        validation_report.missing_notice_ids = missing_notice_ids
+
+    supra_notice.validation_report = validation_report
+    repo.update(daily_supra_notice=supra_notice)

tests/conftest.py

@@ -39,6 +39,7 @@ def notice_repository():
 def ted_document_search():
     return TedAPIAdapter(request_api=FakeRequestAPI())
 
+
 @pytest.fixture
 def raw_notice(ted_document_search, notice_repository, notice_id) -> Notice:
     document_id = notice_id
@@ -47,6 +48,7 @@ def raw_notice(ted_document_search, notice_repository, notice_id) -> Notice:
     raw_notice = notice_repository.get(reference=document_id)
     return raw_notice
 
+
 @pytest.fixture
 def indexed_notice(raw_notice) -> Notice:
     raw_notice.set_xml_metadata(XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))

tests/unit/supra_notice_manager/conftest.py

@@ -0,0 +1,14 @@
+import pytest
+
+from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository
+from tests.fakes.fake_ted_api import FakeRequestAPI
+
+
+@pytest.fixture
+def daily_supra_notice_repository(mongodb_client) -> DailySupraNoticeRepository:
+    return DailySupraNoticeRepository(mongodb_client=mongodb_client)
+
+
+@pytest.fixture
+def fake_request_api():
+    return FakeRequestAPI()

tests/unit/supra_notice_manager/test_daily_supra_notice_manager.py

@@ -5,9 +5,8 @@
     create_and_store_in_mongo_db_daily_supra_notice
 
 
-def test_daily_supra_notice_manager(mongodb_client):
+def test_daily_supra_notice_manager(mongodb_client, daily_supra_notice_repository):
     notice_ids = ["1", "2", "3"]
     create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
-    daily_supra_notice_repository = DailySupraNoticeRepository(mongodb_client=mongodb_client)
     result = daily_supra_notice_repository.get(reference=datetime.combine(datetime.today(), time()))
-    assert result
+    assert result

tests/unit/supra_notice_manager/test_supra_notice_validator.py

@@ -0,0 +1,32 @@
+from datetime import datetime, time
+
+import pytest
+
+from ted_sws.supra_notice_manager.services.daily_supra_notice_manager import \
+    create_and_store_in_mongo_db_daily_supra_notice
+from ted_sws.supra_notice_manager.services.supra_notice_validator import validate_and_update_daily_supra_notice
+
+
+def test_supra_notice_validator(mongodb_client, daily_supra_notice_repository, fake_request_api):
+    today = datetime.combine(datetime.today(), time())
+
+    with pytest.raises(ValueError):
+        validate_and_update_daily_supra_notice(today, mongodb_client, fake_request_api)
+
+    api_document_id = "067623-2022"
+
+    notice_ids = ["1", "2", "3"]
+    create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
+    validate_and_update_daily_supra_notice(today, mongodb_client, fake_request_api)
+    result = daily_supra_notice_repository.get(reference=today)
+    assert result
+    assert len(result.validation_report.missing_notice_ids) > 0
+    assert api_document_id in result.validation_report.missing_notice_ids
+    assert not result.validation_report.is_valid()
+
+    notice_ids = [api_document_id]
+    create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
+    validate_and_update_daily_supra_notice(today, mongodb_client, fake_request_api)
+    result = daily_supra_notice_repository.get(reference=today)
+    assert result
+    assert result.validation_report.is_valid()