Merge pull request #197 from meaningfy-ws/feature/TED-379

Feature/ted 379

Author: CaptainOfHacks

dags/selector_daily_fetch_orchestrator.py

@@ -10,7 +10,6 @@
 from ted_sws import config
 from ted_sws.core.model.notice import NoticeStatus
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
-from ted_sws.data_sampler.services.notice_xml_indexer import index_notice
 from ted_sws.event_manager.adapters.event_log_decorator import event_log
 from ted_sws.event_manager.model.event_message import TechnicalEventMessage, EventMessageMetadata, \
     EventMessageProcessType

ted_sws/notice_fetcher/adapters/ted_api_abc.py

@@ -17,7 +17,6 @@ def __call__(self, api_url: str, api_query: dict) -> dict:
         """
 
 
-
 class TedAPIAdapterABC(abc.ABC):
     @abc.abstractmethod
     def get_by_id(self, document_id: str) -> dict:

ted_sws/notice_validator/adapters/xpath_coverage_runner.py

@@ -81,8 +81,13 @@ def xpath_coverage(self, report: XPATHCoverageReport, notice_xpaths: XPATH_TYPE,
         report.xpath_covered = list(self.conceptual_xpaths & unique_notice_xpaths)
         report.xpath_not_covered = list(unique_notice_xpaths - self.conceptual_xpaths)
         report.xpath_extra = list(self.conceptual_xpaths - unique_notice_xpaths)
-        if len(unique_notice_xpaths):
-            report.coverage = len(report.xpath_covered) / len(unique_notice_xpaths)
+        unique_notice_xpaths_len = len(unique_notice_xpaths)
+        xpath_covered_len = len(report.xpath_covered)
+        conceptual_xpaths_len = len(self.conceptual_xpaths)
+        if unique_notice_xpaths_len:
+            report.coverage = xpath_covered_len / unique_notice_xpaths_len
+        if conceptual_xpaths_len:
+            report.conceptual_coverage = xpath_covered_len / conceptual_xpaths_len
 
     @classmethod
     def based_xpaths(cls, xpaths: List[str], base_xpath: str) -> List[str]:

ted_sws/notice_validator/model/coverage_report.py

@@ -26,3 +26,4 @@ class XPATHCoverageReport(PropertyBaseModel):
     xpath_not_covered: Optional[List[str]]
     xpath_extra: Optional[List[str]]
     coverage: Optional[float]
+    conceptual_coverage: Optional[float]

ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2

@@ -73,9 +73,17 @@
 <ul>
     <li>Date created: {{ created_at }}</li>
     <li>Mapping suite identifier: {{ mapping_suite_id }}</li>
-    <li>Coverage: <strong>{{ coverage }}</strong>
-    <div><hr></div></li>
-    <li>Notice identifier(s):
+    <li>
+        <div><hr></div>
+        <h4>Overall coverage</h4>
+        <h3>{{ "%.2f"|format((coverage|float * 100)) }}%</h3>
+        <p>(Number of Conceptual XPATHs found in notices) / (Total number of unique XPATHs in provided notices)</p>
+    <li><div><hr></div>
+        <h4>Conceptual coverage</h4>
+        <h3>{{ "%.2f"|format((conceptual_coverage|float * 100)) }}%</h3>
+        <p>(Number of Conceptual XPATHs found in notices) / (Total number of unique Conceptual XPATHs)</p>
+    </li>
+    <li><div><hr></div>Notice identifier(s):
         <div data-role="collapsible" data-state="{% if notice_id|length > 1 %}collapsed{% endif %}"
             class="collapsible-wrapper">
             <h4><a href="#"></a></h4>
@@ -97,16 +105,17 @@
     <tr>
         <th>Title</th>
         <th>XPATH</th>
-        <th>Count</th>
-        <th>Notice</th>
-        <th>Result</th>
+        <th>Found</th>
+        <th>Notice count</th>
+        <th>Notices</th>
     </tr>
     </thead>
     <tbody>
     {% for item in xpath_assertions %}
     <tr>
         <td>{{ item.title }}</td>
         <td>{{ item.xpath }}</td>
+        <td class="{% if item.query_result %}success{% else %}error{% endif %}">{{ item.query_result }}</td>
         <td>{{ item.count }}</td>
         <td class="notice-hit">{% if item.notice_hit %}
             <div data-role="collapsible" data-state="collapsed">
@@ -116,7 +125,6 @@
                 </div>
             </div>
         {% endif %}</td>
-        <td class="{% if item.query_result %}success{% else %}error{% endif %}">{{ item.query_result }}</td>
     </tr>
     {% endfor %}
     </tbody>

ted_sws/supra_notice_manager/services/supra_notice_validator.py

@@ -0,0 +1,38 @@
+from datetime import datetime, date, time
+from typing import Union
+
+from pymongo import MongoClient
+
+from ted_sws.core.model.supra_notice import SupraNoticeValidationReport, DailySupraNotice
+from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository
+from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, RequestAPI
+
+day_type = Union[datetime, date]
+
+
+def validate_and_update_daily_supra_notice(notice_publication_day: day_type, mongodb_client: MongoClient,
+                                           request_api: RequestAPI = None):
+    if isinstance(notice_publication_day, date):
+        notice_publication_day = datetime.combine(notice_publication_day, time())
+
+    repo = DailySupraNoticeRepository(mongodb_client=mongodb_client)
+    supra_notice: DailySupraNotice = repo.get(reference=notice_publication_day)
+
+    if not supra_notice:
+        raise ValueError("SupraNotice not found in Database!")
+
+    fetched_notice_ids_list = supra_notice.notice_ids or []
+    fetched_notice_ids = set(fetched_notice_ids_list)
+
+    ted_api_adapter: TedAPIAdapter = TedAPIAdapter(request_api=request_api)
+    documents = ted_api_adapter.get_by_wildcard_date(wildcard_date=notice_publication_day.strftime("%Y%m%d*"))
+    api_notice_ids_list = [document["ND"] for document in documents] if documents and len(documents) else []
+    api_notice_ids = set(api_notice_ids_list)
+
+    validation_report = SupraNoticeValidationReport(object_data="")
+    missing_notice_ids = api_notice_ids - fetched_notice_ids
+    if len(missing_notice_ids):
+        validation_report.missing_notice_ids = missing_notice_ids
+
+    supra_notice.validation_report = validation_report
+    repo.update(daily_supra_notice=supra_notice)

tests/conftest.py

@@ -39,6 +39,7 @@ def notice_repository():
 def ted_document_search():
     return TedAPIAdapter(request_api=FakeRequestAPI())
 
+
 @pytest.fixture
 def raw_notice(ted_document_search, notice_repository, notice_id) -> Notice:
     document_id = notice_id
@@ -47,6 +48,7 @@ def raw_notice(ted_document_search, notice_repository, notice_id) -> Notice:
     raw_notice = notice_repository.get(reference=document_id)
     return raw_notice
 
+
 @pytest.fixture
 def indexed_notice(raw_notice) -> Notice:
     raw_notice.set_xml_metadata(XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))

tests/unit/supra_notice_manager/conftest.py

@@ -0,0 +1,14 @@
+import pytest
+
+from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository
+from tests.fakes.fake_ted_api import FakeRequestAPI
+
+
+@pytest.fixture
+def daily_supra_notice_repository(mongodb_client) -> DailySupraNoticeRepository:
+    return DailySupraNoticeRepository(mongodb_client=mongodb_client)
+
+
+@pytest.fixture
+def fake_request_api():
+    return FakeRequestAPI()

tests/unit/supra_notice_manager/test_daily_supra_notice_manager.py

@@ -5,9 +5,8 @@
     create_and_store_in_mongo_db_daily_supra_notice
 
 
-def test_daily_supra_notice_manager(mongodb_client):
+def test_daily_supra_notice_manager(mongodb_client, daily_supra_notice_repository):
     notice_ids = ["1", "2", "3"]
     create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
-    daily_supra_notice_repository = DailySupraNoticeRepository(mongodb_client=mongodb_client)
     result = daily_supra_notice_repository.get(reference=datetime.combine(datetime.today(), time()))
-    assert result
+    assert result

tests/unit/supra_notice_manager/test_supra_notice_validator.py

@@ -0,0 +1,32 @@
+from datetime import datetime, time
+
+import pytest
+
+from ted_sws.supra_notice_manager.services.daily_supra_notice_manager import \
+    create_and_store_in_mongo_db_daily_supra_notice
+from ted_sws.supra_notice_manager.services.supra_notice_validator import validate_and_update_daily_supra_notice
+
+
+def test_supra_notice_validator(mongodb_client, daily_supra_notice_repository, fake_request_api):
+    today = datetime.combine(datetime.today(), time())
+
+    with pytest.raises(ValueError):
+        validate_and_update_daily_supra_notice(today, mongodb_client, fake_request_api)
+
+    api_document_id = "067623-2022"
+
+    notice_ids = ["1", "2", "3"]
+    create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
+    validate_and_update_daily_supra_notice(today, mongodb_client, fake_request_api)
+    result = daily_supra_notice_repository.get(reference=today)
+    assert result
+    assert len(result.validation_report.missing_notice_ids) > 0
+    assert api_document_id in result.validation_report.missing_notice_ids
+    assert not result.validation_report.is_valid()
+
+    notice_ids = [api_document_id]
+    create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
+    validate_and_update_daily_supra_notice(today, mongodb_client, fake_request_api)
+    result = daily_supra_notice_repository.get(reference=today)
+    assert result
+    assert result.validation_report.is_valid()