Skip to content

Commit f6982c2

Browse files
kaleanychkaleanych
authored
Merge pull request #215 from OP-TED/feature/TED-639
Feature/ted 639
2 parents c5fb63a + 5ef18e2 commit f6982c2

9 files changed

Lines changed: 46 additions & 27 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ create-env-airflow:
9393

9494
build-airflow: guard-ENVIRONMENT create-env-airflow build-externals
9595
@ echo -e "$(BUILD_PRINT) Build Airflow services $(END_BUILD_PRINT)"
96-
@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} build --no-cache --force-rm
96+
@ docker build -t meaningfy/airflow ./infra/airflow/
9797
@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} up -d --force-recreate
9898

9999
start-airflow: build-externals

‎dags/selector_daily_fetch_orchestrator.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import datetime
1+
from datetime import datetime, timedelta
22
from typing import List
33

44
from airflow.decorators import dag, task
@@ -40,13 +40,15 @@ def fetch_notice_from_ted():
4040
if WILD_CARD_PARAM in dag_params.keys():
4141
current_datetime_wildcard = dag_params[WILD_CARD_PARAM]
4242
else:
43-
current_datetime_wildcard = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y%m%d*")
43+
current_datetime_wildcard = (datetime.now() - timedelta(days=1)).strftime("%Y%m%d*")
44+
notice_publication_date = datetime.strptime(current_datetime_wildcard, "%Y%m%d*").date()
4445
mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
4546
notice_ids = NoticeFetcher(notice_repository=NoticeRepository(mongodb_client=mongodb_client),
4647
ted_api_adapter=TedAPIAdapter(
4748
request_api=TedRequestAPI())).fetch_notices_by_date_wild_card(
4849
wildcard_date=current_datetime_wildcard)
49-
create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
50+
create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client,
51+
notice_publication_date=notice_publication_date)
5052
return notice_ids
5153

5254
@task

‎infra/airflow/docker-compose.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ x-airflow-common:
4545
# Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
4646
# and uncomment the "build" line below, Then run `docker-compose build` to build the images.
4747
#image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.2-python3.8}
48-
build: .
48+
#build: .
49+
image: meaningfy/airflow:latest
4950
env_file:
5051
- ../../.env
5152
environment:

‎ted_sws/core/model/supra_notice.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
""" This module implements aggregates over groups of notices and the appropriate business needs, on those groups """
99
import abc
10-
from datetime import datetime, time
10+
from datetime import datetime, date
1111
from typing import List, Optional
1212

1313
from ted_sws.core.model import PropertyBaseModel
@@ -45,5 +45,7 @@ class DailySupraNotice(SupraNotice):
4545
"""
4646
This is an aggregate over the notices published in TED in a specific day.
4747
"""
48-
notice_publication_day: datetime = datetime.combine(datetime.today(), time())
48+
notice_publication_date: date
4949
validation_report: Optional[SupraNoticeValidationReport]
50+
51+

‎ted_sws/data_manager/adapters/supra_notice_repository.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from datetime import datetime, time
12
from typing import Optional, Iterator
23

34
from pymongo import MongoClient, ASCENDING
@@ -6,7 +7,7 @@
67
from ted_sws.core.model.supra_notice import DailySupraNotice
78
from ted_sws.data_manager.adapters.repository_abc import DailySupraNoticeRepositoryABC
89

9-
DAILY_SUPRA_NOTICE_ID = "notice_publication_day"
10+
DAILY_SUPRA_NOTICE_ID = "notice_publication_date"
1011

1112

1213
class DailySupraNoticeRepository(DailySupraNoticeRepositoryABC):
@@ -26,6 +27,8 @@ def __init__(self, mongodb_client: MongoClient, database_name: str = _database_n
2627

2728
def _update_daily_supra_notice(self, daily_supra_notice: DailySupraNotice, upsert: bool = False):
2829
daily_supra_notice_dict = daily_supra_notice.dict()
30+
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID] = datetime.combine(
31+
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID], time())
2932
self.collection.update_one({DAILY_SUPRA_NOTICE_ID: daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID]},
3033
{"$set": daily_supra_notice_dict}, upsert=upsert)
3134

@@ -43,19 +46,18 @@ def update(self, daily_supra_notice: DailySupraNotice):
4346
:param daily_supra_notice:
4447
:return:
4548
"""
46-
daily_supra_notice_exist = self.collection.find_one(
47-
{DAILY_SUPRA_NOTICE_ID: daily_supra_notice.notice_publication_day})
48-
if daily_supra_notice_exist is not None:
49-
self._update_daily_supra_notice(daily_supra_notice=daily_supra_notice)
49+
self._update_daily_supra_notice(daily_supra_notice=daily_supra_notice)
5050

5151
def get(self, reference) -> Optional[DailySupraNotice]:
5252
"""
5353
This method allows a daily_supra_notice to be obtained based on an identification reference.
5454
:param reference:
5555
:return: DailySupraNotice
5656
"""
57+
reference = datetime.combine(reference, time())
5758
result_dict = self.collection.find_one({DAILY_SUPRA_NOTICE_ID: reference})
5859
if result_dict is not None:
60+
result_dict[DAILY_SUPRA_NOTICE_ID] = result_dict[DAILY_SUPRA_NOTICE_ID].date()
5961
daily_supra_notice = DailySupraNotice.parse_obj(result_dict)
6062
return daily_supra_notice
6163
return None

‎ted_sws/supra_notice_manager/services/daily_supra_notice_manager.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,21 @@
11
from typing import List
22

33
from pymongo import MongoClient
4-
4+
from datetime import date, datetime, time
55
from ted_sws.core.model.supra_notice import DailySupraNotice
66
from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository
77

88

9-
def create_and_store_in_mongo_db_daily_supra_notice(notice_ids: List[str], mongodb_client: MongoClient):
9+
def create_and_store_in_mongo_db_daily_supra_notice(notice_ids: List[str], mongodb_client: MongoClient,
10+
notice_publication_date: date = date.today()):
1011
"""
1112
This function creates and stores a DailySupraNotice in MongoDB.
1213
:param notice_ids:
1314
:param mongodb_client:
15+
:param notice_publication_date:
1416
:return:
1517
"""
1618
daily_supra_notice_repository = DailySupraNoticeRepository(mongodb_client=mongodb_client)
17-
daily_supra_notice = DailySupraNotice(notice_ids=notice_ids)
19+
daily_supra_notice = DailySupraNotice(notice_publication_date=notice_publication_date,
20+
notice_ids=notice_ids)
1821
daily_supra_notice_repository.add(daily_supra_notice=daily_supra_notice)
19-

‎tests/unit/data_manager/conftest.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from datetime import datetime
1+
from datetime import datetime, date
22

33
import mongomock
44
import pymongo
@@ -46,4 +46,4 @@ def fake_mapping_suite():
4646

4747
@pytest.fixture
4848
def daily_supra_notice():
49-
return DailySupraNotice(notice_ids=["1", "2", "3"])
49+
return DailySupraNotice(notice_ids=["1", "2", "3"], notice_publication_date=date.today())

‎tests/unit/data_manager/test_supra_notice_repository.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,20 @@
66
def test_daily_supra_notice_repository(mongodb_client, daily_supra_notice):
77
daily_supra_notice_repository = DailySupraNoticeRepository(mongodb_client=mongodb_client)
88
daily_supra_notice_repository.add(daily_supra_notice=daily_supra_notice)
9-
result_supra_notice = daily_supra_notice_repository.get(reference=daily_supra_notice.notice_publication_day)
9+
result_supra_notice = daily_supra_notice_repository.get(reference=daily_supra_notice.notice_publication_date)
1010
assert result_supra_notice
1111
assert len(result_supra_notice.notice_ids) == 3
12-
assert result_supra_notice.notice_publication_day == daily_supra_notice.notice_publication_day
12+
assert result_supra_notice.notice_publication_date == daily_supra_notice.notice_publication_date
1313
assert result_supra_notice.created_at == daily_supra_notice.created_at
1414
assert result_supra_notice.notice_ids == daily_supra_notice.notice_ids
1515
daily_supra_notice_repository.update(daily_supra_notice=daily_supra_notice)
16-
result_supra_notice = daily_supra_notice_repository.get(reference=daily_supra_notice.notice_publication_day)
16+
result_supra_notice = daily_supra_notice_repository.get(reference=daily_supra_notice.notice_publication_date)
1717
assert result_supra_notice
1818
assert len(result_supra_notice.notice_ids) == 3
19-
assert result_supra_notice.notice_publication_day == daily_supra_notice.notice_publication_day
19+
assert result_supra_notice.notice_publication_date == daily_supra_notice.notice_publication_date
2020
assert result_supra_notice.created_at == daily_supra_notice.created_at
2121
assert result_supra_notice.notice_ids == daily_supra_notice.notice_ids
2222
result = list(daily_supra_notice_repository.list())
2323
assert len(result) == 1
24-
result = daily_supra_notice_repository.get(reference=datetime.datetime.now())
24+
result = daily_supra_notice_repository.get(reference=datetime.date(2020,1,1))
2525
assert result is None

‎tests/unit/supra_notice_manager/test_daily_supra_notice_manager.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,22 @@
1-
from datetime import datetime, time
1+
from datetime import date
22

3-
from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository
43
from ted_sws.supra_notice_manager.services.daily_supra_notice_manager import \
54
create_and_store_in_mongo_db_daily_supra_notice
65

76

87
def test_daily_supra_notice_manager(mongodb_client, daily_supra_notice_repository):
98
notice_ids = ["1", "2", "3"]
10-
create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client)
11-
result = daily_supra_notice_repository.get(reference=datetime.combine(datetime.today(), time()))
9+
notice_publication_date = date(2020, 1, 1)
10+
create_and_store_in_mongo_db_daily_supra_notice(notice_ids=notice_ids, mongodb_client=mongodb_client,
11+
notice_publication_date=notice_publication_date
12+
)
13+
for result in daily_supra_notice_repository.list():
14+
assert result
15+
result = daily_supra_notice_repository.get(reference=notice_publication_date)
1216
assert result
17+
assert result.notice_publication_date == notice_publication_date
18+
notice_ids.append("4")
19+
result.notice_ids = notice_ids
20+
daily_supra_notice_repository.update(daily_supra_notice=result)
21+
result = daily_supra_notice_repository.get(reference=notice_publication_date)
22+
assert result.notice_ids == notice_ids

0 commit comments

Comments
 (0)