Skip to content

Commit 8d4c3eb

Browse files
duprijilduprijil
committed
Update tests to not fail and make better var naming
1 parent ed05bcb commit 8d4c3eb

5 files changed

Lines changed: 62 additions & 69 deletions

File tree

‎dags/fetch_notices_by_date_range.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from dags import DEFAULT_DAG_ARGUMENTS
1010
from dags.dags_utils import get_dag_param
1111
from dags.fetch_notices_by_date import WILD_CARD_DAG_KEY, TRIGGER_COMPLETE_WORKFLOW_DAG_KEY, \
12-
DAG_NAME as FETCH_NOTICES_BY_DATE_DAG_NAME
12+
FETCHER_DAG_NAME as FETCH_NOTICES_BY_DATE_DAG_NAME
1313
from ted_sws.event_manager.adapters.event_log_decorator import event_log
1414
from ted_sws.event_manager.model.event_message import TechnicalEventMessage, EventMessageMetadata, \
1515
EventMessageProcessType

‎tests/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
TEST_DATA_PATH = TESTS_PATH / 'test_data'
2121

22+
AIRFLOW_DAG_FOLDER = TESTS_PATH.parent.resolve() / "dags"
23+
2224

2325
class temporary_copy(object):
2426
"""

‎tests/unit/dags/conftest.py‎

Lines changed: 18 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,30 @@
1-
import logging
2-
import os
3-
4-
from airflow.exceptions import AirflowTimetableInvalid
5-
from airflow.models import DagBag
1+
import pytest
2+
from airflow.models import DagBag, Variable
63
from airflow.timetables.trigger import CronTriggerTimetable
7-
from airflow.utils import db
8-
from psutil.tests import pytest
94

105
from dags.daily_materialized_views_update import DAILY_MATERIALISED_VIEWS_DAG_NAME
116
from dags.fetch_notices_by_date import FETCHER_DAG_NAME
12-
from tests import TESTS_PATH
13-
14-
AIRFLOW_DAG_FOLDER = TESTS_PATH.parent.resolve() / "dags"
15-
16-
17-
@pytest.fixture()
18-
def dag_bag():
19-
os.environ["AIRFLOW_HOME"] = str(AIRFLOW_DAG_FOLDER)
20-
os.environ["AIRFLOW__CORE__LOAD_EXAMPLES"] = "False"
21-
# Initialising the Airflow DB so that it works properly with the new AIRFLOW_HOME
22-
logging.disable(logging.CRITICAL)
23-
db.resetdb()
24-
db.initdb()
25-
logging.disable(logging.NOTSET)
26-
dag_bag = DagBag(dag_folder=AIRFLOW_DAG_FOLDER, include_examples=False,
27-
read_dags_from_db=False)
28-
return dag_bag
7+
from tests import AIRFLOW_DAG_FOLDER
298

309

31-
# @pytest.fixture
32-
# def dag_bag():
33-
# return DagBag(dag_folder=AIRFLOW_DAG_FOLDER, include_examples=False, read_dags_from_db=False)
10+
@pytest.fixture
11+
def dag_bag(dag_materialised_view_update_schedule_variable_name, dag_fetch_schedule_variable_name) -> DagBag:
12+
Variable.delete(key=dag_materialised_view_update_schedule_variable_name)
13+
Variable.delete(key=dag_fetch_schedule_variable_name)
14+
return DagBag(
15+
dag_folder=AIRFLOW_DAG_FOLDER,
16+
include_examples=False,
17+
read_dags_from_db=False,
18+
collect_dags=True)
3419

3520

3621
@pytest.fixture
37-
def fetcher_dag_id():
22+
def fetcher_dag_name() -> str:
3823
return FETCHER_DAG_NAME
3924

25+
4026
@pytest.fixture
41-
def daily_materialised_views_dag_id():
27+
def daily_materialised_views_dag_id() -> str:
4228
return DAILY_MATERIALISED_VIEWS_DAG_NAME
4329

4430

@@ -49,7 +35,7 @@ def example_cron_table() -> str:
4935

5036
@pytest.fixture
5137
def example_wrong_cron_table() -> str:
52-
return "1234"
38+
return "wrong_cron"
5339

5440

5541
@pytest.fixture
@@ -58,9 +44,8 @@ def example_dag_cron_table(example_cron_table) -> CronTriggerTimetable:
5844

5945

6046
@pytest.fixture
61-
def airflow_timetable_import_error_name() -> str:
62-
return AirflowTimetableInvalid.__name__
63-
47+
def airflow_timetable_import_error_message() -> str:
48+
return "FormatException"
6449

6550

6651
@pytest.fixture

‎tests/unit/dags/test_daily_materialised_views_update_schedule.py‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,45 +11,44 @@ def test_daily_materialised_view_change_timetable_from_airflow_variable_after_re
1111
dag_materialised_view_update_schedule_variable_name: str,
1212
daily_materialised_views_dag_id: str,
1313
example_dag_cron_table: CronTriggerTimetable,
14-
airflow_timetable_import_error_name: str):
14+
airflow_timetable_import_error_message: str):
1515
daily_materialised_view_dag: DAG = dag_bag.get_dag(dag_id=daily_materialised_views_dag_id)
16+
1617
assert daily_materialised_view_dag is not None
1718
assert daily_materialised_view_dag.schedule_interval != example_dag_cron_table._expression
1819

1920
Variable.set(key=dag_materialised_view_update_schedule_variable_name, value=example_dag_cron_table._expression)
2021
dag_bag.collect_dags(only_if_updated=False)
21-
2222
daily_materialised_view_dag: DAG = dag_bag.get_dag(dag_id=daily_materialised_views_dag_id)
23+
2324
assert daily_materialised_view_dag is not None
2425
assert daily_materialised_view_dag.schedule_interval == example_dag_cron_table._expression
25-
26-
assert all(airflow_timetable_import_error_name not in error for error in dag_bag.import_errors.values())
26+
assert all(airflow_timetable_import_error_message not in error for error in dag_bag.import_errors.values())
2727

2828

2929
def test_daily_materialised_view_change_timetable_from_env_variable_after_reparse(dag_bag: DagBag,
3030
dag_materialised_view_update_schedule_variable_name: str,
3131
daily_materialised_views_dag_id: str,
3232
example_dag_cron_table: CronTriggerTimetable,
33-
airflow_timetable_import_error_name: str):
33+
airflow_timetable_import_error_message: str):
3434
fetcher_dag: DAG = dag_bag.get_dag(dag_id=daily_materialised_views_dag_id)
35+
3536
assert fetcher_dag is not None
3637
assert fetcher_dag.schedule_interval != example_dag_cron_table._expression
3738

3839
os.environ[dag_materialised_view_update_schedule_variable_name] = example_dag_cron_table._expression
3940
dag_bag.collect_dags(only_if_updated=False)
40-
4141
fetcher_dag: DAG = dag_bag.get_dag(dag_id=daily_materialised_views_dag_id)
42+
4243
assert fetcher_dag is not None
4344
assert fetcher_dag.schedule_interval == example_dag_cron_table._expression
44-
45-
assert all(airflow_timetable_import_error_name not in error for error in dag_bag.import_errors.values())
46-
del os.environ[dag_materialised_view_update_schedule_variable_name]
45+
assert all(airflow_timetable_import_error_message not in error for error in dag_bag.import_errors.values())
4746

4847

4948
def test_daily_materialised_view_has_default_timetable_if_no_variable_is_set_after_reparse(dag_bag: DagBag,
5049
dag_materialised_view_update_schedule_variable_name: str,
5150
daily_materialised_views_dag_id: str,
52-
airflow_timetable_import_error_name: str):
51+
airflow_timetable_import_error_message: str):
5352
env_var_value = os.getenv(dag_materialised_view_update_schedule_variable_name)
5453
is_env_var_set: bool = True if env_var_value is not None else False
5554
if is_env_var_set:
@@ -59,11 +58,12 @@ def test_daily_materialised_view_has_default_timetable_if_no_variable_is_set_aft
5958
if is_airflow_var_set:
6059
Variable.delete(key=dag_materialised_view_update_schedule_variable_name)
6160

62-
dag_bag.collect_dags()
61+
dag_bag.collect_dags(only_if_updated=False)
6362
fetcher_dag: DAG = dag_bag.get_dag(dag_id=daily_materialised_views_dag_id)
63+
6464
assert fetcher_dag is not None
6565
assert fetcher_dag.schedule_interval == DAG_MATERIALIZED_VIEW_UPDATE_DEFAULT_TIMETABLE
66-
assert all(airflow_timetable_import_error_name not in error for error in dag_bag.import_errors.values())
66+
assert all(airflow_timetable_import_error_message not in error for error in dag_bag.import_errors.values())
6767

6868
if is_env_var_set:
6969
os.environ[dag_materialised_view_update_schedule_variable_name] = env_var_value
@@ -75,11 +75,13 @@ def test_daily_materialised_view_gets_incorrect_timetable_after_reparse(dag_bag:
7575
dag_materialised_view_update_schedule_variable_name: str,
7676
daily_materialised_views_dag_id: str,
7777
example_wrong_cron_table: str,
78-
airflow_timetable_import_error_name: str):
78+
airflow_timetable_import_error_message: str):
7979
fetcher_dag: DAG = dag_bag.get_dag(dag_id=daily_materialised_views_dag_id)
80+
8081
assert fetcher_dag is not None
8182

8283
Variable.set(key=dag_materialised_view_update_schedule_variable_name, value=example_wrong_cron_table)
84+
8385
dag_bag.collect_dags(only_if_updated=False)
8486

85-
assert any(airflow_timetable_import_error_name in error for error in dag_bag.import_errors.values())
87+
assert any(airflow_timetable_import_error_message in error for error in dag_bag.import_errors.values())

‎tests/unit/dags/test_fetch_notices_by_date_schedule.py‎

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,48 +8,50 @@
88

99

1010
def test_fetcher_change_timetable_from_airflow_variable_after_reparse(dag_bag: DagBag,
11-
dag_fetch_schedule_variable_name,
12-
fetcher_dag_id: str,
11+
dag_fetch_schedule_variable_name: str,
12+
fetcher_dag_name: str,
1313
example_dag_cron_table: CronTriggerTimetable,
14-
airflow_timetable_import_error_name: str):
15-
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_id)
14+
airflow_timetable_import_error_message: str):
15+
dag_bag.collect_dags(only_if_updated=False)
16+
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_name)
17+
1618
assert fetcher_dag is not None
1719
assert fetcher_dag.schedule_interval != example_dag_cron_table._expression
1820

1921
Variable.set(key=dag_fetch_schedule_variable_name, value=example_dag_cron_table._expression)
2022
dag_bag.collect_dags(only_if_updated=False)
23+
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_name)
2124

22-
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_id)
2325
assert fetcher_dag is not None
2426
assert fetcher_dag.schedule_interval == example_dag_cron_table._expression
25-
26-
assert all(airflow_timetable_import_error_name not in error for error in dag_bag.import_errors.values())
27+
assert all(airflow_timetable_import_error_message not in error for error in dag_bag.import_errors.values())
2728

2829

2930
def test_fetcher_change_timetable_from_env_variable_after_reparse(dag_bag: DagBag,
3031
dag_fetch_schedule_variable_name: str,
31-
fetcher_dag_id: str,
32+
fetcher_dag_name: str,
3233
example_dag_cron_table: CronTriggerTimetable,
33-
airflow_timetable_import_error_name: str):
34-
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_id)
34+
airflow_timetable_import_error_message: str):
35+
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_name)
36+
3537
assert fetcher_dag is not None
3638
assert fetcher_dag.schedule_interval != example_dag_cron_table._expression
3739

3840
os.environ[dag_fetch_schedule_variable_name] = example_dag_cron_table._expression
3941
dag_bag.collect_dags(only_if_updated=False)
42+
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_name)
4043

41-
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_id)
4244
assert fetcher_dag is not None
4345
assert fetcher_dag.schedule_interval == example_dag_cron_table._expression
46+
assert all(airflow_timetable_import_error_message not in error for error in dag_bag.import_errors.values())
4447

45-
assert all(airflow_timetable_import_error_name not in error for error in dag_bag.import_errors.values())
4648
del os.environ[dag_fetch_schedule_variable_name]
4749

4850

4951
def test_fetcher_has_default_timetable_if_no_variable_is_set_after_reparse(dag_bag: DagBag,
5052
dag_fetch_schedule_variable_name: str,
51-
fetcher_dag_id: str,
52-
airflow_timetable_import_error_name: str):
53+
fetcher_dag_name: str,
54+
airflow_timetable_import_error_message: str):
5355
env_var_value = os.getenv(dag_fetch_schedule_variable_name)
5456
is_env_var_set: bool = True if env_var_value is not None else False
5557
if is_env_var_set:
@@ -59,11 +61,12 @@ def test_fetcher_has_default_timetable_if_no_variable_is_set_after_reparse(dag_b
5961
if is_airflow_var_set:
6062
Variable.delete(key=dag_fetch_schedule_variable_name)
6163

62-
dag_bag.collect_dags()
63-
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_id)
64+
dag_bag.collect_dags(only_if_updated=False)
65+
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_name)
66+
6467
assert fetcher_dag is not None
6568
assert fetcher_dag.schedule_interval == DAG_FETCH_DEFAULT_TIMETABLE
66-
assert all(airflow_timetable_import_error_name not in error for error in dag_bag.import_errors.values())
69+
assert all(airflow_timetable_import_error_message not in error for error in dag_bag.import_errors.values())
6770

6871
if is_env_var_set:
6972
os.environ[dag_fetch_schedule_variable_name] = env_var_value
@@ -73,13 +76,14 @@ def test_fetcher_has_default_timetable_if_no_variable_is_set_after_reparse(dag_b
7376

7477
def test_fetcher_gets_incorrect_timetable_after_reparse(dag_bag: DagBag,
7578
dag_fetch_schedule_variable_name: str,
76-
fetcher_dag_id: str,
79+
fetcher_dag_name: str,
7780
example_wrong_cron_table: str,
78-
airflow_timetable_import_error_name: str):
79-
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_id)
81+
airflow_timetable_import_error_message: str):
82+
fetcher_dag: DAG = dag_bag.get_dag(dag_id=fetcher_dag_name)
83+
8084
assert fetcher_dag is not None
8185

8286
Variable.set(key=dag_fetch_schedule_variable_name, value=example_wrong_cron_table)
8387
dag_bag.collect_dags(only_if_updated=False)
8488

85-
assert any(airflow_timetable_import_error_name in error for error in dag_bag.import_errors.values())
89+
assert any(airflow_timetable_import_error_message in error for error in dag_bag.import_errors.values())

0 commit comments

Comments
 (0)