Always prefix with source in test util

Author: ghukill

tests/conftest.py

@@ -62,7 +62,6 @@ def fixed_local_dataset(tmp_path) -> TIMDEXDataset:
         timdex_dataset.write(
             generate_sample_records(
                 num_records=1_000,
-                timdex_record_id_prefix=source,
                 source=source,
                 run_date="2024-12-01",
                 run_id=run_id,
@@ -82,19 +81,6 @@ def _records_iter(num_records):
     return _records_iter
 
 
-@pytest.fixture
-def sample_records_iter_without_partitions():
-    """Simulates an iterator of X number of DatasetRecord instances WITHOUT partition
-    values included."""
-
-    def _records_iter(num_records):
-        return generate_sample_records(
-            num_records, run_date="invalid run-date", year=None, month=None, day=None
-        )
-
-    return _records_iter
-
-
 @pytest.fixture
 def dataset_with_runs_location(tmp_path) -> str:
     """Fixture to simulate a dataset with multiple full and daily ETL runs."""
@@ -139,7 +125,6 @@ def dataset_with_runs_location(tmp_path) -> str:
         num_records, source, run_date, run_type, action, run_id = params
         records = generate_sample_records(
             num_records,
-            timdex_record_id_prefix=source,
             source=source,
             run_date=run_date,
             run_type=run_type,
@@ -195,7 +180,6 @@ def dataset_with_same_day_runs(tmp_path) -> TIMDEXDataset:
         num_records, source, run_date, run_type, action, run_id, run_timestamp = params
         records = generate_sample_records(
             num_records,
-            timdex_record_id_prefix=source,
             source=source,
             run_date=run_date,
             run_type=run_type,

tests/test_write.py

@@ -98,9 +98,7 @@ def test_dataset_write_partition_for_multiple_sources(
 
     # perform write for source="libguides" and run_date="2024-12-01"
     written_files_source_b = new_local_dataset.write(
-        generate_sample_records(
-            num_records=7, timdex_record_id_prefix="libguides", source="libguides"
-        )
+        generate_sample_records(num_records=7, source="libguides")
     )
     new_local_dataset.load()
 

tests/utils.py

@@ -11,7 +11,6 @@
 
 def generate_sample_records(
     num_records: int,
-    timdex_record_id_prefix: str = "alma",
     source: str | None = "alma",
     run_date: str | None = "2024-12-01",
     run_type: str | None = "daily",
@@ -25,7 +24,7 @@ def generate_sample_records(
 
     for x in range(num_records):
         yield DatasetRecord(
-            timdex_record_id=f"{timdex_record_id_prefix}:{x}",
+            timdex_record_id=f"{source}:{x}",
             source_record=b"<record><title>Hello World.</title></record>",
             transformed_record=b"""{"title":["Hello World."]}""",
             source=source,
@@ -53,7 +52,6 @@ def generate_sample_records_with_simulated_partitions(
         source = random.choice(sources)
         yield from generate_sample_records(
             num_records=batch_size,
-            timdex_record_id_prefix=source,
             source=source,
             run_date=random.choice(run_dates),
             run_type=random.choice(run_types),