Merge pull request #160 from MITLibraries/TIMX-533-rework-dataset-load

TIMX 533 - Load pyarrow dataset on TIMDEXDataset init

Author: ghukill

README.md

@@ -110,12 +110,6 @@ timdex_dataset = TIMDEXDataset("s3://my-bucket/path/to/dataset")
 
 # or, local dataset (e.g. testing or development)
 timdex_dataset = TIMDEXDataset("/path/to/dataset")
-
-# load the dataset, which discovers all parquet files
-timdex_dataset.load()
-
-# or, load the dataset but ensure that only current records are ever yielded
-timdex_dataset.load(current_records=True)
 ```
 
 All read methods for `TIMDEXDataset` allow for the same group of filters which are defined in `timdex_dataset_api.dataset.DatasetFilters`.  Examples are shown below.

pyproject.toml

@@ -54,7 +54,7 @@ line-length = 90
 [tool.mypy]
 disallow_untyped_calls = true
 disallow_untyped_defs = true
-exclude = ["tests/", "output/"]
+exclude = ["tests/", "output/", "migrations/"]
 
 [[tool.mypy.overrides]]
 module = []
@@ -95,6 +95,8 @@ ignore = [
     "PLR0915",
     "S321",
     "S608",
+    "TD002",
+    "TD003",
     "TRY003"
 ]
 

tests/conftest.py

@@ -82,7 +82,6 @@ def timdex_dataset(tmp_path, timdex_dataset_config) -> TIMDEXDataset:
         ),
         write_append_deltas=False,
     )
-    dataset.load()
     return dataset
 
 
@@ -110,8 +109,6 @@ def timdex_dataset_multi_source(tmp_path) -> TIMDEXDataset:
             ),
             write_append_deltas=False,
         )
-
-    dataset.load()
     return dataset
 
 
@@ -165,8 +162,6 @@ def timdex_dataset_with_runs(tmp_path, timdex_dataset_config_small) -> TIMDEXDat
             ),
             write_append_deltas=False,
         )
-
-    dataset.load()
     return dataset
 
 
@@ -202,8 +197,6 @@ def timdex_dataset_same_day_runs(tmp_path) -> TIMDEXDataset:
             ),
             write_append_deltas=False,
         )
-
-    dataset.load()
     return dataset