Rename metadata rebuild method and improve refresh methods

Why these changes are being introduced:

The former TIMDEXDatasetMetadata method name recreate_static_database_file()
was too narrowly focused.  This method is responsible for rebuilding the
entire dataset metadata structure.

How this addresses that need:
* Renames method
* Updates refresh() methods on both TIMDEXDataset and TIMDEXDatasetMetadata
to be more fully inclusive

Side effects of this change:
* None

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/TIMX-543

Author: ghukill

tests/conftest.py

@@ -113,7 +113,7 @@ def timdex_dataset_multi_source(tmp_path_factory) -> TIMDEXDataset:
         )
 
     # ensure static metadata database exists for read methods
-    dataset.metadata.recreate_static_database_file()
+    dataset.metadata.rebuild_dataset_metadata()
     dataset.metadata.refresh()
 
     return dataset
@@ -223,7 +223,7 @@ def timdex_dataset_same_day_runs(tmp_path) -> TIMDEXDataset:
 def timdex_metadata(timdex_dataset_with_runs) -> TIMDEXDatasetMetadata:
     """TIMDEXDatasetMetadata with static database file created."""
     metadata = TIMDEXDatasetMetadata(timdex_dataset_with_runs.location)
-    metadata.recreate_static_database_file()
+    metadata.rebuild_dataset_metadata()
     metadata.refresh()
     return metadata
 
@@ -233,7 +233,7 @@ def timdex_dataset_with_runs_with_metadata(
     timdex_dataset_with_runs,
 ) -> TIMDEXDataset:
     """TIMDEXDataset with runs and static metadata created for read tests."""
-    timdex_dataset_with_runs.metadata.recreate_static_database_file()
+    timdex_dataset_with_runs.metadata.rebuild_dataset_metadata()
     timdex_dataset_with_runs.metadata.refresh()
     return timdex_dataset_with_runs
 

tests/test_metadata.py

@@ -43,7 +43,7 @@ def test_tdm_s3_dataset_structure_properties(s3_bucket_mocked):
 
 def test_tdm_create_metadata_database_file_success(caplog, timdex_metadata_empty):
     caplog.set_level("DEBUG")
-    timdex_metadata_empty.recreate_static_database_file()
+    timdex_metadata_empty.rebuild_dataset_metadata()
 
 
 def test_tdm_init_metadata_file_found_success(timdex_metadata):

tests/test_read.py

@@ -254,7 +254,7 @@ def test_dataset_load_current_records_gets_correct_same_day_full_run(
     timdex_dataset_same_day_runs,
 ):
     # ensure metadata exists for this dataset
-    timdex_dataset_same_day_runs.metadata.recreate_static_database_file()
+    timdex_dataset_same_day_runs.metadata.rebuild_dataset_metadata()
     timdex_dataset_same_day_runs.metadata.refresh()
     df = timdex_dataset_same_day_runs.read_dataframe(
         table="current_records", run_type="full"
@@ -265,7 +265,7 @@ def test_dataset_load_current_records_gets_correct_same_day_full_run(
 def test_dataset_load_current_records_gets_correct_same_day_daily_runs_ordering(
     timdex_dataset_same_day_runs,
 ):
-    timdex_dataset_same_day_runs.metadata.recreate_static_database_file()
+    timdex_dataset_same_day_runs.metadata.rebuild_dataset_metadata()
     timdex_dataset_same_day_runs.metadata.refresh()
     first_record = next(
         timdex_dataset_same_day_runs.read_dicts_iter(

timdex_dataset_api/dataset.py

@@ -143,6 +143,10 @@ def location_scheme(self) -> Literal["file", "s3"]:
     def data_records_root(self) -> str:
         return f"{self.location.removesuffix('/')}/data/records"  # type: ignore[union-attr]
 
+    def refresh(self) -> None:
+        """Fully reload TIMDEXDataset instance."""
+        self.__init__(self.location)  # type: ignore[misc]
+
     def create_data_structure(self) -> None:
         """Ensure ETL records data structure exists in TIMDEX dataset."""
         if self.location_scheme == "file":

timdex_dataset_api/metadata.py

@@ -249,16 +249,14 @@ def refresh(self) -> None:
         self.conn = self.setup_duckdb_context()
         self._sa_metadata = sa_reflect_duckdb_conn(self.conn, schema="metadata")
 
-    def recreate_static_database_file(self) -> None:
-        """Create/recreate the static metadata database file.
-
-        The following work is performed:
-            1. Create a local working directory
-            2. Open a DuckDB connection with a database file in this local working dir
-            3. Create tables and views by scanning ETL data in dataset/data/records
-            4. Close DuckDB connection ensuring a fully formed, local database file
-            5. Upload DuckDB database file to target destination, making that the new
-            static metadata database file
+    def rebuild_dataset_metadata(self) -> None:
+        """Fully rebuild dataset metadata.
+
+        Work includes:
+            - remove any append deltas, understanding a full metadata rebuild
+                will pickup that data from the ETL records themselves
+            - build a local, temporary static metadata database file, then overwrite the
+                canonical version in the dataset (e.g. in S3)
         """
         if self.location_scheme == "s3":
             s3_client = S3Client()
@@ -272,7 +270,6 @@ def recreate_static_database_file(self) -> None:
 
             with duckdb.connect(local_db_path) as conn:
                 self.configure_duckdb_connection(conn)
-                conn.execute("""SET threads = 64;""")
 
                 self._create_full_dataset_table(conn)
 
@@ -299,6 +296,9 @@ def _create_full_dataset_table(self, conn: DuckDBPyConnection) -> None:
         start_time = time.perf_counter()
         logger.info("creating table of full dataset metadata")
 
+        # temporarily increase thread count
+        conn.execute("""SET threads = 64;""")
+
         query = f"""
             create or replace table records as (
                 select
@@ -312,6 +312,9 @@ def _create_full_dataset_table(self, conn: DuckDBPyConnection) -> None:
             """
         conn.execute(query)
 
+        # reset thread count
+        conn.execute(f"""SET threads = {self.config.duckdb_connection_threads};""")
+
         row_count = conn.query("""select count(*) from records;""").fetchone()[0]  # type: ignore[index]
         logger.info(
             f"'records' table created - rows: {row_count}, "