Simplify logging setup

Why these changes are being introduced:

As a library that is installed by other applications, it is beneficial to
have this library logging follow the levels and handlers set by the calling
context.  That said, it can also be helpful to set a level like INFO for the
calling context, but DEBUG for this library.

What we definitely want to avoid is this library setting up a handler and
outputting logs, only to have them duplicated in the calling context.

How this addresses that need:
* Remove setting up of handlers; rely on calling context for handlers

Side effects of this change:
* For local development logging statements will not show up without
some kind of manual logging.basicConfig() call or handler setup.
* For calling contexts Transmogrifier, pipeline lambdas, and TIM to show this
library's DEBUG logs, they will need to adjust their configure_logger() functions
to avoid filtering out loggers from different namespaces.

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/TIMX-427

Author: ghukill

tests/test_write.py

@@ -45,7 +45,7 @@ def test_dataset_write_record_batches_uses_batch_size(
     total_records = 101
     batch_size = 50
     batches = list(
-        new_local_dataset.get_dataset_record_batches(
+        new_local_dataset.create_record_batches(
             sample_records_iter(total_records), batch_size=batch_size
         )
     )

timdex_dataset_api/config.py

@@ -3,29 +3,22 @@
 
 
 def configure_logger(name: str) -> logging.Logger:
-    """Prepares and returns a logger instance for a given module name.
+    """Prepares a logger instance.
 
-    This approach is suitable for an installed and imported library such as this, where
-    any calling application logging levels and handlers should be utilized.
+    If the env var TDA_LOG_LEVEL is set, the logging level will override the logging
+    level of the calling context.
 
     Args:
         name (str): The name of the logger, typically __name__ is passed by caller
     """
     logger = logging.getLogger(name)
-    logger.addHandler(logging.NullHandler())
 
-    log_level = os.getenv("TDA_LOG_LEVEL", "INFO").strip().upper()
-    if log_level not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
-        raise ValueError(f"Invalid log level: '{log_level}'")
-    logger.setLevel(getattr(logging, log_level))
-
-    handler = logging.StreamHandler()
-    handler.setFormatter(
-        logging.Formatter(
-            "%(asctime)s %(levelname)s %(name)s.%(funcName)s() "
-            "line %(lineno)d: %(message)s"
-        )
-    )
-    logger.addHandler(handler)
+    # set logger level if env var 'TDA_LOG_LEVEL' is set
+    log_level = os.getenv("TDA_LOG_LEVEL")
+    if log_level:
+        log_level = log_level.strip().upper()
+        if log_level not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
+            raise ValueError(f"Invalid log level: '{log_level}'")
+        logger.setLevel(getattr(logging, log_level))
 
     return logger

timdex_dataset_api/dataset.py

@@ -316,7 +316,7 @@ def write(
                 "Dataset location must be the root of a single dataset for writing"
             )
 
-        record_batches_iter = self.get_dataset_record_batches(
+        record_batches_iter = self.create_record_batches(
             records_iter,
             batch_size=batch_size,
         )
@@ -341,7 +341,7 @@ def write(
         self.log_write_statistics(start_time)
         return self._written_files  # type: ignore[return-value]
 
-    def get_dataset_record_batches(
+    def create_record_batches(
         self,
         records_iter: Iterator["DatasetRecord"],
         *,
@@ -360,14 +360,11 @@ def get_dataset_record_batches(
                 group size in final parquet files
         """
         for i, record_batch in enumerate(itertools.batched(records_iter, batch_size)):
-            batch_start_time = time.perf_counter()
             batch = pa.RecordBatch.from_pylist(
                 [record.to_dict() for record in record_batch]
             )
-            logger.debug(
-                f"Batch {i + 1} yielded for writing, "
-                f"elapsed: {round(time.perf_counter()-batch_start_time, 6)}s"
-            )
+            message = f"Yielding batch {i+1} for dataset writing."
+            logger.debug(message)
             yield batch
 
     def log_write_statistics(self, start_time: float) -> None: