VER: Release 0.19.0

See release notes.

Author: nmacholl

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 0.19.0 - 2023-08-25
+
+This release includes improvements to handling large DBN data and adds support for future datasets.
+
+#### Enhancements
+- Added `count` parameter to `DBNStore.to_df` and `DBNStore.to_ndarray` to help process large files incrementally
+- Improved memory usage of `DBNStore.to_csv` and `DBNStore.to_json`
+- Added the `Publisher`, `Venue`, and `Dataset` enums
+- Replace null prices with `NaN` when `pretty_px=True` in `DBNStore.to_df()`
+- Upgraded `databento-dbn` to 0.8.3
+
+#### Bug fixes
+- Fixed issue where exception messages were displaying JSON encoded data
+- Fixed typo in `BATY.PITCH.BATY` publisher
+- Reduced floating error when converting prices to floats with `pretty_px=True`
+
+#### Breaking changes
+- `DBNStore.to_df` now always utf-8 decodes string fields
+
 ## 0.18.1 - 2023-08-16
 
 #### Bug fixes

README.md

@@ -32,7 +32,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.8 and
 The minimum dependencies as found in the `pyproject.toml` are also listed below:
 - python = "^3.8"
 - aiohttp = "^3.8.3"
-- databento-dbn = "0.8.2"
+- databento-dbn = "0.8.3"
 - numpy= ">=1.23.5"
 - pandas = ">=1.5.3"
 - requests = ">=2.24.0"

databento/__init__.py

@@ -20,7 +20,6 @@
 
 from databento.common import bentologging
 from databento.common.dbnstore import DBNStore
-from databento.common.enums import Dataset
 from databento.common.enums import Delivery
 from databento.common.enums import FeedMode
 from databento.common.enums import HistoricalGateway
@@ -33,6 +32,9 @@
 from databento.common.error import BentoError
 from databento.common.error import BentoHttpError
 from databento.common.error import BentoServerError
+from databento.common.publishers import Dataset
+from databento.common.publishers import Publisher
+from databento.common.publishers import Venue
 from databento.historical.api import API_VERSION
 from databento.historical.client import Historical
 from databento.live import DBNRecord
@@ -49,7 +51,9 @@
     "BentoHttpError",
     "BentoServerError",
     "Compression",
+    "Publisher",
     "Dataset",
+    "Venue",
     "Delivery",
     "Encoding",
     "FeedMode",

databento/common/data.py

@@ -14,6 +14,12 @@
 from databento.live import DBNRecord
 
 
+DEFINITION_TYPE_MAX_MAP = {
+    x[0]: np.iinfo(x[1]).max
+    for x in InstrumentDefMsg._dtypes
+    if not isinstance(x[1], str)
+}
+
 SCHEMA_STRUCT_MAP: dict[Schema, type[DBNRecord]] = {
     Schema.DEFINITION: InstrumentDefMsg,
     Schema.IMBALANCE: ImbalanceMsg,
@@ -51,42 +57,6 @@
     Schema.STATISTICS: StatMsg._dtypes,
 }
 
-DEFINITION_CHARARRAY_COLUMNS = [
-    "currency",
-    "settl_currency",
-    "secsubtype",
-    "raw_symbol",
-    "group",
-    "exchange",
-    "asset",
-    "cfi",
-    "security_type",
-    "unit_of_measure",
-    "underlying",
-    "strike_price_currency",
-    "instrument_class",
-    "match_algorithm",
-    "security_update_action",
-    "user_defined_instrument",
-]
-
-DEFINITION_PRICE_COLUMNS = [
-    "min_price_increment",
-    "high_limit_price",
-    "low_limit_price",
-    "max_price_variation",
-    "trading_reference_price",
-    "min_price_increment_amount",
-    "price_ratio",
-    "strike_price",
-]
-
-DEFINITION_TYPE_MAX_MAP = {
-    x[0]: np.iinfo(x[1]).max
-    for x in InstrumentDefMsg._dtypes
-    if not isinstance(x[1], str)
-}
-
 SCHEMA_COLUMNS = {
     Schema.MBO: MBOMsg._ordered_fields,
     Schema.MBP_1: MBP1Msg._ordered_fields,