VER: Release 0.12.0

See release notes.

Author: nmacholl

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.12.0 - 2023-05-01
+- Added `Live` client for connecting to Databento's live service
+- Upgraded `databento-dbn` to 0.5.0
+- Upgraded `DBNStore` to support mixed schema types to support live data
+- Changed iteration `DBNStore` to return record types from `databento-dbn` instead of numpy arrays
+- Removed `dtype` property from `DBNStore`
+- Removed `record_size` property from `DBNStore`
+- Renamed the `cost` field to `cost_usd` for `batch.submit_job` and `batch.list_jobs` (value now expressed as US dollars)
+- Removed `bad` condition variant from `batch.get_dataset_condition`
+- Added `degraded`, `pending` and `missing` condition variants for `batch.get_dataset_condition`
+- Added `last_modified_date` field to `batch.get_dataset_condition` response
+- Deprecated `SType.PRODUCT_ID` to `SType.INSTRUMENT_ID`
+- Deprecated `SType.NATIVE` to `SType.RAW_SYMBOL`
+- Deprecated `SType.SMART` to `SType.PARENT` and `SType.CONTINUOUS`
+- Removed unused `LiveGateway` enum
+- Removed `STATSTICS` from `Schema` enum
+- Removed `STATUS` from `Schema` enum
+- Removed `GATEWAY_ERROR` from `Schema` enum
+- Removed `SYMBOL_MAPPING` from `Schema` enum
+
 ## 0.11.0 - 2023-04-13
 - Changed `end` and `end_date` to optional to support new forward-fill behaviour
 - Upgraded `zstandard` to 0.20.0

README.md

@@ -31,7 +31,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.7 and
 The minimum dependencies as found in the `requirements.txt` are also listed below:
 - Python (>=3.7)
 - aiohttp (>=3.7.2)
-- databento-dbn (==0.4.3)
+- databento-dbn (==0.5.0)
 - numpy (>=1.17.0)
 - pandas (>=1.1.3)
 - requests (>=2.24.0)

databento/__init__.py

@@ -1,6 +1,7 @@
 import logging
+import warnings
 
-from databento.common import utility
+from databento.common import bentologging
 from databento.common.dbnstore import DBNStore
 from databento.common.enums import (
     Compression,
@@ -9,7 +10,6 @@
     Encoding,
     FeedMode,
     HistoricalGateway,
-    LiveGateway,
     Packaging,
     RecordFlags,
     RollRule,
@@ -26,12 +26,29 @@
 )
 from databento.historical.api import API_VERSION
 from databento.historical.client import Historical
+from databento.live.client import Live
+from databento.live.dbn import DBNRecord, DBNStruct
 from databento.version import __version__  # noqa
+from databento_dbn import (
+    ErrorMsg,
+    ImbalanceMsg,
+    InstrumentDefMsg,
+    MBOMsg,
+    MBP1Msg,
+    MBP10Msg,
+    Metadata,
+    OHLCVMsg,
+    SymbolMappingMsg,
+    SystemMsg,
+    TradeMsg,
+)
 
 
 __all__ = [
     "API_VERSION",
     "DBNStore",
+    "DBNRecord",
+    "DBNStruct",
     "BentoClientError",
     "BentoError",
     "BentoHttpError",
@@ -44,18 +61,33 @@
     "RecordFlags",
     "Historical",
     "HistoricalGateway",
-    "LiveGateway",
+    "Live",
     "Packaging",
     "RollRule",
     "Schema",
     "SplitDuration",
     "SType",
     "SymbologyResolution",
+    # DBN Record Types
+    "Metadata",
+    "MBOMsg",
+    "MBP1Msg",
+    "MBP10Msg",
+    "TradeMsg",
+    "OHLCVMsg",
+    "InstrumentDefMsg",
+    "ImbalanceMsg",
+    "ErrorMsg",
+    "SystemMsg",
+    "SymbolMappingMsg",
 ]
 
 # Setup logging
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
+# Setup deprecation warnings
+warnings.simplefilter("always", DeprecationWarning)
+
 # Convenience imports
-enable_logging = utility.enable_logging
+enable_logging = bentologging.enable_logging
 from_dbn = DBNStore.from_file

databento/common/utility.py databento/common/bentologging.pydatabento/common/utility.py renamed to databento/common/bentologging.py

@@ -0,0 +1,61 @@
+"""Functions for handling challenge-response authentication"""
+import argparse
+import hashlib
+import os
+import sys
+
+
+BUCKET_ID_LENGTH = 5
+
+
+def get_challenge_response(challenge: str, key: str) -> str:
+    """
+    Return the response for a given challenge-response
+    authentication mechanism (CRAM) code provided by
+    a Databento service.
+
+    A valid API key is hashed with the challenge string.
+
+    Parameters
+    ----------
+    challenge : str
+        The CRAM challenge string.
+    key : str
+        The user API key for authentication.
+
+    Returns
+    -------
+    str
+
+    """
+    bucket_id = key[-BUCKET_ID_LENGTH:]
+    sha = hashlib.sha256(f"{challenge}|{key}".encode("utf-8")).hexdigest()
+    return f"{sha}-{bucket_id}"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Script for computing a CRAM response.",
+    )
+    parser.add_argument(
+        "challenge",
+        help="The CRAM challenge string",
+    )
+    parser.add_argument(
+        "key",
+        nargs="?",
+        default=os.environ.get("DATABENTO_API_KEY"),
+        help="An API key; defaults to the value of DATABENTO_API_KEY if set",
+    )
+    arguments = parser.parse_args(sys.argv[1:])
+
+    if arguments.key is None:
+        parser.print_usage()
+        exit(1)
+
+    print(
+        get_challenge_response(
+            challenge=arguments.challenge,
+            key=arguments.key,
+        ),
+    )

databento/common/cram.py

@@ -40,7 +40,7 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     ("length", np.uint8),
     ("rtype", np.uint8),
     ("publisher_id", np.uint16),
-    ("product_id", np.uint32),
+    ("instrument_id", np.uint32),
     ("ts_event", np.uint64),
 ]
 
@@ -78,14 +78,6 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     ("volume", np.int64),
 ]
 
-STATUS_MSG: List[Tuple[str, Union[type, str]]] = RECORD_HEADER + [
-    ("ts_recv", np.uint64),
-    ("group", "S1"),  # 1 byte chararray
-    ("trading_status", np.uint8),
-    ("halt_reason", np.uint8),
-    ("trading_event", np.uint8),
-]
-
 DEFINITION_MSG: List[Tuple[str, Union[type, str]]] = RECORD_HEADER + [
     ("ts_recv", np.uint64),
     ("min_price_increment", np.int64),
@@ -199,23 +191,11 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     Schema.OHLCV_1M: OHLCV_MSG,
     Schema.OHLCV_1H: OHLCV_MSG,
     Schema.OHLCV_1D: OHLCV_MSG,
-    Schema.STATUS: STATUS_MSG,
     Schema.DEFINITION: DEFINITION_MSG,
     Schema.IMBALANCE: IMBALANCE_MSG,
-    Schema.GATEWAY_ERROR: RECORD_HEADER
-    + [
-        ("error", "S64"),
-    ],
-    Schema.SYMBOL_MAPPING: RECORD_HEADER
-    + [
-        ("stype_in_symbol", "S22"),
-        ("stype_out_symbol", "S22"),
-        ("dummy", "S4"),
-        ("start_ts", np.uint64),
-        ("end_ts", np.uint64),
-    ],
 }
 
+
 DEFINITION_CHARARRAY_COLUMNS = [
     "currency",
     "settl_currency",
@@ -271,7 +251,7 @@ def get_deriv_ba_fields(level: int) -> List[str]:
     "ts_event",
     "ts_in_delta",
     "publisher_id",
-    "product_id",
+    "instrument_id",
     "action",
     "side",
     "depth",
@@ -283,15 +263,14 @@ def get_deriv_ba_fields(level: int) -> List[str]:
 
 OHLCV_HEADER_COLUMNS = [
     "publisher_id",
-    "product_id",
+    "instrument_id",
     "open",
     "high",
     "low",
     "close",
     "volume",
 ]
 
-STATUS_DROP_COLUMNS = ["ts_recv"]
 DEFINITION_DROP_COLUMNS = [
     "ts_recv",
     "length",
@@ -309,10 +288,6 @@ def get_deriv_ba_fields(level: int) -> List[str]:
     "dummy",
 ]
 
-STATUS_COLUMNS = [
-    x for x in (np.dtype(STATUS_MSG).names or ()) if x not in STATUS_DROP_COLUMNS
-]
-
 DEFINITION_COLUMNS = [
     x
     for x in (np.dtype(DEFINITION_MSG).names or ())
@@ -323,14 +298,13 @@ def get_deriv_ba_fields(level: int) -> List[str]:
     x for x in (np.dtype(IMBALANCE_MSG).names or ()) if x not in IMBALANCE_DROP_COLUMNS
 ]
 
-
 COLUMNS = {
     Schema.MBO: [
         "ts_event",
         "ts_in_delta",
         "publisher_id",
         "channel_id",
-        "product_id",
+        "instrument_id",
         "order_id",
         "action",
         "side",
@@ -357,7 +331,6 @@ def get_deriv_ba_fields(level: int) -> List[str]:
     Schema.OHLCV_1M: OHLCV_HEADER_COLUMNS,
     Schema.OHLCV_1H: OHLCV_HEADER_COLUMNS,
     Schema.OHLCV_1D: OHLCV_HEADER_COLUMNS,
-    Schema.STATUS: STATUS_COLUMNS,
     Schema.DEFINITION: DEFINITION_COLUMNS,
     Schema.IMBALANCE: IMBALANCE_COLUMNS,
 }