Merge branch 'release/1.7'

Author: gbeced

.github/workflows/runtests.yml

@@ -25,17 +25,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-        # windows-latest due to static image export hanging when using kaleido.
-        os: [ubuntu-latest, macos-13]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        # windows-latest is not enabled due to static image export hanging when using kaleido: https://github.com/plotly/Kaleido/issues/126
+        os: [ubuntu-latest, macos-latest]
 
     runs-on: ${{ matrix.os }}
     name: Test on ${{ matrix.os }} with Python ${{ matrix.python-version }}.
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Update system
@@ -48,7 +48,7 @@ jobs:
         pip install poetry
     - name: Initialize the virtual environment.
       run: |
-        poetry install --no-root --all-extras
+        poetry install --no-root --all-extras --without=docs
     - name: Static checks
       run: |
         poetry run -- mypy basana

CHANGELOG.md

@@ -1,10 +1,26 @@
 # Changelog
 
+## 1.7
+
+### Features
+
+* Added support for Binance order and user data events via websockets.
+* Added support for backtesting order events.
+* Added loan ids to order info.
+
+### Bug fixes
+
+* `bitstamp.orders.Order.amount` was returning the order amount left to be executed instead of the original amount.
+
+### Misc
+
+* Updated dependencies and minimum Python version.
+
 ## 1.6.2
 
 ### Bug fixes
 
-* VolumeShareImpact.calculate_price and VolumeShareImpact.calculate_amount were failing when there was no available liquidity.
+* `VolumeShareImpact.calculate_price` and `VolumeShareImpact.calculate_amount` were failing when there was no available liquidity.
 
 ## 1.6.1
 
@@ -25,7 +41,7 @@
 
 ## 1.5.0
 
-* basana.backtesting.fees.Percentage now supports a minimum fee.
+* `basana.backtesting.fees.Percentage` now supports a minimum fee.
 
 ## 1.4.1
 

DEVELOPERS.md

@@ -2,7 +2,7 @@
 
 ## Requirements
 
-* Python 3.8.1 or greater.
+* Python 3.9 or greater.
 * [Poetry](https://python-poetry.org/) for dependency and package management.
 * Optionally, [Invoke](https://www.pyinvoke.org/).
 

basana/backtesting/charts.py

@@ -94,15 +94,15 @@ def add_traces(self, figure: go.Figure, row: int):
         if self._include_buys:
             x, y = self._get_order_fills(OrderOperation.BUY).get_x_y()
             figure.add_trace(
-                go.Scatter(x=x, y=y, name="Buy", mode="markers", marker=dict(symbol="arrow-up")),
+                go.Scatter(x=x, y=y, name="Buy", mode="markers", marker=dict(symbol="arrow-up", color="green")),
                 row=row, col=1
             )
 
         # Add a trace with sell prices.
         if self._include_sells:
             x, y = self._get_order_fills(OrderOperation.SELL).get_x_y()
             figure.add_trace(
-                go.Scatter(x=x, y=y, name="Sell", mode="markers", marker=dict(symbol="arrow-down")),
+                go.Scatter(x=x, y=y, name="Sell", mode="markers", marker=dict(symbol="arrow-down", color="red")),
                 row=row, col=1
             )
 

basana/backtesting/exchange.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 from decimal import Decimal
-from typing import cast, Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple
+from typing import cast, Callable, Dict, List, Optional, Sequence, Tuple
 import dataclasses
 import logging
 import uuid
@@ -29,9 +29,11 @@
 
 logger = logging.getLogger(__name__)
 
-BarEventHandler = Callable[[bar.BarEvent], Awaitable[Any]]
+BarEventHandler = bar.BarEventHandler
 Error = errors.Error
 LiquidityStrategyFactory = Callable[[], liquidity.LiquidityStrategy]
+OrderEvent = order_mgr.OrderEvent
+OrderEventHandler = order_mgr.OrderEventHandler
 OrderInfo = orders.OrderInfo
 OrderOperation = enums.OrderOperation
 
@@ -93,7 +95,7 @@ class Exchange:
     """
     def __init__(
             self,
-            dispatcher: dispatcher.EventDispatcher,
+            dispatcher: dispatcher.BacktestingDispatcher,
             initial_balances: Dict[str, Decimal],
             liquidity_strategy_factory: LiquidityStrategyFactory = liquidity.VolumeShareImpact,
             fee_strategy: fees.FeeStrategy = fees.NoFee(),
@@ -344,6 +346,14 @@ def subscribe_to_bar_events(self, pair: Pair, event_handler: BarEventHandler):
             self._bar_event_source[pair] = event_source
         self._dispatcher.subscribe(event_source, cast(dispatcher.EventHandler, event_handler))
 
+    def subscribe_to_order_events(self, event_handler: OrderEventHandler):
+        """
+        Registers an async callable that will be called when an order is accepted or updated.
+
+        :param event_handler: The event handler.
+        """
+        self._order_mgr.subscribe_to_order_events(event_handler)
+
     async def get_pair_info(self, pair: Pair) -> PairInfo:
         """
         Returns information about a trading pair.
@@ -427,7 +437,7 @@ async def _on_bar_event(self, event: event.Event):
     def _get_all_orders(self) -> Sequence[orders.Order]:
         return list(self._order_mgr.get_all_orders())
 
-    def _get_dispatcher(self) -> dispatcher.EventDispatcher:
+    def _get_dispatcher(self) -> dispatcher.BacktestingDispatcher:
         return self._dispatcher
 
     def _get_balance(self, symbol: str) -> Balance:

basana/backtesting/lending/base.py

@@ -96,7 +96,7 @@ def calculate_collateral(self, prices: prices.Prices) -> ValueMapDict:
 
 @dataclasses.dataclass
 class ExchangeContext:
-    dispatcher: dispatcher.EventDispatcher
+    dispatcher: dispatcher.BacktestingDispatcher
     account_balances: account_balances.AccountBalances
     prices: prices.Prices
     config: config.Config

basana/backtesting/order_mgr.py

@@ -15,27 +15,30 @@
 # limitations under the License.
 
 from decimal import Decimal
-from typing import cast, Callable, Dict, Generator, Iterable, List, Optional
+from typing import Any, Awaitable, cast, Callable, Dict, Generator, Iterable, List, Optional
 import dataclasses
+import datetime
 import decimal
 import logging
 
 from basana.backtesting import account_balances, config, errors, fees, helpers, lending, loan_mgr, liquidity, prices
-from basana.backtesting.orders import Order
+from basana.backtesting.orders import Order, OrderInfo
 from basana.backtesting.value_map import ValueMap, ValueMapDict
 from basana.core import bar, dispatcher, helpers as core_helpers, logs
 from basana.core.enums import OrderOperation
 from basana.core.pair import Pair
+import basana as bs
 
 
 logger = logging.getLogger(__name__)
 
 LiquidityStrategyFactory = Callable[[], liquidity.LiquidityStrategy]
+OrderEventHandler = Callable[["OrderEvent"], Awaitable[Any]]
 
 
 @dataclasses.dataclass
 class ExchangeContext:
-    dispatcher: dispatcher.EventDispatcher
+    dispatcher: dispatcher.BacktestingDispatcher
     account_balances: account_balances.AccountBalances
     prices: prices.Prices
     fee_strategy: fees.FeeStrategy
@@ -44,12 +47,24 @@ class ExchangeContext:
     config: config.Config
 
 
+class OrderEvent(bs.Event):
+    """
+    An event for order updates.
+    """
+
+    def __init__(self, when: datetime.datetime, order: OrderInfo):
+        super().__init__(when)
+        #: The order.
+        self.order: OrderInfo = order
+
+
 class OrderManager:
     def __init__(self, exchange_ctx: ExchangeContext):
         self._ctx = exchange_ctx
         self._liquidity_strategies: Dict[Pair, liquidity.LiquidityStrategy] = {}
         self._orders = helpers.ExchangeObjectContainer[Order]()
         self._holds_by_order: Dict[str, ValueMap] = {}
+        self._order_updates = core_helpers.LazyProxy(bs.FifoQueueEventSource)
 
     def on_bar_event(self, bar_event: bar.BarEvent):
         if (liquidity_strategy := self._liquidity_strategies.get(bar_event.bar.pair)) is None:
@@ -69,7 +84,12 @@ def add_order(self, order: Order):
                 self._ctx.account_balances.update(hold_updates=required_balances)
                 self._holds_by_order[order.id] = required_balances
 
+            # The order got accepted.
             self._orders.add(order)
+            # Checking dispatcher.now_available is necessary to avoid calling dispatcher.now() when no events have been
+            # processed yet.
+            if self._order_updates.initialized and self._ctx.dispatcher.now_available and self._ctx.dispatcher.now():
+                self._order_updates.push(OrderEvent(self._ctx.dispatcher.now(), order.get_order_info()))
 
         except errors.NotEnoughBalance as e:
             logger.debug(logs.StructuredMessage(
@@ -95,6 +115,14 @@ def cancel_order(self, order_id: str):
         order.cancel()
         self._order_closed(order)
 
+    def subscribe_to_order_events(self, event_handler: OrderEventHandler):
+        """
+        Registers an async callable that will be called when an order is updated.
+
+        :param event_handler: The event handler.
+        """
+        self._ctx.dispatcher.subscribe(self._order_updates.obj, cast(dispatcher.EventHandler, event_handler))
+
     def _update_balances(self, order: Order, balance_updates: ValueMapDict):
         # If we have holds associated with the order, it may be time to release some/all of those.
         hold_updates = {}
@@ -148,31 +176,43 @@ def _borrow(self, required_balances: ValueMap, order: Order):
                 self._ctx.loan_mgr.cancel_loan(loan_id)
             raise
 
-    def _repay_loans(self, symbol: str):
+        # Add loans to order.
+        for loan_id in loan_ids:
+            order.add_loan(loan_id)
+
+    def _repay_loans(self, order: Order):
+        if order.operation == OrderOperation.BUY:
+            credit_symbol = order.pair.base_symbol
+        else:
+            credit_symbol = order.pair.quote_symbol
+
         candidate_loans = [
             loan for loan in self._ctx.loan_mgr.get_loans(is_open=True)
-            if loan.borrowed_symbol == symbol
+            if loan.borrowed_symbol == credit_symbol
         ]
         # Try to cancel bigger loans first.
         candidate_loans.sort(key=lambda loan: loan.borrowed_amount, reverse=True)
+        loan_ids: List[str] = []
         for loan in candidate_loans:
             try:
                 self._ctx.loan_mgr.repay_loan(loan.id)
+                loan_ids.append(loan.id)
                 loan = cast(lending.LoanInfo, self._ctx.loan_mgr.get_loan(loan.id))
                 logger.debug(logs.StructuredMessage("Repayed loan", loan=dataclasses.asdict(loan)))
             except errors.NotEnoughBalance:
                 pass
 
+        # Add loans to order.
+        for loan_id in loan_ids:
+            order.add_loan(loan_id)
+
     def _order_closed(self, order: Order):
         # The order is closed and there might be balances on hold that have to be released.
         self._update_balances(order, {})
-        # If the order has auto_repay set and is filled, either fully or partially, then we need to cancel any open
-        # loans matching the order's base/quote currency.
+        # If the order has auto_repay set and is filled, either fully or partially, then we need to cancel matching open
+        # loans
         if order.auto_repay and order.amount_filled:
-            if order.operation == OrderOperation.BUY:
-                self._repay_loans(order.pair.base_symbol)
-            else:
-                self._repay_loans(order.pair.quote_symbol)
+            self._repay_loans(order)
 
     def _process_order(
             self, order: Order, bar_event: bar.BarEvent, liquidity_strategy: liquidity.LiquidityStrategy
@@ -182,6 +222,9 @@ def order_not_filled():
             logger.debug(logs.StructuredMessage("Order not filled", order_id=order.id, order_state=order.state))
             if not order.is_open:
                 self._order_closed(order)
+                # Push the order event since the order is now closed.
+                if self._order_updates.initialized:
+                    self._order_updates.push(OrderEvent(bar_event.when, order.get_order_info()))
 
         # Calculate balance updates for the current bar.
         logger.debug(logs.StructuredMessage(
@@ -225,6 +268,10 @@ def order_not_filled():
             if not order.is_open:
                 self._order_closed(order)
 
+            # Push the order event since the order got updated.
+            if self._order_updates.initialized:
+                self._order_updates.push(OrderEvent(bar_event.when, order.get_order_info()))
+
         except errors.NotEnoughBalance as e:
             logger.debug(logs.StructuredMessage(
                 "Balance short processing order", order=order.get_debug_info(), error=str(e)

basana/backtesting/orders.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 from decimal import Decimal
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Set
 import abc
 import dataclasses
 import datetime
@@ -60,6 +60,8 @@ class OrderInfo:
     limit_price: Optional[Decimal] = None
     #: The stop price.
     stop_price: Optional[Decimal] = None
+    #: The ids of the associated loans.
+    loan_ids: List[str] = dataclasses.field(default_factory=list)
 
     @property
     def fill_price(self) -> Optional[Decimal]:
@@ -95,6 +97,7 @@ def __init__(
         self._fills: List[Fill] = []
         self._auto_borrow = auto_borrow
         self._auto_repay = auto_repay
+        self._loan_ids: Set[str] = set()
 
     @property
     def id(self) -> str:
@@ -163,12 +166,16 @@ def add_fill(self, when: datetime.datetime, balance_updates: Dict[str, Decimal],
             self._state = OrderState.COMPLETED
         self._fills.append(Fill(when=when, balance_updates=balance_updates, fees=fees))
 
+    def add_loan(self, loan_id: str):
+        self._loan_ids.add(loan_id)
+
     def get_order_info(self) -> OrderInfo:
         return OrderInfo(
             id=self.id, is_open=self._state == OrderState.OPEN, operation=self.operation,
             amount=self.amount, amount_filled=self.amount_filled, amount_remaining=self.amount_pending,
             quote_amount_filled=self.quote_amount_filled,
-            fees={symbol: -amount for symbol, amount in self._fees.items() if amount}
+            fees={symbol: -amount for symbol, amount in self._fees.items() if amount},
+            loan_ids=[loan_id for loan_id in self._loan_ids]
         )
 
     @abc.abstractmethod

basana/core/bar.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 from decimal import Decimal
-from typing import Any, List, Optional, Tuple
+from typing import Any, Awaitable, Callable, List, Optional, Tuple
 import asyncio
 import datetime
 import logging
@@ -163,3 +163,6 @@ async def main(self):
             self._flush(begin, end)
             begin += datetime.timedelta(seconds=self._bar_duration)
             end += datetime.timedelta(seconds=self._bar_duration)
+
+
+BarEventHandler = Callable[[BarEvent], Awaitable[Any]]