Lean  $LEAN_TAG$
SecurityTransactionManager.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14 */
15 
16 using System;
17 using System.Collections.Generic;
18 using System.Linq;
19 using System.Threading;
21 using QuantConnect.Logging;
22 using QuantConnect.Orders;
23 using Python.Runtime;
24 
26 {
27  /// <summary>
28  /// Algorithm Transactions Manager - Recording Transactions
29  /// </summary>
31  {
32  private class TransactionRecordEntry
33  {
34  public decimal ProfitLoss;
35  public bool IsWin;
36  }
37 
38  private readonly Dictionary<DateTime, TransactionRecordEntry> _transactionRecord;
39  private readonly IAlgorithm _algorithm;
40  private int _orderId;
41  private int _groupOrderManagerId;
42  private readonly SecurityManager _securities;
43  private TimeSpan _marketOrderFillTimeout = TimeSpan.MinValue;
44 
45  private IOrderProcessor _orderProcessor;
46 
47  /// <summary>
48  /// Gets the time the security information was last updated
49  /// </summary>
50  public DateTime UtcTime
51  {
52  get { return _securities.UtcTime; }
53  }
54 
55  /// <summary>
56  /// Initialise the transaction manager for holding and processing orders.
57  /// </summary>
59  {
60  _algorithm = algorithm;
61 
62  //Private reference for processing transactions
63  _securities = security;
64 
65  //Internal storage for transaction records:
66  _transactionRecord = new Dictionary<DateTime, TransactionRecordEntry>();
67  }
68 
69  /// <summary>
70  /// Trade record of profits and losses for each trade statistics calculations
71  /// </summary>
72  /// <remarks>Will return a shallow copy, modifying the returned container
73  /// will have no effect <see cref="AddTransactionRecord"/></remarks>
74  public Dictionary<DateTime, decimal> TransactionRecord
75  {
76  get
77  {
78  lock (_transactionRecord)
79  {
80  return _transactionRecord.ToDictionary(x => x.Key, x => x.Value.ProfitLoss);
81  }
82  }
83  }
84 
85  /// <summary>
86  /// Gets the number or winning transactions
87  /// </summary>
88  public int WinCount
89  {
90  get
91  {
92  lock (_transactionRecord)
93  {
94  return _transactionRecord.Values.Count(x => x.IsWin);
95  }
96  }
97  }
98 
99  /// <summary>
100  /// Gets the number of losing transactions
101  /// </summary>
102  public int LossCount
103  {
104  get
105  {
106  lock (_transactionRecord)
107  {
108  return _transactionRecord.Values.Count(x => !x.IsWin);
109  }
110  }
111  }
112 
113  /// <summary>
114  /// Trade record of profits and losses for each trade statistics calculations that are considered winning trades
115  /// </summary>
116  public Dictionary<DateTime, decimal> WinningTransactions
117  {
118  get
119  {
120  lock (_transactionRecord)
121  {
122  return _transactionRecord.Where(x => x.Value.IsWin).ToDictionary(x => x.Key, x => x.Value.ProfitLoss);
123  }
124  }
125  }
126 
127  /// <summary>
128  /// Trade record of profits and losses for each trade statistics calculations that are considered losing trades
129  /// </summary>
130  public Dictionary<DateTime, decimal> LosingTransactions
131  {
132  get
133  {
134  lock (_transactionRecord)
135  {
136  return _transactionRecord.Where(x => !x.Value.IsWin).ToDictionary(x => x.Key, x => x.Value.ProfitLoss);
137  }
138  }
139  }
140 
141  /// <summary>
142  /// Configurable minimum order value to ignore bad orders, or orders with unrealistic sizes
143  /// </summary>
144  /// <remarks>Default minimum order size is $0 value</remarks>
145  [Obsolete("MinimumOrderSize is obsolete and will not be used, please use Settings.MinimumOrderMarginPortfolioPercentage instead")]
146  public decimal MinimumOrderSize { get; }
147 
148  /// <summary>
149  /// Configurable minimum order size to ignore bad orders, or orders with unrealistic sizes
150  /// </summary>
151  /// <remarks>Default minimum order size is 0 shares</remarks>
152  [Obsolete("MinimumOrderQuantity is obsolete and will not be used, please use Settings.MinimumOrderMarginPortfolioPercentage instead")]
153  public int MinimumOrderQuantity { get; }
154 
155  /// <summary>
156  /// Get the last order id.
157  /// </summary>
158  public int LastOrderId
159  {
160  get
161  {
162  return _orderId;
163  }
164  }
165 
166  /// <summary>
167  /// Configurable timeout for market order fills
168  /// </summary>
169  /// <remarks>Default value is 5 seconds</remarks>
170  public TimeSpan MarketOrderFillTimeout
171  {
172  get
173  {
174  return _marketOrderFillTimeout;
175  }
176  set
177  {
178  _marketOrderFillTimeout = value;
179  }
180  }
181 
182  /// <summary>
183  /// Processes the order request
184  /// </summary>
185  /// <param name="request">The request to be processed</param>
186  /// <returns>The order ticket for the request</returns>
188  {
189  if (_algorithm != null && _algorithm.IsWarmingUp)
190  {
191  throw new Exception(OrderResponse.WarmingUp(request).ToString());
192  }
193 
194  var submit = request as SubmitOrderRequest;
195  if (submit != null)
196  {
197  SetOrderId(submit);
198  }
199  return _orderProcessor.Process(request);
200  }
201 
202  /// <summary>
203  /// Sets the order id for the specified submit request
204  /// </summary>
205  /// <param name="request">Request to set the order id for</param>
206  /// <remarks>This method is public so we can request an order id from outside the assembly, for testing for example</remarks>
207  public void SetOrderId(SubmitOrderRequest request)
208  {
209  // avoid setting the order id if it's already been set
210  if (request.OrderId < 1)
211  {
212  request.SetOrderId(GetIncrementOrderId());
213  }
214  }
215 
216  /// <summary>
217  /// Add an order to collection and return the unique order id or negative if an error.
218  /// </summary>
219  /// <param name="request">A request detailing the order to be submitted</param>
220  /// <returns>New unique, increasing orderid</returns>
222  {
223  return ProcessRequest(request);
224  }
225 
226  /// <summary>
227  /// Update an order yet to be filled such as stop or limit orders.
228  /// </summary>
229  /// <param name="request">Request detailing how the order should be updated</param>
230  /// <remarks>Does not apply if the order is already fully filled</remarks>
232  {
233  return ProcessRequest(request);
234  }
235 
236  /// <summary>
237  /// Added alias for RemoveOrder -
238  /// </summary>
239  /// <param name="orderId">Order id we wish to cancel</param>
240  /// <param name="orderTag">Tag to indicate from where this method was called</param>
241  public OrderTicket CancelOrder(int orderId, string orderTag = null)
242  {
243  return RemoveOrder(orderId, orderTag);
244  }
245 
246  /// <summary>
247  /// Cancels all open orders for all symbols
248  /// </summary>
249  /// <returns>List containing the cancelled order tickets</returns>
250  public List<OrderTicket> CancelOpenOrders()
251  {
252  if (_algorithm != null && _algorithm.IsWarmingUp)
253  {
255  }
256 
257  var cancelledOrders = new List<OrderTicket>();
258  foreach (var ticket in GetOpenOrderTickets())
259  {
260  ticket.Cancel(Messages.SecurityTransactionManager.OrderCanceledByCancelOpenOrders(_algorithm.UtcTime));
261  cancelledOrders.Add(ticket);
262  }
263  return cancelledOrders;
264  }
265 
266  /// <summary>
267  /// Cancels all open orders for the specified symbol
268  /// </summary>
269  /// <param name="symbol">The symbol whose orders are to be cancelled</param>
270  /// <param name="tag">Custom order tag</param>
271  /// <returns>List containing the cancelled order tickets</returns>
272  public List<OrderTicket> CancelOpenOrders(Symbol symbol, string tag = null)
273  {
274  if (_algorithm != null && _algorithm.IsWarmingUp)
275  {
277  }
278 
279  var cancelledOrders = new List<OrderTicket>();
280  foreach (var ticket in GetOpenOrderTickets(x => x.Symbol == symbol))
281  {
282  ticket.Cancel(tag);
283  cancelledOrders.Add(ticket);
284  }
285  return cancelledOrders;
286  }
287 
288  /// <summary>
289  /// Remove this order from outstanding queue: user is requesting a cancel.
290  /// </summary>
291  /// <param name="orderId">Specific order id to remove</param>
292  /// <param name="tag">Tag request</param>
293  public OrderTicket RemoveOrder(int orderId, string tag = null)
294  {
295  return ProcessRequest(new CancelOrderRequest(_securities.UtcTime, orderId, tag ?? string.Empty));
296  }
297 
298  /// <summary>
299  /// Gets an enumerable of <see cref="OrderTicket"/> matching the specified <paramref name="filter"/>
300  /// </summary>
301  /// <param name="filter">The filter predicate used to find the required order tickets</param>
302  /// <returns>An enumerable of <see cref="OrderTicket"/> matching the specified <paramref name="filter"/></returns>
303  public IEnumerable<OrderTicket> GetOrderTickets(Func<OrderTicket, bool> filter = null)
304  {
305  return _orderProcessor.GetOrderTickets(filter ?? (x => true));
306  }
307 
308  /// <summary>
309  /// Gets an enumerable of <see cref="OrderTicket"/> matching the specified <paramref name="filter"/>
310  /// </summary>
311  /// <param name="filter">The Python function filter used to find the required order tickets</param>
312  /// <returns>An enumerable of <see cref="OrderTicket"/> matching the specified <paramref name="filter"/></returns>
313  public IEnumerable<OrderTicket> GetOrderTickets(PyObject filter)
314  {
315  return _orderProcessor.GetOrderTickets(filter.ConvertToDelegate<Func<OrderTicket, bool>>());
316  }
317 
318  /// <summary>
319  /// Get an enumerable of open <see cref="OrderTicket"/> for the specified symbol
320  /// </summary>
321  /// <param name="symbol">The symbol for which to return the order tickets</param>
322  /// <returns>An enumerable of open <see cref="OrderTicket"/>.</returns>
323  public IEnumerable<OrderTicket> GetOpenOrderTickets(Symbol symbol)
324  {
325  return GetOpenOrderTickets(x => x.Symbol == symbol);
326  }
327 
328  /// <summary>
329  /// Gets an enumerable of opened <see cref="OrderTicket"/> matching the specified <paramref name="filter"/>
330  /// </summary>
331  /// <param name="filter">The filter predicate used to find the required order tickets</param>
332  /// <returns>An enumerable of opened <see cref="OrderTicket"/> matching the specified <paramref name="filter"/></returns>
333  public IEnumerable<OrderTicket> GetOpenOrderTickets(Func<OrderTicket, bool> filter = null)
334  {
335  return _orderProcessor.GetOpenOrderTickets(filter ?? (x => true));
336  }
337 
338  /// <summary>
339  /// Gets an enumerable of opened <see cref="OrderTicket"/> matching the specified <paramref name="filter"/>
340  /// However, this method can be confused with the override that takes a Symbol as parameter. For this reason
341  /// it first checks if it can convert the parameter into a symbol. If that conversion cannot be aplied it
342  /// assumes the parameter is a Python function object and not a Python representation of a Symbol.
343  /// </summary>
344  /// <param name="filter">The Python function filter used to find the required order tickets</param>
345  /// <returns>An enumerable of opened <see cref="OrderTicket"/> matching the specified <paramref name="filter"/></returns>
346  public IEnumerable<OrderTicket> GetOpenOrderTickets(PyObject filter)
347  {
348  Symbol pythonSymbol;
349  if (filter.TryConvert(out pythonSymbol))
350  {
351  return GetOpenOrderTickets(pythonSymbol);
352  }
353  return _orderProcessor.GetOpenOrderTickets(filter.ConvertToDelegate<Func<OrderTicket, bool>>());
354  }
355 
356  /// <summary>
357  /// Gets the remaining quantity to be filled from open orders, i.e. order size minus quantity filled
358  /// </summary>
359  /// <param name="filter">Filters the order tickets to be included in the aggregate quantity remaining to be filled</param>
360  /// <returns>Total quantity that hasn't been filled yet for all orders that were not filtered</returns>
361  public decimal GetOpenOrdersRemainingQuantity(Func<OrderTicket, bool> filter = null)
362  {
363  return GetOpenOrderTickets(filter)
364  .Aggregate(0m, (d, t) => d + t.Quantity - t.QuantityFilled);
365  }
366 
367  /// <summary>
368  /// Gets the remaining quantity to be filled from open orders, i.e. order size minus quantity filled
369  /// However, this method can be confused with the override that takes a Symbol as parameter. For this reason
370  /// it first checks if it can convert the parameter into a symbol. If that conversion cannot be aplied it
371  /// assumes the parameter is a Python function object and not a Python representation of a Symbol.
372  /// </summary>
373  /// <param name="filter">Filters the order tickets to be included in the aggregate quantity remaining to be filled</param>
374  /// <returns>Total quantity that hasn't been filled yet for all orders that were not filtered</returns>
375  public decimal GetOpenOrdersRemainingQuantity(PyObject filter)
376  {
377  Symbol pythonSymbol;
378  if (filter.TryConvert(out pythonSymbol))
379  {
380  return GetOpenOrdersRemainingQuantity(pythonSymbol);
381  }
382 
383  return GetOpenOrderTickets(filter)
384  .Aggregate(0m, (d, t) => d + t.Quantity - t.QuantityFilled);
385  }
386 
387  /// <summary>
388  /// Gets the remaining quantity to be filled from open orders for a Symbol, i.e. order size minus quantity filled
389  /// </summary>
390  /// <param name="symbol">Symbol to get the remaining quantity of currently open orders</param>
391  /// <returns>Total quantity that hasn't been filled yet for orders matching the Symbol</returns>
392  public decimal GetOpenOrdersRemainingQuantity(Symbol symbol)
393  {
394  return GetOpenOrdersRemainingQuantity(t => t.Symbol == symbol);
395  }
396 
397  /// <summary>
398  /// Gets the order ticket for the specified order id. Returns null if not found
399  /// </summary>
400  /// <param name="orderId">The order's id</param>
401  /// <returns>The order ticket with the specified id, or null if not found</returns>
402  public OrderTicket GetOrderTicket(int orderId)
403  {
404  return _orderProcessor.GetOrderTicket(orderId);
405  }
406 
407  /// <summary>
408  /// Wait for a specific order to be either Filled, Invalid or Canceled
409  /// </summary>
410  /// <param name="orderId">The id of the order to wait for</param>
411  /// <returns>True if we successfully wait for the fill, false if we were unable
412  /// to wait. This may be because it is not a market order or because the timeout
413  /// was reached</returns>
414  public bool WaitForOrder(int orderId)
415  {
416  var orderTicket = GetOrderTicket(orderId);
417  if (orderTicket == null)
418  {
419  Log.Error($@"SecurityTransactionManager.WaitForOrder(): {
420  Messages.SecurityTransactionManager.UnableToLocateOrderTicket(orderId)}");
421 
422  return false;
423  }
424 
425  if (!orderTicket.OrderClosed.WaitOne(_marketOrderFillTimeout))
426  {
427  if(_marketOrderFillTimeout > TimeSpan.Zero)
428  {
429  Log.Error($@"SecurityTransactionManager.WaitForOrder(): {Messages.SecurityTransactionManager.OrderNotFilledWithinExpectedTime(_marketOrderFillTimeout)}");
430  }
431 
432  return false;
433  }
434 
435  return true;
436  }
437 
438  /// <summary>
439  /// Get a list of all open orders for a symbol.
440  /// </summary>
441  /// <param name="symbol">The symbol for which to return the orders</param>
442  /// <returns>List of open orders.</returns>
443  public List<Order> GetOpenOrders(Symbol symbol)
444  {
445  return GetOpenOrders(x => x.Symbol == symbol);
446  }
447 
448  /// <summary>
449  /// Gets open orders matching the specified filter. Specifying null will return an enumerable
450  /// of all open orders.
451  /// </summary>
452  /// <param name="filter">Delegate used to filter the orders</param>
453  /// <returns>All filtered open orders this order provider currently holds</returns>
454  public List<Order> GetOpenOrders(Func<Order, bool> filter = null)
455  {
456  filter = filter ?? (x => true);
457  return _orderProcessor.GetOpenOrders(x => filter(x));
458  }
459 
460  /// <summary>
461  /// Gets open orders matching the specified filter. However, this method can be confused with the
462  /// override that takes a Symbol as parameter. For this reason it first checks if it can convert
463  /// the parameter into a symbol. If that conversion cannot be aplied it assumes the parameter is
464  /// a Python function object and not a Python representation of a Symbol.
465  /// </summary>
466  /// <param name="filter">Python function object used to filter the orders</param>
467  /// <returns>All filtered open orders this order provider currently holds</returns>
468  public List<Order> GetOpenOrders(PyObject filter)
469  {
470  Symbol pythonSymbol;
471  if (filter.TryConvert(out pythonSymbol))
472  {
473  return GetOpenOrders(pythonSymbol);
474  }
475  Func<Order, bool> csharpFilter = filter.ConvertToDelegate<Func<Order, bool>>();
476  return _orderProcessor.GetOpenOrders(x => csharpFilter(x));
477  }
478 
479  /// <summary>
480  /// Gets the current number of orders that have been processed
481  /// </summary>
482  public int OrdersCount
483  {
484  get { return _orderProcessor.OrdersCount; }
485  }
486 
487  /// <summary>
488  /// Get the order by its id
489  /// </summary>
490  /// <param name="orderId">Order id to fetch</param>
491  /// <returns>A clone of the order with the specified id, or null if no match is found</returns>
492  public Order GetOrderById(int orderId)
493  {
494  return _orderProcessor.GetOrderById(orderId);
495  }
496 
497  /// <summary>
498  /// Gets the order by its brokerage id
499  /// </summary>
500  /// <param name="brokerageId">The brokerage id to fetch</param>
501  /// <returns>The first order matching the brokerage id, or null if no match is found</returns>
502  public List<Order> GetOrdersByBrokerageId(string brokerageId)
503  {
504  return _orderProcessor.GetOrdersByBrokerageId(brokerageId);
505  }
506 
507  /// <summary>
508  /// Gets all orders matching the specified filter. Specifying null will return an enumerable
509  /// of all orders.
510  /// </summary>
511  /// <param name="filter">Delegate used to filter the orders</param>
512  /// <returns>All orders this order provider currently holds by the specified filter</returns>
513  public IEnumerable<Order> GetOrders(Func<Order, bool> filter = null)
514  {
515  return _orderProcessor.GetOrders(filter ?? (x => true));
516  }
517 
518  /// <summary>
519  /// Gets all orders matching the specified filter.
520  /// </summary>
521  /// <param name="filter">Python function object used to filter the orders</param>
522  /// <returns>All orders this order provider currently holds by the specified filter</returns>
523  public IEnumerable<Order> GetOrders(PyObject filter)
524  {
525  return _orderProcessor.GetOrders(filter.ConvertToDelegate<Func<Order, bool>>());
526  }
527 
528  /// <summary>
529  /// Get a new order id, and increment the internal counter.
530  /// </summary>
531  /// <returns>New unique int order id.</returns>
532  public int GetIncrementOrderId()
533  {
534  return Interlocked.Increment(ref _orderId);
535  }
536 
537  /// <summary>
538  /// Get a new group order manager id, and increment the internal counter.
539  /// </summary>
540  /// <returns>New unique int group order manager id.</returns>
542  {
543  return Interlocked.Increment(ref _groupOrderManagerId);
544  }
545 
546  /// <summary>
547  /// Sets the <see cref="IOrderProvider"/> used for fetching orders for the algorithm
548  /// </summary>
549  /// <param name="orderProvider">The <see cref="IOrderProvider"/> to be used to manage fetching orders</param>
550  public void SetOrderProcessor(IOrderProcessor orderProvider)
551  {
552  _orderProcessor = orderProvider;
553  }
554 
555  /// <summary>
556  /// Record the transaction value and time in a list to later be processed for statistics creation.
557  /// </summary>
558  /// <remarks>
559  /// Bit of a hack -- but using datetime as dictionary key is dangerous as you can process multiple orders within a second.
560  /// For the accounting / statistics generating purposes its not really critical to know the precise time, so just add a millisecond while there's an identical key.
561  /// </remarks>
562  /// <param name="time">Time of order processed </param>
563  /// <param name="transactionProfitLoss">Profit Loss.</param>
564  /// <param name="isWin">
565  /// Whether the transaction is a win.
566  /// For options exercise, this might not depend only on the profit/loss value
567  /// </param>
568  public void AddTransactionRecord(DateTime time, decimal transactionProfitLoss, bool isWin)
569  {
570  lock (_transactionRecord)
571  {
572  var clone = time;
573  while (_transactionRecord.ContainsKey(clone))
574  {
575  clone = clone.AddMilliseconds(1);
576  }
577  _transactionRecord.Add(clone, new TransactionRecordEntry { ProfitLoss = transactionProfitLoss, IsWin = isWin });
578  }
579  }
580 
581  /// <summary>
582  /// Set live mode state of the algorithm
583  /// </summary>
584  /// <param name="isLiveMode">True, live mode is enabled</param>
585  public void SetLiveMode(bool isLiveMode)
586  {
587  if (isLiveMode)
588  {
589  if(MarketOrderFillTimeout == TimeSpan.MinValue)
590  {
591  // set default value in live trading
592  MarketOrderFillTimeout = TimeSpan.FromSeconds(5);
593  }
594  }
595  else
596  {
597  // always zero in backtesting, fills happen synchronously, there's no dedicated thread like in live
598  MarketOrderFillTimeout = TimeSpan.Zero;
599  }
600  }
601  }
602 }