Lean  $LEAN_TAG$
QCAlgorithm.History.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14 */
15 
16 using System;
17 using NodaTime;
18 using System.Linq;
19 using QuantConnect.Data;
20 using QuantConnect.Util;
24 using System.Collections.Generic;
25 using QuantConnect.Python;
26 using Python.Runtime;
27 
28 namespace QuantConnect.Algorithm
29 {
30  public partial class QCAlgorithm
31  {
32  private bool _dataDictionaryTickWarningSent;
33 
34  /// <summary>
35  /// Gets or sets the history provider for the algorithm
36  /// </summary>
38  {
39  get;
40  set;
41  }
42 
43  /// <summary>
44  /// Gets whether or not this algorithm is still warming up
45  /// </summary>
46  [DocumentationAttribute(HistoricalData)]
47  public bool IsWarmingUp
48  {
49  get;
50  private set;
51  }
52 
53  /// <summary>
54  /// Sets the warm up period to the specified value
55  /// </summary>
56  /// <param name="timeSpan">The amount of time to warm up, this does not take into account market hours/weekends</param>
57  [DocumentationAttribute(HistoricalData)]
58  public void SetWarmup(TimeSpan timeSpan)
59  {
60  SetWarmUp(timeSpan, null);
61  }
62 
63  /// <summary>
64  /// Sets the warm up period to the specified value
65  /// </summary>
66  /// <param name="timeSpan">The amount of time to warm up, this does not take into account market hours/weekends</param>
67  [DocumentationAttribute(HistoricalData)]
68  public void SetWarmUp(TimeSpan timeSpan)
69  {
70  SetWarmup(timeSpan);
71  }
72 
73  /// <summary>
74  /// Sets the warm up period to the specified value
75  /// </summary>
76  /// <param name="timeSpan">The amount of time to warm up, this does not take into account market hours/weekends</param>
77  /// <param name="resolution">The resolution to request</param>
78  [DocumentationAttribute(HistoricalData)]
79  public void SetWarmup(TimeSpan timeSpan, Resolution? resolution)
80  {
81  SetWarmup(null, timeSpan, resolution);
82  }
83 
84  /// <summary>
85  /// Sets the warm up period to the specified value
86  /// </summary>
87  /// <param name="timeSpan">The amount of time to warm up, this does not take into account market hours/weekends</param>
88  /// <param name="resolution">The resolution to request</param>
89  [DocumentationAttribute(HistoricalData)]
90  public void SetWarmUp(TimeSpan timeSpan, Resolution? resolution)
91  {
92  SetWarmup(timeSpan, resolution);
93  }
94 
95  /// <summary>
96  /// Sets the warm up period by resolving a start date that would send that amount of data into
97  /// the algorithm. The highest (smallest) resolution in the securities collection will be used.
98  /// For example, if an algorithm has minute and daily data and 200 bars are requested, that would
99  /// use 200 minute bars.
100  /// </summary>
101  /// <param name="barCount">The number of data points requested for warm up</param>
102  [DocumentationAttribute(HistoricalData)]
103  public void SetWarmup(int barCount)
104  {
105  SetWarmUp(barCount, null);
106  }
107 
108  /// <summary>
109  /// Sets the warm up period by resolving a start date that would send that amount of data into
110  /// the algorithm. The highest (smallest) resolution in the securities collection will be used.
111  /// For example, if an algorithm has minute and daily data and 200 bars are requested, that would
112  /// use 200 minute bars.
113  /// </summary>
114  /// <param name="barCount">The number of data points requested for warm up</param>
115  [DocumentationAttribute(HistoricalData)]
116  public void SetWarmUp(int barCount)
117  {
118  SetWarmup(barCount);
119  }
120 
121  /// <summary>
122  /// Sets the warm up period by resolving a start date that would send that amount of data into
123  /// the algorithm.
124  /// </summary>
125  /// <param name="barCount">The number of data points requested for warm up</param>
126  /// <param name="resolution">The resolution to request</param>
127  [DocumentationAttribute(HistoricalData)]
128  public void SetWarmup(int barCount, Resolution? resolution)
129  {
130  SetWarmup(barCount, null, resolution);
131  }
132 
133  /// <summary>
134  /// Sets the warm up period by resolving a start date that would send that amount of data into
135  /// the algorithm.
136  /// </summary>
137  /// <param name="barCount">The number of data points requested for warm up</param>
138  /// <param name="resolution">The resolution to request</param>
139  [DocumentationAttribute(HistoricalData)]
140  public void SetWarmUp(int barCount, Resolution? resolution)
141  {
142  SetWarmup(barCount, resolution);
143  }
144 
145  /// <summary>
146  /// Sets <see cref="IAlgorithm.IsWarmingUp"/> to false to indicate this algorithm has finished its warm up
147  /// </summary>
148  [DocumentationAttribute(HistoricalData)]
149  public void SetFinishedWarmingUp()
150  {
151  IsWarmingUp = false;
152  }
153 
154  /// <summary>
155  /// Message for exception that is thrown when the implicit conversion between symbol and string fails
156  /// </summary>
157  private readonly string _symbolEmptyErrorMessage = "Cannot create history for the given ticker. " +
158  "Either explicitly use a symbol object to make the history request " +
159  "or ensure the symbol has been added using the AddSecurity() method before making the history request.";
160 
161  /// <summary>
162  /// Gets the history requests required for provide warm up data for the algorithm
163  /// </summary>
164  /// <returns></returns>
165  [DocumentationAttribute(HistoricalData)]
166  private bool TryGetWarmupHistoryStartTime(out DateTime result)
167  {
168  result = Time;
169 
170  if (_warmupBarCount.HasValue)
171  {
172  var symbols = Securities.Keys;
173  if (symbols.Count != 0)
174  {
175  var startTimeUtc = CreateBarCountHistoryRequests(symbols, _warmupBarCount.Value, Settings.WarmupResolution)
176  .DefaultIfEmpty()
177  .Min(request => request == null ? default : request.StartTimeUtc);
178  if(startTimeUtc != default)
179  {
180  result = startTimeUtc.ConvertFromUtc(TimeZone);
181  return true;
182  }
183  }
184 
185  var defaultResolutionToUse = UniverseSettings.Resolution;
186  if (Settings.WarmupResolution.HasValue)
187  {
188  defaultResolutionToUse = Settings.WarmupResolution.Value;
189  }
190 
191  // if the algorithm has no added security, let's take a look at the universes to determine
192  // what the start date should be used. Defaulting to always open
193  result = Time - _warmupBarCount.Value * defaultResolutionToUse.ToTimeSpan();
194 
195  foreach (var universe in _pendingUniverseAdditions.Concat(UniverseManager.Values))
196  {
197  var config = universe.Configuration;
198  var resolution = universe.Configuration.Resolution;
199  if (Settings.WarmupResolution.HasValue)
200  {
201  resolution = Settings.WarmupResolution.Value;
202  }
203  var exchange = MarketHoursDatabase.GetExchangeHours(config);
204  var start = _historyRequestFactory.GetStartTimeAlgoTz(config.Symbol, _warmupBarCount.Value, resolution, exchange, config.DataTimeZone);
205  // we choose the min start
206  result = result < start ? result : start;
207  }
208  return true;
209  }
210  if (_warmupTimeSpan.HasValue)
211  {
212  result = Time - _warmupTimeSpan.Value;
213  return true;
214  }
215 
216  return false;
217  }
218 
219  /// <summary>
220  /// Get the history for all configured securities over the requested span.
221  /// This will use the resolution and other subscription settings for each security.
222  /// The symbols must exist in the Securities collection.
223  /// </summary>
224  /// <param name="span">The span over which to request data. This is a calendar span, so take into consideration weekends and such</param>
225  /// <param name="resolution">The resolution to request</param>
226  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
227  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
228  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
229  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
230  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
231  /// For example, 0 will use the front month, 1 will use the back month contract</param>
232  /// <returns>An enumerable of slice containing data over the most recent span for all configured securities</returns>
233  [DocumentationAttribute(HistoricalData)]
234  public IEnumerable<Slice> History(TimeSpan span, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null,
235  DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
236  {
237  return History(Securities.Keys, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
238  dataNormalizationMode, contractDepthOffset).Memoize();
239  }
240 
241  /// <summary>
242  /// Get the history for all configured securities over the requested span.
243  /// This will use the resolution and other subscription settings for each security.
244  /// The symbols must exist in the Securities collection.
245  /// </summary>
246  /// <param name="periods">The number of bars to request</param>
247  /// <param name="resolution">The resolution to request</param>
248  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
249  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
250  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
251  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
252  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
253  /// For example, 0 will use the front month, 1 will use the back month contract</param>
254  /// <returns>An enumerable of slice containing data over the most recent span for all configured securities</returns>
255  [DocumentationAttribute(HistoricalData)]
256  public IEnumerable<Slice> History(int periods, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null,
257  DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
258  {
259  return History(Securities.Keys, periods, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
260  contractDepthOffset).Memoize();
261  }
262 
263  /// <summary>
264  /// Gets the historical data for all symbols of the requested type over the requested span.
265  /// The symbol's configured values for resolution and fill forward behavior will be used
266  /// The symbols must exist in the Securities collection.
267  /// </summary>
268  /// <param name="span">The span over which to retrieve recent historical data</param>
269  /// <param name="resolution">The resolution to request</param>
270  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
271  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
272  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
273  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
274  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
275  /// For example, 0 will use the front month, 1 will use the back month contract</param>
276  /// <returns>An enumerable of slice containing the requested historical data</returns>
277  [DocumentationAttribute(HistoricalData)]
278  public IEnumerable<DataDictionary<T>> History<T>(TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
279  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode ? dataNormalizationMode = null,
280  int? contractDepthOffset = null)
281  where T : IBaseData
282  {
283  return History<T>(Securities.Keys, span, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
284  contractDepthOffset).Memoize();
285  }
286 
287  /// <summary>
288  /// Gets the historical data for the specified symbols over the requested span.
289  /// The symbols must exist in the Securities collection.
290  /// </summary>
291  /// <typeparam name="T">The data type of the symbols</typeparam>
292  /// <param name="symbols">The symbols to retrieve historical data for</param>
293  /// <param name="span">The span over which to retrieve recent historical data</param>
294  /// <param name="resolution">The resolution to request</param>
295  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
296  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
297  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
298  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
299  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
300  /// For example, 0 will use the front month, 1 will use the back month contract</param>
301  /// <returns>An enumerable of slice containing the requested historical data</returns>
302  [DocumentationAttribute(HistoricalData)]
303  public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, TimeSpan span, Resolution? resolution = null,
304  bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
305  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
306  where T : IBaseData
307  {
308  return History<T>(symbols, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
309  dataNormalizationMode, contractDepthOffset).Memoize();
310  }
311 
312  /// <summary>
313  /// Gets the historical data for the specified symbols. The exact number of bars will be returned for
314  /// each symbol. This may result in some data start earlier/later than others due to when various
315  /// exchanges are open. The symbols must exist in the Securities collection.
316  /// </summary>
317  /// <typeparam name="T">The data type of the symbols</typeparam>
318  /// <param name="symbols">The symbols to retrieve historical data for</param>
319  /// <param name="periods">The number of bars to request</param>
320  /// <param name="resolution">The resolution to request</param>
321  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
322  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
323  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
324  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
325  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
326  /// For example, 0 will use the front month, 1 will use the back month contract</param>
327  /// <returns>An enumerable of slice containing the requested historical data</returns>
328  [DocumentationAttribute(HistoricalData)]
329  public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, int periods, Resolution? resolution = null,
330  bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
331  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
332  where T : IBaseData
333  {
334  CheckPeriodBasedHistoryRequestResolution(symbols, resolution);
335  var requests = CreateBarCountHistoryRequests(symbols, typeof(T), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
336  dataNormalizationMode, contractDepthOffset);
337  return GetDataTypedHistory<T>(requests);
338  }
339 
340  /// <summary>
341  /// Gets the historical data for the specified symbols between the specified dates. The symbols must exist in the Securities collection.
342  /// </summary>
343  /// <typeparam name="T">The data type of the symbols</typeparam>
344  /// <param name="symbols">The symbols to retrieve historical data for</param>
345  /// <param name="start">The start time in the algorithm's time zone</param>
346  /// <param name="end">The end time in the algorithm's time zone</param>
347  /// <param name="resolution">The resolution to request</param>
348  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
349  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
350  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
351  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
352  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
353  /// For example, 0 will use the front month, 1 will use the back month contract</param>
354  /// <returns>An enumerable of slice containing the requested historical data</returns>
355  [DocumentationAttribute(HistoricalData)]
356  public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, DateTime start, DateTime end, Resolution? resolution = null,
357  bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
358  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
359  where T : IBaseData
360  {
361  var requests = CreateDateRangeHistoryRequests(symbols, typeof(T), start, end, resolution, fillForward, extendedMarketHours,
362  dataMappingMode, dataNormalizationMode, contractDepthOffset);
363  return GetDataTypedHistory<T>(requests);
364  }
365 
366  /// <summary>
367  /// Gets the historical data for the specified symbol over the request span. The symbol must exist in the Securities collection.
368  /// </summary>
369  /// <typeparam name="T">The data type of the symbol</typeparam>
370  /// <param name="symbol">The symbol to retrieve historical data for</param>
371  /// <param name="span">The span over which to retrieve recent historical data</param>
372  /// <param name="resolution">The resolution to request</param>
373  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
374  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
375  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
376  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
377  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
378  /// For example, 0 will use the front month, 1 will use the back month contract</param>
379  /// <returns>An enumerable of slice containing the requested historical data</returns>
380  [DocumentationAttribute(HistoricalData)]
381  public IEnumerable<T> History<T>(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
382  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
383  int? contractDepthOffset = null)
384  where T : IBaseData
385  {
386  return History<T>(symbol, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
387  dataNormalizationMode, contractDepthOffset).Memoize();
388  }
389 
390  /// <summary>
391  /// Gets the historical data for the specified symbol. The exact number of bars will be returned.
392  /// The symbol must exist in the Securities collection.
393  /// </summary>
394  /// <param name="symbol">The symbol to retrieve historical data for</param>
395  /// <param name="periods">The number of bars to request</param>
396  /// <param name="resolution">The resolution to request</param>
397  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
398  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
399  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
400  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
401  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
402  /// For example, 0 will use the front month, 1 will use the back month contract</param>
403  /// <returns>An enumerable of slice containing the requested historical data</returns>
404  [DocumentationAttribute(HistoricalData)]
405  public IEnumerable<TradeBar> History(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null,
406  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
407  int? contractDepthOffset = null)
408  {
409  if (symbol == null) throw new ArgumentException(_symbolEmptyErrorMessage);
410 
411  resolution = GetResolution(symbol, resolution);
412  CheckPeriodBasedHistoryRequestResolution(new[] { symbol }, resolution);
413  var marketHours = GetMarketHours(symbol);
414  var start = _historyRequestFactory.GetStartTimeAlgoTz(symbol, periods, resolution.Value, marketHours.ExchangeHours,
415  marketHours.DataTimeZone, extendedMarketHours);
416 
417  return History(symbol, start, Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
418  contractDepthOffset);
419  }
420 
421  /// <summary>
422  /// Gets the historical data for the specified symbol. The exact number of bars will be returned.
423  /// The symbol must exist in the Securities collection.
424  /// </summary>
425  /// <typeparam name="T">The data type of the symbol</typeparam>
426  /// <param name="symbol">The symbol to retrieve historical data for</param>
427  /// <param name="periods">The number of bars to request</param>
428  /// <param name="resolution">The resolution to request</param>
429  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
430  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
431  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
432  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
433  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
434  /// For example, 0 will use the front month, 1 will use the back month contract</param>
435  /// <returns>An enumerable of slice containing the requested historical data</returns>
436  [DocumentationAttribute(HistoricalData)]
437  public IEnumerable<T> History<T>(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null,
438  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
439  int? contractDepthOffset = null)
440  where T : IBaseData
441  {
442  resolution = GetResolution(symbol, resolution);
443  CheckPeriodBasedHistoryRequestResolution(new[] { symbol }, resolution);
444  var requests = CreateBarCountHistoryRequests(new [] { symbol }, typeof(T), periods, resolution, fillForward, extendedMarketHours,
445  dataMappingMode, dataNormalizationMode, contractDepthOffset);
446  return GetDataTypedHistory<T>(requests, symbol);
447  }
448 
449  /// <summary>
450  /// Gets the historical data for the specified symbol between the specified dates. The symbol must exist in the Securities collection.
451  /// </summary>
452  /// <param name="symbol">The symbol to retrieve historical data for</param>
453  /// <param name="start">The start time in the algorithm's time zone</param>
454  /// <param name="end">The end time in the algorithm's time zone</param>
455  /// <param name="resolution">The resolution to request</param>
456  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
457  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
458  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
459  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
460  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
461  /// For example, 0 will use the front month, 1 will use the back month contract</param>
462  /// <returns>An enumerable of slice containing the requested historical data</returns>
463  [DocumentationAttribute(HistoricalData)]
464  public IEnumerable<T> History<T>(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null,
465  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
466  int? contractDepthOffset = null)
467  where T : IBaseData
468  {
469  var requests = CreateDateRangeHistoryRequests(new[] { symbol }, typeof(T), start, end, resolution, fillForward, extendedMarketHours,
470  dataMappingMode, dataNormalizationMode, contractDepthOffset);
471  return GetDataTypedHistory<T>(requests, symbol);
472  }
473 
474  /// <summary>
475  /// Gets the historical data for the specified symbol over the request span. The symbol must exist in the Securities collection.
476  /// </summary>
477  /// <param name="symbol">The symbol to retrieve historical data for</param>
478  /// <param name="span">The span over which to retrieve recent historical data</param>
479  /// <param name="resolution">The resolution to request</param>
480  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
481  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
482  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
483  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
484  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
485  /// For example, 0 will use the front month, 1 will use the back month contract</param>
486  /// <returns>An enumerable of slice containing the requested historical data</returns>
487  [DocumentationAttribute(HistoricalData)]
488  public IEnumerable<TradeBar> History(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
489  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
490  int? contractDepthOffset = null)
491  {
492  return History(symbol, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
493  contractDepthOffset);
494  }
495 
496  /// <summary>
497  /// Gets the historical data for the specified symbol over the request span. The symbol must exist in the Securities collection.
498  /// </summary>
499  /// <param name="symbol">The symbol to retrieve historical data for</param>
500  /// <param name="start">The start time in the algorithm's time zone</param>
501  /// <param name="end">The end time in the algorithm's time zone</param>
502  /// <param name="resolution">The resolution to request</param>
503  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
504  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
505  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
506  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
507  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
508  /// For example, 0 will use the front month, 1 will use the back month contract</param>
509  /// <returns>An enumerable of slice containing the requested historical data</returns>
510  [DocumentationAttribute(HistoricalData)]
511  public IEnumerable<TradeBar> History(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null,
512  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
513  int? contractDepthOffset = null)
514  {
515  var securityType = symbol.ID.SecurityType;
516  if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd)
517  {
518  Error("Calling History<TradeBar> method on a Forex or CFD security will return an empty result. Please use the generic version with QuoteBar type parameter.");
519  }
520 
521  var resolutionToUse = resolution ?? GetResolution(symbol, resolution);
522  if (resolutionToUse == Resolution.Tick)
523  {
524  throw new InvalidOperationException("Calling History<TradeBar> method with Resolution.Tick will return an empty result." +
525  " Please use the generic version with Tick type parameter or provide a list of Symbols to use the Slice history request API.");
526  }
527 
528  return History(new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
529  contractDepthOffset).Get(symbol).Memoize();
530  }
531 
532  /// <summary>
533  /// Gets the historical data for the specified symbols over the requested span.
534  /// The symbol's configured values for resolution and fill forward behavior will be used
535  /// The symbols must exist in the Securities collection.
536  /// </summary>
537  /// <param name="symbols">The symbols to retrieve historical data for</param>
538  /// <param name="span">The span over which to retrieve recent historical data</param>
539  /// <param name="resolution">The resolution to request</param>
540  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
541  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
542  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
543  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
544  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
545  /// For example, 0 will use the front month, 1 will use the back month contract</param>
546  /// <returns>An enumerable of slice containing the requested historical data</returns>
547  [DocumentationAttribute(HistoricalData)]
548  public IEnumerable<Slice> History(IEnumerable<Symbol> symbols, TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
549  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
550  int? contractDepthOffset = null)
551  {
552  return History(symbols, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
553  dataNormalizationMode, contractDepthOffset).Memoize();
554  }
555 
556  /// <summary>
557  /// Gets the historical data for the specified symbols. The exact number of bars will be returned for
558  /// each symbol. This may result in some data start earlier/later than others due to when various
559  /// exchanges are open. The symbols must exist in the Securities collection.
560  /// </summary>
561  /// <param name="symbols">The symbols to retrieve historical data for</param>
562  /// <param name="periods">The number of bars to request</param>
563  /// <param name="resolution">The resolution to request</param>
564  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
565  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
566  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
567  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
568  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
569  /// For example, 0 will use the front month, 1 will use the back month contract</param>
570  /// <returns>An enumerable of slice containing the requested historical data</returns>
571  [DocumentationAttribute(HistoricalData)]
572  public IEnumerable<Slice> History(IEnumerable<Symbol> symbols, int periods, Resolution? resolution = null, bool? fillForward = null,
573  bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
574  int? contractDepthOffset = null)
575  {
576  CheckPeriodBasedHistoryRequestResolution(symbols, resolution);
577  return History(CreateBarCountHistoryRequests(symbols, periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
578  dataNormalizationMode, contractDepthOffset)).Memoize();
579  }
580 
581  /// <summary>
582  /// Gets the historical data for the specified symbols between the specified dates. The symbols must exist in the Securities collection.
583  /// </summary>
584  /// <param name="symbols">The symbols to retrieve historical data for</param>
585  /// <param name="start">The start time in the algorithm's time zone</param>
586  /// <param name="end">The end time in the algorithm's time zone</param>
587  /// <param name="resolution">The resolution to request</param>
588  /// <param name="fillForward">True to fill forward missing data, false otherwise</param>
589  /// <param name="extendedMarketHours">True to include extended market hours data, false otherwise</param>
590  /// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
591  /// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
592  /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
593  /// For example, 0 will use the front month, 1 will use the back month contract</param>
594  /// <returns>An enumerable of slice containing the requested historical data</returns>
595  [DocumentationAttribute(HistoricalData)]
596  public IEnumerable<Slice> History(IEnumerable<Symbol> symbols, DateTime start, DateTime end, Resolution? resolution = null,
597  bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
598  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
599  {
600  return History(CreateDateRangeHistoryRequests(symbols, start, end, resolution, fillForward, extendedMarketHours, dataMappingMode,
601  dataNormalizationMode, contractDepthOffset)).Memoize();
602  }
603 
604  /// <summary>
605  /// Executes the specified history request
606  /// </summary>
607  /// <param name="request">the history request to execute</param>
608  /// <returns>An enumerable of slice satisfying the specified history request</returns>
609  [DocumentationAttribute(HistoricalData)]
610  public IEnumerable<Slice> History(HistoryRequest request)
611  {
612  return History(new[] { request }).Memoize();
613  }
614 
615  /// <summary>
616  /// Executes the specified history requests
617  /// </summary>
618  /// <param name="requests">the history requests to execute</param>
619  /// <returns>An enumerable of slice satisfying the specified history request</returns>
620  [DocumentationAttribute(HistoricalData)]
621  public IEnumerable<Slice> History(IEnumerable<HistoryRequest> requests)
622  {
623  return History(requests, TimeZone).Memoize();
624  }
625 
626  /// <summary>
627  /// Yields data to warmup a security for all it's subscribed data types
628  /// </summary>
629  /// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param>
630  /// <returns>Securities historical data</returns>
631  [DocumentationAttribute(AddingData)]
632  [DocumentationAttribute(HistoricalData)]
633  public IEnumerable<BaseData> GetLastKnownPrices(Security security)
634  {
635  return GetLastKnownPrices(security.Symbol);
636  }
637 
638  /// <summary>
639  /// Yields data to warmup a security for all it's subscribed data types
640  /// </summary>
641  /// <param name="symbol">The symbol we want to get seed data for</param>
642  /// <returns>Securities historical data</returns>
643  [DocumentationAttribute(AddingData)]
644  [DocumentationAttribute(HistoricalData)]
645  public IEnumerable<BaseData> GetLastKnownPrices(Symbol symbol)
646  {
647  if (!HistoryRequestValid(symbol) || HistoryProvider == null)
648  {
649  return Enumerable.Empty<BaseData>();
650  }
651 
652  var result = new Dictionary<TickType, BaseData>();
653  Resolution? resolution = null;
654  Func<int, bool> requestData = period =>
655  {
656  var historyRequests = CreateBarCountHistoryRequests(new[] { symbol }, period)
657  .Select(request =>
658  {
659  // For speed and memory usage, use Resolution.Minute as the minimum resolution
660  request.Resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)request.Resolution);
661  // force no fill forward behavior
662  request.FillForwardResolution = null;
663 
664  resolution = request.Resolution;
665  return request;
666  })
667  // request only those tick types we didn't get the data we wanted
668  .Where(request => !result.ContainsKey(request.TickType))
669  .ToList();
670  foreach (var slice in History(historyRequests))
671  {
672  for (var i = 0; i < historyRequests.Count; i++)
673  {
674  var historyRequest = historyRequests[i];
675  var data = slice.Get(historyRequest.DataType);
676  if (data.ContainsKey(symbol))
677  {
678  // keep the last data point per tick type
679  result[historyRequest.TickType] = (BaseData)data[symbol];
680  }
681  }
682  }
683  // true when all history requests tick types have a data point
684  return historyRequests.All(request => result.ContainsKey(request.TickType));
685  };
686 
687  if (!requestData(5))
688  {
689  if (resolution.HasValue)
690  {
691  // If the first attempt to get the last know price returns null, it maybe the case of an illiquid security.
692  // We increase the look-back period for this case accordingly to the resolution to cover 3 trading days
693  var periods =
694  resolution.Value == Resolution.Daily ? 3 :
695  resolution.Value == Resolution.Hour ? 24 : 1440;
696  requestData(periods);
697  }
698  else
699  {
700  // this shouldn't happen but just in case
702  $"QCAlgorithm.GetLastKnownPrices(): no history request was created for symbol {symbol} at {Time}");
703  }
704  }
705  // return the data ordered by time ascending
706  return result.Values.OrderBy(data => data.Time);
707  }
708 
709  /// <summary>
710  /// Get the last known price using the history provider.
711  /// Useful for seeding securities with the correct price
712  /// </summary>
713  /// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param>
714  /// <returns>A single <see cref="BaseData"/> object with the last known price</returns>
715  [Obsolete("This method is obsolete please use 'GetLastKnownPrices' which will return the last data point" +
716  " for each type associated with the requested security")]
717  [DocumentationAttribute(AddingData)]
718  [DocumentationAttribute(HistoricalData)]
720  {
721  return GetLastKnownPrices(security.Symbol)
722  // since we are returning a single data point let's respect order
723  .OrderByDescending(data => GetTickTypeOrder(data.Symbol.SecurityType, LeanData.GetCommonTickTypeForCommonDataTypes(data.GetType(), data.Symbol.SecurityType)))
724  .LastOrDefault();
725  }
726 
727  /// <summary>
728  /// Centralized logic to get data typed history given a list of requests for the specified symbol.
729  /// This method is used to keep backwards compatibility for those History methods that expect an ArgumentException to be thrown
730  /// when the security and the requested data type do not match
731  /// </summary>
732  /// <remarks>
733  /// This method will check for Python custom data types in order to call the right Slice.Get dynamic method
734  /// </remarks>
735  private IEnumerable<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests, Symbol symbol)
736  where T : IBaseData
737  {
738  var type = typeof(T);
739 
740  var historyRequests = requests.Where(x => x != null).ToList();
741  if (historyRequests.Count == 0)
742  {
743  throw new ArgumentException($"No history data could be fetched. " +
744  $"This could be due to the specified security not being of the requested type. Symbol: {symbol} Requested Type: {type.Name}");
745  }
746 
747  var slices = History(historyRequests, TimeZone);
748 
749  IEnumerable<T> result = null;
750 
751  // If T is a custom data coming from Python (a class derived from PythonData), T will get here as PythonData
752  // and not the actual custom type. We take care of this especial case by using a dynamic version of GetDataTypedHistory that
753  // receives the Python type, and we get it from the history requests.
754  if (type == typeof(PythonData))
755  {
756  result = GetPythonCustomDataTypeHistory(slices, historyRequests, symbol).OfType<T>();
757  }
758  // TODO: This is a patch to fix the issue with the Slice.GetImpl method returning only the last tick
759  // for each symbol instead of the whole list of ticks.
760  // The actual issue is Slice.GetImpl, so patch this can be removed right after it is properly addressed.
761  // A proposed solution making the Tick class a BaseDataCollection and make the Ticks class a dictionary Symbol->Tick instead of
762  // Symbol->List<Tick> so we can use the Slice.Get methods to collect all ticks in every slice instead of only the last one.
763  else if (type == typeof(Tick))
764  {
765  result = (IEnumerable<T>)slices.Select(x => x.Ticks).Where(x => x.ContainsKey(symbol)).SelectMany(x => x[symbol]);
766  }
767  else
768  {
769  result = slices.Get<T>(symbol);
770  }
771 
772  return result.Memoize();
773  }
774 
775  /// <summary>
776  /// Centralized logic to get data typed history for a given list of requests.
777  /// </summary>
778  /// <remarks>
779  /// This method will check for Python custom data types in order to call the right Slice.Get dynamic method
780  /// </remarks>
781  private IEnumerable<DataDictionary<T>> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests)
782  where T : IBaseData
783  {
784  var historyRequests = requests.Where(x => x != null).ToList();
785  var slices = History(historyRequests, TimeZone);
786 
787  IEnumerable<DataDictionary<T>> result = null;
788 
789  if (typeof(T) == typeof(PythonData))
790  {
791  result = GetPythonCustomDataTypeHistory(slices, historyRequests).OfType<DataDictionary<T>>();
792  }
793  else
794  {
795  if (typeof(T) == typeof(Tick) && !_dataDictionaryTickWarningSent)
796  {
797  _dataDictionaryTickWarningSent = true;
798  Debug("Warning: Multiple symbols Tick history will return the last tick per timestep. To access all ticks remove the 'Tick' type to use the History() returning Slice, all ticks can be accessed with Slice.Ticks.");
799  }
800  result = slices.Get<T>();
801  }
802 
803  return result.Memoize();
804  }
805 
806  [DocumentationAttribute(HistoricalData)]
807  private IEnumerable<Slice> History(IEnumerable<HistoryRequest> requests, DateTimeZone timeZone)
808  {
809  var sentMessage = false;
810  var hasPythonDataRequest = false;
811  // filter out any universe securities that may have made it this far
812  var filteredRequests = requests.Where(hr => HistoryRequestValid(hr.Symbol)).ToList();
813  for (var i = 0; i < filteredRequests.Count; i++)
814  {
815  var request = filteredRequests[i];
816  // prevent future requests
817  if (request.EndTimeUtc > UtcTime)
818  {
819  var endTimeUtc = UtcTime;
820  var startTimeUtc = request.StartTimeUtc;
821  if (request.StartTimeUtc > request.EndTimeUtc)
822  {
823  startTimeUtc = request.EndTimeUtc;
824  }
825 
826  filteredRequests[i] = new HistoryRequest(startTimeUtc, endTimeUtc,
827  request.DataType, request.Symbol, request.Resolution, request.ExchangeHours,
828  request.DataTimeZone, request.FillForwardResolution, request.IncludeExtendedMarketHours,
829  request.IsCustomData, request.DataNormalizationMode, request.TickType, request.DataMappingMode,
830  request.ContractDepthOffset);
831 
832  if (!sentMessage)
833  {
834  sentMessage = true;
835  Debug("Request for future history modified to end now.");
836  }
837  }
838 
839  if (!hasPythonDataRequest)
840  {
841  hasPythonDataRequest = request.IsCustomData && typeof(PythonData).IsAssignableFrom(request.DataType);
842  }
843  }
844 
845  // filter out future data to prevent look ahead bias
846  var history = HistoryProvider.GetHistory(filteredRequests, timeZone);
847 
848  if (hasPythonDataRequest && PythonEngine.IsInitialized)
849  {
850  // add protection against potential python deadlocks
851  return WrapPythonDataHistory(history);
852  }
853 
854  return history;
855  }
856 
857  /// <summary>
858  /// Helper method to create history requests from a date range
859  /// </summary>
860  private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, DateTime startAlgoTz, DateTime endAlgoTz,
861  Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
862  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
863  {
864  return CreateDateRangeHistoryRequests(symbols, typeof(BaseData), startAlgoTz, endAlgoTz, resolution, fillForward, extendedMarketHours,
865  dataMappingMode, dataNormalizationMode, contractDepthOffset);
866  }
867 
868  /// <summary>
869  /// Helper method to create history requests from a date range with custom data type
870  /// </summary>
871  private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType, DateTime startAlgoTz, DateTime endAlgoTz,
872  Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
873  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
874  {
875  return symbols.Where(HistoryRequestValid).SelectMany(x =>
876  {
877  var requests = new List<HistoryRequest>();
878 
879  foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution))
880  {
881  var request = _historyRequestFactory.CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x), resolution,
882  fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
883  requests.Add(request);
884  }
885 
886  return requests;
887  });
888  }
889 
890  /// <summary>
891  /// Helper methods to create a history request for the specified symbols and bar count
892  /// </summary>
893  private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols, int periods, Resolution? resolution = null,
894  bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
895  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
896  {
897  return CreateBarCountHistoryRequests(symbols, typeof(BaseData), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
898  dataNormalizationMode, contractDepthOffset);
899  }
900 
901  /// <summary>
902  /// Helper methods to create a history request for the specified symbols and bar count with custom data type
903  /// </summary>
904  private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType, int periods,
905  Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
906  DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
907  {
908  return symbols.Where(HistoryRequestValid).SelectMany(symbol =>
909  {
910  var res = GetResolution(symbol, resolution);
911  var exchange = GetExchangeHours(symbol);
912  var configs = GetMatchingSubscriptions(symbol, requestedType, resolution).ToList();
913  if (configs.Count == 0)
914  {
915  return Enumerable.Empty<HistoryRequest>();
916  }
917 
918  var start = _historyRequestFactory.GetStartTimeAlgoTz(symbol, periods, res, exchange, configs.First().DataTimeZone, extendedMarketHours);
919  var end = Time;
920 
921  return configs.Select(config => _historyRequestFactory.CreateHistoryRequest(config, start, end, exchange, res, fillForward,
922  extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset));
923  });
924  }
925 
926  private int GetTickTypeOrder(SecurityType securityType, TickType tickType)
927  {
928  return SubscriptionManager.AvailableDataTypes[securityType].IndexOf(tickType);
929  }
930 
931  private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null)
932  {
933  var matchingSubscriptions = SubscriptionManager.SubscriptionDataConfigService
934  // we add internal subscription so that history requests are covered, this allows us to warm them up too
935  .GetSubscriptionDataConfigs(symbol, includeInternalConfigs:true)
936  // find all subscriptions matching the requested type with a higher resolution than requested
937  .OrderByDescending(s => s.Resolution)
938  // lets make sure to respect the order of the data types
939  .ThenByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType))
940  .Where(s => SubscriptionDataConfigTypeFilter(type, s.Type));
941 
942  var internalConfig = new List<SubscriptionDataConfig>();
943  var userConfig = new List<SubscriptionDataConfig>();
944  foreach (var config in matchingSubscriptions)
945  {
946  if (config.IsInternalFeed)
947  {
948  internalConfig.Add(config);
949  }
950  else
951  {
952  userConfig.Add(config);
953  }
954  }
955 
956  // if we have any user defined subscription configuration we use it, else we use internal ones if any
957  List<SubscriptionDataConfig> configs = null;
958  if(userConfig.Count != 0)
959  {
960  configs = userConfig;
961  }
962  else if (internalConfig.Count != 0)
963  {
964  configs = internalConfig;
965  }
966 
967  // we use the subscription manager registered configurations here, we can not rely on the Securities collection
968  // since this might be called when creating a security and warming it up
969  if (configs != null && configs.Count != 0)
970  {
971  if (resolution.HasValue
972  && (resolution == Resolution.Daily || resolution == Resolution.Hour)
973  && symbol.SecurityType == SecurityType.Equity)
974  {
975  // for Daily and Hour resolution, for equities, we have to
976  // filter out any existing subscriptions that could be of Quote type
977  // This could happen if they were Resolution.Minute/Second/Tick
978  return configs.Where(s => s.TickType != TickType.Quote);
979  }
980 
981  return configs;
982  }
983  else
984  {
985  var entry = MarketHoursDatabase.GetEntry(symbol, new []{ type });
986  resolution = GetResolution(symbol, resolution);
987 
988  return SubscriptionManager
989  .LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution.Value, symbol.IsCanonical())
990  .Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1))
991  .Select(x => new SubscriptionDataConfig(
992  x.Item1,
993  symbol,
994  resolution.Value,
995  entry.DataTimeZone,
996  entry.ExchangeHours.TimeZone,
997  UniverseSettings.FillForward,
998  UniverseSettings.ExtendedMarketHours,
999  true,
1000  false,
1001  x.Item2,
1002  true,
1003  UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType)));
1004  }
1005  }
1006 
1007  /// <summary>
1008  /// Helper method to determine if the provided config type passes the filter of the target type
1009  /// </summary>
1010  /// <remarks>If the target type is <see cref="BaseData"/>, <see cref="OpenInterest"/> config types will return false.
1011  /// This is useful to filter OpenInterest by default from history requests unless it's explicitly requested</remarks>
1012  private bool SubscriptionDataConfigTypeFilter(Type targetType, Type configType)
1013  {
1014  var targetIsGenericType = targetType == typeof(BaseData);
1015 
1016  return targetType.IsAssignableFrom(configType) && (!targetIsGenericType || configType != typeof(OpenInterest));
1017  }
1018 
1019  private SecurityExchangeHours GetExchangeHours(Symbol symbol)
1020  {
1021  return GetMarketHours(symbol).ExchangeHours;
1022  }
1023 
1024  private MarketHoursDatabase.Entry GetMarketHours(Symbol symbol)
1025  {
1026  var hoursEntry = MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
1027 
1028  // user can override the exchange hours in algorithm, i.e. HistoryAlgorithm
1029  Security security;
1030  if (Securities.TryGetValue(symbol, out security))
1031  {
1032  return new MarketHoursDatabase.Entry(hoursEntry.DataTimeZone, security.Exchange.Hours);
1033  }
1034 
1035  return hoursEntry;
1036  }
1037 
1038  private Resolution GetResolution(Symbol symbol, Resolution? resolution)
1039  {
1040  Security security;
1041  if (Securities.TryGetValue(symbol, out security))
1042  {
1043  if (resolution != null)
1044  {
1045  return resolution.Value;
1046  }
1047 
1048  Resolution? result = null;
1049  var hasNonInternal = false;
1050  foreach (var config in SubscriptionManager.SubscriptionDataConfigService
1051  .GetSubscriptionDataConfigs(symbol, includeInternalConfigs: true)
1052  // we process non internal configs first
1053  .OrderBy(config => config.IsInternalFeed ? 1 : 0))
1054  {
1055  if (!config.IsInternalFeed || !hasNonInternal)
1056  {
1057  // once we find a non internal config we ignore internals
1058  hasNonInternal |= !config.IsInternalFeed;
1059  if (!result.HasValue || config.Resolution < result)
1060  {
1061  result = config.Resolution;
1062  }
1063  }
1064  }
1065 
1066  return result ?? UniverseSettings.Resolution;
1067  }
1068  else
1069  {
1070  return resolution ?? UniverseSettings.Resolution;
1071  }
1072  }
1073 
1074  /// <summary>
1075  /// Validate a symbol for a history request.
1076  /// Universe and canonical symbols are only valid for future security types
1077  /// </summary>
1078  private bool HistoryRequestValid(Symbol symbol)
1079  {
1080  return symbol.SecurityType == SecurityType.Future || !UniverseManager.ContainsKey(symbol) && !symbol.IsCanonical();
1081  }
1082 
1083  /// <summary>
1084  /// Will set warmup settings validating the algorithm has not finished initialization yet
1085  /// </summary>
1086  private void SetWarmup(int? barCount, TimeSpan? timeSpan, Resolution? resolution)
1087  {
1088  if (_locked)
1089  {
1090  throw new InvalidOperationException("QCAlgorithm.SetWarmup(): This method cannot be used after algorithm initialized");
1091  }
1092 
1093  _warmupTimeSpan = timeSpan;
1094  _warmupBarCount = barCount;
1095  Settings.WarmupResolution = resolution;
1096  }
1097 
1098  /// <summary>
1099  /// Throws if a period bases history request is made for tick resolution, which is not allowed.
1100  /// </summary>
1101  private void CheckPeriodBasedHistoryRequestResolution(IEnumerable<Symbol> symbols, Resolution? resolution)
1102  {
1103  if (symbols.Any(symbol => GetResolution(symbol, resolution) == Resolution.Tick))
1104  {
1105  throw new InvalidOperationException("History functions that accept a 'periods' parameter can not be used with Resolution.Tick");
1106  }
1107  }
1108 
1109  /// <summary>
1110  /// Centralized logic to get data typed history given a list of requests for the specified symbol.
1111  /// This method is used to keep backwards compatibility for those History methods that expect an ArgumentException to be thrown
1112  /// when the security and the requested data type do not match
1113  /// </summary>
1114  /// <remarks>
1115  /// This method is only used for Python algorithms, specially for those requesting custom data type history.
1116  /// The reason for using this method is that custom data type Python history calls to
1117  /// <see cref="History{T}(QuantConnect.Symbol, int, Resolution?)"/> will always use <see cref="PythonData"/> (the custom data base class)
1118  /// as the T argument, because the custom data class is a Python type, which will cause the history data in the slices to not be matched
1119  /// to the actual requested type, resulting in an empty list of slices.
1120  /// </remarks>
1121  private static IEnumerable<dynamic> GetPythonCustomDataTypeHistory(IEnumerable<Slice> slices, List<HistoryRequest> requests,
1122  Symbol symbol = null)
1123  {
1124  if (requests.Count == 0 || requests.Any(x => x.DataType != requests[0].DataType))
1125  {
1126  throw new ArgumentException("QCAlgorithm.GetPythonCustomDataTypeHistory(): All history requests must be for the same data type");
1127  }
1128 
1129  var pythonType = requests[0].DataType;
1130 
1131  if (symbol == null)
1132  {
1133  return slices.Get(pythonType);
1134  }
1135 
1136  return slices.Get(pythonType, symbol);
1137  }
1138 
1139  /// <summary>
1140  /// Wraps the resulting history enumerable in case of a Python custom data history request.
1141  /// We need to get and release the Python GIL when parallel history requests are enabled to avoid deadlocks
1142  /// in the custom data readers.
1143  /// </summary>
1144  private static IEnumerable<Slice> WrapPythonDataHistory(IEnumerable<Slice> history)
1145  {
1146  using var enumerator = history.GetEnumerator();
1147 
1148  var hasData = true;
1149  while (hasData)
1150  {
1151  // TODO: we don't really need the GIL. We should find a way to check whether we have the lock and only call this wrapper method if we do.
1152  using (Py.GIL())
1153  {
1154  var state = PythonEngine.BeginAllowThreads();
1155  hasData = enumerator.MoveNext();
1156  PythonEngine.EndAllowThreads(state);
1157  }
1158 
1159  if (hasData)
1160  {
1161  yield return enumerator.Current;
1162  }
1163  }
1164  }
1165  }
1166 }