24 using System.Collections.Generic;
30 public partial class QCAlgorithm
32 private bool _dataDictionaryTickWarningSent;
157 private readonly
string _symbolEmptyErrorMessage =
"Cannot create history for the given ticker. " +
158 "Either explicitly use a symbol object to make the history request " +
159 "or ensure the symbol has been added using the AddSecurity() method before making the history request.";
166 private bool TryGetWarmupHistoryStartTime(out DateTime result)
170 if (_warmupBarCount.HasValue)
173 if (symbols.Count != 0)
177 .Min(request => request ==
null ?
default : request.StartTimeUtc);
178 if(startTimeUtc !=
default)
180 result = startTimeUtc.ConvertFromUtc(
TimeZone);
193 result =
Time - _warmupBarCount.Value * defaultResolutionToUse.ToTimeSpan();
197 var config = universe.Configuration;
198 var resolution = universe.Configuration.Resolution;
204 var start = _historyRequestFactory.
GetStartTimeAlgoTz(config.Symbol, _warmupBarCount.Value, resolution, exchange, config.DataTimeZone);
206 result = result < start ? result : start;
210 if (_warmupTimeSpan.HasValue)
212 result =
Time - _warmupTimeSpan.Value;
233 [DocumentationAttribute(HistoricalData)]
234 public IEnumerable<Slice>
History(TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
238 dataNormalizationMode, contractDepthOffset).Memoize();
256 public IEnumerable<Slice>
History(
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
259 return History(
Securities.
Keys, periods, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
260 contractDepthOffset).Memoize();
278 public IEnumerable<DataDictionary<T>>
History<T>(TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
280 int? contractDepthOffset =
null)
283 return History<T>(
Securities.
Keys, span, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
284 contractDepthOffset).Memoize();
303 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols, TimeSpan span,
Resolution? resolution =
null,
304 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
308 return History<T>(symbols,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
309 dataNormalizationMode, contractDepthOffset).Memoize();
329 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
330 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
334 CheckPeriodBasedHistoryRequestResolution(symbols, resolution);
335 var requests = CreateBarCountHistoryRequests(symbols, typeof(T), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
336 dataNormalizationMode, contractDepthOffset);
337 return GetDataTypedHistory<T>(requests);
356 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols, DateTime start, DateTime end,
Resolution? resolution =
null,
357 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
361 var requests = CreateDateRangeHistoryRequests(symbols, typeof(T), start, end, resolution, fillForward, extendedMarketHours,
362 dataMappingMode, dataNormalizationMode, contractDepthOffset);
363 return GetDataTypedHistory<T>(requests);
383 int? contractDepthOffset =
null)
386 return History<T>(symbol,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
387 dataNormalizationMode, contractDepthOffset).Memoize();
407 int? contractDepthOffset =
null)
409 if (symbol ==
null)
throw new ArgumentException(_symbolEmptyErrorMessage);
411 resolution = GetResolution(symbol, resolution);
412 CheckPeriodBasedHistoryRequestResolution(
new[] { symbol }, resolution);
413 var marketHours = GetMarketHours(symbol);
414 var start = _historyRequestFactory.
GetStartTimeAlgoTz(symbol, periods, resolution.
Value, marketHours.ExchangeHours,
415 marketHours.DataTimeZone, extendedMarketHours);
417 return History(symbol, start,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
418 contractDepthOffset);
439 int? contractDepthOffset =
null)
442 resolution = GetResolution(symbol, resolution);
443 CheckPeriodBasedHistoryRequestResolution(
new[] { symbol }, resolution);
444 var requests = CreateBarCountHistoryRequests(
new [] { symbol }, typeof(T), periods, resolution, fillForward, extendedMarketHours,
445 dataMappingMode, dataNormalizationMode, contractDepthOffset);
446 return GetDataTypedHistory<T>(requests, symbol);
466 int? contractDepthOffset =
null)
469 var requests = CreateDateRangeHistoryRequests(
new[] { symbol }, typeof(T), start, end, resolution, fillForward, extendedMarketHours,
470 dataMappingMode, dataNormalizationMode, contractDepthOffset);
471 return GetDataTypedHistory<T>(requests, symbol);
490 int? contractDepthOffset =
null)
492 return History(symbol,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
493 contractDepthOffset);
511 public IEnumerable<TradeBar>
History(
Symbol symbol, DateTime start, DateTime end,
Resolution? resolution =
null,
bool? fillForward =
null,
513 int? contractDepthOffset =
null)
518 Error(
"Calling History<TradeBar> method on a Forex or CFD security will return an empty result. Please use the generic version with QuoteBar type parameter.");
521 var resolutionToUse = resolution ?? GetResolution(symbol, resolution);
524 throw new InvalidOperationException(
"Calling History<TradeBar> method with Resolution.Tick will return an empty result." +
525 " Please use the generic version with Tick type parameter or provide a list of Symbols to use the Slice history request API.");
528 return History(
new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
529 contractDepthOffset).Get(symbol).Memoize();
548 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols, TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
550 int? contractDepthOffset =
null)
552 return History(symbols,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
553 dataNormalizationMode, contractDepthOffset).Memoize();
572 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
574 int? contractDepthOffset =
null)
576 CheckPeriodBasedHistoryRequestResolution(symbols, resolution);
577 return History(CreateBarCountHistoryRequests(symbols, periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
578 dataNormalizationMode, contractDepthOffset)).Memoize();
596 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols, DateTime start, DateTime end,
Resolution? resolution =
null,
597 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
600 return History(CreateDateRangeHistoryRequests(symbols, start, end, resolution, fillForward, extendedMarketHours, dataMappingMode,
601 dataNormalizationMode, contractDepthOffset)).Memoize();
612 return History(
new[] { request }).Memoize();
621 public IEnumerable<Slice>
History(IEnumerable<HistoryRequest> requests)
649 return Enumerable.Empty<
BaseData>();
652 var result =
new Dictionary<TickType, BaseData>();
654 Func<int, bool> requestData = period =>
656 var historyRequests = CreateBarCountHistoryRequests(
new[] { symbol }, period)
662 request.FillForwardResolution =
null;
664 resolution = request.Resolution;
668 .Where(request => !result.ContainsKey(request.TickType))
670 foreach (var slice
in History(historyRequests))
672 for (var i = 0; i < historyRequests.Count; i++)
674 var historyRequest = historyRequests[i];
675 var data = slice.Get(historyRequest.DataType);
676 if (data.ContainsKey(symbol))
679 result[historyRequest.TickType] = (
BaseData)data[symbol];
684 return historyRequests.All(request => result.ContainsKey(request.TickType));
689 if (resolution.HasValue)
695 resolution.Value ==
Resolution.Hour ? 24 : 1440;
696 requestData(periods);
702 $
"QCAlgorithm.GetLastKnownPrices(): no history request was created for symbol {symbol} at {Time}");
706 return result.Values.OrderBy(data => data.Time);
715 [Obsolete(
"This method is obsolete please use 'GetLastKnownPrices' which will return the last data point" +
716 " for each type associated with the requested security")]
735 private IEnumerable<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests,
Symbol symbol)
738 var type = typeof(T);
740 var historyRequests = requests.Where(x => x !=
null).ToList();
741 if (historyRequests.Count == 0)
743 throw new ArgumentException($
"No history data could be fetched. " +
744 $
"This could be due to the specified security not being of the requested type. Symbol: {symbol} Requested Type: {type.Name}");
749 IEnumerable<T> result =
null;
756 result = GetPythonCustomDataTypeHistory(slices, historyRequests, symbol).OfType<T>();
763 else if (type == typeof(
Tick))
765 result = (IEnumerable<T>)slices.Select(x => x.Ticks).Where(x => x.ContainsKey(symbol)).SelectMany(x => x[symbol]);
769 result = slices.Get<T>(symbol);
772 return result.Memoize();
781 private IEnumerable<DataDictionary<T>> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests)
784 var historyRequests = requests.Where(x => x !=
null).ToList();
787 IEnumerable<DataDictionary<T>> result =
null;
791 result = GetPythonCustomDataTypeHistory(slices, historyRequests).OfType<
DataDictionary<T>>();
795 if (typeof(T) == typeof(
Tick) && !_dataDictionaryTickWarningSent)
797 _dataDictionaryTickWarningSent =
true;
798 Debug(
"Warning: Multiple symbols Tick history will return the last tick per timestep. To access all ticks remove the 'Tick' type to use the History() returning Slice, all ticks can be accessed with Slice.Ticks.");
800 result = slices.Get<T>();
803 return result.Memoize();
806 [DocumentationAttribute(HistoricalData)]
807 private IEnumerable<Slice>
History(IEnumerable<HistoryRequest> requests, DateTimeZone timeZone)
809 var sentMessage =
false;
810 var hasPythonDataRequest =
false;
812 var filteredRequests = requests.Where(hr => HistoryRequestValid(hr.Symbol)).ToList();
813 for (var i = 0; i < filteredRequests.Count; i++)
815 var request = filteredRequests[i];
817 if (request.EndTimeUtc >
UtcTime)
820 var startTimeUtc = request.StartTimeUtc;
821 if (request.StartTimeUtc > request.EndTimeUtc)
823 startTimeUtc = request.EndTimeUtc;
826 filteredRequests[i] =
new HistoryRequest(startTimeUtc, endTimeUtc,
827 request.DataType, request.Symbol, request.Resolution, request.ExchangeHours,
828 request.DataTimeZone, request.FillForwardResolution, request.IncludeExtendedMarketHours,
829 request.IsCustomData, request.DataNormalizationMode, request.TickType, request.DataMappingMode,
830 request.ContractDepthOffset);
835 Debug(
"Request for future history modified to end now.");
839 if (!hasPythonDataRequest)
841 hasPythonDataRequest = request.IsCustomData && typeof(
PythonData).IsAssignableFrom(request.DataType);
848 if (hasPythonDataRequest && PythonEngine.IsInitialized)
851 return WrapPythonDataHistory(history);
860 private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, DateTime startAlgoTz, DateTime endAlgoTz,
861 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
864 return CreateDateRangeHistoryRequests(symbols, typeof(
BaseData), startAlgoTz, endAlgoTz, resolution, fillForward, extendedMarketHours,
865 dataMappingMode, dataNormalizationMode, contractDepthOffset);
871 private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType, DateTime startAlgoTz, DateTime endAlgoTz,
872 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
875 return symbols.Where(HistoryRequestValid).SelectMany(x =>
877 var requests =
new List<HistoryRequest>();
879 foreach (var config
in GetMatchingSubscriptions(x, requestedType, resolution))
881 var request = _historyRequestFactory.
CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x), resolution,
882 fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
883 requests.Add(request);
893 private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
894 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
897 return CreateBarCountHistoryRequests(symbols, typeof(
BaseData), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
898 dataNormalizationMode, contractDepthOffset);
904 private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType,
int periods,
905 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
908 return symbols.Where(HistoryRequestValid).SelectMany(symbol =>
910 var res = GetResolution(symbol, resolution);
911 var exchange = GetExchangeHours(symbol);
912 var configs = GetMatchingSubscriptions(symbol, requestedType, resolution).ToList();
913 if (configs.Count == 0)
918 var start = _historyRequestFactory.
GetStartTimeAlgoTz(symbol, periods, res, exchange, configs.First().DataTimeZone, extendedMarketHours);
921 return configs.Select(config => _historyRequestFactory.
CreateHistoryRequest(config, start, end, exchange, res, fillForward,
922 extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset));
931 private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(
Symbol symbol, Type type,
Resolution? resolution =
null)
937 .OrderByDescending(s => s.Resolution)
939 .ThenByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType))
940 .Where(s => SubscriptionDataConfigTypeFilter(type, s.Type));
942 var internalConfig =
new List<SubscriptionDataConfig>();
943 var userConfig =
new List<SubscriptionDataConfig>();
944 foreach (var config
in matchingSubscriptions)
946 if (config.IsInternalFeed)
948 internalConfig.Add(config);
952 userConfig.Add(config);
957 List<SubscriptionDataConfig> configs =
null;
958 if(userConfig.Count != 0)
960 configs = userConfig;
962 else if (internalConfig.Count != 0)
964 configs = internalConfig;
969 if (configs !=
null && configs.Count != 0)
971 if (resolution.HasValue
978 return configs.Where(s => s.TickType !=
TickType.Quote);
986 resolution = GetResolution(symbol, resolution);
990 .Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1))
996 entry.ExchangeHours.TimeZone,
1003 UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType)));
1012 private bool SubscriptionDataConfigTypeFilter(Type targetType, Type configType)
1014 var targetIsGenericType = targetType == typeof(
BaseData);
1016 return targetType.IsAssignableFrom(configType) && (!targetIsGenericType || configType != typeof(
OpenInterest));
1021 return GetMarketHours(symbol).ExchangeHours;
1043 if (resolution !=
null)
1045 return resolution.Value;
1049 var hasNonInternal =
false;
1053 .OrderBy(config => config.IsInternalFeed ? 1 : 0))
1055 if (!config.IsInternalFeed || !hasNonInternal)
1058 hasNonInternal |= !config.IsInternalFeed;
1059 if (!result.HasValue || config.Resolution < result)
1061 result = config.Resolution;
1078 private bool HistoryRequestValid(
Symbol symbol)
1090 throw new InvalidOperationException(
"QCAlgorithm.SetWarmup(): This method cannot be used after algorithm initialized");
1093 _warmupTimeSpan = timeSpan;
1094 _warmupBarCount = barCount;
1101 private void CheckPeriodBasedHistoryRequestResolution(IEnumerable<Symbol> symbols,
Resolution? resolution)
1103 if (symbols.Any(symbol => GetResolution(symbol, resolution) ==
Resolution.Tick))
1105 throw new InvalidOperationException(
"History functions that accept a 'periods' parameter can not be used with Resolution.Tick");
1121 private static IEnumerable<dynamic> GetPythonCustomDataTypeHistory(IEnumerable<Slice> slices, List<HistoryRequest> requests,
1124 if (requests.Count == 0 || requests.Any(x => x.DataType != requests[0].DataType))
1126 throw new ArgumentException(
"QCAlgorithm.GetPythonCustomDataTypeHistory(): All history requests must be for the same data type");
1129 var pythonType = requests[0].DataType;
1133 return slices.Get(pythonType);
1136 return slices.Get(pythonType, symbol);
1144 private static IEnumerable<Slice> WrapPythonDataHistory(IEnumerable<Slice> history)
1146 using var enumerator = history.GetEnumerator();
1154 var state = PythonEngine.BeginAllowThreads();
1155 hasData = enumerator.MoveNext();
1156 PythonEngine.EndAllowThreads(state);
1161 yield
return enumerator.Current;