24 using System.Collections.Generic;
32 public partial class QCAlgorithm
34 private bool _dataDictionaryTickWarningSent;
159 private readonly
string _symbolEmptyErrorMessage =
"Cannot create history for the given ticker. " +
160 "Either explicitly use a symbol object to make the history request " +
161 "or ensure the symbol has been added using the AddSecurity() method before making the history request.";
168 private bool TryGetWarmupHistoryStartTime(out DateTime result)
172 if (_warmupBarCount.HasValue)
175 if (symbols.Count != 0)
179 .Min(request => request ==
null ?
default : request.StartTimeUtc);
180 if(startTimeUtc !=
default)
182 result = startTimeUtc.ConvertFromUtc(
TimeZone);
195 result =
Time - _warmupBarCount.Value * defaultResolutionToUse.ToTimeSpan();
199 var config = universe.Configuration;
200 var resolution = universe.Configuration.Resolution;
206 var start = _historyRequestFactory.
GetStartTimeAlgoTz(config.Symbol, _warmupBarCount.Value, resolution, exchange, config.DataTimeZone);
208 result = result < start ? result : start;
212 if (_warmupTimeSpan.HasValue)
214 result =
Time - _warmupTimeSpan.Value;
235 [DocumentationAttribute(HistoricalData)]
236 public IEnumerable<Slice>
History(TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
240 dataNormalizationMode, contractDepthOffset).Memoize();
258 public IEnumerable<Slice>
History(
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
261 return History(
Securities.
Keys, periods, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
262 contractDepthOffset).Memoize();
281 public IEnumerable<BaseDataCollection>
History(
Universe universe,
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
286 CheckPeriodBasedHistoryRequestResolution(symbols, resolution, universe.
Configuration.
Type);
287 var requests = CreateBarCountHistoryRequests(symbols, universe.
Configuration.
Type, periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
288 dataNormalizationMode, contractDepthOffset);
289 return GetDataTypedHistory<BaseDataCollection>(requests).Select(x => x.Values.Single());
308 public IEnumerable<BaseDataCollection>
History(
Universe universe, TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
310 int? contractDepthOffset =
null)
312 return History(universe,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
331 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
336 dataMappingMode, dataNormalizationMode, contractDepthOffset);
337 return GetDataTypedHistory<BaseDataCollection>(requests).Select(x => x.Values.Single());
355 public IEnumerable<DataDictionary<T>>
History<T>(TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
357 int? contractDepthOffset =
null)
360 return History<T>(
Securities.
Keys, span, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
361 contractDepthOffset).Memoize();
380 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols, TimeSpan span,
Resolution? resolution =
null,
381 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
385 return History<T>(symbols,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
386 dataNormalizationMode, contractDepthOffset).Memoize();
406 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
407 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
411 CheckPeriodBasedHistoryRequestResolution(symbols, resolution, typeof(
T));
412 var requests = CreateBarCountHistoryRequests(symbols, typeof(
T), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
413 dataNormalizationMode, contractDepthOffset);
414 return GetDataTypedHistory<T>(requests);
433 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols, DateTime start, DateTime end,
Resolution? resolution =
null,
434 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
439 dataMappingMode, dataNormalizationMode, contractDepthOffset);
440 return GetDataTypedHistory<T>(requests);
460 int? contractDepthOffset =
null)
463 return History<T>(symbol,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
464 dataNormalizationMode, contractDepthOffset).Memoize();
484 int? contractDepthOffset =
null)
486 if (symbol ==
null)
throw new ArgumentException(_symbolEmptyErrorMessage);
488 resolution = GetResolution(symbol, resolution, typeof(
TradeBar));
489 CheckPeriodBasedHistoryRequestResolution(
new[] { symbol }, resolution, typeof(
TradeBar));
490 var marketHours = GetMarketHours(symbol);
491 var start = _historyRequestFactory.
GetStartTimeAlgoTz(symbol, periods, resolution.
Value, marketHours.ExchangeHours,
492 marketHours.DataTimeZone, extendedMarketHours);
494 return History(symbol, start,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
495 contractDepthOffset);
516 int? contractDepthOffset =
null)
519 resolution = GetResolution(symbol, resolution, typeof(
T));
520 CheckPeriodBasedHistoryRequestResolution(
new[] { symbol }, resolution, typeof(
T));
521 var requests = CreateBarCountHistoryRequests(
new [] { symbol }, typeof(
T), periods, resolution, fillForward, extendedMarketHours,
522 dataMappingMode, dataNormalizationMode, contractDepthOffset);
523 return GetDataTypedHistory<T>(requests, symbol);
543 int? contractDepthOffset =
null)
547 dataMappingMode, dataNormalizationMode, contractDepthOffset);
548 return GetDataTypedHistory<T>(requests, symbol);
567 int? contractDepthOffset =
null)
569 return History(symbol,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
570 contractDepthOffset);
588 public IEnumerable<TradeBar>
History(
Symbol symbol, DateTime start, DateTime end,
Resolution? resolution =
null,
bool? fillForward =
null,
590 int? contractDepthOffset =
null)
595 Error(
"Calling History<TradeBar> method on a Forex or CFD security will return an empty result. Please use the generic version with QuoteBar type parameter.");
598 var resolutionToUse = resolution ?? GetResolution(symbol, resolution, typeof(
TradeBar));
601 throw new InvalidOperationException(
"Calling History<TradeBar> method with Resolution.Tick will return an empty result." +
602 " Please use the generic version with Tick type parameter or provide a list of Symbols to use the Slice history request API.");
605 return History(
new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
606 contractDepthOffset).Get(symbol).Memoize();
625 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols, TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
627 int? contractDepthOffset =
null)
629 return History(symbols,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
630 dataNormalizationMode, contractDepthOffset).Memoize();
649 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
651 int? contractDepthOffset =
null)
653 CheckPeriodBasedHistoryRequestResolution(symbols, resolution,
null);
654 return History(CreateBarCountHistoryRequests(symbols, periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
655 dataNormalizationMode, contractDepthOffset)).Memoize();
673 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols, DateTime start, DateTime end,
Resolution? resolution =
null,
674 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
678 dataNormalizationMode, contractDepthOffset)).Memoize();
689 return History(
new[] { request }).Memoize();
698 public IEnumerable<Slice>
History(IEnumerable<HistoryRequest> requests)
726 return Enumerable.Empty<
BaseData>();
729 var result =
new Dictionary<TickType, BaseData>();
731 Func<int, bool> requestData = period =>
733 var historyRequests = CreateBarCountHistoryRequests(
new[] { symbol }, period)
739 request.FillForwardResolution =
null;
741 resolution = request.Resolution;
745 .Where(request => !result.ContainsKey(request.TickType))
747 foreach (var slice
in History(historyRequests))
749 for (var i = 0; i < historyRequests.Count; i++)
751 var historyRequest = historyRequests[i];
752 var data = slice.Get(historyRequest.DataType);
753 if (data.ContainsKey(symbol))
756 result[historyRequest.TickType] = (
BaseData)data[symbol];
761 return historyRequests.All(request => result.ContainsKey(request.TickType));
766 if (resolution.HasValue)
772 resolution.Value ==
Resolution.Hour ? 24 : 1440;
773 requestData(periods);
779 $
"QCAlgorithm.GetLastKnownPrices(): no history request was created for symbol {symbol} at {Time}");
783 return result.Values.OrderBy(data => data.Time);
792 [Obsolete(
"This method is obsolete please use 'GetLastKnownPrices' which will return the last data point" +
793 " for each type associated with the requested security")]
812 private IEnumerable<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests,
Symbol symbol)
815 var type = typeof(
T);
817 var historyRequests = requests.Where(x => x !=
null).ToList();
818 if (historyRequests.Count == 0)
820 throw new ArgumentException($
"No history data could be fetched. " +
821 $
"This could be due to the specified security not being of the requested type. Symbol: {symbol} Requested Type: {type.Name}");
826 IEnumerable<T> result =
null;
833 result = GetPythonCustomDataTypeHistory(slices, historyRequests, symbol).OfType<
T>();
840 else if (type == typeof(
Tick))
842 result = (IEnumerable<T>)slices.Select(x => x.Ticks).Where(x => x.ContainsKey(symbol)).SelectMany(x => x[symbol]);
846 result = slices.Get<
T>(symbol);
849 return result.Memoize();
858 protected IEnumerable<DataDictionary<T>> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests)
861 var historyRequests = requests.Where(x => x !=
null).ToList();
864 IEnumerable<DataDictionary<T>> result =
null;
868 result = GetPythonCustomDataTypeHistory(slices, historyRequests).OfType<
DataDictionary<T>>();
872 if (typeof(
T) == typeof(
Tick) && !_dataDictionaryTickWarningSent)
874 _dataDictionaryTickWarningSent =
true;
875 Debug(
"Warning: Multiple symbols Tick history will return the last tick per timestep. To access all ticks remove the 'Tick' type to use the History() returning Slice, all ticks can be accessed with Slice.Ticks.");
880 result = (IEnumerable<DataDictionary<T>>)slices.GetUniverseData();
884 result = slices.Get<
T>();
888 return result.Memoize();
891 private IEnumerable<Slice>
History(IEnumerable<HistoryRequest> requests, DateTimeZone timeZone)
894 var filteredRequests = GetFilterestRequests(requests);
899 if (PythonEngine.IsInitialized)
904 return WrapPythonDataHistory(history);
910 private IEnumerable<HistoryRequest> GetFilterestRequests(IEnumerable<HistoryRequest> requests)
912 var sentMessage =
false;
913 foreach (var request
in requests.Where(hr => HistoryRequestValid(hr.Symbol)))
916 if (request.EndTimeUtc >
UtcTime)
919 var startTimeUtc = request.StartTimeUtc;
920 if (request.StartTimeUtc > request.EndTimeUtc)
922 startTimeUtc = request.EndTimeUtc;
926 request.DataType, request.Symbol, request.Resolution, request.ExchangeHours,
927 request.DataTimeZone, request.FillForwardResolution, request.IncludeExtendedMarketHours,
928 request.IsCustomData, request.DataNormalizationMode, request.TickType, request.DataMappingMode,
929 request.ContractDepthOffset);
934 Debug(
"Request for future history modified to end now.");
939 yield
return request;
948 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
952 dataMappingMode, dataNormalizationMode, contractDepthOffset);
959 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
962 return symbols.Where(HistoryRequestValid).SelectMany(x =>
964 var requests =
new List<HistoryRequest>();
966 foreach (var config
in GetMatchingSubscriptions(x, requestedType, resolution))
968 var request = _historyRequestFactory.
CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x, requestedType), resolution,
969 fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
970 requests.Add(request);
980 private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
981 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
984 return CreateBarCountHistoryRequests(symbols, typeof(
BaseData), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
985 dataNormalizationMode, contractDepthOffset);
991 private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType,
int periods,
992 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
995 return symbols.Where(HistoryRequestValid).SelectMany(symbol =>
997 var res = GetResolution(symbol, resolution, requestedType);
998 var exchange = GetExchangeHours(symbol, requestedType);
999 var configs = GetMatchingSubscriptions(symbol, requestedType, resolution).ToList();
1000 if (configs.Count == 0)
1005 var start = _historyRequestFactory.
GetStartTimeAlgoTz(symbol, periods, res, exchange, configs.First().DataTimeZone, extendedMarketHours);
1008 return configs.Select(config => _historyRequestFactory.
CreateHistoryRequest(config, start, end, exchange, res, fillForward,
1009 extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset));
1018 private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(
Symbol symbol, Type type,
Resolution? resolution =
null)
1024 .OrderByDescending(s => s.Resolution)
1026 .ThenByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType));
1028 var matchingSubscriptions = subscriptions.Where(s => SubscriptionDataConfigTypeFilter(type, s.Type));
1030 var internalConfig =
new List<SubscriptionDataConfig>();
1031 var userConfig =
new List<SubscriptionDataConfig>();
1032 foreach (var config
in matchingSubscriptions)
1034 if (config.IsInternalFeed)
1036 internalConfig.Add(config);
1040 userConfig.Add(config);
1045 List<SubscriptionDataConfig> configs =
null;
1046 if(userConfig.Count != 0)
1048 configs = userConfig;
1050 else if (internalConfig.Count != 0)
1052 configs = internalConfig;
1057 if (configs !=
null && configs.Count != 0)
1059 if (resolution.HasValue
1066 return configs.Where(s => s.TickType !=
TickType.Quote);
1069 if (symbol.IsCanonical() && configs.Count > 1)
1072 return configs.Where(s => s.Type != typeof(
ZipEntryName));
1080 resolution = GetResolution(symbol, resolution, type);
1090 entry.ExchangeHours.TimeZone,
1097 UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType))};
1102 .Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1))
1108 entry.ExchangeHours.TimeZone,
1115 UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType)));
1124 private bool SubscriptionDataConfigTypeFilter(Type targetType, Type configType)
1126 var targetIsGenericType = targetType == typeof(
BaseData);
1128 return targetType.IsAssignableFrom(configType) && (!targetIsGenericType || configType != typeof(
OpenInterest));
1133 return GetMarketHours(symbol, type).ExchangeHours;
1138 var hoursEntry = type !=
null
1155 if (resolution !=
null)
1157 return resolution.Value;
1161 var hasNonInternal =
false;
1165 .OrderBy(config => config.IsInternalFeed ? 1 : 0))
1167 if (!config.IsInternalFeed || !hasNonInternal)
1170 hasNonInternal |= !config.IsInternalFeed;
1171 if (!result.HasValue || config.Resolution < result)
1173 result = config.Resolution;
1184 if(resolution !=
null)
1186 return resolution.Value;
1197 var instance = type.GetBaseDataInstance();
1198 return instance.DefaultResolution();
1212 private bool HistoryRequestValid(
Symbol symbol)
1214 return symbol.SecurityType ==
SecurityType.Future || !symbol.IsCanonical();
1224 throw new InvalidOperationException(
"QCAlgorithm.SetWarmup(): This method cannot be used after algorithm initialized");
1227 _warmupTimeSpan = timeSpan;
1228 _warmupBarCount = barCount;
1235 private void CheckPeriodBasedHistoryRequestResolution(IEnumerable<Symbol> symbols,
Resolution? resolution, Type requestedType)
1237 if (symbols.Any(symbol => GetResolution(symbol, resolution, requestedType) ==
Resolution.Tick))
1239 throw new InvalidOperationException(
"History functions that accept a 'periods' parameter can not be used with Resolution.Tick");
1255 private static IEnumerable<dynamic> GetPythonCustomDataTypeHistory(IEnumerable<Slice> slices, List<HistoryRequest> requests,
1258 if (requests.Count == 0 || requests.Any(x => x.DataType != requests[0].DataType))
1260 throw new ArgumentException(
"QCAlgorithm.GetPythonCustomDataTypeHistory(): All history requests must be for the same data type");
1263 var pythonType = requests[0].DataType;
1267 return slices.Get(pythonType);
1270 return slices.Get(pythonType, symbol);
1278 private static IEnumerable<Slice> WrapPythonDataHistory(IEnumerable<Slice> history)
1280 using var enumerator = history.GetEnumerator();
1288 var state = PythonEngine.BeginAllowThreads();
1291 hasData = enumerator.MoveNext();
1296 PythonEngine.EndAllowThreads(state);
1301 yield
return enumerator.Current;