24 using System.Collections.Generic;
32 public partial class QCAlgorithm
34 private bool _dataDictionaryTickWarningSent;
159 private readonly
string _symbolEmptyErrorMessage =
"Cannot create history for the given ticker. " +
160 "Either explicitly use a symbol object to make the history request " +
161 "or ensure the symbol has been added using the AddSecurity() method before making the history request.";
168 private bool TryGetWarmupHistoryStartTime(out DateTime result)
172 if (_warmupBarCount.HasValue)
175 if (symbols.Count != 0)
179 .Min(request => request ==
null ?
default : request.StartTimeUtc);
180 if (startTimeUtc !=
default)
182 result = startTimeUtc.ConvertFromUtc(
TimeZone);
195 result =
Time - _warmupBarCount.Value * defaultResolutionToUse.ToTimeSpan();
199 var config = universe.Configuration;
200 var resolution = universe.Configuration.Resolution;
206 var start = _historyRequestFactory.
GetStartTimeAlgoTz(config.Symbol, _warmupBarCount.Value, resolution, exchange, config.DataTimeZone, config.Type);
208 result = result < start ? result : start;
212 if (_warmupTimeSpan.HasValue)
214 result =
Time - _warmupTimeSpan.Value;
235 [DocumentationAttribute(HistoricalData)]
236 public IEnumerable<Slice>
History(TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
240 dataNormalizationMode, contractDepthOffset).Memoize();
258 public IEnumerable<Slice>
History(
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
261 return History(
Securities.
Keys, periods, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
262 contractDepthOffset).Memoize();
281 public IEnumerable<BaseDataCollection>
History(
Universe universe,
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
286 CheckPeriodBasedHistoryRequestResolution(symbols, resolution, universe.
Configuration.
Type);
287 var requests = CreateBarCountHistoryRequests(symbols, universe.
Configuration.
Type, periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
288 dataNormalizationMode, contractDepthOffset);
289 return GetDataTypedHistory<BaseDataCollection>(requests).Select(x => x.Values.Single());
308 public IEnumerable<BaseDataCollection>
History(
Universe universe, TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
310 int? contractDepthOffset =
null)
312 return History(universe,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
331 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
336 dataMappingMode, dataNormalizationMode, contractDepthOffset);
337 return GetDataTypedHistory<BaseDataCollection>(requests).Select(x => x.Values.Single());
355 public IEnumerable<DataDictionary<T>>
History<T>(TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
357 int? contractDepthOffset =
null)
360 return History<T>(
Securities.
Keys, span, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
361 contractDepthOffset).Memoize();
380 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols, TimeSpan span,
Resolution? resolution =
null,
381 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
385 return History<T>(symbols,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
386 dataNormalizationMode, contractDepthOffset).Memoize();
406 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
407 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
411 CheckPeriodBasedHistoryRequestResolution(symbols, resolution, typeof(
T));
412 var requests = CreateBarCountHistoryRequests(symbols, typeof(
T), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
413 dataNormalizationMode, contractDepthOffset);
414 return GetDataTypedHistory<T>(requests);
433 public IEnumerable<DataDictionary<T>>
History<T>(IEnumerable<Symbol> symbols, DateTime start, DateTime end,
Resolution? resolution =
null,
434 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
439 dataMappingMode, dataNormalizationMode, contractDepthOffset);
440 return GetDataTypedHistory<T>(requests);
460 int? contractDepthOffset =
null)
463 return History<T>(symbol,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
464 dataNormalizationMode, contractDepthOffset).Memoize();
484 int? contractDepthOffset =
null)
486 if (symbol ==
null)
throw new ArgumentException(_symbolEmptyErrorMessage);
488 resolution = GetResolution(symbol, resolution, typeof(
TradeBar));
489 CheckPeriodBasedHistoryRequestResolution(
new[] { symbol }, resolution, typeof(
TradeBar));
490 var marketHours = GetMarketHours(symbol);
491 var start = _historyRequestFactory.
GetStartTimeAlgoTz(symbol, periods, resolution.
Value, marketHours.ExchangeHours,
492 marketHours.DataTimeZone, typeof(
TradeBar), extendedMarketHours);
494 return History(symbol, start,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
495 contractDepthOffset);
516 int? contractDepthOffset =
null)
519 resolution = GetResolution(symbol, resolution, typeof(
T));
520 CheckPeriodBasedHistoryRequestResolution(
new[] { symbol }, resolution, typeof(
T));
521 var requests = CreateBarCountHistoryRequests(
new[] { symbol }, typeof(
T), periods, resolution, fillForward, extendedMarketHours,
522 dataMappingMode, dataNormalizationMode, contractDepthOffset);
523 return GetDataTypedHistory<T>(requests, symbol);
543 int? contractDepthOffset =
null)
547 dataMappingMode, dataNormalizationMode, contractDepthOffset);
548 return GetDataTypedHistory<T>(requests, symbol);
567 int? contractDepthOffset =
null)
569 return History(symbol,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
570 contractDepthOffset);
588 public IEnumerable<TradeBar>
History(
Symbol symbol, DateTime start, DateTime end,
Resolution? resolution =
null,
bool? fillForward =
null,
590 int? contractDepthOffset =
null)
595 Error(
"Calling History<TradeBar> method on a Forex or CFD security will return an empty result. Please use the generic version with QuoteBar type parameter.");
598 var resolutionToUse = resolution ?? GetResolution(symbol, resolution, typeof(
TradeBar));
601 throw new InvalidOperationException(
"Calling History<TradeBar> method with Resolution.Tick will return an empty result." +
602 " Please use the generic version with Tick type parameter or provide a list of Symbols to use the Slice history request API.");
605 return History(
new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
606 contractDepthOffset).Get(symbol).Memoize();
625 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols, TimeSpan span,
Resolution? resolution =
null,
bool? fillForward =
null,
627 int? contractDepthOffset =
null)
629 return History(symbols,
Time - span,
Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
630 dataNormalizationMode, contractDepthOffset).Memoize();
649 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
bool? fillForward =
null,
651 int? contractDepthOffset =
null)
653 CheckPeriodBasedHistoryRequestResolution(symbols, resolution,
null);
654 return History(CreateBarCountHistoryRequests(symbols, periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
655 dataNormalizationMode, contractDepthOffset)).Memoize();
673 public IEnumerable<Slice>
History(IEnumerable<Symbol> symbols, DateTime start, DateTime end,
Resolution? resolution =
null,
674 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
678 dataNormalizationMode, contractDepthOffset)).Memoize();
689 return History(
new[] { request }).Memoize();
698 public IEnumerable<Slice>
History(IEnumerable<HistoryRequest> requests)
726 return Enumerable.Empty<
BaseData>();
729 var result =
new Dictionary<TickType, BaseData>();
731 Func<int, bool> requestData = period =>
733 var historyRequests = CreateBarCountHistoryRequests(
new[] { symbol }, period)
739 request.FillForwardResolution =
null;
741 resolution = request.Resolution;
745 .Where(request => !result.ContainsKey(request.TickType))
747 foreach (var slice
in History(historyRequests))
749 for (var i = 0; i < historyRequests.Count; i++)
751 var historyRequest = historyRequests[i];
752 var data = slice.Get(historyRequest.DataType);
753 if (data.ContainsKey(symbol))
756 result[historyRequest.TickType] = (
BaseData)data[symbol];
761 return historyRequests.All(request => result.ContainsKey(request.TickType));
766 if (resolution.HasValue)
772 resolution.Value ==
Resolution.Hour ? 24 : 1440;
773 requestData(periods);
779 $
"QCAlgorithm.GetLastKnownPrices(): no history request was created for symbol {symbol} at {Time}");
783 return result.Values.OrderBy(data => data.Time);
792 [Obsolete(
"This method is obsolete please use 'GetLastKnownPrices' which will return the last data point" +
793 " for each type associated with the requested security")]
812 private IEnumerable<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests,
Symbol symbol)
815 var type = typeof(
T);
817 var historyRequests = requests.Where(x => x !=
null).ToList();
818 if (historyRequests.Count == 0)
820 throw new ArgumentException($
"No history data could be fetched. " +
821 $
"This could be due to the specified security not being of the requested type. Symbol: {symbol} Requested Type: {type.Name}");
826 IEnumerable<T> result =
null;
833 result = GetPythonCustomDataTypeHistory(slices, historyRequests, symbol).OfType<
T>();
840 else if (type == typeof(
Tick))
842 result = (IEnumerable<T>)slices.Select(x => x.Ticks).Where(x => x.ContainsKey(symbol)).SelectMany(x => x[symbol]);
846 result = slices.Get<
T>(symbol);
849 return result.Memoize();
858 protected IEnumerable<DataDictionary<T>> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests)
861 var historyRequests = requests.Where(x => x !=
null).ToList();
864 IEnumerable<DataDictionary<T>> result =
null;
868 result = GetPythonCustomDataTypeHistory(slices, historyRequests).OfType<
DataDictionary<T>>();
872 if (typeof(
T) == typeof(
Tick) && !_dataDictionaryTickWarningSent)
874 _dataDictionaryTickWarningSent =
true;
875 Debug(
"Warning: Multiple symbols Tick history will return the last tick per timestep. To access all ticks remove the 'Tick' type to use the History() returning Slice, all ticks can be accessed with Slice.Ticks.");
880 result = (IEnumerable<DataDictionary<T>>)slices.GetUniverseData();
884 result = slices.Get<
T>();
888 return result.Memoize();
891 private IEnumerable<Slice>
History(IEnumerable<HistoryRequest> requests, DateTimeZone timeZone)
894 var filteredRequests = GetFilterestRequests(requests);
899 if (PythonEngine.IsInitialized)
904 return WrapPythonDataHistory(history);
910 private IEnumerable<HistoryRequest> GetFilterestRequests(IEnumerable<HistoryRequest> requests)
912 var sentMessage =
false;
913 foreach (var request
in requests.Where(hr => HistoryRequestValid(hr.Symbol)))
916 if (request.EndTimeUtc >
UtcTime)
919 var startTimeUtc = request.StartTimeUtc;
920 if (request.StartTimeUtc > request.EndTimeUtc)
922 startTimeUtc = request.EndTimeUtc;
926 request.DataType, request.Symbol, request.Resolution, request.ExchangeHours,
927 request.DataTimeZone, request.FillForwardResolution, request.IncludeExtendedMarketHours,
928 request.IsCustomData, request.DataNormalizationMode, request.TickType, request.DataMappingMode,
929 request.ContractDepthOffset);
934 Debug(
"Request for future history modified to end now.");
939 yield
return request;
948 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
952 var symbolsArray = symbols.ToArray();
962 dataNormalizationMode,
963 contractDepthOffset);
970 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
973 return symbols.Where(HistoryRequestValid).SelectMany(x =>
975 var requests =
new List<HistoryRequest>();
977 foreach (var config
in GetMatchingSubscriptions(x, requestedType, resolution))
979 var request = _historyRequestFactory.
CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x, requestedType), resolution,
980 fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
981 requests.Add(request);
991 private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols,
int periods,
Resolution? resolution =
null,
992 bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
996 var symbolsArray = symbols.ToArray();
997 return CreateBarCountHistoryRequests(
1003 extendedMarketHours,
1005 dataNormalizationMode,
1006 contractDepthOffset);
1012 private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType,
int periods,
1013 Resolution? resolution =
null,
bool? fillForward =
null,
bool? extendedMarketHours =
null,
DataMappingMode? dataMappingMode =
null,
1016 return symbols.Where(HistoryRequestValid).SelectMany(symbol =>
1019 var configs = GetMatchingSubscriptions(symbol, requestedType, resolution).ToList();
1020 if (configs.Count == 0)
1025 return configs.Select(config =>
1028 var type = requestedType ?? config.Type;
1029 var res = GetResolution(symbol, resolution, type);
1030 var exchange = GetExchangeHours(symbol, type);
1031 var start = _historyRequestFactory.
GetStartTimeAlgoTz(symbol, periods, res, exchange, config.DataTimeZone,
1032 config.Type, extendedMarketHours);
1035 return _historyRequestFactory.
CreateHistoryRequest(config, start, end, exchange, res, fillForward,
1036 extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
1046 private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(
Symbol symbol, Type type,
Resolution? resolution =
null)
1052 .OrderByDescending(s => s.Resolution)
1054 .ThenByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType));
1056 var matchingSubscriptions = subscriptions.Where(s => SubscriptionDataConfigTypeFilter(type, s.Type));
1058 var internalConfig =
new List<SubscriptionDataConfig>();
1059 var userConfig =
new List<SubscriptionDataConfig>();
1060 foreach (var config
in matchingSubscriptions)
1062 if (config.IsInternalFeed)
1064 internalConfig.Add(config);
1068 userConfig.Add(config);
1073 List<SubscriptionDataConfig> configs =
null;
1074 if (userConfig.Count != 0)
1076 configs = userConfig;
1078 else if (internalConfig.Count != 0)
1080 configs = internalConfig;
1085 if (configs !=
null && configs.Count != 0)
1087 if (resolution.HasValue && symbol.SecurityType ==
SecurityType.Equity)
1095 return configs.Where(s => s.TickType !=
TickType.Quote);
1100 if (type ==
null && !configs.Any(config => config.TickType ==
TickType.Quote))
1104 var baseFillForward = configs[0].FillDataForward;
1105 var baseExtendedMarketHours = configs[0].ExtendedMarketHours;
1113 entry.ExchangeHours.TimeZone,
1115 baseExtendedMarketHours,
1118 configs.Add(newConfig);
1121 return configs.OrderByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType));
1125 if (symbol.IsCanonical() && configs.Count > 1)
1128 return configs.Where(s => s.Type != typeof(
ZipEntryName));
1135 resolution = GetResolution(symbol, resolution, type);
1141 var isCustom = Extensions.IsCustomDataType(symbol, type);
1150 entry.ExchangeHours.TimeZone,
1157 UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType))};
1162 .Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1))
1165 var configType = x.Item1;
1174 entry.ExchangeHours.TimeZone,
1181 UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType));
1184 .OrderByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType));
1193 private bool SubscriptionDataConfigTypeFilter(Type targetType, Type configType)
1195 if (targetType ==
null)
1200 var targetIsGenericType = targetType == typeof(
BaseData);
1202 return targetType.IsAssignableFrom(configType) && (!targetIsGenericType || configType != typeof(
OpenInterest));
1207 return GetMarketHours(symbol, type).ExchangeHours;
1212 var hoursEntry = type !=
null
1229 if (resolution !=
null)
1231 return resolution.Value;
1235 var hasNonInternal =
false;
1239 .OrderBy(config => config.IsInternalFeed ? 1 : 0))
1241 if (!config.IsInternalFeed || !hasNonInternal)
1244 hasNonInternal |= !config.IsInternalFeed;
1245 if (!result.HasValue || config.Resolution < result)
1247 result = config.Resolution;
1258 if (resolution !=
null)
1260 return resolution.Value;
1271 var instance = type.GetBaseDataInstance();
1272 return instance.DefaultResolution();
1286 private bool HistoryRequestValid(
Symbol symbol)
1291 !symbol.IsCanonical();
1301 throw new InvalidOperationException(
"QCAlgorithm.SetWarmup(): This method cannot be used after algorithm initialized");
1304 _warmupTimeSpan = timeSpan;
1305 _warmupBarCount = barCount;
1312 private void CheckPeriodBasedHistoryRequestResolution(IEnumerable<Symbol> symbols,
Resolution? resolution, Type requestedType)
1314 if (symbols.Any(symbol => GetResolution(symbol, resolution, requestedType) ==
Resolution.Tick))
1316 throw new InvalidOperationException(
"History functions that accept a 'periods' parameter can not be used with Resolution.Tick");
1332 private static IEnumerable<dynamic> GetPythonCustomDataTypeHistory(IEnumerable<Slice> slices, List<HistoryRequest> requests,
1335 if (requests.Count == 0 || requests.Any(x => x.DataType != requests[0].DataType))
1337 throw new ArgumentException(
"QCAlgorithm.GetPythonCustomDataTypeHistory(): All history requests must be for the same data type");
1340 var pythonType = requests[0].DataType;
1344 return slices.Get(pythonType);
1347 return slices.Get(pythonType, symbol);
1355 private static IEnumerable<Slice> WrapPythonDataHistory(IEnumerable<Slice> history)
1357 using var enumerator = history.GetEnumerator();
1365 var state = PythonEngine.BeginAllowThreads();
1368 hasData = enumerator.MoveNext();
1373 PythonEngine.EndAllowThreads(state);
1378 yield
return enumerator.Current;