Lean  $LEAN_TAG$
QCAlgorithm.Framework.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14 */
15 
16 using System;
17 using System.Collections.Generic;
18 using System.Linq;
25 using QuantConnect.Data;
28 using QuantConnect.Util;
29 
30 namespace QuantConnect.Algorithm
31 {
32  public partial class QCAlgorithm
33  {
34  private bool _isEmitWarmupInsightWarningSent;
35  private bool _isEmitDelistedInsightWarningSent;
36 
37  /// <summary>
38  /// Enables additional logging of framework models including:
39  /// All insights, portfolio targets, order events, and any risk management altered targets
40  /// </summary>
41  [DocumentationAttribute(Logging)]
42  public bool DebugMode { get; set; }
43 
44  /// <summary>
45  /// Gets or sets the universe selection model.
46  /// </summary>
47  [DocumentationAttribute(AlgorithmFramework)]
49 
50  /// <summary>
51  /// Gets or sets the alpha model
52  /// </summary>
53  [DocumentationAttribute(AlgorithmFramework)]
54  public IAlphaModel Alpha { get; set; }
55 
56  /// <summary>
57  /// Gets the insight manager
58  /// </summary>
59  [DocumentationAttribute(AlgorithmFramework)]
60  public InsightManager Insights { get; private set; }
61 
62  /// <summary>
63  /// Gets or sets the portfolio construction model
64  /// </summary>
65  [DocumentationAttribute(AlgorithmFramework)]
67 
68  /// <summary>
69  /// Gets or sets the execution model
70  /// </summary>
71  [DocumentationAttribute(AlgorithmFramework)]
72  public IExecutionModel Execution { get; set; }
73 
74  /// <summary>
75  /// Gets or sets the risk management model
76  /// </summary>
77  [DocumentationAttribute(AlgorithmFramework)]
78  public IRiskManagementModel RiskManagement { get; set; }
79 
80  /// <summary>
81  /// Called by setup handlers after Initialize and allows the algorithm a chance to organize
82  /// the data gather in the Initialize method
83  /// </summary>
84  [DocumentationAttribute(AlgorithmFramework)]
86  {
87  foreach (var universe in UniverseSelection.CreateUniverses(this))
88  {
89  lock(_pendingUniverseAdditionsLock)
90  {
91  // on purpose we don't call 'AddUniverse' here so that these universes don't get registered as user added
92  // this is so that later during 'UniverseSelection.CreateUniverses' we wont remove them from UniverseManager
93  _pendingUniverseAdditions.Add(universe);
94  }
95  }
96 
97  if (DebugMode)
98  {
99  InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}");
100  }
101  }
102 
103  /// <summary>
104  /// Used to send data updates to algorithm framework models
105  /// </summary>
106  /// <param name="slice">The current data slice</param>
107  [DocumentationAttribute(AlgorithmFramework)]
108  [DocumentationAttribute(HandlingData)]
109  public void OnFrameworkData(Slice slice)
110  {
112  {
113  var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol);
114 
115  // remove deselected universes by symbol
116  foreach (var ukvp in UniverseManager)
117  {
118  var universeSymbol = ukvp.Key;
119  if (_userAddedUniverses.Contains(universeSymbol))
120  {
121  // prevent removal of qc algorithm created user defined universes
122  continue;
123  }
124 
125  if (ukvp.Value.DisposeRequested)
126  {
127  // have to remove in the next loop after the universe is marked as disposed, when 'Dispose()' is called it will trigger universe selection
128  // and deselect all symbols, sending the removed security changes, which are picked up by the AlgorithmManager and tags securities
129  // as non tradable as long as they are not active in any universe (uses UniverseManager.ActiveSecurities)
130  // but they will remain tradable if a position is still being hold since they won't be remove from the UniverseManager
131  // but this last part will not happen if we remove the universe from the UniverseManager right away, since it won't be part of 'UniverseManager'.
132  // And we have to remove the universe even if it's present at 'universes' because that one is another New universe that should get added!
133  // 'UniverseManager' will skip duplicate entries getting added.
134  UniverseManager.Remove(universeSymbol);
135  }
136 
137  Universe universe;
138  if (!universes.TryGetValue(universeSymbol, out universe))
139  {
140  // mark this universe as disposed to remove all child subscriptions
141  ukvp.Value.Dispose();
142  }
143  }
144 
145  // add newly selected universes
146  foreach (var ukvp in universes)
147  {
148  // note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here
149  UniverseManager.Add(ukvp);
150  }
151  }
152 
153  // update scores
155 
156  // we only want to run universe selection if there's no data available in the slice
157  if (!slice.HasData)
158  {
159  return;
160  }
161 
162  // insight timestamping handled via InsightsGenerated event handler
163  var insightsEnumerable = Alpha.Update(this, slice);
164  // for performance only call 'ToArray' if not empty enumerable (which is static)
165  var insights = insightsEnumerable == Enumerable.Empty<Insight>()
166  ? new Insight[] { } : insightsEnumerable.ToArray();
167 
168  // only fire insights generated event if we actually have insights
169  if (insights.Length != 0)
170  {
171  insights = InitializeInsights(insights);
172  OnInsightsGenerated(insights);
173  }
174 
175  ProcessInsights(insights);
176  }
177 
178  /// <summary>
179  /// They different framework models will process the new provided insight.
180  /// The <see cref="IPortfolioConstructionModel"/> will create targets,
181  /// the <see cref="IRiskManagementModel"/> will adjust the targets
182  /// and the <see cref="IExecutionModel"/> will execute the <see cref="IPortfolioTarget"/>
183  /// </summary>
184  /// <param name="insights">The insight to process</param>
185  [DocumentationAttribute(AlgorithmFramework)]
186  private void ProcessInsights(Insight[] insights)
187  {
188  // construct portfolio targets from insights
189  var targetsEnumerable = PortfolioConstruction.CreateTargets(this, insights);
190  // for performance only call 'ToArray' if not empty enumerable (which is static)
191  var targets = targetsEnumerable == Enumerable.Empty<IPortfolioTarget>()
192  ? new IPortfolioTarget[] {} : targetsEnumerable.ToArray();
193 
194  // set security targets w/ those generated via portfolio construction module
195  foreach (var target in targets)
196  {
197  var security = Securities[target.Symbol];
198  security.Holdings.Target = target;
199  }
200 
201  if (DebugMode)
202  {
203  // debug printing of generated targets
204  if (targets.Length > 0)
205  {
206  Log($"{Time}: PORTFOLIO: {string.Join(" | ", targets.Select(t => t.ToString()).OrderBy(t => t))}");
207  }
208  }
209 
210  var riskTargetOverridesEnumerable = RiskManagement.ManageRisk(this, targets);
211  // for performance only call 'ToArray' if not empty enumerable (which is static)
212  var riskTargetOverrides = riskTargetOverridesEnumerable == Enumerable.Empty<IPortfolioTarget>()
213  ? new IPortfolioTarget[] { } : riskTargetOverridesEnumerable.ToArray();
214 
215  // override security targets w/ those generated via risk management module
216  foreach (var target in riskTargetOverrides)
217  {
218  var security = Securities[target.Symbol];
219  security.Holdings.Target = target;
220  }
221 
222  if (DebugMode)
223  {
224  // debug printing of generated risk target overrides
225  if (riskTargetOverrides.Length > 0)
226  {
227  Log($"{Time}: RISK: {string.Join(" | ", riskTargetOverrides.Select(t => t.ToString()).OrderBy(t => t))}");
228  }
229  }
230 
231  IPortfolioTarget[] riskAdjustedTargets;
232  // for performance we check the length before
233  if (riskTargetOverrides.Length != 0
234  || targets.Length != 0)
235  {
236  // execute on the targets, overriding targets for symbols w/ risk targets
237  riskAdjustedTargets = riskTargetOverrides.Concat(targets).DistinctBy(pt => pt.Symbol).ToArray();
238  }
239  else
240  {
241  riskAdjustedTargets = new IPortfolioTarget[] { };
242  }
243 
244  if (DebugMode)
245  {
246  // only log adjusted targets if we've performed an adjustment
247  if (riskTargetOverrides.Length > 0)
248  {
249  Log($"{Time}: RISK ADJUSTED TARGETS: {string.Join(" | ", riskAdjustedTargets.Select(t => t.ToString()).OrderBy(t => t))}");
250  }
251  }
252 
253  Execution.Execute(this, riskAdjustedTargets);
254  }
255 
256  /// <summary>
257  /// Used to send security changes to algorithm framework models
258  /// </summary>
259  /// <param name="changes">Security additions/removals for this time step</param>
260  [DocumentationAttribute(AlgorithmFramework)]
261  [DocumentationAttribute(Universes)]
263  {
264  if (DebugMode)
265  {
266  Debug($"{Time}: {changes}");
267  }
268 
269  Alpha.OnSecuritiesChanged(this, changes);
271  Execution.OnSecuritiesChanged(this, changes);
272  RiskManagement.OnSecuritiesChanged(this, changes);
273  }
274 
275  /// <summary>
276  /// Sets the universe selection model
277  /// </summary>
278  /// <param name="universeSelection">Model defining universes for the algorithm</param>
279  [DocumentationAttribute(AlgorithmFramework)]
280  [DocumentationAttribute(Universes)]
281  public void SetUniverseSelection(IUniverseSelectionModel universeSelection)
282  {
283  UniverseSelection = universeSelection;
284  }
285 
286  /// <summary>
287  /// Adds a new universe selection model
288  /// </summary>
289  /// <param name="universeSelection">Model defining universes for the algorithm to add</param>
290  [DocumentationAttribute(AlgorithmFramework)]
291  [DocumentationAttribute(Universes)]
292  public void AddUniverseSelection(IUniverseSelectionModel universeSelection)
293  {
294  if (UniverseSelection.GetType() != typeof(NullUniverseSelectionModel))
295  {
296  var compositeUniverseSelection = UniverseSelection as CompositeUniverseSelectionModel;
297  if (compositeUniverseSelection != null)
298  {
299  compositeUniverseSelection.AddUniverseSelection(universeSelection);
300  }
301  else
302  {
304  }
305  }
306  else
307  {
308  UniverseSelection = universeSelection;
309  }
310  }
311 
312  /// <summary>
313  /// Sets the alpha model
314  /// </summary>
315  /// <param name="alpha">Model that generates alpha</param>
316  [DocumentationAttribute(AlgorithmFramework)]
317  public void SetAlpha(IAlphaModel alpha)
318  {
319  Alpha = alpha;
320  }
321 
322  /// <summary>
323  /// Adds a new alpha model
324  /// </summary>
325  /// <param name="alpha">Model that generates alpha to add</param>
326  [DocumentationAttribute(AlgorithmFramework)]
327  public void AddAlpha(IAlphaModel alpha)
328  {
329  if (Alpha.GetType() != typeof(NullAlphaModel))
330  {
331  var compositeAlphaModel = Alpha as CompositeAlphaModel;
332  if (compositeAlphaModel != null)
333  {
334  compositeAlphaModel.AddAlpha(alpha);
335  }
336  else
337  {
338  Alpha = new CompositeAlphaModel(Alpha, alpha);
339  }
340  }
341  else
342  {
343  Alpha = alpha;
344  }
345  }
346 
347  /// <summary>
348  /// Sets the portfolio construction model
349  /// </summary>
350  /// <param name="portfolioConstruction">Model defining how to build a portfolio from insights</param>
351  [DocumentationAttribute(AlgorithmFramework)]
352  [DocumentationAttribute(TradingAndOrders)]
353  public void SetPortfolioConstruction(IPortfolioConstructionModel portfolioConstruction)
354  {
355  PortfolioConstruction = portfolioConstruction;
356  }
357 
358  /// <summary>
359  /// Sets the execution model
360  /// </summary>
361  /// <param name="execution">Model defining how to execute trades to reach a portfolio target</param>
362  [DocumentationAttribute(AlgorithmFramework)]
363  [DocumentationAttribute(TradingAndOrders)]
364  public void SetExecution(IExecutionModel execution)
365  {
366  Execution = execution;
367  }
368 
369  /// <summary>
370  /// Sets the risk management model
371  /// </summary>
372  /// <param name="riskManagement">Model defining how risk is managed</param>
373  [DocumentationAttribute(AlgorithmFramework)]
374  [DocumentationAttribute(TradingAndOrders)]
375  public void SetRiskManagement(IRiskManagementModel riskManagement)
376  {
377  RiskManagement = riskManagement;
378  }
379 
380  /// <summary>
381  /// Adds a new risk management model
382  /// </summary>
383  /// <param name="riskManagement">Model defining how risk is managed to add</param>
384  [DocumentationAttribute(AlgorithmFramework)]
385  [DocumentationAttribute(TradingAndOrders)]
386  public void AddRiskManagement(IRiskManagementModel riskManagement)
387  {
388  if (RiskManagement.GetType() != typeof(NullRiskManagementModel))
389  {
390  var compositeRiskModel = RiskManagement as CompositeRiskManagementModel;
391  if (compositeRiskModel != null)
392  {
393  compositeRiskModel.AddRiskManagement(riskManagement);
394  }
395  else
396  {
398  }
399  }
400  else
401  {
402  RiskManagement = riskManagement;
403  }
404  }
405 
406  /// <summary>
407  /// Manually emit insights from an algorithm.
408  /// This is typically invoked before calls to submit orders in algorithms written against
409  /// QCAlgorithm that have been ported into the algorithm framework.
410  /// </summary>
411  /// <param name="insights">The array of insights to be emitted</param>
412  [DocumentationAttribute(AlgorithmFramework)]
413  public void EmitInsights(params Insight[] insights)
414  {
415  if (IsWarmingUp)
416  {
417  if (!_isEmitWarmupInsightWarningSent)
418  {
419  Error("Warning: insights emitted during algorithm warmup are ignored.");
420  _isEmitWarmupInsightWarningSent = true;
421  }
422  return;
423  }
424 
425  insights = InitializeInsights(insights);
426  OnInsightsGenerated(insights);
427  ProcessInsights(insights);
428  }
429 
430  /// <summary>
431  /// Manually emit insights from an algorithm.
432  /// This is typically invoked before calls to submit orders in algorithms written against
433  /// QCAlgorithm that have been ported into the algorithm framework.
434  /// </summary>
435  /// <param name="insight">The insight to be emitted</param>
436  [DocumentationAttribute(AlgorithmFramework)]
437  public void EmitInsights(Insight insight)
438  {
439  EmitInsights(new[] { insight });
440  }
441 
442  /// <summary>
443  /// Helper method used to validate insights and prepare them to be emitted
444  /// </summary>
445  /// <param name="insights">insights preparing to be emitted</param>
446  /// <returns>Validated insights</returns>
447  private Insight[] InitializeInsights(Insight[] insights)
448  {
449  List<Insight> validInsights = null;
450  for (var i = 0; i < insights.Length; i++)
451  {
452  var security = Securities[insights[i].Symbol];
453  if (security.IsDelisted)
454  {
455  if (!_isEmitDelistedInsightWarningSent)
456  {
457  Error($"QCAlgorithm.EmitInsights(): Warning: cannot emit insights for delisted securities, these will be discarded");
458  _isEmitDelistedInsightWarningSent = true;
459  }
460 
461  // If this is our first invalid insight, create the list and fill it with previous values
462  if (validInsights == null)
463  {
464  validInsights = new List<Insight>() {};
465  for (var j = 0; j < i; j++)
466  {
467  validInsights.Add(insights[j]);
468  }
469  }
470  }
471  else
472  {
473  // Initialize the insight fields
474  insights[i] = InitializeInsightFields(insights[i], security);
475 
476  // If we already had an invalid insight, this will have been initialized storing the valid ones.
477  if (validInsights != null)
478  {
479  validInsights.Add(insights[i]);
480  }
481  }
482  }
483 
484  return validInsights == null ? insights : validInsights.ToArray();
485 
486  }
487 
488  /// <summary>
489  /// Helper class used to set values not required to be set by alpha models
490  /// </summary>
491  /// <param name="insight">The <see cref="Insight"/> to set the values for</param>
492  /// <param name="security">The <see cref="Security"/> instance associated with this insight</param>
493  /// <returns>The same <see cref="Insight"/> instance with the values set</returns>
494  private Insight InitializeInsightFields(Insight insight, Security security)
495  {
496  insight.GeneratedTimeUtc = UtcTime;
497  switch (insight.Type)
498  {
499  case InsightType.Price:
500  insight.ReferenceValue = security.Price;
501  break;
502  case InsightType.Volatility:
503  insight.ReferenceValue = security.VolatilityModel.Volatility;
504  break;
505  }
506  insight.SourceModel = string.IsNullOrEmpty(insight.SourceModel) ? Alpha.GetModelName() : insight.SourceModel;
507 
508  var exchangeHours = MarketHoursDatabase.GetExchangeHours(insight.Symbol.ID.Market, insight.Symbol, insight.Symbol.SecurityType);
509  insight.SetPeriodAndCloseTime(exchangeHours);
510  return insight;
511  }
512  }
513 }