17 using System.Collections.Generic;
18 using System.Globalization;
45 .Select(x => x.ToLowerInvariant()).Union(
new[] {
"alternative" }).ToHashSet();
55 var clone = data.
Clone();
56 clone.
Time = data.
Time.ConvertTo(exchangeTimeZone, dataTimeZone);
57 return GenerateLine(clone, clone.Symbol.ID.SecurityType, resolution);
70 var index = line.IndexOf(
',', StringComparison.InvariantCulture);
71 return date.AddTicks(Convert.ToInt64(10000 * decimal.Parse(line.AsSpan(0, index))));
76 throw new ArgumentOutOfRangeException(nameof(resolution), resolution,
null);
85 var milliseconds = data.
Time.TimeOfDay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);
94 var tick = (
Tick) data;
97 return ToCsv(milliseconds, Scale(tick.LastPrice), tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ?
"1" :
"0");
101 return ToCsv(milliseconds, Scale(tick.BidPrice), tick.BidSize, Scale(tick.AskPrice), tick.AskSize, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ?
"1" :
"0");
107 if (tradeBar !=
null)
109 return ToCsv(milliseconds, Scale(tradeBar.Open), Scale(tradeBar.High), Scale(tradeBar.Low), Scale(tradeBar.Close), tradeBar.Volume);
112 if (quoteBar !=
null)
114 return ToCsv(milliseconds,
115 ToScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
116 ToScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
123 if (bigTradeBar !=
null)
125 return ToCsv(longTime, Scale(bigTradeBar.Open), Scale(bigTradeBar.High), Scale(bigTradeBar.Low), Scale(bigTradeBar.Close), bigTradeBar.Volume);
128 if (bigQuoteBar !=
null)
130 return ToCsv(longTime,
131 ToScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
132 ToScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
143 var tick = data as
Tick;
146 throw new ArgumentException($
"{securityType} tick could not be created", nameof(data));
148 if (tick.TickType ==
TickType.Trade)
150 return ToCsv(milliseconds, tick.LastPrice, tick.Quantity, tick.Suspicious ?
"1" :
"0");
152 if (tick.TickType ==
TickType.Quote)
154 return ToCsv(milliseconds, tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.Suspicious ?
"1" :
"0");
156 throw new ArgumentException($
"{securityType} tick could not be created");
160 if (quoteBar !=
null)
162 return ToCsv(milliseconds,
163 ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
164 ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
167 if (tradeBar !=
null)
169 return ToCsv(milliseconds, tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
171 throw new ArgumentException($
"{securityType} minute/second bar could not be created", nameof(data));
176 if (bigQuoteBar !=
null)
178 return ToCsv(longTime,
179 ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
180 ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
183 if (bigTradeBar !=
null)
185 return ToCsv(longTime,
192 throw new ArgumentException($
"{securityType} hour/daily bar could not be created", nameof(data));
200 var tick = data as
Tick;
203 throw new ArgumentException(
"Expected data of type 'Tick'", nameof(data));
205 return ToCsv(milliseconds, tick.BidPrice, tick.AskPrice);
212 throw new ArgumentException(
"Expected data of type 'QuoteBar'", nameof(data));
214 return ToCsv(milliseconds,
215 ToNonScaledCsv(bar.Bid), bar.LastBidSize,
216 ToNonScaledCsv(bar.Ask), bar.LastAskSize);
223 throw new ArgumentException(
"Expected data of type 'QuoteBar'", nameof(data));
225 return ToCsv(longTime,
226 ToNonScaledCsv(bigBar.Bid), bigBar.LastBidSize,
227 ToNonScaledCsv(bigBar.Ask), bigBar.LastAskSize);
235 var tick = (
Tick) data;
236 return ToCsv(milliseconds, tick.LastPrice, tick.Quantity,
string.Empty,
string.Empty,
"0");
242 throw new ArgumentException(
"Expected data of type 'TradeBar'", nameof(data));
244 return ToCsv(milliseconds, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume);
248 return ToCsv(longTime, bigTradeBar.Open, bigTradeBar.High, bigTradeBar.Low, bigTradeBar.Close, bigTradeBar.Volume);
257 var tick = (
Tick)data;
258 if (tick.TickType ==
TickType.Trade)
260 return ToCsv(milliseconds,
261 Scale(tick.LastPrice), tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ?
"1" :
"0");
263 if (tick.TickType ==
TickType.Quote)
265 return ToCsv(milliseconds,
266 Scale(tick.BidPrice), tick.BidSize, Scale(tick.AskPrice), tick.AskSize, tick.ExchangeCode, tick.Suspicious ?
"1" :
"0");
268 if (tick.TickType ==
TickType.OpenInterest)
270 return ToCsv(milliseconds, tick.Value);
278 if (quoteBar !=
null)
280 return ToCsv(milliseconds,
281 ToScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
282 ToScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
285 if (tradeBar !=
null)
287 return ToCsv(milliseconds,
288 Scale(tradeBar.Open), Scale(tradeBar.High), Scale(tradeBar.Low), Scale(tradeBar.Close), tradeBar.Volume);
291 if (openInterest !=
null)
293 return ToCsv(milliseconds, openInterest.Value);
301 if (bigQuoteBar !=
null)
303 return ToCsv(longTime,
304 ToScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
305 ToScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
308 if (bigTradeBar !=
null)
310 return ToCsv(longTime, ToScaledCsv(bigTradeBar), bigTradeBar.Volume);
313 if (bigOpenInterest !=
null)
315 return ToCsv(longTime, bigOpenInterest.Value);
320 throw new ArgumentOutOfRangeException(nameof(resolution), resolution,
null);
328 var tick = (
Tick)data;
329 if (tick.TickType ==
TickType.Trade)
331 return ToCsv(milliseconds,
332 tick.LastPrice, tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ?
"1" :
"0");
334 if (tick.TickType ==
TickType.Quote)
336 return ToCsv(milliseconds,
337 tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.ExchangeCode, tick.Suspicious ?
"1" :
"0");
339 if (tick.TickType ==
TickType.OpenInterest)
341 return ToCsv(milliseconds, tick.Value);
349 if (quoteBar !=
null)
351 return ToCsv(milliseconds,
352 ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
353 ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
356 if (tradeBar !=
null)
358 return ToCsv(milliseconds,
359 tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
362 if (openInterest !=
null)
364 return ToCsv(milliseconds, openInterest.Value);
372 if (bigQuoteBar !=
null)
374 return ToCsv(longTime,
375 ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
376 ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
379 if (bigTradeBar !=
null)
381 return ToCsv(longTime, ToNonScaledCsv(bigTradeBar), bigTradeBar.Volume);
384 if (bigOpenInterest !=
null)
386 return ToCsv(longTime, bigOpenInterest.Value);
391 throw new ArgumentOutOfRangeException(nameof(resolution), resolution,
null);
399 var tick = (
Tick)data;
400 if (tick.TickType ==
TickType.Trade)
402 return ToCsv(milliseconds,
403 tick.LastPrice, tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ?
"1":
"0");
405 if (tick.TickType ==
TickType.Quote)
407 return ToCsv(milliseconds,
408 tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.ExchangeCode, tick.Suspicious ?
"1" :
"0");
410 if (tick.TickType ==
TickType.OpenInterest)
412 return ToCsv(milliseconds, tick.Value);
420 if (quoteBar !=
null)
422 return ToCsv(milliseconds,
423 ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
424 ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
427 if (tradeBar !=
null)
429 return ToCsv(milliseconds,
430 tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
433 if (openInterest !=
null)
435 return ToCsv(milliseconds, openInterest.Value);
443 if (bigQuoteBar !=
null)
445 return ToCsv(longTime,
446 ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
447 ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
450 if (bigTradeBar !=
null)
452 return ToCsv(longTime, ToNonScaledCsv(bigTradeBar), bigTradeBar.Volume);
455 if (bigOpenInterest !=
null)
457 return ToCsv(longTime, bigOpenInterest.Value);
462 throw new ArgumentOutOfRangeException(nameof(resolution), resolution,
null);
467 throw new ArgumentOutOfRangeException(nameof(securityType), securityType,
null);
470 throw new NotImplementedException(Invariant(
471 $
"LeanData.GenerateLine has not yet been implemented for security type: {securityType} at resolution: {resolution}"
499 if (baseDataType == typeof(
Tick) ||
545 var securityType = symbol.
SecurityType.SecurityTypeToLower();
547 var market = symbol.
ID.
Market.ToLowerInvariant();
548 var res = resolution.ResolutionToLower();
549 var directory = Path.Combine(securityType, market, res);
558 return !isHourOrDaily ? Path.Combine(directory, symbol.
Value.ToLowerInvariant()) : directory;
562 return !isHourOrDaily ? Path.Combine(directory, symbol.
ID.
Symbol.ToLowerInvariant()) : directory;
566 return !isHourOrDaily ? Path.Combine(directory, symbol.
Underlying.
Value.ToLowerInvariant()) : directory;
575 return Path.Combine(directory, futureOptionPath);
579 return !isHourOrDaily ? Path.Combine(directory, symbol.
ID.
Symbol.ToLowerInvariant()) : directory;
583 throw new ArgumentOutOfRangeException();
596 symbol.
Value.ToLowerInvariant() +
".csv");
612 var directory = Path.Combine(securityType.SecurityTypeToLower(), market.ToLowerInvariant(), resolution.ResolutionToLower());
615 directory = Path.Combine(directory, symbol.ToLowerInvariant());
618 return Path.Combine(directory,
GenerateZipFileName(symbol, securityType, date, resolution));
639 return Invariant($
"{formattedDate}_{symbol.Value.ToLowerInvariant()}_{tickType}_{resolution}.csv");
644 return $
"{symbol.Value.ToLowerInvariant()}.csv";
647 return Invariant($
"{formattedDate}_{symbol.Value.ToLowerInvariant()}_{resolution.ResolutionToLower()}_{tickType.TickTypeToLower()}.csv");
654 return string.Join(
"_",
656 tickType.TickTypeToLower(),
664 return string.Join(
"_",
667 resolution.ResolutionToLower(),
668 tickType.TickTypeToLower(),
678 var optionTickerBasedPath = symbol.
ID.
Symbol.ToLowerInvariant();
682 return string.Join(
"_",
683 optionTickerBasedPath,
684 tickType.TickTypeToLower(),
692 return string.Join(
"_",
694 optionTickerBasedPath,
695 resolution.ResolutionToLower(),
696 tickType.TickTypeToLower(),
710 string expirationTag;
711 var expiryDate = symbol.
ID.
Date;
715 var contractYearMonth = expiryDate.AddMonths(monthsToAdd).ToStringInvariant(
DateFormat.
YearMonth);
717 expirationTag = $
"{contractYearMonth}_{expiryDate.ToStringInvariant(DateFormat.EightCharacter)}";
721 expirationTag =
"perp";
726 return string.Join(
"_",
727 symbol.
ID.
Symbol.ToLowerInvariant(),
728 tickType.TickTypeToLower(),
733 return string.Join(
"_",
735 symbol.
ID.
Symbol.ToLowerInvariant(),
736 resolution.ResolutionToLower(),
737 tickType.TickTypeToLower(),
743 throw new ArgumentOutOfRangeException();
752 var tickTypeString = tickType.TickTypeToLower();
765 return $
"{symbol.Value.ToLowerInvariant()}.zip";
768 return $
"{formattedDate}_{tickTypeString}.zip";
772 return $
"{symbol.Value.ToLowerInvariant()}_{tickTypeString}.zip";
775 return $
"{formattedDate}_{tickTypeString}.zip";
781 return $
"{optionPath}_{date.Year}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
784 return $
"{formattedDate}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
791 var optionTickerBasedPath = symbol.
ID.
Symbol.ToLowerInvariant();
792 return $
"{optionTickerBasedPath}_{date.Year}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
795 return $
"{formattedDate}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
801 return $
"{symbol.ID.Symbol.ToLowerInvariant()}_{tickTypeString}.zip";
804 return $
"{formattedDate}_{tickTypeString}.zip";
808 throw new ArgumentOutOfRangeException();
819 return $
"{symbol.ToLowerInvariant()}.zip";
824 if (tickType ==
null)
835 var suffix = Invariant($
"_{tickType.Value.TickTypeToLower()}.zip");
836 return zipFileName + suffix;
863 var parts = zipEntryName.Replace(
".csv",
string.Empty).Split(
'_');
871 var style = parts[2].ParseOptionStyle();
872 var right = parts[3].ParseOptionRight();
879 var style = parts[4].ParseOptionStyle();
880 var right = parts[5].ParseOptionRight();
891 var futureExpiry = futureExpiryFunc(expiryYearMonth);
898 var futureExpiry = futureExpiryFunc(expiryYearMonth);
903 throw new NotImplementedException(Invariant(
904 $
"ReadSymbolFromZipEntry is not implemented for {symbol.ID.SecurityType} {symbol.ID.Market} {resolution}"
912 private static long Scale(decimal value)
914 return (
long)(value*10000);
920 private static string ToCsv(params
object[] args)
923 for (var i = 0; i < args.Length; i++)
926 if (value is decimal)
928 args[i] = ((decimal) value).Normalize();
932 var argsFormatted = args.Select(x => Convert.ToString(x, CultureInfo.InvariantCulture));
933 return string.Join(
",", argsFormatted);
939 private static string ToScaledCsv(
IBar bar)
943 return ToCsv(
string.Empty,
string.Empty,
string.Empty,
string.Empty);
946 return ToCsv(Scale(bar.
Open), Scale(bar.
High), Scale(bar.
Low), Scale(bar.
Close));
953 private static string ToNonScaledCsv(
IBar bar)
957 return ToCsv(
string.Empty,
string.Empty,
string.Empty,
string.Empty);
988 if (type == typeof(
Tick))
1004 if (securityType.Equals(
"alternative", StringComparison.InvariantCultureIgnoreCase))
1020 market =
string.Empty;
1024 var info = SplitDataPath(fileName);
1031 var foundMarket = info.Find(x => existingMarkets.Contains(x.ToLowerInvariant()));
1032 if (foundMarket !=
null)
1034 market = foundMarket;
1039 Log.
Error($
"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {e.GetBaseException()}");
1062 resolution =
default;
1066 if (!
TryParsePath(filePath, out symbol, out date, out resolution))
1072 var fileName = Path.GetFileNameWithoutExtension(filePath);
1073 if (fileName.Contains(
"_"))
1076 var tickTypePosition = 1;
1077 if (resolution >=
Resolution.Hour && symbol.SecurityType.IsOption())
1081 tickTypePosition = 2;
1083 tickType = (
TickType)Enum.Parse(typeof(
TickType), fileName.Split(
'_')[tickTypePosition],
true);
1089 catch (Exception ex)
1091 Log.
Debug($
"LeanData.TryParsePath(): Error encountered while parsing the path {filePath}. Error: {ex.GetBaseException()}");
1107 date =
default(DateTime);
1111 var info = SplitDataPath(fileName);
1116 if(startIndex == -1)
1120 Log.
Debug($
"LeanData.TryParsePath(): Failed to parse '{fileName}' unexpected SecurityType");
1129 var isUniverses =
false;
1130 if (!Enum.TryParse(info[startIndex + 2],
true, out resolution))
1133 isUniverses = info[startIndex + 2].Equals(
"universes", StringComparison.InvariantCultureIgnoreCase);
1140 Log.
Debug($
"LeanData.TryParsePath(): Failed to parse '{fileName}' unexpected Resolution");
1152 var fileNameNoPath = info[info.Count - 1].Split(
'_').First();
1154 if (!DateTime.TryParseExact(fileNameNoPath,
1156 DateTimeFormatInfo.InvariantInfo,
1157 DateTimeStyles.None,
1161 ticker = fileNameNoPath;
1166 ticker = info[info.Count - 2];
1172 market = info[startIndex + 1];
1173 var components = info[startIndex + 3].Split(
'_');
1176 ticker = components[0];
1181 var dateIndex = securityType ==
SecurityType.FutureOption ? startIndex + 5 : startIndex + 4;
1187 var year =
int.Parse(components[1], CultureInfo.InvariantCulture);
1188 date =
new DateTime(year, 01, 01);
1200 var underlyingFuture =
Symbol.
CreateFuture(underlyingTicker, market, underlyingFutureExpiryDate);
1212 Type dataType =
null;
1213 if (isUniverses && info[startIndex + 3].
Equals(
"etf", StringComparison.InvariantCultureIgnoreCase))
1217 symbol = CreateSymbol(ticker, securityType, market, dataType, date);
1221 catch (Exception ex)
1223 Log.
Debug($
"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {ex.GetBaseException()}");
1248 private static Symbol CreateSymbol(
string ticker,
SecurityType securityType,
string market, Type dataType, DateTime mappingResolveDate =
default)
1257 return Symbol.
Create(ticker, securityType, market, baseDataType: dataType);
1261 private static List<string> SplitDataPath(
string fileName)
1263 var pathSeparators =
new[] {
'/',
'\\' };
1266 fileName = fileName.Replace(fileName.GetExtension(),
string.Empty);
1269 while (fileName.First() ==
'.' || pathSeparators.Any(x => x == fileName.First()))
1271 fileName = fileName.Remove(0, 1);
1275 return fileName.Split(pathSeparators, StringSplitOptions.RemoveEmptyEntries).ToList();
1332 public static void ParseKey(
string key, out
string fileName, out
string entryName)
1344 var hashIndex = key.LastIndexOf(
"#", StringComparison.Ordinal);
1345 if (hashIndex != -1)
1347 entryName = key.Substring(hashIndex + 1);
1348 fileName = key.Substring(0, hashIndex);
1365 var getConsolidatedBar = () =>
1367 if (lastAggregated != consolidator.Consolidated && consolidator.Consolidated !=
null)
1370 lastAggregated = consolidator.Consolidated;
1371 lastAggregated.
Symbol = symbol;
1372 return lastAggregated;
1377 foreach (var dataPoint
in dataPoints)
1379 consolidator.Update(dataPoint);
1380 var consolidated = getConsolidatedBar();
1381 if (consolidated !=
null)
1383 yield
return (T)consolidated;
1388 consolidator.Scan(Time.EndOfTime);
1389 var lastConsolidated = getConsolidatedBar();
1390 if (lastConsolidated !=
null)
1392 yield
return (T)lastConsolidated;
1396 consolidator.DisposeSafely();