17 using System.Collections.Generic;
21 using System.Net.Http;
22 using Newtonsoft.Json;
23 using Newtonsoft.Json.Linq;
25 using RestSharp.Extensions;
35 using System.Threading;
36 using System.Net.Http.Headers;
37 using System.Collections.Concurrent;
47 private readonly BlockingCollection<Lazy<HttpClient>> _clientPool;
48 private string _dataFolder;
60 _clientPool =
new BlockingCollection<Lazy<HttpClient>>(
new ConcurrentQueue<Lazy<HttpClient>>(), 5);
61 for (
int i = 0; i < _clientPool.BoundedCapacity; i++)
63 _clientPool.Add(
new Lazy<HttpClient>());
70 public virtual void Initialize(
int userId,
string token,
string dataFolder)
73 _dataFolder = dataFolder?.Replace(
"\\",
"/", StringComparison.InvariantCulture);
76 JsonConvert.DefaultSettings = () =>
new JsonSerializerSettings
98 var request =
new RestRequest(
"projects/create", Method.POST)
100 RequestFormat = DataFormat.Json
105 if (
string.IsNullOrEmpty(organizationId))
107 jsonParams = JsonConvert.SerializeObject(
new
115 jsonParams = JsonConvert.SerializeObject(
new
123 request.AddParameter(
"application/json", jsonParams, ParameterType.RequestBody);
137 var request =
new RestRequest(
"projects/read", Method.POST)
139 RequestFormat = DataFormat.Json
142 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
145 }), ParameterType.RequestBody);
158 var request =
new RestRequest(
"projects/read", Method.POST)
160 RequestFormat = DataFormat.Json
178 var request =
new RestRequest(
"files/create", Method.POST)
180 RequestFormat = DataFormat.Json
183 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
188 }), ParameterType.RequestBody);
205 var request =
new RestRequest(
"files/update", Method.POST)
207 RequestFormat = DataFormat.Json
210 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
214 newName = newFileName
215 }), ParameterType.RequestBody);
232 var request =
new RestRequest(
"files/update", Method.POST)
234 RequestFormat = DataFormat.Json
237 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
241 content = newFileContents
242 }), ParameterType.RequestBody);
257 var request =
new RestRequest(
"files/read", Method.POST)
259 RequestFormat = DataFormat.Json
262 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
265 }), ParameterType.RequestBody);
278 var request =
new RestRequest(
"projects/nodes/read", Method.POST)
280 RequestFormat = DataFormat.Json
283 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
286 }), ParameterType.RequestBody);
301 var request =
new RestRequest(
"projects/nodes/update", Method.POST)
303 RequestFormat = DataFormat.Json
306 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
310 }), ParameterType.RequestBody);
325 var request =
new RestRequest(
"files/read", Method.POST)
327 RequestFormat = DataFormat.Json
330 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
334 }), ParameterType.RequestBody);
345 var request =
new RestRequest(
"lean/versions/read", Method.POST)
347 RequestFormat = DataFormat.Json
363 var request =
new RestRequest(
"files/delete", Method.POST)
365 RequestFormat = DataFormat.Json
368 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
372 }), ParameterType.RequestBody);
386 var request =
new RestRequest(
"projects/delete", Method.POST)
388 RequestFormat = DataFormat.Json
391 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
394 }), ParameterType.RequestBody);
408 var request =
new RestRequest(
"compile/create", Method.POST)
410 RequestFormat = DataFormat.Json
413 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
416 }), ParameterType.RequestBody);
431 var request =
new RestRequest(
"compile/read", Method.POST)
433 RequestFormat = DataFormat.Json
436 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
440 }), ParameterType.RequestBody);
454 throw new NotImplementedException($
"{nameof(Api)} does not support sending notifications");
467 var request =
new RestRequest(
"backtests/create", Method.POST)
469 RequestFormat = DataFormat.Json
472 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
477 }), ParameterType.RequestBody);
482 result.Backtest.Success = result.Success;
483 result.Backtest.Errors = result.Errors;
486 return result.Backtest;
499 var request =
new RestRequest(
"backtests/read", Method.POST)
501 RequestFormat = DataFormat.Json
504 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
508 }), ParameterType.RequestBody);
515 result.Backtest =
new Backtest { BacktestId = backtestId };
518 else if (getCharts && result.Backtest.Completed)
521 var updatedCharts =
new Dictionary<string, Chart>();
524 foreach (var chart
in result.Backtest.Charts)
526 if (!chart.Value.Series.IsNullOrEmpty())
531 var chartRequest =
new RestRequest(
"backtests/read", Method.POST)
533 RequestFormat = DataFormat.Json
536 chartRequest.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
541 }), ParameterType.RequestBody);
546 updatedCharts.Add(chart.Key, chartResponse.Backtest.Charts[chart.Key]);
551 foreach(var updatedChart
in updatedCharts)
553 result.Backtest.Charts[updatedChart.Key] = updatedChart.Value;
558 result.Backtest.Success = result.Success;
559 result.Backtest.Errors = result.Errors;
562 return result.Backtest;
575 public List<ApiOrderResponse>
ReadBacktestOrders(
int projectId,
string backtestId,
int start = 0,
int end = 100)
577 var request =
new RestRequest(
"backtests/read/orders", Method.POST)
579 RequestFormat = DataFormat.Json
582 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
588 }), ParameterType.RequestBody);
590 return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(
ReadBacktestOrders)).Orders;
604 var request =
new RestRequest(
"backtests/update", Method.POST)
606 RequestFormat = DataFormat.Json
609 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
615 }), ParameterType.RequestBody);
630 var request =
new RestRequest(
"backtests/list", Method.POST)
632 RequestFormat = DataFormat.Json
635 var obj =
new Dictionary<string, object>()
637 {
"projectId", projectId },
638 {
"includeStatistics", includeStatistics }
641 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
656 var request =
new RestRequest(
"backtests/delete", Method.POST)
658 RequestFormat = DataFormat.Json
661 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
665 }), ParameterType.RequestBody);
680 var request =
new RestRequest(
"backtests/tags/update", Method.POST)
682 RequestFormat = DataFormat.Json
685 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
690 }), ParameterType.RequestBody);
720 Dictionary<string, object> brokerageSettings,
721 string versionId =
"-1",
722 Dictionary<string, object> dataProviders =
null)
724 var request =
new RestRequest(
"live/create", Method.POST)
726 RequestFormat = DataFormat.Json
729 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
738 ), ParameterType.RequestBody);
768 return CreateLiveAlgorithm(projectId, compileId, nodeId, ConvertToDictionary(brokerageSettings), versionId, dataProviders !=
null ? ConvertToDictionary(dataProviders) :
null);
775 private static Dictionary<string, object> ConvertToDictionary(PyObject brokerageSettings)
779 var stringBrokerageSettings = brokerageSettings.ToString();
780 return JsonConvert.DeserializeObject<Dictionary<string, object>>(stringBrokerageSettings);
793 DateTime? startTime =
null,
794 DateTime? endTime =
null)
797 if (status.HasValue &&
803 throw new ArgumentException(
804 "The Api only supports Algorithm Statuses of Running, Stopped, RuntimeError and Liquidated");
807 var request =
new RestRequest(
"live/read", Method.POST)
809 RequestFormat = DataFormat.Json
815 JObject obj =
new JObject
817 {
"start", epochStartTime },
818 {
"end", epochEndTime }
823 obj.Add(
"status", status.ToString());
826 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
841 var request =
new RestRequest(
"live/read", Method.POST)
843 RequestFormat = DataFormat.Json
846 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
850 }), ParameterType.RequestBody);
865 public List<ApiOrderResponse>
ReadLiveOrders(
int projectId,
int start = 0,
int end = 100)
867 var request =
new RestRequest(
"live/read/orders", Method.POST)
869 RequestFormat = DataFormat.Json
872 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
877 }), ParameterType.RequestBody);
879 return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(
ReadLiveOrders)).Orders;
890 var request =
new RestRequest(
"live/update/liquidate", Method.POST)
892 RequestFormat = DataFormat.Json
895 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
898 }), ParameterType.RequestBody);
912 var request =
new RestRequest(
"live/update/stop", Method.POST)
914 RequestFormat = DataFormat.Json
917 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
920 }), ParameterType.RequestBody);
935 public LiveLog ReadLiveLogs(
int projectId,
string algorithmId, DateTime? startTime =
null, DateTime? endTime =
null)
940 var request =
new RestRequest(
"live/read/log", Method.POST)
942 RequestFormat = DataFormat.Json
945 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
950 start = epochStartTime,
952 }), ParameterType.RequestBody);
966 if (filePath ==
null)
968 throw new ArgumentException(
"Api.ReadDataLink(): Filepath must not be null");
974 var request =
new RestRequest(
"data/read", Method.POST)
976 RequestFormat = DataFormat.Json
979 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
984 }), ParameterType.RequestBody);
996 if (filePath ==
null)
998 throw new ArgumentException(
"Api.ReadDataDirectory(): Filepath must not be null");
1006 if (filePath.Count(x => x ==
'/') < 3)
1008 throw new ArgumentException($
"Api.ReadDataDirectory(): Data directory requested must be at least" +
1009 $
" three directories deep. FilePath: {filePath}");
1012 var request =
new RestRequest(
"data/list", Method.POST)
1014 RequestFormat = DataFormat.Json
1017 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1020 }), ParameterType.RequestBody);
1031 var request =
new RestRequest(
"data/prices", Method.POST)
1033 RequestFormat = DataFormat.Json
1036 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1039 }), ParameterType.RequestBody);
1053 var request =
new RestRequest(
"backtests/read/report", Method.POST)
1055 RequestFormat = DataFormat.Json
1058 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1062 }), ParameterType.RequestBody);
1065 var finish = DateTime.UtcNow.AddMinutes(1);
1066 while (DateTime.UtcNow < finish && !report.
Success)
1068 Thread.Sleep(10000);
1087 if (!dataLink.Success)
1089 Log.
Trace($
"Api.DownloadData(): Failed to get link for {filePath}. " +
1090 $
"Errors: {string.Join(',', dataLink.Errors)}");
1095 var directory = Path.GetDirectoryName(filePath);
1096 if (!Directory.Exists(directory))
1098 Directory.CreateDirectory(directory);
1101 var client = BorrowClient();
1105 var uri =
new Uri(dataLink.Url);
1106 using var dataStream = client.Value.GetStreamAsync(uri);
1109 dataStream.Result.CopyTo(fileStream);
1113 Log.
Error($
"Api.DownloadData(): Failed to download zip for path ({filePath})");
1118 ReturnClient(client);
1134 ChartSubscription =
"*"
1165 public virtual void SendStatistics(
string algorithmId, decimal unrealized, decimal fees, decimal netProfit, decimal holdings, decimal equity, decimal netReturn, decimal volume,
int trades,
double sharpe)
1177 public virtual void SendUserEmail(
string algorithmId,
string subject,
string body)
1190 public virtual string Download(
string address, IEnumerable<KeyValuePair<string, string>> headers,
string userName,
string password)
1192 return Encoding.UTF8.GetString(
DownloadBytes(address, headers, userName, password));
1204 public virtual byte[]
DownloadBytes(
string address, IEnumerable<KeyValuePair<string, string>> headers,
string userName,
string password)
1206 var client = BorrowClient();
1209 client.Value.DefaultRequestHeaders.Clear();
1212 client.Value.DefaultRequestHeaders.TryAddWithoutValidation(
"user-agent",
"QCAlgorithm.Download(): User Agent Header");
1214 if (headers !=
null)
1216 foreach (var header
in headers)
1218 client.Value.DefaultRequestHeaders.Add(header.Key, header.Value);
1222 if (!userName.IsNullOrEmpty() || !password.IsNullOrEmpty())
1224 var credentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($
"{userName}:{password}"));
1225 client.Value.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue(
"Basic", credentials);
1228 return client.Value.GetByteArrayAsync(
new Uri(address)).Result;
1230 catch (Exception exception)
1232 var message = $
"Api.DownloadBytes(): Failed to download data from {address}";
1233 if (!userName.IsNullOrEmpty() || !password.IsNullOrEmpty())
1235 message += $
" with username: {userName} and password {password}";
1238 throw new WebException($
"{message}. Please verify the source for missing http:// or https://", exception);
1242 client.Value.DefaultRequestHeaders.Clear();
1243 ReturnClient(client);
1254 _clientPool.CompleteAdding();
1255 foreach (var client
in _clientPool.GetConsumingEnumerable())
1257 if (client.IsValueCreated)
1259 client.Value.DisposeSafely();
1262 _clientPool.DisposeSafely();
1273 var data = $
"{token}:{timestamp.ToStringInvariant()}";
1274 return data.ToSHA256();
1283 var request =
new RestRequest(
"account/read", Method.POST)
1285 RequestFormat = DataFormat.Json
1288 if (organizationId !=
null)
1290 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new { organizationId }), ParameterType.RequestBody);
1304 var request =
new RestRequest(
"organizations/read", Method.POST)
1306 RequestFormat = DataFormat.Json
1309 if (organizationId !=
null)
1311 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new { organizationId }), ParameterType.RequestBody);
1315 return response.Organization;
1336 decimal? targetValue,
1339 HashSet<OptimizationParameter> parameters,
1340 IReadOnlyList<Constraint> constraints)
1342 var request =
new RestRequest(
"optimizations/estimate", Method.POST)
1344 RequestFormat = DataFormat.Json
1347 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1358 }), ParameterType.RequestBody);
1361 return response.Estimate;
1385 decimal? targetValue,
1388 HashSet<OptimizationParameter> parameters,
1389 IReadOnlyList<Constraint> constraints,
1390 decimal estimatedCost,
1394 var request =
new RestRequest(
"optimizations/create", Method.POST)
1396 RequestFormat = DataFormat.Json
1399 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1413 }), ParameterType.RequestBody);
1416 return result.Optimizations.FirstOrDefault();
1426 var request =
new RestRequest(
"optimizations/list", Method.POST)
1428 RequestFormat = DataFormat.Json
1431 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1434 }), ParameterType.RequestBody);
1437 return result.Optimizations;
1447 var request =
new RestRequest(
"optimizations/read", Method.POST)
1449 RequestFormat = DataFormat.Json
1452 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1455 }), ParameterType.RequestBody);
1458 return response.Optimization;
1468 var request =
new RestRequest(
"optimizations/abort", Method.POST)
1470 RequestFormat = DataFormat.Json
1473 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1476 }), ParameterType.RequestBody);
1490 var request =
new RestRequest(
"optimizations/update", Method.POST)
1492 RequestFormat = DataFormat.Json
1495 var obj =
new JObject
1497 {
"optimizationId", optimizationId }
1500 if (name.HasValue())
1502 obj.Add(
"name", name);
1505 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
1518 var request =
new RestRequest(
"optimizations/delete", Method.POST)
1520 RequestFormat = DataFormat.Json
1523 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1526 }), ParameterType.RequestBody);
1539 public bool GetObjectStore(
string organizationId, List<string> keys,
string destinationFolder =
null)
1541 var request =
new RestRequest(
"object/get", Method.POST)
1543 RequestFormat = DataFormat.Json
1546 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1550 }), ParameterType.RequestBody);
1554 if (result ==
null || !result.Success)
1556 Log.
Error($
"Api.GetObjectStore(): Failed to get the jobId to request the download URL for the object store files."
1557 + (result !=
null ? $
" Errors: {string.Join(",
", result.Errors)}" :
""));
1561 var jobId = result.JobId;
1562 var getUrlRequest =
new RestRequest(
"object/get", Method.POST)
1564 RequestFormat = DataFormat.Json
1566 getUrlRequest.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1570 }), ParameterType.RequestBody);
1572 var frontier = DateTime.UtcNow + TimeSpan.FromMinutes(5);
1573 while (
string.IsNullOrEmpty(result?.Url) && (DateTime.UtcNow < frontier))
1579 if (result ==
null ||
string.IsNullOrEmpty(result.Url))
1581 Log.
Error($
"Api.GetObjectStore(): Failed to get the download URL from the jobId {jobId}."
1582 + (result !=
null ? $
" Errors: {string.Join(",
", result.Errors)}" :
""));
1586 var directory = destinationFolder ?? Directory.GetCurrentDirectory();
1587 var client = BorrowClient();
1591 if (client.Value.Timeout != TimeSpan.FromMinutes(20))
1593 client.Value.Timeout = TimeSpan.FromMinutes(20);
1597 var uri =
new Uri(result.Url);
1598 using var byteArray = client.Value.GetByteArrayAsync(uri);
1604 Log.
Error($
"Api.GetObjectStore(): Failed to download zip for path ({directory}). Error: {e.Message}");
1609 ReturnClient(client);
1624 var request =
new RestRequest(
"object/properties", Method.POST)
1626 RequestFormat = DataFormat.Json
1629 request.AddParameter(
"organizationId", organizationId);
1630 request.AddParameter(
"key", key);
1634 if (result ==
null || !result.Success)
1636 Log.
Error($
"Api.ObjectStore(): Failed to get the properties for the object store key {key}." + (result !=
null ? $
" Errors: {string.Join(",
", result.Errors)}" :
""));
1650 var request =
new RestRequest(
"object/set", Method.POST)
1652 RequestFormat = DataFormat.Json
1655 request.AddParameter(
"organizationId", organizationId);
1656 request.AddParameter(
"key", key);
1657 request.AddFileBytes(
"objectData", objectData,
"objectData");
1658 request.AlwaysMultipartFormData =
true;
1672 var request =
new RestRequest(
"object/delete", Method.POST)
1674 RequestFormat = DataFormat.Json
1677 var obj =
new Dictionary<string, object>
1679 {
"organizationId", organizationId },
1683 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
1697 var request =
new RestRequest(
"object/list", Method.POST)
1699 RequestFormat = DataFormat.Json
1702 var obj =
new Dictionary<string, object>
1704 {
"organizationId", organizationId },
1708 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
1722 if (filePath ==
null)
1724 Log.
Error(
"Api.FormatPathForDataRequest(): Cannot format null string");
1730 dataFolder = dataFolder.Replace(
"\\",
"/", StringComparison.InvariantCulture);
1731 filePath = filePath.Replace(
"\\",
"/", StringComparison.InvariantCulture);
1734 if (filePath.StartsWith(dataFolder, StringComparison.InvariantCulture))
1736 filePath = filePath.Substring(dataFolder.Length);
1740 filePath = filePath.TrimStart(
'/');
1747 private T MakeRequestOrThrow<T>(RestRequest request,
string callerName)
1752 var errors =
string.Empty;
1753 if (result !=
null && result.Errors !=
null && result.Errors.Count > 0)
1755 errors = $
". Errors: ['{string.Join(",
", result.Errors)}']";
1757 throw new WebException($
"{callerName} api request failed{errors}");
1766 private Lazy<HttpClient> BorrowClient()
1768 using var cancellationTokenSource =
new CancellationTokenSource(TimeSpan.FromMinutes(10));
1769 return _clientPool.Take(cancellationTokenSource.Token);
1775 private void ReturnClient(Lazy<HttpClient> client)
1777 _clientPool.Add(client);