Skip to content

Commit

Permalink
Historical data when symbol was restructured (renamed) (#11)
Browse files Browse the repository at this point in the history 
* feat: historical data for renamed symbol name
feet: symbolMapper with updating cache flag
feat: test

* remove: not used param in tests

* refactor: reducing call in enumerable
refactor: test of renamed symbol
refactor: test, additional helper mthd

* refactor: getInstance of mapFileProvider
fix: test validation on null instand of empty

* test:refactor: Renamed Symbol Historical request

* feat: `volatile` for preventing spamming flags

* fix: getOptionContact capture/return wrong Symbol
  • Loading branch information
@Romazes
Romazes authored Mar 18, 2024
1 parent e0a3180 commit a2d1210
Show file tree
Hide file tree
Showing 8 changed files with 149 additions and 47 deletions.
2 changes: 1 addition & 1 deletion QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period,

[TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void DownloadsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
public void DownloadsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType)
{
var data = DownloadIndexHistoryData(resolution, period, tickType);

Expand Down
76 changes: 60 additions & 16 deletions QuantConnect.Polygon.Tests/PolygonHistoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
using System.Collections.Generic;
using RestSharp;
using QuantConnect.Tests;
using NodaTime;

namespace QuantConnect.Lean.DataSource.Polygon.Tests
{
Expand All @@ -48,7 +49,10 @@ public void SetUp()
[TearDown]
public void TearDown()
{
_historyProvider.Dispose();
if (_historyProvider != null)
{
_historyProvider.Dispose();
}
}

internal static TestCaseData[] HistoricalDataTestCases
Expand Down Expand Up @@ -92,6 +96,28 @@ public void GetsHistoricalData(Symbol symbol, Resolution resolution, TimeSpan pe
AssertHistoricalDataResults(history.Select(x => x.AllData).SelectMany(x => x).ToList(), resolution, _historyProvider.DataPointCount);
}

[TestCase("GOOGL", "2014/4/1", "2016/4/1", Resolution.Daily, Description = "The stock split on July 15 2022. [GOOG -> GOOGL]")]
[TestCase("GOOGL", "2014/4/1", "2014/4/4", Resolution.Hour)]
public void GetsRenamedSymbolHistoricalData(string ticker, DateTime startDateTime, DateTime endDateTime, Resolution resolution)
{
var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA);

var request = CreateHistoryRequest(symbol, resolution, TickType.Trade, startDateTime, endDateTime);

var history = _historyProvider.GetHistory(new[] { request }, TimeZones.NewYork)?.ToList();

Log.Trace("Data points retrieved: " + history.Count);

Assert.IsNotNull(history);
Assert.IsNotEmpty(history);
Assert.That(history.First().Time.Date, Is.EqualTo(startDateTime));
Assert.That(history.Last().Time.Date, Is.LessThanOrEqualTo(endDateTime));
Assert.That(history.First().AllData.First().Symbol.Value, Is.EqualTo("GOOG"));
Assert.That(history.Last().AllData.First().Symbol.Value, Is.EqualTo("GOOGL"));

AssertHistoricalDataResults(history.Select(x => x.AllData).SelectMany(x => x).ToList(), resolution);
}

internal static void AssertHistoricalDataResults(List<BaseData> history, Resolution resolution, int? expectedCount = null)
{
// Assert that we got some data
Expand Down Expand Up @@ -141,18 +167,17 @@ internal static TestCaseData[] IndexHistoricalDataTestCases

[TestCaseSource(nameof(IndexHistoricalDataTestCases))]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeNull)
{
var history = GetIndexHistory(resolution, period, tickType);

Log.Trace("Data points retrieved: " + history.Count);

if (shouldBeEmpty)
if (shouldBeNull)
{
Assert.That(history, Is.Empty);
Assert.IsNull(history);
}
else
{
Log.Trace("Data points retrieved: " + history.Count);
AssertHistoricalDataResults(history.Select(x => x.AllData).SelectMany(x => x).ToList(), resolution, _historyProvider.DataPointCount);
}
}
Expand All @@ -163,16 +188,16 @@ internal static TestCaseData[] IndexHistoricalInvalidDataTestCases
{
return new[]
{
new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote),
new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote),
new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote),
};
}
}

[TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void GetsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
public void GetsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType)
{
var history = GetIndexHistory(resolution, period, tickType);

Expand Down Expand Up @@ -312,24 +337,43 @@ internal List<Slice> GetIndexHistory(Resolution resolution, TimeSpan period, Tic
internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period)
{
var end = new DateTime(2023, 12, 15, 16, 0, 0);

if (resolution == Resolution.Daily)
{
end = end.Date.AddDays(1);
}
var dataType = LeanData.GetDataType(resolution, tickType);

return new HistoryRequest(end.Subtract(period),
end,
return CreateHistoryRequest(symbol, resolution, tickType, end.Subtract(period), end);
}

internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime,
SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null)
{
if (exchangeHours == null)
{
exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork);
}

if (dataTimeZone == null)
{
dataTimeZone = TimeZones.NewYork;
}

var dataType = LeanData.GetDataType(resolution, tickType);
return new HistoryRequest(
startDateTime,
endDateTime,
dataType,
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
exchangeHours,
dataTimeZone,
null,
true,
false,
DataNormalizationMode.Adjusted,
tickType);
tickType
);
}

private class TestPolygonRestApiClient : PolygonRestApiClient
Expand Down
43 changes: 39 additions & 4 deletions QuantConnect.Polygon.Tests/PolygonOptionChainProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@
*
*/

using Moq;
using System;
using RestSharp;
using System.Linq;
using NUnit.Framework;
using QuantConnect.Configuration;
using QuantConnect.Logging;
using System;
using QuantConnect.Configuration;
using System.Collections.Generic;
using System.Linq;

namespace QuantConnect.Lean.DataSource.Polygon.Tests
{
Expand Down Expand Up @@ -52,6 +54,7 @@ public void TearDown()
("AAPL", SecurityType.Equity),
("IBM", SecurityType.Equity),
("GOOG", SecurityType.Equity),
("GOOGL", SecurityType.Equity),
("SPX", SecurityType.Index),
("VIX", SecurityType.Index),
("DAX", SecurityType.Index),
Expand Down Expand Up @@ -103,7 +106,7 @@ public void GetsOptionChainGivenTheOptionSymbol(Symbol option)
[Test]
public void GetsFullSPXOptionChain()
{
var chain = GetOptionChain(Symbol.Create("SPX", SecurityType.Index, Market.USA), new DateTime(2024, 01, 03));
var chain = GetOptionChain(Symbol.Create("SPX", SecurityType.Index, Market.USA), new DateTime(2024, 03, 15));

// SPX has a lot of options, let's make sure we get more than 1000 contracts (which is the pagination limit)
// to assert that multiple requests are being made.
Expand All @@ -120,5 +123,37 @@ public void GetsFullSPXOptionChain()

Assert.That(spxw.Count + spx.Count, Is.EqualTo(chain.Count));
}

[TestCaseSource(nameof(Underlyings))]
public void ValidateQueryParameterToSpecificSymbolValue(Symbol underlyingSymbol)
{
IRestRequest request = default;
var mock = new Mock<PolygonRestApiClient>("api-key");

mock.Setup(m => m.DownloadAndParseData<OptionChainResponse>(It.IsAny<RestRequest>()))
.Callback((RestRequest r) => request = r);

var optionChainProvider = new PolygonOptionChainProvider(mock.Object, new PolygonSymbolMapper());

var expiryDate = new DateTime(2024, 03, 15);
var option = Symbol.CreateOption(underlyingSymbol, Market.USA, OptionStyle.American, OptionRight.Call, 1000m, expiryDate);

var optionContracts = optionChainProvider.GetOptionContractList(option, expiryDate).ToList();

Assert.IsNotNull(optionContracts);
Assert.IsTrue(request.Parameters[0].Value.ToString().EndsWith(option.Underlying.Value));
}

[TestCaseSource(nameof(Underlyings))]
public void ValidateGetOptionContractsReturnsAppropriateSymbol(Symbol underlyingSymbol)
{
var referenceDate = new DateTime(2024, 03, 15);
var optionContracts = _optionChainProvider.GetOptionContractList(underlyingSymbol, referenceDate).ToList();

foreach (var optionContract in optionContracts)
{
Assert.That(optionContract.Underlying, Is.EqualTo(underlyingSymbol));
}
}
}
}
12 changes: 8 additions & 4 deletions QuantConnect.Polygon.Tests/PolygonSymbolMapperTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,18 @@ public void ConvertsPolygonSymbolToLeanSymbol(string polygonSymbol, Symbol leanS
Assert.That(convertedSymbol, Is.EqualTo(leanSymbol));
}

[TestCase("O:SPY240104C00467000", SecurityType.Option, ExpectedResult = SecurityType.Equity)]
[TestCase("O:SPX240104C04685000", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)]
[TestCase("O:SPXW240104C04685000", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)]
public SecurityType ConvertsOptionSymbolWithCorrectUnderlyingSecurityType(string ticker, SecurityType optionType)
[TestCase("O:SPY240104C00467000", "SPY", SecurityType.Option, ExpectedResult = SecurityType.Equity)]
[TestCase("O:GOOGL240105C00050000", "GOOGL", SecurityType.Option, ExpectedResult = SecurityType.Equity)]
[TestCase("O:GOOG240105C00070000", "GOOG", SecurityType.Option, ExpectedResult = SecurityType.Equity)]
[TestCase("O:SPX240104C04685000", "SPX", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)]
[TestCase("O:SPXW240104C04685000", "SPX", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)]
public SecurityType ConvertsOptionSymbolWithCorrectUnderlyingSecurityType(string ticker, string expectedTicker, SecurityType optionType)
{
var mapper = new PolygonSymbolMapper();
var symbol = mapper.GetLeanSymbol(ticker, optionType, Market.USA, new DateTime(2024, 01, 04), 400m, OptionRight.Call);

Assert.That(symbol.Underlying.Value, Is.EqualTo(expectedTicker));

return symbol.Underlying.SecurityType;
}

Expand Down
27 changes: 16 additions & 11 deletions QuantConnect.Polygon/PolygonDataProvider.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/*
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
Expand Down Expand Up @@ -66,11 +66,16 @@ public partial class PolygonDataProvider : IDataQueueHandler
private bool _initialized;
private bool _disposed;

private bool _unsupportedSecurityTypeMessageLogged;
private bool _unsupportedTickTypeMessagedLogged;
private bool _unsupportedDataTypeMessageLogged;
private bool _potentialUnsupportedResolutionMessageLogged;
private bool _potentialUnsupportedTickTypeMessageLogged;
private volatile bool _unsupportedSecurityTypeMessageLogged;
private volatile bool _unsupportedTickTypeMessagedLogged;
private volatile bool _unsupportedDataTypeMessageLogged;
private volatile bool _potentialUnsupportedResolutionMessageLogged;
private volatile bool _potentialUnsupportedTickTypeMessageLogged;

/// <summary>
/// <inheritdoc cref="IMapFileProvider"/>
/// </summary>
private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart<IMapFileProvider>();

/// <summary>
/// The time provider instance. Used for improved testability
Expand Down Expand Up @@ -417,8 +422,8 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick
{
if (!_unsupportedSecurityTypeMessageLogged)
{
Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported security type: {securityType}");
_unsupportedSecurityTypeMessageLogged = true;
Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported security type: {securityType}");
}
return false;
}
Expand All @@ -427,8 +432,8 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick
{
if (!_unsupportedTickTypeMessagedLogged)
{
Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported tick type: {tickType}");
_unsupportedTickTypeMessagedLogged = true;
Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported tick type: {tickType}");
}
return false;
}
Expand All @@ -439,26 +444,26 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick
{
if (!_unsupportedDataTypeMessageLogged)
{
Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported data type: {dataType}");
_unsupportedDataTypeMessageLogged = true;
Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported data type: {dataType}");
}
return false;
}

if (resolution < Resolution.Second && !_potentialUnsupportedResolutionMessageLogged)
{
_potentialUnsupportedResolutionMessageLogged = true;
Log.Trace("PolygonDataProvider.IsSupported(): " +
$"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " +
$"An Advanced Polygon.io subscription plan is required to stream tick data.");
_potentialUnsupportedResolutionMessageLogged = true;
}

if (tickType == TickType.Quote && !_potentialUnsupportedTickTypeMessageLogged)
{
_potentialUnsupportedTickTypeMessageLogged = true;
Log.Trace("PolygonDataProvider.IsSupported(): " +
$"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " +
$"An Advanced Polygon.io subscription plan is required to stream quote data.");
_potentialUnsupportedTickTypeMessageLogged = true;
}

return true;
Expand Down
18 changes: 10 additions & 8 deletions QuantConnect.Polygon/PolygonHistoryProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ public partial class PolygonDataProvider : SynchronizingHistoryProvider
/// <summary>
/// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC.
/// </summary>
private bool _invalidStartTimeErrorFired;
private volatile bool _invalidStartTimeErrorFired;

/// <summary>
/// Indicates whether an error has been fired due to invalid conditions if the TickType is <seealso cref="TickType.Quote"/> and the <seealso cref="Resolution"/> is greater than one second.
/// </summary>
private bool _invalidTickTypeAndResolutionErrorFired;
private volatile bool _invalidTickTypeAndResolutionErrorFired;

/// <summary>
/// Gets the total number of data points emitted by this history provider
Expand All @@ -63,12 +63,14 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
var subscriptions = new List<Subscription>();
foreach (var request in requests)
{
var history = GetHistory(request);
if (history == null)
var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? Enumerable.Empty<BaseData>());

var subscription = CreateSubscription(request, history);
if (!subscription.MoveNext())
{
continue;
}
var subscription = CreateSubscription(request, history);

subscriptions.Add(subscription);
}

Expand Down Expand Up @@ -99,8 +101,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
{
if (!_invalidTickTypeAndResolutionErrorFired)
{
Log.Error("PolygonDataProvider.GetHistory(): Quote data above second resolution is not supported.");
_invalidTickTypeAndResolutionErrorFired = true;
Log.Error("PolygonDataProvider.GetHistory(): Quote data above second resolution is not supported.");
}
return null;
}
Expand All @@ -109,8 +111,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
{
if (!_invalidStartTimeErrorFired)
{
Log.Error($"{nameof(PolygonDataProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned");
_invalidStartTimeErrorFired = true;
Log.Error($"{nameof(PolygonDataProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned");
}
return null;
}
Expand Down Expand Up @@ -179,7 +181,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
/// </summary>
private IEnumerable<TradeBar> GetAggregates(HistoryRequest request)
{
var ticker = _symbolMapper.GetBrokerageSymbol(request.Symbol);
var ticker = _symbolMapper.GetBrokerageSymbol(request.Symbol, true);
var resolutionTimeSpan = request.Resolution.ToTimeSpan();
// Aggregates API gets timestamps in milliseconds
var start = Time.DateTimeToUnixTimeStampMilliseconds(request.StartTimeUtc.RoundDown(resolutionTimeSpan));
Expand Down
2 changes: 1 addition & 1 deletion QuantConnect.Polygon/PolygonOptionChainProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public IEnumerable<Symbol> GetOptionContractList(Symbol symbol, DateTime date)
var optionsSecurityType = underlying.SecurityType == SecurityType.Index ? SecurityType.IndexOption : SecurityType.Option;

var request = new RestRequest("/v3/reference/options/contracts", Method.GET);
request.AddQueryParameter("underlying_ticker", underlying.ID.Symbol);
request.AddQueryParameter("underlying_ticker", underlying.Value);
request.AddQueryParameter("as_of", date.ToStringInvariant("yyyy-MM-dd"));
request.AddQueryParameter("limit", "1000");

Expand Down
Loading

0 comments on commit a2d1210

Please sign in to comment.