diff --git a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs index 6664e69..d6d258e 100644 --- a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs @@ -81,7 +81,7 @@ public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, [TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))] [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] - public void DownloadsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) + public void DownloadsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType) { var data = DownloadIndexHistoryData(resolution, period, tickType); diff --git a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs index c16c4d4..c1a671e 100644 --- a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs @@ -28,6 +28,7 @@ using System.Collections.Generic; using RestSharp; using QuantConnect.Tests; +using NodaTime; namespace QuantConnect.Lean.DataSource.Polygon.Tests { @@ -48,7 +49,10 @@ public void SetUp() [TearDown] public void TearDown() { - _historyProvider.Dispose(); + if (_historyProvider != null) + { + _historyProvider.Dispose(); + } } internal static TestCaseData[] HistoricalDataTestCases @@ -92,6 +96,28 @@ public void GetsHistoricalData(Symbol symbol, Resolution resolution, TimeSpan pe AssertHistoricalDataResults(history.Select(x => x.AllData).SelectMany(x => x).ToList(), resolution, _historyProvider.DataPointCount); } + [TestCase("GOOGL", "2014/4/1", "2016/4/1", Resolution.Daily, Description = "The stock split on July 15 2022. [GOOG -> GOOGL]")] + [TestCase("GOOGL", "2014/4/1", "2014/4/4", Resolution.Hour)] + public void GetsRenamedSymbolHistoricalData(string ticker, DateTime startDateTime, DateTime endDateTime, Resolution resolution) + { + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + + var request = CreateHistoryRequest(symbol, resolution, TickType.Trade, startDateTime, endDateTime); + + var history = _historyProvider.GetHistory(new[] { request }, TimeZones.NewYork)?.ToList(); + + Log.Trace("Data points retrieved: " + history.Count); + + Assert.IsNotNull(history); + Assert.IsNotEmpty(history); + Assert.That(history.First().Time.Date, Is.EqualTo(startDateTime)); + Assert.That(history.Last().Time.Date, Is.LessThanOrEqualTo(endDateTime)); + Assert.That(history.First().AllData.First().Symbol.Value, Is.EqualTo("GOOG")); + Assert.That(history.Last().AllData.First().Symbol.Value, Is.EqualTo("GOOGL")); + + AssertHistoricalDataResults(history.Select(x => x.AllData).SelectMany(x => x).ToList(), resolution); + } + internal static void AssertHistoricalDataResults(List history, Resolution resolution, int? expectedCount = null) { // Assert that we got some data @@ -141,18 +167,17 @@ internal static TestCaseData[] IndexHistoricalDataTestCases [TestCaseSource(nameof(IndexHistoricalDataTestCases))] [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] - public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) + public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeNull) { var history = GetIndexHistory(resolution, period, tickType); - Log.Trace("Data points retrieved: " + history.Count); - - if (shouldBeEmpty) + if (shouldBeNull) { - Assert.That(history, Is.Empty); + Assert.IsNull(history); } else { + Log.Trace("Data points retrieved: " + history.Count); AssertHistoricalDataResults(history.Select(x => x.AllData).SelectMany(x => x).ToList(), resolution, _historyProvider.DataPointCount); } } @@ -163,16 +188,16 @@ internal static TestCaseData[] IndexHistoricalInvalidDataTestCases { return new[] { - new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote, true), - new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote, true), - new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote, true), + new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote), + new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote), + new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote), }; } } [TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))] [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] - public void GetsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) + public void GetsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType) { var history = GetIndexHistory(resolution, period, tickType); @@ -312,24 +337,43 @@ internal List GetIndexHistory(Resolution resolution, TimeSpan period, Tic internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period) { var end = new DateTime(2023, 12, 15, 16, 0, 0); + if (resolution == Resolution.Daily) { end = end.Date.AddDays(1); } - var dataType = LeanData.GetDataType(resolution, tickType); - return new HistoryRequest(end.Subtract(period), - end, + return CreateHistoryRequest(symbol, resolution, tickType, end.Subtract(period), end); + } + + internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime, + SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null) + { + if (exchangeHours == null) + { + exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork); + } + + if (dataTimeZone == null) + { + dataTimeZone = TimeZones.NewYork; + } + + var dataType = LeanData.GetDataType(resolution, tickType); + return new HistoryRequest( + startDateTime, + endDateTime, dataType, symbol, resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), - TimeZones.NewYork, + exchangeHours, + dataTimeZone, null, true, false, DataNormalizationMode.Adjusted, - tickType); + tickType + ); } private class TestPolygonRestApiClient : PolygonRestApiClient diff --git a/QuantConnect.Polygon.Tests/PolygonOptionChainProviderTests.cs b/QuantConnect.Polygon.Tests/PolygonOptionChainProviderTests.cs index 0db508b..6e5ae6e 100644 --- a/QuantConnect.Polygon.Tests/PolygonOptionChainProviderTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonOptionChainProviderTests.cs @@ -14,12 +14,14 @@ * */ +using Moq; +using System; +using RestSharp; +using System.Linq; using NUnit.Framework; -using QuantConnect.Configuration; using QuantConnect.Logging; -using System; +using QuantConnect.Configuration; using System.Collections.Generic; -using System.Linq; namespace QuantConnect.Lean.DataSource.Polygon.Tests { @@ -52,6 +54,7 @@ public void TearDown() ("AAPL", SecurityType.Equity), ("IBM", SecurityType.Equity), ("GOOG", SecurityType.Equity), + ("GOOGL", SecurityType.Equity), ("SPX", SecurityType.Index), ("VIX", SecurityType.Index), ("DAX", SecurityType.Index), @@ -103,7 +106,7 @@ public void GetsOptionChainGivenTheOptionSymbol(Symbol option) [Test] public void GetsFullSPXOptionChain() { - var chain = GetOptionChain(Symbol.Create("SPX", SecurityType.Index, Market.USA), new DateTime(2024, 01, 03)); + var chain = GetOptionChain(Symbol.Create("SPX", SecurityType.Index, Market.USA), new DateTime(2024, 03, 15)); // SPX has a lot of options, let's make sure we get more than 1000 contracts (which is the pagination limit) // to assert that multiple requests are being made. @@ -120,5 +123,37 @@ public void GetsFullSPXOptionChain() Assert.That(spxw.Count + spx.Count, Is.EqualTo(chain.Count)); } + + [TestCaseSource(nameof(Underlyings))] + public void ValidateQueryParameterToSpecificSymbolValue(Symbol underlyingSymbol) + { + IRestRequest request = default; + var mock = new Mock("api-key"); + + mock.Setup(m => m.DownloadAndParseData(It.IsAny())) + .Callback((RestRequest r) => request = r); + + var optionChainProvider = new PolygonOptionChainProvider(mock.Object, new PolygonSymbolMapper()); + + var expiryDate = new DateTime(2024, 03, 15); + var option = Symbol.CreateOption(underlyingSymbol, Market.USA, OptionStyle.American, OptionRight.Call, 1000m, expiryDate); + + var optionContracts = optionChainProvider.GetOptionContractList(option, expiryDate).ToList(); + + Assert.IsNotNull(optionContracts); + Assert.IsTrue(request.Parameters[0].Value.ToString().EndsWith(option.Underlying.Value)); + } + + [TestCaseSource(nameof(Underlyings))] + public void ValidateGetOptionContractsReturnsAppropriateSymbol(Symbol underlyingSymbol) + { + var referenceDate = new DateTime(2024, 03, 15); + var optionContracts = _optionChainProvider.GetOptionContractList(underlyingSymbol, referenceDate).ToList(); + + foreach (var optionContract in optionContracts) + { + Assert.That(optionContract.Underlying, Is.EqualTo(underlyingSymbol)); + } + } } } \ No newline at end of file diff --git a/QuantConnect.Polygon.Tests/PolygonSymbolMapperTests.cs b/QuantConnect.Polygon.Tests/PolygonSymbolMapperTests.cs index 3acbc72..a245591 100644 --- a/QuantConnect.Polygon.Tests/PolygonSymbolMapperTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonSymbolMapperTests.cs @@ -47,14 +47,18 @@ public void ConvertsPolygonSymbolToLeanSymbol(string polygonSymbol, Symbol leanS Assert.That(convertedSymbol, Is.EqualTo(leanSymbol)); } - [TestCase("O:SPY240104C00467000", SecurityType.Option, ExpectedResult = SecurityType.Equity)] - [TestCase("O:SPX240104C04685000", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)] - [TestCase("O:SPXW240104C04685000", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)] - public SecurityType ConvertsOptionSymbolWithCorrectUnderlyingSecurityType(string ticker, SecurityType optionType) + [TestCase("O:SPY240104C00467000", "SPY", SecurityType.Option, ExpectedResult = SecurityType.Equity)] + [TestCase("O:GOOGL240105C00050000", "GOOGL", SecurityType.Option, ExpectedResult = SecurityType.Equity)] + [TestCase("O:GOOG240105C00070000", "GOOG", SecurityType.Option, ExpectedResult = SecurityType.Equity)] + [TestCase("O:SPX240104C04685000", "SPX", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)] + [TestCase("O:SPXW240104C04685000", "SPX", SecurityType.IndexOption, ExpectedResult = SecurityType.Index)] + public SecurityType ConvertsOptionSymbolWithCorrectUnderlyingSecurityType(string ticker, string expectedTicker, SecurityType optionType) { var mapper = new PolygonSymbolMapper(); var symbol = mapper.GetLeanSymbol(ticker, optionType, Market.USA, new DateTime(2024, 01, 04), 400m, OptionRight.Call); + Assert.That(symbol.Underlying.Value, Is.EqualTo(expectedTicker)); + return symbol.Underlying.SecurityType; } diff --git a/QuantConnect.Polygon/PolygonDataProvider.cs b/QuantConnect.Polygon/PolygonDataProvider.cs index ab55d64..b23717c 100644 --- a/QuantConnect.Polygon/PolygonDataProvider.cs +++ b/QuantConnect.Polygon/PolygonDataProvider.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -66,11 +66,16 @@ public partial class PolygonDataProvider : IDataQueueHandler private bool _initialized; private bool _disposed; - private bool _unsupportedSecurityTypeMessageLogged; - private bool _unsupportedTickTypeMessagedLogged; - private bool _unsupportedDataTypeMessageLogged; - private bool _potentialUnsupportedResolutionMessageLogged; - private bool _potentialUnsupportedTickTypeMessageLogged; + private volatile bool _unsupportedSecurityTypeMessageLogged; + private volatile bool _unsupportedTickTypeMessagedLogged; + private volatile bool _unsupportedDataTypeMessageLogged; + private volatile bool _potentialUnsupportedResolutionMessageLogged; + private volatile bool _potentialUnsupportedTickTypeMessageLogged; + + /// + /// + /// + private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); /// /// The time provider instance. Used for improved testability @@ -417,8 +422,8 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick { if (!_unsupportedSecurityTypeMessageLogged) { - Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported security type: {securityType}"); _unsupportedSecurityTypeMessageLogged = true; + Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported security type: {securityType}"); } return false; } @@ -427,8 +432,8 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick { if (!_unsupportedTickTypeMessagedLogged) { - Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported tick type: {tickType}"); _unsupportedTickTypeMessagedLogged = true; + Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported tick type: {tickType}"); } return false; } @@ -439,26 +444,26 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick { if (!_unsupportedDataTypeMessageLogged) { - Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported data type: {dataType}"); _unsupportedDataTypeMessageLogged = true; + Log.Trace($"PolygonDataProvider.IsSupported(): Unsupported data type: {dataType}"); } return false; } if (resolution < Resolution.Second && !_potentialUnsupportedResolutionMessageLogged) { + _potentialUnsupportedResolutionMessageLogged = true; Log.Trace("PolygonDataProvider.IsSupported(): " + $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + $"An Advanced Polygon.io subscription plan is required to stream tick data."); - _potentialUnsupportedResolutionMessageLogged = true; } if (tickType == TickType.Quote && !_potentialUnsupportedTickTypeMessageLogged) { + _potentialUnsupportedTickTypeMessageLogged = true; Log.Trace("PolygonDataProvider.IsSupported(): " + $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + $"An Advanced Polygon.io subscription plan is required to stream quote data."); - _potentialUnsupportedTickTypeMessageLogged = true; } return true; diff --git a/QuantConnect.Polygon/PolygonHistoryProvider.cs b/QuantConnect.Polygon/PolygonHistoryProvider.cs index 1e0eadc..59fe76a 100644 --- a/QuantConnect.Polygon/PolygonHistoryProvider.cs +++ b/QuantConnect.Polygon/PolygonHistoryProvider.cs @@ -32,12 +32,12 @@ public partial class PolygonDataProvider : SynchronizingHistoryProvider /// /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// - private bool _invalidStartTimeErrorFired; + private volatile bool _invalidStartTimeErrorFired; /// /// Indicates whether an error has been fired due to invalid conditions if the TickType is and the is greater than one second. /// - private bool _invalidTickTypeAndResolutionErrorFired; + private volatile bool _invalidTickTypeAndResolutionErrorFired; /// /// Gets the total number of data points emitted by this history provider @@ -63,12 +63,14 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) var subscriptions = new List(); foreach (var request in requests) { - var history = GetHistory(request); - if (history == null) + var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? Enumerable.Empty()); + + var subscription = CreateSubscription(request, history); + if (!subscription.MoveNext()) { continue; } - var subscription = CreateSubscription(request, history); + subscriptions.Add(subscription); } @@ -99,8 +101,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) { if (!_invalidTickTypeAndResolutionErrorFired) { - Log.Error("PolygonDataProvider.GetHistory(): Quote data above second resolution is not supported."); _invalidTickTypeAndResolutionErrorFired = true; + Log.Error("PolygonDataProvider.GetHistory(): Quote data above second resolution is not supported."); } return null; } @@ -109,8 +111,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) { if (!_invalidStartTimeErrorFired) { - Log.Error($"{nameof(PolygonDataProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); _invalidStartTimeErrorFired = true; + Log.Error($"{nameof(PolygonDataProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); } return null; } @@ -179,7 +181,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// private IEnumerable GetAggregates(HistoryRequest request) { - var ticker = _symbolMapper.GetBrokerageSymbol(request.Symbol); + var ticker = _symbolMapper.GetBrokerageSymbol(request.Symbol, true); var resolutionTimeSpan = request.Resolution.ToTimeSpan(); // Aggregates API gets timestamps in milliseconds var start = Time.DateTimeToUnixTimeStampMilliseconds(request.StartTimeUtc.RoundDown(resolutionTimeSpan)); diff --git a/QuantConnect.Polygon/PolygonOptionChainProvider.cs b/QuantConnect.Polygon/PolygonOptionChainProvider.cs index 8312c7f..23427af 100644 --- a/QuantConnect.Polygon/PolygonOptionChainProvider.cs +++ b/QuantConnect.Polygon/PolygonOptionChainProvider.cs @@ -68,7 +68,7 @@ public IEnumerable GetOptionContractList(Symbol symbol, DateTime date) var optionsSecurityType = underlying.SecurityType == SecurityType.Index ? SecurityType.IndexOption : SecurityType.Option; var request = new RestRequest("/v3/reference/options/contracts", Method.GET); - request.AddQueryParameter("underlying_ticker", underlying.ID.Symbol); + request.AddQueryParameter("underlying_ticker", underlying.Value); request.AddQueryParameter("as_of", date.ToStringInvariant("yyyy-MM-dd")); request.AddQueryParameter("limit", "1000"); diff --git a/QuantConnect.Polygon/PolygonSymbolMapper.cs b/QuantConnect.Polygon/PolygonSymbolMapper.cs index 141a4e1..58130ec 100644 --- a/QuantConnect.Polygon/PolygonSymbolMapper.cs +++ b/QuantConnect.Polygon/PolygonSymbolMapper.cs @@ -40,9 +40,21 @@ public string GetBrokerageSymbol(Symbol symbol) throw new ArgumentException($"Invalid symbol: {(symbol == null ? "null" : symbol.ToString())}"); } + return GetBrokerageSymbol(symbol, false); + } + + /// + /// Converts a Lean symbol instance to a brokerage symbol with updating of cached symbol collection + /// + /// + /// + /// + /// + public string GetBrokerageSymbol(Symbol symbol, bool isUpdateCachedSymbol) + { lock (_locker) { - if (!_brokerageSymbolsCache.TryGetValue(symbol, out var brokerageSymbol)) + if (!_brokerageSymbolsCache.TryGetValue(symbol, out var brokerageSymbol) || isUpdateCachedSymbol) { var ticker = symbol.Value.Replace(" ", ""); switch (symbol.SecurityType) @@ -118,7 +130,7 @@ public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, s if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) { var leanBaseSymbol = securityType == SecurityType.Equity ? null : GetLeanSymbol(brokerageSymbol); - var underlyingSymbolStr = underlying?.ID.Symbol ?? leanBaseSymbol?.Underlying.ID.Symbol; + var underlyingSymbolStr = underlying?.Value ?? leanBaseSymbol?.Underlying.Value; switch (securityType) {