diff --git a/Algorithm.CSharp/AddBetaIndicatorNewAssetsRegressionAlgorithm.cs b/Algorithm.CSharp/AddBetaIndicatorNewAssetsRegressionAlgorithm.cs index f9020dfd7406..213ab2824913 100644 --- a/Algorithm.CSharp/AddBetaIndicatorNewAssetsRegressionAlgorithm.cs +++ b/Algorithm.CSharp/AddBetaIndicatorNewAssetsRegressionAlgorithm.cs @@ -1,11 +1,11 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); + * + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -107,7 +107,7 @@ public override void OnOrderEvent(OrderEvent orderEvent) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 77; + public int AlgorithmHistoryDataPoints => 26; /// /// Final status of the algorithm @@ -122,18 +122,18 @@ public override void OnOrderEvent(OrderEvent orderEvent) {"Total Orders", "436"}, {"Average Win", "0.28%"}, {"Average Loss", "-0.01%"}, - {"Compounding Annual Return", "1.926%"}, + {"Compounding Annual Return", "1.925%"}, {"Drawdown", "1.000%"}, - {"Expectancy", "1.650"}, + {"Expectancy", "1.649"}, {"Start Equity", "10000.00"}, - {"End Equity", "10411.11"}, - {"Net Profit", "4.111%"}, + {"End Equity", "10410.99"}, + {"Net Profit", "4.110%"}, {"Sharpe Ratio", "0.332"}, {"Sortino Ratio", "0.313"}, {"Probabilistic Sharpe Ratio", "74.084%"}, {"Loss Rate", "90%"}, {"Win Rate", "10%"}, - {"Profit-Loss Ratio", "25.26"}, + {"Profit-Loss Ratio", "25.25"}, {"Alpha", "0.003"}, {"Beta", "0.001"}, {"Annual Standard Deviation", "0.01"}, @@ -146,7 +146,7 @@ public override void OnOrderEvent(OrderEvent orderEvent) {"Lowest Capacity Asset", "BTCUSD 2XR"}, {"Portfolio Turnover", "2.22%"}, {"Drawdown Recovery", "139"}, - {"OrderListHash", "9fce77ef8817cf0159897fc64d01f5e9"} + {"OrderListHash", "896ecc92440d51ed26644aac5b8706e4"} }; } } diff --git a/Algorithm.CSharp/AutomaticSeedBaseRegressionAlgorithm.cs b/Algorithm.CSharp/AutomaticSeedBaseRegressionAlgorithm.cs new file mode 100644 index 000000000000..a39b4c19f68e --- /dev/null +++ b/Algorithm.CSharp/AutomaticSeedBaseRegressionAlgorithm.cs @@ -0,0 +1,137 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Interfaces; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Data.Market; +using QuantConnect.Securities; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting that security are automatically seeded by default + /// + public abstract class AutomaticSeedBaseRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + protected virtual bool ShouldHaveTradeData { get; } + protected virtual bool ShouldHaveQuoteData { get; } + protected virtual bool ShouldHaveOpenInterestData { get; } + + public override void OnSecuritiesChanged(SecurityChanges changes) + { + var gotTrades = false; + var gotQuotes = false; + var gotOpenInterest = false; + + foreach (var addedSecurity in changes.AddedSecurities.Where(x => !x.Symbol.IsCanonical() || x.Symbol.SecurityType == SecurityType.Future)) + { + if (addedSecurity.Price == 0) + { + throw new RegressionTestException("Security was not seeded"); + } + + if (!addedSecurity.HasData) + { + throw new RegressionTestException("Security does not have TradeBar or QuoteBar or OpenInterest data"); + } + + gotTrades |= addedSecurity.Cache.GetData() != null; + gotQuotes |= addedSecurity.Cache.GetData() != null; + gotOpenInterest |= addedSecurity.Cache.GetData() != null; + } + + if (changes.AddedSecurities.Count > 0) + { + if (ShouldHaveTradeData && !gotTrades) + { + throw new RegressionTestException("No contract had TradeBar data"); + } + + if (ShouldHaveQuoteData && !gotQuotes) + { + throw new RegressionTestException("No contract had QuoteBar data"); + } + + if (ShouldHaveOpenInterestData && !gotOpenInterest) + { + throw new RegressionTestException("No contract had OpenInterest data"); + } + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public abstract long DataPoints { get; } + + /// + /// Data Points count of the algorithm history + /// + public abstract int AlgorithmHistoryDataPoints { get; } + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public virtual Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Algorithm.CSharp/BasicSetAccountCurrencyAlgorithm.cs b/Algorithm.CSharp/BasicSetAccountCurrencyAlgorithm.cs index b32816e5483e..12a2a95a6a52 100644 --- a/Algorithm.CSharp/BasicSetAccountCurrencyAlgorithm.cs +++ b/Algorithm.CSharp/BasicSetAccountCurrencyAlgorithm.cs @@ -77,7 +77,7 @@ public override void OnData(Slice slice) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 15; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BasicSetAccountCurrencyWithAmountAlgorithm.cs b/Algorithm.CSharp/BasicSetAccountCurrencyWithAmountAlgorithm.cs index def5fcfcb173..039ccaec651a 100644 --- a/Algorithm.CSharp/BasicSetAccountCurrencyWithAmountAlgorithm.cs +++ b/Algorithm.CSharp/BasicSetAccountCurrencyWithAmountAlgorithm.cs @@ -47,7 +47,7 @@ public override void SetAccountCurrency() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 15; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BasicTemplateCryptoAlgorithm.cs b/Algorithm.CSharp/BasicTemplateCryptoAlgorithm.cs index 4268d083e60c..5c3ce2f2c90d 100644 --- a/Algorithm.CSharp/BasicTemplateCryptoAlgorithm.cs +++ b/Algorithm.CSharp/BasicTemplateCryptoAlgorithm.cs @@ -202,7 +202,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 240; + public int AlgorithmHistoryDataPoints => 35; /// /// Final status of the algorithm @@ -220,7 +220,7 @@ public override void OnEndOfAlgorithm() {"Compounding Annual Return", "0%"}, {"Drawdown", "0%"}, {"Expectancy", "0"}, - {"Start Equity", "31588.24"}, + {"Start Equity", "31592.84"}, {"End Equity", "30866.71"}, {"Net Profit", "0%"}, {"Sharpe Ratio", "0"}, diff --git a/Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs b/Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs index 7299325651c0..8bb42742ea27 100644 --- a/Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs +++ b/Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs @@ -154,7 +154,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 340; + public int AlgorithmHistoryDataPoints => 354; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketAlgorithm.cs b/Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketAlgorithm.cs index f3323557cadd..693c63a7cd56 100644 --- a/Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketAlgorithm.cs +++ b/Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketAlgorithm.cs @@ -154,7 +154,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 340; + public int AlgorithmHistoryDataPoints => 354; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BinanceCashAccountFeeRegressionAlgorithm.cs b/Algorithm.CSharp/BinanceCashAccountFeeRegressionAlgorithm.cs index 6e7ae0c2c23b..1b7aad6998f8 100644 --- a/Algorithm.CSharp/BinanceCashAccountFeeRegressionAlgorithm.cs +++ b/Algorithm.CSharp/BinanceCashAccountFeeRegressionAlgorithm.cs @@ -46,7 +46,7 @@ public override void Initialize() /// /// Data Points count of the algorithm history /// - public override int AlgorithmHistoryDataPoints => 28; + public override int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BinanceMarginAccountFeeRegressionAlgorithm.cs b/Algorithm.CSharp/BinanceMarginAccountFeeRegressionAlgorithm.cs index 4c30043f2586..823b2389af92 100644 --- a/Algorithm.CSharp/BinanceMarginAccountFeeRegressionAlgorithm.cs +++ b/Algorithm.CSharp/BinanceMarginAccountFeeRegressionAlgorithm.cs @@ -46,7 +46,7 @@ public override void Initialize() /// /// Data Points count of the algorithm history /// - public override int AlgorithmHistoryDataPoints => 28; + public override int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BitfinexCashAccountFeeRegressionAlgorithm.cs b/Algorithm.CSharp/BitfinexCashAccountFeeRegressionAlgorithm.cs index f56e8e1ed08a..1d2ca5ef72a1 100644 --- a/Algorithm.CSharp/BitfinexCashAccountFeeRegressionAlgorithm.cs +++ b/Algorithm.CSharp/BitfinexCashAccountFeeRegressionAlgorithm.cs @@ -45,7 +45,7 @@ public override void Initialize() /// /// Data Points count of the algorithm history /// - public override int AlgorithmHistoryDataPoints => 28; + public override int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BitfinexMarginAccountFeeRegressionAlgorithm.cs b/Algorithm.CSharp/BitfinexMarginAccountFeeRegressionAlgorithm.cs index 7536fe642e78..c6aeca81b5e2 100644 --- a/Algorithm.CSharp/BitfinexMarginAccountFeeRegressionAlgorithm.cs +++ b/Algorithm.CSharp/BitfinexMarginAccountFeeRegressionAlgorithm.cs @@ -45,7 +45,7 @@ public override void Initialize() /// /// Data Points count of the algorithm history /// - public override int AlgorithmHistoryDataPoints => 28; + public override int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/BybitCryptoFuturesRegressionAlgorithm.cs b/Algorithm.CSharp/BybitCryptoFuturesRegressionAlgorithm.cs index b77d810a5fc8..ccda05848fda 100644 --- a/Algorithm.CSharp/BybitCryptoFuturesRegressionAlgorithm.cs +++ b/Algorithm.CSharp/BybitCryptoFuturesRegressionAlgorithm.cs @@ -226,7 +226,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 10; /// /// Final status of the algorithm @@ -244,7 +244,7 @@ public override void OnEndOfAlgorithm() {"Compounding Annual Return", "0%"}, {"Drawdown", "0%"}, {"Expectancy", "0"}, - {"Start Equity", "100285.86"}, + {"Start Equity", "100285.85"}, {"End Equity", "100285.26"}, {"Net Profit", "0%"}, {"Sharpe Ratio", "0"}, diff --git a/Algorithm.CSharp/BybitCryptoRegressionAlgorithm.cs b/Algorithm.CSharp/BybitCryptoRegressionAlgorithm.cs index 09385dcd106f..c9fb4c24878e 100644 --- a/Algorithm.CSharp/BybitCryptoRegressionAlgorithm.cs +++ b/Algorithm.CSharp/BybitCryptoRegressionAlgorithm.cs @@ -135,7 +135,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 10; /// /// Final status of the algorithm @@ -153,7 +153,7 @@ public override void OnEndOfAlgorithm() {"Compounding Annual Return", "0%"}, {"Drawdown", "0%"}, {"Expectancy", "0"}, - {"Start Equity", "117171.12"}, + {"Start Equity", "117170.74"}, {"End Equity", "117244.52"}, {"Net Profit", "0%"}, {"Sharpe Ratio", "0"}, diff --git a/Algorithm.CSharp/BybitCustomDataCryptoRegressionAlgorithm.cs b/Algorithm.CSharp/BybitCustomDataCryptoRegressionAlgorithm.cs index af3e61d55f38..63303bd45752 100644 --- a/Algorithm.CSharp/BybitCustomDataCryptoRegressionAlgorithm.cs +++ b/Algorithm.CSharp/BybitCustomDataCryptoRegressionAlgorithm.cs @@ -147,7 +147,7 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 10; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/CancelOpenOrdersRegressionAlgorithm.cs b/Algorithm.CSharp/CancelOpenOrdersRegressionAlgorithm.cs index 5f87df40d595..5fd071b242c9 100644 --- a/Algorithm.CSharp/CancelOpenOrdersRegressionAlgorithm.cs +++ b/Algorithm.CSharp/CancelOpenOrdersRegressionAlgorithm.cs @@ -130,7 +130,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 20; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/CoarseFineOptionUniverseChainRegressionAlgorithm.cs b/Algorithm.CSharp/CoarseFineOptionUniverseChainRegressionAlgorithm.cs index ca659fe4512d..dba0e2d74b0a 100644 --- a/Algorithm.CSharp/CoarseFineOptionUniverseChainRegressionAlgorithm.cs +++ b/Algorithm.CSharp/CoarseFineOptionUniverseChainRegressionAlgorithm.cs @@ -47,6 +47,11 @@ public override void Initialize() _aapl = QuantConnect.Symbol.Create("AAPL", SecurityType.Equity, Market.USA); UniverseSettings.Resolution = Resolution.Minute; + // Let's disable initial price seeding, the algorithm will wait until both equity + // and options are added an have prices to do the tests, we don't want the equity + // having prices before the options are added. + Settings.SeedInitialPrices = false; + SetStartDate(2014, 06, 04); // TWX is selected the 4th and 5th and aapl after that. // If the algo ends on the 6th, TWX subscriptions will not be removed before OnEndOfAlgorithm is called: diff --git a/Algorithm.CSharp/CoarseSelectionsAutomaticSeedRegressionAlgorithm.cs b/Algorithm.CSharp/CoarseSelectionsAutomaticSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..16a9b60a43b2 --- /dev/null +++ b/Algorithm.CSharp/CoarseSelectionsAutomaticSeedRegressionAlgorithm.cs @@ -0,0 +1,120 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Data.UniverseSelection; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting that securities added via coarse selection get automatically seeded by default + /// + public class CoarseSelectionsAutomaticSeedRegressionAlgorithm : AutomaticSeedBaseRegressionAlgorithm + { + private readonly Queue> _coarseSelections = new(new[] { "AAPL", "GOOG", "AIG", "BAC", "FB", "IBM" } + .Select(x => QuantConnect.Symbol.Create(x, SecurityType.Equity, Market.USA)) + .BatchBy(2)); + + private HashSet _addedSecurities = new(); + + protected override bool ShouldHaveTradeData => true; + // Daily resolution, only trade data is available + protected override bool ShouldHaveQuoteData => false; + protected override bool ShouldHaveOpenInterestData => false; + + public override void Initialize() + { + SetStartDate(2015, 01, 01); + SetEndDate(2015, 03, 01); + SetCash(100000); + + Settings.SeedInitialPrices = true; + UniverseSettings.Resolution = Resolution.Daily; + + AddUniverse((coarse) => + { + var selection = _coarseSelections.Dequeue(); + _coarseSelections.Enqueue(selection); + return selection; + }); + } + + public override void OnSecuritiesChanged(SecurityChanges changes) + { + base.OnSecuritiesChanged(changes); + + foreach (var addedSecurity in changes.AddedSecurities.Where(x => !x.Symbol.IsCanonical())) + { + _addedSecurities.Add(addedSecurity.Symbol); + } + } + + public override void OnEndOfAlgorithm() + { + if (!_coarseSelections.SelectMany(x => x).Order().SequenceEqual(_addedSecurities.Order())) + { + throw new RegressionTestException("Not all securities were added"); + } + } + + /// + /// Data Points count of all timeslices of algorithm + /// + public override long DataPoints => 358; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 390; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public override Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "-1.066"}, + {"Tracking Error", "0.116"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Algorithm.CSharp/CoinbaseCryptoYearMarketTradingRegressionAlgorithm.cs b/Algorithm.CSharp/CoinbaseCryptoYearMarketTradingRegressionAlgorithm.cs index 71aff42b6187..598444164a60 100644 --- a/Algorithm.CSharp/CoinbaseCryptoYearMarketTradingRegressionAlgorithm.cs +++ b/Algorithm.CSharp/CoinbaseCryptoYearMarketTradingRegressionAlgorithm.cs @@ -104,7 +104,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 43; + public int AlgorithmHistoryDataPoints => 11; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/Collective2PortfolioSignalExportDemonstrationAlgorithm.cs b/Algorithm.CSharp/Collective2PortfolioSignalExportDemonstrationAlgorithm.cs index 221427ea306c..7e4a1ab222be 100644 --- a/Algorithm.CSharp/Collective2PortfolioSignalExportDemonstrationAlgorithm.cs +++ b/Algorithm.CSharp/Collective2PortfolioSignalExportDemonstrationAlgorithm.cs @@ -153,7 +153,7 @@ public override void OnData(Slice slice) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 11147; + public int AlgorithmHistoryDataPoints => 50; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/Collective2SignalExportDemonstrationAlgorithm.cs b/Algorithm.CSharp/Collective2SignalExportDemonstrationAlgorithm.cs index cdfff0c7ce25..3db21ab2f84c 100644 --- a/Algorithm.CSharp/Collective2SignalExportDemonstrationAlgorithm.cs +++ b/Algorithm.CSharp/Collective2SignalExportDemonstrationAlgorithm.cs @@ -176,7 +176,7 @@ public override void OnData(Slice slice) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 11147; + public int AlgorithmHistoryDataPoints => 50; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/ContinuousFutureRolloverBaseRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverBaseRegressionAlgorithm.cs index 8975109da32c..3b8134c69420 100644 --- a/Algorithm.CSharp/ContinuousFutureRolloverBaseRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ContinuousFutureRolloverBaseRegressionAlgorithm.cs @@ -45,17 +45,23 @@ public abstract class ContinuousFutureRolloverBaseRegressionAlgorithm : QCAlgori protected abstract Offset ExchangeToDataTimeZoneOffset { get; } + protected virtual bool SeedIntialPrices { get; } + private DateTimeZone DataTimeZone => TimeZones.Utc; private DateTimeZone ExchangeTimeZone => DateTimeZone.ForOffset(ExchangeToDataTimeZoneOffset); private bool RolloverHappened => _rolloverTime != DateTime.MinValue; + private BaseData MappedContractSeededData; + public override void Initialize() { SetStartDate(2013, 10, 8); SetEndDate(2013, 12, 20); + Settings.SeedInitialPrices = SeedIntialPrices; + _originalMhdbEntry = MarketHoursDatabase.GetEntry(Market.CME, Ticker, SecurityType.Future); var exchangeHours = new SecurityExchangeHours(ExchangeTimeZone, _originalMhdbEntry.ExchangeHours.Holidays, @@ -115,6 +121,9 @@ public override void OnData(Slice slice) $"Expected {expectedMappingOldSymbol} -> {expectedMappingNewSymbol} " + $"but was {symbolChangedEvent.OldSymbol} -> {symbolChangedEvent.NewSymbol}"); } + + var mappedContract = Securities[_continuousContract.Mapped]; + MappedContractSeededData = mappedContract.GetLastData(); } var mappedFuture = Securities[_continuousContract.Mapped]; @@ -148,7 +157,10 @@ public override void OnData(Slice slice) } else if (mappedFuturePrice != 0 || !RolloverHappened) { - if (continuousContractPrice != mappedFuturePrice) + var mappedFutureData = mappedFuture.GetLastData(); + // We only do this check is default securities seeding is desabled, else the mapped contract will have historical data + if ((!Settings.SeedInitialPrices || !ReferenceEquals(MappedContractSeededData, mappedFutureData)) && + continuousContractPrice != mappedFuturePrice) { var continuousContractLastData = _continuousContract.GetLastData(); throw new RegressionTestException($"[{Time}] -- Prices do not match. " + @@ -221,7 +233,7 @@ private void ResetMarketHoursDatabase() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 0; + public virtual int AlgorithmHistoryDataPoints => 0; /// /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm diff --git a/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataRegressionAlgorithm.cs index c96e620c275a..c982f8b4d0c6 100644 --- a/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataRegressionAlgorithm.cs @@ -34,5 +34,10 @@ public class ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataRegressionA /// Data Points count of all timeslices of algorithm /// public override long DataPoints => 483; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 0; } } diff --git a/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..2ffbe7e930c9 --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,33 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the exchange time zone is ahead of the data time zone. + /// + public class ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverDailyExchangeTimeZoneAheadOfDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 15; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..def2d23bca51 --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the exchange time zone is behind of the data time zone. + /// + public class ContinuousFutureRolloverDailyExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverDailyExchangeTimeZoneBehindOfDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 17; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneSameAsDataWithIntialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneSameAsDataWithIntialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..1a5713bac8e3 --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverDailyExchangeTimeZoneSameAsDataWithIntialSeedRegressionAlgorithm.cs @@ -0,0 +1,35 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the data time zone is the same as the exchange time zone. + /// + public class ContinuousFutureRolloverDailyExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverDailyExchangeTimeZoneSameAsDataRegressionAlgorithm + + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 17; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..9e088413d171 --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the exchange time zone is ahead of the data time zone. + /// + public class ContinuousFutureRolloverHourExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverHourExchangeTimeZoneAheadOfDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 65; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..9ed875f9d432 --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the exchange time zone is behind of the data time zone. + /// + public class ContinuousFutureRolloverHourExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverHourExchangeTimeZoneBehindOfDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 64; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..894d12d113f0 --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverHourExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the data time zone is the same as the exchange time zone. + /// + public class ContinuousFutureRolloverHourExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverHourExchangeTimeZoneSameAsDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 64; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..6774f5fcdc9e --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the exchange time zone is ahead of the data time zone. + /// + public class ContinuousFutureRolloverMinuteExchangeTimeZoneAheadOfDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverMinuteExchangeTimeZoneAheadOfDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 1958; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..7531da4271dd --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the exchange time zone is behind of the data time zone. + /// + public class ContinuousFutureRolloverMinuteExchangeTimeZoneBehindOfDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverMinuteExchangeTimeZoneBehindOfDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 892; + } +} diff --git a/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm.cs b/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..cc2cc354c353 --- /dev/null +++ b/Algorithm.CSharp/ContinuousFutureRolloverMinuteExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Base class for regression algorithms testing that when a continuous future rollover happens, + /// the continuous contract is updated correctly with the new contract data. + /// The algorithms asserts the behavior for the case when the data time zone is the same as the exchange time zone. + /// + public class ContinuousFutureRolloverMinuteExchangeTimeZoneSameAsDataWithInitialSeedRegressionAlgorithm + : ContinuousFutureRolloverMinuteExchangeTimeZoneSameAsDataRegressionAlgorithm + { + protected override bool SeedIntialPrices => true; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 668; + } +} diff --git a/Algorithm.CSharp/CustomDataAutomaticSeedRegressionAlgorithm.cs b/Algorithm.CSharp/CustomDataAutomaticSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..11751eaade2d --- /dev/null +++ b/Algorithm.CSharp/CustomDataAutomaticSeedRegressionAlgorithm.cs @@ -0,0 +1,232 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using Newtonsoft.Json; +using QuantConnect.Data; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression test to assert that custom data is seeded by default + /// + public class CustomDataAutomaticSeedRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + public override void Initialize() + { + SetStartDate(2020, 01, 05); + SetEndDate(2020, 01, 10); + SetCash(100000); + + Settings.SeedInitialPrices = true; + + var resolution = Resolution.Daily; + var customData = AddData("BTC", resolution); + + if (!customData.HasData || customData.Price == 0) + { + throw new RegressionTestException("Custom data was not seeded with data on addition"); + } + + var seedData = customData.GetLastData(); + if (seedData is not Bitcoin) + { + throw new RegressionTestException("Custom data was not seeded with correct data type"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 50; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 5; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "-9.259"}, + {"Tracking Error", "0.073"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + + /// + /// Custom Data Type: Bitcoin data from Quandl - http://www.quandl.com/help/api-for-bitcoin-data + /// + public class Bitcoin : BaseData + { + [JsonProperty("timestamp")] + public int Timestamp { get; set; } + [JsonProperty("open")] + public decimal Open { get; set; } + [JsonProperty("high")] + public decimal High { get; set; } + [JsonProperty("low")] + public decimal Low { get; set; } + public decimal Mid { get; set; } + + [JsonProperty("last")] + public decimal Close { get; set; } + [JsonProperty("bid")] + public decimal Bid { get; set; } + [JsonProperty("ask")] + public decimal Ask { get; set; } + [JsonProperty("vwap")] + public decimal WeightedPrice { get; set; } + [JsonProperty("volume")] + public decimal VolumeBTC { get; set; } + + /// + /// The end time of this data. Some data covers spans (trade bars) + /// and as such we want to know the entire time span covered + /// + /// + /// This property is overriden to allow different values for Time and EndTime + /// if they are set in the Reader. In the base implementation EndTime equals Time + /// + public override DateTime EndTime { get; set; } + + /// + /// 1. DEFAULT CONSTRUCTOR: Custom data types need a default constructor. + /// We search for a default constructor so please provide one here. It won't be used for data, just to generate the "Factory". + /// + public Bitcoin() + { + Symbol = "BTC"; + } + + /// + /// 2. RETURN THE STRING URL SOURCE LOCATION FOR YOUR DATA: + /// This is a powerful and dynamic select source file method. If you have a large dataset, 10+mb we recommend you break it into smaller files. E.g. One zip per year. + /// We can accept raw text or ZIP files. We read the file extension to determine if it is a zip file. + /// + /// Configuration object + /// Date of this source file + /// true if we're in live mode, false for backtesting mode + /// String URL of source file. + public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) + { + if (isLiveMode) + { + return new SubscriptionDataSource("https://www.bitstamp.net/api/ticker/", SubscriptionTransportMedium.Rest); + } + + //return "http://my-ftp-server.com/futures-data-" + date.ToString("Ymd") + ".zip"; + // OR simply return a fixed small data file. Large files will slow down your backtest + return new SubscriptionDataSource("https://www.quantconnect.com/api/v2/proxy/nasdaq/api/v3/datatables/QDL/BITFINEX.csv?code=BTCUSD&api_key=qAWKpUfmSVFnU3bRQwKy") + { + Sort = true + }; + } + + /// + /// 3. READER METHOD: Read 1 line from data source and convert it into Object. + /// Each line of the CSV File is presented in here. The backend downloads your file, loads it into memory and then line by line + /// feeds it into your algorithm + /// + /// string line from the data source file submitted above + /// Subscription data, symbol name, data type + /// Current date we're requesting. This allows you to break up the data source into daily files. + /// true if we're in live mode, false for backtesting mode + /// New Bitcoin Object which extends BaseData. + public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) + { + var coin = new Bitcoin(); + if (isLiveMode) + { + //Example Line Format: + //{"high": "441.00", "last": "421.86", "timestamp": "1411606877", "bid": "421.96", "vwap": "428.58", "volume": "14120.40683975", "low": "418.83", "ask": "421.99"} + try + { + coin = JsonConvert.DeserializeObject(line); + coin.EndTime = DateTime.UtcNow.ConvertFromUtc(config.ExchangeTimeZone); + coin.Value = coin.Close; + } + catch { /* Do nothing, possible error in json decoding */ } + return coin; + } + + //Example Line Format: + // code date high low mid last bid ask volume + // BTCUSD 2024-10-08 63248.0 61940.0 62246.5 62245.0 62246.0 62247.0 477.91102114 + try + { + string[] data = line.Split(','); + coin.Time = DateTime.Parse(data[1], CultureInfo.InvariantCulture); + coin.EndTime = coin.Time.AddDays(1); + coin.High = Convert.ToDecimal(data[2], CultureInfo.InvariantCulture); + coin.Low = Convert.ToDecimal(data[3], CultureInfo.InvariantCulture); + coin.Mid = Convert.ToDecimal(data[4], CultureInfo.InvariantCulture); + coin.Close = Convert.ToDecimal(data[5], CultureInfo.InvariantCulture); + coin.Bid = Convert.ToDecimal(data[6], CultureInfo.InvariantCulture); + coin.Ask = Convert.ToDecimal(data[7], CultureInfo.InvariantCulture); + coin.VolumeBTC = Convert.ToDecimal(data[8], CultureInfo.InvariantCulture); + coin.Value = coin.Close; + } + catch { /* Do nothing, skip first title row */ } + + return coin; + } + } + } +} diff --git a/Algorithm.CSharp/CustomDataPropertiesRegressionAlgorithm.cs b/Algorithm.CSharp/CustomDataPropertiesRegressionAlgorithm.cs index 1534c82ea0dc..fba19955843a 100644 --- a/Algorithm.CSharp/CustomDataPropertiesRegressionAlgorithm.cs +++ b/Algorithm.CSharp/CustomDataPropertiesRegressionAlgorithm.cs @@ -198,7 +198,6 @@ public class Bitcoin : BaseData /// public Bitcoin() { - Symbol = "BTC"; } /// @@ -237,7 +236,7 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, /// New Bitcoin Object which extends BaseData. public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) { - var coin = new Bitcoin(); + var coin = new Bitcoin() { Symbol = config.Symbol }; if (isLiveMode) { //Example Line Format: diff --git a/Algorithm.CSharp/CustomDataWorksWithDifferentExchangesRegressionAlgorithm.cs b/Algorithm.CSharp/CustomDataWorksWithDifferentExchangesRegressionAlgorithm.cs index 71cf23798821..fc7a4360f09e 100644 --- a/Algorithm.CSharp/CustomDataWorksWithDifferentExchangesRegressionAlgorithm.cs +++ b/Algorithm.CSharp/CustomDataWorksWithDifferentExchangesRegressionAlgorithm.cs @@ -83,7 +83,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/CustomSecurityInitializerAlgorithm.cs b/Algorithm.CSharp/CustomSecurityInitializerAlgorithm.cs index e30e808caf64..ae4ba351d154 100644 --- a/Algorithm.CSharp/CustomSecurityInitializerAlgorithm.cs +++ b/Algorithm.CSharp/CustomSecurityInitializerAlgorithm.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * diff --git a/Algorithm.CSharp/DailyHistoryForDailyResolutionRegressionAlgorithm.cs b/Algorithm.CSharp/DailyHistoryForDailyResolutionRegressionAlgorithm.cs index 93479ffcf571..093d906eda8f 100644 --- a/Algorithm.CSharp/DailyHistoryForDailyResolutionRegressionAlgorithm.cs +++ b/Algorithm.CSharp/DailyHistoryForDailyResolutionRegressionAlgorithm.cs @@ -101,7 +101,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 564; + public int AlgorithmHistoryDataPoints => 458; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/DailyHistoryForMinuteResolutionRegressionAlgorithm.cs b/Algorithm.CSharp/DailyHistoryForMinuteResolutionRegressionAlgorithm.cs index 5394ffa04328..35c4e458624c 100644 --- a/Algorithm.CSharp/DailyHistoryForMinuteResolutionRegressionAlgorithm.cs +++ b/Algorithm.CSharp/DailyHistoryForMinuteResolutionRegressionAlgorithm.cs @@ -103,7 +103,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 43089; + public int AlgorithmHistoryDataPoints => 15307; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/DailyResolutionVsTimeSpanRegressionAlgorithm.cs b/Algorithm.CSharp/DailyResolutionVsTimeSpanRegressionAlgorithm.cs index a937d9ba2150..eeba2f06f68a 100644 --- a/Algorithm.CSharp/DailyResolutionVsTimeSpanRegressionAlgorithm.cs +++ b/Algorithm.CSharp/DailyResolutionVsTimeSpanRegressionAlgorithm.cs @@ -38,7 +38,7 @@ public override void Initialize() Settings.DailyPreciseEndTime = DailyPreciseEndTime; - // First RSI: Updates at market close (4 PM) by default + // First RSI: Updates at market close (4 PM) by default // If DailyPreciseEndTime is false, updates at midnight (12:00 AM) RelativeStrengthIndex1 = new RelativeStrengthIndex(14, MovingAverageType.Wilders); RegisterIndicator(Spy, RelativeStrengthIndex1, Resolution.Daily); diff --git a/Algorithm.CSharp/DailyResolutionVsTimeSpanWithSecondEquityAlgorithm.cs b/Algorithm.CSharp/DailyResolutionVsTimeSpanWithSecondEquityAlgorithm.cs index d4431df85d44..a5b6bbc19eae 100644 --- a/Algorithm.CSharp/DailyResolutionVsTimeSpanWithSecondEquityAlgorithm.cs +++ b/Algorithm.CSharp/DailyResolutionVsTimeSpanWithSecondEquityAlgorithm.cs @@ -17,7 +17,7 @@ namespace QuantConnect.Algorithm.CSharp { - public class DailyResolutionVsTimeSpanWithSecondResolutionEquityAlgorithm : DailyResolutionVsTimeSpanRegressionAlgorithm + public class DailyResolutionVsTimeSpanWithSecondEquityAlgorithm : DailyResolutionVsTimeSpanRegressionAlgorithm { protected override void InitializeBaseSettings() { @@ -62,7 +62,8 @@ protected override void InitializeBaseSettings() {"Estimated Strategy Capacity", "$0"}, {"Lowest Capacity Asset", ""}, {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} }; } -} \ No newline at end of file +} diff --git a/Algorithm.CSharp/EmitInsightCryptoCashAccountType.cs b/Algorithm.CSharp/EmitInsightCryptoCashAccountType.cs index 6ea12e46f3e1..528bfc773784 100644 --- a/Algorithm.CSharp/EmitInsightCryptoCashAccountType.cs +++ b/Algorithm.CSharp/EmitInsightCryptoCashAccountType.cs @@ -74,7 +74,7 @@ public override void OnData(Slice slice) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 15; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/FeeModelNotUsingAccountCurrency.cs b/Algorithm.CSharp/FeeModelNotUsingAccountCurrency.cs index 37a7b086d104..9b62692c94f9 100644 --- a/Algorithm.CSharp/FeeModelNotUsingAccountCurrency.cs +++ b/Algorithm.CSharp/FeeModelNotUsingAccountCurrency.cs @@ -157,7 +157,7 @@ public override OrderFee GetOrderFee(OrderFeeParameters parameters) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 10; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/ForexInternalFeedOnDataHigherResolutionRegressionAlgorithm.cs b/Algorithm.CSharp/ForexInternalFeedOnDataHigherResolutionRegressionAlgorithm.cs index 67cd5c68356d..0d1f992f0a4e 100644 --- a/Algorithm.CSharp/ForexInternalFeedOnDataHigherResolutionRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ForexInternalFeedOnDataHigherResolutionRegressionAlgorithm.cs @@ -141,7 +141,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 10; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/ForexInternalFeedOnDataSameResolutionRegressionAlgorithm.cs b/Algorithm.CSharp/ForexInternalFeedOnDataSameResolutionRegressionAlgorithm.cs index ca10ee0aaf4d..1664a83f5921 100644 --- a/Algorithm.CSharp/ForexInternalFeedOnDataSameResolutionRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ForexInternalFeedOnDataSameResolutionRegressionAlgorithm.cs @@ -140,7 +140,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 10; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/ForexMultiResolutionRegressionAlgorithm.cs b/Algorithm.CSharp/ForexMultiResolutionRegressionAlgorithm.cs index 2bd99298e9ee..9f4d5dea4e4b 100644 --- a/Algorithm.CSharp/ForexMultiResolutionRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ForexMultiResolutionRegressionAlgorithm.cs @@ -93,7 +93,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 189; + public int AlgorithmHistoryDataPoints => 55; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/FractionalQuantityRegressionAlgorithm.cs b/Algorithm.CSharp/FractionalQuantityRegressionAlgorithm.cs index 684749ad17ce..da5b1d807ca9 100644 --- a/Algorithm.CSharp/FractionalQuantityRegressionAlgorithm.cs +++ b/Algorithm.CSharp/FractionalQuantityRegressionAlgorithm.cs @@ -101,7 +101,7 @@ private void DataConsolidated(object sender, TradeBar e) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 10; /// /// Final status of the algorithm @@ -119,7 +119,7 @@ private void DataConsolidated(object sender, TradeBar e) {"Compounding Annual Return", "1497.266%"}, {"Drawdown", "5.500%"}, {"Expectancy", "1.339"}, - {"Start Equity", "100000.0"}, + {"Start Equity", "100000.00"}, {"End Equity", "113775.23"}, {"Net Profit", "13.775%"}, {"Sharpe Ratio", "4.906"}, diff --git a/Algorithm.CSharp/FuturesAutomaticSeedRegressionAlgorithm.cs b/Algorithm.CSharp/FuturesAutomaticSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..e6233f05cdd9 --- /dev/null +++ b/Algorithm.CSharp/FuturesAutomaticSeedRegressionAlgorithm.cs @@ -0,0 +1,87 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Securities; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting that futures and future option contracts added via universe selection + /// get automatically seeded by default + /// + public class FuturesAutomaticSeedRegressionAlgorithm : AutomaticSeedBaseRegressionAlgorithm + { + private bool _futureContractsAdded; + private bool _fopsContractsAdded; + + protected override bool ShouldHaveTradeData => true; + protected override bool ShouldHaveQuoteData => false; + protected override bool ShouldHaveOpenInterestData => true; + + public override void Initialize() + { + SetStartDate(2020, 01, 07); + SetEndDate(2020, 01, 07); + SetCash(100000); + + Settings.SeedInitialPrices = true; + + var futures = AddFuture(Futures.Indices.SP500EMini); + futures.SetFilter(0, 365); + + AddFutureOption(futures.Symbol, universe => universe.Strikes(-5, +5)); + } + + public override void OnSecuritiesChanged(SecurityChanges changes) + { + base.OnSecuritiesChanged(changes); + + if (!_futureContractsAdded || !_fopsContractsAdded) + { + foreach (var addedSecurity in changes.AddedSecurities) + { + // Just making sure we had the data to select and seed futures and future options + _futureContractsAdded |= addedSecurity.Symbol.SecurityType == SecurityType.Future; + _fopsContractsAdded |= addedSecurity.Symbol.SecurityType == SecurityType.FutureOption; + } + } + } + + public override void OnEndOfAlgorithm() + { + if (!_futureContractsAdded) + { + throw new RegressionTestException("No option contracts were added"); + } + + if (!_fopsContractsAdded) + { + throw new RegressionTestException("No future option contracts were added"); + } + } + + /// + /// Data Points count of all timeslices of algorithm + /// + public override long DataPoints => 448; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 453; + } +} diff --git a/Algorithm.CSharp/HSIFutureDailyRegressionAlgorithm.cs b/Algorithm.CSharp/HSIFutureDailyRegressionAlgorithm.cs index 2529ddccfa1c..2c851fdfc754 100644 --- a/Algorithm.CSharp/HSIFutureDailyRegressionAlgorithm.cs +++ b/Algorithm.CSharp/HSIFutureDailyRegressionAlgorithm.cs @@ -36,7 +36,7 @@ public class HSIFutureDailyRegressionAlgorithm : HSIFutureHourRegressionAlgorith /// /// Data Points count of the algorithm history /// - public override int AlgorithmHistoryDataPoints => 17; + public override int AlgorithmHistoryDataPoints => 115; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/HSIFutureHourRegressionAlgorithm.cs b/Algorithm.CSharp/HSIFutureHourRegressionAlgorithm.cs index 637b42ed7b31..7cc7211d1371 100644 --- a/Algorithm.CSharp/HSIFutureHourRegressionAlgorithm.cs +++ b/Algorithm.CSharp/HSIFutureHourRegressionAlgorithm.cs @@ -166,7 +166,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes) /// /// Data Points count of the algorithm history /// - public virtual int AlgorithmHistoryDataPoints => 30; + public virtual int AlgorithmHistoryDataPoints => 133; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/HistoryProviderManagerRegressionAlgorithm.cs b/Algorithm.CSharp/HistoryProviderManagerRegressionAlgorithm.cs index 07499a1fd0ad..1490fd859681 100644 --- a/Algorithm.CSharp/HistoryProviderManagerRegressionAlgorithm.cs +++ b/Algorithm.CSharp/HistoryProviderManagerRegressionAlgorithm.cs @@ -77,7 +77,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 14061; + public int AlgorithmHistoryDataPoints => 100; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/ImmediateExecutionModelWorksWithBinanceFeeModel.cs b/Algorithm.CSharp/ImmediateExecutionModelWorksWithBinanceFeeModel.cs index fcd4655a458b..d6a75735c42e 100644 --- a/Algorithm.CSharp/ImmediateExecutionModelWorksWithBinanceFeeModel.cs +++ b/Algorithm.CSharp/ImmediateExecutionModelWorksWithBinanceFeeModel.cs @@ -75,7 +75,7 @@ public override void OnOrderEvent(OrderEvent orderEvent) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/IndexOptionChainApisConsistencyRegressionAlgorithm.cs b/Algorithm.CSharp/IndexOptionChainApisConsistencyRegressionAlgorithm.cs index b6a759012d6e..d5b6c4b78931 100644 --- a/Algorithm.CSharp/IndexOptionChainApisConsistencyRegressionAlgorithm.cs +++ b/Algorithm.CSharp/IndexOptionChainApisConsistencyRegressionAlgorithm.cs @@ -36,5 +36,10 @@ protected override Option GetOption() /// Data Points count of all timeslices of algorithm /// public override long DataPoints => 2862; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 2; } } diff --git a/Algorithm.CSharp/IndicatorSelectorsWorkWithDifferentOptions.cs b/Algorithm.CSharp/IndicatorSelectorsWorkWithDifferentOptions.cs index d16d5f66607e..06cba9c1b22d 100644 --- a/Algorithm.CSharp/IndicatorSelectorsWorkWithDifferentOptions.cs +++ b/Algorithm.CSharp/IndicatorSelectorsWorkWithDifferentOptions.cs @@ -191,7 +191,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 351; + public int AlgorithmHistoryDataPoints => 296; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/InsufficientMarginOrderUpdateRegressionAlgorithm.cs b/Algorithm.CSharp/InsufficientMarginOrderUpdateRegressionAlgorithm.cs index e20792fb57e4..e71d106cd8cc 100644 --- a/Algorithm.CSharp/InsufficientMarginOrderUpdateRegressionAlgorithm.cs +++ b/Algorithm.CSharp/InsufficientMarginOrderUpdateRegressionAlgorithm.cs @@ -159,7 +159,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 5; /// /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm diff --git a/Algorithm.CSharp/LargeQuantityOptionStrategyAlgorithm.cs b/Algorithm.CSharp/LargeQuantityOptionStrategyAlgorithm.cs index f1062955a0f2..ef4a70c268c5 100644 --- a/Algorithm.CSharp/LargeQuantityOptionStrategyAlgorithm.cs +++ b/Algorithm.CSharp/LargeQuantityOptionStrategyAlgorithm.cs @@ -111,7 +111,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 7; + public int AlgorithmHistoryDataPoints => 175; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/OptionEquityBaseStrategyRegressionAlgorithm.cs b/Algorithm.CSharp/OptionEquityBaseStrategyRegressionAlgorithm.cs index 279a5a315077..e0d16846d3f7 100644 --- a/Algorithm.CSharp/OptionEquityBaseStrategyRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionEquityBaseStrategyRegressionAlgorithm.cs @@ -80,12 +80,12 @@ protected decimal GetPriceSpreadDifference(params Symbol[] symbols) { if (security.AskPrice != 0) { - spread = security.Price - security.AskPrice; + spread = security.Price - security.Holdings.AveragePrice; } } else if(security.BidPrice != 0) { - spread = security.BidPrice - security.Price; + spread = security.Holdings.AveragePrice - security.Price; } spreadPaid += spread * actualQuantity * security.SymbolProperties.ContractMultiplier; } diff --git a/Algorithm.CSharp/OptionsAutomaticSeedRegressionAlgorithm.cs b/Algorithm.CSharp/OptionsAutomaticSeedRegressionAlgorithm.cs new file mode 100644 index 000000000000..4e123af087ba --- /dev/null +++ b/Algorithm.CSharp/OptionsAutomaticSeedRegressionAlgorithm.cs @@ -0,0 +1,101 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Data; +using QuantConnect.Data.UniverseSelection; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting that option contracts added via universe selection get automatically seeded by default + /// + public class OptionsAutomaticSeedRegressionAlgorithm : AutomaticSeedBaseRegressionAlgorithm + { + private bool _contractsAdded; + + protected override bool ShouldHaveTradeData => true; + protected override bool ShouldHaveQuoteData => true; + protected override bool ShouldHaveOpenInterestData => true; + + public override void Initialize() + { + SetStartDate(2015, 12, 28); + SetEndDate(2015, 12, 28); + SetCash(100000); + + Settings.SeedInitialPrices = true; + UniverseSettings.Resolution = Resolution.Minute; + + var equity = AddEquity("GOOG"); + + // This security should haven been seeded right away + if (!equity.HasData || equity.Price == 0) + { + throw new RegressionTestException("Equity security was not seeded"); + } + + var option = AddOption(equity.Symbol); + + option.SetFilter(u => u.Strikes(-2, +2).Expiration(0, 180)); + } + + public override void OnData(Slice slice) + { + if (Time.TimeOfDay.Hours > 12) + { + var anotherEquity = AddEquity("SPY", Resolution.Daily); + + // This security should haven been seeded right away + if (!anotherEquity.HasData || anotherEquity.Price == 0) + { + throw new RegressionTestException("Equity security was not seeded"); + } + } + } + + public override void OnSecuritiesChanged(SecurityChanges changes) + { + base.OnSecuritiesChanged(changes); + + if (!_contractsAdded) + { + foreach (var addedSecurity in changes.AddedSecurities) + { + // Just making sure we had the data to select and seed options + _contractsAdded |= addedSecurity.Symbol.SecurityType == SecurityType.Option; + } + } + } + + public override void OnEndOfAlgorithm() + { + if (!_contractsAdded) + { + throw new RegressionTestException("No option contracts were added"); + } + } + + /// + /// Data Points count of all timeslices of algorithm + /// + public override long DataPoints => 4044; + + /// + /// Data Points count of the algorithm history + /// + public override int AlgorithmHistoryDataPoints => 218; + } +} diff --git a/Algorithm.CSharp/OrderSubmissionDataRegressionAlgorithm.cs b/Algorithm.CSharp/OrderSubmissionDataRegressionAlgorithm.cs index 0e1112e41ec0..5fb54088b678 100644 --- a/Algorithm.CSharp/OrderSubmissionDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OrderSubmissionDataRegressionAlgorithm.cs @@ -86,7 +86,7 @@ private void PlaceTrade(string ticker) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/RegressionTests/CorrelationLastComputedValueRegressionAlgorithm.cs b/Algorithm.CSharp/RegressionTests/CorrelationLastComputedValueRegressionAlgorithm.cs index e532ef49817b..2989c9dbdc6c 100644 --- a/Algorithm.CSharp/RegressionTests/CorrelationLastComputedValueRegressionAlgorithm.cs +++ b/Algorithm.CSharp/RegressionTests/CorrelationLastComputedValueRegressionAlgorithm.cs @@ -97,7 +97,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 72; + public int AlgorithmHistoryDataPoints => 21; /// /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm diff --git a/Algorithm.CSharp/RegressionTests/CustomData/CustomDataUnlinkedTradeBarIconicTypeConsolidationRegressionAlgorithm.cs b/Algorithm.CSharp/RegressionTests/CustomData/CustomDataUnlinkedTradeBarIconicTypeConsolidationRegressionAlgorithm.cs index ffc4d735b3fb..650cff0faf4d 100644 --- a/Algorithm.CSharp/RegressionTests/CustomData/CustomDataUnlinkedTradeBarIconicTypeConsolidationRegressionAlgorithm.cs +++ b/Algorithm.CSharp/RegressionTests/CustomData/CustomDataUnlinkedTradeBarIconicTypeConsolidationRegressionAlgorithm.cs @@ -66,7 +66,7 @@ public override void OnData(Slice slice) /// /// Incrementally updating data /// - private class IncrementallyGeneratedCustomData : UnlinkedDataTradeBar + public class IncrementallyGeneratedCustomData : UnlinkedDataTradeBar { private const decimal _start = 10.01m; private static decimal _step; diff --git a/Algorithm.CSharp/SecurityInitializationOnReAdditionForEquityRegressionAlgorithm.cs b/Algorithm.CSharp/SecurityInitializationOnReAdditionForEquityRegressionAlgorithm.cs index aa5ab9fe6138..a27f4abeab20 100644 --- a/Algorithm.CSharp/SecurityInitializationOnReAdditionForEquityRegressionAlgorithm.cs +++ b/Algorithm.CSharp/SecurityInitializationOnReAdditionForEquityRegressionAlgorithm.cs @@ -192,7 +192,7 @@ protected virtual void AssertSecurityInitializationCount(Dictionary /// Data Points count of the algorithm history /// - public virtual int AlgorithmHistoryDataPoints => 3848; + public virtual int AlgorithmHistoryDataPoints => 2948; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/SecurityInitializationOnReAdditionForManuallyAddedFutureContractRegressionAlgorithm.cs b/Algorithm.CSharp/SecurityInitializationOnReAdditionForManuallyAddedFutureContractRegressionAlgorithm.cs index 3546d5423298..edc4333ff03f 100644 --- a/Algorithm.CSharp/SecurityInitializationOnReAdditionForManuallyAddedFutureContractRegressionAlgorithm.cs +++ b/Algorithm.CSharp/SecurityInitializationOnReAdditionForManuallyAddedFutureContractRegressionAlgorithm.cs @@ -49,7 +49,7 @@ protected override Security AddSecurity() /// /// Data Points count of the algorithm history /// - public override int AlgorithmHistoryDataPoints => 48; + public override int AlgorithmHistoryDataPoints => 80; /// /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm diff --git a/Algorithm.CSharp/SetAccountCurrencyCashBuyingPowerModelRegressionAlgorithm.cs b/Algorithm.CSharp/SetAccountCurrencyCashBuyingPowerModelRegressionAlgorithm.cs index 9210b5a0ff33..0f1c623141ed 100644 --- a/Algorithm.CSharp/SetAccountCurrencyCashBuyingPowerModelRegressionAlgorithm.cs +++ b/Algorithm.CSharp/SetAccountCurrencyCashBuyingPowerModelRegressionAlgorithm.cs @@ -239,7 +239,7 @@ public override void OnOrderEvent(OrderEvent orderEvent) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 15; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/SetAccountCurrencySecurityMarginModelRegressionAlgorithm.cs b/Algorithm.CSharp/SetAccountCurrencySecurityMarginModelRegressionAlgorithm.cs index a07fef771bfe..860ec82a273e 100644 --- a/Algorithm.CSharp/SetAccountCurrencySecurityMarginModelRegressionAlgorithm.cs +++ b/Algorithm.CSharp/SetAccountCurrencySecurityMarginModelRegressionAlgorithm.cs @@ -198,7 +198,7 @@ public override void OnOrderEvent(OrderEvent orderEvent) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 60; + public int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/StartingCapitalRegressionAlgorithm.cs b/Algorithm.CSharp/StartingCapitalRegressionAlgorithm.cs index 0b6e2bafeff9..de6681e61c76 100644 --- a/Algorithm.CSharp/StartingCapitalRegressionAlgorithm.cs +++ b/Algorithm.CSharp/StartingCapitalRegressionAlgorithm.cs @@ -77,7 +77,7 @@ public override void OnData(Slice slice) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 180; + public int AlgorithmHistoryDataPoints => 25; /// /// Final status of the algorithm @@ -95,7 +95,7 @@ public override void OnData(Slice slice) {"Compounding Annual Return", "0%"}, {"Drawdown", "0%"}, {"Expectancy", "0"}, - {"Start Equity", "76970482.10"}, + {"Start Equity", "77011732.10"}, {"End Equity", "71490762.61"}, {"Net Profit", "0%"}, {"Sharpe Ratio", "0"}, diff --git a/Algorithm.CSharp/TwoLegCurrencyConversionRegressionAlgorithm.cs b/Algorithm.CSharp/TwoLegCurrencyConversionRegressionAlgorithm.cs index ec6492821a38..cad8d5c8a156 100644 --- a/Algorithm.CSharp/TwoLegCurrencyConversionRegressionAlgorithm.cs +++ b/Algorithm.CSharp/TwoLegCurrencyConversionRegressionAlgorithm.cs @@ -111,7 +111,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 120; + public int AlgorithmHistoryDataPoints => 20; /// /// Final status of the algorithm @@ -129,7 +129,7 @@ public override void OnEndOfAlgorithm() {"Compounding Annual Return", "0%"}, {"Drawdown", "0%"}, {"Expectancy", "0"}, - {"Start Equity", "132348.63"}, + {"Start Equity", "132337.76"}, {"End Equity", "131620.05"}, {"Net Profit", "0%"}, {"Sharpe Ratio", "0"}, diff --git a/Algorithm.CSharp/UnsettledCashWhenQuoteCurrencyIsNotAccountCurrencyAlgorithm.cs b/Algorithm.CSharp/UnsettledCashWhenQuoteCurrencyIsNotAccountCurrencyAlgorithm.cs index 858bcd24f6d8..2bb42245d952 100644 --- a/Algorithm.CSharp/UnsettledCashWhenQuoteCurrencyIsNotAccountCurrencyAlgorithm.cs +++ b/Algorithm.CSharp/UnsettledCashWhenQuoteCurrencyIsNotAccountCurrencyAlgorithm.cs @@ -169,7 +169,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 7594; + public int AlgorithmHistoryDataPoints => 50; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/WarmupConversionRatesRegressionAlgorithm.cs b/Algorithm.CSharp/WarmupConversionRatesRegressionAlgorithm.cs index 96c1b0fb5cd1..9c96947d10d8 100644 --- a/Algorithm.CSharp/WarmupConversionRatesRegressionAlgorithm.cs +++ b/Algorithm.CSharp/WarmupConversionRatesRegressionAlgorithm.cs @@ -86,7 +86,7 @@ public override void OnData(Slice slice) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 180; + public int AlgorithmHistoryDataPoints => 20; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/WarmupDataTypesRegressionAlgorithm.cs b/Algorithm.CSharp/WarmupDataTypesRegressionAlgorithm.cs index b00eec8b0849..439e7f53c625 100644 --- a/Algorithm.CSharp/WarmupDataTypesRegressionAlgorithm.cs +++ b/Algorithm.CSharp/WarmupDataTypesRegressionAlgorithm.cs @@ -104,7 +104,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public virtual int AlgorithmHistoryDataPoints => 41; + public virtual int AlgorithmHistoryDataPoints => 5; /// /// Final status of the algorithm diff --git a/Algorithm.CSharp/WarmupTrainRegressionAlgorithm.cs b/Algorithm.CSharp/WarmupTrainRegressionAlgorithm.cs index f5a8a55f05b0..584b04234ae9 100644 --- a/Algorithm.CSharp/WarmupTrainRegressionAlgorithm.cs +++ b/Algorithm.CSharp/WarmupTrainRegressionAlgorithm.cs @@ -73,7 +73,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 7494; + public int AlgorithmHistoryDataPoints => 48; /// /// Final status of the algorithm diff --git a/Algorithm.Python/ConsolidateRegressionAlgorithm.py b/Algorithm.Python/ConsolidateRegressionAlgorithm.py index 90b54060b310..25d4e390cecb 100644 --- a/Algorithm.Python/ConsolidateRegressionAlgorithm.py +++ b/Algorithm.Python/ConsolidateRegressionAlgorithm.py @@ -69,17 +69,12 @@ def initialize(self): # custom data self._custom_data_consolidator = 0 - custom_symbol = self.add_data(Bitcoin, "BTC", Resolution.DAILY).symbol + custom_symbol = self.add_data(Bitcoin, "BTC", Resolution.MINUTE).symbol self.consolidate(custom_symbol, timedelta(1), lambda bar: self.increment_counter(1)) - self._custom_data_consolidator2 = 0 - self.consolidate(custom_symbol, Resolution.DAILY, lambda bar: self.increment_counter(2)) - def increment_counter(self, id): if id == 1: self._custom_data_consolidator += 1 - if id == 2: - self._custom_data_consolidator2 += 1 def update_trade_bar(self, bar, position): self._smas[position].update(bar.end_time, bar.volume) @@ -113,9 +108,6 @@ def on_end_of_algorithm(self): if self._custom_data_consolidator == 0: raise ValueError("Custom data consolidator did not consolidate any data") - if self._custom_data_consolidator2 == 0: - raise ValueError("Custom data consolidator 2 did not consolidate any data") - for i, sma in enumerate(self._smas): if sma.samples != self._expected_consolidation_counts[i]: raise AssertionError(f"Expected {self._expected_consolidation_counts[i]} samples in each SMA but found {sma.samples} in SMA in index {i}") diff --git a/Algorithm.Python/CustomDataUniverseRegressionAlgorithm.py b/Algorithm.Python/CustomDataUniverseRegressionAlgorithm.py index 871fd2b22174..c7f16e7080a9 100644 --- a/Algorithm.Python/CustomDataUniverseRegressionAlgorithm.py +++ b/Algorithm.Python/CustomDataUniverseRegressionAlgorithm.py @@ -67,16 +67,18 @@ def on_end_of_algorithm(self): if len(self._selection_time) != 0: raise ValueError(f"Unexpected selection times, missing {len(self._selection_time)}") - def OnSecuritiesChanged(self, changes): - for security in changes.AddedSecurities: + def on_securities_changed(self, changes): + for security in changes.added_securities: if security.symbol.security_type == SecurityType.BASE: continue - self.current_underlying_symbols.add(security.Symbol) + self.current_underlying_symbols.add(security.symbol) - for security in changes.RemovedSecurities: - if security.symbol.security_type == SecurityType.BASE: + for security in changes.removed_securities: + if (security.symbol.security_type == SecurityType.BASE or + # This check can be removed after GH issue #9055 is resolved + not security.symbol in self.current_underlying_symbols): continue - self.current_underlying_symbols.remove(security.Symbol) + self.current_underlying_symbols.remove(security.symbol) class MyPyCustomData(PythonData): diff --git a/Algorithm.Python/HistoryWithCustomDataSourceRegressionAlgorithm.py b/Algorithm.Python/HistoryWithCustomDataSourceRegressionAlgorithm.py index 7574caf39008..282b1ecf95bc 100644 --- a/Algorithm.Python/HistoryWithCustomDataSourceRegressionAlgorithm.py +++ b/Algorithm.Python/HistoryWithCustomDataSourceRegressionAlgorithm.py @@ -55,6 +55,7 @@ def get_source(self, config, date, is_live_mode): def reader(self, config, line, date, is_live_mode): trade_bar = TradeBar.parse_equity(config, line, date) data = CustomData() + data.Symbol = config.symbol data.time = trade_bar.time data.value = trade_bar.value data.close = trade_bar.close diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index 75d178995e62..d9d206e75445 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -25,12 +25,17 @@ using QuantConnect.Python; using Python.Runtime; using QuantConnect.Data.UniverseSelection; -using QuantConnect.Data.Auxiliary; +using QuantConnect.Configuration; namespace QuantConnect.Algorithm { public partial class QCAlgorithm { + private readonly int SeedLookbackPeriod = Config.GetInt("seed-lookback-period", 5); + private readonly int SeedRetryMinuteLookbackPeriod = Config.GetInt("seed-retry-minute-lookback-period", 24 * 60); + private readonly int SeedRetryHourLookbackPeriod = Config.GetInt("seed-retry-hour-lookback-period", 24); + private readonly int SeedRetryDailyLookbackPeriod = Config.GetInt("seed-retry-daily-lookback-period", 10); + private bool _dataDictionaryTickWarningSent; /// @@ -713,7 +718,7 @@ public IEnumerable GetLastKnownPrices(Security security) } /// - /// Yields data to warmup a security for all it's subscribed data types + /// Yields data to warm up a security for all its subscribed data types /// /// The symbol we want to get seed data for /// Securities historical data @@ -726,61 +731,45 @@ public IEnumerable GetLastKnownPrices(Symbol symbol) return Enumerable.Empty(); } - var result = new Dictionary(); - Resolution? resolution = null; - Func requestData = period => - { - var historyRequests = CreateBarCountHistoryRequests(new[] { symbol }, period) - .Select(request => - { - // For speed and memory usage, use Resolution.Minute as the minimum resolution - request.Resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)request.Resolution); - // force no fill forward behavior - request.FillForwardResolution = null; + var data = GetLastKnownPrices([symbol]); + return data.Values.FirstOrDefault() ?? Enumerable.Empty(); + } - resolution = request.Resolution; - return request; - }) - // request only those tick types we didn't get the data we wanted - .Where(request => !result.ContainsKey(request.TickType)) - .ToList(); - foreach (var slice in History(historyRequests)) - { - for (var i = 0; i < historyRequests.Count; i++) - { - var historyRequest = historyRequests[i]; - var data = slice.Get(historyRequest.DataType); - if (data.ContainsKey(symbol)) - { - // keep the last data point per tick type - result[historyRequest.TickType] = (BaseData)data[symbol]; - } - } - } - // true when all history requests tick types have a data point - return historyRequests.All(request => result.ContainsKey(request.TickType)); - }; + /// + /// Yields data to warm up multiple securities for all their subscribed data types + /// + /// The securities we want to get seed data for + /// Securities historical data + [DocumentationAttribute(AddingData)] + [DocumentationAttribute(HistoricalData)] + public Dictionary> GetLastKnownPrices(IEnumerable securities) + { + return GetLastKnownPrices(securities.Select(s => s.Symbol)); + } - if (!requestData(5)) + /// + /// Yields data to warm up multiple securities for all their subscribed data types + /// + /// The symbols we want to get seed data for + /// Securities historical data + [DocumentationAttribute(AddingData)] + [DocumentationAttribute(HistoricalData)] + public Dictionary> GetLastKnownPrices(IEnumerable symbols) + { + if (HistoryProvider == null) { - if (resolution.HasValue) - { - // If the first attempt to get the last know price returns null, it maybe the case of an illiquid security. - // We increase the look-back period for this case accordingly to the resolution to cover 3 trading days - var periods = - resolution.Value == Resolution.Daily ? 3 : - resolution.Value == Resolution.Hour ? 24 : 1440; - requestData(periods); - } - else - { - // this shouldn't happen but just in case - QuantConnect.Logging.Log.Error( - $"QCAlgorithm.GetLastKnownPrices(): no history request was created for symbol {symbol} at {Time}"); - } + return new Dictionary>(); } - // return the data ordered by time ascending - return result.Values.OrderBy(data => data.Time); + + var data = new Dictionary<(Symbol, Type, TickType), BaseData>(); + GetLastKnownPricesImpl(symbols, data); + + return data + .GroupBy(kvp => kvp.Key.Item1) + .ToDictionary(g => g.Key, + g => g.OrderBy(kvp => kvp.Value.Time) + .ThenBy(kvp => GetTickTypeOrder(kvp.Key.Item1.SecurityType, kvp.Key.Item3)) + .Select(kvp => kvp.Value)); } /// @@ -795,12 +784,149 @@ public IEnumerable GetLastKnownPrices(Symbol symbol) [DocumentationAttribute(HistoricalData)] public BaseData GetLastKnownPrice(Security security) { - return GetLastKnownPrices(security.Symbol) + return GetLastKnownPrice(security.Symbol); + } + + /// + /// Get the last known price using the history provider. + /// Useful for seeding securities with the correct price + /// + /// Symbol for which to retrieve historical data + /// A single object with the last known price + [Obsolete("This method is obsolete please use 'GetLastKnownPrices' which will return the last data point" + + " for each type associated with the requested security")] + [DocumentationAttribute(AddingData)] + [DocumentationAttribute(HistoricalData)] + public BaseData GetLastKnownPrice(Symbol symbol) + { + return GetLastKnownPrices(symbol) // since we are returning a single data point let's respect order .OrderByDescending(data => GetTickTypeOrder(data.Symbol.SecurityType, LeanData.GetCommonTickTypeForCommonDataTypes(data.GetType(), data.Symbol.SecurityType))) .LastOrDefault(); } + private void GetLastKnownPricesImpl(IEnumerable symbols, Dictionary<(Symbol, Type, TickType), BaseData> result, + int attempts = 0, IEnumerable failedRequests = null) + { + IEnumerable historyRequests; + var isRetry = failedRequests != null; + + symbols = symbols?.Where(x => !x.IsCanonical() || x.SecurityType == SecurityType.Future); + + if (attempts == 0) + { + historyRequests = CreateBarCountHistoryRequests(symbols, SeedLookbackPeriod, + fillForward: false, useAllSubscriptions: true) + .SelectMany(request => + { + // Make open interest request daily, higher resolutions will need greater periods to return data + if (request.DataType == typeof(OpenInterest) && request.Resolution < Resolution.Daily) + { + return CreateBarCountHistoryRequests([request.Symbol], typeof(OpenInterest), SeedLookbackPeriod, + Resolution.Daily, fillForward: false, useAllSubscriptions: true); + } + + if (request.Resolution < Resolution.Minute) + { + var dataType = request.DataType; + if (dataType == typeof(Tick)) + { + dataType = request.TickType == TickType.Trade ? typeof(TradeBar) : typeof(QuoteBar); + } + + return CreateBarCountHistoryRequests([request.Symbol], dataType, SeedLookbackPeriod, + Resolution.Minute, fillForward: false, useAllSubscriptions: true); + } + + return [request]; + }); + } + else if (attempts == 1) + { + // If the first attempt to get the last know price returns no data, it maybe the case of an illiquid security. + // We increase the look-back period for this case accordingly to the resolution to cover a longer period + historyRequests = failedRequests + .GroupBy(request => request.Symbol) + .Select(group => + { + var symbolRequests = group.ToArray(); + var resolution = symbolRequests[0].Resolution; + var periods = resolution == Resolution.Daily + ? SeedRetryDailyLookbackPeriod + : resolution == Resolution.Hour ? SeedRetryHourLookbackPeriod : SeedRetryMinuteLookbackPeriod; + return CreateBarCountHistoryRequests([group.Key], periods, resolution, fillForward: false, useAllSubscriptions: true) + .Where(request => symbolRequests.Any(x => x.DataType == request.DataType)); + }) + .SelectMany(x => x); + } + else + { + // Fall back to bigger daily requests as a last resort + historyRequests = CreateBarCountHistoryRequests(failedRequests.Select(x => x.Symbol).Distinct(), + Math.Min(60, 5 * SeedRetryDailyLookbackPeriod), Resolution.Daily, fillForward: false, useAllSubscriptions: true); + } + + var requests = historyRequests.ToArray(); + if (requests.Length == 0) + { + return; + } + + foreach (var slice in History(requests)) + { + for (var i = 0; i < requests.Length; i++) + { + var historyRequest = requests[i]; + + // keep the last data point per tick type + BaseData data = null; + + if (historyRequest.DataType == typeof(QuoteBar)) + { + if (slice.QuoteBars.TryGetValue(historyRequest.Symbol, out var quoteBar)) + { + data = quoteBar; + } + } + else if (historyRequest.DataType == typeof(TradeBar)) + { + if (slice.Bars.TryGetValue(historyRequest.Symbol, out var quoteBar)) + { + data = quoteBar; + } + } + else if (historyRequest.DataType == typeof(OpenInterest)) + { + if (slice.Ticks.TryGetValue(historyRequest.Symbol, out var openInterests)) + { + data = openInterests[0]; + } + } + // No Tick data, resolution is limited to Minute as minimum + else + { + var typeData = slice.Get(historyRequest.DataType); + if (typeData.ContainsKey(historyRequest.Symbol)) + { + data = typeData[historyRequest.Symbol]; + } + } + + if (data != null) + { + result[(historyRequest.Symbol, historyRequest.DataType, historyRequest.TickType)] = data; + } + } + } + + if (attempts < 2) + { + // Give it another try to get data for all symbols and all data types + GetLastKnownPricesImpl(null, result, attempts + 1, + requests.Where((request, i) => !result.ContainsKey((request.Symbol, request.DataType, request.TickType)))); + } + } + /// /// Centralized logic to get data typed history given a list of requests for the specified symbol. /// This method is used to keep backwards compatibility for those History methods that expect an ArgumentException to be thrown @@ -990,7 +1116,7 @@ protected IEnumerable CreateDateRangeHistoryRequests(IEnumerable /// private IEnumerable CreateBarCountHistoryRequests(IEnumerable symbols, int periods, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, - DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) + DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null, bool useAllSubscriptions = false) { // Materialize the symbols to avoid multiple enumeration var symbolsArray = symbols.ToArray(); @@ -1003,7 +1129,8 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable @@ -1011,12 +1138,12 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable private IEnumerable CreateBarCountHistoryRequests(IEnumerable symbols, Type requestedType, int periods, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, - DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) + DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null, bool useAllSubscriptions = false) { return symbols.Where(HistoryRequestValid).SelectMany(symbol => { // Match or create configs for the symbol - var configs = GetMatchingSubscriptions(symbol, requestedType, resolution).ToList(); + var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions).ToList(); if (configs.Count == 0) { return Enumerable.Empty(); @@ -1043,7 +1170,7 @@ private int GetTickTypeOrder(SecurityType securityType, TickType tickType) return SubscriptionManager.AvailableDataTypes[securityType].IndexOf(tickType); } - private IEnumerable GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null) + private IEnumerable GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null, bool useAllSubscriptions = false) { var subscriptions = SubscriptionManager.SubscriptionDataConfigService // we add internal subscription so that history requests are covered, this allows us to warm them up too @@ -1068,7 +1195,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb else { // Filter subscriptions matching the requested type - matchingSubscriptions = subscriptions.Where(s => SubscriptionDataConfigTypeFilter(type, s.Type)); + matchingSubscriptions = subscriptions.Where(s => SubscriptionDataConfigTypeFilter(type, s.Type, filterOutOpenInterest: !useAllSubscriptions)); } var internalConfig = new List(); @@ -1091,9 +1218,16 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb { configs = userConfig; } - else if (internalConfig.Count != 0) + if ((useAllSubscriptions || configs == null) && internalConfig.Count != 0) { - configs = internalConfig; + if (configs == null) + { + configs = internalConfig; + } + else + { + configs.AddRange(internalConfig); + } } // we use the subscription manager registered configurations here, we can not rely on the Securities collection @@ -1186,7 +1320,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb .LookupSubscriptionConfigDataTypes(symbol.SecurityType, res, // for continuous contracts, if we are given a type (or none) that's common (like trade/quote), we respect it symbol.IsCanonical() && (symbol.SecurityType != SecurityType.Future || type != null && !LeanData.IsCommonLeanDataType(type))) - .Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1)) + .Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1, filterOutOpenInterest: !useAllSubscriptions)) .Select(x => { var configType = x.Item1; @@ -1220,16 +1354,16 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb /// /// If the target type is , config types will return false. /// This is useful to filter OpenInterest by default from history requests unless it's explicitly requested - private bool SubscriptionDataConfigTypeFilter(Type targetType, Type configType) + private bool SubscriptionDataConfigTypeFilter(Type targetType, Type configType, bool filterOutOpenInterest = true) { if (targetType == null) { - return configType != typeof(OpenInterest); + return !filterOutOpenInterest || configType != typeof(OpenInterest); } var targetIsGenericType = targetType == typeof(BaseData); - return targetType.IsAssignableFrom(configType) && (!targetIsGenericType || configType != typeof(OpenInterest)); + return targetType.IsAssignableFrom(configType) && (!targetIsGenericType || !filterOutOpenInterest || configType != typeof(OpenInterest)); } private SecurityExchangeHours GetExchangeHours(Symbol symbol, Type type = null) diff --git a/Algorithm/QCAlgorithm.Universe.cs b/Algorithm/QCAlgorithm.Universe.cs index 8dac58fd2d71..12d34cd9f6ba 100644 --- a/Algorithm/QCAlgorithm.Universe.cs +++ b/Algorithm/QCAlgorithm.Universe.cs @@ -75,7 +75,7 @@ public void OnEndOfTimeStep() return; } - var requiredHistoryRequests = new Dictionary(); + var securitiesToSeed = new HashSet(); foreach (var security in Securities.Select(kvp => kvp.Value).Union( _pendingUserDefinedUniverseSecurityAdditions.Select(x => x.Security))) @@ -123,22 +123,9 @@ public void OnEndOfTimeStep() ConfigureUnderlyingSecurity(underlyingSecurity); } - if (LiveMode && underlyingSecurity.GetLastData() == null) + if (LiveMode && !Settings.SeedInitialPrices && underlyingSecurity.GetLastData() == null) { - if (requiredHistoryRequests.ContainsKey(underlyingSecurity)) - { - // lets request the higher resolution - var currentResolutionRequest = requiredHistoryRequests[underlyingSecurity]; - if (currentResolutionRequest != Resolution.Minute // Can not be less than Minute - && resolution < currentResolutionRequest) - { - requiredHistoryRequests[underlyingSecurity] = (Resolution)Math.Max((int)resolution, (int)Resolution.Minute); - } - } - else - { - requiredHistoryRequests.Add(underlyingSecurity, (Resolution)Math.Max((int)resolution, (int)Resolution.Minute)); - } + securitiesToSeed.Add(underlyingSecurity); } // set the underlying security on the derivative -- we do this in two places since it's possible // to do AddOptionContract w/out the underlying already added and normalized properly @@ -150,22 +137,9 @@ public void OnEndOfTimeStep() } } - if (!requiredHistoryRequests.IsNullOrEmpty()) + if (!securitiesToSeed.IsNullOrEmpty()) { - // Create requests - var historyRequests = Enumerable.Empty(); - foreach (var byResolution in requiredHistoryRequests.GroupBy(x => x.Value)) - { - historyRequests = historyRequests.Concat( - CreateBarCountHistoryRequests(byResolution.Select(x => x.Key.Symbol), 3, byResolution.Key)); - } - // Request data - var historicLastData = History(historyRequests); - historicLastData.PushThrough(x => - { - var security = requiredHistoryRequests.Keys.FirstOrDefault(y => y.Symbol == x.Symbol); - security?.Cache.AddData(x); - }); + AlgorithmUtils.SeedSecurities(securitiesToSeed, this); } // add subscriptionDataConfig to their respective user defined universes diff --git a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs index 5b11d40c5959..8cd9fce81516 100644 --- a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs +++ b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs @@ -1087,9 +1087,23 @@ public void OnWarmupFinished() /// Get the last known price using the history provider. /// Useful for seeding securities with the correct price /// - /// object for which to retrieve historical data + /// Symbol for which to retrieve historical data /// A single object with the last known price - public BaseData GetLastKnownPrice(Security security) => _baseAlgorithm.GetLastKnownPrice(security); + public BaseData GetLastKnownPrice(Symbol symbol) => _baseAlgorithm.GetLastKnownPrice(symbol); + + /// + /// Yields data to warmup a security for all it's subscribed data types + /// + /// Symbol for which to retrieve historical data + /// Securities historical data + public IEnumerable GetLastKnownPrices(Symbol symbol) => _baseAlgorithm.GetLastKnownPrices(symbol); + + /// + /// Yields data to warm up multiple securities for all their subscribed data types + /// + /// The symbols we want to get seed data for + /// Securities historical data + public Dictionary> GetLastKnownPrices(IEnumerable symbols) => _baseAlgorithm.GetLastKnownPrices(symbols); /// /// Set the runtime error diff --git a/Common/AlgorithmSettings.cs b/Common/AlgorithmSettings.cs index e8eaec146a7e..ae79dfd43544 100644 --- a/Common/AlgorithmSettings.cs +++ b/Common/AlgorithmSettings.cs @@ -173,6 +173,11 @@ public Resolution? WarmUpResolution /// public TimeSpan PerformanceSamplePeriod { get; set; } + /// + /// Determines whether to seed initial prices for all selected and manually added securities. + /// + public bool SeedInitialPrices { get; set; } + /// /// Initializes a new instance of the class /// @@ -189,6 +194,7 @@ public AlgorithmSettings() MinAbsolutePortfolioTargetPercentage = 0.0000000001m; DatabasesRefreshPeriod = _defaultDatabasesRefreshPeriod; IgnoreUnknownAssetHoldings = _defaultIgnoreUnknownAssetHoldings; + SeedInitialPrices = false; } } } diff --git a/Common/AlgorithmUtils.cs b/Common/AlgorithmUtils.cs new file mode 100644 index 000000000000..df62c3b46b93 --- /dev/null +++ b/Common/AlgorithmUtils.cs @@ -0,0 +1,50 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect +{ + /// + /// Provides utility methods for or related to algorithms + /// + public static class AlgorithmUtils + { + /// + /// Seeds the provided securities with their last known prices from the algorithm + /// + /// The securities to seed + /// The algorithm instance + public static void SeedSecurities(IReadOnlyCollection securities, IAlgorithm algorithm) + { + var securitiesToSeed = securities.Where(x => x.Price == 0); + var data = algorithm.GetLastKnownPrices(securitiesToSeed.Select(x => x.Symbol)); + + foreach (var security in securitiesToSeed) + { + if (data.TryGetValue(security.Symbol, out var seedData)) + { + foreach (var datum in seedData) + { + security.SetMarketPrice(datum); + } + } + } + } + } +} diff --git a/Common/Interfaces/IAlgorithm.cs b/Common/Interfaces/IAlgorithm.cs index 4fb640376b27..2ac9f8392b3f 100644 --- a/Common/Interfaces/IAlgorithm.cs +++ b/Common/Interfaces/IAlgorithm.cs @@ -839,9 +839,23 @@ Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool fillForw /// Get the last known price using the history provider. /// Useful for seeding securities with the correct price /// - /// object for which to retrieve historical data + /// The symbol for which to retrieve historical data /// A single object with the last known price - BaseData GetLastKnownPrice(Security security); + BaseData GetLastKnownPrice(Symbol symbol); + + /// + /// Yields data to warmup a security for all it's subscribed data types + /// + /// The symbol for which to retrieve historical data + /// Securities historical data + IEnumerable GetLastKnownPrices(Symbol symbol); + + /// + /// Yields data to warm up multiple securities for all their subscribed data types + /// + /// The symbols we want to get seed data for + /// Securities historical data + Dictionary> GetLastKnownPrices(IEnumerable symbols); /// /// Set the runtime error diff --git a/Common/Interfaces/IAlgorithmSettings.cs b/Common/Interfaces/IAlgorithmSettings.cs index e30072063e1f..7e71531f1078 100644 --- a/Common/Interfaces/IAlgorithmSettings.cs +++ b/Common/Interfaces/IAlgorithmSettings.cs @@ -143,5 +143,10 @@ public interface IAlgorithmSettings /// Performance tracking sample period to use if any, useful to debug performance issues /// TimeSpan PerformanceSamplePeriod { get; set; } + + /// + /// Determines whether to seed initial prices for all selected and manually added securities. + /// + bool SeedInitialPrices { get; set; } } } diff --git a/Common/Interfaces/ISecurityService.cs b/Common/Interfaces/ISecurityService.cs index 5d7f362f389d..466fcb8c7cc1 100644 --- a/Common/Interfaces/ISecurityService.cs +++ b/Common/Interfaces/ISecurityService.cs @@ -34,7 +34,8 @@ Security CreateSecurity(Symbol symbol, List subscriptionDataConfigList, decimal leverage = 0, bool addToSymbolCache = true, - Security underlying = null); + Security underlying = null, + bool seedSecurity = true); /// /// Creates a new security @@ -45,7 +46,8 @@ Security CreateSecurity(Symbol symbol, SubscriptionDataConfig subscriptionDataConfig, decimal leverage = 0, bool addToSymbolCache = true, - Security underlying = null); + Security underlying = null, + bool seedSecurity = true); /// /// Creates a new benchmark security diff --git a/Common/Securities/Cash.cs b/Common/Securities/Cash.cs index 6858ab484c9e..a561c1f87666 100644 --- a/Common/Securities/Cash.cs +++ b/Common/Securities/Cash.cs @@ -316,7 +316,9 @@ public List EnsureCurrencyDataFeed(SecurityManager secur var newSecurity = securityService.CreateSecurity(symbol, config, - addToSymbolCache: false); + addToSymbolCache: false, + // All securities added for currency conversion will be seeded in batch after all are created + seedSecurity: false); Log.Trace("Cash.EnsureCurrencyDataFeed(): " + Messages.Cash.AddingSecuritySymbolForCashCurrencyFeed(symbol, Symbol)); diff --git a/Common/Securities/SecurityService.cs b/Common/Securities/SecurityService.cs index 9ba4d2a80fa0..aab216a7f11b 100644 --- a/Common/Securities/SecurityService.cs +++ b/Common/Securities/SecurityService.cs @@ -73,10 +73,14 @@ private Security CreateSecurity(Symbol symbol, bool addToSymbolCache, Security underlying, bool initializeSecurity, - bool reCreateSecurity) + bool reCreateSecurity, + bool seedSecurity) { var configList = new SubscriptionDataConfigList(symbol); - configList.AddRange(subscriptionDataConfigList); + if (subscriptionDataConfigList != null) + { + configList.AddRange(subscriptionDataConfigList); + } if (!reCreateSecurity && _algorithm != null && _algorithm.Securities.TryGetValue(symbol, out var existingSecurity)) { @@ -88,7 +92,7 @@ private Security CreateSecurity(Symbol symbol, existingSecurity.MakeTradable(); } - InitializeSecurity(initializeSecurity, existingSecurity); + InitializeSecurity(initializeSecurity, existingSecurity, seedSecurity); return existingSecurity; } @@ -231,7 +235,7 @@ private Security CreateSecurity(Symbol symbol, security.AddData(configList); // invoke the security initializer - InitializeSecurity(initializeSecurity, security); + InitializeSecurity(initializeSecurity, security, seedSecurity); CheckCanonicalSecurityModels(security); @@ -262,10 +266,11 @@ public Security CreateSecurity(Symbol symbol, List subscriptionDataConfigList, decimal leverage = 0, bool addToSymbolCache = true, - Security underlying = null) + Security underlying = null, + bool seedSecurity = true) { return CreateSecurity(symbol, subscriptionDataConfigList, leverage, addToSymbolCache, underlying, - initializeSecurity: true, reCreateSecurity: false); + initializeSecurity: true, reCreateSecurity: false, seedSecurity: seedSecurity); } /// @@ -273,9 +278,14 @@ public Security CreateSecurity(Symbol symbol, /// /// Following the obsoletion of Security.Subscriptions, /// both overloads will be merged removing arguments - public Security CreateSecurity(Symbol symbol, SubscriptionDataConfig subscriptionDataConfig, decimal leverage = 0, bool addToSymbolCache = true, Security underlying = null) + public Security CreateSecurity(Symbol symbol, + SubscriptionDataConfig subscriptionDataConfig, + decimal leverage = 0, + bool addToSymbolCache = true, + Security underlying = null, + bool seedSecurity = true) { - return CreateSecurity(symbol, new List { subscriptionDataConfig }, leverage, addToSymbolCache, underlying); + return CreateSecurity(symbol, new List { subscriptionDataConfig }, leverage, addToSymbolCache, underlying, seedSecurity); } /// @@ -291,7 +301,8 @@ public Security CreateBenchmarkSecurity(Symbol symbol) addToSymbolCache: false, underlying: null, initializeSecurity: false, - reCreateSecurity: true); + reCreateSecurity: true, + seedSecurity: false); } /// @@ -328,10 +339,15 @@ private void CheckCanonicalSecurityModels(Security security) } } - private void InitializeSecurity(bool initializeSecurity, Security security) + private void InitializeSecurity(bool initializeSecurity, Security security, bool seedSecurity) { if (initializeSecurity && !security.IsInitialized) { + if (seedSecurity && _algorithm.Settings.SeedInitialPrices) + { + AlgorithmUtils.SeedSecurities([security], _algorithm); + } + _securityInitializerProvider.SecurityInitializer.Initialize(security); security.IsInitialized = true; } diff --git a/Engine/DataFeeds/UniverseSelection.cs b/Engine/DataFeeds/UniverseSelection.cs index 14ca3bd5a3ab..3a8a2e2c16fd 100644 --- a/Engine/DataFeeds/UniverseSelection.cs +++ b/Engine/DataFeeds/UniverseSelection.cs @@ -313,6 +313,8 @@ public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTi Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges); } + SeedAddedSecurities(securityChanges); + return securityChanges; } @@ -496,12 +498,26 @@ private Security GetOrCreateSecurity(Dictionary pendingAdditio Security security; if (!pendingAdditions.TryGetValue(symbol, out security)) { - security = _securityService.CreateSecurity(symbol, new List(), universeSettings.Leverage, symbol.ID.SecurityType.IsOption(), underlying); + security = _securityService.CreateSecurity(symbol, + (List)null, + universeSettings.Leverage, + symbol.ID.SecurityType.IsOption(), + underlying, + // Securities will be seeded after all selections are applied + seedSecurity: false); pendingAdditions.Add(symbol, security); } return security; } + + private void SeedAddedSecurities(SecurityChanges changes) + { + if (_algorithm.Settings.SeedInitialPrices) + { + AlgorithmUtils.SeedSecurities(changes.AddedSecurities, _algorithm); + } + } } } diff --git a/Engine/Setup/BaseSetupHandler.cs b/Engine/Setup/BaseSetupHandler.cs index 7fc933084e64..44998da47fc9 100644 --- a/Engine/Setup/BaseSetupHandler.cs +++ b/Engine/Setup/BaseSetupHandler.cs @@ -29,7 +29,6 @@ using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Data.UniverseSelection; using QuantConnect.Lean.Engine.DataFeeds.WorkScheduling; -using HistoryRequest = QuantConnect.Data.HistoryRequest; using QuantConnect.Securities; namespace QuantConnect.Lean.Engine.Setup @@ -99,95 +98,13 @@ public static void SetupCurrencyConversions( .Distinct() .ToList(); - var historyRequestFactory = new HistoryRequestFactory(algorithm); - var historyRequests = new List(); - foreach (var security in securitiesToUpdate) - { - var configs = algorithm - .SubscriptionManager - .SubscriptionDataConfigService - .GetSubscriptionDataConfigs(security.Symbol, - includeInternalConfigs: true); - - // we need to order and select a specific configuration type - // so the conversion rate is deterministic - var configToUse = configs.OrderBy(x => x.TickType).First(); - var hours = security.Exchange.Hours; - - var resolution = configs.GetHighestResolution(); - var startTime = historyRequestFactory.GetStartTimeAlgoTz( - security.Symbol, - 60, - resolution, - hours, - configToUse.DataTimeZone, - configToUse.Type); - var endTime = algorithm.Time; - - historyRequests.Add(historyRequestFactory.CreateHistoryRequest( - configToUse, - startTime, - endTime, - security.Exchange.Hours, - resolution)); - } - - // Attempt to get history for these requests and update cash - var slices = algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone); - slices.PushThrough(data => - { - foreach (var security in securitiesToUpdate.Where(x => x.Symbol == data.Symbol)) - { - security.SetMarketPrice(data); - } - }); + AlgorithmUtils.SeedSecurities(securitiesToUpdate, algorithm); foreach (var cash in cashToUpdate) { cash.Update(); } - // Any remaining unassigned cash will attempt to fall back to a daily resolution history request to resolve - var unassignedCash = cashToUpdate.Where(x => x.ConversionRate == 0).ToList(); - if (unassignedCash.Any()) - { - Log.Trace( - $"Failed to assign conversion rates for the following cash: {string.Join(",", unassignedCash.Select(x => x.Symbol))}." + - $" Attempting to request daily resolution history to resolve conversion rate"); - - var unassignedCashSymbols = unassignedCash - .SelectMany(x => x.SecuritySymbols) - .ToHashSet(); - - var replacementHistoryRequests = new List(); - foreach (var request in historyRequests.Where(x => - unassignedCashSymbols.Contains(x.Symbol) && x.Resolution < Resolution.Daily)) - { - var newRequest = new HistoryRequest(request.EndTimeUtc.AddDays(-10), request.EndTimeUtc, - request.DataType, - request.Symbol, Resolution.Daily, request.ExchangeHours, request.DataTimeZone, - request.FillForwardResolution, - request.IncludeExtendedMarketHours, request.IsCustomData, request.DataNormalizationMode, - request.TickType); - - replacementHistoryRequests.Add(newRequest); - } - - slices = algorithm.HistoryProvider.GetHistory(replacementHistoryRequests, algorithm.TimeZone); - slices.PushThrough(data => - { - foreach (var security in securitiesToUpdate.Where(x => x.Symbol == data.Symbol)) - { - security.SetMarketPrice(data); - } - }); - - foreach (var cash in unassignedCash) - { - cash.Update(); - } - } - Log.Trace($"BaseSetupHandler.SetupCurrencyConversions():{Environment.NewLine}" + $"Account Type: {algorithm.BrokerageModel.AccountType}{Environment.NewLine}{Environment.NewLine}{algorithm.Portfolio.CashBook}"); // this is useful for debugging diff --git a/Launcher/config.json b/Launcher/config.json index 8a83567be959..fbe48f9ee6e7 100644 --- a/Launcher/config.json +++ b/Launcher/config.json @@ -48,6 +48,12 @@ "symbol-second-limit": 10000, "symbol-tick-limit": 10000, + // retry lookback periods for automatic security seed history requests + "seed-lookback-period": 5, + "seed-retry-minute-lookback-period": 1440, + "seed-retry-hour-lookback-period": 24, + "seed-retry-daily-lookback-period": 10, + // Ignore unknown asset types from brokerages "ignore-unknown-asset-holdings": true, diff --git a/Tests/Algorithm/AlgorithmAddDataTests.cs b/Tests/Algorithm/AlgorithmAddDataTests.cs index 1074a637fd8e..374c654efd26 100644 --- a/Tests/Algorithm/AlgorithmAddDataTests.cs +++ b/Tests/Algorithm/AlgorithmAddDataTests.cs @@ -179,6 +179,7 @@ public void OnEndOfTimeStepSeedsUnderlyingSecuritiesThatHaveNoData() var qcAlgorithm = new QCAlgorithm(); qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm, new MockDataFeed())); qcAlgorithm.SetLiveMode(true); + qcAlgorithm.Settings.SeedInitialPrices = false; var testHistoryProvider = new TestHistoryProvider(); qcAlgorithm.HistoryProvider = testHistoryProvider; @@ -201,6 +202,7 @@ public void OnEndOfTimeStepDoesNotThrowWhenSeedsSameUnderlyingForTwoSecurities() var qcAlgorithm = new QCAlgorithm(); qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm, new MockDataFeed())); qcAlgorithm.SetLiveMode(true); + qcAlgorithm.Settings.SeedInitialPrices = false; var testHistoryProvider = new TestHistoryProvider(); qcAlgorithm.HistoryProvider = testHistoryProvider; var option = qcAlgorithm.AddOption(testHistoryProvider.underlyingSymbol); @@ -215,7 +217,6 @@ public void OnEndOfTimeStepDoesNotThrowWhenSeedsSameUnderlyingForTwoSecurities() qcAlgorithm.OnEndOfTimeStep(); var data = qcAlgorithm.Securities[testHistoryProvider.underlyingSymbol].GetLastData(); - Assert.AreEqual(testHistoryProvider.LastResolutionRequest, Resolution.Minute); Assert.IsNotNull(data); Assert.AreEqual(data.Price, 2); } @@ -728,7 +729,6 @@ private class TestHistoryProvider : HistoryProviderBase public string underlyingSymbol = "GOOG"; public string underlyingSymbol2 = "AAPL"; public override int DataPointCount { get; } - public Resolution LastResolutionRequest; public override void Initialize(HistoryProviderInitializeParameters parameters) { @@ -738,12 +738,11 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) { var now = DateTime.UtcNow; - LastResolutionRequest = requests.First().Resolution; #pragma warning disable CS0618 var tradeBar1 = new TradeBar(now, underlyingSymbol, 1, 1, 1, 1, 1, TimeSpan.FromDays(1)); var tradeBar2 = new TradeBar(now, underlyingSymbol2, 3, 3, 3, 3, 3, TimeSpan.FromDays(1)); var slice1 = new Slice(now, new List { tradeBar1, tradeBar2 }, - new TradeBars(now), new QuoteBars(), + new TradeBars(now) { tradeBar1, tradeBar2 }, new QuoteBars(), new Ticks(), new OptionChains(), new FuturesChains(), new Splits(), new Dividends(now), new Delistings(), @@ -751,7 +750,7 @@ public override IEnumerable GetHistory(IEnumerable reques var tradeBar1_2 = new TradeBar(now, underlyingSymbol, 2, 2, 2, 2, 2, TimeSpan.FromDays(1)); #pragma warning restore CS0618 var slice2 = new Slice(now, new List { tradeBar1_2 }, - new TradeBars(now), new QuoteBars(), + new TradeBars(now) { tradeBar1_2 }, new QuoteBars(), new Ticks(), new OptionChains(), new FuturesChains(), new Splits(), new Dividends(now), new Delistings(), diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index 7e62b3eca925..68ff1646e18c 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -522,6 +522,7 @@ def getTickHistory(algorithm, symbol, start, end): public void TimeSpanHistoryRequestIsCorrectlyBuilt(Resolution resolution, Language language, bool symbolAlreadyAdded) { _algorithm.SetStartDate(2013, 10, 07); + _algorithm.Settings.SeedInitialPrices = false; var symbol = Symbols.SPY; if (symbolAlreadyAdded) @@ -618,6 +619,7 @@ public void BarCountHistoryRequestIsCorrectlyBuilt(Resolution? resolution, Langu bool symbolAlreadyAdded, DateTime dateTime, Resolution? defaultResolution, bool multiSymbol) { _algorithm.SetStartDate(dateTime); + _algorithm.Settings.SeedInitialPrices = false; if (symbolAlreadyAdded) { @@ -695,6 +697,7 @@ public void BarCountHistoryRequestIsCorrectlyBuilt(Resolution? resolution, Langu public void TickHistoryRequestIgnoresFillForward(Language language, bool symbolAlreadyAdded) { _algorithm.SetStartDate(2013, 10, 07); + _algorithm.Settings.SeedInitialPrices = false; var symbol = Symbols.SPY; if (symbolAlreadyAdded) @@ -895,9 +898,10 @@ public void GetLastKnownPricesOption() var option = algorithm.AddOptionContract(Symbols.CreateOptionSymbol("AAPL", OptionRight.Call, 250m, new DateTime(2016, 01, 15))); var lastKnownPrices = algorithm.GetLastKnownPrices(option).ToList(); - Assert.AreEqual(2, lastKnownPrices.Count); + Assert.AreEqual(3, lastKnownPrices.Count); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar))); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar))); + Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(OpenInterest))); } [Test] @@ -919,9 +923,10 @@ public void GetLastKnownPricesFuture() var future = algorithm.AddSecurity(Symbols.CreateFutureSymbol(Futures.Indices.SP500EMini, new DateTime(2013, 12, 20))); var lastKnownPrices = algorithm.GetLastKnownPrices(future).ToList(); - Assert.AreEqual(2, lastKnownPrices.Count); + Assert.AreEqual(3, lastKnownPrices.Count); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar))); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar))); + Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(OpenInterest))); } [TestCase(Language.CSharp)] diff --git a/Tests/Algorithm/AlgorithmRegisterIndicatorTests.cs b/Tests/Algorithm/AlgorithmRegisterIndicatorTests.cs index a5c49c6d4b30..3bdaa6f0156b 100644 --- a/Tests/Algorithm/AlgorithmRegisterIndicatorTests.cs +++ b/Tests/Algorithm/AlgorithmRegisterIndicatorTests.cs @@ -211,7 +211,7 @@ from QuantConnect.Lean.Engine.DataFeeds import * marketHoursDatabase = MarketHoursDatabase.FromDataFolder() symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder() -securityService = SecurityService(algo.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, algo, RegisteredSecurityDataTypesProvider.Null, SecurityCacheProvider(algo.Portfolio)) +securityService = SecurityService(algo.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, algo, RegisteredSecurityDataTypesProvider.Null, SecurityCacheProvider(algo.Portfolio), algorithm=algo) algo.Securities.SetSecurityService(securityService) dataPermissionManager = DataPermissionManager() dataManager = DataManager(None, UniverseSelection(algo, securityService, dataPermissionManager, None), algo, algo.TimeKeeper, marketHoursDatabase, False, RegisteredSecurityDataTypesProvider.Null, dataPermissionManager) diff --git a/Tests/Algorithm/AlgorithmWarmupTests.cs b/Tests/Algorithm/AlgorithmWarmupTests.cs index 30022c250087..f65706cba63d 100644 --- a/Tests/Algorithm/AlgorithmWarmupTests.cs +++ b/Tests/Algorithm/AlgorithmWarmupTests.cs @@ -121,6 +121,7 @@ public void WarmUpInternalSubscriptions() { HistoryProvider = new SubscriptionDataReaderHistoryProvider() }; + algo.Settings.SeedInitialPrices = false; algo.SetStartDate(2013, 10, 08); algo.AddCfd("DE30EUR", Resolution.Second, Market.Oanda); diff --git a/Tests/Common/Data/UniverseSelection/UserDefinedUniverseTests.cs b/Tests/Common/Data/UniverseSelection/UserDefinedUniverseTests.cs index 6bb8385787f2..02b43466e5cf 100644 --- a/Tests/Common/Data/UniverseSelection/UserDefinedUniverseTests.cs +++ b/Tests/Common/Data/UniverseSelection/UserDefinedUniverseTests.cs @@ -53,6 +53,8 @@ public override void Initialize() SetStartDate(2013, 10, 07); SetEndDate(2013, 10, 11); + Settings.SeedInitialPrices = false; + #pragma warning disable CS0618 var spy = AddEquity("SPY", Resolution.Minute, dataNormalizationMode: DataNormalizationMode.Raw).Symbol; diff --git a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs index c62dbc440c4a..1ef131ba1484 100644 --- a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs +++ b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs @@ -33,11 +33,14 @@ using QuantConnect.Securities; using QuantConnect.Securities.Option.StrategyMatcher; using QuantConnect.Securities.Option; -using QuantConnect.Tests.Common.Securities; using QuantConnect.Tests.Engine.DataFeeds; using QuantConnect.Util; using Bitcoin = QuantConnect.Algorithm.CSharp.LiveTradingFeaturesAlgorithm.Bitcoin; using System.Collections; +using QuantConnect.Configuration; +using NodaTime; +using QuantConnect.Data.Market; +using QuantConnect.Data; namespace QuantConnect.Tests.Engine.Setup { @@ -842,6 +845,28 @@ public bool TestLoadExistingHoldingsAndOrders(IBrokerage brokerage, IAlgorithm a return LoadExistingHoldingsAndOrders(brokerage, algorithm, parameters); } } + + private class TestHistoryProvider : HistoryProviderBase + { + public override int DataPointCount { get; } + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + throw new NotImplementedException(); + } + + public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + var requestsList = requests.ToList(); + if (requestsList.Count == 0) + { + return Enumerable.Empty(); + } + + var request = requestsList[0]; + return new List{ new Slice(DateTime.UtcNow, + new List {new QuoteBar(DateTime.MinValue, request.Symbol, new Bar(1, 2, 3, 4), 5, new Bar(1, 2, 3, 4), 5) }, DateTime.UtcNow)}; + } + } } internal class TestBrokerageFactory : BrokerageFactory