From 125cbe46f8177ecc6a198c99077baf478dd66be6 Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Thu, 2 May 2024 19:35:54 -0300 Subject: [PATCH] Minor fix for QC data download (#454) * Minor fix for QC data download * More fixes and improvements --- README.md | 8 ++--- lean/commands/data/download.py | 46 ++++++++++++++++------------ lean/models/api.py | 1 + tests/commands/data/test_download.py | 20 ++++++------ 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index acb08d97..df71f3e4 100644 --- a/README.md +++ b/README.md @@ -880,16 +880,16 @@ Options: --dataset TEXT The name of the dataset to download non-interactively --overwrite Overwrite existing local data -y, --yes Automatically confirm payment confirmation prompts - --data-type [Trade|Quote|OpenInterest] + --data-type [Trade|Quote|Bulk|OpenInterest] Specify the type of historical data --resolution [Tick|Second|Minute|Hour|Daily] Specify the resolution of the historical data --security-type [Equity|Index|Forex|Cfd|Future|Crypto|CryptoFuture|Option|IndexOption|Commodity|FutureOption] Specify the security type of the historical data --market TEXT Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance') - --tickers TEXT Specify comma separated list of tickers to use for historical data request. - --start-date TEXT Specify the start date for the historical data request in the format yyyyMMdd. - --end-date TEXT Specify the end date for the historical data request in the format yyyyMMdd. (defaults + --ticker TEXT Specify comma separated list of tickers to use for historical data request. + --start TEXT Specify the start date for the historical data request in the format yyyyMMdd. + --end TEXT Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today) --image TEXT The LEAN engine image to use (defaults to quantconnect/lean:latest) --update Pull the LEAN engine image before running the Downloader Data Provider diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 4072d5fe..007291da 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -352,7 +352,8 @@ def _select_products_non_interactive(organization: QCFullOrganization, if option.condition is not None and not option.condition.check(option_results): continue - user_input = ctx.params.get(option.id, None) + # if the option id has a '-' in its name, and it's a click option, in the click context it's available with '_' + user_input = ctx.params.get(option.id.replace('-', '_'), ctx.params.get(option.id, None)) if user_input is None: missing_options.append(f"--{option.id} <{option.get_placeholder()}>: {option.description}") @@ -512,13 +513,13 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - help="Specify the security type of the historical data") @option("--market", type=str, help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')") -@option("--tickers", +@option("--ticker", type=str, help="Specify comma separated list of tickers to use for historical data request.") -@option("--start-date", +@option("--start", type=str, help="Specify the start date for the historical data request in the format yyyyMMdd.") -@option("--end-date", +@option("--end", type=str, help="Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today)") @option("--image", @@ -543,9 +544,9 @@ def download(ctx: Context, resolution: Optional[str], security_type: Optional[str], market: Optional[str], - tickers: Optional[str], - start_date: Optional[str], - end_date: Optional[str], + ticker: Optional[str], + start: Optional[str], + end: Optional[str], image: Optional[str], update: bool, no_update: bool, @@ -575,6 +576,9 @@ def download(ctx: Context, """ organization = _get_organization() + if dataset: + data_provider_historical = 'QuantConnect' + if data_provider_historical is None: data_provider_historical = _get_historical_data_provider() @@ -626,17 +630,21 @@ def download(ctx: Context, market = _get_user_input_or_prompt(market, data_provider_support_markets, data_provider_historical, "Select a Market") - if not tickers: - tickers = ','.join(DatasetTextOption(id="id", - label="Enter comma separated list of tickers to use for historical data request.", - description="description", - transform=DatasetTextOptionTransform.Lowercase, - multiple=True).configure_interactive().value) + if not ticker: + ticker = ','.join(DatasetTextOption(id="id", + label="Enter comma separated list of tickers to use for historical data request.", + description="description", + transform=DatasetTextOptionTransform.Uppercase, + multiple=True).configure_interactive().value) + else: + split_tickers = ticker.split(',') + # don't trust user provider tickers without spaces in between + ticker = ','.join([a_ticker.strip().upper() for a_ticker in split_tickers]) - start_date = _configure_date_option(start_date, "start", "Please enter a Start Date in the format") - end_date = _configure_date_option(end_date, "end", "Please enter a End Date in the format") + start = _configure_date_option(start, "start", "Please enter a Start Date in the format") + end = _configure_date_option(end, "end", "Please enter a End Date in the format") - if start_date.value >= end_date.value: + if start.value >= end.value: raise ValueError("Historical start date cannot be greater than or equal to historical end date.") logger = container.logger @@ -674,12 +682,12 @@ def download(ctx: Context, dll_arguments = ["dotnet", "QuantConnect.DownloaderDataProvider.Launcher.dll", "--data-type", data_type, - "--start-date", start_date.value.strftime("%Y%m%d"), - "--end-date", end_date.value.strftime("%Y%m%d"), + "--start-date", start.value.strftime("%Y%m%d"), + "--end-date", end.value.strftime("%Y%m%d"), "--security-type", security_type, "--market", market, "--resolution", resolution, - "--tickers", tickers] + "--tickers", ticker] run_options["commands"].append(' '.join(dll_arguments)) diff --git a/lean/models/api.py b/lean/models/api.py index 4d552a6e..6ec432df 100644 --- a/lean/models/api.py +++ b/lean/models/api.py @@ -410,6 +410,7 @@ class QCMinimalOrganization(WrappedBaseModel): class QCDataType(str, Enum): Trade = "Trade" Quote = "Quote" + Bulk = "Bulk" OpenInterest = "OpenInterest" @classmethod diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index a5c1e6b5..feca0bb7 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -123,9 +123,9 @@ def _create_lean_data_download(data_provider_name: str, "--data-type", data_type, "--resolution", resolution, "--security-type", security_type, - "--tickers", ','.join(tickers), - "--start-date", start_date, - "--end-date", end_date, + "--ticker", ','.join(tickers), + "--start", start_date, + "--end", end_date, ] if market: run_parameters.extend(["--market", market]) @@ -135,7 +135,7 @@ def _create_lean_data_download(data_provider_name: str, return CliRunner().invoke(lean, run_parameters) -@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,tickers,data_provider_parameters", +@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,ticker,data_provider_parameters", [("Polygon", "NYSE", False, "Equity", ["AAPL"], ["--polygon-api-key", "123"]), ("Binance", "Binance", True, "CryptoFuture", ["BTCUSDT"], ["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123", @@ -145,9 +145,9 @@ def _create_lean_data_download(data_provider_name: str, ("Interactive Brokers", "USA", False, "Index", ["INTL", "NVDA"], ["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])]) def test_download_data_non_interactive(data_provider: str, market: str, is_crypto: bool, security_type: str, - tickers: List[str], data_provider_parameters: List[str]): + ticker: List[str], data_provider_parameters: List[str]): run_data_download = _create_lean_data_download( - data_provider, "Trade", "Minute", security_type, tickers, "20240101", "20240202", + data_provider, "Trade", "Minute", security_type, ticker, "20240101", "20240202", _get_data_provider_config(is_crypto), market, data_provider_parameters) assert run_data_download.exit_code == 0 @@ -174,11 +174,11 @@ def test_download_data_non_interactive_wrong_security_type(data_provider: str, w assert wrong_security_type in error_msg -@pytest.mark.parametrize("data_provider,start_date,end_date", +@pytest.mark.parametrize("data_provider,start,end", [("Polygon", "20240101", "20230202"), ("Polygon", "2024-01-01", "2023-02-02")]) -def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start_date: str, end_date: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date, - end_date, _get_data_provider_config(), "USA", +def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start: str, end: str): + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start, + end, _get_data_provider_config(), "USA", extra_run_command=["--polygon-api-key", "123"]) assert run_data_download.exit_code == 1