Skip to content

Commit

Permalink
Remove hardcoded python and dotnet version (#441)
Browse files Browse the repository at this point in the history
* Remove hardcoded python and dotnet version

- Remove hardcoded python and dotnet version, these will be retrieved
  from the docker labels of the image being used

* Move default python dotnet values to constant
  • Loading branch information
Martin-Molinero authored Apr 2, 2024
1 parent 8703d99 commit 65a91d6
Show file tree
Hide file tree
Showing 14 changed files with 117 additions and 114 deletions.
13 changes: 9 additions & 4 deletions lean/commands/library/add.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@

from pathlib import Path
from typing import Any, Dict, Optional, Tuple
from lean.constants import LEAN_STRICT_PYTHON_VERSION
from click import command, argument, option

from lean.click import LeanCommand, PathParameter
from lean.constants import DEFAULT_LEAN_STRICT_PYTHON_VERSION
from lean.container import container
from lean.models.errors import MoreInfoError

Expand Down Expand Up @@ -124,7 +124,7 @@ def _is_pypi_file_compatible(file: Dict[str, Any], required_python_version) -> b
return True


def _get_pypi_package(name: str, version: Optional[str]) -> Tuple[str, str]:
def _get_pypi_package(name: str, version: Optional[str], python_version: str) -> Tuple[str, str]:
"""Retrieves the properly-capitalized name and the latest compatible version of a package from PyPI.
If the version is already given, this method checks whether that version is compatible with the Docker images.
Expand All @@ -148,7 +148,7 @@ def _get_pypi_package(name: str, version: Optional[str]) -> Tuple[str, str]:
pypi_data = loads(response.text)
name = pypi_data["info"]["name"]

required_python_version = StrictVersion(LEAN_STRICT_PYTHON_VERSION)
required_python_version = StrictVersion(python_version)

last_compatible_version = None
last_compatible_version_upload_time = None
Expand Down Expand Up @@ -237,7 +237,12 @@ def _add_pypi_package_to_python_project(project_dir: Path, name: str, version: O
else:
logger.info("Retrieving latest compatible version from PyPI")

name, version = _get_pypi_package(name, version)
project_config = container.project_config_manager.get_project_config(project_dir)
engine_image = container.cli_config_manager.get_engine_image(project_config.get("engine-image", None))
python_version = container.docker_manager.get_image_label(engine_image, 'strict_python_version',
DEFAULT_LEAN_STRICT_PYTHON_VERSION)

name, version = _get_pypi_package(name, version, python_version)

requirements_file = project_dir / "requirements.txt"
logger.info(f"Adding {name} {version} to '{path_manager.get_relative_path(requirements_file)}'")
Expand Down
6 changes: 4 additions & 2 deletions lean/commands/optimize.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,10 @@ def optimize(project: Path,
build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config,
kwargs, logger, environment_name)

run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach)
container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)

run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach,
engine_image)

run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(f"dotnet QuantConnect.Optimizer.Launcher.dll{' --estimate' if estimate else ''}")
Expand All @@ -346,7 +349,6 @@ def optimize(project: Path,
type="bind",
read_only=True)
)
container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)

# Add known additional run options from the extra docker config
LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))
Expand Down
25 changes: 13 additions & 12 deletions lean/commands/research.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,24 @@ def research(project: Path,
for key, value in extra_config:
lean_config[key] = value

project_config_manager = container.project_config_manager
cli_config_manager = container.cli_config_manager

project_config = project_config_manager.get_project_config(algorithm_file.parent)
research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))

container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)

if str(research_image) != DEFAULT_RESEARCH_IMAGE:
logger.warn(f'A custom research image: "{research_image}" is being used!')

run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file,
temp_manager.create_temporary_directory(),
None,
False,
detach)
detach,
research_image)

# Mount the config in the notebooks directory as well
local_config_path = next(m["Source"] for m in run_options["mounts"] if m["Target"].endswith("config.json"))
Expand Down Expand Up @@ -174,17 +186,6 @@ def research(project: Path,
# Add known additional run options from the extra docker config
LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))

project_config_manager = container.project_config_manager
cli_config_manager = container.cli_config_manager

project_config = project_config_manager.get_project_config(algorithm_file.parent)
research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))

if str(research_image) != DEFAULT_RESEARCH_IMAGE:
logger.warn(f'A custom research image: "{research_image}" is being used!')

container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)

try:
container.docker_manager.run_image(research_image, **run_options)
except APIError as error:
Expand Down
14 changes: 12 additions & 2 deletions lean/components/docker/docker_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,16 @@ def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager:
self._temp_manager = temp_manager
self._platform_manager = platform_manager

def get_image_label(self, image: DockerImage, label: str, default: str) -> str:
docker_image = self._get_docker_client().images.get(image.name)

for name, value in docker_image.labels.items():
if name == label:
self._logger.debug(f"Label '{label}' found in image '{image.name}', value {value}")
return value
self._logger.info(f"Label '{label}' not found in image '{image.name}', using default {default}")
return default

def pull_image(self, image: DockerImage) -> None:
"""Pulls a Docker image.
Expand Down Expand Up @@ -563,13 +573,13 @@ def _format_source_path(self, path: str) -> str:
break

return path

def get_container_port(self, container_name: str, internal_port: str) -> Optional[int]:
"""
Returns a containers external port for a mapped internal port
:param container_name: Name of the container
:param internal_port: The internal port of container. If protocol not included
we assume /tcp. ex. 5678/tcp
we assume /tcp. ex. 5678/tcp
:return: The external port that is linked to it, or None if it does not exist
"""

Expand Down
43 changes: 30 additions & 13 deletions lean/components/docker/lean_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
from lean.components.util.project_manager import ProjectManager
from lean.components.util.temp_manager import TempManager
from lean.components.util.xml_manager import XMLManager
from lean.constants import MODULES_DIRECTORY, TERMINAL_LINK_PRODUCT_ID, LEAN_ROOT_PATH, DEFAULT_DATA_DIRECTORY_NAME
from lean.constants import MODULES_DIRECTORY, LEAN_ROOT_PATH, DEFAULT_DATA_DIRECTORY_NAME, \
DEFAULT_LEAN_DOTNET_FRAMEWORK, DEFAULT_LEAN_PYTHON_VERSION
from lean.constants import DOCKER_PYTHON_SITE_PACKAGES_PATH
from lean.models.docker import DockerImage
from lean.models.utils import DebuggingMethod
Expand Down Expand Up @@ -97,7 +98,8 @@ def run_lean(self,
output_dir,
debugging_method,
release,
detach)
detach,
image)

# Add known additional run options from the extra docker config
self.parse_extra_docker_config(run_options, extra_docker_config)
Expand Down Expand Up @@ -175,7 +177,8 @@ def get_basic_docker_config(self,
output_dir: Path,
debugging_method: Optional[DebuggingMethod],
release: bool,
detach: bool) -> Dict[str, Any]:
detach: bool,
image: DockerImage) -> Dict[str, Any]:
"""Creates a basic Docker config to run the engine with.
This method constructs the parts of the Docker config that is the same for both the engine and the optimizer.
Expand All @@ -186,6 +189,7 @@ def get_basic_docker_config(self,
:param debugging_method: the debugging method if debugging needs to be enabled, None if not
:param release: whether C# projects should be compiled in release configuration instead of debug
:param detach: whether LEAN should run in a detached container
:param image: The docker image that will be used
:return: the Docker configuration containing basic configuration to run Lean
"""
from docker.types import Mount
Expand Down Expand Up @@ -309,7 +313,10 @@ def get_basic_docker_config(self,
# Create a C# project used to resolve the dependencies of the modules
run_options["commands"].append("mkdir /ModulesProject")
run_options["commands"].append("dotnet new sln -o /ModulesProject")
run_options["commands"].append("dotnet new classlib -o /ModulesProject -f net6.0 --no-restore")

framework_ver = self._docker_manager.get_image_label(image, 'target_framework',
DEFAULT_LEAN_DOTNET_FRAMEWORK)
run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore")
run_options["commands"].append("rm /ModulesProject/Class1.cs")

# Add all modules to the project, automatically resolving all dependencies
Expand All @@ -325,7 +332,8 @@ def get_basic_docker_config(self,

# Set up language-specific run options
self.setup_language_specific_run_options(run_options, project_dir, algorithm_file,
set_up_common_csharp_options_called, release)
set_up_common_csharp_options_called, release,
image)

# Save the final Lean config to a temporary file so we can mount it into the container
config_path = self._temp_manager.create_temporary_directory() / "config.json"
Expand Down Expand Up @@ -359,11 +367,12 @@ def get_basic_docker_config(self,

return run_options

def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any]) -> None:
def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None:
"""Sets up Docker run options specific to Python projects.
:param project_dir: the path to the project directory
:param run_options: the dictionary to append run options to
:param image: the docker image that will be used
"""

from docker.types import Mount
Expand Down Expand Up @@ -410,10 +419,14 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
"mode": "rw"
}

python_version = self._docker_manager.get_image_label(image, 'python_version',
DEFAULT_LEAN_PYTHON_VERSION)
site_packages_path = DOCKER_PYTHON_SITE_PACKAGES_PATH.replace('{LEAN_PYTHON_VERSION}', python_version)

# Mount a volume to the user packages directory so we don't install packages every time
site_packages_volume = self._docker_manager.create_site_packages_volume(requirements_txt)
run_options["volumes"][site_packages_volume] = {
"bind": f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}",
"bind": f"{site_packages_path}",
"mode": "rw"
}

Expand All @@ -424,7 +437,7 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
# We only need to do this if it hasn't already been done before for this site packages volume
# To keep track of this we create a special file in the site packages directory after installation
# If this file already exists we can skip pip install completely
marker_file = f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}/pip-install-done"
marker_file = f"{site_packages_path}/pip-install-done"
run_options["commands"].extend([
f"! test -f {marker_file} && pip install --user --progress-bar off -r /requirements.txt",
f"touch {marker_file}"
Expand Down Expand Up @@ -452,12 +465,14 @@ def _concat_python_requirements(self, requirements_files: List[Path]) -> str:
requirements = sorted(set(requirements))
return "\n".join(requirements)

def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool) -> None:
def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool,
image: DockerImage) -> None:
"""Sets up Docker run options specific to C# projects.
:param project_dir: the path to the project directory
:param run_options: the dictionary to append run options to
:param release: whether C# projects should be compiled in release configuration instead of debug
:param image: the docker image that will be used
"""
compile_root = self._get_csharp_compile_root(project_dir)

Expand All @@ -474,11 +489,13 @@ def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any],
for path in compile_root.rglob("*.csproj"):
self._ensure_csproj_uses_correct_lean(compile_root, path, csproj_temp_dir, run_options)

framework_ver = self._docker_manager.get_image_label(image, 'target_framework',
DEFAULT_LEAN_DOTNET_FRAMEWORK)
# Set up the MSBuild properties
msbuild_properties = {
"Configuration": "Release" if release else "Debug",
"Platform": "AnyCPU",
"TargetFramework": "net6.0",
"TargetFramework": framework_ver,
"OutputPath": "/Compile/bin",
"GenerateAssemblyInfo": "false",
"GenerateTargetFrameworkAttribute": "false",
Expand Down Expand Up @@ -722,14 +739,14 @@ def _force_disk_provider_if_necessary(self,
lean_config[config_key] = disk_provider

def setup_language_specific_run_options(self, run_options, project_dir, algorithm_file,
set_up_common_csharp_options_called, release) -> None:
set_up_common_csharp_options_called, release, image: DockerImage) -> None:
# Set up language-specific run options
if algorithm_file.name.endswith(".py"):
self.set_up_python_options(project_dir, run_options)
self.set_up_python_options(project_dir, run_options, image)
else:
if not set_up_common_csharp_options_called:
self.set_up_common_csharp_options(run_options)
self.set_up_csharp_options(project_dir, run_options, release)
self.set_up_csharp_options(project_dir, run_options, release, image)

def format_error_before_logging(self, chunk: str):
from lean.components.util import compiler
Expand Down
Loading

0 comments on commit 65a91d6

Please sign in to comment.