From da085207fb7fe1e0cb3d06b68cc42ba658e3cf67 Mon Sep 17 00:00:00 2001 From: JakeTheSnake3p0 Date: Mon, 11 Oct 2021 19:17:15 -0400 Subject: [PATCH 1/4] Ports can now be exposed on docker container in research environment --- README.md | 9 +++++++++ lean/commands/research.py | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 40508192..89c9eb52 100644 --- a/README.md +++ b/README.md @@ -985,6 +985,15 @@ Usage: lean research [OPTIONS] PROJECT By default the official LEAN research image is used. You can override this using the --image option. Alternatively you can set the default research image using `lean config set research-image `. + To expose additional ports to the docker container, update your project's config.json to include the "docker-ports" object. + + For example if you want to expose Tensorboard (defaults to port 6006), + + ... + "docker-ports": { + "6006": "6006" + } + Options: --port INTEGER The port to run Jupyter Lab on (defaults to 8888) --data-provider [Local|QuantConnect|Terminal Link] diff --git a/lean/commands/research.py b/lean/commands/research.py index 3a0a056d..92a182eb 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -157,6 +157,13 @@ def research(project: Path, container.update_manager().pull_docker_image_if_necessary(research_image, update) + # Allow the user to map docker ports + docker_ports = project_config.get('docker-ports') or {} + for outer, inner in docker_ports.items(): + if outer in run_options["ports"]: + raise RuntimeError(f"Port {outer} is already in use, please specify a different host port under 'docker-ports' in your project's config.json") + run_options["ports"][outer] = inner + try: container.docker_manager().run_image(research_image, **run_options) except APIError as error: From 7448da792148b5df20f4644680d3649b3eabeb6c Mon Sep 17 00:00:00 2001 From: JakeTheSnake3p0 Date: Mon, 18 Oct 2021 19:12:18 -0400 Subject: [PATCH 2/4] Environment variables can be set on docker container using project's config file --- README.md | 7 +++++++ lean/commands/research.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 89c9eb52..a5ebf531 100644 --- a/README.md +++ b/README.md @@ -994,6 +994,13 @@ Usage: lean research [OPTIONS] PROJECT "6006": "6006" } + Likewise for environment variables: + + ... + "docker-env": { + "PYTHONHASHSEED": "0" + } + Options: --port INTEGER The port to run Jupyter Lab on (defaults to 8888) --data-provider [Local|QuantConnect|Terminal Link] diff --git a/lean/commands/research.py b/lean/commands/research.py index 92a182eb..f6405794 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -164,6 +164,9 @@ def research(project: Path, raise RuntimeError(f"Port {outer} is already in use, please specify a different host port under 'docker-ports' in your project's config.json") run_options["ports"][outer] = inner + # Allow the user to set environment variables + run_options["environment"].update(project_config.get('docker-env') or {}) + try: container.docker_manager().run_image(research_image, **run_options) except APIError as error: From c8cf1b63215ba56b1235b1ce1372e44e760a668f Mon Sep 17 00:00:00 2001 From: JakeTheSnake3p0 Date: Fri, 17 Dec 2021 20:08:06 -0500 Subject: [PATCH 3/4] Simplified usage of project config.json Removed erroneous documentation placement --- README.md | 16 ---------------- lean/commands/research.py | 10 ---------- lean/components/docker/lean_runner.py | 6 ++++-- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index a5ebf531..40508192 100644 --- a/README.md +++ b/README.md @@ -985,22 +985,6 @@ Usage: lean research [OPTIONS] PROJECT By default the official LEAN research image is used. You can override this using the --image option. Alternatively you can set the default research image using `lean config set research-image `. - To expose additional ports to the docker container, update your project's config.json to include the "docker-ports" object. - - For example if you want to expose Tensorboard (defaults to port 6006), - - ... - "docker-ports": { - "6006": "6006" - } - - Likewise for environment variables: - - ... - "docker-env": { - "PYTHONHASHSEED": "0" - } - Options: --port INTEGER The port to run Jupyter Lab on (defaults to 8888) --data-provider [Local|QuantConnect|Terminal Link] diff --git a/lean/commands/research.py b/lean/commands/research.py index f6405794..3a0a056d 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -157,16 +157,6 @@ def research(project: Path, container.update_manager().pull_docker_image_if_necessary(research_image, update) - # Allow the user to map docker ports - docker_ports = project_config.get('docker-ports') or {} - for outer, inner in docker_ports.items(): - if outer in run_options["ports"]: - raise RuntimeError(f"Port {outer} is already in use, please specify a different host port under 'docker-ports' in your project's config.json") - run_options["ports"][outer] = inner - - # Allow the user to set environment variables - run_options["environment"].update(project_config.get('docker-env') or {}) - try: container.docker_manager().run_image(research_image, **run_options) except APIError as error: diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 404bdf38..623e08e6 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -176,6 +176,8 @@ def get_basic_docker_config(self, :return: the Docker configuration containing basic configuration to run Lean """ project_dir = algorithm_file.parent + project_config = self._project_config_manager.get_project_config(project_dir) + docker_project_config = project_config.get('docker', {}) # Install the required modules when they're needed if lean_config.get("data-provider", None) == "QuantConnect.Lean.Engine.DataFeeds.DownloaderDataProvider" \ @@ -216,11 +218,11 @@ def get_basic_docker_config(self, run_options: Dict[str, Any] = { "detach": detach, "commands": [], - "environment": {}, + "environment": docker_project_config.get("environment", {}), "stop_signal": "SIGINT" if debugging_method is None else "SIGKILL", "mounts": [], "volumes": {}, - "ports": {} + "ports": docker_project_config.get("ports", {}) } # Mount the data directory From dc6c731bc18b195aa262a7dcb9c6b0d6e667fb46 Mon Sep 17 00:00:00 2001 From: Jake Mitchell Date: Fri, 17 Dec 2021 20:09:48 -0500 Subject: [PATCH 4/4] Update lean_runner.py --- lean/components/docker/lean_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 623e08e6..04b4a094 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -177,7 +177,7 @@ def get_basic_docker_config(self, """ project_dir = algorithm_file.parent project_config = self._project_config_manager.get_project_config(project_dir) - docker_project_config = project_config.get('docker', {}) + docker_project_config = project_config.get("docker", {}) # Install the required modules when they're needed if lean_config.get("data-provider", None) == "QuantConnect.Lean.Engine.DataFeeds.DownloaderDataProvider" \