diff --git a/changelog/1.feature.md b/changelog/2.feature.md similarity index 100% rename from changelog/1.feature.md rename to changelog/2.feature.md diff --git a/changelog/4.feature.md b/changelog/4.feature.md new file mode 100644 index 0000000..504f5af --- /dev/null +++ b/changelog/4.feature.md @@ -0,0 +1,2 @@ +Adds the placeholder concept of `Executor`'s which are responsible for running metrics +in different environments. diff --git a/docs/cli/.gitkeep b/docs/cli/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..938e0bb --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,14 @@ +# Configuration + +## Environment Variables + +Environment variables are used to control some aspects of the model. +The default values for these environment variables are generally suitable, +but if you require updating these values we recommend the use of a `.env` file +to make the changes easier to reproduce in future. + +### `CMIP_REF_EXECUTOR` + +Executor to use for running the metrics. + +Defaults to use the local executor ("local"). diff --git a/docs/explanation.md b/docs/explanation.md index 9da03cb..ba4e8f3 100644 --- a/docs/explanation.md +++ b/docs/explanation.md @@ -11,3 +11,21 @@ Points we will aim to cover: We will aim to avoid writing instructions or technical descriptions here, they belong elsewhere. + + +## Execution Environments + +The REF aims to support the execution of metrics in a variety of environments. +This includes local execution, testing, cloud-based execution, and execution on HPC systems. + +The currently supported execution environments are: + +* Local + +The following environments are planned to be supported in the future: + +* Kubernetes (for cloud-based execution) +* Subprocess (for HPC systems) + +The selected executor is defined using the `CMIP_REF_EXECUTOR` environment variable. +See the [Configuration](configuration.md) page for more information. diff --git a/docs/gen_doc_stubs.py b/docs/gen_doc_stubs.py index bfd991a..0d7cefb 100644 --- a/docs/gen_doc_stubs.py +++ b/docs/gen_doc_stubs.py @@ -118,11 +118,14 @@ def write_module_page( fh.write("\n") fh.write(f"::: {package_full_name}") - package_doc_split = package.__doc__.splitlines() - if not package_doc_split[0]: - summary = package_doc_split[1] + if package.__doc__ is None: + summary = "" else: - summary = package_doc_split[0] + package_doc_split = package.__doc__.splitlines() + if not package_doc_split[0]: + summary = package_doc_split[1] + else: + summary = package_doc_split[0] return PackageInfo(package_full_name, package_name, summary) diff --git a/mkdocs.yml b/mkdocs.yml index 3db6ae5..b0461d2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,7 @@ repo_url: https://github.com/CMIP-REF/cmip-ref nav: - CMIP REF: index.md - Installation: installation.md + - Configuration: configuration.md - How-to guides: - how-to-guides/index.md - Tutorials: tutorials.md diff --git a/packages/ref-core/src/ref_core/executor/__init__.py b/packages/ref-core/src/ref_core/executor/__init__.py new file mode 100644 index 0000000..1fd0235 --- /dev/null +++ b/packages/ref-core/src/ref_core/executor/__init__.py @@ -0,0 +1,132 @@ +""" +Execute metrics in different environments + +We support running metrics in different environments, such as locally, +in a separate process, or in a container. +These environments are represented by `Executor` classes. +The `CMIP_REF_EXECUTOR` environment variable determines which executor is used. + +The simplest executor is the `LocalExecutor`, which runs the metric in the same process. +This is useful for local testing and debugging. + +This is a placeholder implementation and will be expanded in the future. +""" + +import os +from typing import Protocol, runtime_checkable + +from .local import LocalExecutor + + +@runtime_checkable +class Executor(Protocol): + """ + An executor is responsible for running a metric. + + The metric may be run locally in the same process or in a separate process or container. + + Notes + ----- + This is an extremely basic interface and will be expanded in the future, as we figure out + our requirements. + """ + + name: str + + def run_metric(self, metric: object, *args, **kwargs) -> object: # type: ignore + """ + Execute a metric + """ + # TODO: Add type hints for metric and return value in follow-up PR + ... + + +class ExecutorManager: + """ + Enables the registration of executors and retrieval by name. + + This is exposed as a singleton instance `ref_core.executor.get_executor` + and `ref_core.executor.register_executor`, + but for testability, you can create your own instance. + """ + + def __init__(self) -> None: + self._executors: dict[str, Executor] = {} + + def register(self, executor: Executor) -> None: + """ + Register an executor with the manager + + Parameters + ---------- + executor + The executor to register + """ + if not isinstance(executor, Executor): # pragma: no cover + raise ValueError("Executor must be an instance of Executor") + self._executors[executor.name.lower()] = executor + + def get(self, name: str) -> Executor: + """ + Get an executor by name + + Parameters + ---------- + name + Name of the executor (case-sensitive) + + Raises + ------ + KeyError + If the executor with the given name is not found + + Returns + ------- + : + The requested executor + """ + return self._executors[name.lower()] + + +_default_manager = ExecutorManager() + +register_executor = _default_manager.register +get_executor = _default_manager.get + + +def run_metric(metric_name: str, *args, **kwargs) -> object: # type: ignore + """ + Run a metric using the default executor + + The executor is determined by the `CMIP_REF_EXECUTOR` environment variable. + The arguments will be updated in the future as the metric execution interface is expanded. + + TODO: migrate to a configuration object rather than relying on environment variables. + + Parameters + ---------- + metric_name + Name of the metric to run. + + Eventually the metric will be sourced from via some kind of registry. + For now, it's just a placeholder. + args + Extra arguments passed to the metric of interest + kwargs + Extra keyword arguments passed to the metric of interest + + Returns + ------- + : + The result of the metric execution + """ + executor_name = os.environ.get("CMIP_REF_EXECUTOR", "local") + + executor = get_executor(executor_name) + # metric = get_metric(metric_name) # TODO: Implement this + metric = kwargs.pop("metric") + + return executor.run_metric(metric, *args, **kwargs) + + +register_executor(LocalExecutor()) diff --git a/packages/ref-core/src/ref_core/executor/local.py b/packages/ref-core/src/ref_core/executor/local.py new file mode 100644 index 0000000..af48231 --- /dev/null +++ b/packages/ref-core/src/ref_core/executor/local.py @@ -0,0 +1,27 @@ +class LocalExecutor: + """ + Run a metric locally, in-process. + + This is mainly useful for debugging and testing. + The production executor will run the metric in a separate process or container, + the exact manner of which is yet to be determined. + """ + + name = "local" + + def run_metric(self, metric, *args, **kwargs): # type: ignore + """ + Run a metric in process + + Parameters + ---------- + metric + args + kwargs + + Returns + ------- + : + Results from running the metric + """ + return metric.run(*args, **kwargs) diff --git a/packages/ref-core/tests/unit/test_executor.py b/packages/ref-core/tests/unit/test_executor.py new file mode 100644 index 0000000..ceaa3d9 --- /dev/null +++ b/packages/ref-core/tests/unit/test_executor.py @@ -0,0 +1,59 @@ +import pytest +from ref_core.executor import Executor, ExecutorManager, run_metric +from ref_core.executor.local import LocalExecutor + + +class MockMetric: + def run(self, *args, **kwargs): + result = { + "args": args, + "kwargs": kwargs, + } + + return result + + +class TestExecutorManager: + def test_executor_register(self): + manager = ExecutorManager() + manager.register(LocalExecutor()) + + assert len(manager._executors) == 1 + assert "local" in manager._executors + assert isinstance(manager.get("local"), LocalExecutor) + + +class TestLocalExecutor: + def test_is_executor(self): + executor = LocalExecutor() + + assert executor.name == "local" + assert isinstance(executor, Executor) + + def test_run_metric(self): + executor = LocalExecutor() + + metric = MockMetric() + result = executor.run_metric(metric, "test", kwarg="test") + + assert result == { + "args": ("test",), + "kwargs": {"kwarg": "test"}, + } + + +@pytest.mark.parametrize("executor_name", ["local", None]) +def test_run_metric_local(monkeypatch, executor_name): + if executor_name: + monkeypatch.setenv("CMIP_REF_EXECUTOR", executor_name) + result = run_metric("example_metric", "test", kwarg="test", metric=MockMetric()) + assert result == { + "args": ("test",), + "kwargs": {"kwarg": "test"}, + } + + +def test_run_metric_unknown(monkeypatch): + monkeypatch.setenv("CMIP_REF_EXECUTOR", "missing") + with pytest.raises(KeyError): + run_metric("anything", "test", kwarg="test")