feat(dask): add more env vars(#821)

reanahub · Sep 15, 2024 · 4147327 · 4147327
1 parent f59fca2
commit 4147327
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 1 deletion.
diff --git a/helm/reana/README.md b/helm/reana/README.md
@@ -60,7 +60,13 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `components.reana_workflow_engine_snakemake.environment` | [REANA-Workflow-Engine-Snakemake](https://github.com/reanahub/reana-workflow-engine-snakemake) environment variables | `{}`            |
 | `components.reana_workflow_engine_snakemake.image`       | [REANA-Workflow-Engine-Snakemake image](https://hub.docker.com/r/reanahub/reana-workflow-engine-snakemake) to use | `docker.io/reanahub/reana-workflow-engine-snakemake:<chart-release-version>` |
 | `compute_backends`                                       | List of supported compute backends (kubernetes, htcondorcern, slurmcern)             | "kubernetes"                                    |
-| `dask.enabled` | Install dask-kubernetes-operator custom resources in the cluster for dask workflows | true
+| `dask.enabled` | Install dask-kubernetes-operator custom resources in the cluster to enable dask workflows | true |
+| `dask.cluster_default_cores_limit` | Default cores limit for dask clusters | 4 |
+| `dask.cluster_max_cores_limit` | Max cores limit for dask clusters | 16 | 
+| `dask.cluster_default_memory_limit` | Default memory limit for dask clusters | "2Gi" |
+| `dask.cluster_max_memory_limit` | Max memory limit for dask clusters | "4Gi" | 
+| `dask.cluster_default_single_worker_cores` | Amount of cores used by a single dask worker by default | 0.5 |
+| `dask.cluster_default_single_worker_memory` | Amount of memory used by a single dask worker by default | "512Mi" | 
 | `db_env_config.REANA_DB_HOST`                            | Environment variable to connect to external databases                                | `<chart-release-name>-db`                       |
 | `db_env_config.REANA_DB_NAME`                            | Environment variable to connect to external databases                                | reana                                           |
 | `db_env_config.REANA_DB_PORT`                            | Environment variable to connect to external databases                                | "5432"                                          |

diff --git a/helm/reana/templates/reana-server.yaml b/helm/reana/templates/reana-server.yaml
@@ -98,6 +98,22 @@ spec:
             value: !!str {{ .Values.kubernetes_jobs_timeout_limit | default 604800 }}  # 604800 seconds = 7 days
           - name: REANA_KUBERNETES_JOBS_MEMORY_LIMIT
             value: {{ .Values.kubernetes_jobs_memory_limit | default "4Gi" }}
+          - name: DASK_ENABLED
+            value: {{ .Values.dask.enabled | quote }}
+          {{- if .Values.dask.enabled }}
+          - name: REANA_DASK_CLUSTER_MAX_CORES_LIMIT
+            value: !!str {{ .Values.dask.cluster_max_cores_limit | default 8 }}
+          - name: REANA_DASK_CLUSTER_MAX_MEMORY_LIMIT
+            value: {{ .Values.dask.cluster_max_memory_limit | default "4Gi" }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_CORES_LIMIT
+            value: !!str {{ .Values.dask.cluster_default_cores_limit | default 4 }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_MEMORY_LIMIT
+            value: {{ .Values.dask.cluster_default_memory_limit | default "2Gi" }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_CORES
+            value: !!str {{ .Values.dask.cluster_default_single_worker_cores | default 0.5 }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_MEMORY
+            value: {{ .Values.dask.cluster_default_single_worker_memory | default "512Mi" }}
+          {{- end }}          
           - name: WORKSPACE_PATHS
             value: {{ .Values.workspaces.paths | toJson | quote }}
           - name: WORKSPACE_RETENTION_PERIOD

diff --git a/helm/reana/templates/reana-workflow-controller.yaml b/helm/reana/templates/reana-workflow-controller.yaml
@@ -173,6 +173,16 @@ spec:
             value: {{ .Values.interactive_sessions.environments | toJson | quote }}
           - name: DASK_ENABLED
             value: {{ .Values.dask.enabled | quote }}
+          {{- if .Values.dask.enabled }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_CORES_LIMIT
+            value: !!str {{ .Values.dask.cluster_default_cores_limit | default 4 }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_MEMORY_LIMIT
+            value: {{ .Values.dask.cluster_default_memory_limit | default "2Gi" }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_CORES
+            value: !!str {{ .Values.dask.cluster_default_single_worker_cores | default 0.5 }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_MEMORY
+            value: {{ .Values.dask.cluster_default_single_worker_memory | default "512Mi" }}
+          {{- end }}        
           {{- if .Values.reana_hostname }}
           - name: REANA_HOSTNAME
             value: {{ .Values.reana_hostname }}

diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml
@@ -175,6 +175,12 @@ traefik:
 
 dask:
   enabled: true
+  cluster_default_cores_limit: 4
+  cluster_max_cores_limit: 8
+  cluster_default_memory_limit: "2Gi"
+  cluster_max_memory_limit: "4Gi"
+  cluster_default_single_worker_cores: 0.5
+  cluster_default_single_worker_memory: "256Mi"
 
 pgbouncer:
   enabled: false