From 18d3b4bcf522dabdc58ca0be3935f9df162b372c Mon Sep 17 00:00:00 2001 From: Marco Donadoni Date: Wed, 7 Feb 2024 11:25:59 +0100 Subject: [PATCH] refactor(docs): move from reST to Markdown (#776) Convert docs from reStructuredText to Markdown so that the changelog file is compatible with Release Please. --- AUTHORS.md | 30 ++ AUTHORS.rst | 31 -- CHANGELOG.md | 579 ++++++++++++++++++++++++++++ CHANGES.rst | 548 -------------------------- CONTRIBUTING.rst => CONTRIBUTING.md | 16 +- MANIFEST.in | 3 +- README.md | 54 +++ README.rst | 71 ---- docs/conf.py | 8 +- docs/index.md | 3 + docs/index.rst | 4 - setup.py | 6 +- 12 files changed, 683 insertions(+), 670 deletions(-) create mode 100644 AUTHORS.md delete mode 100644 AUTHORS.rst delete mode 100644 CHANGES.rst rename CONTRIBUTING.rst => CONTRIBUTING.md (71%) create mode 100644 README.md delete mode 100644 README.rst create mode 100644 docs/index.md delete mode 100644 docs/index.rst diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 00000000..2812931b --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,30 @@ +# Authors + +The list of contributors in alphabetical order: + +- [Adelina Lintuluoto](https://orcid.org/0000-0002-0726-1452) +- [Anton Khodak](https://orcid.org/0000-0003-3263-4553) +- [Audrius Mecionis](https://orcid.org/0000-0002-3759-1663) +- [Camila Diaz](https://orcid.org/0000-0001-5543-797X) +- [Daan Rosendal](https://github.com/DaanRosendal) +- [Dan Leehr](https://orcid.org/0000-0003-3221-9579) +- [Daniel Prelipcean](https://orcid.org/0000-0002-4855-194X) +- [Diego Rodriguez](https://orcid.org/0000-0003-0649-2002) +- [Dinos Kousidis](https://orcid.org/0000-0002-4914-4289) +- [Domenic Gosein](https://orcid.org/0000-0002-1546-0435) +- [Harri Hirvonsalo](https://orcid.org/0000-0002-5503-510X) +- [Jan Okraska](https://orcid.org/0000-0002-1416-3244) +- [Kenyi Hurtado-Anampa](https://orcid.org/0000-0002-9779-3566) +- [Leticia Wanderley](https://orcid.org/0000-0003-4649-6630) +- [Lukas Heinrich](https://orcid.org/0000-0002-4048-7584) +- [Marco Donadoni](https://orcid.org/0000-0003-2922-5505) +- [Marco Vidal](https://orcid.org/0000-0002-9363-4971) +- [Matthew Feickert](https://orcid.org/0000-0003-4124-7862) +- [Michael R. Crusoe](https://orcid.org/0000-0002-2961-9670) +- [Robin Long](https://github.com/longr) +- [Rokas Maciulaitis](https://orcid.org/0000-0003-1064-6967) +- [Ronald Dobos](https://orcid.org/0000-0003-2914-000X) +- [Sasha Baranov](https://github.com/sashabaranov) +- [Sinclert Perez](https://www.linkedin.com/in/sinclert) +- [Tibor Simko](https://orcid.org/0000-0001-7202-5803) +- [Vladyslav Moisieienkov](https://orcid.org/0000-0001-9717-0775) diff --git a/AUTHORS.rst b/AUTHORS.rst deleted file mode 100644 index 4cbd8772..00000000 --- a/AUTHORS.rst +++ /dev/null @@ -1,31 +0,0 @@ -Authors -======= - -The list of contributors in alphabetical order: - -- `Adelina Lintuluoto `_ -- `Anton Khodak `_ -- `Audrius Mecionis `_ -- `Camila Diaz `_ -- `Daan Rosendal `_ -- `Dan Leehr `_ -- `Daniel Prelipcean `_ -- `Diego Rodriguez `_ -- `Dinos Kousidis `_ -- `Domenic Gosein `_ -- `Harri Hirvonsalo `_ -- `Jan Okraska `_ -- `Kenyi Hurtado-Anampa `_ -- `Leticia Wanderley `_ -- `Lukas Heinrich `_ -- `Marco Donadoni `_ -- `Marco Vidal `_ -- `Matthew Feickert `_ -- `Michael R. Crusoe `_ -- `Robin Long `_ -- `Rokas Maciulaitis `_ -- `Ronald Dobos `_ -- `Sasha Baranov `_ -- `Sinclert Perez `_ -- `Tibor Simko `_ -- `Vladyslav Moisieienkov `_ diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29b..61c1da67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,579 @@ +# Changelog + +## 0.9.2 (2023-12-19) + +- Users: + + - Adds web interface form allowing to generate launcher URL for any user-provided public analysis, as well as the Markdown snippet for the corresponding Launch-on-REANA badge. + - Adds web interface option to delete all the runs of a workflow. + - Changes the Launch-on-REANA web interface page to improve how workflow parameters are shown by displaying them inside a table. + - Changes CVMFS support to allow users to automatically mount any available repository. + - Changes the REANA specification schema to use the `draft-07` version of the JSON Schema specification. + - Changes `reana-client validate` command to show detailed errors when the specification file is not a valid YAML file. + - Changes validation of REANA specification to make the `environment` property mandatory for the steps of serial workflows. + - Changes validation of REANA specification to raise a warning for unexpected properties for the steps of serial workflows. + - Changes validation of REANA specification to report improved validation warnings which also indicate where unexpected properties are located in the file. + - Changes workflow restarts to allow for more than nine restarts of the same workflows. + - Changes workflow scheduler logging behaviour to also report the main reason behind scheduling errors to the users. + - Fixes `reana-client list` command to accept case-insensitive column names when sorting the returned workflow runs via the `--sort` option. + - Fixes `reana-client run` wrapper command for workflows that do not contain `inputs` clause in their specification. + - Fixes `reana-client`'s `create_workflow_from_json` API function to always load and send the workflow specification to the server. + - Fixes creation of image thumbnails for output files in Snakemake HTML execution reports. + +- Administrators: + + - Changes several database index definitions in order to improve performance of most common database queries. + - Changes the names of database table, column, index and key constraints in order to follow the SQLAlchemy upstream naming conventions everywhere. + - Changes the `Workflow` table to replace the `run_number` column with two new columns `run_number_major` and `run_number_minor` in order to allow for more than nine restarts of user workflows. + - Changes CVMFS support to allow users to automatically mount any available repository, thanks to CVMFS CSI v2. + - Fixes the mounting of CVMFS volumes for the REANA deployments that use non-default Kubernetes namespace. + - Fixes container image building of cluster components for the arm64 architecture. + - Fixes job monitoring in cases when job creation fails, for example when it is not possible to successfully mount volumes. + - Fixes job status consumer exception while attempting to fetch workflow engine logs for workflows that could not have been successfully scheduled. + - Fixes the creation of Kubernetes jobs by retrying in case of error and by correctly handling the error after reaching the retry limit. + +- Developers: + + - Adds automated multi-platform container image building of cluster components for amd64 and arm64 architectures. + - Adds new `--image-name` option to the `reana-dev docker-push` command to customise the name of the docker image to publish. + - Adds new `--platform` option to the `reana-dev docker-build` and `reana-dev release-docker` commands to build and publish multi-platform images. + - Adds new `--registry` option to the `reana-dev docker-push` and `reana-dev release-docker` commands to specify the registry where the built docker images should be pushed to. + - Adds new `--tags-only` option to the `reana-dev release-docker` command to only print the final docker image names, without pushing the images to the registry. + - Adds new `reana-dev git-create-release-branch` command to create a new Git branch for a new release. + - Adds new `reana-dev git-upgrade-requirements` command to upgrade the `requirements.txt` file before a new release. + - Changes `reana-dev git-fork` and `reana-dev git-create-pr` to use the `gh` CLI client instead of `hub`. + - Changes `reana-dev python-run-tests` command to allow excluding certain Python components. + - Changes `reana-dev python-run-tests` command to allow execution of selected pytests only by passing over `PYTEST_ADDOPTS` environment variable. + - Changes validation of REANA specification to expose functions for loading workflow input parameters and workflow specifications. + - Changes version of `reana-ui` Node.js Docker image from 16 to 18. + - Changes the workflow deletion endpoint to return a different and more appropriate message when deleting all the runs of a workflow. + - Changes the workflow list endpoint on how pagination is performed in order to avoid counting twice the total number of records. + - Fixes `reana-dev python-run-tests` command to create Python-3.8 based virtual environments to use the same version as container images. + +## 0.9.1 (2023-09-27) + +- Users: + + - Adds support for Python 3.12. + - Adds support for previewing PDF files present in a workflow's workspace. + - Adds support for previewing ROOT files present in a workflow's workspace. + - Adds support for signing-in with a custom third-party Keycloak instance. + - Adds a new menu item to the workflow actions popup to allow stopping running workflows. + - Adds `prune` command to delete all intermediate files of a given workflow. Use with care. + - Changes the deletion of a workflow to automatically delete an open interactive session attached to its workspace. + - Changes the workflow deletion message to clarify that attached interactive sessions are also closed when a workflow is deleted. + - Changes the workflow progress bar to always display it as completed for finished workflows. + - Changes the interactive session notification to also report that the session will be closed after a specified number of days of inactivity. + - Changes the workflow-details page to make it possible to scroll through the list of workflow steps in the job logs section. + - Changes the workflow-details page to not automatically refresh the selected job when viewing the related logs, but keeping the user-selected one active. + - Changes the page titles to conform to the same sentence case style. + - Changes the launcher page to show warnings when validating the REANA specification file of the workflow to be launched. + - Changes `open` command to inform user about the auto-closure of interactive sessions after a certain inactivity timeout. + - Changes `validate` command to display non-critical validation warnings when checking the REANA specification file. + - Changes Rucio authentication helper to allow users to override the Rucio server and authentication hosts independently of VO name. + - Fixes job status inconsistency when stopping a workflow by setting the job statuses to `stopped` for any running jobs. + - Fixes calculation of workflow runtime durations for stopped workflows. + - Fixes `list` command to correctly list workflows when sorting them by their run number or by the size of their workspace. + - Fixes `du` command help message typo. + - Fixes `validation --environments` command to correctly handle fully-qualified image names. + - Fixes deletion of failed jobs not being performed when Kerberos is enabled. + - Fixes job monitoring to consider OOM-killed jobs as failed. + - Fixes detection of default Rucio server and authentication host for ATLAS VO. + - Fixes the reported total number of jobs for restarted workflows by excluding cached jobs that were simply reused from previous runs in the workspace and not really executed by Snakemake. + - Fixes an issue where Snakemake workflows could get stuck waiting for already-finished jobs. + +- Administrators: + + - Adds support for Kubernetes clusters 1.26, 1.27, 1.28. + - Adds new configuration option `components.reana_ui.launcher_examples` to customise the demo examples that are shown in the launch page in the REANA UI. + - Adds new configuration option `interactive_sessions.maximum_inactivity_period` to set a limit in days for the maximum inactivity period of interactive sessions after which they will be closed. + - Adds new configuration option `interactive_sessions.cronjob_schedule` to set how often interactive session cleanup should be performed. + - Adds new configuration option `ingress.extra` to define extra Ingress resources, in order to support redirecting HTTP requests to HTTPS with traefik v2 version. + - Adds new configuration option `ingress.tls.hosts` to define hosts that are present in the TLS certificate, in order to support cert-manager's automatic creation of certificates. + - Adds new configuration option `notifications.email_config.smtp_ssl` to use SSL when connecting to the SMTP email server. + - Adds new configuration option `notifications.email_config.smtp_starttls` to use the STARTTLS command to enable encryption after connecting to the SMTP email server. + - Adds new configuration option `components.reana_ui.file_preview_size_limit` to set the maximum file size that can be previewed in the web interface. + - Adds new configuration options `login` and `secrets.login` for configuring Keycloak SSO login with third-party authentication services. + - Adds new `interactive-session-cleanup` command that can be used by REANA administrators to close interactive sessions that are inactive for more than the specified number of days. + - Adds progress meter to the logs of the periodic quota updater. + - Adds the content of the `secrets.gitlab.REANA_GITLAB_HOST` configuration option to the list of GitLab instances from which it is possible to launch a workflow. + - Changes uWSGI configuration to increase buffer size, add vacuum option, etc. + - Changes CPU and disk quota calculations to improve the performance of periodic quota updater. + - Changes the system status report to simplify and clarify the disk usage summary. + - Changes `check-workflows` command to also check the presence of workspaces on the shared volume. + - Changes `check-workflows` command to not show in-sync runs by default. If needed, they can be shown using the new `--show-all` option. + - Changes `reana-admin` command options to require the passing of `--admin-access-token` argument more globally. + - Changes the k8s specification for interactive session pods to include labels for improved subset selection of objects. + - Changes the k8s specification for interactive session ingress resource to include annotations. + - Changes nginx configuration to save bandwidth by serving gzip-compressed static files. + - Changes HTCondor to version 9.0.17 (LTS). + - Fixes uWSGI memory consumption on systems with very high allowed number of open files. + - Fixes cronjob failures due to database connection issues when REANA is deployed with non-default namespace or prefix. + - Fixes `ingress.enabled` option to correctly enable or disable the creation of Ingresses. + - Fixes graceful shutdown for reana-server and reana-workflow-controller. + - Fixes the workflow priority calculation to avoid workflows stuck in the `queued` status when the number of allowed concurrent workflow is set to zero. + - Fixes GitLab integration to automatically redirect the user to the correct URL when the access request is accepted. + - Fixes authentication flow to correctly deny access to past revoked tokens in case the same user has also other new active tokens. + - Fixes email templates to show the correct `kubectl` commands when REANA is deployed inside a non-default namespace or with a custom component name prefix. + - Fixes email sender for system emails to `notifications.email_config.sender` Helm value. + - Fixes email receiver for token request emails to use `notifications.email_config.receiver` Helm value. + - Fixes `quota-set-default-limits` command to propagate default quota limits to all users without custom quota limit values. + - Fixes job status consumer to correctly rollback the database transaction when an error occurs. + - Fixes intermittent Slurm connection issues by DNS-resolving the Slurm head node IPv4 address before establishing connections. + - Fixes Slurm command generation issues when using fully-qualified image names. + - Fixes high memory usage of RabbitMQ by limiting the maximum number of open file descriptors. + - Removes support for Kubernetes version prior to 1.21. + +- Developers: + + - Adds new `prune_workspace` endpoint to allow users to delete all the files of a workflow, specifying whether to also delete the inputs and/or the outputs. + - Adds the timestamp of when the workflow was stopped (`run_stopped_at`) to the workflow list and the workflow status endpoints. + - Adds unique error messages to Kubernetes job monitor to more easily identify source of problems. + - Adds new `--parallel` option to `docker-build`, `cluster-build` and `run-ci` to build multiple docker images in parallel. + - Changes `launch` endpoint to also include the warnings of the validation of the workflow specification. + - Changes OpenAPI specification of the `info` endpoint to return the maximum inactivity time before automatic closure of interactive sessions. + - Changes `apispec` dependency version in order to be compatible with `PyYAML` v6. + - Changes Paramiko to version 3.0.0. + - Changes remote storage file support for Snakemake workflows to use XRootD 5.6.0. + - Fixes `cluster-deploy`, `cluster-undeploy` and `client-setup-environment` commands when using custom instance name or kubernetes namespace. + - Fixes the `git-tag` command to display the component name. + - Fixes container image names to be Podman-compatible. + - Fixes location of HTCondor build dependencies. + +## 0.9.0 (2023-01-26) + +- Users: + + - Adds support for Rucio authentication for workflow jobs. + - Adds support for Kerberos authentication for workflow orchestration. + - Adds support for specifying `slurm_partition` and `slurm_time` for Slurm compute backend jobs. + - Adds support for XRootD remote file locations in Snakemake workflow specification definitions. + - Adds support for Python 3.11. + - Adds Launch on REANA page allowing the submission of workflows via badge-clicking. + - Adds notifications to inform users when critical levels of quota usage is reached. + - Adds 404 Not Found error page. + - Adds tab titles to all the pages. + - Adds the `REANA_WORKSPACE` environment variable to jupyter notebooks and terminals. + - Adds option to sort workflows by most disk and cpu quota usage to the workflow list endpoint. + - Adds support for specifying and listing workspace file retention rules. + - Adds support for `.gitignore` and `.reanaignore` to specify files that should not be uploaded to REANA. + - Adds `retention-rules-list` command to list the retention rules of a workflow. + - Changes REANA specification to allow specifying `retention_days` for the workflow. + - Changes default Slurm partition to `inf-short`. + - Changes GitLab integration to also retrieve user's projects that are in groups and subgroups. + - Changes the workflow-details page to show the logs of the workflow engine. + - Changes the workflow-details page to show file sizes in a human-readable format. + - Changes the workflow-details page to show the workspace's retention rules. + - Changes the workflow-details page to show the duration of the workflow's jobs. + - Changes the workflow-details page to display a label of the workflow launcher URL remote origin. + - Changes the workflow-details page to periodically refresh the content of the page. + - Changes the workflow-details page to refresh after the deletion of a workflow. + - Changes the workflow-list page to add a way to hide deleted workflows. + - Changes the workflow-list page to add new workflows sorting options by most used disk and cpu quota. + - Changes the deletion of a workflow to always clean up the workspace and to update the user disk quota usage. + - Changes the CWD of jupyter's terminals to the directory of the workflow's workspace. + - Changes percentage ranges used to calculate the health status of user resource quota usage. + - Changes `create` and `restart` commands to always upload the REANA specification file. + - Changes `delete` command to always delete the workflow's workspace. + - Changes `delete_workflow` Python API function to always delete the workflow's workspace. + - Changes `download` command to add the possibility to write files to the standard output via `-o -` option. + - Changes `list` command to hide deleted workflows by default. + - Changes `list` command to allow displaying deleted workflows via `--all` and `--show-deleted-runs` options. + - Changes `list` and `status` commands to allow displaying the duration of workflows with the `--include-duration` option. + - Changes `mv` command to allow moving files while a workflow is running. + - Changes `upload` command to prevent uploading symlinks. + - Changes `validation --environment` command to use Docker registry v2 APIs to check that a Docker image exists in DockerHub. + - Fixes `list` command to highlight the workflow specified in `REANA_WORKON` correctly. + - Fixes `secrets-delete` command error message when deleting non existing secrets. + - Fixes `start` command to report failed workflows as errors. + - Fixes `start` and `run` commands to correctly follow the execution of the workflow until termination. + - Fixes `status` command to respect output format provided by the `--format` option. + - Fixes `upload` command to report when input directories are listed under the `files` section in the REANA specification file and vice versa. + - Fixes `validate --environment` command to detect illegal whitespace characters in Docker image names. + - Fixes Kerberos authentication for long-running workflows by renewing the Kerberos ticket periodically. + - Fixes status reporting for failed CWL and Snakemake jobs that were incorrectly considered successful. + - Fixes redirection chain for non-signed-in CERN SSO users to access the desired target page after sign-in. + - Fixes the ordering by size of the files showed in the `Workspace` tab of the workflow-details page. + - Fixes CERN OIDC authentication to possibly allow eduGAIN and social login users. + - Fixes wrong numbering of restarted workflows by limiting the number of times a workflow can be restarted to nine. + +- Administrators: + + - Adds new configuration environment variable `reana_server.environment.REANA_SCHEDULER_REQUEUE_COUNT` to set workflow requeue count in case of scheduling errors or busy cluster situations. + - Adds "infinity" option to `REANA_SCHEDULER_REQUEUE_COUNT` to disable requeue count. + - Adds support for Kubernetes clusters 1.22, 1.23, 1.24, 1.25. + - Adds new configuration option `workspaces.retention_rules.maximum_period` to set a default period for workspace retention rules. + - Adds new configuration option `workspaces.retention_rules.cronjob_schedule` to set how often pending retention rules should be applied. + - Adds configuration environment variable `reana_server.environment.REANA_RATELIMIT_SLOW` to limit API requests to some protected endpoints e.g launch workflow. + - Adds configuration environment variable `reana_server.environment.REANA_WORKFLOW_SCHEDULING_READINESS_CHECK_LEVEL` to define checks that are performed to assess whether the cluster is ready to start new workflows. + - Adds new configuration option `ingress.tls.self_signed_cert` to enable the generation of a self-signed TLS certificate. + - Adds new configuration option `ingress.tls.secret_name` to specify the name of the Kubernetes secret containing the TLS certificate to be used. + - Adds support for configuring an additional volume to be used by the database and the message broker. + - Adds new configuration option `maintenance.enabled` to scale down the cluster for maintenance. + - Adds support for Unicode characters inside email body. + - Adds `queue-consume` command that can be used by REANA administrators to remove specific messages from the queue. + - Adds `retention-rules-apply` command that can be used by REANA administrators to apply pending retention rules. + - Adds `retention-rules-extend` command that can be used by REANA administrators to extend the duration of active retentions rules. + - Adds `check-workflows` command that can be used by REANA administrators to check for out-of-sync workflows and interactive sessions. + - Changes configuration option `quota.workflow_termination_update_policy` to deactivate workflow termination accounting by default. + - Changes Helm template to use PostgreSQL 12.13 version. + - Changes the base image for most of the components to Ubuntu 20.04 LTS and reduces final Docker image size by removing build-time dependencies. + - Changes `reana-auth-vomsproxy` sidecar to the latest stable version to support client-side proxy file generation technique and ESCAPE VOMS. + - Changes OAuth configuration to enable the new CERN SSO. + - Changes job status consumer to improve logging for not-alive workflows. + - Changes the deployment of interactive sessions to improve security by not automounting the Kubernetes service account token. + - Changes the deployment of job-controller to avoid unnecessarily mounting the database's directory. + - Changes the announcements to support limited HTML markup. + - Changes REANA specification loading functionality to allow specifying different working directories. + - Changes global setting of maximum number of parallel jobs to 300 for Snakemake workflow engine. + - Fixes job status consumer by discarding invalid job IDs. + - Fixes GitLab integration error reporting in case user exceeds CPU or Disk quota usage limits. + - Fixes issue when irregular number formats are passed to `REANA_SCHEDULER_REQUEUE_COUNT` configuration environment variable. + - Fixes quota updater to reduce memory usage. + - Fixes conversion of possibly-negative resource usage values to human-readable formats. + - Fixes disk quota updater to prevent setting negative disk quota usage values. + - Removes support for Kubernetes version prior to 1.19. + +- Developers: + + - Adds OpenAPI specification support for `launch` endpoint that allows running workflows from remote sources. + - Adds OpenAPI specification support for `get_workflow_retention_rules` endpoint that allows to retrieve the workspace file retention rules of a workflow. + - Adds the remote origin of workflows submitted via Launch-on-REANA (`launcher_url`) to the workflow list endpoint. + - Adds common utility functions for managing workspace files to `reana-commons`. + - Changes default consumer prefetch count to handle 10 messages instead of 200 in order to reduce the probability of 406 PRECONDITION errors on message acknowledgement. + - Changes `git-upgrade-shared-modules` to generate the correct upper-bound in `setup.py`. + - Changes REANA specification loading and validation functionalities by porting some of the logic to `reana-commons`. + - Changes OpenAPI specification to include missing response schema elements. + - Changes the Kubernetes Python client to use the `networking/v1` API. + - Changes the deployment of interactive sessions to use `networking/v1` Kubernetes API. + - Changes to Flask v2. + - Changes `/api/info` endpoint to also include the kubernetes maximum memory limit, the kubernetes default memory limit and the maximum workspace retention period. + - Changes `start_workflow` endpoint to validate the REANA specification of the workflow. + - Changes `create_workflow` endpoint to populate workspace retention rules for the workflow. + - Changes `start_workflow` endpoint to disallow restarting a workflow when retention rules are pending. + - Changes API rate limiter error messages to be more verbose. + - Changes workflow scheduler to allow defining the checks needed to assess whether the cluster can start new workflows. + - Changes workflow list endpoint to add the possibility to filter by workflow ID. + - Changes the `move_files` endpoint to allow moving files while a workflow is running. + - Changes the k8s specification of interactive sessions' pods to remove the environment variables used for service discovery. + - Changes GitLab integration to use `reana` as pipeline name instead of `default` when setting status of a commit. + - Changes the loading of Snakemake specifications to preserve the current working directory. + - Changes the Invenio dependencies to the latest versions. + - Fixes the submission of jobs by stripping potential leading and trailing whitespaces in Docker image names. + - Fixes `fetchWorkflow` action to fetch a specific workflow instead of the entire user workflow list. (reana-ui) + - Fixes the download of files by changing the default MIME type to `application/octet-stream`. + - Fixes the workflow list endpoint to correctly parse the boolean parameters `include_progress`, `include_workspace_size` and `include_retention_rules`. + +## 0.8.1 (2022-02-15) + +- Users: + + - Adds support for specifying `kubernetes_job_timeout` for Kubernetes compute backend jobs. + - Adds Kubernetes job memory limits validation before accepting workflows for execution. + - Adds support for HTML preview of workspace files in the web user interface. + - Adds an option to search for concrete file names in the workflow's workspace web user interface page. + - Changes the Cluster Health web interface page to display the cluster status information based on resource availability rather than only usage. + - Changes `info` command to include the list of supported compute backends. + - Fixes workflow stuck in pending status due to early Yadage failures. + - Fixes formatting of error messages and sets appropriate exit status codes. + +- Administrators: + + - Adds new configuration option to set default job timeout value for the Kubernetes compute backend jobs (`kubernetes_jobs_timeout_limit`). + - Adds new configuration option to set maximum job timeout that users can assign to their jobs for the Kubernetes compute backend (`kubernetes_jobs_max_user_timeout_limit`). + - Adds new configuration option `compute_backends` to specify the supported list of compute backends for validation purposes. + - Adds new configuration option `reana_server.uwsgi.log_all` to toggle the logging of all the HTTP requests. + - Adds new configuration options `reana_server.uwsgi.log_4xx` and `reana_server.uwsgi.log_5xx` to only log HTTP error requests, i.e. HTTP requests with status code 4XX and 5XX. To make this configuration effective `reana_server.uwsgi.log_all` must be `false`. + - Adds new configuration options `node_label_infrastructuremq` and `node_label_infrastructuredb` to have the possibility to run the Message Broker and the Database pods in specific nodes. + - Changes uWSGI configuration to log all HTTP requests in REANA-Server by default. + - Changes `quota.disk_update` to `quota.periodic_update_policy` to also update the CPU quota. Keeps `quota.disk_update` for backward compatibility. + - Changes the name of configuration option `quota.termination_update_policy` to `quota.workflow_termination_update_policy`. Keeps `quota.termination_update_policy` for backward compatibility. + +- Developers: + + - Adds workflow name validation to the `create_workflow` endpoint, restricting special characters like dots. + - Changes `/api/info` endpoint to return a list of supported compute backends. + - Changes `/api/status` endpoint to calculate the cluster health status based on the availability instead of the usage. + - Changes the way of determining Snakemake job statuses, polling the Job Controller API instead of checking local files. + +## 0.8.0 (2021-11-30) + +- Users: + + - Adds support for running and validating Snakemake workflows. + - Adds support for `outputs.directories` in `reana.yaml` allowing to easily download output directories. + - Adds new command `quota-show` to retrieve information about total CPU and Disk usage and quota limits. + - Adds new command `info` that retrieves general information about the cluster, such as available workspace path settings. + - Changes `validate` command to add the possibility to check the workflow against server capabilities such as desired workspace path via `--server-capabilities` option. + - Changes `list` command to add the possibility to filter by workflow status and search by workflow name via `--filter` option. + - Changes `list` command to add the possibility to filter and display all the runs of a given workflow via `-w` option. + - Changes `list` command to stop including workflow progress and workspace size by default. Please use new options `--include-progress` and `--include-workspace-size` to show this information. + - Changes `list --sessions` command to display the status of interactive sessions. + - Changes `logs` command to display also the start and finish times of individual jobs. + - Changes `ls` command to add the possibility to filter by file name, size and last-modified values via `--filter` option. + - Changes `du` command to add the possibility filter by file name and size via `--filter` option. + - Changes `delete` command to prevent hard-deletion of workflows. + - Changes Yadage workflow specification loading to be done in `reana-commons`. + - Changes CWL workflow engine to `cwltool` version `3.1.20210628163208`. + - Removes support for Python 2.7. Please use Python 3.6 or higher from now on. + +- Administrators: + + - Adds new configuration options `node_label_runtimebatch`, `node_label_runtimejobs`, `node_label_runtimesessions` allowing to set cluster node labels for splitting runtime workload into dedicated workflow batch nodes, workflow job nodes and interactive session nodes. + - Adds new configuration option `workspaces.paths` allowing to set a dictionary of available workspace paths to pairs of `cluster_node_path:cluster_pod_mountpath` for mounting directories from cluster nodes. + - Adds new configuration option `quota.enabled` to enable or disable CPU and Disk quota accounting for users. + - Adds new configuration option `quota.termination_update_policy` to select the quota resources such as CPU and Disk for which the quota usage will be calculated immediately at the workflow termination time. + - Adds new periodic cron job to update Disk quotas nightly. Useful if the `quota.termination_update_policy` does not include Disk quota resource. + - Adds configuration environment variable `reana_server.environment.REANA_WORKFLOW_SCHEDULING_POLICY` allowing to set workflow scheduling policy (first-in first-out, user-balanced and workflow-complexity balanced). + - Adds configuration environment variables `reana_server.environment.REANA_RATELIMIT_GUEST_USER`, `reana_server.environment.REANA_RATELIMIT_AUTHENTICATED_USER` allowing to set REST API rate limit values. + - Adds configuration environment variable `reana_server.environment.REANA_SCHEDULER_REQUEUE_SLEEP` to set a time to wait between processing queued workflows. + - Adds configuration environment variable `reana_workflow_controller.environment.REANA_JOB_STATUS_CONSUMER_PREFETCH_COUNT` allowing to set a prefetch count for the job status consumer. + - Adds support for Kubernetes 1.21 version clusters. + - Adds default `kubernetes_memory_limit` value (4 GiB) that will be used for all user jobs unless they specify otherwise. + - Changes Helm template to use PostgreSQL 12.8 version. + - Changes Helm template for `reana-db` component to allow 300 maximum number of database connections by default. + - Fixes email validation procedure during `create-admin-user` command to recognize more permissive email address formats. + +- Developers: + + - Changes `git-*` commands to add the possibility of excluding certain components via the `--exclude-components` option. + - Changes `git-create-release-commit` command to bump all version files in a component. + - Changes `git-log` command to show diff patch or to pass any wanted argument. + - Changes `helm-upgrade-components` command to also upgrade the image tags in `prefetch-images.sh` script. + +## 0.7.4 (2021-07-07) + +- Users: + + - Adds support for file listing wildcard matching patterns to `ls` command. + - Adds support for directory download and wildcard matching patterns to `download` command. + - Adds support for specifying `kubernetes_memory_limit` for Kubernetes compute backend jobs for CWL, Serial and Yadage workflows. + - Changes `list` command to include deleted workflows by default. + - Changes `validate` command to warn about incorrectly used workflow parameters for each step. + - Changes `validate` command to display more granular workflow validation output. + - Fixes workflow step job command formatting bug for CWL workflows on HTCondor compute backend. + - Fixes `validate` command output for verifying environment image UID values. + - Fixes `upload_to_server()` Python API function to silently skip uploading in case of none-like inputs. + - Fixes `validate` command for environment image validation to not test repetitively the same image found in different steps. + +- Administrators: + + - Adds support for Kubernetes 1.21. + - Adds configuration environment variable to set default job memory limits for the Kubernetes compute backend (`REANA_KUBERNETES_JOBS_MEMORY_LIMIT`). + - Adds configuration environment variable to set maximum custom memory limits that users can assign to their jobs for the Kubernetes compute backend (`REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT`). + - Changes HTCondor compute backend to 8.9.11 and `myschedd` package and configuration to latest versions. + - Fixes Kubernetes job log capture to include information about failures caused by external factors such as out-of-memory situations (`OOMKilled`). + +- Developers: + + - Adds new functions to serialise/deserialise job commands between REANA components. + - Changes client dependencies to unpin six so that client may be installed in more contexts. + - Changes cluster dependencies to remove click and pins several dependencies. + - Changes `reana_ready()` function location to REANA-Server. + +## 0.7.3 (2021-03-24) + +- Users: + + - Adds `reana-client validate` options to detect possible issues with workflow input parameters and environment images. + - Fixes problem with failed jobs being reported as still running in case of network problems. + - Fixes job command encoding issues when dispatching jobs to HTCondor and Slurm backends. + +- Administrators: + + - Adds new configuration to toggle Kubernetes user jobs clean up. + : (`REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES` in `components.reana_workflow_controller.environment`) + - Improves platform resilience. + +- Developers: + + - Adds new command-line options to `reana-dev run-example` command allowing full parallel asynchronous execution of demo examples. + - Adds default configuration for developer deployment mode to keep failed workflow and job pods for easier debugging. + - Changes job status consumer communications to improve overall platform resilience. + +## 0.7.2 (2021-02-04) + +- Administrators: + + - Adds support for deployments on Kubernetes 1.20 clusters. + - Adds deployment option to disable user email confirmation step after sign-up. (`REANA_USER_EMAIL_CONFIRMATION` in `components.reana_server.environment`) + - Adds deployment option to disable user sign-up feature completely. (`components.reana_ui.hide_signup`) + - Adds deployment option to display CERN Privacy Notice for CERN deployments. (`components.reana_ui.cern_ropo`) + +- Developers: + + - Adds support for Python 3.9. + - Fixes minor code warnings. + - Changes CI system to include Python flake8 and Dockerfile hadolint checkers. + +## 0.7.1 (2020-11-10) + +- Users: + + - Adds support for specifying `htcondor_max_runtime` and `htcondor_accounting_group` for HTCondor compute backend jobs. + - Fixes restarting of Yadage and CWL workflows. + - Fixes REANA \<-> GitLab synchronisation for projects having additional external webhooks. + - Changes `ping` command output to include REANA client and server version information. + +- Developers: + + - Fixes conflicting `kombu` installation requirements by requiring Celery version 4. + - Changes `/api/you` endpoint to include REANA server version information. + - Changes continuous integration platform from Travis CI to GitHub Actions. + +## 0.7.0 (2020-10-21) + +- Users: + + - Adds new `restart` command to restart previously run or failed workflows. + - Adds option to `logs` command to filter job logs according to compute backend, docker image, job status and step name. + - Adds option to specify operational options in the `reana.yaml` of the workflow. + - Adds option to specify unpacked Docker images as workflow step requirement. + - Adds option to specify Kubernetes UID for jobs. + - Adds support for VOMS proxy as a new authentication method. + - Adds support for pulling private Docker images. + - Adds pagination on the workflow list and workflow detailed web interface pages. + - Adds user profile page to the web interface. + - Adds page refresh button to workflow detailed page. + - Adds local user web forms for sign-in and sign-up functionalities for local deployments. + - Fixes user experience by preventing dots as part of the workflow name to avoid confusion with restart runs. + - Fixes workflow specification display to show runtime parameters. + - Fixes file preview functionality experience to allow/disallow certain file formats. + - Changes Yadage workflow engine to version 0.20.1. + - Changes CERN HTCondor compute backend to use the new `myschedd` connection library. + - Changes CERN Slurm compute backend to improve job status detection. + - Changes documentation to move large parts to [docs.reana.io](http://docs.reana.io). + - Changes `du` command output format. + - Changes `logs` command to enhance formatting using marks and colours. + - Changes `ping` command to perform user access token validation. + - Changes `diff` command to improve output formatting. + - Changes defaults to accept both `reana.yaml` and `reana.yml` filenames. + - Changes from Bravado to requests to improve download performance. + - Changes file loading to optimise CLI performance. + +- Administrators: + + - Adds Helm chart and switches to Helm-based deployment technique instead of using now-deprecated `reana-cluster`. + - Adds email notification service to inform administrators about system health. + - Adds announcement configuration option to display any desired text on the web UI. + - Adds pinning of all Python dependencies allowing to easily rebuild component images at later times. + - Adds support for local user management and web forms for sign-in and sign-up functionalities. + - Adds support for database upgrades using Alembic. + - Changes installation procedures to move database initialisation and admin creation after Helm installation. + - Changes service exposure to stop exposing unused Invenio-Accounts views. + - Changes runtime job instantiation into the configured runtime namespace. + - Changes CVMFS to be read-only mount. + +- Developers: + + - Adds several new `reana-dev` commands to help with merging, releasing, unit testing. + - Changes base image to use Python 3.8 for all REANA cluster components. + - Changes pre-requisites to node version 12 and latest npm dependencies. + - Changes back-end code formatting to respect `black` coding style. + - Changes front-end code formatting to respect updated `prettier` version coding style. + - Changes test strategy to start PostgreSQL DB container to run tests locally. + - Changes auto-generated component documentation to single-page layout. + +## 0.6.1 (2020-06-09) + +- Administrators: + + - Fixes installation troubles for REANA 0.6.x release series by pinning several dependencies. + - Upgrades REANA-Commons package to latest Kubernetes Python client version. + - Amends documentation for `minikube start` to include VirtualBox hypervisor explicitly. + +## 0.6.0 (2019-12-27) + +- Users: + + - Adds support for HTCondor compute backend for all workflow engines (CWL, Serial, Yadage). + - Adds support for Slurm compute backend for all workflow engines (CWL, Serial, Yadage). + - Allows to run hybrid analysis pipelines where different parts of the workflow can run on different compute backends (HTCondor, Kubernetes, Slurm). + - Adds support for Kerberos authentication mechanism for user workflows. + - Introduces user secrets management commands `secrets-add`, `secrets-list` and `secrets-delete`. + - Fixes `upload` command behaviour for uploading very large files. + - Upgrades CWL workflow engine to 1.0.20191022103248. + - Upgrades Yadage workflow engine to 0.20.0 with Packtivity 0.14.21. + - Adds support for Python 3.8. + - See additional changes in [reana-client 0.6.0 release notes](https://reana-client.readthedocs.io/en/latest/changes.html#version-0-6-0-2019-12-27). + +- Administrators: + + - Upgrades to Kubernetes 1.16 and moves Traefik installation to Helm 3.0.0. + - Creates a new Kubernetes service account for REANA with appropriate permissions. + - Makes database connection details configurable so that REANA can connect to databases external to the cluster. + - Autogenerates deployment secrets if not provided by administrator at cluster creation time. + - Adds an interactive mode on cluster initialisation to allow providing deployment secrets. + - Adds CERN specific Kerberos configuration files and CERN EOS storage support. + - See additional changes in [reana-cluster 0.6.0 release notes](https://reana-cluster.readthedocs.io/en/latest/changes.html#version-0-6-0-2019-12-27). + +- Developers: + + - Modifies the batch workflow runtime pod creation including an instance of job controller running alongside workflow engine using the sidecar pattern. + - Adds generic job manager class and provides example classes for CERN HTCondor and CERN Slurm clusters. + - Provides user secrets to the job container runtime tasks. + - Adds sidecar container to the Kubernetes job pod if Kerberos authentication is required. + - Refactors job monitoring using the singleton pattern. + - Enriches `make` behaviour for developer-oriented installations with live code reload changes and debugging. + - Enriches `git-status` component status reporting for developers. + - See additional changes in [individual REANA 0.6.0 platform components](https://reana.readthedocs.io/en/latest/administratorguide.html#components). + +## 0.5.0 (2019-04-24) + +- Users: + + - Allows to explore workflow results by running interactive Jupyter notebook sessions on the workspace files. + - Allows to declare computing resources needed for workflow runs, such as access to CVMFS repositories. + - Improves `reana-client` command-line client with new options to stop workflows, diff workflows, move and remove files. + - Upgrades CWL engine to 1.0.20181118133959. + - See additional changes in [reana-client 0.5.0 release notes](https://reana-client.readthedocs.io/en/latest/changes.html#version-0-5-0-2019-04-24). + +- Administrators: + + - Upgrades to Kubernetes 1.14, Helm 2.13 and Minikube 1.0. + - Separates cluster infrastructure pods from runtime workflow engine pods that will be created by workflow controller. + - Introduces configurable CVMFS and CephFS shared volume mounts. + - Adds support for optional HTTPS protocol termination. + - Introduces incoming workflow queue for additional safety in case of user storms. + - Makes infrastructure pods container image slimmer to reduce the memory footprint. + - See additional changes in [reana-cluster 0.5.0 release notes](https://reana-cluster.readthedocs.io/en/latest/changes.html#version-0-5-0-2019-04-24). + +- Developers: + + - Enhances development process by using git-submodule-like behaviour for shared components. + - Introduces simple Makefile for (fast) local testing and (slow) nightly building purposes. + - Centralises logging level and common Celery tasks. + - Adds helpers for test suite fixtures and improves code coverage. + - See additional changes in [individual REANA 0.5.0 platform components](https://reana.readthedocs.io/en/latest/administratorguide.html#components). + +## 0.4.0 (2018-11-07) + +- Uses common OpenAPI client in communications between workflow engines and job + controller. +- Improves AMQP re-connection handling. +- Enhances test suite and increases code coverage. +- Changes license to MIT. + +## 0.3.0 (2018-09-27) + +- Introduces new Serial workflow engine for simple sequential workflow needs. +- Enhances progress reporting for CWL, Serial and Yadage workflow engines. +- Simplifies `reana-client` command set and usage scenarios. +- Introduces multi-user capabilities with mandatory access tokens. +- Adds support for multi-node clusters using shared CephFS volumes. +- Adds support for Kubernetes 1.11, Minikube 0.28.2. +- Upgrades CWL workflow engine to use latest `cwltool` version. +- Fixes several bugs such as binary file download with Python 3. + +## 0.2.0 (2018-04-23) + +- Adds support for Common Workflow Language workflows. +- Adds support for persistent user-selected workflow names. +- Enables file and directory input uploading using absolute paths. +- Enriches `reana-client` and `reana-cluster` command set. +- Reduces verbosity level for commands and improves error messages. + +## 0.1.0 (2018-01-30) + +- Initial public release. + diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 11f09dbf..00000000 --- a/CHANGES.rst +++ /dev/null @@ -1,548 +0,0 @@ -Changes -======= - -Version 0.9.2 (2023-12-19) --------------------------- - -- Users: - - Adds web interface form allowing to generate launcher URL for any user-provided public analysis, as well as the Markdown snippet for the corresponding Launch-on-REANA badge. - - Adds web interface option to delete all the runs of a workflow. - - Changes the Launch-on-REANA web interface page to improve how workflow parameters are shown by displaying them inside a table. - - Changes CVMFS support to allow users to automatically mount any available repository. - - Changes the REANA specification schema to use the ``draft-07`` version of the JSON Schema specification. - - Changes ``reana-client validate`` command to show detailed errors when the specification file is not a valid YAML file. - - Changes validation of REANA specification to make the ``environment`` property mandatory for the steps of serial workflows. - - Changes validation of REANA specification to raise a warning for unexpected properties for the steps of serial workflows. - - Changes validation of REANA specification to report improved validation warnings which also indicate where unexpected properties are located in the file. - - Changes workflow restarts to allow for more than nine restarts of the same workflows. - - Changes workflow scheduler logging behaviour to also report the main reason behind scheduling errors to the users. - - Fixes ``reana-client list`` command to accept case-insensitive column names when sorting the returned workflow runs via the ``--sort`` option. - - Fixes ``reana-client run`` wrapper command for workflows that do not contain ``inputs`` clause in their specification. - - Fixes ``reana-client``'s ``create_workflow_from_json`` API function to always load and send the workflow specification to the server. - - Fixes creation of image thumbnails for output files in Snakemake HTML execution reports. -- Administrators: - - Changes several database index definitions in order to improve performance of most common database queries. - - Changes the names of database table, column, index and key constraints in order to follow the SQLAlchemy upstream naming conventions everywhere. - - Changes the ``Workflow`` table to replace the ``run_number`` column with two new columns ``run_number_major`` and ``run_number_minor`` in order to allow for more than nine restarts of user workflows. - - Changes CVMFS support to allow users to automatically mount any available repository, thanks to CVMFS CSI v2. - - Fixes the mounting of CVMFS volumes for the REANA deployments that use non-default Kubernetes namespace. - - Fixes container image building of cluster components for the arm64 architecture. - - Fixes job monitoring in cases when job creation fails, for example when it is not possible to successfully mount volumes. - - Fixes job status consumer exception while attempting to fetch workflow engine logs for workflows that could not have been successfully scheduled. - - Fixes the creation of Kubernetes jobs by retrying in case of error and by correctly handling the error after reaching the retry limit. -- Developers: - - Adds automated multi-platform container image building of cluster components for amd64 and arm64 architectures. - - Adds new ``--image-name`` option to the ``reana-dev docker-push`` command to customise the name of the docker image to publish. - - Adds new ``--platform`` option to the ``reana-dev docker-build`` and ``reana-dev release-docker`` commands to build and publish multi-platform images. - - Adds new ``--registry`` option to the ``reana-dev docker-push`` and ``reana-dev release-docker`` commands to specify the registry where the built docker images should be pushed to. - - Adds new ``--tags-only`` option to the ``reana-dev release-docker`` command to only print the final docker image names, without pushing the images to the registry. - - Adds new ``reana-dev git-create-release-branch`` command to create a new Git branch for a new release. - - Adds new ``reana-dev git-upgrade-requirements`` command to upgrade the ``requirements.txt`` file before a new release. - - Changes ``reana-dev git-fork`` and ``reana-dev git-create-pr`` to use the ``gh`` CLI client instead of ``hub``. - - Changes ``reana-dev python-run-tests`` command to allow excluding certain Python components. - - Changes ``reana-dev python-run-tests`` command to allow execution of selected pytests only by passing over `PYTEST_ADDOPTS` environment variable. - - Changes validation of REANA specification to expose functions for loading workflow input parameters and workflow specifications. - - Changes version of ``reana-ui`` Node.js Docker image from 16 to 18. - - Changes the workflow deletion endpoint to return a different and more appropriate message when deleting all the runs of a workflow. - - Changes the workflow list endpoint on how pagination is performed in order to avoid counting twice the total number of records. - - Fixes ``reana-dev python-run-tests`` command to create Python-3.8 based virtual environments to use the same version as container images. - -Version 0.9.1 (2023-09-27) --------------------------- - -- Users: - - Adds support for Python 3.12. - - Adds support for previewing PDF files present in a workflow's workspace. - - Adds support for previewing ROOT files present in a workflow's workspace. - - Adds support for signing-in with a custom third-party Keycloak instance. - - Adds a new menu item to the workflow actions popup to allow stopping running workflows. - - Adds ``prune`` command to delete all intermediate files of a given workflow. Use with care. - - Changes the deletion of a workflow to automatically delete an open interactive session attached to its workspace. - - Changes the workflow deletion message to clarify that attached interactive sessions are also closed when a workflow is deleted. - - Changes the workflow progress bar to always display it as completed for finished workflows. - - Changes the interactive session notification to also report that the session will be closed after a specified number of days of inactivity. - - Changes the workflow-details page to make it possible to scroll through the list of workflow steps in the job logs section. - - Changes the workflow-details page to not automatically refresh the selected job when viewing the related logs, but keeping the user-selected one active. - - Changes the page titles to conform to the same sentence case style. - - Changes the launcher page to show warnings when validating the REANA specification file of the workflow to be launched. - - Changes ``open`` command to inform user about the auto-closure of interactive sessions after a certain inactivity timeout. - - Changes ``validate`` command to display non-critical validation warnings when checking the REANA specification file. - - Changes Rucio authentication helper to allow users to override the Rucio server and authentication hosts independently of VO name. - - Fixes job status inconsistency when stopping a workflow by setting the job statuses to ``stopped`` for any running jobs. - - Fixes calculation of workflow runtime durations for stopped workflows. - - Fixes ``list`` command to correctly list workflows when sorting them by their run number or by the size of their workspace. - - Fixes ``du`` command help message typo. - - Fixes ``validation --environments`` command to correctly handle fully-qualified image names. - - Fixes deletion of failed jobs not being performed when Kerberos is enabled. - - Fixes job monitoring to consider OOM-killed jobs as failed. - - Fixes detection of default Rucio server and authentication host for ATLAS VO. - - Fixes the reported total number of jobs for restarted workflows by excluding cached jobs that were simply reused from previous runs in the workspace and not really executed by Snakemake. - - Fixes an issue where Snakemake workflows could get stuck waiting for already-finished jobs. -- Administrators: - - Adds support for Kubernetes clusters 1.26, 1.27, 1.28. - - Adds new configuration option ``components.reana_ui.launcher_examples`` to customise the demo examples that are shown in the launch page in the REANA UI. - - Adds new configuration option ``interactive_sessions.maximum_inactivity_period`` to set a limit in days for the maximum inactivity period of interactive sessions after which they will be closed. - - Adds new configuration option ``interactive_sessions.cronjob_schedule`` to set how often interactive session cleanup should be performed. - - Adds new configuration option ``ingress.extra`` to define extra Ingress resources, in order to support redirecting HTTP requests to HTTPS with traefik v2 version. - - Adds new configuration option ``ingress.tls.hosts`` to define hosts that are present in the TLS certificate, in order to support cert-manager's automatic creation of certificates. - - Adds new configuration option ``notifications.email_config.smtp_ssl`` to use SSL when connecting to the SMTP email server. - - Adds new configuration option ``notifications.email_config.smtp_starttls`` to use the STARTTLS command to enable encryption after connecting to the SMTP email server. - - Adds new configuration option ``components.reana_ui.file_preview_size_limit`` to set the maximum file size that can be previewed in the web interface. - - Adds new configuration options ``login`` and ``secrets.login`` for configuring Keycloak SSO login with third-party authentication services. - - Adds new ``interactive-session-cleanup`` command that can be used by REANA administrators to close interactive sessions that are inactive for more than the specified number of days. - - Adds progress meter to the logs of the periodic quota updater. - - Adds the content of the ``secrets.gitlab.REANA_GITLAB_HOST`` configuration option to the list of GitLab instances from which it is possible to launch a workflow. - - Changes uWSGI configuration to increase buffer size, add vacuum option, etc. - - Changes CPU and disk quota calculations to improve the performance of periodic quota updater. - - Changes the system status report to simplify and clarify the disk usage summary. - - Changes ``check-workflows`` command to also check the presence of workspaces on the shared volume. - - Changes ``check-workflows`` command to not show in-sync runs by default. If needed, they can be shown using the new ``--show-all`` option. - - Changes ``reana-admin`` command options to require the passing of ``--admin-access-token`` argument more globally. - - Changes the k8s specification for interactive session pods to include labels for improved subset selection of objects. - - Changes the k8s specification for interactive session ingress resource to include annotations. - - Changes nginx configuration to save bandwidth by serving gzip-compressed static files. - - Changes HTCondor to version 9.0.17 (LTS). - - Fixes uWSGI memory consumption on systems with very high allowed number of open files. - - Fixes cronjob failures due to database connection issues when REANA is deployed with non-default namespace or prefix. - - Fixes ``ingress.enabled`` option to correctly enable or disable the creation of Ingresses. - - Fixes graceful shutdown for reana-server and reana-workflow-controller. - - Fixes the workflow priority calculation to avoid workflows stuck in the ``queued`` status when the number of allowed concurrent workflow is set to zero. - - Fixes GitLab integration to automatically redirect the user to the correct URL when the access request is accepted. - - Fixes authentication flow to correctly deny access to past revoked tokens in case the same user has also other new active tokens. - - Fixes email templates to show the correct ``kubectl`` commands when REANA is deployed inside a non-default namespace or with a custom component name prefix. - - Fixes email sender for system emails to ``notifications.email_config.sender`` Helm value. - - Fixes email receiver for token request emails to use ``notifications.email_config.receiver`` Helm value. - - Fixes ``quota-set-default-limits`` command to propagate default quota limits to all users without custom quota limit values. - - Fixes job status consumer to correctly rollback the database transaction when an error occurs. - - Fixes intermittent Slurm connection issues by DNS-resolving the Slurm head node IPv4 address before establishing connections. - - Fixes Slurm command generation issues when using fully-qualified image names. - - Fixes high memory usage of RabbitMQ by limiting the maximum number of open file descriptors. - - Removes support for Kubernetes version prior to 1.21. -- Developers: - - Adds new ``prune_workspace`` endpoint to allow users to delete all the files of a workflow, specifying whether to also delete the inputs and/or the outputs. - - Adds the timestamp of when the workflow was stopped (``run_stopped_at``) to the workflow list and the workflow status endpoints. - - Adds unique error messages to Kubernetes job monitor to more easily identify source of problems. - - Adds new ``--parallel`` option to ``docker-build``, ``cluster-build`` and ``run-ci`` to build multiple docker images in parallel. - - Changes ``launch`` endpoint to also include the warnings of the validation of the workflow specification. - - Changes OpenAPI specification of the ``info`` endpoint to return the maximum inactivity time before automatic closure of interactive sessions. - - Changes ``apispec`` dependency version in order to be compatible with ``PyYAML`` v6. - - Changes Paramiko to version 3.0.0. - - Changes remote storage file support for Snakemake workflows to use XRootD 5.6.0. - - Fixes ``cluster-deploy``, ``cluster-undeploy`` and ``client-setup-environment`` commands when using custom instance name or kubernetes namespace. - - Fixes the ``git-tag`` command to display the component name. - - Fixes container image names to be Podman-compatible. - - Fixes location of HTCondor build dependencies. - -Version 0.9.0 (2023-01-26) --------------------------- - -- Users: - - Adds support for Rucio authentication for workflow jobs. - - Adds support for Kerberos authentication for workflow orchestration. - - Adds support for specifying ``slurm_partition`` and ``slurm_time`` for Slurm compute backend jobs. - - Adds support for XRootD remote file locations in Snakemake workflow specification definitions. - - Adds support for Python 3.11. - - Adds Launch on REANA page allowing the submission of workflows via badge-clicking. - - Adds notifications to inform users when critical levels of quota usage is reached. - - Adds 404 Not Found error page. - - Adds tab titles to all the pages. - - Adds the ``REANA_WORKSPACE`` environment variable to jupyter notebooks and terminals. - - Adds option to sort workflows by most disk and cpu quota usage to the workflow list endpoint. - - Adds support for specifying and listing workspace file retention rules. - - Adds support for ``.gitignore`` and ``.reanaignore`` to specify files that should not be uploaded to REANA. - - Adds ``retention-rules-list`` command to list the retention rules of a workflow. - - Changes REANA specification to allow specifying ``retention_days`` for the workflow. - - Changes default Slurm partition to ``inf-short``. - - Changes GitLab integration to also retrieve user's projects that are in groups and subgroups. - - Changes the workflow-details page to show the logs of the workflow engine. - - Changes the workflow-details page to show file sizes in a human-readable format. - - Changes the workflow-details page to show the workspace's retention rules. - - Changes the workflow-details page to show the duration of the workflow's jobs. - - Changes the workflow-details page to display a label of the workflow launcher URL remote origin. - - Changes the workflow-details page to periodically refresh the content of the page. - - Changes the workflow-details page to refresh after the deletion of a workflow. - - Changes the workflow-list page to add a way to hide deleted workflows. - - Changes the workflow-list page to add new workflows sorting options by most used disk and cpu quota. - - Changes the deletion of a workflow to always clean up the workspace and to update the user disk quota usage. - - Changes the CWD of jupyter's terminals to the directory of the workflow's workspace. - - Changes percentage ranges used to calculate the health status of user resource quota usage. - - Changes ``create`` and ``restart`` commands to always upload the REANA specification file. - - Changes ``delete`` command to always delete the workflow's workspace. - - Changes ``delete_workflow`` Python API function to always delete the workflow's workspace. - - Changes ``download`` command to add the possibility to write files to the standard output via ``-o -`` option. - - Changes ``list`` command to hide deleted workflows by default. - - Changes ``list`` command to allow displaying deleted workflows via ``--all`` and ``--show-deleted-runs`` options. - - Changes ``list`` and ``status`` commands to allow displaying the duration of workflows with the ``--include-duration`` option. - - Changes ``mv`` command to allow moving files while a workflow is running. - - Changes ``upload`` command to prevent uploading symlinks. - - Changes ``validation --environment`` command to use Docker registry v2 APIs to check that a Docker image exists in DockerHub. - - Fixes ``list`` command to highlight the workflow specified in ``REANA_WORKON`` correctly. - - Fixes ``secrets-delete`` command error message when deleting non existing secrets. - - Fixes ``start`` command to report failed workflows as errors. - - Fixes ``start`` and ``run`` commands to correctly follow the execution of the workflow until termination. - - Fixes ``status`` command to respect output format provided by the ``--format`` option. - - Fixes ``upload`` command to report when input directories are listed under the ``files`` section in the REANA specification file and vice versa. - - Fixes ``validate --environment`` command to detect illegal whitespace characters in Docker image names. - - Fixes Kerberos authentication for long-running workflows by renewing the Kerberos ticket periodically. - - Fixes status reporting for failed CWL and Snakemake jobs that were incorrectly considered successful. - - Fixes redirection chain for non-signed-in CERN SSO users to access the desired target page after sign-in. - - Fixes the ordering by size of the files showed in the ``Workspace`` tab of the workflow-details page. - - Fixes CERN OIDC authentication to possibly allow eduGAIN and social login users. - - Fixes wrong numbering of restarted workflows by limiting the number of times a workflow can be restarted to nine. -- Administrators: - - Adds new configuration environment variable ``reana_server.environment.REANA_SCHEDULER_REQUEUE_COUNT`` to set workflow requeue count in case of scheduling errors or busy cluster situations. - - Adds "infinity" option to ``REANA_SCHEDULER_REQUEUE_COUNT`` to disable requeue count. - - Adds support for Kubernetes clusters 1.22, 1.23, 1.24, 1.25. - - Adds new configuration option ``workspaces.retention_rules.maximum_period`` to set a default period for workspace retention rules. - - Adds new configuration option ``workspaces.retention_rules.cronjob_schedule`` to set how often pending retention rules should be applied. - - Adds configuration environment variable ``reana_server.environment.REANA_RATELIMIT_SLOW`` to limit API requests to some protected endpoints e.g launch workflow. - - Adds configuration environment variable ``reana_server.environment.REANA_WORKFLOW_SCHEDULING_READINESS_CHECK_LEVEL`` to define checks that are performed to assess whether the cluster is ready to start new workflows. - - Adds new configuration option `ingress.tls.self_signed_cert` to enable the generation of a self-signed TLS certificate. - - Adds new configuration option `ingress.tls.secret_name` to specify the name of the Kubernetes secret containing the TLS certificate to be used. - - Adds support for configuring an additional volume to be used by the database and the message broker. - - Adds new configuration option `maintenance.enabled` to scale down the cluster for maintenance. - - Adds support for Unicode characters inside email body. - - Adds ``queue-consume`` command that can be used by REANA administrators to remove specific messages from the queue. - - Adds ``retention-rules-apply`` command that can be used by REANA administrators to apply pending retention rules. - - Adds ``retention-rules-extend`` command that can be used by REANA administrators to extend the duration of active retentions rules. - - Adds ``check-workflows`` command that can be used by REANA administrators to check for out-of-sync workflows and interactive sessions. - - Changes configuration option ``quota.workflow_termination_update_policy`` to deactivate workflow termination accounting by default. - - Changes Helm template to use PostgreSQL 12.13 version. - - Changes the base image for most of the components to Ubuntu 20.04 LTS and reduces final Docker image size by removing build-time dependencies. - - Changes ``reana-auth-vomsproxy`` sidecar to the latest stable version to support client-side proxy file generation technique and ESCAPE VOMS. - - Changes OAuth configuration to enable the new CERN SSO. - - Changes job status consumer to improve logging for not-alive workflows. - - Changes the deployment of interactive sessions to improve security by not automounting the Kubernetes service account token. - - Changes the deployment of job-controller to avoid unnecessarily mounting the database's directory. - - Changes the announcements to support limited HTML markup. - - Changes REANA specification loading functionality to allow specifying different working directories. - - Changes global setting of maximum number of parallel jobs to 300 for Snakemake workflow engine. - - Fixes job status consumer by discarding invalid job IDs. - - Fixes GitLab integration error reporting in case user exceeds CPU or Disk quota usage limits. - - Fixes issue when irregular number formats are passed to ``REANA_SCHEDULER_REQUEUE_COUNT`` configuration environment variable. - - Fixes quota updater to reduce memory usage. - - Fixes conversion of possibly-negative resource usage values to human-readable formats. - - Fixes disk quota updater to prevent setting negative disk quota usage values. - - Removes support for Kubernetes version prior to 1.19. -- Developers: - - Adds OpenAPI specification support for ``launch`` endpoint that allows running workflows from remote sources. - - Adds OpenAPI specification support for ``get_workflow_retention_rules`` endpoint that allows to retrieve the workspace file retention rules of a workflow. - - Adds the remote origin of workflows submitted via Launch-on-REANA (``launcher_url``) to the workflow list endpoint. - - Adds common utility functions for managing workspace files to ``reana-commons``. - - Changes default consumer prefetch count to handle 10 messages instead of 200 in order to reduce the probability of 406 PRECONDITION errors on message acknowledgement. - - Changes `git-upgrade-shared-modules` to generate the correct upper-bound in `setup.py`. - - Changes REANA specification loading and validation functionalities by porting some of the logic to ``reana-commons``. - - Changes OpenAPI specification to include missing response schema elements. - - Changes the Kubernetes Python client to use the ``networking/v1`` API. - - Changes the deployment of interactive sessions to use ``networking/v1`` Kubernetes API. - - Changes to Flask v2. - - Changes ``/api/info`` endpoint to also include the kubernetes maximum memory limit, the kubernetes default memory limit and the maximum workspace retention period. - - Changes ``start_workflow`` endpoint to validate the REANA specification of the workflow. - - Changes ``create_workflow`` endpoint to populate workspace retention rules for the workflow. - - Changes ``start_workflow`` endpoint to disallow restarting a workflow when retention rules are pending. - - Changes API rate limiter error messages to be more verbose. - - Changes workflow scheduler to allow defining the checks needed to assess whether the cluster can start new workflows. - - Changes workflow list endpoint to add the possibility to filter by workflow ID. - - Changes the ``move_files`` endpoint to allow moving files while a workflow is running. - - Changes the k8s specification of interactive sessions' pods to remove the environment variables used for service discovery. - - Changes GitLab integration to use ``reana`` as pipeline name instead of ``default`` when setting status of a commit. - - Changes the loading of Snakemake specifications to preserve the current working directory. - - Changes the Invenio dependencies to the latest versions. - - Fixes the submission of jobs by stripping potential leading and trailing whitespaces in Docker image names. - - Fixes ``fetchWorkflow`` action to fetch a specific workflow instead of the entire user workflow list. (reana-ui) - - Fixes the download of files by changing the default MIME type to ``application/octet-stream``. - - Fixes the workflow list endpoint to correctly parse the boolean parameters ``include_progress``, ``include_workspace_size`` and ``include_retention_rules``. - -Version 0.8.1 (2022-02-15) --------------------------- - -- Users: - - Adds support for specifying ``kubernetes_job_timeout`` for Kubernetes compute backend jobs. - - Adds Kubernetes job memory limits validation before accepting workflows for execution. - - Adds support for HTML preview of workspace files in the web user interface. - - Adds an option to search for concrete file names in the workflow's workspace web user interface page. - - Changes the Cluster Health web interface page to display the cluster status information based on resource availability rather than only usage. - - Changes ``info`` command to include the list of supported compute backends. - - Fixes workflow stuck in pending status due to early Yadage failures. - - Fixes formatting of error messages and sets appropriate exit status codes. -- Administrators: - - Adds new configuration option to set default job timeout value for the Kubernetes compute backend jobs (``kubernetes_jobs_timeout_limit``). - - Adds new configuration option to set maximum job timeout that users can assign to their jobs for the Kubernetes compute backend (``kubernetes_jobs_max_user_timeout_limit``). - - Adds new configuration option ``compute_backends`` to specify the supported list of compute backends for validation purposes. - - Adds new configuration option ``reana_server.uwsgi.log_all`` to toggle the logging of all the HTTP requests. - - Adds new configuration options ``reana_server.uwsgi.log_4xx`` and ``reana_server.uwsgi.log_5xx`` to only log HTTP error requests, i.e. HTTP requests with status code 4XX and 5XX. To make this configuration effective ``reana_server.uwsgi.log_all`` must be ``false``. - - Adds new configuration options ``node_label_infrastructuremq`` and ``node_label_infrastructuredb`` to have the possibility to run the Message Broker and the Database pods in specific nodes. - - Changes uWSGI configuration to log all HTTP requests in REANA-Server by default. - - Changes ``quota.disk_update`` to ``quota.periodic_update_policy`` to also update the CPU quota. Keeps ``quota.disk_update`` for backward compatibility. - - Changes the name of configuration option ``quota.termination_update_policy`` to ``quota.workflow_termination_update_policy``. Keeps ``quota.termination_update_policy`` for backward compatibility. -- Developers: - - Adds workflow name validation to the ``create_workflow`` endpoint, restricting special characters like dots. - - Changes ``/api/info`` endpoint to return a list of supported compute backends. - - Changes ``/api/status`` endpoint to calculate the cluster health status based on the availability instead of the usage. - - Changes the way of determining Snakemake job statuses, polling the Job Controller API instead of checking local files. - -Version 0.8.0 (2021-11-30) --------------------------- - -- Users: - - Adds support for running and validating Snakemake workflows. - - Adds support for ``outputs.directories`` in ``reana.yaml`` allowing to easily download output directories. - - Adds new command ``quota-show`` to retrieve information about total CPU and Disk usage and quota limits. - - Adds new command ``info`` that retrieves general information about the cluster, such as available workspace path settings. - - Changes ``validate`` command to add the possibility to check the workflow against server capabilities such as desired workspace path via ``--server-capabilities`` option. - - Changes ``list`` command to add the possibility to filter by workflow status and search by workflow name via ``--filter`` option. - - Changes ``list`` command to add the possibility to filter and display all the runs of a given workflow via ``-w`` option. - - Changes ``list`` command to stop including workflow progress and workspace size by default. Please use new options ``--include-progress`` and ``--include-workspace-size`` to show this information. - - Changes ``list --sessions`` command to display the status of interactive sessions. - - Changes ``logs`` command to display also the start and finish times of individual jobs. - - Changes ``ls`` command to add the possibility to filter by file name, size and last-modified values via ``--filter`` option. - - Changes ``du`` command to add the possibility filter by file name and size via ``--filter`` option. - - Changes ``delete`` command to prevent hard-deletion of workflows. - - Changes Yadage workflow specification loading to be done in ``reana-commons``. - - Changes CWL workflow engine to ``cwltool`` version ``3.1.20210628163208``. - - Removes support for Python 2.7. Please use Python 3.6 or higher from now on. -- Administrators: - - Adds new configuration options ``node_label_runtimebatch``, ``node_label_runtimejobs``, ``node_label_runtimesessions`` allowing to set cluster node labels for splitting runtime workload into dedicated workflow batch nodes, workflow job nodes and interactive session nodes. - - Adds new configuration option ``workspaces.paths`` allowing to set a dictionary of available workspace paths to pairs of ``cluster_node_path:cluster_pod_mountpath`` for mounting directories from cluster nodes. - - Adds new configuration option ``quota.enabled`` to enable or disable CPU and Disk quota accounting for users. - - Adds new configuration option ``quota.termination_update_policy`` to select the quota resources such as CPU and Disk for which the quota usage will be calculated immediately at the workflow termination time. - - Adds new periodic cron job to update Disk quotas nightly. Useful if the ``quota.termination_update_policy`` does not include Disk quota resource. - - Adds configuration environment variable ``reana_server.environment.REANA_WORKFLOW_SCHEDULING_POLICY`` allowing to set workflow scheduling policy (first-in first-out, user-balanced and workflow-complexity balanced). - - Adds configuration environment variables ``reana_server.environment.REANA_RATELIMIT_GUEST_USER``, ``reana_server.environment.REANA_RATELIMIT_AUTHENTICATED_USER`` allowing to set REST API rate limit values. - - Adds configuration environment variable ``reana_server.environment.REANA_SCHEDULER_REQUEUE_SLEEP`` to set a time to wait between processing queued workflows. - - Adds configuration environment variable ``reana_workflow_controller.environment.REANA_JOB_STATUS_CONSUMER_PREFETCH_COUNT`` allowing to set a prefetch count for the job status consumer. - - Adds support for Kubernetes 1.21 version clusters. - - Adds default ``kubernetes_memory_limit`` value (4 GiB) that will be used for all user jobs unless they specify otherwise. - - Changes Helm template to use PostgreSQL 12.8 version. - - Changes Helm template for ``reana-db`` component to allow 300 maximum number of database connections by default. - - Fixes email validation procedure during ``create-admin-user`` command to recognize more permissive email address formats. -- Developers: - - Changes ``git-*`` commands to add the possibility of excluding certain components via the ``--exclude-components`` option. - - Changes ``git-create-release-commit`` command to bump all version files in a component. - - Changes ``git-log`` command to show diff patch or to pass any wanted argument. - - Changes ``helm-upgrade-components`` command to also upgrade the image tags in ``prefetch-images.sh`` script. - -Version 0.7.4 (2021-07-07) --------------------------- - -- Users: - - Adds support for file listing wildcard matching patterns to ``ls`` command. - - Adds support for directory download and wildcard matching patterns to ``download`` command. - - Adds support for specifying ``kubernetes_memory_limit`` for Kubernetes compute backend jobs for CWL, Serial and Yadage workflows. - - Changes ``list`` command to include deleted workflows by default. - - Changes ``validate`` command to warn about incorrectly used workflow parameters for each step. - - Changes ``validate`` command to display more granular workflow validation output. - - Fixes workflow step job command formatting bug for CWL workflows on HTCondor compute backend. - - Fixes ``validate`` command output for verifying environment image UID values. - - Fixes ``upload_to_server()`` Python API function to silently skip uploading in case of none-like inputs. - - Fixes ``validate`` command for environment image validation to not test repetitively the same image found in different steps. -- Administrators: - - Adds support for Kubernetes 1.21. - - Adds configuration environment variable to set default job memory limits for the Kubernetes compute backend (``REANA_KUBERNETES_JOBS_MEMORY_LIMIT``). - - Adds configuration environment variable to set maximum custom memory limits that users can assign to their jobs for the Kubernetes compute backend (``REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT``). - - Changes HTCondor compute backend to 8.9.11 and `myschedd` package and configuration to latest versions. - - Fixes Kubernetes job log capture to include information about failures caused by external factors such as out-of-memory situations (`OOMKilled`). -- Developers: - - Adds new functions to serialise/deserialise job commands between REANA components. - - Changes client dependencies to unpin six so that client may be installed in more contexts. - - Changes cluster dependencies to remove click and pins several dependencies. - - Changes ``reana_ready()`` function location to REANA-Server. - -Version 0.7.3 (2021-03-24) --------------------------- - -- Users: - - Adds ``reana-client validate`` options to detect possible issues with workflow input parameters and environment images. - - Fixes problem with failed jobs being reported as still running in case of network problems. - - Fixes job command encoding issues when dispatching jobs to HTCondor and Slurm backends. -- Administrators: - - Adds new configuration to toggle Kubernetes user jobs clean up. - (``REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES`` in ``components.reana_workflow_controller.environment``) - - Improves platform resilience. -- Developers: - - Adds new command-line options to ``reana-dev run-example`` command allowing full parallel asynchronous execution of demo examples. - - Adds default configuration for developer deployment mode to keep failed workflow and job pods for easier debugging. - - Changes job status consumer communications to improve overall platform resilience. - -Version 0.7.2 (2021-02-04) --------------------------- - -- Administrators: - - Adds support for deployments on Kubernetes 1.20 clusters. - - Adds deployment option to disable user email confirmation step after sign-up. - (``REANA_USER_EMAIL_CONFIRMATION`` in ``components.reana_server.environment``) - - Adds deployment option to disable user sign-up feature completely. - (``components.reana_ui.hide_signup``) - - Adds deployment option to display CERN Privacy Notice for CERN deployments. - (``components.reana_ui.cern_ropo``) -- Developers: - - Adds support for Python 3.9. - - Fixes minor code warnings. - - Changes CI system to include Python flake8 and Dockerfile hadolint checkers. - -Version 0.7.1 (2020-11-10) --------------------------- - -- Users: - - Adds support for specifying ``htcondor_max_runtime`` and ``htcondor_accounting_group`` for HTCondor compute backend jobs. - - Fixes restarting of Yadage and CWL workflows. - - Fixes REANA <-> GitLab synchronisation for projects having additional external webhooks. - - Changes ``ping`` command output to include REANA client and server version information. -- Developers: - - Fixes conflicting ``kombu`` installation requirements by requiring Celery version 4. - - Changes ``/api/you`` endpoint to include REANA server version information. - - Changes continuous integration platform from Travis CI to GitHub Actions. - -Version 0.7.0 (2020-10-21) --------------------------- - -- Users: - - Adds new ``restart`` command to restart previously run or failed workflows. - - Adds option to ``logs`` command to filter job logs according to compute backend, docker image, job status and step name. - - Adds option to specify operational options in the ``reana.yaml`` of the workflow. - - Adds option to specify unpacked Docker images as workflow step requirement. - - Adds option to specify Kubernetes UID for jobs. - - Adds support for VOMS proxy as a new authentication method. - - Adds support for pulling private Docker images. - - Adds pagination on the workflow list and workflow detailed web interface pages. - - Adds user profile page to the web interface. - - Adds page refresh button to workflow detailed page. - - Adds local user web forms for sign-in and sign-up functionalities for local deployments. - - Fixes user experience by preventing dots as part of the workflow name to avoid confusion with restart runs. - - Fixes workflow specification display to show runtime parameters. - - Fixes file preview functionality experience to allow/disallow certain file formats. - - Changes Yadage workflow engine to version 0.20.1. - - Changes CERN HTCondor compute backend to use the new ``myschedd`` connection library. - - Changes CERN Slurm compute backend to improve job status detection. - - Changes documentation to move large parts to `docs.reana.io `_. - - Changes ``du`` command output format. - - Changes ``logs`` command to enhance formatting using marks and colours. - - Changes ``ping`` command to perform user access token validation. - - Changes ``diff`` command to improve output formatting. - - Changes defaults to accept both ``reana.yaml`` and ``reana.yml`` filenames. - - Changes from Bravado to requests to improve download performance. - - Changes file loading to optimise CLI performance. -- Administrators: - - Adds Helm chart and switches to Helm-based deployment technique instead of using now-deprecated ``reana-cluster``. - - Adds email notification service to inform administrators about system health. - - Adds announcement configuration option to display any desired text on the web UI. - - Adds pinning of all Python dependencies allowing to easily rebuild component images at later times. - - Adds support for local user management and web forms for sign-in and sign-up functionalities. - - Adds support for database upgrades using Alembic. - - Changes installation procedures to move database initialisation and admin creation after Helm installation. - - Changes service exposure to stop exposing unused Invenio-Accounts views. - - Changes runtime job instantiation into the configured runtime namespace. - - Changes CVMFS to be read-only mount. -- Developers: - - Adds several new ``reana-dev`` commands to help with merging, releasing, unit testing. - - Changes base image to use Python 3.8 for all REANA cluster components. - - Changes pre-requisites to node version 12 and latest npm dependencies. - - Changes back-end code formatting to respect ``black`` coding style. - - Changes front-end code formatting to respect updated ``prettier`` version coding style. - - Changes test strategy to start PostgreSQL DB container to run tests locally. - - Changes auto-generated component documentation to single-page layout. - -Version 0.6.1 (2020-06-09) --------------------------- - -- Administrators: - - Fixes installation troubles for REANA 0.6.x release series by pinning several dependencies. - - Upgrades REANA-Commons package to latest Kubernetes Python client version. - - Amends documentation for `minikube start` to include VirtualBox hypervisor explicitly. - -Version 0.6.0 (2019-12-27) --------------------------- - -- Users: - - Adds support for HTCondor compute backend for all workflow engines (CWL, Serial, Yadage). - - Adds support for Slurm compute backend for all workflow engines (CWL, Serial, Yadage). - - Allows to run hybrid analysis pipelines where different parts of the workflow can run on different compute backends (HTCondor, Kubernetes, Slurm). - - Adds support for Kerberos authentication mechanism for user workflows. - - Introduces user secrets management commands ``secrets-add``, ``secrets-list`` and ``secrets-delete``. - - Fixes ``upload`` command behaviour for uploading very large files. - - Upgrades CWL workflow engine to 1.0.20191022103248. - - Upgrades Yadage workflow engine to 0.20.0 with Packtivity 0.14.21. - - Adds support for Python 3.8. - - See additional changes in `reana-client 0.6.0 release notes `_. -- Administrators: - - Upgrades to Kubernetes 1.16 and moves Traefik installation to Helm 3.0.0. - - Creates a new Kubernetes service account for REANA with appropriate permissions. - - Makes database connection details configurable so that REANA can connect to databases external to the cluster. - - Autogenerates deployment secrets if not provided by administrator at cluster creation time. - - Adds an interactive mode on cluster initialisation to allow providing deployment secrets. - - Adds CERN specific Kerberos configuration files and CERN EOS storage support. - - See additional changes in `reana-cluster 0.6.0 release notes `_. -- Developers: - - Modifies the batch workflow runtime pod creation including an instance of job controller running alongside workflow engine using the sidecar pattern. - - Adds generic job manager class and provides example classes for CERN HTCondor and CERN Slurm clusters. - - Provides user secrets to the job container runtime tasks. - - Adds sidecar container to the Kubernetes job pod if Kerberos authentication is required. - - Refactors job monitoring using the singleton pattern. - - Enriches ``make`` behaviour for developer-oriented installations with live code reload changes and debugging. - - Enriches ``git-status`` component status reporting for developers. - - See additional changes in `individual REANA 0.6.0 platform components `_. - -Version 0.5.0 (2019-04-24) --------------------------- - -- Users: - - Allows to explore workflow results by running interactive Jupyter notebook sessions on the workspace files. - - Allows to declare computing resources needed for workflow runs, such as access to CVMFS repositories. - - Improves ``reana-client`` command-line client with new options to stop workflows, diff workflows, move and remove files. - - Upgrades CWL engine to 1.0.20181118133959. - - See additional changes in `reana-client 0.5.0 release notes `_. -- Administrators: - - Upgrades to Kubernetes 1.14, Helm 2.13 and Minikube 1.0. - - Separates cluster infrastructure pods from runtime workflow engine pods that will be created by workflow controller. - - Introduces configurable CVMFS and CephFS shared volume mounts. - - Adds support for optional HTTPS protocol termination. - - Introduces incoming workflow queue for additional safety in case of user storms. - - Makes infrastructure pods container image slimmer to reduce the memory footprint. - - See additional changes in `reana-cluster 0.5.0 release notes `_. -- Developers: - - Enhances development process by using git-submodule-like behaviour for shared components. - - Introduces simple Makefile for (fast) local testing and (slow) nightly building purposes. - - Centralises logging level and common Celery tasks. - - Adds helpers for test suite fixtures and improves code coverage. - - See additional changes in `individual REANA 0.5.0 platform components `_. - -Version 0.4.0 (2018-11-07) --------------------------- - -- Uses common OpenAPI client in communications between workflow engines and job - controller. -- Improves AMQP re-connection handling. -- Enhances test suite and increases code coverage. -- Changes license to MIT. - -Version 0.3.0 (2018-09-27) --------------------------- - -- Introduces new Serial workflow engine for simple sequential workflow needs. -- Enhances progress reporting for CWL, Serial and Yadage workflow engines. -- Simplifies ``reana-client`` command set and usage scenarios. -- Introduces multi-user capabilities with mandatory access tokens. -- Adds support for multi-node clusters using shared CephFS volumes. -- Adds support for Kubernetes 1.11, Minikube 0.28.2. -- Upgrades CWL workflow engine to use latest ``cwltool`` version. -- Fixes several bugs such as binary file download with Python 3. - -Version 0.2.0 (2018-04-23) --------------------------- - -- Adds support for Common Workflow Language workflows. -- Adds support for persistent user-selected workflow names. -- Enables file and directory input uploading using absolute paths. -- Enriches ``reana-client`` and ``reana-cluster`` command set. -- Reduces verbosity level for commands and improves error messages. - -Version 0.1.0 (2018-01-30) --------------------------- - -- Initial public release. - -.. admonition:: Please beware - - Please note that REANA is in an early alpha stage of its development. The - developer preview releases are meant for early adopters and testers. Please - don't rely on released versions for any production purposes yet. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.md similarity index 71% rename from CONTRIBUTING.rst rename to CONTRIBUTING.md index f466bad2..228327cd 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.md @@ -1,14 +1,11 @@ -Contributing -============ +# Contributing -Issues ------- +## Issues Bug reports, issues, feature requests, and other contributions are welcome. If you find a demonstrable problem that is caused by the REANA code, please: -1. Search for `already reported problems - `_. +1. Search for [already reported problems](https://github.com/search?q=org%3Areanahub+is%3Aissue+is%3Aopen). 2. Check if the issue has been fixed or is still reproducible on the latest `master` branch. 3. Create an issue, ideally with **a test case**. @@ -16,9 +13,8 @@ a demonstrable problem that is caused by the REANA code, please: If you create a pull request fixing a bug or implementing a feature, you can run the tests to ensure that everything is operating correctly: -.. code-block:: console - - $ ./run-tests.sh +```console +$ ./run-tests.sh +``` Each pull request should preserve or increase code coverage. - diff --git a/MANIFEST.in b/MANIFEST.in index c41e5ce6..c831a72e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,7 +7,6 @@ include *.html include *.json include *.md -include *.rst include *.sh include *.yaml include *.yml @@ -19,7 +18,7 @@ prune docs/_build recursive-include reana *.py recursive-include docs *.py recursive-include docs *.png -recursive-include docs *.rst +recursive-include docs *.md recursive-include docs *.txt recursive-include tests *.py recursive-include helm *.yaml diff --git a/README.md b/README.md new file mode 100644 index 00000000..52591bdc --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +[![image](docs/logo-reana.png)](http://docs.reana.io) + +# REANA - Reusable Analyses + +[![image](https://github.com/reanahub/reana/workflows/CI/badge.svg)](https://github.com/reanahub/reana/actions) +[![image](https://readthedocs.org/projects/reana/badge/?version=latest)](https://reana.readthedocs.io/en/latest/?badge=latest) +[![image](https://codecov.io/gh/reanahub/reana/branch/master/graph/badge.svg)](https://codecov.io/gh/reanahub/reana) +[![image](https://img.shields.io/badge/discourse-forum-blue.svg)](https://forum.reana.io) +[![image](https://img.shields.io/github/license/reanahub/reana.svg)](https://github.com/reanahub/reana/blob/master/LICENSE) +[![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +## About + +[REANA](http://www.reana.io) is a reusable and reproducible research data analysis +platform. It helps researchers to structure their input data, analysis code, +containerised environments and computational workflows so that the analysis can be +instantiated and run on remote compute clouds. + +REANA was born to target the use case of particle physics analyses, but is applicable to +any scientific discipline. The system paves the way towards reusing and reinterpreting +preserved data analyses even several years after the original publication. + +## Features + +- structure research data analysis in reusable manner +- instantiate computational workflows on remote clouds +- rerun analyses with modified input data, parameters or code +- support for several compute clouds (Kubernetes/OpenStack) +- support for several workflow specifications (CWL, Serial, Yadage, Snakemake) +- support for several shared storage systems (Ceph) +- support for several container technologies (Docker) + +## Getting started + +You can +[install REANA locally](https://docs.reana.io/administration/deployment/deploying-locally/), +[deploy it at scale on premises](https://docs.reana.io/administration/deployment/deploying-at-scale/) +(in about 10 minutes) or use . Once the system is ready, you can +follow the guide to run +[your first example](https://docs.reana.io/getting-started/first-example/). For more in +depth information visit the [official REANA documentation](https://docs.reana.io/). + +## Community + +- Discuss [on Forum](https://forum.reana.io/) +- Chat on [Mattermost](https://mattermost.web.cern.ch/it-dep/channels/reana) or + [Gitter](https://gitter.im/reanahub/reana) +- Follow us [on Twitter](https://twitter.com/reanahub) + +## Useful links + +- [REANA home page](http://www.reana.io/) +- [REANA documentation](http://docs.reana.io/) +- [REANA on DockerHub](https://hub.docker.com/u/reanahub/) diff --git a/README.rst b/README.rst deleted file mode 100644 index cb8b7220..00000000 --- a/README.rst +++ /dev/null @@ -1,71 +0,0 @@ -.. image:: docs/logo-reana.png - :target: http://docs.reana.io - :align: center - -=========================== - REANA - Reusable Analyses -=========================== - -.. image:: https://github.com/reanahub/reana/workflows/CI/badge.svg - :target: https://github.com/reanahub/reana/actions - -.. image:: https://readthedocs.org/projects/reana/badge/?version=latest - :target: https://reana.readthedocs.io/en/latest/?badge=latest - -.. image:: https://codecov.io/gh/reanahub/reana/branch/master/graph/badge.svg - :target: https://codecov.io/gh/reanahub/reana - -.. image:: https://badges.gitter.im/Join%20Chat.svg - :target: https://gitter.im/reanahub/reana?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge - -.. image:: https://img.shields.io/github/license/reanahub/reana.svg - :target: https://github.com/reanahub/reana/blob/master/LICENSE - -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - -About ------ - -`REANA `_ is a reusable and reproducible research data -analysis platform. It helps researchers to structure their input data, analysis -code, containerised environments and computational workflows so that the -analysis can be instantiated and run on remote compute clouds. - -REANA was born to target the use case of particle physics analyses, but is -applicable to any scientific discipline. The system paves the way towards -reusing and reinterpreting preserved data analyses even several years after the -original publication. - -Features --------- - -- structure research data analysis in reusable manner -- instantiate computational workflows on remote clouds -- rerun analyses with modified input data, parameters or code -- support for several compute clouds (Kubernetes/OpenStack) -- support for several workflow specifications (CWL, Serial, Yadage, Snakemake) -- support for several shared storage systems (Ceph) -- support for several container technologies (Docker) - -Getting started ---------------- - -You can `install REANA locally `_, `deploy it at scale on premises -`_ (in about 10 minutes) or use https://reana.cern.ch. Once the system -is ready, you can follow the guide to run `your first example `_. -For more in depth information visit the `official REANA documentation `_. - -Community ---------- - -- Discuss `on Forum `_ -- Chat on `Mattermost `_ or `Gitter `_ -- Follow us `on Twitter `_ - -Useful links ------------- - -- `REANA home page `_ -- `REANA documentation `_ -- `REANA on DockerHub `_ diff --git a/docs/conf.py b/docs/conf.py index 5d3d3242..09351ff5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + "myst_parser", "sphinx.ext.autodoc", "sphinx.ext.coverage", "sphinx.ext.doctest", @@ -55,14 +56,17 @@ # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = [".rst", ".md"] + +# Allow using ::: in Markdown files +myst_enable_extensions = ["colon_fence"] # The master toctree document. master_doc = "index" # General information about the project. project = "reana" -copyright = "2017-2023 info@reana.io" +copyright = "2017-2024, info@reana.io" author = "info@reana.io" # The version info for the project you're documenting, acts as replacement for diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..640c37ed --- /dev/null +++ b/docs/index.md @@ -0,0 +1,3 @@ +# REANA + +REANA documentation has been moved, please visit our new documentation website [docs.reana.io](https://docs.reana.io). diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index b0a92ca8..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,4 +0,0 @@ -REANA -===== - -REANA documentation has been moved, please visit our new documentation website `docs.reana.io `_. \ No newline at end of file diff --git a/setup.py b/setup.py index efd47c44..b770ae0d 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,8 @@ from setuptools import find_packages, setup -readme = open("README.rst").read() -history = open("CHANGES.rst").read() +readme = open("README.md").read() +history = open("CHANGELOG.md").read() tests_require = [ "pytest-reana>=0.9.2,<0.10.0", @@ -23,6 +23,7 @@ extras_require = { "docs": [ + "myst-parser", "Sphinx>=1.5.1", ], "tests": tests_require, @@ -66,6 +67,7 @@ version=version, description=__doc__, long_description=readme + "\n\n" + history, + long_description_content_type="text/markdown", author="REANA", author_email="info@reana.io", url="http://www.reana.io/",