From c57c1cc81d739c9a100c616e0993fecf243113d4 Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Wed, 6 Nov 2024 22:37:24 -0700 Subject: [PATCH 1/4] Consolidate Redis roles --- docs/redis_guide.md | 64 +++------------ playbooks/redis.yml | 23 +----- .../defaults/{main.yml => main/redis.yml} | 6 +- .../defaults/main/sentinel.yml} | 0 roles/redis/tasks/configure-sentinel.yml | 62 +++++++++++++++ roles/redis/tasks/download-packages.yml | 2 +- roles/redis/tasks/main.yml | 13 +++- roles/redis/tasks/redis-using-remi-repo.yml | 2 +- roles/redis/tasks/redis-using-source.yml | 8 -- roles/redis/templates/redis.conf.j2 | 32 +++++--- .../templates/sentinel.conf.j2 | 36 ++++----- roles/redis_auth/README.md | 0 roles/redis_auth/tasks/main.yml | 78 ------------------- roles/redis_replication/README.md | 0 roles/redis_replication/tasks/main.yml | 74 ------------------ 15 files changed, 125 insertions(+), 275 deletions(-) rename roles/redis/defaults/{main.yml => main/redis.yml} (90%) rename roles/{redis_replication/defaults/main.yml => redis/defaults/main/sentinel.yml} (100%) create mode 100644 roles/redis/tasks/configure-sentinel.yml rename roles/{redis_replication => redis}/templates/sentinel.conf.j2 (96%) delete mode 100644 roles/redis_auth/README.md delete mode 100644 roles/redis_auth/tasks/main.yml delete mode 100644 roles/redis_replication/README.md delete mode 100644 roles/redis_replication/tasks/main.yml diff --git a/docs/redis_guide.md b/docs/redis_guide.md index 41ca507..5ca1a28 100644 --- a/docs/redis_guide.md +++ b/docs/redis_guide.md @@ -1,20 +1,18 @@ # Overview -The playbook and roles in this section install and configure Redis for the Itential Automation Platform. There are currently three Redis-related roles: +The playbook and roles in this section install and configure Redis for the Itential Automation Platform. There is one Redis-related role: -* `redis` – Installs Redis and performs a base configuration. -* `redis_auth` – Configures Redis authentication. -* `redis_replication` – Configures Redis replication. +* `redis` – Installs Redis and performs a base configuration. Optionally configures authentication and replication. # Roles ## Redis Role -The `redis` role performs a base install of Redis including any OS packages required. It will compile and install any custom SELinux profiles. It creates the appropriate Linux users, directories, log files, and systemd services. It uses a template to generate a configuration file with some potential features available in other roles commented out. It will start the redis service when complete. +The `redis` role performs a base install of Redis including any OS packages required. It will compile and install any custom SELinux profiles. It creates the appropriate Linux users, directories, log files, and systemd services. It uses a template to generate a configuration file with some potential features available in other roles commented out. It will start the Redis service when complete. -## Redis Auth Role +### Authentication -The `redis_auth` role performs tasks to require authentication (username and password) when communicating with the Redis server. It adjusts the Redis config file and adds each of the required users and applies appropriate ACLs (see table). The "default" Redis user is disabled. It modifies the Redis config file to use the appropriate user while doing replication. It adjusts the Sentinel config file to enable the correct Sentinel user to monitor the redis cluster, if required. It disables the default user in both Redis and Redis Sentinel. It will restart the redis service and the Sentinel service (if required) when complete. +Optionally, the `redis` role performs tasks to require authentication (username and password) when communicating with the Redis server. It adjusts the Redis config file and adds each of the required users and applies appropriate ACLs (see table). The "default" Redis user is disabled. It modifies the Redis config file to use the appropriate user while doing replication. It adjusts the Sentinel config file to enable the correct Sentinel user to monitor the redis cluster, if required. It disables the default user in both Redis and Redis Sentinel. More info on Redis authorization: https://redis.io/docs/manual/security/ @@ -24,12 +22,13 @@ More info on Redis authorization: https://redis.io/docs/manual/security/ | itential | itential | Has access to all keys, all channels, and all commands except: -asking -cluster -readonly -readwrite -bgrewriteaof -bgsave -failover -flushall -flushdb -psync -replconf -replicaof -save -shutdown -sync | repluser | repluser | Has access to the minimum set of commands to perform replication. | sentineluser | sentineluser | Has access to the minimum set of commands to perform sentinel monitoring. +| prometheus | prometheus | Has access to the minimum set of commands to perform Redis and Sentinel monitoring with Prometheus. Required by the optional redis_exporter service. :::(Warning) (⚠ Warning: ) It is assumed that these default passwords will be changed to meet more rigorous standards. These are intended to be defaults strictly used just for ease of the installation. It is highly recommended that sensitive data be encrypted using Ansible Vault. -## Redis Replication Role +### Replication -The `redis_replication` role performs the steps required to create a Redis replica set. It uses a template to generate a Redis Sentinel config file. It modifies the Redis config file to turn off protected-mode. It assumes that the first host defined in the inventory file is the initial primary. It will update the config file for the non-primary Redis servers to replicate from the primary using hostname. It will restart Redis and Redis Sentinel when complete. +Optionally, the `redis` role performs the steps required to create a Redis replica set. It uses a template to generate a Redis Sentinel config file. It modifies the Redis config file to turn off protected-mode. It assumes that the first host defined in the inventory file is the initial primary. It will update the config file for the non-primary Redis servers to replicate from the primary using hostname. It will start Redis Sentinel when complete. For more information on Redis replication: https://redis.io/docs/manual/replication/ @@ -57,15 +56,15 @@ The following table lists the default variables that are shared between the Redi | Variable | Group | Type | Description | Default Value | :------- | :---- | :--- | :---------- | :------------ -| `redis_auth` | `all` | Boolean | Flag to enable Redis authentication. When set to to `true`, the `redis_auth` role will be executed. | `false` -| `redis_replication` | `all` | Boolean | Flag to enable Redis replication. When set to `true`, the `redis_replication` role will be executed. | `false` +| `redis_auth` | `all` | Boolean | Flag to enable Redis authentication. When set to to `true`, Redis authentication will be configured. | `false` +| `redis_replication` | `all` | Boolean | Flag to enable Redis replication. When set to `true`, Redis replication will be configured and the Redis Sentinel service started. | `false` | `redis_tls` | `all` | Boolean | Flag to enable TLS connections. | `false` ## Redis Role Variables The variables in this section may be overridden in the inventory in the `redis` group vars. -The following table lists the default variables located in `roles/redis/defaults/main.yml`. +The following table lists the default variables located in `roles/redis/defaults/main/redis.yml` and `roles/redis/defaults/main/sentinel.yml`. | Variable | Group | Type | Description | Default Value | :------- | :---- | :--- | :---------- | :------------ @@ -82,23 +81,8 @@ The following table lists the default variables located in `roles/redis/defaults | `redis_bind_ipv6` | `redis` | Boolean | Flag to enable IPv6. | `true` | `redis_bind_addr_source` | `redis` | String | The bind address source. Will default to the Ansible `inventory_hostname` unless explicitly set to `default_ipv4_address`. | `inventory_hostname` | `redis_bind_addrs` | `redis` | String | A space-separated list of hostnames/IP addresses on which Redis listeners will be created. If `redis_bind_ipv6` is set to `true`, `::1` will be added to the addresses. The `redis_bind_addr_source` will also be added to the addresses. | `127.0.0.1` -| `iap_redis_packages` | `redis` | List of Strings | The Linux packages to install. | `redis`
`jemalloc` | `redis_install_method` | `redis` | String | The method to use to install Redis.
Set to `remi_repo` to use the Remi repo.
Set to `source` to install from source. | `remi_repo` -| `epel_repo_url` | `redis` | String | The URL of the EPEL repo RPM.
Note: this is only used when the `redis_install_method` is set to `remi_repo`. | `https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm` - - -## Redis Auth Role Variables - -There are no default variables for the Redis Auth role other than the Redis common default variables. - -## Redis Replication Role Variables - -The variables in this section may be overridden in the inventory in the `redis` group vars. - -The following table lists the default variables located in `roles/redis_replication/defaults/main.yml`. - -| Variable | Group | Type | Description | Default Value -| :------- | :---- | :--- | :---------- | :------------ +| `redis_epel_repo_url` | `redis` | String | The URL of the EPEL repo RPM.
Note: this is only used when the `redis_install_method` is set to `remi_repo`. | `https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm` | `redis_sentinel_conf_file` | `redis` | String | The location of the Redis Sentinel configuration file. | `{{ redis_conf_path }}/sentinel.conf` | `redis_sentinel_port` | `redis` | Integer | The Redis Sentinel listen port | `26379` @@ -170,27 +154,3 @@ To execute all Redis roles, run the `redis` playbook: ``` ansible-playbook itential.deployer.redis -i ``` - -You can also run select Redis roles by using the following tags: - -* `redis_install` -* `redis_auth` -* `redis_replication` - -To execute only the `redis` role (skipping the `redis_auth` and `redis_replication` roles), run the `itential.deployer.redis` playbook with the `redis_install` tag: - -``` -ansible-playbook itential.deployer.redis -i --tags redis_install -``` - -To execute only the Redis Auth role (skipping the Redis and Redis Replication roles), run the `itential.deployer.redis` playbook with the `redis_auth` tag: - -``` -ansible-playbook itential.deployer.redis -i --tags redis_auth -``` - -To execute only the Redis Replication role (skipping the Redis and Redis Auth roles), run the `itential.deployer.redis` playbook with the `redis_replication` tag: - -``` -ansible-playbook itential.deployer.redis -i --tags redis_replication -``` diff --git a/playbooks/redis.yml b/playbooks/redis.yml index 83fff85..b37bdbd 100644 --- a/playbooks/redis.yml +++ b/playbooks/redis.yml @@ -8,27 +8,6 @@ roles: # Pull in the common vars - role: itential.deployer.common_vars - tags: - - always - # Perform a base installation of Redis + # Perform a base installation of Redis and optionally configure authorization and replication - role: itential.deployer.redis - tags: - - redis - - redis_install - - # Perform installation of Redis Sentinel for Redis HA and replication - # https://redis.io/docs/manual/sentinel/ - - role: itential.deployer.redis_replication - when: redis_replication | bool - tags: - - redis - - redis_replication - - # Configure Redis to require a username & password (authorization) - # https://redis.io/docs/manual/security/acl/ - - role: itential.deployer.redis_auth - when: redis_auth | bool - tags: - - redis - - redis_auth diff --git a/roles/redis/defaults/main.yml b/roles/redis/defaults/main/redis.yml similarity index 90% rename from roles/redis/defaults/main.yml rename to roles/redis/defaults/main/redis.yml index 521245b..dca5b0a 100644 --- a/roles/redis/defaults/main.yml +++ b/roles/redis/defaults/main/redis.yml @@ -39,10 +39,6 @@ redis_bind_addr_source: inventory_hostname # to the redis_bind_addrs depending on the redis_bind_addr_source. redis_bind_addrs: 127.0.0.1 -iap_redis_packages: - - redis - - jemalloc - # Offline install settings # Refer to the offline variables in common_vars for additional settings packages_path: "{{ itential_packages_path }}/{{ iap_release }}/redis" @@ -52,4 +48,4 @@ packages_path: "{{ itential_packages_path }}/{{ iap_release }}/redis" redis_install_method: source # The EPEL repo is only required when the redis_install_method is set to 'remi_repo' -epel_repo_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" +redis_epel_repo_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" diff --git a/roles/redis_replication/defaults/main.yml b/roles/redis/defaults/main/sentinel.yml similarity index 100% rename from roles/redis_replication/defaults/main.yml rename to roles/redis/defaults/main/sentinel.yml diff --git a/roles/redis/tasks/configure-sentinel.yml b/roles/redis/tasks/configure-sentinel.yml new file mode 100644 index 0000000..2830678 --- /dev/null +++ b/roles/redis/tasks/configure-sentinel.yml @@ -0,0 +1,62 @@ +# Copyright (c) 2024, Itential, Inc +# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) +--- +- name: Create Redis Sentinel systemd file + ansible.builtin.template: + src: redis-sentinel.service.j2 + dest: /usr/lib/systemd/system/redis-sentinel.service + owner: root + group: root + mode: "0644" + +- name: Use template to generate sentinel.conf + ansible.builtin.template: + src: sentinel.conf.j2 + dest: "{{ redis_sentinel_conf_file }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0640" + backup: true + when: + - groups['redis'] is defined + - inventory_hostname in groups['redis'] + vars: + master_name: "{{ hostvars[groups['redis'][0]].inventory_hostname }}" + +- name: Use template to generate sentinel.conf for secondary DR + ansible.builtin.template: + src: sentinel.conf.j2 + dest: "{{ redis_sentinel_conf_file }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0640" + backup: true + when: + - groups['redis_secondary'] is defined + - inventory_hostname in groups['redis_secondary'] + vars: + master_name: "{{ hostvars[groups['redis_secondary'][0]].inventory_hostname }}" + +# Check if firewalld is running, if it is then open the appropriate ports +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Open Port on FirewallD Public Zone + ansible.posix.firewalld: + port: "{{ redis_sentinel_port }}/tcp" + permanent: true + state: enabled + zone: public + immediate: true + when: + - ansible_facts.services["firewalld.service"] is defined + - ansible_facts.services["firewalld.service"].state == "running" + - ansible_facts.services["firewalld.service"].status == "enabled" + +- name: Enable Redis Sentinel + throttle: 1 + ansible.builtin.systemd: + name: redis-sentinel + enabled: true + state: restarted + daemon_reload: true diff --git a/roles/redis/tasks/download-packages.yml b/roles/redis/tasks/download-packages.yml index 1872feb..1c0d7c3 100644 --- a/roles/redis/tasks/download-packages.yml +++ b/roles/redis/tasks/download-packages.yml @@ -30,7 +30,7 @@ block: - name: Install EPEL repo ansible.builtin.dnf: - name: "{{ epel_repo_url }}" + name: "{{ redis_epel_repo_url }}" state: present update_cache: true disable_gpg_check: true diff --git a/roles/redis/tasks/main.yml b/roles/redis/tasks/main.yml index d02a22f..1a2648a 100644 --- a/roles/redis/tasks/main.yml +++ b/roles/redis/tasks/main.yml @@ -79,8 +79,7 @@ - name: Include tasks to install Redis using Remi repo (online) ansible.builtin.include_tasks: file: redis-using-remi-repo.yml - when: - - redis_install_method == "remi_repo" + when: redis_install_method == "remi_repo" - name: Include tasks to install Redis using source ansible.builtin.include_tasks: @@ -92,7 +91,7 @@ file: redis-offline.yml when: offline_install -- name: Ensure the directories for the config exists +- name: Ensure the configuration directory exists ansible.builtin.file: path: "{{ redis_conf_path }}" state: directory @@ -133,6 +132,14 @@ state: restarted daemon_reload: true +- name: Configure Redis Sentinel + tags: configure_redis_sentinel + when: redis_replication | bool + block: + - name: Include tasks to configure Sentinel + ansible.builtin.include_tasks: + file: configure-sentinel.yml + - name: Determine redis version ansible.builtin.shell: cmd: set -o pipefail && redis-server --version | cut -d" " -f3 | cut -d"=" -f2 diff --git a/roles/redis/tasks/redis-using-remi-repo.yml b/roles/redis/tasks/redis-using-remi-repo.yml index e78620d..aa0bfbe 100644 --- a/roles/redis/tasks/redis-using-remi-repo.yml +++ b/roles/redis/tasks/redis-using-remi-repo.yml @@ -3,7 +3,7 @@ --- - name: Install EPEL repo ansible.builtin.dnf: - name: "{{ epel_repo_url }}" + name: "{{ redis_epel_repo_url }}" state: present update_cache: true disable_gpg_check: true diff --git a/roles/redis/tasks/redis-using-source.yml b/roles/redis/tasks/redis-using-source.yml index 104a05b..d6dc6b4 100644 --- a/roles/redis/tasks/redis-using-source.yml +++ b/roles/redis/tasks/redis-using-source.yml @@ -88,11 +88,3 @@ owner: root group: root mode: "0644" - -- name: Create Redis Sentinel systemd file - ansible.builtin.template: - src: redis-sentinel.service.j2 - dest: /usr/lib/systemd/system/redis-sentinel.service - owner: root - group: root - mode: "0644" diff --git a/roles/redis/templates/redis.conf.j2 b/roles/redis/templates/redis.conf.j2 index bf9b6d0..3913e08 100644 --- a/roles/redis/templates/redis.conf.j2 +++ b/roles/redis/templates/redis.conf.j2 @@ -489,6 +489,13 @@ dir {{ redis_data_dir }} # and resynchronize with them. # # replicaof +{% if redis_replication %} +{% if groups['redis'] is defined and inventory_hostname in groups['redis'] and inventory_hostname not in groups['redis'][0] %} +replicaof {{ hostvars[groups['redis'][0]].inventory_hostname }} {{ redis_port }} +{% elif groups['redis_secondary'] is defined and inventory_hostname in groups['redis_secondary'] and inventory_hostname not in groups['redis_secondary'][0] %} +replicaof {{ hostvars[groups['redis_secondary'][0]].inventory_hostname }} {{ redis_port }} +{% endif %} +{% endif %} # If the master is password protected (using the "requirepass" configuration # directive below) it is possible to tell the replica to authenticate before @@ -507,6 +514,10 @@ dir {{ redis_data_dir }} # # When masteruser is specified, the replica will authenticate against its # master using the new AUTH form: AUTH . +{% if redis_auth and redis_replication %} +masterauth {{ redis_user_repluser_password }} +masteruser repluser +{% endif %} # When a replica loses its connection with the master, or when the replication # is still in progress, the replica can act in two different ways: @@ -889,17 +900,16 @@ replica-announce-port {{ redis_port }} # # For more information about ACL configuration please refer to # the Redis web site at https://redis.io/topics/acl -########################### -# Itential user definitions -# uncomment if using auth with Itential redis -########################### -# user default off -# user admin on allkeys allchannels allcommands >{{ redis_user_admin_password }} -# user itential on allkeys allchannels allcommands -asking -cluster -readonly -readwrite -bgrewriteaof -bgsave -failover -flushall -flushdb -psync -replconf -replicaof -save -shutdown -sync >{{ redis_user_itential_password }} -# user repluser on allchannels +psync +replconf +ping >{{ redis_user_repluser_password }} -# user sentineluser on allchannels +multi +slaveof +ping +exec +subscribe +config|rewrite +role +publish +info +client|setname +client|kill +script|kill >{{ redis_user_sentineluser_password }} -# user prometheus on -@all +@connection +memory -readonly +strlen +config|get +xinfo +pfcount -quit +zcard +type +xlen -readwrite -command +client -wait +scard +llen +hlen +get +eval +slowlog +cluster|info -hello -echo +info +latency +scan -reset -auth -asking >{{ redis_user_prometheus_password }} - +{% if redis_auth %} +user default off +user admin on allkeys allchannels allcommands >{{ redis_user_admin_password }} +user itential on allkeys allchannels allcommands -asking -cluster -readonly -readwrite -bgrewriteaof -bgsave -failover -flushall -flushdb -psync -replconf -replicaof -save -shutdown -sync >{{ redis_user_itential_password }} +user prometheus on -@all +@connection +memory -readonly +strlen +config|get +xinfo +pfcount -quit +zcard +type +xlen -readwrite -command +client -wait +scard +llen +hlen +get +eval +slowlog +cluster|info -hello -echo +info +latency +scan -reset -auth -asking >{{ redis_user_prometheus_password }} +{% if redis_replication %} +user repluser on allchannels +psync +replconf +ping >{{ redis_user_repluser_password }} +user sentineluser on allchannels +multi +slaveof +ping +exec +subscribe +config|rewrite +role +publish +info +client|setname +client|kill +script|kill >{{ redis_user_sentineluser_password }} +{% endif %} +{% endif %} # ACL LOG # diff --git a/roles/redis_replication/templates/sentinel.conf.j2 b/roles/redis/templates/sentinel.conf.j2 similarity index 96% rename from roles/redis_replication/templates/sentinel.conf.j2 rename to roles/redis/templates/sentinel.conf.j2 index c55bb81..f6aca0a 100644 --- a/roles/redis_replication/templates/sentinel.conf.j2 +++ b/roles/redis/templates/sentinel.conf.j2 @@ -35,8 +35,6 @@ pidfile {{ redis_pid_dir }}/redis-sentinel.pid # output for logging but daemonize, logs will be sent to /dev/null logfile {{ redis_log_dir }}/sentinel.log -sentinel announce-ip {{ inventory_hostname }} -sentinel announce-port {{ redis_sentinel_port }} # # The above two configuration directives are useful in environments where, # because of NAT, Sentinel is reachable from outside via a non-local address. @@ -56,6 +54,8 @@ sentinel announce-port {{ redis_sentinel_port }} # Example: # # sentinel announce-ip 1.2.3.4 +sentinel announce-ip {{ inventory_hostname }} +sentinel announce-port {{ redis_sentinel_port }} # dir # Every long running process should have a well-defined working directory. @@ -83,9 +83,7 @@ dir /tmp # The valid charset is A-z 0-9 and the three characters ".-_". sentinel monitor mymaster {{ master_name }} {{ redis_port }} 2 -{% if redis_auth %} -sentinel auth-pass mymaster {{ redis_user_sentineluser_password }} -{% endif%} + # sentinel auth-pass # # Set the password to use to authenticate with the master and replicas. @@ -104,10 +102,10 @@ sentinel auth-pass mymaster {{ redis_user_sentineluser_password }} # Example: # # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd - {% if redis_auth %} -sentinel auth-user mymaster sentineluser -{% endif %}# +sentinel auth-pass mymaster {{ redis_user_sentineluser_password }} +{% endif %} + # sentinel auth-user # # This is useful in order to authenticate to instances having ACL capabilities, @@ -119,13 +117,13 @@ sentinel auth-user mymaster sentineluser # # user sentinel-user >somepassword +client +subscribe +publish \ # +ping +info +multi +slaveof +config +client +exec on -########################### -# Itential user definitions -# uncomment if using auth with Itential redis -########################### -# user default off -# user admin on >{{ redis_user_sentineladmin_password }} ~* &* +@all -# user sentineluser on >{{ redis_user_sentineluser_password }} &* -@all +auth +client|getname +client|id +client|setname +command +hello +ping +role +sentinel|is-master-down-by-addr +sentinel|get-master-addr-by-name +sentinel|master +sentinel|myid +sentinel|replicas +sentinel|sentinels +{% if redis_auth %} +sentinel auth-user mymaster sentineluser + +user default off +user admin on >{{ redis_user_sentineladmin_password }} ~* &* +@all +user sentineluser on >{{ redis_user_sentineluser_password }} &* -@all +auth +client|getname +client|id +client|setname +command +hello +ping +role +sentinel|is-master-down-by-addr +sentinel|get-master-addr-by-name +sentinel|master +sentinel|myid +sentinel|replicas +sentinel|sentinels +{% endif %} # sentinel down-after-milliseconds # @@ -193,18 +191,16 @@ acllog-max-len 128 # # The requirepass is not compatable with aclfile option and the ACL LOAD # command, these will cause requirepass to be ignored. - -{% if redis_auth %} -sentinel sentinel-user sentineluser -{% endif %} +# # sentinel sentinel-user # # You can configure Sentinel to authenticate with other Sentinels with specific # user name. - {% if redis_auth %} +sentinel sentinel-user sentineluser sentinel sentinel-pass {{ redis_user_sentineluser_password }} {% endif %} + # sentinel sentinel-pass # # The password for Sentinel to authenticate with other Sentinels. If sentinel-user diff --git a/roles/redis_auth/README.md b/roles/redis_auth/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/roles/redis_auth/tasks/main.yml b/roles/redis_auth/tasks/main.yml deleted file mode 100644 index 114f871..0000000 --- a/roles/redis_auth/tasks/main.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2024, Itential, Inc -# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) ---- -- name: Update Redis configuration - notify: Restart Redis - block: - # The authorization users are already included in the redis.conf file from the - # role that installed redis. This task is simply uncommenting those lines to - # enable/disable the necessary users. - - name: Disable "default" user, enable other users - ansible.builtin.lineinfile: - path: "{{ redis_conf_file }}" - regexp: ^#\s({{ item }}.*)$ - line: \1 - backrefs: true - with_items: - - user default - - user admin - - user itential - - user repluser - - user sentineluser - - user prometheus - - # When auth is turned on its required to use a specially designed user who - # has the required permissions to perform the replication. - - name: Modify redis.conf to add replication user (if required) - ansible.builtin.lineinfile: - path: "{{ redis_conf_file }}" - regexp: "# masteruser " - line: "masteruser repluser" - when: redis_replication | bool - - - name: Modify redis.conf to add replication user's password (if required) - ansible.builtin.lineinfile: - path: "{{ redis_conf_file }}" - regexp: "# masterauth " - line: "masterauth {{ redis_user_repluser_password }}" - when: redis_replication | bool - - # Starting in Redis 7.0 the default user was being automatically inserted - # into the redis.conf when missing. This will remove that extra line - # when auth is enabled to avoid a configuration error that will prevent - # Redis from starting. - - name: Modify redis.conf and remove any extra default user lines - ansible.builtin.lineinfile: - path: "{{ redis_conf_file }}" - regexp: "user default on nopass.+$" - state: absent - when: redis_replication | bool - -- name: Update Redis Sentinel configuration - notify: Restart Sentinel - block: - # Starting in Redis 7.0 the default user was being automatically inserted - # into the sentinel.conf when missing. This will remove that extra line - # when auth is enabled to avoid a configuration error that will prevent - # Redis from starting. - - name: Modify sentinel.conf and remove any extra default user lines - ansible.builtin.lineinfile: - path: "{{ redis_sentinel_conf_file }}" - regexp: "user default on nopass.+$" - state: absent - when: redis_replication | bool - - # The authorization users are already included in the sentinel.conf file from - # the role that installed redis. This task is simply uncommenting those lines - # to enable/disable the necessary sentinel users. - - name: Disable "default" user, enable other sentinel users - ansible.builtin.lineinfile: - path: "{{ redis_sentinel_conf_file }}" - regexp: ^#\s({{ item }}.*)$ - line: '\1' - backrefs: true - with_items: - - user default - - user admin - - user sentineluser - when: redis_replication | bool diff --git a/roles/redis_replication/README.md b/roles/redis_replication/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/roles/redis_replication/tasks/main.yml b/roles/redis_replication/tasks/main.yml deleted file mode 100644 index 926d89f..0000000 --- a/roles/redis_replication/tasks/main.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2024, Itential, Inc -# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) ---- -- name: Configure Redis clustering - notify: - - Restart Redis - - Restart Sentinel - block: - - name: Use template to generate sentinel.conf - ansible.builtin.template: - src: sentinel.conf.j2 - dest: "{{ redis_sentinel_conf_file }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0640" - backup: true - when: - - groups['redis'] is defined - - inventory_hostname in groups['redis'] - vars: - master_name: "{{ hostvars[groups['redis'][0]].inventory_hostname }}" - - - name: Use template to generate sentinel.conf for secondary DR - ansible.builtin.template: - src: sentinel.conf.j2 - dest: "{{ redis_sentinel_conf_file }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0640" - backup: true - when: - - groups['redis_secondary'] is defined - - inventory_hostname in groups['redis_secondary'] - vars: - master_name: "{{ hostvars[groups['redis_secondary'][0]].inventory_hostname }}" - - # Only the replicas should have this "replicaof" config line. The first host - # defined in the hosts file is chosen as the Redis primary, so, this should - # only happen on the others, not the first. - - name: Update replicaof line for all replicas redis.conf files - ansible.builtin.lineinfile: - path: "{{ redis_conf_file }}" - regexp: "# replicaof " - line: "replicaof {{ hostvars[groups['redis'][0]].inventory_hostname }} {{ redis_port }}" - when: - - groups['redis'] is defined - - inventory_hostname in groups['redis'] - - inventory_hostname not in groups['redis'][0] - - - name: Update replicaof line for all replicas redis.conf files for secondary DR - ansible.builtin.lineinfile: - path: "{{ redis_conf_file }}" - regexp: "# replicaof " - line: "replicaof {{ hostvars[groups['redis_secondary'][0]].inventory_hostname }} {{ redis_port }}" - when: - - groups['redis_secondary'] is defined - - inventory_hostname in groups['redis_secondary'] - - inventory_hostname not in groups['redis_secondary'][0] - - # Check if firewalld is running, if it is then open the appropriate ports - - name: Gather service facts - ansible.builtin.service_facts: - - - name: Open Port on FirewallD Public Zone - ansible.posix.firewalld: - port: "{{ redis_sentinel_port }}/tcp" - permanent: true - state: enabled - zone: public - immediate: true - when: - - ansible_facts.services["firewalld.service"] is defined - - ansible_facts.services["firewalld.service"].state == "running" - - ansible_facts.services["firewalld.service"].status == "enabled" From 1b9dc6e4e16a4b4cb65d7a3e417f50d0c3c4193b Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Thu, 7 Nov 2024 00:07:38 -0700 Subject: [PATCH 2/4] Use handlers in redis role Separate out basic configuration tasks in redis role Fix issue with determining the Redis version when installing from source --- roles/redis/handlers/main.yml | 8 +- roles/redis/tasks/basic-configuration.yml | 79 +++++++++ roles/redis/tasks/configure-sentinel.yml | 93 +++++------ roles/redis/tasks/main.yml | 186 +++++++--------------- 4 files changed, 181 insertions(+), 185 deletions(-) create mode 100644 roles/redis/tasks/basic-configuration.yml diff --git a/roles/redis/handlers/main.yml b/roles/redis/handlers/main.yml index 0101def..6262baf 100644 --- a/roles/redis/handlers/main.yml +++ b/roles/redis/handlers/main.yml @@ -1,14 +1,18 @@ # Copyright (c) 2024, Itential, Inc # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) --- -- name: Restart Redis +- name: Enable and Start Redis + throttle: 1 ansible.builtin.systemd: name: redis enabled: true state: restarted + daemon_reload: true -- name: Restart Sentinel +- name: Enable and Start Redis Sentinel + throttle: 1 ansible.builtin.systemd: name: redis-sentinel enabled: true state: restarted + daemon_reload: true diff --git a/roles/redis/tasks/basic-configuration.yml b/roles/redis/tasks/basic-configuration.yml new file mode 100644 index 0000000..d6f8972 --- /dev/null +++ b/roles/redis/tasks/basic-configuration.yml @@ -0,0 +1,79 @@ +# Copyright (c) 2024, Itential, Inc +# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) +--- +# Kernel Adjust +# Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. +# Being disabled, it can can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328 +- name: Adjust Memory Overcommit + ansible.posix.sysctl: + name: vm.overcommit_memory + value: 1 + +- name: Install custom SELinux profiles + ansible.builtin.include_role: + name: selinux + tags: configure_selinux + +# Check if firewalld is running, if it is then open the appropriate ports +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Open Ports on FirewallD Public Zone + ansible.posix.firewalld: + port: "{{ item }}" + permanent: true + state: enabled + zone: public + immediate: true + loop: + - "{{ redis_port }}/tcp" + - "{{ redis_replication | bool | ternary(rabbitmq_mgt_console_port ~ '/tcp', omit) }}" + when: + - ansible_facts.services["firewalld.service"] is defined + - ansible_facts.services["firewalld.service"].state == "running" + - ansible_facts.services["firewalld.service"].status == "enabled" + +- name: Create Redis group + ansible.builtin.group: + name: "{{ redis_group }}" + +- name: Create Redis user + ansible.builtin.user: + name: "{{ redis_owner }}" + group: "{{ redis_group }}" + state: present + +- name: Create Redis data directory + ansible.builtin.file: + state: directory + path: "{{ redis_data_dir }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0755" + when: redis_data_dir != "/var/lib/redis" or redis_install_method == "source" + +- name: Create Redis log directory + ansible.builtin.file: + state: directory + path: "{{ redis_log_dir }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0755" + when: redis_log_dir != "/var/log/redis" or redis_install_method == "source" + +- name: Create Redis pid directory + ansible.builtin.file: + state: directory + path: "{{ redis_pid_dir }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0755" + when: redis_pid_dir != "/var/run" or redis_install_method == "source" + +- name: Create Redis configuration directory + ansible.builtin.file: + path: "{{ redis_conf_path }}" + state: directory + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0760" diff --git a/roles/redis/tasks/configure-sentinel.yml b/roles/redis/tasks/configure-sentinel.yml index 2830678..0c2180c 100644 --- a/roles/redis/tasks/configure-sentinel.yml +++ b/roles/redis/tasks/configure-sentinel.yml @@ -1,62 +1,41 @@ # Copyright (c) 2024, Itential, Inc # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) --- -- name: Create Redis Sentinel systemd file - ansible.builtin.template: - src: redis-sentinel.service.j2 - dest: /usr/lib/systemd/system/redis-sentinel.service - owner: root - group: root - mode: "0644" +- name: Configure Sentinel + notify: Enable and Start Redis Sentinel + block: + - name: Create Redis Sentinel systemd file + ansible.builtin.template: + src: redis-sentinel.service.j2 + dest: /usr/lib/systemd/system/redis-sentinel.service + owner: root + group: root + mode: "0644" -- name: Use template to generate sentinel.conf - ansible.builtin.template: - src: sentinel.conf.j2 - dest: "{{ redis_sentinel_conf_file }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0640" - backup: true - when: - - groups['redis'] is defined - - inventory_hostname in groups['redis'] - vars: - master_name: "{{ hostvars[groups['redis'][0]].inventory_hostname }}" + - name: Use template to generate sentinel.conf + ansible.builtin.template: + src: sentinel.conf.j2 + dest: "{{ redis_sentinel_conf_file }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0640" + backup: true + when: + - groups['redis'] is defined + - inventory_hostname in groups['redis'] + vars: + master_name: "{{ hostvars[groups['redis'][0]].inventory_hostname }}" -- name: Use template to generate sentinel.conf for secondary DR - ansible.builtin.template: - src: sentinel.conf.j2 - dest: "{{ redis_sentinel_conf_file }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0640" - backup: true - when: - - groups['redis_secondary'] is defined - - inventory_hostname in groups['redis_secondary'] - vars: - master_name: "{{ hostvars[groups['redis_secondary'][0]].inventory_hostname }}" - -# Check if firewalld is running, if it is then open the appropriate ports -- name: Gather service facts - ansible.builtin.service_facts: - -- name: Open Port on FirewallD Public Zone - ansible.posix.firewalld: - port: "{{ redis_sentinel_port }}/tcp" - permanent: true - state: enabled - zone: public - immediate: true - when: - - ansible_facts.services["firewalld.service"] is defined - - ansible_facts.services["firewalld.service"].state == "running" - - ansible_facts.services["firewalld.service"].status == "enabled" - -- name: Enable Redis Sentinel - throttle: 1 - ansible.builtin.systemd: - name: redis-sentinel - enabled: true - state: restarted - daemon_reload: true + - name: Use template to generate sentinel.conf for secondary DR + ansible.builtin.template: + src: sentinel.conf.j2 + dest: "{{ redis_sentinel_conf_file }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0640" + backup: true + when: + - groups['redis_secondary'] is defined + - inventory_hostname in groups['redis_secondary'] + vars: + master_name: "{{ hostvars[groups['redis_secondary'][0]].inventory_hostname }}" diff --git a/roles/redis/tasks/main.yml b/roles/redis/tasks/main.yml index 1a2648a..9521087 100644 --- a/roles/redis/tasks/main.yml +++ b/roles/redis/tasks/main.yml @@ -7,130 +7,59 @@ with_first_found: - "release-{{ iap_release }}.yml" - "release-undefined.yml" + tags: always - name: Check for valid Redis release ansible.builtin.fail: msg: "Deployer does not support installing Redis on IAP version {{ iap_release }}" when: invalid_redis_release is defined + tags: always - name: Install base OS packages ansible.builtin.include_role: name: os tags: install_base_os_packages -# Kernel Adjust -# Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. -# Being disabled, it can can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328 -- name: Adjust Memory Overcommit - ansible.posix.sysctl: - name: vm.overcommit_memory - value: 1 - -- name: Configure SELinux - tags: configure_selinux - block: - - name: Install custom SELinux profiles - ansible.builtin.include_role: - name: selinux - -- name: Create Redis group - ansible.builtin.group: - name: "{{ redis_group }}" - -- name: Create Redis user - ansible.builtin.user: - name: "{{ redis_owner }}" - group: "{{ redis_group }}" - state: present - -- name: Create Redis data directory - ansible.builtin.file: - state: directory - path: "{{ redis_data_dir }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0755" - when: redis_data_dir != "/var/lib/redis" or redis_install_method == "source" - -- name: Create Redis log directory - ansible.builtin.file: - state: directory - path: "{{ redis_log_dir }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0755" - when: redis_log_dir != "/var/log/redis" or redis_install_method == "source" - -- name: Create Redis pid directory - ansible.builtin.file: - state: directory - path: "{{ redis_pid_dir }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0755" - when: redis_pid_dir != "/var/run" or redis_install_method == "source" - -- name: Install Redis - tags: install_redis +- name: Install and configure Redis + notify: Enable and Start Redis + tags: install_configure_redis block: - - name: Include tasks to install Redis (online) - when: not offline_install + - name: Basic configuration + tags: basic_configuration block: - - name: Include tasks to install Redis using Remi repo (online) + - name: Include tasks to perform basic configuration ansible.builtin.include_tasks: - file: redis-using-remi-repo.yml - when: redis_install_method == "remi_repo" + file: basic-configuration.yml - - name: Include tasks to install Redis using source + - name: Install Redis + tags: install_redis + block: + - name: Include tasks to install Redis (online) + when: not offline_install + block: + - name: Include tasks to install Redis using Remi repo (online) + ansible.builtin.include_tasks: + file: redis-using-remi-repo.yml + when: redis_install_method == "remi_repo" + + - name: Include tasks to install Redis using source + ansible.builtin.include_tasks: + file: redis-using-source.yml + when: redis_install_method == "source" + + - name: Include tasks to install Redis (offline) ansible.builtin.include_tasks: - file: redis-using-source.yml - when: redis_install_method == "source" - - - name: Include tasks to install Redis (offline) - ansible.builtin.include_tasks: - file: redis-offline.yml - when: offline_install - -- name: Ensure the configuration directory exists - ansible.builtin.file: - path: "{{ redis_conf_path }}" - state: directory - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0760" - -- name: Use template to generate redis.conf - ansible.builtin.template: - src: redis.conf.j2 - dest: "{{ redis_conf_file }}" - owner: "{{ redis_owner }}" - group: "{{ redis_group }}" - mode: "0640" - backup: true - -# Check if firewalld is running, if it is then open the appropriate ports -- name: Gather service facts - ansible.builtin.service_facts: - -- name: Open Port on FirewallD Public Zone - ansible.posix.firewalld: - port: "{{ redis_port }}/tcp" - permanent: true - state: enabled - zone: public - immediate: true - when: - - ansible_facts.services["firewalld.service"] is defined - - ansible_facts.services["firewalld.service"].state == "running" - - ansible_facts.services["firewalld.service"].status == "enabled" - -- name: Enable redis - throttle: 1 - ansible.builtin.systemd: - name: redis - enabled: true - state: restarted - daemon_reload: true + file: redis-offline.yml + when: offline_install + + - name: Use template to generate redis.conf + ansible.builtin.template: + src: redis.conf.j2 + dest: "{{ redis_conf_file }}" + owner: "{{ redis_owner }}" + group: "{{ redis_group }}" + mode: "0640" + backup: true - name: Configure Redis Sentinel tags: configure_redis_sentinel @@ -140,21 +69,26 @@ ansible.builtin.include_tasks: file: configure-sentinel.yml -- name: Determine redis version - ansible.builtin.shell: - cmd: set -o pipefail && redis-server --version | cut -d" " -f3 | cut -d"=" -f2 - register: result - check_mode: false - changed_when: false - failed_when: result.rc != 0 and result.rc != 127 - -- name: Set redis version variable - ansible.builtin.set_fact: - redis_server_version: "{{ result.stdout }}" - -- name: Write redis release information - ansible.builtin.lineinfile: - path: "{{ itential_release_file }}" - line: "REDIS={{ redis_server_version }}" - create: true - mode: "0644" +- name: Update Itential release file + tags: update_release_file + block: + - name: Determine redis version + ansible.builtin.shell: + cmd: 'set -o pipefail && redis-server --version | cut -d" " -f3 | cut -d"=" -f2' + register: result + check_mode: false + changed_when: false + failed_when: result.rc != 0 + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin" + + - name: Set redis version variable + ansible.builtin.set_fact: + redis_server_version: "{{ result.stdout }}" + + - name: Write redis release information + ansible.builtin.lineinfile: + path: "{{ itential_release_file }}" + line: "REDIS={{ redis_server_version }}" + create: true + mode: "0644" From f55836949961162025ab89cc91bc63e2c9d74f16 Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Thu, 7 Nov 2024 15:09:17 -0700 Subject: [PATCH 3/4] Update Redis readme --- docs/redis_guide.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/redis_guide.md b/docs/redis_guide.md index 5ca1a28..9691d67 100644 --- a/docs/redis_guide.md +++ b/docs/redis_guide.md @@ -1,16 +1,16 @@ # Overview -The playbook and roles in this section install and configure Redis for the Itential Automation Platform. There is one Redis-related role: +The playbook and role in this section install and configure Redis for the Itential Automation Platform. There is one Redis-related role: * `redis` – Installs Redis and performs a base configuration. Optionally configures authentication and replication. -# Roles +# Redis Role -## Redis Role +## Base Install The `redis` role performs a base install of Redis including any OS packages required. It will compile and install any custom SELinux profiles. It creates the appropriate Linux users, directories, log files, and systemd services. It uses a template to generate a configuration file with some potential features available in other roles commented out. It will start the Redis service when complete. -### Authentication +## Authentication Optionally, the `redis` role performs tasks to require authentication (username and password) when communicating with the Redis server. It adjusts the Redis config file and adds each of the required users and applies appropriate ACLs (see table). The "default" Redis user is disabled. It modifies the Redis config file to use the appropriate user while doing replication. It adjusts the Sentinel config file to enable the correct Sentinel user to monitor the redis cluster, if required. It disables the default user in both Redis and Redis Sentinel. @@ -26,7 +26,7 @@ More info on Redis authorization: https://redis.io/docs/manual/security/ :::(Warning) (⚠ Warning: ) It is assumed that these default passwords will be changed to meet more rigorous standards. These are intended to be defaults strictly used just for ease of the installation. It is highly recommended that sensitive data be encrypted using Ansible Vault. -### Replication +## Replication Optionally, the `redis` role performs the steps required to create a Redis replica set. It uses a template to generate a Redis Sentinel config file. It modifies the Redis config file to turn off protected-mode. It assumes that the first host defined in the inventory file is the initial primary. It will update the config file for the non-primary Redis servers to replicate from the primary using hostname. It will start Redis Sentinel when complete. @@ -149,7 +149,7 @@ all: # Running the Playbook -To execute all Redis roles, run the `redis` playbook: +To execute the Redis role, run the `redis` playbook: ``` ansible-playbook itential.deployer.redis -i From d161f7807f77b103500777c4e306d53fd6df56a4 Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Thu, 7 Nov 2024 15:25:10 -0700 Subject: [PATCH 4/4] Remove redis_replication and redis_auth references --- playbooks/install_active_standby.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/playbooks/install_active_standby.yml b/playbooks/install_active_standby.yml index 18a4c66..547fe53 100644 --- a/playbooks/install_active_standby.yml +++ b/playbooks/install_active_standby.yml @@ -6,27 +6,10 @@ become: true roles: ### REDIS - # Perform a base installation of Redis + # Perform a base installation of Redis and optionally configure authorization and replication - role: itential.deployer.redis tags: - redis - - redis_install - - # Perform installation of Redis Sentinel for Redis HA and replication - # https://redis.io/docs/manual/sentinel/ - - role: itential.deployer.redis_replication - when: redis_replication | bool - tags: - - redis - - redis_replication - - # Configure Redis to require a username & password (authorization) - # https://redis.io/docs/manual/security/acl/ - - role: itential.deployer.redis_auth - when: redis_auth | bool - tags: - - redis - - redis_auth - name: Install RabbitMQ hosts: rabbitmq, rabbitmq_secondary