Configuration for cross-DC clusters

roles/infinispan/README.md

@@ -8,11 +8,17 @@ Role Defaults
 -------------
 
 | Variable | Description | Default |
-|:---------|:------------|:---------|
+|:---------|:------------|:--------|
 |`infinispan_keycloak_caches`| Creates remote caches for keycloak | `False` |
-|`override_jdg_port`| Alternate port for the service | `11222` |
-|`override_jdg_jgroups_port`| Alternate port for the jgroups cluster | `7800` |
-|`override_jdg_bind_addr`| Alternate bind address for the daemon | `localhost` |
+|`jdg_port`| Alternate port for the service | `11222` |
+|`jdg_jgroups_port`| Alternate port for the jgroups cluster | `7800` |
+|`jdg_jgroups_relay_port`| Alternate port for the jgroups relaying cluster | `7801` |
+|`jdg_bind_addr`| Alternate bind address for the daemon | `localhost` |
+|`jdg_jgroups_relay`| Enable cross-DC relaying | `False` |
+|`jdg_jgroups_relay_sites`| List of site names for cross-DC relaying | `[]` |
+|`jdg_jgroups_relay_site`| Site the inventory host is in when cross-DC is enabled | `''` |
+|`jdg_jgroups_jdbcping`| Enable clustering using JDBC PING discovery | `False` |
+
 
 
 Role Variables
@@ -33,6 +39,17 @@ infinispan_users:
   - { name: 'testuser2', password: 'test', roles: 'application' }
 ```
 
+The following are required when `jdg_jgroups_jdbcping` is enabled:
+
+| Variable | Description | Default |
+|:---------|:------------|:--------|
+|`mariadb_jdbc_url`| URL for connecting to database | `jdbc:mariadb://localhost:3306/keycloak` |
+|`mariadb_db_user`| Username for connecting to database | `keycloak-user` |
+|`mariadb_db_pass`| Password for connecting to database | `keycloak-pass` |
+
+When setting up cross-DC relaying, remember to also setup mariadb in active-active mode (ie. with galera cluster), and switch the JDBC to url to the `sequential` scheme.
+
+
 Dependencies
 ------------
 

roles/infinispan/defaults/main.yml

@@ -15,53 +15,22 @@ infinispan_installation_path: "/opt/infinispan-server-{{ infinispan_version }}/"
 jdg_app_download_dir: "/opt/apps"
 jdg_enable: "{{ True if jdg_rhn_id is defined else False }}"
 jdg_healthcheck: True
-jdg:
-  home: "{{ jdg_default_installation_path if jdg_rhn_id is defined else infinispan_installation_path }}"
-  version: "{{ jdg_version if jdg_rhn_id is defined else infinispan_version }}"
-  bind_addr: "{{ override_jdg_bind_addr | default('localhost') }}"
-  port: "{{ override_jdg_port | default('11222') }}"
-  jgroups_bind_addr: "{{ override_jdg_jgroups_bind_addr | default(ansible_all_ipv4_addresses[0]) }}"
-  jgroups_port: "{{ override_jdg_jgroups_port | default('7800') }}"
-  config:
-    name: "{{ override_jdg_config | default('jdg.xml') }}"
-    template: "{{ override_jdg_config_template | default('templates/infinispan.xml.j2') }}"
-    jgroups_template: "{{ override_jdg_config_jgroups_template | default('templates/jgroups-jdbcping.xml.j2') }}"
-    users: "{{ override_jdg_config_users_properties | default('users.properties') }}"
-    groups: "{{ override_jdg_config_group_properties | default('groups.properties') }}"
-  user:
-    name: "{{ override_jdg_user | default('jdg') }}"
-  group:
-    name: "{{ override_jdg_group | default('jdg') }}"
-  service:
-    name: "{{ override_jdg_service_name | default('jdg' if jdg_rhn_id is defined else 'infinispan') }}"
-  users: "{{ [ jdg_supervisor ] + infinispan_users }}"
-
-jdg_supervisor:
-  name: 'supervisor'
-  password: "{{ supervisor_password }}"
-  roles: 'admin'
+jdg_bind_address: localhost
+jdg_port: 11222
+jdg_jgroups_port: 7800
+jdg_jgroups_relay_port: 7801
 
+# flag to enable protocol encryption
 jdg_default_realm_tls: False
 
-jdg_jgroups_jdbcping: "{{ True if infinispan_keycloak_caches else False }}"
-jdg_jgroups_jdbc:
-  mariadb:
-    enabled: "{{ jdg_jgroups_jdbcping }}"
-    driver_class: org.mariadb.jdbc.Driver
-    driver_jar_filename: "mariadb-java-client-2.7.4.jar"
-    driver_jar_url: "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.7.4/mariadb-java-client-2.7.4.jar"
-    connection_url: "{{ mariadb_jdbc_url | default('jdbc:mariadb://localhost:3306/keycloak') }}"
-    db_user: "{{ mariadb_db_user | default('keycloak-user') }}"
-    db_password: "{{ mariadb_db_pass | default('keycloak-pass') }}"
+# flag to enable clustering on jdbc discovery
+jdg_jgroups_jdbcping: "{{ True if infinispan_keycloak_caches or jdg_jgroups_relay else False }}"
+# flag to enable cross-DC relay
+jdg_jgroups_relay: False
+# list of sites (clusters) in the cross-DC global view
+jdg_jgroups_relay_sites: []
+# site (cluster) this node belongs to
+jdg_jgroups_relay_site: ''
 
+# flag to enable keycloak integration
 infinispan_keycloak_caches: False
-jdg_keycloak_cache:
-  enabled: "{{ infinispan_keycloak_caches }}"
-  caches:
-    - sessions
-    - offlineSessions
-    - clientSessions
-    - offlineClientSessions
-    - loginFailures
-    - actionTokens
-    - work

roles/infinispan/tasks/firewalld.yml

@@ -22,3 +22,4 @@
   loop:
     - "{{ jdg.port }}/tcp"
     - "{{ jdg.jgroups_port }}/tcp"
+    - "{{ jdg.jgroups_relay_port }}/tcp"

roles/infinispan/tasks/main.yml

@@ -55,21 +55,6 @@
   notify:
     - restart infinispan
 
-- name: "Ensures {{ jdg.service.name }} jgroups configuration is deployed"
-  template:
-    src: "{{ jdg.config.jgroups_template }}"
-    dest: "{{ jdg.home }}/server/conf/jgroups-jdbcping.xml"
-    owner: "{{ jdg.user.name }}"
-    group: "{{ jdg.group.name }}"
-    mode: 0644
-  when:
-    - jdg.config is defined
-    - jdg.config.name is defined
-    - jdg.service.name is defined
-    - jdg_jgroups_jdbcping
-  notify:
-    - restart infinispan
-
 - name: Download mariadb driver jar to target
   get_url:
     url: "{{ jdg_jgroups_jdbc.mariadb.driver_jar_url }}"

roles/infinispan/templates/groups.properties.j2

@@ -1,4 +1,4 @@
-#
+# {{ ansible_managed }}
 # Properties declaration of users roles for the realm 'default' which is the default realm.
 #
 # Users can be added to this properties file at any time, updates after the server has started

roles/infinispan/templates/infinispan.service.j2

@@ -1,11 +1,12 @@
+# {{ ansible_managed }}
 # This script allows you to run Infinispan Server as a systemd service.
 # Modify environment properties in this script as appropriate.
 # Copy this script to the following location: /etc/systemd/system
 # Activate with 'systemctl daemon-reload'
 #               'systemctl start|enable {{ jdg.service.name }}'
 
 [Unit]
-Description=Infinispan Server Service
+Description={{ jdg.service.name }} service
 After=network.target
 
 [Service]

roles/infinispan/templates/infinispan.xml.j2

@@ -1,38 +1,106 @@
+<!-- {{ ansible_managed }} -->
 <infinispan
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="urn:infinispan:config:12.1 https://infinispan.org/schemas/infinispan-config-12.1.xsd
                             urn:infinispan:server:12.1 https://infinispan.org/schemas/infinispan-server-12.1.xsd"
         xmlns="urn:infinispan:config:12.1"
         xmlns:server="urn:infinispan:server:12.1">
 
+    <jgroups>
 {% if jdg_keycloak_cache.enabled or jdg_jgroups_jdbcping %}
-   <jgroups>
-      <stack-file name="datagrid-jdbcping" path="jgroups-jdbcping.xml"/>
-   </jgroups>
+        <!-- TCP local cluster with JDBCPING discovery -->
+        <stack name="datagrid" extends="tcp">
+            <JDBC_PING  stack.combine="REPLACE" stack.position="MPING"
+                connection_url="{{ jdg_jgroups_jdbc.mariadb.connection_url }}"
+                connection_username="{{ jdg_jgroups_jdbc.mariadb.db_user }}"
+                connection_password="{{ jdg_jgroups_jdbc.mariadb.db_password }}"
+                connection_driver="{{ jdg_jgroups_jdbc.mariadb.driver_class }}"
+                initialize_sql="CREATE TABLE IF NOT EXISTS JGROUPSPING (
+                        own_addr varchar(200) NOT NULL,
+                        cluster_name varchar(200) NOT NULL,
+                        updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+                        ping_data varbinary(5000) DEFAULT NULL,
+                        PRIMARY KEY (own_addr, cluster_name))
+                        ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
+                insert_single_sql="INSERT INTO JGROUPSPING (own_addr, cluster_name, ping_data) values (?, ?, ?)"
+                delete_single_sql="DELETE FROM JGROUPSPING WHERE own_addr=? AND cluster_name=?"
+                select_all_pingdata_sql="SELECT ping_data FROM JGROUPSPING WHERE cluster_name=?" />
+            <FD_SOCK stack.combine="REMOVE"/>
+            <pbcast.GMS join_timeout="30000" />
+            <RSVP timeout="60000" resend_interval="500" ack_on_delivery="true" />
+        </stack>
 {% endif %}
+{% if jdg_jgroups_relay and jdg_jgroups_jdbcping  %}
+        <!-- TCP RELAY with JDBCPING discovery -->
+        <stack name="datagridrelay" extends="tcp">
+            <JDBC_PING  stack.combine="REPLACE" stack.position="MPING"
+                connection_url="{{ jdg_jgroups_jdbc.mariadb.connection_url }}"
+                connection_username="{{ jdg_jgroups_jdbc.mariadb.db_user }}"
+                connection_password="{{ jdg_jgroups_jdbc.mariadb.db_password }}"
+                connection_driver="{{ jdg_jgroups_jdbc.mariadb.driver_class }}"
+                initialize_sql="CREATE TABLE IF NOT EXISTS JGROUPSPING (
+                        own_addr varchar(200) NOT NULL,
+                        cluster_name varchar(200) NOT NULL,
+                        updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+                        ping_data varbinary(5000) DEFAULT NULL,
+                        PRIMARY KEY (own_addr, cluster_name))
+                        ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
+                insert_single_sql="INSERT INTO JGROUPSPING (own_addr, cluster_name, ping_data) values (?, ?, ?)"
+                delete_single_sql="DELETE FROM JGROUPSPING WHERE own_addr=? AND cluster_name=?"
+                select_all_pingdata_sql="SELECT ping_data FROM JGROUPSPING WHERE cluster_name=?" />
+        </stack>
+        <!-- x-site declaration -->
+        <stack name="xsite" extends="datagrid">
+            <relay.RELAY2 site="{{ jdg_jgroups_relay_site }}" xmlns="urn:org:jgroups" max_site_masters="1000" can_become_site_master="true"
+                can_forward_local_cluster="true" relay_multicasts="false" />
+            <FORWARD_TO_COORD />
+            <remote-sites default-stack="datagridrelay">
+{% for site in jdg_jgroups_relay_sites %}
+                <remote-site name="{{ site }}"/>
+{% endfor %}
+            </remote-sites>
+        </stack>
+{% endif %}
+    </jgroups>
 
     <cache-container name="default" statistics="true">
-{% if jdg_keycloak_cache.enabled or jdg_jgroups_jdbcping %}
-      <transport cluster="${infinispan.cluster.name:datagrid-cluster}" stack="${infinispan.cluster.stack:datagrid-jdbcping}" node-name="${infinispan.node.name:}"/>
+{% if jdg_jgroups_relay and jdg_jgroups_jdbcping %}
+      <transport cluster="{{ jdg_jgroups_relay_site }}" stack="xsite" node-name="{{ inventory_hostname }}"  machine="{{ inventory_hostname }}" site="{{ jdg_jgroups_relay_site }}"/>
+{% else if jdg_keycloak_cache.enabled or jdg_jgroups_jdbcping %}
+      <transport cluster="datagrid" stack="datagrid" node-name="{{ inventory_hostname }}" machine="{{ inventory_hostname }}"/>
 {% else %}
-      <transport cluster="${infinispan.cluster.name:datagrid-cluster}" stack="${infinispan.cluster.stack:tcp}" node-name="${infinispan.node.name:}"/>
+      <transport cluster="datagrid" stack="tcp" node-name="{{ inventory_hostname }}"/>
 {% endif %}
       <security>
         <authorization>
-          <role name="supervisor" permissions="READ WRITE EXEC CREATE"/>
+          <role name="{{ jdg_supervisor.name }}" permissions="READ WRITE EXEC CREATE"/>
 	    </authorization>
       </security>
     </cache-container>
 
 {% if jdg_keycloak_cache.enabled %}
     <!-- keycloak caches -->
     <cache-container name="keycloak" statistics="true">
-      <transport cluster="${infinispan.cluster.name:datagrid-cluster}" stack="${infinispan.cluster.stack:tcp}" node-name="${infinispan.node.name:}"/>
+{% if jdg_jgroups_relay %}
+      <transport cluster="{{ jdg_jgroups_relay_site }}" stack="xsite" node-name="{{ inventory_hostname }}"  machine="{{ inventory_hostname }}" site="{{ jdg_jgroups_relay_site }}"/>
+{% else %}
+      <transport cluster="datagrid" stack="datagrid" node-name="{{ inventory_hostname }}" machine="{{ inventory_hostname }}"/>
+{% endif %}
 {% for keycloak_cache in jdg_keycloak_cache.caches %}
       <replicated-cache name="{{ keycloak_cache }}" mode="SYNC">
         <transaction mode="NONE" locking="PESSIMISTIC"/>
         <locking acquire-timeout="0" />
+        <backups>
+{% for site in jdg_jgroups_relay_sites %}
+            {% if site != jdg_jgroups_relay_site %}<backup site="{{ site }}" strategy="ASYNC"/>{% endif %}
+{% endfor %}
+        </backups>
       </replicated-cache>
+      <security>
+        <authorization>
+          <role name="{{ jdg_supervisor.name }}" permissions="READ WRITE EXEC CREATE"/>
+        </authorization>
+      </security>
 {% endfor %}
     </cache-container>
 {% endif %}

roles/infinispan/templates/users.properties.j2

@@ -1,3 +1,4 @@
+# {{ ansible_managed }}
 # This file stores usernames and passwords (either encrypted or plain-text)
 #
 # Users can be added to this properties file at any time, updates after the server has started

roles/infinispan/vars/main.yml

@@ -1,3 +1,51 @@
 ---
 supervisor_password:
-infinispan_users: []
+infinispan_users: []
+
+jdg_supervisor:
+  name: 'supervisor'
+  password: "{{ supervisor_password }}"
+  roles: 'admin'
+
+jdg:
+  home: "{{ jdg_default_installation_path if jdg_rhn_id is defined else infinispan_installation_path }}"
+  version: "{{ jdg_version if jdg_rhn_id is defined else infinispan_version }}"
+  bind_addr: "{{ jdg_bind_address }}"
+  port: "{{ jdg_port }}"
+  jgroups_bind_addr: "{{ override_jdg_jgroups_bind_addr | default(ansible_all_ipv4_addresses[0]) }}"
+  jgroups_port: "{{ jdg_jgroups_port }}"
+  jgroups_relay_port: "{{ jdg_jgroups_relay_port }}"
+  config:
+    name: "{{ override_jdg_config | default('jdg.xml') }}"
+    template: "{{ override_jdg_config_template | default('templates/infinispan.xml.j2') }}"
+    jgroups_template: "{{ override_jdg_config_jgroups_template | default('templates/jgroups-jdbcping.xml.j2') }}"
+    users: "{{ override_jdg_config_users_properties | default('users.properties') }}"
+    groups: "{{ override_jdg_config_group_properties | default('groups.properties') }}"
+  user:
+    name: "{{ override_jdg_user | default('jdg') }}"
+  group:
+    name: "{{ override_jdg_group | default('jdg') }}"
+  service:
+    name: "{{ override_jdg_service_name | default('jdg' if jdg_rhn_id is defined else 'infinispan') }}"
+  users: "{{ [ jdg_supervisor ] + infinispan_users }}"
+
+jdg_jgroups_jdbc:
+  mariadb:
+    enabled: "{{ jdg_jgroups_jdbcping }}"
+    driver_class: org.mariadb.jdbc.Driver
+    driver_jar_filename: "mariadb-java-client-2.7.4.jar"
+    driver_jar_url: "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.7.4/mariadb-java-client-2.7.4.jar"
+    connection_url: "{{ mariadb_jdbc_url | default('jdbc:mariadb://localhost:3306/keycloak') }}"
+    db_user: "{{ mariadb_db_user | default('keycloak-user') }}"
+    db_password: "{{ mariadb_db_pass | default('keycloak-pass') }}"
+
+jdg_keycloak_cache:
+  enabled: "{{ infinispan_keycloak_caches }}"
+  caches:
+    - sessions
+    - offlineSessions
+    - clientSessions
+    - offlineClientSessions
+    - loginFailures
+    - actionTokens
+    - work