From 0688fb7dab19b983fdd64247714f7b4878f3b564 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 9 Mar 2024 14:15:48 +0530 Subject: [PATCH 1/7] remove es 2 based config --- .../templates/config/elasticsearch.yml.j2 | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 b/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 index eb6ef5212f..4e1ea78bf4 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 +++ b/src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2 @@ -10,13 +10,8 @@ node.name: "{{ elasticsearch_node_name }}" {% if elasticsearch_node_zone is defined %} # Used for shard allocation awareness # https://www.elastic.co/guide/en/elasticsearch/reference/current/allocation-awareness.html -{% if elasticsearch_version == '2.4.6'%} -node.zone: "{{ elasticsearch_node_zone }}" -{% endif %} -{% if elasticsearch_version is version('5.0.0', '>=') %} node.attr.zone: "{{ elasticsearch_node_zone }}" {% endif %} -{% endif %} # disable multiple nodes starting from the same installation location node.max_local_storage_nodes: 1 @@ -40,10 +35,7 @@ discovery.zen.minimum_master_nodes: {{ [2, groups.elasticsearch|map('extract', h discovery.zen.fd.ping_timeout: 90s discovery.zen.fd.ping_interval: 10s discovery.zen.fd.ping_retries: 20 -{% if elasticsearch_version == '2.4.6' %} -discovery.zen.ping.multicast.enabled: false -discovery.zen.ping.timeout: 90s -{% endif %} + {% if elasticsearch_version == '5.6.16' %} discovery.zen.ping_timeout: 90s {% endif %} @@ -70,14 +62,6 @@ network.host: "{{ inventory_hostname }}" network.host: "{{ lookup('dig', inventory_hostname, wantlist=True)[0] }}" {% endif %} -{% if elasticsearch_version == '2.4.6'%} -index.search.slowlog.threshold.query.warn: 10s -index.search.slowlog.threshold.query.info: 5s -index.search.slowlog.threshold.query.debug: 2s -index.search.slowlog.threshold.query.trace: 500ms -index.max_result_window: 1050000 -{% endif %} - {% if elasticsearch_fielddata_cache_size|default('') %} indices.fielddata.cache.size: {{ elasticsearch_fielddata_cache_size }} {% endif %} @@ -102,13 +86,8 @@ cloud: {% endif %} -# Only allow indices to be created by an explicit "create" command -{% if elasticsearch_version == '2.4.6'%} -action.auto_create_index: false -{% endif %} -{% if elasticsearch_version is version('5.0.0', '>=') %} +# Only auto create elastic search internal indices action.auto_create_index: .watches,.triggered_watches,.watcher-history-* -{% endif %} {% if es_local_repo and not backup_es_s3%} From 710b84111f910bca46c11ba7f7ea43ffb8ab00d1 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 9 Mar 2024 14:26:06 +0530 Subject: [PATCH 2/7] make es 5 as default --- quick_monolith_install/sample-environment/public.yml.j2 | 4 ++-- .../ansible/roles/elasticsearch/defaults/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/quick_monolith_install/sample-environment/public.yml.j2 b/quick_monolith_install/sample-environment/public.yml.j2 index b23e199587..5f00fffd06 100644 --- a/quick_monolith_install/sample-environment/public.yml.j2 +++ b/quick_monolith_install/sample-environment/public.yml.j2 @@ -2,8 +2,8 @@ {% endraw %} elasticsearch_memory: '2048m' elasticsearch_cluster_name: '{{env_name}}-es' -elasticsearch_version: 2.4.6 -elasticsearch_download_sha256: 5f7e4bb792917bb7ffc2a5f612dfec87416d54563f795d6a70637befef4cfc6f. +elasticsearch_version: 5.6.16 +elasticsearch_download_sha256: 6b035a59337d571ab70cea72cc55225c027ad142fbb07fd8984e54261657c77f. formplayer_java_version: {% raw %}"{{ java_17_bin_path }}/java" {% endraw %} diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml b/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml index 994a1cc48d..fb85dcede5 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml +++ b/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml @@ -1,9 +1,9 @@ --- -elasticsearch_version: 2.4.6 +elasticsearch_version: 5.6.16 # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html#_give_half_your_memory_to_lucene # Either half the machines RAM or 24 GB. If changing in the future, never want to go over 30 GB elasticsearch_memory: "{{ [ansible_memory_mb.real.total // 2, 24576] | min }}m" -elasticsearch_download_sha256: 5f7e4bb792917bb7ffc2a5f612dfec87416d54563f795d6a70637befef4cfc6f. +elasticsearch_download_sha256: 6b035a59337d571ab70cea72cc55225c027ad142fbb07fd8984e54261657c77f. elasticsearch_home: "/opt/elasticsearch-{{ elasticsearch_version }}" elasticsearch_conf_dir: "/etc/elasticsearch-{{ elasticsearch_version }}" elasticsearch_data_dir: "{{ encrypted_root }}/elasticsearch-{{ elasticsearch_version }}" From f218fc504e641b23c2309fb4b5493ff4f95910e9 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 9 Mar 2024 14:26:54 +0530 Subject: [PATCH 3/7] set ELASTICSEARCH_MAJOR_VERSION=5 for localsettings template files --- quick_monolith_install/sample-environment/public.yml.j2 | 2 +- .../ansible/roles/commcarehq/templates/localsettings.py.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quick_monolith_install/sample-environment/public.yml.j2 b/quick_monolith_install/sample-environment/public.yml.j2 index 5f00fffd06..74dde7767a 100644 --- a/quick_monolith_install/sample-environment/public.yml.j2 +++ b/quick_monolith_install/sample-environment/public.yml.j2 @@ -82,7 +82,7 @@ localsettings: COUCH_STALE_QUERY: 'update_after' DEPLOY_MACHINE_NAME: {% raw %}"{{ ansible_hostname }}" {% endraw %} - ELASTICSEARCH_MAJOR_VERSION: 2 + ELASTICSEARCH_MAJOR_VERSION: 5 EMAIL_SMTP_HOST: 'localhost' EMAIL_SMTP_PORT: 25 EMAIL_USE_TLS: no diff --git a/src/commcare_cloud/ansible/roles/commcarehq/templates/localsettings.py.j2 b/src/commcare_cloud/ansible/roles/commcarehq/templates/localsettings.py.j2 index 5cf0e9593f..fc9ba76720 100644 --- a/src/commcare_cloud/ansible/roles/commcarehq/templates/localsettings.py.j2 +++ b/src/commcare_cloud/ansible/roles/commcarehq/templates/localsettings.py.j2 @@ -463,7 +463,7 @@ ELASTICSEARCH_PORT = {{ ELASTICSEARCH_PORT }} {% if 'ELASTICSEARCH_MAJOR_VERSION' in localsettings %} ELASTICSEARCH_MAJOR_VERSION = {{ localsettings.ELASTICSEARCH_MAJOR_VERSION }} {% else %} -ELASTICSEARCH_MAJOR_VERSION = 2 +ELASTICSEARCH_MAJOR_VERSION = 5 {% endif %} ES_SETTINGS = {{ ES_SETTINGS }} From 30ae3ba05a46d725d05661f8e2c48a61299cbc15 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 9 Mar 2024 14:29:08 +0530 Subject: [PATCH 4/7] remove es 2 download instructions from es playbook --- .../ansible/roles/elasticsearch/tasks/main.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml b/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml index a02fa8bbd8..3eeac3f2aa 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml +++ b/src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml @@ -24,14 +24,6 @@ value: '262144' state: present become: yes - -- name: Download Elasticsearch 2.4.x - become: yes - get_url: - url: "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{ elasticsearch_version }}.tar.gz" - dest: "/home/{{ cchq_user }}/downloads/elasticsearch-{{ elasticsearch_version }}.tar.gz" - sha256sum: "{{ elasticsearch_download_sha256 }}" - when: elasticsearch_version == "2.4.6" - name: Download Elasticsearch 5 become: yes @@ -83,9 +75,6 @@ tags: - es_conf -- include: misc_v2.yml - when: elasticsearch_version == '2.4.6' - - include: misc_v5.yml when: elasticsearch_version == '5.6.16' From 74e6bd3adb7faa8cdcc6a8928dabbcf2561d1f1e Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 9 Mar 2024 14:29:32 +0530 Subject: [PATCH 5/7] remove unused file --- .../roles/elasticsearch/tasks/misc_v2.yml | 47 ------------------- 1 file changed, 47 deletions(-) delete mode 100644 src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v2.yml diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v2.yml b/src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v2.yml deleted file mode 100644 index 58b325eb03..0000000000 --- a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v2.yml +++ /dev/null @@ -1,47 +0,0 @@ - -- name: Copy Elasticsearch logging config - become: yes - template: - src: config/logging.yml.j2 - dest: "{{ elasticsearch_conf_dir }}/logging.yml" - owner: elasticsearch - group: elasticsearch - register: copy_elasticsearch_logging_result - tags: - - es_conf - -- name: Disable Java Heap Dump - replace: - path: "{{ elasticsearch_home }}/bin/elasticsearch.in.sh" - regexp: '^JAVA_OPTS="\$JAVA_OPTS -XX:\+HeapDumpOnOutOfMemoryError"' - replace: '#JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"' - -- name: Enable Explicit GC - replace: - path: "{{ elasticsearch_home }}/bin/elasticsearch.in.sh" - regexp: '^JAVA_OPTS="\$JAVA_OPTS -XX:\+DisableExplicitGC"' - replace: '#JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"' - -- name: Install elasticsearch python client - become: yes - pip: - name: elasticsearch>=2.0.0,<3.0.0 - -- name: Check for installed plugins - shell: "{{ elasticsearch_home }}/bin/plugin list" - register: installed_plugins - -- name: Install elasticsearch aws plugin - become: yes - shell: "{{ elasticsearch_home }}/bin/plugin install cloud-aws" - when: (backup_es_s3) and (installed_plugins.stdout is defined) and ('cloud-aws' not in installed_plugins.stdout) and elasticsearch_version == '2.4.6' - -- name: Install elasticsearch swift plugin - become: yes - shell: "{{ elasticsearch_home }}/bin/plugin install https://github.com/dimagi/elasticsearch-repository-swift/releases/download/swift-repository-plugin-{{ swift_plugin_version }}/repository-swift-3.0.0-es{{ swift_plugin_version }}.zip" - when: (backup_es_swift) and (installed_plugins.stdout is defined) and ('repository-swift' not in installed_plugins.stdout) and elasticsearch_version == '2.4.6' - -- name: Install elasticsearch phonetic plugin - become: yes - shell: "{{ elasticsearch_home }}/bin/plugin install analysis-phonetic" - when: (installed_plugins.stdout is defined) and ('analysis-phonetic' not in installed_plugins.stdout) and es_plugins_phonetic_enabled From 37d9acf4130225dd9d6e1d5273705edfad77e883 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 9 Mar 2024 14:33:10 +0530 Subject: [PATCH 6/7] remove variables and details of plugins we no longer use --- .../ansible/roles/elasticsearch/defaults/main.yml | 2 -- .../ansible/roles/elasticsearch/tasks/misc_v7.yml | 10 ---------- 2 files changed, 12 deletions(-) diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml b/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml index fb85dcede5..ec802266eb 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml +++ b/src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml @@ -11,8 +11,6 @@ elasticsearch_tcp_port: 9300 elasticsearch_http_port: 9200 #es_snapshot_bucket: "dimagi-{{ deploy_env }}-es-snapshots" es_repository_name: "{{ deploy_env }}_es_snapshot" -aws_plugin_version: "2.7.1" -swift_plugin_version: "2.4.6" backup_es_swift: no elasticsearch_enable_inline_groovy_scripts: True elasticsearch_service_name: elasticsearch diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v7.yml b/src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v7.yml index 67a710590a..c156a2c675 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v7.yml +++ b/src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v7.yml @@ -29,13 +29,3 @@ - name: Check for installed plugins shell: "{{ elasticsearch_home }}/bin/elasticsearch-plugin list" register: installed_plugins - -- name: Install elasticsearch aws plugin - become: yes - shell: "{{ elasticsearch_home }}/bin/elasticsearch-plugin install elasticsearch/elasticsearch-cloud-aws/{{ aws_plugin_version }}" - when: (backup_es_s3) and (installed_plugins.stdout is defined) and ('cloud-aws' not in installed_plugins.stdout) - -- name: Install elasticsearch swift plugin - become: yes - shell: "{{ elasticsearch_home }}/bin/elasticsearch-plugin install https://github.com/dimagi/elasticsearch-repository-swift/releases/download/swift-repository-plugin-{{ swift_plugin_version }}/repository-swift-3.0.0-es{{ swift_plugin_version }}.zip" - when: (backup_es_swift) and (installed_plugins.stdout is defined) and ('repository-swift' not in installed_plugins.stdout) From 10dec90ad545b91b59080c04afe865ab2ea70611 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 9 Mar 2024 14:35:41 +0530 Subject: [PATCH 7/7] update service file --- .../templates/systemd/elasticsearch.service.j2 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/commcare_cloud/ansible/roles/elasticsearch/templates/systemd/elasticsearch.service.j2 b/src/commcare_cloud/ansible/roles/elasticsearch/templates/systemd/elasticsearch.service.j2 index 01768e2448..5cf8a359e5 100644 --- a/src/commcare_cloud/ansible/roles/elasticsearch/templates/systemd/elasticsearch.service.j2 +++ b/src/commcare_cloud/ansible/roles/elasticsearch/templates/systemd/elasticsearch.service.j2 @@ -12,21 +12,12 @@ Environment=ES_PATH_CONF={{ elasticsearch_conf_dir}} {% if elasticsearch_version == '5.6.16'%} Environment=ES_JVM_OPTIONS={{ elasticsearch_conf_dir}}/jvm.options {% endif %} -{% if elasticsearch_version == '2.4.6'%} -Environment=ES_HEAP_SIZE={{ elasticsearch_memory }} -{% endif %} WorkingDirectory={{ elasticsearch_home }} User=elasticsearch Group=elasticsearch -{% if elasticsearch_version == '2.4.6'%} -ExecStart={{ elasticsearch_home }}/bin/elasticsearch --path.conf=${CONF_DIR} --path.home=${ES_HOME} -{% endif %} - -{% if elasticsearch_version == '5.6.16'%} ExecStart={{ elasticsearch_home }}/bin/elasticsearch -Epath.conf=${CONF_DIR} -{% endif %} Restart=always RestartSec=10