dovecot: replace solr fts with flatcurve (xapian) (#5680)

* fts-flatcurve: inital implementation

* fts: removed solr from compose.yml

* flatcurve: added heap and proc logic to dovecot

* added logic for update.sh & generate for Flatcurve

* delete old iteration of fts-flatcurve.conf

* updated default fts.conf

* updated .gitignore to exclude fts.conf for further git updates

* Remove autogeneration of fts.conf (disable override)

* cleanup all left solr stuff

* renamed SKIP_FLATCURVE to SKIP_FTS

* cleanup leftovers solr in lang files

* moved lazy_expunge plugin only to mail_plugins

* added fts timeout value

* compose: remove dev image of dovecot

* updated japanese translation

.github/workflows/image_builds.yml

@@ -23,7 +23,6 @@ jobs:
           - "postfix-mailcow"
           - "rspamd-mailcow"
           - "sogo-mailcow"
-          - "solr-mailcow"
           - "unbound-mailcow"
           - "watchdog-mailcow"
     runs-on: ubuntu-latest

.gitignore

@@ -23,6 +23,7 @@ data/conf/dovecot/sni.conf
 data/conf/dovecot/sogo-sso.conf
 data/conf/dovecot/sogo_trusted_ip.conf
 data/conf/dovecot/sql
+data/conf/dovecot/conf.d/fts.conf
 data/conf/nextcloud-*.bak
 data/conf/nginx/*.active
 data/conf/nginx/*.bak

data/Dockerfiles/dovecot/Dockerfile

@@ -105,7 +105,6 @@ RUN addgroup -g 5000 vmail \
   dovecot-submissiond \
   dovecot-pigeonhole-plugin \
   dovecot-pop3d \
-  dovecot-fts-solr \
   dovecot-fts-flatcurve \
   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -110,21 +110,16 @@ EOF
 
 echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone
 
-if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY]) ]]; then
-echo -e "\e[33mActivating Flatcurve as FTS Backend...\e[0m"
-echo -e "\e[33mDepending on your previous setup a full reindex might be needed... \e[0m"
-echo -e "\e[34mVisit https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-fts/#fts-related-dovecot-commands to learn how to reindex\e[0m"
-echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
-echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
-echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
-elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+if [[ "${SKIP_FTS}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+echo -e "\e[33mDetecting SKIP_FTS=y... not enabling Flatcurve (FTS) then...\e[0m"
 echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
 echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
 echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 else
-echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_solr listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
-echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_solr listescape replication' > /etc/dovecot/mail_plugins_imap
-echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_solr notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
+echo -e "\e[32mDetecting SKIP_FTS=n... enabling Flatcurve (FTS)\e[0m"
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 fi
 chmod 644 /etc/dovecot/mail_plugins /etc/dovecot/mail_plugins_imap /etc/dovecot/mail_plugins_lmtp /templates/quarantine.tpl
 
@@ -247,51 +242,6 @@ function script_deinit()
 end
 EOF
 
-# Temporarily set FTS depending on user choice inside mailcow.conf. Will be removed as soon as Solr is dropped
-if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
-cat <<EOF > /etc/dovecot/conf.d/fts.conf
-# Autogenerated by mailcow
-plugin {
-    fts_autoindex = yes
-    fts_autoindex_exclude = \Junk
-    fts_autoindex_exclude2 = \Trash
-    fts = flatcurve
-
-    # Maximum term length can be set via the 'maxlen' argument (maxlen is
-    # specified in bytes, not number of UTF-8 characters)
-    fts_tokenizer_email_address = maxlen=100
-    fts_tokenizer_generic = algorithm=simple maxlen=30
-
-    # These are not flatcurve settings, but required for Dovecot FTS. See
-    # Dovecot FTS Configuration link above for further information.
-    fts_languages = en es de
-    fts_tokenizers = generic email-address
-
-    # OPTIONAL: Recommended default FTS core configuration
-    fts_filters = normalizer-icu snowball stopwords
-    fts_filters_en = lowercase snowball english-possessive stopwords
-}
-EOF
-elif [[ ! "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
-cat <<EOF > /etc/dovecot/conf.d/fts.conf
-# Autogenerated by mailcow
-plugin {
-  fts = solr
-  fts_autoindex = yes
-  fts_autoindex_exclude = \Junk
-  fts_autoindex_exclude2 = \Trash
-  fts_solr = url=http://solr:8983/solr/dovecot-fts/
-
-  fts_tokenizers = generic email-address
-  fts_tokenizer_generic = algorithm=simple
-
-  fts_filters = normalizer-icu snowball stopwords
-  fts_filters_en = lowercase snowball english-possessive stopwords
-}
-EOF
-fi
-
-
 # Replace patterns in app-passdb.lua
 sed -i "s/__DBUSER__/${DBUSER}/g" /etc/dovecot/lua/passwd-verify.lua
 sed -i "s/__DBPASS__/${DBPASS}/g" /etc/dovecot/lua/passwd-verify.lua
@@ -398,6 +348,15 @@ mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
 EOF
 fi
 
+# Setting variables for indexer-worker inside fts.conf automatically according to mailcow.conf settings
+if [[ "${SKIP_FTS}" =~ ^([nN][oO]|[nN])+$ ]]; then
+  echo -e "\e[94mConfiguring FTS Settings...\e[0m"
+  echo -e "\e[94mSetting FTS Memory Limit (per process) to ${FTS_HEAP} MB\e[0m"
+  sed -i "s/vsz_limit\s*=\s*[0-9]*\s*MB*/vsz_limit=${FTS_HEAP} MB/" /etc/dovecot/conf.d/fts.conf
+  echo -e "\e[94mSetting FTS Process Limit to ${FTS_PROCS}\e[0m"
+  sed -i "s/process_limit\s*=\s*[0-9]*/process_limit=${FTS_PROCS}/" /etc/dovecot/conf.d/fts.conf
+fi
+
 # 401 is user dovecot
 if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
 	openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem

data/Dockerfiles/dovecot/optimize-fts.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ && ! "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+if [[ "${SKIP_FTS}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
     exit 0
 else
     doveadm fts optimize -A
-fi
+fi