diff --git a/contributors/runbtc.txt b/contributors/runbtc.txt new file mode 100644 index 0000000000..7578d13808 --- /dev/null +++ b/contributors/runbtc.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 5, 2024. + +Signed: runbtc \ No newline at end of file diff --git a/production/install b/production/install index 30754863c3..778671c397 100755 --- a/production/install +++ b/production/install @@ -375,20 +375,30 @@ LIQUIDTESTNET_ASSET_REGISTRY_DB_NAME=asset_registry_testnet_db ##### OS packages ##### ####################### +# packages needed to run this script +DEBIAN_PREREQ_PKG=() +DEBIAN_PREREQ_PKG+=(dialog) + # packages needed for mempool ecosystem DEBIAN_PKG=() -DEBIAN_PKG+=(zsh vim curl screen openssl python3 dialog cron) +DEBIAN_PKG+=(zsh vim curl screen openssl python3 cron acl) DEBIAN_PKG+=(build-essential git git-lfs clang cmake jq) DEBIAN_PKG+=(autotools-dev autoconf automake pkg-config bsdmainutils) DEBIAN_PKG+=(libevent-dev libdb-dev libssl-dev libtool autotools-dev) DEBIAN_PKG+=(libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev) DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python3-certbot-nginx rsync ufw) -DEBIAN_PKG+=(geoipupdate) +DEBIAN_PKG+=(geoipupdate redis) +DEBIAN_PKG+=(cargo) +DEBIAN_PKG+=(openjdk-11-jdk) DEBIAN_UNFURL_PKG=() DEBIAN_UNFURL_PKG+=(cups chromium-bsu libatk1.0 libatk-bridge2.0 libxkbcommon-dev libxcomposite-dev) DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev libasound-dev) +# packages needed to run this script +FREEBSD_PREREQ_PKG=() +FREEBSD_PREREQ_PKG+=(dialog) + # packages needed for mempool ecosystem FREEBSD_PKG=() FREEBSD_PKG+=(zsh sudo git git-lfs screen curl wget calc neovim) @@ -470,6 +480,18 @@ osPackageInstallAll() esac } +osPackageInstallPreReq() +{ + case $OS in + FreeBSD) + osPackageInstall ${FREEBSD_PREREQ_PKG[@]} + ;; + Debian) + osPackageInstall ${DEBIAN_PREREQ_PKG[@]} + ;; + esac +} + # osUserCreate username home_directory main_groupname [additional_group] osUserCreate() { @@ -633,6 +655,14 @@ zfsCreateFilesystems() ext4CreateDir() { mkdir -p "/backup" "${ELEMENTS_HOME}" "${BITCOIN_HOME}" "${MINFEE_HOME}" "${ELECTRS_HOME}" "${MEMPOOL_HOME}" "${MYSQL_HOME}" "${BITCOIN_ELECTRS_HOME}" "${ELEMENTS_HOME}/liquidv1" "${ELEMENTS_ELECTRS_HOME}" + mkdir -p /var/cache/nginx/services /var/cache/nginx/api + chown "${NGINX_USER}:${NGINX_GROUP}" /var/cache/nginx/services /var/cache/nginx/api + + # create /bitcoin/socket with custom ACL for electrs unix sockets + mkdir -p "${BITCOIN_HOME}/socket" + + # create /elements/socket with custom ACL for electrs unix sockets + mkdir -p "${ELEMENTS_HOME}/socket" # Bitcoin Mainnet if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then @@ -714,6 +744,11 @@ ext4CreateDir() fi } +################################# +# Install prerequisite packages # +################################# + +osPackageInstallPreReq ##### Perform sanity checks before trying anything @@ -1038,6 +1073,9 @@ case $OS in echo "[*] Installing syslog configuration" osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/linux/rsyslog.conf" /etc/rsyslog.d/10-mempool.conf osSudo "${ROOT_USER}" sed -i.orig -e 's/^\*\.\*;auth,authpriv\.none/*\.*;auth,authpriv\.none,local7\.none/' /etc/rsyslog.d/50-default.conf + + echo "[*] Installing redis configuration" + osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/linux/redis.conf" /etc/redis/redis.conf ;; esac @@ -1379,9 +1417,6 @@ if [ "${BISQ_INSTALL}" = ON ];then echo "[*] Cloning Bisq top-level repo" osSudo "${BISQ_USER}" git clone --branch "${BISQ_REPO_BRANCH}" "${BISQ_REPO_URL}" "${BISQ_HOME}/${BISQ_REPO_NAME}" - echo "[*] Installing OpenJDK from Bisq install_java_linux.sh script" - osSudo "${ROOT_USER}" "${BISQ_HOME}/${BISQ_REPO_NAME}/scripts/install_java_linux.sh" - echo "[*] Checking out Bisq ${BISQ_LATEST_RELEASE}" osSudo "${BISQ_USER}" sh -c "cd ${BISQ_HOME}/${BISQ_REPO_NAME} && git checkout ${BISQ_LATEST_RELEASE}" @@ -1402,9 +1437,13 @@ if [ "${BISQ_INSTALL}" = ON ];then ;; Debian) + echo "[*] Installing Bisq service file" osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/bisq.service" "${DEBIAN_SERVICE_HOME}/bisq.service" - osSudo "${ROOT_USER}" sed -i.orig "s/#Requires=bitcoin.service/Requires=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service" - osSudo "${ROOT_USER}" sed -i.orig "s/#BindsTo=bitcoin.service/BindsTo=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service" + if [ "${BITCOIN_INSTALL}" = ON ];then + osSudo "${ROOT_USER}" sed -i.orig "s/After=network.target/After=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service" + osSudo "${ROOT_USER}" sed -i.orig "s/#Requires=bitcoin.service/Requires=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service" + osSudo "${ROOT_USER}" sed -i.orig "s/#BindsTo=bitcoin.service/BindsTo=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service" + fi osSudo "${ROOT_USER}" sed -i.orig "s/__BISQ_REPO_NAME__/${BISQ_REPO_NAME}/" "${DEBIAN_SERVICE_HOME}/bisq.service" osSudo "${ROOT_USER}" sed -i.orig "s!__BISQ_HOME__!${BISQ_HOME}!" "${DEBIAN_SERVICE_HOME}/bisq.service" @@ -1412,18 +1451,20 @@ if [ "${BISQ_INSTALL}" = ON ];then osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/bisq.env" "${DEBIAN_ENV_HOME}/bisq.env" osSudo "${ROOT_USER}" sed -i.orig "s!__BISQ_APP_NAME__!${BISQ_APP_NAME}!" "${DEBIAN_ENV_HOME}/bisq.env" osSudo "${ROOT_USER}" sed -i.orig "s!__BISQ_HOME__!${BISQ_HOME}!" "${DEBIAN_ENV_HOME}/bisq.env" - - echo "[*] Configuring Bisq environment file with Bitcoin RPC credentials" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_P2P_HOST__/${BITCOIN_MAINNET_P2P_HOST}/" "${DEBIAN_ENV_HOME}/bisq.env" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_P2P_PORT__/${BITCOIN_MAINNET_P2P_PORT}/" "${DEBIAN_ENV_HOME}/bisq.env" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_HOST__/${BITCOIN_MAINNET_RPC_HOST}/" "${DEBIAN_ENV_HOME}/bisq.env" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PORT__/${BITCOIN_MAINNET_RPC_PORT}/" "${DEBIAN_ENV_HOME}/bisq.env" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${DEBIAN_ENV_HOME}/bisq.env" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${DEBIAN_ENV_HOME}/bisq.env" + osSudo "${ROOT_USER}" sed -i.orig "s/BISQ_DUMP_BLOCKCHAIN=false/BISQ_DUMP_BLOCKCHAIN=true/" "${DEBIAN_ENV_HOME}/bisq.env" + osSudo "${ROOT_USER}" sed -i.orig "s/BISQ_DUMP_STATISTICS=false/BISQ_DUMP_STATISTICS=true/" "${DEBIAN_ENV_HOME}/bisq.env" - #echo "[*] Updating Bitcoin configuration for Bisq" - #osSudo "${ROOT_USER}" sed -i.orig "s/#blocknotify/blocknotify/" "${BITCOIN_HOME}/bitcoin.conf" - #osSudo "${BITCOIN_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/blocknotify.sh" "${BITCOIN_HOME}/blocknotify.sh" + if [ "${BITCOIN_INSTALL}" = ON ];then + echo "[*] Updating Bitcoin configuration for Bisq" + osSudo "${ROOT_USER}" sed -i.orig "1iblocknotify=${BITCOIN_HOME}/blocknotify.sh %s" "${BITCOIN_HOME}/bitcoin.conf" + osSudo "${BITCOIN_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/blocknotify.sh" "${BITCOIN_HOME}/blocknotify.sh" + fi ;; esac @@ -1435,7 +1476,7 @@ fi if [ "${UNFURL_INSTALL}" = ON ];then - echo "[*] Insalling Unfurl source" + echo "[*] Installing Unfurl source" case $OS in FreeBSD) @@ -1520,7 +1561,7 @@ fi # Bitcoin instance for Mainnet Minfee # ####################################### -if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then +if [ "${BITCOIN_MAINNET_MINFEE_ENABLE}" = ON ];then echo "[*] Installing Bitcoin Minfee service" case $OS in @@ -1588,8 +1629,8 @@ fi # Bitcoin instance for Liquid Testnet # ####################################### -if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Liquid service" +if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then + echo "[*] Installing Bitcoin Liquid Testnet service" case $OS in FreeBSD) @@ -1880,13 +1921,25 @@ echo "[*] Setting permissions for electrs sockets" case $OS in FreeBSD) - setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket" - chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket" - setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket" - chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket" + if [ "${BITCOIN_INSTALL}" = ON ];then + setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket" + chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket" + fi + if [ "${ELEMENTS_INSTALL}" = ON ];then + setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket" + chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket" + fi ;; Debian) + if [ "${BITCOIN_INSTALL}" = ON ];then + setfacl -Rdm "u:bitcoin:rwx,u:mempool:rwx,u:www-data:rwx,o::rw-" "${BITCOIN_HOME}/socket" + chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket" + fi + if [ "${ELEMENTS_INSTALL}" = ON ];then + setfacl -Rdm "u:elements:rwx,u:mempool:rwx,u:www-data:rwx,o::rw-" "${ELEMENTS_HOME}/socket" + chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket" + fi ;; esac diff --git a/production/linux/redis.conf b/production/linux/redis.conf new file mode 100644 index 0000000000..4426857d39 --- /dev/null +++ b/production/linux/redis.conf @@ -0,0 +1,79 @@ +save 3600 1 300 100 60 10000 +unixsocket /var/run/redis/redis.sock +unixsocketperm 666 + +bind 127.0.0.1 -::1 +protected-mode yes +port 6379 +tcp-backlog 511 +timeout 0 +tcp-keepalive 300 +daemonize yes +pidfile /var/run/redis/redis.pid +loglevel notice +logfile /var/log/redis/redis.log +databases 16 +always-show-logo no +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +#locale-collate "" +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +dbfilename dump.rdb +rdb-del-sync-files no +dir /var/lib/redis/ +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +repl-disable-tcp-nodelay no +replica-priority 100 +acllog-max-len 128 +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no +lazyfree-lazy-user-del no +lazyfree-lazy-user-flush no +oom-score-adj no +oom-score-adj-values 0 200 800 +disable-thp yes +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no + +slowlog-log-slower-than 10000 +slowlog-max-len 128 +latency-monitor-threshold 0 +notify-keyspace-events "" +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +#set-max-listpack-entries 128 +#set-max-listpack-value 64 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +dynamic-hz yes +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +jemalloc-bg-thread yes diff --git a/production/mempool-build-all b/production/mempool-build-all index 601f15b9a8..1ad66af937 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -1,4 +1,19 @@ #!/usr/bin/env zsh + +case `uname -s` in + + FreeBSD) + DB_SOCKET=/var/run/mysql/mysql.sock + REDIS_SOCKET=/tmp/redis.sock + ;; + + Linux) + DB_SOCKET=/run/mysqld/mysqld.sock + REDIS_SOCKET=/var/run/redis/redis.sock + ;; + +esac + PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin HOSTNAME=$(hostname) LOCATION=$(hostname|cut -d . -f2) @@ -112,6 +127,8 @@ build_backend() -e "s!__MEMPOOL_LIQUIDTESTNET_PASS__!${MEMPOOL_LIQUIDTESTNET_PASS}!" \ -e "s!__MEMPOOL_BISQ_USER__!${MEMPOOL_BISQ_USER}!" \ -e "s!__MEMPOOL_BISQ_PASS__!${MEMPOOL_BISQ_PASS}!" \ + -e "s!__DB_SOCKET__!${DB_SOCKET}!" \ + -e "s!__REDIS_SOCKET__!${REDIS_SOCKET}!" \ "mempool-config.json" fi npm install --omit=dev --omit=optional || exit 1 diff --git a/production/mempool-config.bisq.json b/production/mempool-config.bisq.json index 4913cb9869..6ff25dcb4a 100644 --- a/production/mempool-config.bisq.json +++ b/production/mempool-config.bisq.json @@ -22,7 +22,7 @@ "DATABASE": { "ENABLED": false, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_BISQ_USER__", "PASSWORD": "__MEMPOOL_BISQ_PASS__", "DATABASE": "mempool_bisq" diff --git a/production/mempool-config.liquid.json b/production/mempool-config.liquid.json index 9051bba741..f70256a412 100644 --- a/production/mempool-config.liquid.json +++ b/production/mempool-config.liquid.json @@ -71,7 +71,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_LIQUID_USER__", "PASSWORD": "__MEMPOOL_LIQUID_PASS__", "DATABASE": "mempool_liquid" diff --git a/production/mempool-config.liquidtestnet.json b/production/mempool-config.liquidtestnet.json index ae6d7b1ac4..103aade69e 100644 --- a/production/mempool-config.liquidtestnet.json +++ b/production/mempool-config.liquidtestnet.json @@ -71,7 +71,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_LIQUIDTESTNET_USER__", "PASSWORD": "__MEMPOOL_LIQUIDTESTNET_PASS__", "DATABASE": "mempool_liquidtestnet" diff --git a/production/mempool-config.mainnet-lightning.json b/production/mempool-config.mainnet-lightning.json index b113a3c8a1..a1d258ea3b 100644 --- a/production/mempool-config.mainnet-lightning.json +++ b/production/mempool-config.mainnet-lightning.json @@ -84,7 +84,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_MAINNET_LIGHTNING_USER__", "PASSWORD": "__MEMPOOL_MAINNET_LIGHTNING_PASS__", "DATABASE": "mempool_mainnet_lightning" diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index b6ee68da1e..f60926e73c 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -85,7 +85,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_MAINNET_USER__", "PASSWORD": "__MEMPOOL_MAINNET_PASS__", "DATABASE": "mempool" @@ -144,7 +144,7 @@ }, "REDIS": { "ENABLED": true, - "UNIX_SOCKET_PATH": "/tmp/redis.sock", + "UNIX_SOCKET_PATH": "__REDIS_SOCKET__", "BATCH_QUERY_BASE_SIZE": 5000 }, "MEMPOOL_SERVICES": { diff --git a/production/mempool-config.signet-lightning.json b/production/mempool-config.signet-lightning.json index c087a9104b..2679c97cc2 100644 --- a/production/mempool-config.signet-lightning.json +++ b/production/mempool-config.signet-lightning.json @@ -79,7 +79,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_SIGNET_LIGHTNING_USER__", "PASSWORD": "__MEMPOOL_SIGNET_LIGHTNING_PASS__", "DATABASE": "mempool_signet_lightning" diff --git a/production/mempool-config.signet.json b/production/mempool-config.signet.json index 57cb374434..2cff03e821 100644 --- a/production/mempool-config.signet.json +++ b/production/mempool-config.signet.json @@ -73,7 +73,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_SIGNET_USER__", "PASSWORD": "__MEMPOOL_SIGNET_PASS__", "DATABASE": "mempool_signet" diff --git a/production/mempool-config.testnet-lightning.json b/production/mempool-config.testnet-lightning.json index 130410dc41..e127a7c011 100644 --- a/production/mempool-config.testnet-lightning.json +++ b/production/mempool-config.testnet-lightning.json @@ -79,7 +79,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_TESTNET_LIGHTNING_USER__", "PASSWORD": "__MEMPOOL_TESTNET_LIGHTNING_PASS__", "DATABASE": "mempool_testnet_lightning" diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index 1a216c54b2..0c61f74bfc 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -73,7 +73,7 @@ "DATABASE": { "ENABLED": true, "HOST": "127.0.0.1", - "SOCKET": "/var/run/mysql/mysql.sock", + "SOCKET": "__DB_SOCKET__", "USERNAME": "__MEMPOOL_TESTNET_USER__", "PASSWORD": "__MEMPOOL_TESTNET_PASS__", "DATABASE": "mempool_testnet"