diff --git a/addons/backup-and-maintenance.sh b/addons/backup-and-maintenance.sh index 02256723e6e1..4926f19e12e6 100755 --- a/addons/backup-and-maintenance.sh +++ b/addons/backup-and-maintenance.sh @@ -12,6 +12,65 @@ # Licensed under the GPL # +source /usr/local/pf/addons/functions/helpers.functions + +############################################################################# +### Help +############################################################################# + +help(){ + cat <<-EOF +$0 Backup a PacketFence instance + +Usage: $0 -f /path/to/backup_file.tgz [OPTION]... + +Options: + -h,--help Display this help + --db Backup only database from PacketFence database + --conf Backup only configuration from PacketFence configuration +EOF +} + +############################################################################# +### Handle args +############################################################################# +do_full_backup=1 +do_db_backup=0 +do_config_backup=0 +do_replication=0 +mariabackup_installed=false + +# Parse option +TEMP=$(getopt -o f:h --long file:,help,db,conf \ + -n "$0" -- "$@") || (echo "getopt failed." && exit 1) + +# Note the quotes around `$TEMP': they are essential! +eval set -- "$TEMP" + +while true ; do + case "$1" in + -h|--help) + help ; exit 0 ; shift + ;; + --db) + do_db_backup=1 ; do_full_backup=0 ; shift + ;; + --conf) + do_config_backup=1 ; do_full_backup=0 ; shift + ;; + --) + shift ; break + ;; + *) + echo "Wrong usage !" ; help ; exit 1 + ;; + esac +done + +############################################################################# +### Variables +############################################################################# + NB_DAYS_TO_KEEP_DB=7 NB_DAYS_TO_KEEP_FILES=7 PF_DIRECTORY='/usr/local/pf/' @@ -23,72 +82,62 @@ REP_USER=$($PF_DIRECTORY/bin/get_pf_conf active_active galera_replication_userna REP_PWD=$($PF_DIRECTORY/bin/get_pf_conf active_active galera_replication_password) BACKUP_DIRECTORY=${BACKUP_DIRECTORY:-/root/backup/} BACKUP_DB_FILENAME='packetfence-db-dump' -BACKUP_PF_FILENAME='packetfence-files-dump' +BACKUP_CONF_FILENAME='packetfence-conf-dump' ARCHIVE_DIRECTORY=$BACKUP_DIRECTORY ARCHIVE_DB_FILENAME='packetfence-archive' MARIABACKUP_INSTALLED=0 BACKUPRC=1 -# For replication -ACTIVATE_REPLICATION=0 -REPLICATION_USER='' -NODE1_HOSTNAME='' -NODE2_HOSTNAME='' -NODE1_IP='' -NODE2_IP='' - -# to detect MariaDB remote DB -if [ "$DB_HOST" != "localhost" ] && [ "$DB_HOST" != "100.64.0.1" ]; then - MARIADB_REMOTE_CLUSTER=1 -else - MARIADB_REMOTE_CLUSTER=0 -fi - -# Create the backup directory -if [ ! -d "$BACKUP_DIRECTORY" ]; then - mkdir -p $BACKUP_DIRECTORY - echo -e "$BACKUP_DIRECTORY , created. \n" -else - echo -e "$BACKUP_DIRECTORY , folder already created. \n" -fi - -PF_USED_SPACE=`du -s $PF_DIRECTORY --exclude=logs --exclude=var | awk '{ print $1 }'` -BACKUPS_AVAILABLE_SPACE=`df --output=avail $BACKUP_DIRECTORY | awk 'NR == 2 { print $1 }'` - -if (( $BACKUPS_AVAILABLE_SPACE > (( $PF_USED_SPACE / 2 )) )); then - # Backup complete PacketFence installation except logs - current_tgz=$BACKUP_DIRECTORY/$BACKUP_PF_FILENAME-`date +%F_%Hh%M`.tgz - if [ ! -f $BACKUP_DIRECTORY$BACKUP_PF_FILENAME ]; then - tar -czf $current_tgz --exclude='logs/*' --exclude='var/*' --exclude='.git/*' --exclude='conf/certmanager/*' --directory $PF_DIRECTORY . - BACKUPRC=$? - if (( $BACKUPRC > 0 )); then - echo "ERROR: PacketFence files backup was not successful" >&2 - echo "ERROR: PacketFence files backup was not successful" > /usr/local/pf/var/backup_files.status - else - echo -e $BACKUP_PF_FILENAME "have been created in $BACKUP_DIRECTORY \n" - echo "OK" > /usr/local/pf/var/backup_files.status - find $BACKUP_DIRECTORY -name "packetfence-files-dump-*.tgz" -mtime +$NB_DAYS_TO_KEEP_FILES -print0 | xargs -0r rm -f - echo -e "$BACKUP_PF_FILENAME older than $NB_DAYS_TO_KEEP_FILES days have been removed. \n" - fi +die() { + echo "$(basename $0): $@" >&2 ; exit 1 +} + +create_backup_directory(){ + # Create the backup directory + if [ ! -d "$BACKUP_DIRECTORY" ]; then + mkdir -p $BACKUP_DIRECTORY + echo -e "$BACKUP_DIRECTORY , created. \n" else - echo -e $BACKUP_DIRECTORY$BACKUP_PF_FILENAME ", file already created. \n" + echo -e "$BACKUP_DIRECTORY , folder already created. \n" fi -else - echo "ERROR: There is not enough space in $BACKUP_DIRECTORY to safely backup files. Skipping the backup." >&2 - echo "ERROR: There is not enough space in $BACKUP_DIRECTORY to safely backup files. Skipping the backup." > /usr/local/pf/var/backup_files.status -fi +} -die() { - echo "$(basename $0): $@" >&2 ; exit 1 +should_backup_config(){ + PF_USED_SPACE=`du -s $PF_DIRECTORY --exclude=logs --exclude=var | awk '{ print $1 }'` + BACKUPS_AVAILABLE_SPACE=`df --output=avail $BACKUP_DIRECTORY | awk 'NR == 2 { print $1 }'` + + if (( $BACKUPS_AVAILABLE_SPACE > (( $PF_USED_SPACE / 2 )) )); then + # Backup complete PacketFence installation except logs + current_tgz=$BACKUP_DIRECTORY/$BACKUP_CONF_FILENAME-`date +%F_%Hh%M`.tgz + if [ ! -f $BACKUP_DIRECTORY$BACKUP_CONF_FILENAME ]; then + tar -czf $current_tgz --exclude='logs/*' --exclude='var/*' --exclude='.git/*' --exclude='conf/certmanager/*' --directory $PF_DIRECTORY . + BACKUPRC=$? + if (( $BACKUPRC > 0 )); then + echo "ERROR: PacketFence files backup was not successful" >&2 + echo "ERROR: PacketFence files backup was not successful" > /usr/local/pf/var/backup_files.status + else + echo -e $BACKUP_CONF_FILENAME "have been created in $BACKUP_DIRECTORY \n" + echo "OK" > /usr/local/pf/var/backup_files.status + find $BACKUP_DIRECTORY -name "packetfence-conf-dump-*.tgz" -mtime +$NB_DAYS_TO_KEEP_FILES -print0 | xargs -0r rm -f + echo -e "$BACKUP_CONF_FILENAME older than $NB_DAYS_TO_KEEP_FILES days have been removed. \n" + fi + else + echo -e $BACKUP_DIRECTORY$BACKUP_CONF_FILENAME ", file already created. \n" + fi + else + echo "ERROR: There is not enough space in $BACKUP_DIRECTORY to safely backup files. Skipping the backup." >&2 + echo "ERROR: There is not enough space in $BACKUP_DIRECTORY to safely backup files. Skipping the backup." > /usr/local/pf/var/backup_files.status + fi } -should_backup(){ +should_backup_database(){ # Default choices SHOULD_BACKUP=1 MARIADB_LOCAL_CLUSTER=0 MARIADB_DISABLE_GALERA=1 - if [ $MARIADB_REMOTE_CLUSTER -eq 1 ]; then + # to detect MariaDB remote DB + if [ "$DB_HOST" != "localhost" ] && [ "$DB_HOST" != "100.64.0.1" ]; then echo "Remote database detected: backup should be done on database server itself." exit $BACKUPRC fi @@ -107,12 +156,17 @@ should_backup(){ else echo -e "First server of the cluster : database backup will start.\n" fi - else + fi + # Is the database running on the current server and should we be running a backup ? + if [ $SHOULD_BACKUP -eq 1 ] && { [ -f /var/run/mysqld/mysqld.pid ] || [ -f /var/run/mariadb/mariadb.pid ] || [ -f /var/lib/mysql/`hostname`.pid ]; }; then echo "Database backup will start" + backup_database + else + echo "Nothing to do" fi } -backup_db(){ +backup_database(){ # Check to see if Mariabackup is installed if hash mariabackup 2>/dev/null; then echo -e "Mariabackup is available. Will proceed using it for DB backup to avoid locking tables and easier recovery process. \n" @@ -176,25 +230,7 @@ backup_db(){ fi } -should_backup -# Is the database running on the current server and should we be running a backup ? -if [ $SHOULD_BACKUP -eq 1 ] && { [ -f /var/run/mysqld/mysqld.pid ] || [ -f /var/run/mariadb/mariadb.pid ] || [ -f /var/lib/mysql/`hostname`.pid ]; }; then - backup_db -else - echo "Nothing to do" -fi - -# Replicate the db backups between both servers -if [ $ACTIVATE_REPLICATION == 1 ]; then - if [ $HOSTNAME == $NODE1_HOSTNAME ]; then - replicate_to=$NODE2_IP - elif [ $HOSTNAME == $NODE2_HOSTNAME ]; then - replicate_to=$NODE1_IP - else - echo "Cannot recognize hostname. This script is made for $NODE1_HOSTNAME and $NODE2_HOSTNAME. Exiting" >&2 - exit 1 - fi; - eval "rsync -auv -e ssh --delete --include '$BACKUP_DB_FILENAME*' --exclude='*' $BACKUP_DIRECTORY $REPLICATION_USER@$replicate_to:$BACKUP_DIRECTORY" -fi +should_backup_config +should_backup_database exit $BACKUPRC diff --git a/addons/exportable-backup.sh b/addons/exportable-backup.sh new file mode 100755 index 000000000000..73a2b351cc4e --- /dev/null +++ b/addons/exportable-backup.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# +# Backup of $PF_DIRECTORY and $DB_NAME that can be used in export/import procedure +# +# - compressed $PF_DIRECTORY to $BACKUP_DIRECTORY +# - compressed mysqldump to $BACKUP_DIRECTORY +# - prepare files for backup and exportation, rotate and clean +# +# Copyright (C) 2005-2024 Inverse inc. +# +# Author: Inverse inc. +# +# Licensed under the GPL +# + +source /usr/local/pf/addons/functions/helpers.functions + +BACKUP_DIRECTORY=${BACKUP_DIRECTORY:-/root/backup/} +BACKUP_DB_FILENAME='packetfence-db-dump' +BACKUP_CONF_FILENAME='packetfence-conf-dump' +BACKUP_OLD_CONF_FILENAME='packetfence-files-dump' +BACKUP_PF_FILENAME='packetfence-exportable-backup' +NB_DAYS_TO_KEEP_BACKUP=${NB_DAYS_TO_KEEP_BACKUP:-7} +BACKUPRC=1 + +############################################################################# +### Replicate +############################################################################# +replicate_backup(){ + REPLICATION_USER=${REPLICATION_USER:-root} + NODE1_HOSTNAME=${NODE1_HOSTNAME:-node1_hostname} + NODE2_HOSTNAME=${NODE2_HOSTNAME:-node2_hostname} + NODE1_IP=${NODE1_IP:-node1_ip_address} + NODE2_IP=${NODE2_IP:-node2_ip_address} + + if [ $HOSTNAME == $NODE1_HOSTNAME ]; then + replicate_to=$NODE2_IP + elif [ $HOSTNAME == $NODE2_HOSTNAME ]; then + replicate_to=$NODE1_IP + else + echo "Cannot recognize hostname. This script is made for $NODE1_HOSTNAME and $NODE2_HOSTNAME. Exiting" >&2 + exit 1 + fi; + eval "rsync -auv -e ssh --delete --include '$BACKUP_DB_FILENAME*' --exclude='*' $BACKUP_DIRECTORY $REPLICATION_USER@$replicate_to:$BACKUP_DIRECTORY" + exit $BACKUPRC +} + +############################################################################# +### Help +############################################################################# +help(){ + cat <<-EOF +$0 Backup a PacketFence instance + +Usage: $0 -f /path/to/backup_file.tgz [OPTION]... + +Options: + -f,--file Backup in a specific PacketFence (by default it will be under /root/backup/) + -h,--help Display this help + --db Backup only database from PacketFence database + --conf Backup only configuration from PacketFence configuration + --replication Replicate the backup accross two nodes. + +EOF +} + +############################################################################# +### Clean old exportable backup archive +############################################################################# +clean_old_backup_archive(){ + find $BACKUP_DIRECTORY -name "$BACKUP_PF_FILENAME-*.tgz" -mtime +$NB_DAYS_TO_KEEP_BACKUP -delete +} + +############################################################################# +### Disk space +############################################################################# +check_disk_space(){ + BACKUPS_AVAILABLE_SPACE=`df --output=avail $BACKUP_DIRECTORY | awk 'NR == 2 { print $1 }'` + MYSQL_USED_SPACE=`du -s /var/lib/mysql | awk '{ print $1 }'` + CONF_USED_SPACE=`du -s $PF_DIRECTORY --exclude=logs --exclude=var | awk '{ print $1 }'` + if (( $BACKUPS_AVAILABLE_SPACE < (( (( $MYSQL_USED_SPACE + $CONF_USED_SPACE )) /2 )) )); then + echo "There is not enough space in $BACKUP_DIRECTORY to safely backup exportable. Skipping backup." >&2 + echo "There is not enough space in $BACKUP_DIRECTORY to safely backup exportable. Skipping backup." > /usr/local/pf/var/backup_pf.status + exit $BACKUPRC + fi +} + + +create_backup_directory(){ + # Create the backup directory + if [ ! -d "$BACKUP_DIRECTORY" ]; then + mkdir -p $BACKUP_DIRECTORY + echo -e "$BACKUP_DIRECTORY , created. \n" + else + echo -e "$BACKUP_DIRECTORY , folder already created. \n" + fi +} + + +############################################################################# +### Cleaning +############################################################################# +clean_backup(){ + echo "Start backup cleaning" + find $BACKUP_DIRECTORY -name "$BACKUP_PF_FILENAME-*.tgz" -mtime +$NB_DAYS_TO_KEEP_BACKUP -delete + echo "Old backup cleaned" + find $BACKUP_DIRECTORY -name "$BACKUP_DB_FILENAME-*.sql.gz" -delete + echo "Temp db backup cleaned" + find $BACKUP_DIRECTORY -name "$BACKUP_CONF_FILENAME-*.tgz" -delete + echo "Temp config backup cleaned" + find $BACKUP_DIRECTORY -name "$BACKUP_OLD_CONF_FILENAME-*.tgz" -mtime +$NB_DAYS_TO_KEEP_BACKUP -delete + echo "Old config backup cleaned" + echo "Backup cleaning is done" +} + +############################################################################# +### Handle args +############################################################################# +do_full_backup=1 +do_db_backup=0 +do_config_backup=0 +do_replication=0 +BACKUP_FILE=${BACKUP_FILE:-} + +# Parse option +TEMP=$(getopt -o f:h --long file:,help,db,conf,replication \ + -n "$0" -- "$@") || (echo "getopt failed." && exit 1) + +# Note the quotes around `$TEMP': they are essential! +eval set -- "$TEMP" + +while true ; do + case "$1" in + -f|--file) + # first shift is mandatory to get file path + shift + BACKUP_FILE="$1" ; shift + ;; + -h|--help) + help ; exit 0 ; shift + ;; + --db) + do_db_backup=1 ; do_full_backup=0 ; shift + ;; + --conf) + do_config_backup=1 ; do_full_backup=0 ; shift + ;; + --replication) + do_replication=1 ; shift + ;; + --) + shift ; break + ;; + *) + echo "Wrong usage !" ; help ; exit 1 + ;; + esac +done + +if [ -z "$BACKUP_FILE" ]; then + echo "Default directory $BACKUP_DIRECTORY will be used." + BACKUP_FILE=$BACKUP_DIRECTORY/$BACKUP_PF_FILENAME-`date +%F_%Hh%M`.tgz + echo "The backup file will be $BACKUP_FILE" +fi + +############################################################################# +### Main +############################################################################# +create_backup_directory +if check_disk_space; then + /bin/bash /usr/local/pf/addons/backup-and-maintenance.sh + if [ ! -f $BACKUP_FILE ]; then + /bin/bash /usr/local/pf/addons/full-import/export.sh $BACKUP_FILE + else + echo -e $BACKUP_FILE ", file already created. \n" + fi + clean_backup + if [ $do_replication == 1 ]; then + replicate_backup + fi + echo "Exportable backup is done" +fi diff --git a/addons/full-import/export.sh b/addons/full-import/export.sh index 12b1ecb22b76..36b125e100a8 100755 --- a/addons/full-import/export.sh +++ b/addons/full-import/export.sh @@ -6,31 +6,28 @@ if [ -z "$1" ]; then exit 1 fi -if [ "$2" = "--force" ];then - echo "Force flag enabled" - mtime="" -else - mtime="-mtime -1" -fi - set -o nounset -o pipefail -o errexit source /usr/local/pf/addons/functions/helpers.functions source /usr/local/pf/addons/functions/database.functions output="$1" +BACKUP_DB_FILENAME='packetfence-db-dump-*' +BACKUP_CONF_FILENAME='packetfence-conf-dump-*' -db_dump=`find /root/backup -name 'packetfence-db-dump-*' -printf "%T@ %p\n" | sort -n | tail -1 | awk '{ print $2 }'` +echo "Search last database dump available." +last_db_dump=`find /root/backup -name $BACKUP_DB_FILENAME -printf "%T@ %p\n" | sort -n | tail -1 | awk '{ print $2 }'` -if [ -z "$db_dump" ]; then - echo "Unable to find a database dump that was done in the last 24 hours. Add --force to ignore this." +if [ -z "$last_db_dump" ]; then + echo "Unable to find a database dump." exit 1 fi -files_dump=`find /root/backup -name 'packetfence-files-dump-*' -printf "%T@ %p\n" | sort -n | tail -1 | awk '{ print $2 }'` +echo "Search last config dump available." +last_conf_dump=`find /root/backup -name $BACKUP_CONF_FILENAME -printf "%T@ %p\n" | sort -n | tail -1 | awk '{ print $2 }'` -if [ -z "$files_dump" ]; then - echo "Unable to find a files dump that was done in the last 24 hours. Add --force to ignore this." +if [ -z "$last_conf_dump" ]; then + echo "Unable to find a config dump." exit 1 fi @@ -46,12 +43,12 @@ pushd $build_dir main_splitter echo "Copying dump files to temporary export directory" -cp -a $db_dump $build_dir/ -cp -a $files_dump $build_dir/ +cp -a $last_db_dump $build_dir/ +cp -a $last_conf_dump $build_dir/ mariadb_args="" -if echo "$db_dump" | grep '\.sql.gz$' >/dev/null; then +if echo "$last_db_dump" | grep '\.sql.gz$' >/dev/null; then if ! test_db_connection_no_creds; then echo -n "Please enter the root password for MariaDB:" read -s mariadb_root_pass @@ -85,12 +82,12 @@ for f in $add_files; do done main_splitter -echo "Creating export archive" +echo "Creating exportable backup archive" tar -cvzf $output * check_code $? main_splitter -echo "Done exporting to $output" +echo "Done backuping to $output" popd > /dev/null diff --git a/addons/full-import/import.sh b/addons/full-import/import.sh index c2317cdb72ce..b015e3679a3f 100755 --- a/addons/full-import/import.sh +++ b/addons/full-import/import.sh @@ -40,7 +40,7 @@ prepare_import() { ls -l | grep -v export.tgz main_splitter - files_dump=`ls packetfence-files-*` + files_dump=`ls packetfence-conf-*` echo "Found files dump '$files_dump'" echo "Extracting files dump" @@ -79,31 +79,36 @@ import_db() { if echo "$db_dump" | grep '\.sql$' >/dev/null; then echo "The database dump uses mysqldump" #TODO /tmp/grants.sql should be included in the export - import_mysqldump grants.sql $db_dump usr/local/pf/conf/pf.conf + import_mysqldump grants.sql $db_dump usr/local/pf/conf/pf.conf $do_db_restore elif echo "$db_dump" | grep '\.xbstream$' >/dev/null; then echo "The database uses mariabackup" - # permit to remove mariabackup if everything goes well - # or to uninstall it if a failure occurs during installation + # permit to remove mariabackup if everything goes well + # or to uninstall it if a failure occurs during installation if install_mariabackup $pf_version_in_export; then - mariabackup_installed=true - else - uninstall_mariabackup $pf_version_in_export - exit 1 - fi + mariabackup_installed=true + else + uninstall_mariabackup $pf_version_in_export + exit 1 + fi import_mariabackup $db_dump else echo "Unable to detect format of the database dump" exit 1 fi - handle_devel_upgrade `egrep -o '[0-9]+\.[0-9]+\.[0-9]+$' /usr/local/pf/conf/pf-release | egrep -o '^[0-9]+\.[0-9]+'` + if [ "$do_db_restore" -eq 0 ] ; then + handle_devel_upgrade `egrep -o '[0-9]+\.[0-9]+\.[0-9]+$' /usr/local/pf/conf/pf-release | egrep -o '^[0-9]+\.[0-9]+'` - #TODO: check the version of the export, we want to support only 10.3.0 and above - #TODO: check if galera is enabled and stop if its the case + #TODO: check the version of the export, we want to support only 10.3.0 and above + #TODO: check if galera is enabled and stop if its the case - main_splitter - db_name=`get_db_name usr/local/pf/conf/pf.conf` - upgrade_database $db_name + main_splitter + db_name=`get_db_name usr/local/pf/conf/pf.conf` + upgrade_database $db_name + else + main_splitter + echo "Only database restoration has been selected. No upgrade on database will be done" + fi } import_config() { @@ -132,31 +137,44 @@ import_config() { restore_certificates } + finalize_import() { - main_splitter - echo "Finalizing import" + if [ "$do_db_restore" -eq 0 ] ; then + main_splitter + echo "Finalizing import" - sub_splitter - echo "Applying fixpermissions" - /usr/local/pf/bin/pfcmd fixpermissions + sub_splitter + echo "Applying fixpermissions" + /usr/local/pf/bin/pfcmd fixpermissions - sub_splitter - echo "Restarting packetfence-redis-cache" - systemctl restart packetfence-redis-cache + sub_splitter + echo "Restarting packetfence-redis-cache" + systemctl restart packetfence-redis-cache - sub_splitter - echo "Restarting packetfence-config" - systemctl restart packetfence-config + sub_splitter + echo "Restarting packetfence-config" + systemctl restart packetfence-config - sub_splitter - echo "Reloading configuration" - configreload + sub_splitter + echo "Reloading configuration" + configreload - main_splitter - echo "Completed import of the database and the configuration! Complete any necessary adjustments and restart PacketFence" + main_splitter + echo "Completed import of the database and the configuration! Complete any necessary adjustments and restart PacketFence" - # Done with everything, time to cleanup! - systemctl cat monit > /dev/null 2>&1 && systemctl enable monit + # Done with everything, time to cleanup! + systemctl cat monit > /dev/null 2>&1 && systemctl enable monit + else + main_splitter + echo "Finalizing db restoration" + + sub_splitter + echo "Restarting service packetfence-httpd.admin_dispatcher" + systemctl restart packetfence-httpd.admin_dispatcher + echo "Restarting packetfence-haproxy-admin service" + systemctl start packetfence-haproxy-admin + echo "Completed import of the database! Complete any necessary adjustments and restart PacketFence" + fi popd > /dev/null } @@ -169,9 +187,10 @@ Usage: $0 -f /path/to/export.tgz [OPTION]... Options: -f,--file Import a PacketFence export (mandatory) -h,--help Display this help - --db Import only database from PacketFence export - --conf Import only configuration from PacketFence export - --skip-adjust-conf Don't run adjustments on configuration (only use it if you know what you are doing) + --db Import only database from PacketFence export + --db-restore Restore only database from PacketFence export without upgrade process + --conf Import only configuration from PacketFence export + --skip-adjust-conf Don't run adjustments on configuration (only use it if you know what you are doing) EOF } @@ -181,6 +200,7 @@ EOF ############################################################################# do_full_import=1 do_db_import=0 +do_db_restore=0 do_config_import=0 do_adjust_config=1 mariabackup_installed=false @@ -189,7 +209,7 @@ EXPORT_FILE=${EXPORT_FILE:-} # Parse option # TEMP=$(getopt -o f:h --long file:,help,db,conf \ # -n "$0" -- "$@") || (echo "getopt failed." && exit 1) -TEMP=$(getopt -o f:h --long file:,help,db,conf,skip-adjust-conf \ +TEMP=$(getopt -o f:h --long file:,help,db,db-restore,conf,skip-adjust-conf \ -n "$0" -- "$@") || (echo "getopt failed." && exit 1) # Note the quotes around `$TEMP': they are essential! @@ -198,15 +218,18 @@ eval set -- "$TEMP" while true ; do case "$1" in -f|--file) - # first shift is mandatory to get file path - shift - EXPORT_FILE="$1" ; shift + # first shift is mandatory to get file path + shift + EXPORT_FILE="$1" ; shift ;; -h|--help) - help ; exit 0 ; shift + help ; exit 0 ; shift ;; --db) - do_db_import=1 ; do_full_import=0 ; shift + do_db_restore=0 ; do_db_import=1 ; do_full_import=0 ; shift + ;; + --db-restore) + do_db_restore=1 ; do_db_import=1 ; do_full_import=0 ; shift ;; --conf) do_config_import=1 ; do_full_import=0 ; shift @@ -214,11 +237,11 @@ while true ; do --skip-adjust-conf) do_adjust_config=0 ; shift ;; - --) - shift ; break + --) + shift ; break ;; - *) - echo "Wrong usage !" ; help ; exit 1 + *) + echo "Wrong usage !" ; help ; exit 1 ;; esac done diff --git a/addons/full-upgrade/run-upgrade.sh b/addons/full-upgrade/run-upgrade.sh index 5b68b6dfa941..6a847e909ba0 100755 --- a/addons/full-upgrade/run-upgrade.sh +++ b/addons/full-upgrade/run-upgrade.sh @@ -232,8 +232,7 @@ echo "Stopping the PacketFence services" main_splitter export_to="/root/packetfence-pre-upgrade-backup-`date '+%s'`.tgz" echo "Generating full pre-upgrade backup to $export_to" -/usr/local/pf/addons/backup-and-maintenance.sh -/usr/local/pf/addons/full-import/export.sh $export_to +/usr/local/pf/addons/exportable-backup.sh -f $export_to main_splitter INCLUDE_OS_UPDATE="${INCLUDE_OS_UPDATE:-}" diff --git a/addons/functions/database.functions b/addons/functions/database.functions index cf2b72a8dd97..3cc1f90a20bf 100644 --- a/addons/functions/database.functions +++ b/addons/functions/database.functions @@ -45,11 +45,6 @@ function import_mysqldump() { # We reimport the schema so that we have the functions and triggers if the dump doesn't contain the triggers if ! egrep "CREATE.*TRIGGER.*(AFTER|BEFORE)" $dump_file > /dev/null; then echo "Dump file was made without triggers and procedures" - # Detect minor version of the restored data - restoring_version=`egrep -o '[0-9]+\.[0-9]+\.[0-9]+$' usr/local/pf/conf/pf-release | egrep -o '^[0-9]+\.[0-9]+'` - echo "Importing bare schema for $restoring_version" - mysql ${mariadb_args} $db_name < /usr/local/pf/db/pf-schema-$restoring_version.sql - check_code $? sub_splitter echo "Replacing CREATE TABLE and DROP TABLE statements" @@ -65,14 +60,6 @@ function import_mysqldump() { sed -i 's/DELETE IGNORE FROM `action`;//g' $dump_file sed -i 's/DELETE IGNORE FROM `activation`;//g' $dump_file - # We need to drop the data from all the tables that contain data in our bare schema - # This is because the mysqldump import already contains this data - mysql ${mariadb_args} $db_name -e 'delete ignore from node_category;' - mysql ${mariadb_args} $db_name -e 'delete ignore from password;' - mysql ${mariadb_args} $db_name -e 'delete ignore from person;' - mysql ${mariadb_args} $db_name -e 'delete ignore from pf_version;' - mysql ${mariadb_args} $db_name -e 'delete ignore from radreply;' - mysql ${mariadb_args} $db_name -e 'delete ignore from sms_carrier;' else echo "Dump file includes triggers and procedures" fi @@ -257,3 +244,13 @@ function uninstall_current_mariabackup() { yum remove -q -y MariaDB-backup fi } + +function handle_devel_db_schema() { + local restoring_version=$1 + if [ ! -f /usr/local/pf/db/pf-schema-$restoring_version.sql ]; then + main_splitter + echo "The db schema for $restoring_version does not exist in /usr/local/pf/db/. pf-schema-X.Y.sql will be used." + sub_splitter + cp /usr/local/pf/db/pf-schema-X.Y.sql /usr/local/pf/db/pf-schema-$restoring_version.sql + fi +} diff --git a/debian/packetfence.cron.d b/debian/packetfence.cron.d index 5eea0b9d654b..a3782adaf709 100644 --- a/debian/packetfence.cron.d +++ b/debian/packetfence.cron.d @@ -3,7 +3,7 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -30 0 * * * root /usr/local/pf/addons/backup-and-maintenance.sh +30 0 * * * root /usr/local/pf/addons/exportable-backup.sh # Renew any Let's Encrypt certificates the first of the month 1 0 1 * * root /usr/local/pf/bin/pfcmd renew_lets_encrypt diff --git a/docs/cluster/layer_3_clusters.asciidoc b/docs/cluster/layer_3_clusters.asciidoc index 1baa48459708..8576ea2f9914 100644 --- a/docs/cluster/layer_3_clusters.asciidoc +++ b/docs/cluster/layer_3_clusters.asciidoc @@ -416,7 +416,7 @@ First restart packetfence-mariadb on all of the servers in the main cluster. systemctl restart packetfence-mariadb -Run the /usr/local/pf/addons/backup-and-maintenance.sh script on the master node of the main cluster. If you do not know which server is the master run this command on all nodes in the main cluster and only the master will create a backup file (eg: /root/backup/packetfence-db-dump-innobackup-YYYY-MM-DD_HHhss.xbstream.gz). +Run the /usr/local/pf/addons/exportable-backup.sh script on the master node of the main cluster. If you do not know which server is the master run this command on all nodes in the main cluster and only the master will create a backup file (eg: /root/backup/packetfence-exportable-backup-YYYY-MM-DD_HHhss.tgz). Transfer this file to the remote server (eg: /root/backup/) Connect to the remote server and perform the following to sync the configuration from the master cluster: @@ -425,18 +425,8 @@ Connect to the remote server and perform the following to sync the configuration /usr/local/pf/bin/pfcmd configreload hard Then the following command to import the backup: - - mkdir /root/backup/restore - cd /root/backup/restore - cp ../packetfence-db-dump-innobackup-YYYY-MM-DD_HHhss.xbstream.gz . - gunzip packetfence-db-dump-innobackup-YYYY-MM-DD_HHhss.xbstream.gz - mbstream -x < packetfence-db-dump-innobackup-YYYY-MM-DD_HHhss.xbstream - mv packetfence-db-dump-innobackup-YYYY-MM-DD_HHhss.xbstream ../ - mariabackup --prepare --target-dir=./ - systemctl stop packetfence-mariadb - rm -fr /var/lib/mysql/* - mariabackup --innobackupex --defaults-file=/usr/local/pf/var/conf/mariadb.conf --move-back --force-non-empty-directories ./ - chown -R mysql: /var/lib/mysql + + /usr/local/pf/addons/full-import/import.sh --db -f /root/backup/packetfence-exportable-backup-YYYY-MM-DD_HHhss.tgz systemctl start packetfence-mariadb On the master node of the main cluster, grant replication for the replication user: diff --git a/docs/cluster/maintenance_and_operations.asciidoc b/docs/cluster/maintenance_and_operations.asciidoc index d2d0dcabcb90..c94e54b8cf20 100644 --- a/docs/cluster/maintenance_and_operations.asciidoc +++ b/docs/cluster/maintenance_and_operations.asciidoc @@ -146,20 +146,21 @@ If you need to externalize those backups, they are in: ---- /root/backup ---- -Files description: +File description: -* `packetfence-db-dump-innobackup-DATE_00h30.xbstream.gz` are the SQL dump of your MariaDB database. -* `packetfence-files-dump-DATE_00h30.tgz` are the dump of the PacketFence files. +* packetfence-exportable-backup-DATE_00h30.tgz is an exportable packetfence backup that contains: + * `packetfence-db-dump-innobackup-DATE_00h30.xbstream.gz` are the SQL dump of your MariaDB database. + * `packetfence-config-dump-DATE_00h30.tgz` are the dump of the PacketFence files. ==== Manual backups In case you need to make a "manual" backup, you can type the following command: ---- -/usr/local/pf/addons/backup-and-maintenance.sh +/usr/local/pf/addons/exportable-backup.sh ---- -As the daily automatic backups, you will find the files in: +As the daily automatic backups, you will find the file in: ---- /root/backup/ diff --git a/docs/installation/export_import_mechanism.asciidoc b/docs/installation/export_import_mechanism.asciidoc index c1bcf76586fd..31af47a5b1bc 100644 --- a/docs/installation/export_import_mechanism.asciidoc +++ b/docs/installation/export_import_mechanism.asciidoc @@ -93,14 +93,7 @@ configuration in your export, run: [source,bash] ---- -/usr/local/pf/addons/backup-and-maintenance.sh ----- - -Next, run the export script: - -[source,bash] ----- -/usr/local/pf/addons/full-import/export.sh /tmp/export.tgz +/usr/local/pf/addons/exportable-backup.sh -f /tmp/export.tgz ---- The command above will create your export archive in [filename]`/tmp/export.tgz`. You will diff --git a/docs/installation/performance_optimizations.asciidoc b/docs/installation/performance_optimizations.asciidoc index 876cd8e77b29..08b90a3f275d 100644 --- a/docs/installation/performance_optimizations.asciidoc +++ b/docs/installation/performance_optimizations.asciidoc @@ -297,7 +297,7 @@ This will grind PacketFence to a halt so you want to avoid that at all cost. One ==== Using MariaDB-backup -When dealing with a large database, the database backup and maintenance script (`/usr/local/pf/addons/backup-and-maintenance.sh`) which uses mysqldump may create a long lock on your database which may cause service to hang. +When dealing with a large database, the database backup and maintenance script (`/usr/local/pf/addons/exportable-backup.sh`) which uses mysqldump may create a long lock on your database which may cause service to hang. This is fixed easily by using MariaDB-backup which can complete a full database backup without locking your tables. @@ -325,9 +325,9 @@ Once this is done, grant the proper rights to the `pf` user (or the one you conf MariaDB> GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'pf'@'localhost'; MariaDB> FLUSH PRIVILEGES; -Next, run the maintenance script [filename]`/usr/local/pf/addons/backup-and-maintenance.sh` and ensure that the following line is part of the output: +Next, run the maintenance script [filename]`/usr/local/pf/addons/exportable-backup.sh` and ensure that the following line is part of the output: - innobackupex: completed OK! + Exportable backup is done If the backup fails, check [filename]`/usr/local/pf/logs/innobackup.log` for details and refer to the MariaDB-backup documentation for troubleshooting. diff --git a/packetfence.cron.d b/packetfence.cron.d index 7146e3746eaf..c42d3fc247be 100644 --- a/packetfence.cron.d +++ b/packetfence.cron.d @@ -4,7 +4,7 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin # Database backup and maintenance script -30 00 * * * root /usr/local/pf/addons/backup-and-maintenance.sh +30 00 * * * root /usr/local/pf/addons/exportable-backup.sh # Renew any Let's Encrypt certificates the first of the month 1 0 1 * * root /usr/local/pf/bin/pfcmd renew_lets_encrypt diff --git a/t/venom/test_suites/backup/backup_db.yml b/t/venom/test_suites/backup/backup_db.yml index 60919974a50d..2d8c985e3d2c 100644 --- a/t/venom/test_suites/backup/backup_db.yml +++ b/t/venom/test_suites/backup/backup_db.yml @@ -3,4 +3,4 @@ testcases: - name: backup steps: - type: exec - script: /usr/local/pf/addons/backup-and-maintenance.sh + script: /usr/local/pf/addons/exportable-backup.sh diff --git a/t/venom/test_suites/backup_db_and_restore/00_backup_db_and_restore.yml b/t/venom/test_suites/backup_db_and_restore/00_backup_db_and_restore.yml index ea1c18641602..be2b76fdc97e 100644 --- a/t/venom/test_suites/backup_db_and_restore/00_backup_db_and_restore.yml +++ b/t/venom/test_suites/backup_db_and_restore/00_backup_db_and_restore.yml @@ -20,52 +20,20 @@ testcases: assertions: - result.statuscode ShouldEqual 201 -- name: backup +- name: create exportable backup steps: - type: exec - script: /usr/local/pf/addons/backup-and-maintenance.sh + script: /usr/local/pf/addons/exportable-backup.sh - name: get_backup_name steps: - type: exec - script: 'find {{.backup_db_and_restore.backup_dir}} -name "packetfence-db-dump-*.sql.gz" -newermt "-1 minute"' - vars: - backup_name: - from: result.systemout + script: 'find {{.backup_db_and_restore.backup_dir}} -name "packetfence-exportable-backup-*.tgz" -newermt "-1 minute"' -- name: unzip_db_backup +- name: import steps: - type: exec - script: 'gunzip {{.get_backup_name.backup_name}}' - -# we only get filename without path -- name: get_backup_name_uncompressed - steps: - - type: exec - script: 'basename {{.get_backup_name.backup_name}} .gz' - vars: - backup_name_uncompressed: - from: result.systemout - -- name: drop_pf_db - steps: - - type: exec - script: mysql -e "DROP DATABASE pf;" - -- name: create_pf_db - steps: - - type: exec - script: mysql -e "CREATE DATABASE pf;" - -- name: restore_schema - steps: - - type: exec - script: 'mysql pf < /usr/local/pf/db/pf-schema.sql' - -- name: restore_db - steps: - - type: exec - script: 'mysql pf < {{.backup_db_and_restore.backup_dir}}/{{.get_backup_name_uncompressed.backup_name_uncompressed}}' + script: '/usr/local/pf/addons/full-import/import.sh --db-restore -f {{.get_backup_name.result.systemout}}' - name: search_user_in_db steps: diff --git a/t/venom/test_suites/backup_db_and_restore/TESTSUITE.md b/t/venom/test_suites/backup_db_and_restore/TESTSUITE.md index d56d1a7554d6..77d38f432678 100644 --- a/t/venom/test_suites/backup_db_and_restore/TESTSUITE.md +++ b/t/venom/test_suites/backup_db_and_restore/TESTSUITE.md @@ -5,12 +5,8 @@ MariaDB running and available using UNIX socket ## Scenario steps 1. Create user in DB using API -2. Backup files and DB with backup-and-maintenance script -3. Check DB file has been created by backup script -4. Unzip DB file -5. Drop DB -6. Recreate DB based on current schema -7. Restore data from backup DB file +2. Backup files and DB with exportable-backup script +7. Import only db from backup 8. Check that user created at first step is still here using API: validate that application is running after DB restore