rsunk

#!/usr/bin/env bash
#
#  Synopsis:
#	Push /d1/d2/... to user@rsync.net:sunk/host/$(hostname -f)/root/d1/d2
#  Description:
#  	Rsync a precious, stable directory to an rsync.net server and send
#  	status email.  Files that change during the sync will generate
#  	an error, unless given the option --ignore-24.
# 
#  	Files are sunk in archive mode to user@rsync.net:sunk/host/$(hostname).
#	Directories are	created as needed.  Stale files on the remote server
#	are deleted.  An itemized summary of all sunk files is mailed to those
#	who want to know (see option --notify).
#
#  	Directory level filter rules in .rsunk-filter are observed.
#	Remember, paths in .rsunk-filter are relative to directory
#	containing the .rsunk-filter file .  In other words, absolute paths 
#	are meaningless.
#
#  	Obviously, the script assumes the remote server supports an rsync.net
#  	like environment;  i.e, certain programs can be run on the remote
#  	server,	like du, mkdir, find.
#
#	For more pedestrian file-system level rsync backups, see
#	https://github.com/jmscott/work/rsync-fs
#  Usage:
#	rsunk								\
#		--ignore-24						\
#		--notify jmscott@setspace.com				\
#		--notify jmscott@rj2tech.com				\
#		--ssh-auth 8008@usw-s008.rsync.net			\
#		--source-path /						\
#			>>/usr/local/log/rsunk-root.log
#	rsunk --notify jmscott --source-path /home/svn --ssh-auth jmscott
#	rsunk --notify jmscott --source-path /home/jmscott/backup 	\
#		--ssh-auth jmscott --itemize
#  Options:
#	--source-path <dir>	Path to root of source directory
#	--ignore-24		Ignore rsnc exit status 24 (file disappeared)
#	--notify <email>	Send summary report to <email>.  Multiple ok.
#	--ssh-auth		Authorization at rsync.net
#  See:
#	https://github.com/jmscott/work/rsunk
#	https://rsync.net
#  Note:
#	Add option --template to generate reasonable /.rsunk-filter file.
#
#	Add option --rsync-filter-path.  Not sure if global /.rsunk-filter
#	makes too many assumptions about access to root.  In principle
#	extra merge rules can be added to /.rsunk-filter, but not total sure
#	how all that works.
#

START_EPOCH=$(date +%s)
PROG=$(basename $0)
SOURCE_PATH=
TARGET=
SSH_AUTH=
DU=

#  ignore exit status 24 (some files changed)
IGNORE_24=no

RSYNC_ARGS="
	--numeric-ids
	--verbose
	--archive
	--delete
	--delete-excluded
	--itemize-changes
	--human-readable
	--compress
"

TARGET_PATH=sunk/host/$(hostname -f)/root/
RSYNC_OUT=
RSYNC_TEE=cat
ITEMIZE=no

log()
{
	echo "$(date +'%Y/%m/%d %H:%M:%S'): $@"
}

leave()
{
	rm -f $RSYNC_OUT
	log "good bye, cruel world: elapsed $(elapsed-english $START_EPOCH)"
}

do_mail()
{
	SUBJECT="$1"

	if [ -z "$NOTIFY" ];  then
		log 'ERROR: do_mail: --option notify required' >&2
		exit 1
	fi
	(
		cat
		cat <<END

Begin Process:

	Elapsed: $(elapsed-english $START_EPOCH)
  Hostname FQDN: $(hostname --fqdn)
           User: $USER
    Script Path: $0
     Start Time: $RSYNC_START_TIME
       End Time: $(date --rfc-3339=seconds)
       SSH Auth: $SSH_AUTH
    Source Path: $SOURCE_PATH
         Target: $TARGET
     Rsync Args: $RSYNC_ARGS
END
	#
	#  Tack on disk usage, start end times
	#
	test -n "$DU" && echo "	Disk Usage: $DU"
	#
	#  Concatenate rsync output. 
	# 
	if [ -s $RSYNC_OUT ];  then
		cat <<END

	Begin rsync stdout/err:

END
		cat $RSYNC_OUT
		cat <<END

	  End rsync stdout/err:
END
	fi

	#
	#  Process environment
	#
	cat <<END

	Begin Environment:
END
		env
		cat <<END
	  End Environment:

END Process:
END
	) | tr -d '\r' | mail -s "$SUBJECT" $NOTIFY
}

die()
{
	log "ERROR: $@" >&2

	#
	#
	#
	MSG="$@"
	if [ -n "$SOURCE_PATH" ];  then
		MSG=$(basename $SOURCE_PATH)
	fi
	do_mail "ERROR: $PROG: $MSG" <<END
In process $PROG#$$ the following error occured:

    $@

For more details, see the output of standard error of this script log file,
probably defined in a crontab for the user $USER.

-$USER
END
	exit 1
}

trap leave EXIT

while [ "$1" ];  do
	ARG="$1"
	shift
	case "$ARG" in
	--notify)
		N="$1"
		test -n "$N" || die 'option --notify: missing user@host'
		NOTIFY="$NOTIFY $N"
		shift
		;;
	--ssh-auth)				# user@host
		test -z "$SSH_AUTH" ||
				die 'option --ssh-auth: given more than once'
		test -n "$1" || die 'option --ssh-auth: missing auth'

		SSH_AUTH="$1"
		shift
		;;
	--source-path)		#  file path
		test -z "$SOURCE_PATH" 					||
				die 'option --source-path: given more than once'
		case "$1" in
		'')
			die 'option --source-path: missing /d1/d2/...'
			;;
		/)
			;;
		/*/)
			die 'option --source-path: path can not end in /'
			;;
		/*)
			;;
		*)
			die "option --source-path: not full path: $SOURCE_PATH"
			;;
		esac
		SOURCE_PATH="$1"
		shift
		;;
	--one-file-system)
		RSYNC_ARGS="$RSYNC_ARGS --one-file-system"
		;;
	--dry-run)
		RSYNC_ARGS="$RSYNC_ARGS --dry-run"
		;;
	--ignore-24)
		IGNORE_24=yes
		;;
	--itemize)
		ITEMIZE=yes
		RSYNC_OUT=${TMPDIR:=/tmp}/$PROG.$$
		RSYNC_TEE="tee $RSYNC_OUT"
		;;
	--exclude)
		test -n "$1" || die 'option --exclude: missing pattern'
		RSYNC_ARGS="$RSYNC_ARGS --exclude $1"
		shift
		;;
	--*)
		die "unknown option: $ARG"
		;;
	*)
		die "unknown argument: $ARG"
		;;
	esac
done

log 'hello, world'

test -n "$NOTIFY" || die 'missing option --notify'
log "notify: $NOTIFY"

test -n "$SSH_AUTH" || die 'missing option --ssh-auth'
log "ssh auth: $SSH_AUTH"
log "rsync args: $RSYNC_ARGS"
log "ignore exit status 24: $IGNORE_24"
log "start epoch: $START_EPOCH"

test -n "$SOURCE_PATH" || die 'missing option: --source-path'
log "source path: $SOURCE_PATH"
log "itemize: $ITEMIZE"

log "target path: $TARGET_PATH"

TARGET="$SSH_AUTH:$TARGET_PATH"
log "rsync target: $TARGET"
log 'starting rsync ...'

RSYNC_START_TIME=$(date --rfc-3339=seconds)
#
#
#
log "make sure target path $TARGET_PATH exists on rsync.net"
ssh $SSH_AUTH "mkdir -p $TARGET_PATH" ||
			die "rsync.net mkdir -p failed: exit status=$?"

#
#  Call rsync.
#
#  Note:
#	Notice the explicit adding of the command line argument
#
#		--filter='dir-merge /.rsunk-filter'
#
#	See gripe at top of file.
#
set -x
time rsync --filter='dir-merge /.rsunk-filter'				\
		$RSYNC_ARGS $SOURCE_PATH $TARGET 2>&1 | $RSYNC_TEE
STATUS="${PIPESTATUS[*]}"
set +x
case $STATUS in
'0 0')
	log 'done, no errors in rsync'
	;;
'24 0')
	if [ $IGNORE_24 = yes ];  then
		log "WARN: some files changed during transfer (ok)"
	else
		die "rsync: some files changed during transfer"
	fi
	;;
*)
	die "rsync failed: exit status=$STATUS"
	;;
esac

log 'calculating disk usage'
DU=$(ssh $SSH_AUTH du -sh $TARGET_PATH/$(basename $SOURCE_PATH))
log "disk usage: $DU"

echo 'Done. No Errors.'							|
		do_mail "OK: $PROG: $DU in $(elapsed-english $START_EPOCH)"