Shortly before this time last year I revolutionised my backup infrastructure. This year I finally got around to scripting updating the off-site version.

I’m afraid this blog post largely just consists of the script, with a brief list of pre-requisites to punctuate it.

Pre-requisites

  • Off-site backup disk partitions labelled backuppc-offsite[0-9] where [0-9] is a single digit (sequential for each off-site copy).

The script

#!/bin/bash

set -e

MYLOCK="/tmp/$(basename $0)_lock"
CONFIG_BACKUP_TARGET="/var/lib/backuppc/etc-backuppc.tgz"
BACKUPPC_VG_NAME="backuppc"
BACKUPPC_LV_NAME="store"

# LVM Snapshot settings
SNAPSHOT_SIZE="500G"
SNAPSHOT_NAME=backup

# Luks settings
REMOVABLE_DEV_NAME="backuppc-backup-removable"

# Devices
MAPPER_BASE="/dev/mapper/"
BACKUPPC_LV_BASE="${MAPPER_BASE}${BACKUPPC_VG_NAME}-"
BACKUPPC_LV_DEV="${BACKUPPC_LV_BASE}${BACKUPPC_LV_NAME}"
SNAPSHOT_LV_DEV="${BACKUPPC_LV_BASE}${SNAPSHOT_NAME}"
REMOVABLE_DEV="${MAPPER_BASE}${REMOVABLE_DEV_NAME}"

dated_stderr_out() {
	echo $( date "+%F %R:%S.%N" ) "$@" >&2
}

error() {
	dated_stderr_out "ERROR:" "$@"
}

warning() {
	dated_stderr_out "WARN:" "$@"
}

progress() {
	dated_stderr_out "$@..."
}

update_pid() {
	echo $$ > "$MYLOCK/pid"
}

find_usb_disk() {
	file_list=(/dev/disk/by-partlabel/backuppc-offsite[0-9])
	if [[ ${#file_list[@]} -ne 1 ]]
	then
		error "More than one partition matching /dev/disk/by-partlabel/backuppc-offsite[0-9] found"
		exit 1
	else
		OFFSITE_PART=${file_list[0]}
		OFFSITE_DISK="/dev/$( lsblk -no pkname "$OFFSITE_PART" )"
	fi
}

if [[ $UID -ne 0 ]]
then
	error "Script must be run as root"
	exit 1
fi

# mkdir is atomic, useful for locking purposes - see: http://mywiki.wooledge.org/BashFAQ/045
if ! mkdir "$MYLOCK"
then
	if [ -f "$MYLOCK/pid" ]
	then
		if ! [ -d /proc/$( cat "$MYLOCK/pid" ) ]
		then
			# Trying to update the pid file would not be atomic, so just delete it and signal to rerun script.  Anything else might result in a race condition.
			warning "Lock file found but no such pid running"
			# Try and get an atomic update lock
			if mkdir "$MYLOCK/pid_update"
			then
				progress "Got update lock"
				update_pid
				# Deliberately leave the pid_update lock so nothing else can acquire it until this script ends
			else
				error "Lock directory found at $MYLOCK, pid in pid file does not exist but could not acquire lock to update."
				exit 1
			fi
		else
			error "Lock directory found at $MYLOCK but no pid file inside.  Aborting (possible race condition and another script is starting?)"
			exit 1
		fi
	else
		error "Unknown problem creating lock directory at $MYLOCK"
		exit 1
	fi
else
	# Got the lock directory atomically - write our pid out
	update_pid
fi

progress "Backing up config"
in_progress_target="${CONFIG_BACKUP_TARGET}-inprogress"
[ -e "$in_progress_target" ] && rm "$in_progress_target"
tar -zcf "$in_progress_target" /etc/backuppc
mv "$in_progress_target" "$CONFIG_BACKUP_TARGET"

progress "Taking LVM snapshot"
lvcreate -L$SNAPSHOT_SIZE -s -n "$SNAPSHOT_NAME" "$BACKUPPC_LV_DEV"

progress "Opening encrypted USB disk"
find_usb_disk
cryptsetup luksOpen "$OFFSITE_PART" "$REMOVABLE_DEV_NAME"

progress "Cloning partition (this will take a while)"
time e2image -ra -pc "${SNAPSHOT_LV_DEV}" "${REMOVABLE_DEV}"

progress "Running fsck on clone"
fsck -t ext4 "${REMOVABLE_DEV}" -f -n

progress "Closing encrypted USB disk"
cryptsetup luksClose "${REMOVABLE_DEV}"

progress "Powering off USB disk"
udisksctl power-off -b "${OFFSITE_DISK}"

progress "Removing snapshot"
lvremove -y "${SNAPSHOT_LV_DEV}"

progress "Releasing script lock"
rm -rf "$MYLOCK"