From a78dff2ee06eedc0af8283bc27a934e321a8ccc8 Mon Sep 17 00:00:00 2001
From: Florian Pritz <bluewind@xinu.at>
Date: Wed, 13 Apr 2016 18:57:00 +0200
Subject: backup.sh: Add borg support; improve fs whitelisting

Signed-off-by: Florian Pritz <bluewind@xinu.at>
---
 backup.sh | 91 ++++++++++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 58 insertions(+), 33 deletions(-)

(limited to 'backup.sh')

diff --git a/backup.sh b/backup.sh
index 760d78e..e191f3a 100755
--- a/backup.sh
+++ b/backup.sh
@@ -6,6 +6,9 @@
 # Important steps:
 #  - define a host "backup" in root's .ssh/config
 #  - read the script and adjust to your needs
+#  - export BORG_REPO in your environment or create a wrapper in $PATH that
+#    does it for calls to borg (my preference). For details on the variable
+#    see man borg
 
 set -e
 
@@ -17,26 +20,28 @@ main() {
 	TMPDIR="$(mktemp -d "/tmp/${0##*/}.XXXXXX")"
 	trap "rm -rf '${TMPDIR}'" EXIT TERM
 
-	# ensure duplicity keeps its cache at a central location
-	export HOME=/root
-
 	# if you want to encrypt the backups remove --no-encryption in the duplicity call
 	# and uncomment the lines that contain PASSPHRASE
 	#PASSPHRASE="randomstringhere"
 
 	# these mountpoints will be excluded
 	excludeMountpoints=(
-	/tmp/
-	/sys/
-	/dev/
-	/proc/
-	/run/
-	/mnt/levant/nfs/
-	/media/
+	/tmp
+	/sys
+	/dev
+	/proc
+	/run
+	/mnt/levant/nfs
+	/media
 	)
 
-	# mountpoints of these types do not need to be excluded
-	fsWhitelist=(ext4 btrfs)
+	# these mountpoints will be included
+	includeMountpoints=(
+	/
+	/boot
+	/home
+	/mnt/data
+	)
 
 	# first line that matches wins
 	IFS='' read -r -d '' excludeList <<EOF || true
@@ -46,10 +51,21 @@ main() {
 - /home/*/.claws-mail/imapcache
 - /root/.cache/*
 - /var/cache/pacman/pkg/*
+EOF
+
+	# same as above, but for borg backup. Note that borg uses a different
+	# syntax and does not support including lower level directories
+	# (TODO: verify this claim)
+	IFS='' read -r -d '' excludeList_borg <<EOF || true
+sh:/home/flo/tmp/*
+sh:/home/*/.cache/*
+sh:/root/.cache/*
+sh:/var/cache/pacman/pkg/*
 EOF
 
 	exclude_mountpoints
 	echo "$excludeList" > "$TMPDIR/exclude-list"
+	echo "$excludeList_borg" > "$TMPDIR/exclude-list-borg"
 
 	# save some data that's useful for restores
 	local backupDataDir=/root/backup-data/
@@ -62,31 +78,29 @@ EOF
 	df -a > "$backupDataDir/df"
 	findmnt -l > "$backupDataDir/findmnt"
 
-	# this does not ignore /proc and network mounts so it's not that useful :(
-	#find / | gzip > /root/full-file-list.txt.gz
-
-	local backupdir="$HOSTNAME-backup/full-backup"
+	backup_borg /
 
-	backup / "sftp://backup/$backupdir/" --exclude-filelist "$TMPDIR/exclude-list"
-	ssh backup "touch $backupdir/last-backup-timestamp"
+	#local backupdir="$HOSTNAME-backup/full-backup"
+	#backup_duplicity / "sftp://backup/$backupdir/" --exclude-filelist "$TMPDIR/exclude-list"
+	#ssh backup "touch $backupdir/last-backup-timestamp"
 }
 
-backup() {
+backup_duplicity() {
 	local src=$1
 	local dest=$2
 	shift 2
 	local -a options=()
 
-	if [[ $(date +%u ) == '1' ]]; then
-		# try to only run full backups on monday (1)
+	if [[ $(date +%d ) == '1' ]]; then
+		# try to only run full backups on day 1 each month
 		options+=(--full-if-older-than 2D)
 	else
 		# force a full backup once in a while
-		options+=(--full-if-older-than 20D)
+		options+=(--full-if-older-than 30D)
 	fi
 
 	#export PASSPHRASE
-	duplicity \
+	HOME=/root duplicity \
 		-v5 \
 		--numeric-owner \
 		--volsize 250 \
@@ -95,10 +109,25 @@ backup() {
 		--no-encryption \
 		"${options[@]}" "$@" "$src" "$dest"
 
-	duplicity --force remove-older-than 120D "$dest"
+	HOME=/root duplicity --force remove-older-than 120D "$dest"
 	#export PASSPHRASE=""
 }
 
+backup_borg() {
+	local src=$1
+
+	borg create \
+		-v \
+		--numeric-owner \
+		--compression lz4 \
+		--stats \
+		--progress \
+		--exclude-from "$TMPDIR/exclude-list-borg" \
+		"::backup-$(date "+%Y%m%d-%H%M%S")" "$src"
+
+	borg prune -v --keep-within 3m
+}
+
 ### support functions below ###
 
 ##
@@ -129,21 +158,17 @@ exclude_mountpoints() {
 	local error=0
 
 	for fs in "${excludeMountpoints[@]}"; do
-		if [[ $fs != */ ]]; then
-			error=1
-			echo "Error: excludeMountpoints entry doesn't end with /: $fs" >&2
-		fi
-		excludeList+="- $fs*"$'\n'
+		excludeList+="- $fs/*"$'\n'
+		excludeList_borg+="sh:$fs/*"$'\n'
 	done
 
 	while read line; do
 		local mountpoint=$(echo "$line" | cut -d\  -f2 | sed 's#\040# #g;')
-		local type=$(echo "$line" | cut -d\  -f3)
 
-		if ! in_array $type "${fsWhitelist[@]}"; then
-			if ! in_array_startswith "$mountpoint/" "${excludeMountpoints[@]}"; then
+		if ! in_array $mountpoint "${includeMountpoints[@]}"; then
+			if ! in_array_startswith "$mountpoint/" "${excludeMountpoints[@]/%//}"; then
 				error=1
-				echo "Warning: mountpoint not excluded: $mountpoint" >&2
+				echo "Warning: mountpoint not excluded or included: $mountpoint" >&2
 			fi
 		fi
 	done </etc/mtab
-- 
cgit v1.2.3-24-g4f1b