dillard
d96229f370
A killed restore left its leases for up to two hours, so the first backup of the recovered server was refused. A backup from before 1.0.3 records no empty folders: the test now expects that.
378 lines
26 KiB
Bash
Executable File
378 lines
26 KiB
Bash
Executable File
#!/bin/bash
|
|
# Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a
|
|
# SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup
|
|
# location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7.
|
|
# Usage: full-restore-roundtrip.sh <nextcloud-image-tag> [sqlite|pgsql|mysql] [legacy|sftp|salvage|salvage-files|to-pgsql|to-mysql|to-sqlite]
|
|
# e.g. full-restore-roundtrip.sh 35-apache pgsql
|
|
# With "salvage-files" one pack is damaged and another is gone from the location. The restore must
|
|
# stop and point at --salvage; run again as the guide says, it restores the database and every file
|
|
# that can be read, and lists the rest (exit 2). NGB_FRT_DATA_MB sets the amount of data (default 200).
|
|
# With "sftp" the location is an SFTP server instead of a shared directory, and the origin has a
|
|
# file spread over several packs: the restore reads them ahead on connections of its own.
|
|
# With "to-<db>" the restore instance runs that database whatever the origin runs (a MySQL or
|
|
# SQLite backup records no ID counters; after the restore new rows must not collide with restored
|
|
# ones, whatever the database on either side).
|
|
# With "salvage" the snapshot's database dump is damaged on the location: the restore must stop
|
|
# without changing anything, and with --salvage put the files in a folder of their own.
|
|
# With "legacy" the origin runs NG Backup 1.0.0-rc.1 (0.9.7), whose backups do not record the
|
|
# structure of their tables: the restore must then leave the tables of the disabled app out
|
|
# with --skip-missing-tables, as for a real backup made before 1.0.0-rc.2.
|
|
# With pgsql or mysql the origin also has what broke real disaster recoveries: the Mail app with a synced
|
|
# account (foreign keys between its tables) and Deck installed and then switched off (tables of
|
|
# an app that is not enabled). The restore instance has neither; --install-apps adds Mail.
|
|
set -euo pipefail
|
|
IMAGE="nextcloud:${1:-35-apache}"
|
|
DB="${2:-sqlite}"
|
|
MODE="${3:-}"
|
|
LEGACY=""
|
|
[ "$MODE" = legacy ] && LEGACY=legacy
|
|
APP=$(cd "$(dirname "$0")/../.." && pwd)
|
|
ORIGIN_APP="$APP"
|
|
NET=ngb-frt-net
|
|
SHARED=$(mktemp -d)
|
|
chmod 777 "$SHARED"
|
|
fail() { echo "FAIL: $*"; exit 1; }
|
|
ok() { echo "ok $*"; }
|
|
|
|
cleanup() {
|
|
docker rm -fv ngb-frt-origin ngb-frt-restore ngb-frt-db-origin ngb-frt-db-restore ngb-frt-imap ngb-frt-sftp >/dev/null 2>&1 || true
|
|
docker network rm "$NET" >/dev/null 2>&1 || true
|
|
docker run --rm -v "$SHARED:/target" alpine sh -c 'rm -rf /target/*' >/dev/null 2>&1 || true
|
|
rmdir "$SHARED" 2>/dev/null || true
|
|
rm -f /tmp/ngb-frt-kit.json /tmp/ngb-frt-pass /tmp/ngb-frt-sums
|
|
[ -n "${LEGACY_DIR:-}" ] && rm -rf "$LEGACY_DIR"
|
|
return 0
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [ "$LEGACY" = legacy ]; then
|
|
LEGACY_DIR=$(mktemp -d)
|
|
curl -fsSL https://github.com/dillardblom/ng_backup/releases/download/v1.0.0-rc.1/ng_backup-0.9.7.tar.gz | tar -xz -C "$LEGACY_DIR"
|
|
ORIGIN_APP="$LEGACY_DIR/ng_backup"
|
|
fi
|
|
|
|
run_on() { docker exec -u www-data "$1" php occ "${@:2}"; }
|
|
# The app store sends the whole app list (~38 MB) and Nextcloud gives up after 120 s: on a slow day
|
|
# an install fails for reasons outside this test. Up to three tries.
|
|
install_app() { # container app
|
|
for i in 1 2 3; do run_on "$1" app:install "$2" >/dev/null 2>&1 && return 0; sleep 20; done
|
|
return 1
|
|
}
|
|
|
|
# Database settings for an instance: SQLite inside the container, or its own PostgreSQL container
|
|
db_env() {
|
|
if [ "$DB" = pgsql ]; then
|
|
docker run -d --name "ngb-frt-db-$1" --network "$NET" -e POSTGRES_DB=nextcloud -e POSTGRES_USER=nextcloud \
|
|
-e POSTGRES_PASSWORD=nextcloud postgres:16-alpine >/dev/null
|
|
for i in $(seq 1 30); do docker exec "ngb-frt-db-$1" pg_isready -q 2>/dev/null && break; sleep 2; done
|
|
echo "-e POSTGRES_HOST=ngb-frt-db-$1 -e POSTGRES_DB=nextcloud -e POSTGRES_USER=nextcloud -e POSTGRES_PASSWORD=nextcloud"
|
|
elif [ "$DB" = mysql ]; then
|
|
docker run -d --name "ngb-frt-db-$1" --network "$NET" -e MARIADB_DATABASE=nextcloud -e MARIADB_USER=nextcloud \
|
|
-e MARIADB_PASSWORD=nextcloud -e MARIADB_ROOT_PASSWORD=nextcloud mariadb:11 >/dev/null
|
|
for i in $(seq 1 60); do docker exec "ngb-frt-db-$1" mariadb-admin ping -unextcloud -pnextcloud --silent >/dev/null 2>&1 && break; sleep 2; done
|
|
echo "-e MYSQL_HOST=ngb-frt-db-$1 -e MYSQL_DATABASE=nextcloud -e MYSQL_USER=nextcloud -e MYSQL_PASSWORD=nextcloud"
|
|
else
|
|
echo "-e SQLITE_DATABASE=nextcloud"
|
|
fi
|
|
}
|
|
|
|
docker network create "$NET" >/dev/null
|
|
LOCATION=(local -a null::null -o datadir=/mnt/shared)
|
|
if [ "$MODE" = sftp ]; then
|
|
docker run -d --name ngb-frt-sftp --network "$NET" atmoz/sftp:alpine ngbackup:ngbackuppw:::upload >/dev/null
|
|
LOCATION=(sftp -a password::password -o host=ngb-frt-sftp -o root=/upload -o user=ngbackup -o password=ngbackuppw)
|
|
fi
|
|
ORIGIN_DB=$(db_env origin)
|
|
# shellcheck disable=SC2086
|
|
docker run -d --name ngb-frt-origin --network "$NET" $ORIGIN_DB \
|
|
-e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \
|
|
-v "$ORIGIN_APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null
|
|
for i in $(seq 1 90); do run_on ngb-frt-origin status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done
|
|
docker exec ngb-frt-origin chown www-data:www-data /var/www/html/custom_apps
|
|
run_on ngb-frt-origin app:enable ng_backup >/dev/null
|
|
ok "origin instance up ($DB${LEGACY:+, NG Backup 0.9.7})"
|
|
|
|
docker exec ngb-frt-origin sh -c "printf 'full restore roundtrip passphrase 2026\n' > /tmp/pass"
|
|
run_on ngb-frt-origin backup:key:init --passphrase-file=/tmp/pass --label=CI >/dev/null || fail "key init"
|
|
run_on ngb-frt-origin backup:key:kit --output=/tmp/kit.json >/dev/null
|
|
CODE=$(run_on ngb-frt-origin backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
|
|
run_on ngb-frt-origin backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "confirm"
|
|
run_on ngb-frt-origin backup:target:add offsite "${LOCATION[@]}" --path=repo >/dev/null || fail "target add"
|
|
ok "key, kit and location on the origin"
|
|
|
|
NC_PASS=$(openssl rand -hex 16)
|
|
docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-origin php occ user:add --password-from-env frtest >/dev/null
|
|
docker exec ngb-frt-origin sh -c "mkdir -p /var/www/html/data/frtest/files/Docs /var/www/html/data/frtest/files/Empty/Nested && head -c 200000 /dev/urandom > /var/www/html/data/frtest/files/Docs/report.bin && echo 'hello disaster recovery' > /var/www/html/data/frtest/files/Docs/note.txt && chown -R www-data:www-data /var/www/html/data/frtest"
|
|
if [ "$MODE" = sftp ]; then
|
|
# about three packs of 32 MiB
|
|
docker exec ngb-frt-origin sh -c "head -c 90000000 /dev/urandom > /var/www/html/data/frtest/files/Docs/large.bin && chown www-data:www-data /var/www/html/data/frtest/files/Docs/large.bin"
|
|
LARGE_SUM=$(docker exec ngb-frt-origin sha256sum /var/www/html/data/frtest/files/Docs/large.bin | cut -d' ' -f1)
|
|
fi
|
|
if [ "$MODE" = salvage-files ]; then
|
|
# Half in large files that fill packs of their own, half in files of 1 MiB; incompressible
|
|
DATA_MB=${NGB_FRT_DATA_MB:-200}
|
|
docker exec -e DATA_MB="$DATA_MB" ngb-frt-origin sh -c '
|
|
cd /var/www/html/data/frtest/files && mkdir -p Large Small/a Small/b
|
|
i=0; while [ $((i * 50)) -lt $((DATA_MB / 2)) ]; do head -c 52428800 /dev/urandom > Large/f$i.bin; i=$((i + 1)); done
|
|
i=0; while [ $i -lt $((DATA_MB / 2)) ]; do head -c 1048576 /dev/urandom > Small/$([ $((i % 2)) = 0 ] && echo a || echo b)/f$i.bin; i=$((i + 1)); done
|
|
chown -R www-data:www-data . && find . -type f -print0 | xargs -0 sha256sum > /tmp/frt-sums'
|
|
docker cp ngb-frt-origin:/tmp/frt-sums /tmp/ngb-frt-sums
|
|
ok "origin has $DATA_MB MiB of test data, $(wc -l < /tmp/ngb-frt-sums) files"
|
|
fi
|
|
run_on ngb-frt-origin files:scan frtest >/dev/null
|
|
|
|
# SQL on an instance's own database (pgsql/mysql only)
|
|
sql() {
|
|
if [ "$DB" = pgsql ]; then docker exec "ngb-frt-db-$1" psql -U nextcloud -d nextcloud -Atc "$2"
|
|
else docker exec "ngb-frt-db-$1" mariadb -unextcloud -pnextcloud -N -B nextcloud -e "$2"; fi
|
|
}
|
|
if [ "$DB" != sqlite ]; then
|
|
# Mail with a synced account: oc_mail_accounts points at oc_mail_mailboxes (foreign keys),
|
|
# while the restore loads tables in name order, accounts before mailboxes.
|
|
docker run -d --name ngb-frt-imap --network "$NET" -e MAILNAME=mail.domain.tld -e MAIL_ADDRESS=user@domain.tld \
|
|
-e MAIL_PASS=mypassword ghcr.io/christophwurst/docker-imap-devel:latest >/dev/null
|
|
install_app ngb-frt-origin mail || fail "install mail on the origin (app store unreachable?)"
|
|
run_on ngb-frt-origin config:system:set app.mail.verify-tls-peer --type=boolean --value=false >/dev/null
|
|
sleep 10
|
|
run_on ngb-frt-origin mail:account:create frtest Test user@domain.tld ngb-frt-imap 993 ssl user@domain.tld mypassword \
|
|
ngb-frt-imap 25 tls user@domain.tld mypassword >/dev/null || fail "mail account"
|
|
run_on ngb-frt-origin mail:account:sync -f 1 >/dev/null || fail "mail sync"
|
|
[ "$(sql origin 'select count(*) from oc_mail_accounts where drafts_mailbox_id is not null')" = 1 ] \
|
|
|| fail "the mail account does not point at its mailboxes"
|
|
# Deck installed and then switched off: its tables stay in the database and in the backup
|
|
install_app ngb-frt-origin deck || fail "install deck on the origin (app store unreachable?)"
|
|
# a 4-byte character (emoji) in a table the restore has to create: utf8mb4 on MySQL/MariaDB
|
|
sql origin "insert into oc_deck_boards (title, owner, color, archived, deleted_at, last_modified) values ('Plan 📌', 'frtest', '0087C5', false, 0, 0)" >/dev/null
|
|
run_on ngb-frt-origin app:disable deck >/dev/null
|
|
ok "origin has a synced Mail account and the tables of a disabled app"
|
|
fi
|
|
|
|
RUN_OUT=$(run_on ngb-frt-origin backup:run offsite -l disaster-test)
|
|
SNAP=$(printf '%s\n' "$RUN_OUT" | head -1 | awk -F'snapshot ' '{print $2}' | tr -d ',')
|
|
[ -n "$SNAP" ] || fail "no snapshot id from backup:run"
|
|
ok "test data backed up, snapshot $SNAP"
|
|
if [ "$MODE" = salvage ]; then
|
|
# The database dump of this snapshot damaged on the location: a byte flipped in its manifest
|
|
docker exec -u www-data ngb-frt-origin php -r '$p = "/mnt/shared/repo/db/" . $argv[1]; $d = file_get_contents($p); $i = intdiv(strlen($d), 2); $d[$i] = ~$d[$i]; file_put_contents($p, $d);' "$SNAP" \
|
|
|| fail "damage the database dump"
|
|
ok "database dump of the snapshot damaged on the location"
|
|
fi
|
|
if [ "$MODE" = salvage-files ]; then
|
|
# The two largest packs hold user files for sure: bytes flipped in one, the other gone
|
|
docker exec -u www-data ngb-frt-origin php -r '
|
|
$packs = glob("/mnt/shared/repo/packs/*/*");
|
|
usort($packs, fn ($a, $b) => filesize($b) <=> filesize($a));
|
|
count($packs) >= 3 or exit(1);
|
|
$h = fopen($packs[0], "r+b");
|
|
for ($o = 4096; $o < filesize($packs[0]); $o += 1048576) { fseek($h, $o); $b = fread($h, 16); fseek($h, $o); fwrite($h, ~$b); }
|
|
fclose($h);
|
|
unlink($packs[1]);' || fail "damage the packs"
|
|
ok "one pack damaged and one pack gone on the location"
|
|
fi
|
|
|
|
docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json
|
|
docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass
|
|
ORIGIN_DBTYPE=$DB
|
|
case "$MODE" in to-*) DB=${MODE#to-} ;; esac
|
|
RESTORE_DB=$(db_env restore)
|
|
# shellcheck disable=SC2086
|
|
docker run -d --name ngb-frt-restore --network "$NET" $RESTORE_DB \
|
|
-e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \
|
|
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null
|
|
for i in $(seq 1 90); do run_on ngb-frt-restore status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done
|
|
docker exec ngb-frt-restore chown www-data:www-data /var/www/html/custom_apps
|
|
run_on ngb-frt-restore app:enable ng_backup >/dev/null
|
|
docker cp /tmp/ngb-frt-kit.json ngb-frt-restore:/tmp/kit.json
|
|
docker exec ngb-frt-restore chmod 644 /tmp/kit.json
|
|
docker cp /tmp/ngb-frt-pass ngb-frt-restore:/tmp/pass
|
|
ok "fresh restore instance up"
|
|
|
|
run_on ngb-frt-restore backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass >/dev/null || fail "key import"
|
|
ADD_OUT=$(run_on ngb-frt-restore backup:target:add offsite "${LOCATION[@]}" --path=repo 2>&1) || { printf '%s\n' "$ADD_OUT"; fail "target add on the restore instance"; }
|
|
LIST_OUT=$(run_on ngb-frt-restore backup:list offsite 2>&1)
|
|
printf '%s\n' "$LIST_OUT" | grep -q "$SNAP" || {
|
|
printf 'target:add said:\n%s\nbackup:list said:\n%s\n' "$ADD_OUT" "$LIST_OUT"
|
|
docker exec ngb-frt-restore sh -c 'grep -h ng_backup /var/www/html/data/nextcloud.log | tail -5' || true
|
|
docker exec ngb-frt-restore sh -c 'ls -la /mnt/shared/repo /mnt/shared/repo/snapshots' || true
|
|
fail "the origin's snapshot is not listed after connecting the location"
|
|
}
|
|
ok "recovery kit imported, existing repository adopted, snapshot visible"
|
|
|
|
if [ "$MODE" = salvage ]; then
|
|
set +e
|
|
OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction 2>&1); CODE=$?
|
|
set -e
|
|
[ "$CODE" = 1 ] || { printf '%s\n' "$OUT" | tail -20; fail "a restore with a damaged database dump did not stop (exit $CODE)"; }
|
|
printf '%s\n' "$OUT" | grep -- "--salvage" >/dev/null || { printf '%s\n' "$OUT" | tail -20; fail "the stopped restore does not point at --salvage"; }
|
|
run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode stayed on after a restore that changed nothing"
|
|
ok "damaged database dump: the restore stops before changing anything and points at --salvage"
|
|
set +e
|
|
OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction --salvage 2>&1); CODE=$?
|
|
set -e
|
|
[ "$CODE" = 2 ] || { printf '%s\n' "$OUT" | tail -30; fail "restore:full --salvage exited with $CODE, not 2"; }
|
|
SALVAGE=/var/www/html/data/ngb-salvage-${SNAP:0:12}
|
|
printf '%s\n' "$OUT" | grep "$SALVAGE" >/dev/null || { printf '%s\n' "$OUT" | tail -30; fail "the output does not name the salvage folder"; }
|
|
[ "$(docker exec ngb-frt-restore cat "$SALVAGE/data/frtest/files/Docs/note.txt")" = "hello disaster recovery" ] || fail "the user's file is not in the salvage folder"
|
|
docker exec ngb-frt-restore test -f "$SALVAGE/config/config.php" || fail "config.php is not in the salvage folder"
|
|
docker exec ngb-frt-restore test -d "$SALVAGE/data/frtest/files/Empty/Nested" || fail "the empty folder is not in the salvage folder"
|
|
run_on ngb-frt-restore user:list | grep frtest >/dev/null && fail "the fresh database was changed (the backed-up user is there)"
|
|
run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the salvage"
|
|
run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after the salvage"
|
|
ok "with --salvage: the fresh database untouched, data and config of the backup in $SALVAGE (exit 2)"
|
|
echo "FULL RESTORE ROUNDTRIP PASS ($DB, salvage)"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$MODE" = salvage-files ]; then
|
|
set +e
|
|
OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction 2>&1); CODE=$?
|
|
set -e
|
|
[ "$CODE" = 1 ] || { printf '%s\n' "$OUT" | tail -20; fail "a restore with damaged data did not stop (exit $CODE)"; }
|
|
printf '%s\n' "$OUT" | grep -- "--salvage" >/dev/null || { printf '%s\n' "$OUT" | tail -20; fail "the stopped restore does not point at --salvage"; }
|
|
run_on ngb-frt-restore status 2>&1 | grep "maintenance: true" >/dev/null || fail "maintenance mode is off after a restore that stopped halfway"
|
|
ok "damaged data: the restore stops, keeps maintenance mode on and points at --salvage"
|
|
|
|
# As the guide and the message say: maintenance mode off and again, now with --salvage
|
|
run_on ngb-frt-restore maintenance:mode --off >/dev/null || fail "maintenance mode off"
|
|
set +e
|
|
OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction --salvage --force 2>&1); CODE=$?
|
|
set -e
|
|
[ "$CODE" = 2 ] || { printf '%s\n' "$OUT" | tail -30; fail "restore:full --salvage exited with $CODE, not 2"; }
|
|
printf '%s\n' "$OUT" | grep "could not be restored" >/dev/null || { printf '%s\n' "$OUT" | tail -30; fail "the damaged files are not listed"; }
|
|
printf '%s\n' "$OUT" | grep "ngb-salvage-" >/dev/null && { printf '%s\n' "$OUT" | tail -30; fail "the database was given up, only files are damaged"; }
|
|
docker cp /tmp/ngb-frt-sums ngb-frt-restore:/tmp/frt-sums
|
|
printf '%s\n' "$OUT" | docker exec -i ngb-frt-restore sh -c 'cat > /tmp/frt-out'
|
|
# Every file: listed and not there, or there and bit-identical
|
|
CHECK=$(docker exec ngb-frt-restore php -r '
|
|
$root = "/var/www/html/data/frtest/files/"; $listed = []; $reasons = [];
|
|
foreach (file("/tmp/frt-out", FILE_IGNORE_NEW_LINES) as $l) {
|
|
if (preg_match("#^ (?:data/)?frtest/files/(.+?): (.*)$#", $l, $m)) { $listed[$m[1]] = true; $reasons[] = $m[2]; }
|
|
}
|
|
$same = 0; $gone = 0; $bad = [];
|
|
foreach (file("/tmp/frt-sums", FILE_IGNORE_NEW_LINES) as $l) {
|
|
[$sum, $p] = preg_split("/\s+/", $l, 2); $p = substr($p, 2);
|
|
if (isset($listed[$p])) { file_exists($root . $p) ? $bad[] = "listed but there: $p" : $gone++; continue; }
|
|
if (!file_exists($root . $p)) { $bad[] = "not listed and not there: $p"; continue; }
|
|
hash_file("sha256", $root . $p) === $sum ? $same++ : $bad[] = "different: $p";
|
|
}
|
|
$walk = new RecursiveIteratorIterator(new RecursiveDirectoryIterator("/var/www/html/data", FilesystemIterator::SKIP_DOTS), RecursiveIteratorIterator::SELF_FIRST);
|
|
foreach ($walk as $f) { if (str_contains($f->getFilename(), ".ngb-restore-")) { $bad[] = "temporary left: " . $f->getPathname(); } }
|
|
echo "same=$same gone=$gone bad=", count($bad), "\n", implode("\n", array_slice($bad, 0, 10)), "\n";
|
|
echo "reasons: ", implode(" | ", array_unique(array_map(fn ($r) => substr(preg_replace("#[0-9a-f]{16,}#", "<id>", $r), 0, 70), $reasons))), "\n";')
|
|
printf '%s\n' "$CHECK"
|
|
printf '%s\n' "$CHECK" | grep -q " bad=0" || fail "files after the salvage are not as listed"
|
|
printf '%s\n' "$CHECK" | grep -qE "same=[1-9]" || fail "nothing was restored"
|
|
printf '%s\n' "$CHECK" | grep -qE "gone=[1-9]" || fail "no user file was hit by the damage"
|
|
ok "with --salvage: every file listed and absent, or there and identical"
|
|
run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the salvage"
|
|
run_on ngb-frt-restore user:list | grep frtest >/dev/null || fail "the backed-up user is missing: the database was not restored"
|
|
run_on ngb-frt-restore user:list | grep admin >/dev/null || fail "the admin is missing: the database was not restored"
|
|
docker exec ngb-frt-restore test -d /var/www/html/data/frtest/files/Empty/Nested || fail "the empty folder did not come back with --salvage"
|
|
[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt 2>/dev/null)" = "hello disaster recovery" ] \
|
|
|| printf '%s\n' "$OUT" | grep "Docs/note.txt" >/dev/null || fail "note.txt is neither restored nor listed"
|
|
run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after the salvage"
|
|
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after the salvage"
|
|
ok "the database and config are the old server's, the instance works"
|
|
echo "FULL RESTORE ROUNDTRIP PASS ($DB, salvage-files)"
|
|
exit 0
|
|
fi
|
|
|
|
RESTORE_OPTS=(--no-interaction --install-apps)
|
|
if [ "$LEGACY" = legacy ] && [ "$DB" != sqlite ]; then
|
|
# Without --skip-missing-tables an older backup must be refused, naming the app's tables
|
|
REFUSED=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" "${RESTORE_OPTS[@]}" 2>&1) \
|
|
&& fail "an older backup with tables of unknown structure was restored without --skip-missing-tables"
|
|
printf '%s\n' "$REFUSED" | grep -q "oc_deck_boards" || { printf '%s\n' "$REFUSED" | tail -20; fail "the refusal does not name the missing tables"; }
|
|
ok "older backup refused without --skip-missing-tables, missing tables named"
|
|
RESTORE_OPTS+=(--skip-missing-tables)
|
|
fi
|
|
if [ "$DB" = sqlite ]; then
|
|
# Another process writing while the restore starts (a cron job that was already running, or
|
|
# the installer still finishing after it says "installed"): the restore must wait for it, not
|
|
# stop with "database is locked". Short write transactions on a table of its own, until the
|
|
# restore is done (/tmp/frt-side-stop; Nextcloud's own commands after it are not under test).
|
|
docker exec -d -u www-data ngb-frt-restore php -r '
|
|
$p = new PDO("sqlite:/var/www/html/data/nextcloud.db"); $p->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
|
|
$p->exec("CREATE TABLE IF NOT EXISTS frt_side_writer (t REAL)");
|
|
for ($end = microtime(true) + 300; microtime(true) < $end && !file_exists("/tmp/frt-side-stop");) {
|
|
$p->beginTransaction(); $p->exec("INSERT INTO frt_side_writer VALUES (" . microtime(true) . ")"); usleep(300000); $p->commit(); usleep(20000);
|
|
}
|
|
touch("/tmp/frt-side-done");'
|
|
sleep 1
|
|
fi
|
|
sftp_logins() { docker logs ngb-frt-sftp 2>&1 | grep -c "Accepted password" || true; }
|
|
[ "$MODE" = sftp ] && LOGINS_BEFORE=$(sftp_logins)
|
|
OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" "${RESTORE_OPTS[@]}" 2>&1) || {
|
|
printf '%s\n' "$OUT" | tail -40
|
|
docker exec ngb-frt-restore sh -c 'tail -c 6000 /var/www/html/data/nextcloud.log' || true
|
|
fail "restore:full"
|
|
}
|
|
printf '%s\n' "$OUT" | grep -q "maintenance mode while the restore runs" || fail "the restore did not say it runs in maintenance mode"
|
|
run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the restore"
|
|
run_on ngb-frt-restore config:app:get ng_backup restore_full_started >/dev/null 2>&1 && fail "the restore left its 'started' marker behind"
|
|
ok "backup:restore:full completed, in maintenance mode and out of it again"
|
|
if [ "$DB" = sqlite ]; then
|
|
docker exec ngb-frt-restore touch /tmp/frt-side-stop
|
|
for i in $(seq 1 20); do docker exec ngb-frt-restore test -f /tmp/frt-side-done && break; sleep 0.5; done
|
|
WRITES=$(docker exec -u www-data ngb-frt-restore php -r 'echo (new PDO("sqlite:/var/www/html/data/nextcloud.db"))->query("SELECT COUNT(*) FROM frt_side_writer")->fetchColumn();')
|
|
[ "${WRITES:-0}" -gt 3 ] || fail "the other writer did not run alongside the restore ($WRITES writes)"
|
|
ok "the restore waited for another process writing to SQLite ($WRITES writes alongside)"
|
|
fi
|
|
|
|
# Everything below must work in a FRESH occ process (a new PHP process per docker exec), since
|
|
# earlier bugs here only showed up after config.php's secret was rewritten mid-restore.
|
|
run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after restore (own key/target undecryptable?)"
|
|
# not grep -q: it stops reading at the first match, and occ, still writing, then fails (pipefail)
|
|
run_on ngb-frt-restore user:list | grep frtest >/dev/null || fail "the backed-up user is missing from the restored database"
|
|
docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.txt || fail "restored file is missing"
|
|
[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt)" = "hello disaster recovery" ] || fail "restored file content differs"
|
|
if [ "$LEGACY" = legacy ]; then
|
|
# Backups made before 1.0.3 do not record empty folders
|
|
docker exec ngb-frt-restore test -e /var/www/html/data/frtest/files/Empty/Nested && fail "an empty folder came back from a backup that does not record them"
|
|
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore"
|
|
else
|
|
docker exec ngb-frt-restore test -d /var/www/html/data/frtest/files/Empty/Nested || fail "the empty folder did not come back"
|
|
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore"
|
|
docker exec ngb-frt-restore test -d /var/www/html/data/frtest/files/Empty/Nested || fail "files:scan removed the empty folder"
|
|
ok "the empty folder came back and stays after files:scan"
|
|
fi
|
|
# New rows after the restore: a new file (file cache) and a new user must not collide with
|
|
# restored rows (ID counters after the data).
|
|
docker exec ngb-frt-restore sh -c "echo new > /var/www/html/data/frtest/files/Docs/after-restore.txt && chown www-data /var/www/html/data/frtest/files/Docs/after-restore.txt"
|
|
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "a new file cannot be added after the restore"
|
|
if [ "$DB" != sqlite ]; then
|
|
[ "$(sql restore "select count(*) from oc_filecache where path = 'files/Docs/after-restore.txt'")" = 1 ] \
|
|
|| fail "a new file did not make it into the file cache after the restore (ID counters?)"
|
|
fi
|
|
docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-restore php occ user:add --password-from-env frnew >/dev/null || fail "a new user cannot be added after the restore"
|
|
ok "database, files and ng_backup's own state are all intact and usable after the restore"
|
|
|
|
# Mail and Deck were only set up on an origin with a database server (see above)
|
|
if [ "$ORIGIN_DBTYPE" != sqlite ] && [ "$DB" != sqlite ] && [ "$LEGACY" = legacy ]; then
|
|
[ "$(sql restore "select count(*) from information_schema.tables where table_name = 'oc_deck_boards'")" = 0 ] || fail "a table of unknown structure was created"
|
|
[ "$(sql restore "select count(*) from oc_mail_accounts where drafts_mailbox_id is not null")" = 1 ] || fail "the mail account was not restored"
|
|
ok "older backup: tables of the disabled app left out, Mail restored"
|
|
elif [ "$ORIGIN_DBTYPE" != sqlite ] && [ "$DB" != sqlite ]; then
|
|
[ "$(sql restore "select count(*) from information_schema.tables where table_name = 'oc_deck_boards'")" = 1 ] || fail "the tables of the disabled app were not created"
|
|
[ "$(sql restore "select title from oc_deck_boards")" = "Plan 📌" ] || fail "a 4-byte character did not survive in a created table"
|
|
[ "$(sql restore "select count(*) from oc_mail_accounts where drafts_mailbox_id is not null")" = 1 ] || fail "the mail account was not restored"
|
|
[ "$(sql restore "select count(*) from information_schema.table_constraints where constraint_type = 'FOREIGN KEY' and table_name like 'oc_mail_%'")" -gt 0 ] \
|
|
|| fail "the foreign keys of Mail are missing after the restore"
|
|
printf '%s\n' "$OUT" | grep -q "Created .* tables" || fail "the restore did not report the tables it created"
|
|
ok "tables of a disabled app created, Mail restored with its foreign keys"
|
|
fi
|
|
|
|
if [ "$MODE" = sftp ]; then
|
|
[ "$(docker exec ngb-frt-restore sha256sum /var/www/html/data/frtest/files/Docs/large.bin | cut -d' ' -f1)" = "$LARGE_SUM" ] \
|
|
|| fail "the large file differs after the restore"
|
|
LOGINS=$(( $(sftp_logins) - LOGINS_BEFORE ))
|
|
[ "$LOGINS" -gt 1 ] || fail "the restore did not read ahead on connections of its own ($LOGINS SFTP logins)"
|
|
docker exec ngb-frt-restore sh -c 'ls -a /var/www/html/data' | grep -q ngb-restore-spool && fail "the spool folder was left behind"
|
|
ok "over SFTP: read ahead on connections of its own ($LOGINS SFTP logins during the restore), large file identical, no spool left"
|
|
fi
|
|
|
|
run_on ngb-frt-restore backup:run offsite -l post-restore-check >/dev/null || fail "a new backup after the restore fails"
|
|
ok "the restored instance can run a new backup of its own"
|
|
|
|
echo "FULL RESTORE ROUNDTRIP PASS ($ORIGIN_DBTYPE${MODE:+, $MODE})"
|