dillard
d8ef50732f
Backups now record each table's structure in the dump manifest. The full restore creates tables that do not exist on the new instance (left behind by apps that were removed or are switched off), lists all problems at once, and no longer stops on apps that are missing. Foreign keys are not checked while rows are loaded in name order, and checked again before the restore commits (PostgreSQL: dropped and added back in the transaction). --install-apps installs or enables the apps that were enabled at backup time; --skip-missing-tables leaves out tables of older backups whose structure is unknown. nextcloud-aio is never required. backup:target:add without a key points to backup:key:import, and says that connecting can take a while. Removing a location drops its cached index; a migration removes the cache of locations that are gone. The full-restore test now also runs on PostgreSQL with Mail (foreign keys) and a disabled app.
141 lines
8.7 KiB
Bash
Executable File
141 lines
8.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a
|
|
# SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup
|
|
# location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7.
|
|
# Usage: full-restore-roundtrip.sh <nextcloud-image-tag> [sqlite|pgsql]
|
|
# e.g. full-restore-roundtrip.sh 35-apache pgsql
|
|
# With pgsql the origin also has what broke real disaster recoveries: the Mail app with a synced
|
|
# account (foreign keys between its tables) and Deck installed and then switched off (tables of
|
|
# an app that is not enabled). The restore instance has neither; --install-apps adds Mail.
|
|
set -euo pipefail
|
|
IMAGE="nextcloud:${1:-35-apache}"
|
|
DB="${2:-sqlite}"
|
|
APP=$(cd "$(dirname "$0")/../.." && pwd)
|
|
NET=ngb-frt-net
|
|
SHARED=$(mktemp -d)
|
|
chmod 777 "$SHARED"
|
|
fail() { echo "FAIL: $*"; exit 1; }
|
|
ok() { echo "ok $*"; }
|
|
|
|
cleanup() {
|
|
docker rm -fv ngb-frt-origin ngb-frt-restore ngb-frt-db-origin ngb-frt-db-restore ngb-frt-imap >/dev/null 2>&1 || true
|
|
docker network rm "$NET" >/dev/null 2>&1 || true
|
|
docker run --rm -v "$SHARED:/target" alpine sh -c 'rm -rf /target/*' >/dev/null 2>&1 || true
|
|
rmdir "$SHARED" 2>/dev/null || true
|
|
rm -f /tmp/ngb-frt-kit.json /tmp/ngb-frt-pass
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
run_on() { docker exec -u www-data "$1" php occ "${@:2}"; }
|
|
|
|
# Database settings for an instance: SQLite inside the container, or its own PostgreSQL container
|
|
db_env() {
|
|
if [ "$DB" = pgsql ]; then
|
|
docker run -d --name "ngb-frt-db-$1" --network "$NET" -e POSTGRES_DB=nextcloud -e POSTGRES_USER=nextcloud \
|
|
-e POSTGRES_PASSWORD=nextcloud postgres:16-alpine >/dev/null
|
|
for i in $(seq 1 30); do docker exec "ngb-frt-db-$1" pg_isready -q 2>/dev/null && break; sleep 2; done
|
|
echo "-e POSTGRES_HOST=ngb-frt-db-$1 -e POSTGRES_DB=nextcloud -e POSTGRES_USER=nextcloud -e POSTGRES_PASSWORD=nextcloud"
|
|
else
|
|
echo "-e SQLITE_DATABASE=nextcloud"
|
|
fi
|
|
}
|
|
|
|
docker network create "$NET" >/dev/null
|
|
ORIGIN_DB=$(db_env origin)
|
|
# shellcheck disable=SC2086
|
|
docker run -d --name ngb-frt-origin --network "$NET" $ORIGIN_DB \
|
|
-e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \
|
|
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null
|
|
for i in $(seq 1 90); do run_on ngb-frt-origin status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done
|
|
docker exec ngb-frt-origin chown www-data:www-data /var/www/html/custom_apps
|
|
run_on ngb-frt-origin app:enable ng_backup >/dev/null
|
|
ok "origin instance up ($DB)"
|
|
|
|
docker exec ngb-frt-origin sh -c "printf 'full restore roundtrip passphrase 2026\n' > /tmp/pass"
|
|
run_on ngb-frt-origin backup:key:init --passphrase-file=/tmp/pass --label=CI >/dev/null || fail "key init"
|
|
run_on ngb-frt-origin backup:key:kit --output=/tmp/kit.json >/dev/null
|
|
CODE=$(run_on ngb-frt-origin backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
|
|
run_on ngb-frt-origin backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "confirm"
|
|
run_on ngb-frt-origin backup:target:add offsite local -a null::null -o datadir=/mnt/shared --path=repo >/dev/null || fail "target add"
|
|
ok "key, kit and location on the origin"
|
|
|
|
NC_PASS=$(openssl rand -hex 16)
|
|
docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-origin php occ user:add --password-from-env frtest >/dev/null
|
|
docker exec ngb-frt-origin sh -c "mkdir -p /var/www/html/data/frtest/files/Docs && head -c 200000 /dev/urandom > /var/www/html/data/frtest/files/Docs/report.bin && echo 'hello disaster recovery' > /var/www/html/data/frtest/files/Docs/note.txt && chown -R www-data:www-data /var/www/html/data/frtest"
|
|
run_on ngb-frt-origin files:scan frtest >/dev/null
|
|
|
|
if [ "$DB" = pgsql ]; then
|
|
# Mail with a synced account: oc_mail_accounts points at oc_mail_mailboxes (foreign keys),
|
|
# while the restore loads tables in name order, accounts before mailboxes.
|
|
docker run -d --name ngb-frt-imap --network "$NET" -e MAILNAME=mail.domain.tld -e MAIL_ADDRESS=user@domain.tld \
|
|
-e MAIL_PASS=mypassword ghcr.io/christophwurst/docker-imap-devel:latest >/dev/null
|
|
run_on ngb-frt-origin app:install mail >/dev/null || fail "install mail on the origin"
|
|
run_on ngb-frt-origin config:system:set app.mail.verify-tls-peer --type=boolean --value=false >/dev/null
|
|
sleep 10
|
|
run_on ngb-frt-origin mail:account:create frtest Test user@domain.tld ngb-frt-imap 993 ssl user@domain.tld mypassword \
|
|
ngb-frt-imap 25 tls user@domain.tld mypassword >/dev/null || fail "mail account"
|
|
run_on ngb-frt-origin mail:account:sync -f 1 >/dev/null || fail "mail sync"
|
|
[ "$(docker exec ngb-frt-db-origin psql -U nextcloud -d nextcloud -Atc 'select count(*) from oc_mail_accounts where drafts_mailbox_id is not null')" = 1 ] \
|
|
|| fail "the mail account does not point at its mailboxes"
|
|
# Deck installed and then switched off: its tables stay in the database and in the backup
|
|
run_on ngb-frt-origin app:install deck >/dev/null || fail "install deck on the origin"
|
|
run_on ngb-frt-origin app:disable deck >/dev/null
|
|
ok "origin has a synced Mail account and the tables of a disabled app"
|
|
fi
|
|
|
|
RUN_OUT=$(run_on ngb-frt-origin backup:run offsite -l disaster-test)
|
|
SNAP=$(printf '%s\n' "$RUN_OUT" | head -1 | awk -F'snapshot ' '{print $2}' | tr -d ',')
|
|
[ -n "$SNAP" ] || fail "no snapshot id from backup:run"
|
|
ok "test data backed up, snapshot $SNAP"
|
|
|
|
docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json
|
|
docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass
|
|
RESTORE_DB=$(db_env restore)
|
|
# shellcheck disable=SC2086
|
|
docker run -d --name ngb-frt-restore --network "$NET" $RESTORE_DB \
|
|
-e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \
|
|
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null
|
|
for i in $(seq 1 90); do run_on ngb-frt-restore status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done
|
|
docker exec ngb-frt-restore chown www-data:www-data /var/www/html/custom_apps
|
|
run_on ngb-frt-restore app:enable ng_backup >/dev/null
|
|
docker cp /tmp/ngb-frt-kit.json ngb-frt-restore:/tmp/kit.json
|
|
docker exec ngb-frt-restore chmod 644 /tmp/kit.json
|
|
docker cp /tmp/ngb-frt-pass ngb-frt-restore:/tmp/pass
|
|
ok "fresh restore instance up"
|
|
|
|
run_on ngb-frt-restore backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass >/dev/null || fail "key import"
|
|
run_on ngb-frt-restore backup:target:add offsite local -a null::null -o datadir=/mnt/shared --path=repo >/dev/null || fail "target add on the restore instance"
|
|
run_on ngb-frt-restore backup:list offsite | grep -q "$SNAP" || fail "the origin's snapshot is not listed after connecting the location"
|
|
ok "recovery kit imported, existing repository adopted, snapshot visible"
|
|
|
|
OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction --install-apps 2>&1) || {
|
|
printf '%s\n' "$OUT" | tail -40
|
|
docker exec ngb-frt-restore sh -c 'tail -c 6000 /var/www/html/data/nextcloud.log' || true
|
|
fail "restore:full"
|
|
}
|
|
ok "backup:restore:full completed"
|
|
|
|
# Everything below must work in a FRESH occ process (a new PHP process per docker exec), since
|
|
# earlier bugs here only showed up after config.php's secret was rewritten mid-restore.
|
|
run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after restore (own key/target undecryptable?)"
|
|
run_on ngb-frt-restore user:list | grep -q frtest || fail "the backed-up user is missing from the restored database"
|
|
docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.txt || fail "restored file is missing"
|
|
[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt)" = "hello disaster recovery" ] || fail "restored file content differs"
|
|
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore"
|
|
ok "database, files and ng_backup's own state are all intact and usable after the restore"
|
|
|
|
if [ "$DB" = pgsql ]; then
|
|
Q() { docker exec ngb-frt-db-restore psql -U nextcloud -d nextcloud -Atc "$1"; }
|
|
[ "$(Q "select count(*) from pg_tables where tablename = 'oc_deck_boards'")" = 1 ] || fail "the tables of the disabled app were not created"
|
|
[ "$(Q "select count(*) from oc_mail_accounts where drafts_mailbox_id is not null")" = 1 ] || fail "the mail account was not restored"
|
|
[ "$(Q "select count(*) from pg_constraint where contype = 'f' and conrelid::regclass::text like 'oc_mail_%'")" -gt 0 ] \
|
|
|| fail "the foreign keys of Mail are missing after the restore"
|
|
printf '%s\n' "$OUT" | grep -q "Created .* tables" || fail "the restore did not report the tables it created"
|
|
ok "tables of a disabled app created, Mail restored with its foreign keys"
|
|
fi
|
|
|
|
run_on ngb-frt-restore backup:run offsite -l post-restore-check >/dev/null || fail "a new backup after the restore fails"
|
|
ok "the restored instance can run a new backup of its own"
|
|
|
|
echo "FULL RESTORE ROUNDTRIP PASS ($DB)"
|