Files
nextcloud-ng_backup/tests/integration/recovery-scenarios.sh
T
dillard 99f1655d04
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / psalm (push) Has been cancelled
CI / frontend (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
CI / integration-targets (push) Has been cancelled
CI / integration-user (32) (push) Has been cancelled
CI / integration-user (33) (push) Has been cancelled
CI / integration-user (34) (push) Has been cancelled
CI / integration-user (35) (push) Has been cancelled
CI / integration-full-restore (push) Has been cancelled
Review fixes: a missed passphrase change follows at the next backup, a key from before 1.1.0 counts as used, schedules edited in their own time zone
- A location that a passphrase change could not reach gets it at its next backup
- A key from before 1.1.0 is never replaced as an unused one
- An unreadable settings file on a location is told apart from a missing one
- A schedule is shown and saved in the zone it was saved in
- Saving an unchanged schedule keeps its moment; --timezone needs a schedule
- Schedule coverage uses the time the backup began reading its data
- Tests: catch-up scenario, upgrade test restores the new file, autumn clock change
2026-11-08 17:18:13 +01:00

302 lines
20 KiB
Bash
Executable File

#!/bin/bash
# Getting the backup key onto a new server, with and without the recovery kit, and what the
# administrator is told in each case. Every scenario starts on a fresh Nextcloud (SQLite) with a
# copy of the origin's location; nothing is restored (full-restore-roundtrip.sh does that).
# Usage: recovery-scenarios.sh [nextcloud-image-tag] e.g. recovery-scenarios.sh 35-apache
#
# kit kit + passphrase: backup:key:import, then backup:target:add asks nothing
# kit-wrong kit with a wrong passphrase: refused, no key left behind
# no-kit passphrase only: target:add asks for it (or --passphrase-file), a wrong one
# changes nothing; backups wait for a new kit
# no-kit-old as no-kit, on a location backed up only before 1.1.0 (no settings file):
# the deletion delay is the default, and the admin is told so
# no-config-no-kit the location's config is gone and there is no kit: told that only the kit helps
# no-config-kit config gone, kit there: --repair-config writes it again, the backups open
# other-kit the kit of another server: refused, told to use the old server's kit
# empty nothing at the location: told to check the details, not to make a new key
# unused-key-kit a new key was made by mistake: key:import refuses and names
# --replace-unused-key; with it the kit's key takes its place
# unused-key-no-kit the same without a kit: target:add with --replace-unused-key and a passphrase
# of the old server; a wrong one keeps the unused key
# used-key a key that protected a location is never replaced, also after the location was removed
# slot-catch-up a passphrase added while a location was unreachable reaches it at its next
# backup; a new server then gets the key from it with that passphrase
set -euo pipefail
IMAGE="nextcloud:${1:-35-apache}"
APP=$(cd "$(dirname "$0")/../.." && pwd)
SHARED=$(mktemp -d)
chmod 777 "$SHARED"
PASS='recovery scenarios passphrase 2026'
fail() { echo "FAIL: $*"; exit 1; }
ok() { echo "ok $*"; }
cleanup() {
docker rm -fv ngb-rs-origin ngb-rs-other ngb-rs-new >/dev/null 2>&1 || true
docker run --rm -v "$SHARED:/target" alpine sh -c 'rm -rf /target/*' >/dev/null 2>&1 || true
rmdir "$SHARED" 2>/dev/null || true
return 0
}
trap cleanup EXIT
run_on() { docker exec -u www-data "$1" php occ "${@:2}"; }
# Output of a command that must fail; the scenario fails when it succeeds.
must_fail() { # container what args...
local out
if out=$(run_on "$1" "${@:3}" 2>&1); then printf '%s\n' "$out"; fail "$2: the command succeeded"; fi
printf '%s\n' "$out"
}
expect() { # output text what
printf '%s\n' "$1" | grep -qF -- "$2" || { printf '%s\n' "$1"; fail "$3: the output does not say \"$2\""; }
}
not_expect() {
printf '%s\n' "$1" | grep -qF -- "$2" && { printf '%s\n' "$1"; fail "$3: the output says \"$2\""; }
return 0
}
start() { # name
docker rm -fv "$1" >/dev/null 2>&1 || true
docker run -d --name "$1" -e SQLITE_DATABASE=nextcloud -e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null
for i in $(seq 1 90); do run_on "$1" status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done
docker exec "$1" chown www-data:www-data /var/www/html/custom_apps
run_on "$1" app:enable ng_backup >/dev/null
docker exec "$1" sh -c "printf '%s\n' '$PASS' > /tmp/pass; printf 'not the passphrase at all\n' > /tmp/wrongpass; chmod 644 /tmp/pass /tmp/wrongpass"
}
key_of() { run_on "$1" backup:status 2>&1 | sed -n 's/^Key: \([0-9a-f-]*\),.*/\1/p'; }
no_key() { # container what
local s
s=$(run_on "$1" backup:status 2>&1 || true)
expect "$s" "Key: not created" "$2: a key was left behind"
[ -z "$(run_on "$1" backup:target:list 2>&1 | grep offsite || true)" ] || fail "$2: a location was left behind"
}
location() { echo "local -a null::null -o datadir=/mnt/shared/$1 --path=repo"; }
# A fresh server with its own copy of the origin's location
scenario() { # name
docker exec ngb-rs-origin sh -c "rm -rf /mnt/shared/$1 && mkdir -p /mnt/shared/$1 && cp -a /mnt/shared/origin/repo /mnt/shared/$1/ && chown -R www-data /mnt/shared/$1"
start ngb-rs-new
docker exec -i ngb-rs-new sh -c 'cat > /tmp/kit.json && chmod 644 /tmp/kit.json' < "$SHARED/kit.json"
}
# The origin: a key with a deletion delay of its own, a kit, one backup
start ngb-rs-origin
run_on ngb-rs-origin backup:key:init --passphrase-file=/tmp/pass --label=Safe --delete-delay-days=21 >/dev/null || fail "key init"
run_on ngb-rs-origin backup:key:kit --output=/tmp/kit.json >/dev/null
CODE=$(run_on ngb-rs-origin backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
run_on ngb-rs-origin backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "confirm"
docker exec ngb-rs-origin sh -c 'mkdir -p /mnt/shared/origin && chown www-data /mnt/shared/origin'
# shellcheck disable=SC2046
run_on ngb-rs-origin backup:target:add offsite $(location origin) >/dev/null || fail "target add on the origin"
run_on ngb-rs-origin backup:run offsite >/dev/null || fail "backup on the origin"
SNAP=$(run_on ngb-rs-origin backup:list offsite | grep -o '[0-9a-f]\{32\}' | head -1)
FP=$(key_of ngb-rs-origin)
docker exec ngb-rs-origin cat /tmp/kit.json > "$SHARED/kit.json"
docker exec ngb-rs-origin test -f /mnt/shared/origin/repo/settings || fail "the origin's location has no settings file"
ok "origin: key $FP, kit, snapshot $SNAP, settings on the location"
# kit: the passphrase once, for the kit; target:add asks nothing more
scenario kit
OUT=$(run_on ngb-rs-new backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass 2>&1) || { printf '%s\n' "$OUT"; fail "kit: key import"; }
expect "$OUT" "fingerprint $FP" "kit"
expect "$OUT" "21 days (from the kit)" "kit"
expect "$OUT" "backup:target:add" "kit: the next step"
# shellcheck disable=SC2046
OUT=$(run_on ngb-rs-new backup:target:add offsite $(location kit) --no-interaction 2>&1) || { printf '%s\n' "$OUT"; fail "kit: target add"; }
not_expect "$OUT" "Passphrase" "kit: target:add asked for a passphrase"
expect "$OUT" "found and adopted" "kit"
run_on ngb-rs-new backup:list offsite | grep -q "$SNAP" || fail "kit: the snapshot is not listed"
run_on ngb-rs-new backup:run offsite >/dev/null || fail "kit: a backup after the kit import"
ok "kit: imported with the passphrase, location added without asking again, backups run"
# kit-wrong
scenario kit-wrong
OUT=$(must_fail ngb-rs-new "kit-wrong" backup:key:import /tmp/kit.json --passphrase-file=/tmp/wrongpass)
expect "$OUT" "Wrong passphrase" "kit-wrong"
no_key ngb-rs-new "kit-wrong"
ok "kit-wrong: refused, nothing left behind"
# no-kit
scenario no-kit
docker exec ngb-rs-new rm /tmp/kit.json
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "no-kit without a passphrase" backup:target:add offsite $(location no-kit) --no-interaction)
expect "$OUT" "this location holds backups" "no-kit: explains where the key comes from"
expect "$OUT" "--passphrase-file" "no-kit: says how to give the passphrase"
no_key ngb-rs-new "no-kit without a passphrase"
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "no-kit wrong passphrase" backup:target:add offsite $(location no-kit) --passphrase-file=/tmp/wrongpass)
expect "$OUT" "Wrong passphrase" "no-kit wrong passphrase"
expect "$OUT" "Nothing was changed" "no-kit wrong passphrase"
no_key ngb-rs-new "no-kit wrong passphrase"
# shellcheck disable=SC2046
OUT=$(run_on ngb-rs-new backup:target:add offsite $(location no-kit) --passphrase-file=/tmp/pass 2>&1) || { printf '%s\n' "$OUT"; fail "no-kit: target add"; }
expect "$OUT" "fingerprint $FP" "no-kit"
expect "$OUT" "21 days (from the location)" "no-kit"
expect "$OUT" "download a new recovery kit" "no-kit: asks for a new kit"
run_on ngb-rs-new backup:list offsite | grep -q "$SNAP" || fail "no-kit: the snapshot is not listed"
OUT=$(must_fail ngb-rs-new "no-kit backup before a new kit" backup:run offsite)
expect "$OUT" "Confirm the recovery kit" "no-kit: a backup before a new kit"
CODE=$(run_on ngb-rs-new backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
run_on ngb-rs-new backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "no-kit: confirm the new kit"
run_on ngb-rs-new backup:run offsite >/dev/null || fail "no-kit: a backup after the new kit"
ok "no-kit: passphrase asked, wrong one changes nothing, key and delay from the location, backups wait for a new kit"
# no-kit-old: a location from before 1.1.0 has no settings file
scenario no-kit-old
docker exec ngb-rs-new rm /tmp/kit.json /mnt/shared/no-kit-old/repo/settings
# shellcheck disable=SC2046
OUT=$(run_on ngb-rs-new backup:target:add offsite $(location no-kit-old) --passphrase-file=/tmp/pass 2>&1) || { printf '%s\n' "$OUT"; fail "no-kit-old: target add"; }
expect "$OUT" "fingerprint $FP" "no-kit-old"
expect "$OUT" "7 days (the default" "no-kit-old: the default delay and why"
docker exec ngb-rs-new test -f /mnt/shared/no-kit-old/repo/settings || fail "no-kit-old: the settings were not written when the location was added"
ok "no-kit-old: key from the location, default delay with the reason, settings written now"
# no-config-no-kit
scenario no-config-no-kit
docker exec ngb-rs-new rm /tmp/kit.json /mnt/shared/no-config-no-kit/repo/config
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "no-config-no-kit" backup:target:add offsite $(location no-config-no-kit) --passphrase-file=/tmp/pass)
expect "$OUT" "its config file is gone" "no-config-no-kit"
expect "$OUT" "only the recovery kit has the key" "no-config-no-kit"
expect "$OUT" "--repair-config" "no-config-no-kit"
no_key ngb-rs-new "no-config-no-kit"
docker exec ngb-rs-new test ! -f /mnt/shared/no-config-no-kit/repo/config || fail "no-config-no-kit: a config was written"
ok "no-config-no-kit: told the kit is needed, nothing written"
# no-config-kit
scenario no-config-kit
docker exec ngb-rs-new rm /mnt/shared/no-config-kit/repo/config
run_on ngb-rs-new backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass >/dev/null || fail "no-config-kit: key import"
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "no-config-kit without --repair-config" backup:target:add offsite $(location no-config-kit))
expect "$OUT" "--repair-config" "no-config-kit: points at --repair-config"
docker exec ngb-rs-new test ! -f /mnt/shared/no-config-kit/repo/config || fail "no-config-kit: a config was written without --repair-config"
# shellcheck disable=SC2046
OUT=$(run_on ngb-rs-new backup:target:add offsite $(location no-config-kit) --repair-config 2>&1) || { printf '%s\n' "$OUT"; fail "no-config-kit: --repair-config"; }
expect "$OUT" "written again" "no-config-kit"
expect "$OUT" "found and adopted" "no-config-kit"
run_on ngb-rs-new backup:list offsite | grep -q "$SNAP" || fail "no-config-kit: the snapshot is not listed after the repair"
run_on ngb-rs-new backup:browse offsite "$SNAP" >/dev/null 2>&1 || fail "no-config-kit: the snapshot cannot be read after the repair"
ok "no-config-kit: refused without --repair-config, config written again with it, the backups open"
# other-kit: the kit of another server
start ngb-rs-other
run_on ngb-rs-other backup:key:init --passphrase-file=/tmp/pass >/dev/null || fail "other: key init"
run_on ngb-rs-other backup:key:kit --output=/tmp/other.json >/dev/null
docker exec ngb-rs-other cat /tmp/other.json > "$SHARED/other.json"
docker rm -fv ngb-rs-other >/dev/null
scenario other-kit
docker exec ngb-rs-new rm /tmp/kit.json
docker exec -i ngb-rs-new sh -c 'cat > /tmp/other.json && chmod 644 /tmp/other.json' < "$SHARED/other.json"
run_on ngb-rs-new backup:key:import /tmp/other.json --passphrase-file=/tmp/pass >/dev/null || fail "other-kit: key import"
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "other-kit" backup:target:add offsite $(location other-kit))
expect "$OUT" "different key" "other-kit"
expect "$OUT" "must be the old server's" "other-kit: points at the right kit"
[ -z "$(run_on ngb-rs-new backup:target:list 2>&1 | grep offsite || true)" ] || fail "other-kit: a location was left behind"
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "other-kit repair" backup:target:add offsite $(location other-kit) --repair-config)
expect "$OUT" "leave out --repair-config" "other-kit: a config that is there is not repaired"
ok "other-kit: refused, told to use the old server's kit, the config untouched"
# empty
scenario empty
docker exec ngb-rs-new sh -c 'mkdir -p /mnt/shared/nothing && chown www-data /mnt/shared/nothing'
OUT=$(must_fail ngb-rs-new "empty" backup:target:add offsite local -a null::null -o datadir=/mnt/shared/nothing --path=repo --passphrase-file=/tmp/pass)
expect "$OUT" "no backups at this location" "empty"
expect "$OUT" "Check the connection details and --path" "empty"
no_key ngb-rs-new "empty"
ok "empty: told to check the details, no key made"
# unused-key-kit: a key made by mistake, then the kit
scenario unused-key-kit
run_on ngb-rs-new backup:key:init --passphrase-file=/tmp/pass >/dev/null || fail "unused-key-kit: key init"
MISTAKE=$(key_of ngb-rs-new)
OUT=$(must_fail ngb-rs-new "unused-key-kit without the option" backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass)
expect "$OUT" "never used" "unused-key-kit"
expect "$OUT" "--replace-unused-key" "unused-key-kit: names the option"
[ "$(key_of ngb-rs-new)" = "$MISTAKE" ] || fail "unused-key-kit: the key changed without the option"
OUT=$(must_fail ngb-rs-new "unused-key-kit wrong passphrase" backup:key:import /tmp/kit.json --passphrase-file=/tmp/wrongpass --replace-unused-key)
expect "$OUT" "Wrong passphrase" "unused-key-kit wrong passphrase"
[ "$(key_of ngb-rs-new)" = "$MISTAKE" ] || fail "unused-key-kit: a wrong passphrase changed the key"
OUT=$(run_on ngb-rs-new backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass --replace-unused-key 2>&1) || { printf '%s\n' "$OUT"; fail "unused-key-kit: replace"; }
expect "$OUT" "The unused key was replaced" "unused-key-kit"
expect "$OUT" "fingerprint $FP" "unused-key-kit"
expect "$OUT" "21 days (from the kit)" "unused-key-kit"
# shellcheck disable=SC2046
OUT=$(run_on ngb-rs-new backup:target:add offsite $(location unused-key-kit) --no-interaction 2>&1) || { printf '%s\n' "$OUT"; fail "unused-key-kit: target add"; }
expect "$OUT" "found and adopted" "unused-key-kit"
run_on ngb-rs-new backup:run offsite >/dev/null || fail "unused-key-kit: a backup after the replacement"
ok "unused-key-kit: refused without the option, a wrong passphrase changes nothing, replaced with it, backups run"
# unused-key-no-kit: a key made by mistake, no kit
scenario unused-key-no-kit
docker exec ngb-rs-new rm /tmp/kit.json
run_on ngb-rs-new backup:key:init --passphrase-file=/tmp/pass >/dev/null || fail "unused-key-no-kit: key init"
MISTAKE=$(key_of ngb-rs-new)
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "unused-key-no-kit without the option" backup:target:add offsite $(location unused-key-no-kit) --no-interaction)
expect "$OUT" "different key" "unused-key-no-kit"
expect "$OUT" "never used" "unused-key-no-kit: says the key can make way"
expect "$OUT" "--replace-unused-key" "unused-key-no-kit: names the option"
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "unused-key-no-kit wrong passphrase" backup:target:add offsite $(location unused-key-no-kit) --replace-unused-key --passphrase-file=/tmp/wrongpass)
expect "$OUT" "Nothing was changed" "unused-key-no-kit wrong passphrase"
[ "$(key_of ngb-rs-new)" = "$MISTAKE" ] || fail "unused-key-no-kit: a wrong passphrase changed the key"
[ -z "$(run_on ngb-rs-new backup:target:list 2>&1 | grep offsite || true)" ] || fail "unused-key-no-kit: a location was left behind"
# shellcheck disable=SC2046
OUT=$(run_on ngb-rs-new backup:target:add offsite $(location unused-key-no-kit) --replace-unused-key --passphrase-file=/tmp/pass 2>&1) || { printf '%s\n' "$OUT"; fail "unused-key-no-kit: replace"; }
expect "$OUT" "The unused key was replaced" "unused-key-no-kit"
expect "$OUT" "fingerprint $FP" "unused-key-no-kit"
expect "$OUT" "download a new recovery kit" "unused-key-no-kit: asks for a new kit"
run_on ngb-rs-new backup:list offsite | grep -q "$SNAP" || fail "unused-key-no-kit: the snapshot is not listed"
OUT=$(must_fail ngb-rs-new "unused-key-no-kit backup before a new kit" backup:run offsite)
expect "$OUT" "Confirm the recovery kit" "unused-key-no-kit: the old key's kit confirmation does not count"
ok "unused-key-no-kit: hint on the refusal, a wrong passphrase changes nothing, replaced with a passphrase, a new kit is needed"
# used-key: a key that protected a location stays, also after that location is removed
scenario used-key
docker exec ngb-rs-new sh -c 'mkdir -p /mnt/shared/own && chown www-data /mnt/shared/own'
run_on ngb-rs-new backup:key:init --passphrase-file=/tmp/pass >/dev/null || fail "used-key: key init"
OWN=$(key_of ngb-rs-new)
run_on ngb-rs-new backup:target:add own local -a null::null -o datadir=/mnt/shared/own --path=repo >/dev/null || fail "used-key: own location"
run_on ngb-rs-new backup:target:remove own >/dev/null || fail "used-key: remove the own location"
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "used-key target:add" backup:target:add offsite $(location used-key) --replace-unused-key --passphrase-file=/tmp/pass)
expect "$OUT" "never replaced" "used-key target:add"
# shellcheck disable=SC2046
OUT=$(must_fail ngb-rs-new "used-key target:add without the option" backup:target:add offsite $(location used-key) --no-interaction)
not_expect "$OUT" "--replace-unused-key" "used-key: offers a replacement for a used key"
OUT=$(must_fail ngb-rs-new "used-key key:import" backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass --replace-unused-key)
expect "$OUT" "never replaced" "used-key key:import"
[ "$(key_of ngb-rs-new)" = "$OWN" ] || fail "used-key: the key changed"
ok "used-key: never replaced, by kit or passphrase, also after its location was removed"
# slot-catch-up: a passphrase change that missed a location reaches it at the next backup
start ngb-rs-new
docker exec ngb-rs-new sh -c "printf '%s\n' 'catch up passphrase of the cto' > /tmp/pass2 && chmod 644 /tmp/pass2 && rm -rf /mnt/shared/catchup && mkdir -p /mnt/shared/catchup && chown www-data /mnt/shared/catchup"
run_on ngb-rs-new backup:key:init --passphrase-file=/tmp/pass --label=Safe >/dev/null || fail "slot-catch-up: key init"
CODE=$(run_on ngb-rs-new backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
run_on ngb-rs-new backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "slot-catch-up: confirm"
# shellcheck disable=SC2046
run_on ngb-rs-new backup:target:add offsite $(location catchup) >/dev/null || fail "slot-catch-up: target add"
run_on ngb-rs-new backup:run offsite >/dev/null || fail "slot-catch-up: first backup"
BEFORE=$(docker exec ngb-rs-new md5sum /mnt/shared/catchup/repo/config | cut -d' ' -f1)
docker exec ngb-rs-new chmod 000 /mnt/shared/catchup/repo
OUT=$(must_fail ngb-rs-new "slot-catch-up: slot add with the location unreachable" backup:key:slot:add --label=CTO --passphrase-file=/tmp/pass2)
expect "$OUT" "tried again at their next backup" "slot-catch-up: says the change follows"
docker exec ngb-rs-new chmod 755 /mnt/shared/catchup/repo
[ "$(docker exec ngb-rs-new md5sum /mnt/shared/catchup/repo/config | cut -d' ' -f1)" = "$BEFORE" ] || fail "slot-catch-up: the location changed while unreachable"
CODE=$(run_on ngb-rs-new backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
run_on ngb-rs-new backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "slot-catch-up: confirm the new kit"
run_on ngb-rs-new backup:run offsite >/dev/null || fail "slot-catch-up: backup after the location came back"
[ "$(docker exec ngb-rs-new md5sum /mnt/shared/catchup/repo/config | cut -d' ' -f1)" != "$BEFORE" ] || fail "slot-catch-up: the backup did not bring the new passphrase to the location"
CATCHFP=$(key_of ngb-rs-new)
start ngb-rs-other
docker exec ngb-rs-other sh -c "printf '%s\n' 'catch up passphrase of the cto' > /tmp/pass2 && chmod 644 /tmp/pass2"
# shellcheck disable=SC2046
OUT=$(run_on ngb-rs-other backup:target:add offsite $(location catchup) --passphrase-file=/tmp/pass2 2>&1) || { printf '%s\n' "$OUT"; fail "slot-catch-up: a new server with the new passphrase"; }
expect "$OUT" "fingerprint $CATCHFP" "slot-catch-up: the new server"
ok "slot-catch-up: a passphrase that missed a location reaches it at the next backup and opens it on a new server"
echo "RECOVERY SCENARIOS PASS"