From b625fe7666daa7167064132ae063d0372bdcba1a Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Fri, 6 Nov 2026 10:27:41 +0100 Subject: [PATCH] A restore that stops at a file gone from the location points at --salvage; test a full restore with a damaged and a missing pack The DR test of 1.0.2 over SMB is done: result in PLAN, empty folders as a known issue (#27). --- .github/workflows/ci.yml | 2 + CHANGELOG.md | 5 ++ PLAN.md | 15 ++++ docs/DISASTER-RECOVERY.md | 3 + lib/Command/Restore.php | 9 ++- lib/Command/RestoreFull.php | 2 +- tests/integration/full-restore-roundtrip.sh | 82 ++++++++++++++++++++- 7 files changed, 114 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cd6278..11cdb41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,3 +140,5 @@ jobs: run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite to-pgsql - name: Full restore with a damaged database dump (--salvage) run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite salvage + - name: Full restore with a damaged and a missing pack (--salvage) + run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite salvage-files diff --git a/CHANGELOG.md b/CHANGELOG.md index fecf689..7dcd79e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,11 @@ shown under 1.0.0. The git tag of a release candidate goes after the date. - `occ backup:target:add` only estimates the time left for reading the index once enough of it is in. The first file also carries the time to connect, which made it say "about 642 min left" for something done within a minute. +- A restore that stops because a file is gone from the location now points at `--salvage`, as it + already did for damaged data. + +### Known issues +- Empty folders are not backed up or restored; to be fixed in 1.0.3 (#27). ## [1.0.1] - 2026-11-01 diff --git a/PLAN.md b/PLAN.md index cb5402d..1b0028c 100644 --- a/PLAN.md +++ b/PLAN.md @@ -415,6 +415,21 @@ in README "Known limitations": lengte -1. Opgelost (controle + nieuwe verbinding), plus hervatten: `--force` laat bestanden met dezelfde grootte en mtime staan. Hervat 2026-11-05 07:21 UTC na `maintenance:mode --off` volgens de gids; binnen 10 min voorbij de 392 GiB. SMB op sub5 na de test weer uitzetten. + **Resultaat (klaar 2026-11-06 02:11 UTC, nazorg 02:27):** hervatte run 18 u 50 min, exit 0, 150.755 + bestanden; 7-13 MiB/s (langzamer vanaf ~18 uur Duitse tijd, oorzaak niet vastgesteld). Vergeleken + met nc01: 150.467 van 150.758 bestanden gelijk in grootte en mtime, 397 op hash zonder verschil; + gebruikers, agenda's, contacten, shares en Deck gelijk; bestanden, agenda en contacten via WebDAV + getest als gebruiker. `backup:restore --salvage` op 16.904 bestanden: exit 0, geen verschil. Niet + vastgesteld: de 50.080 "updated" bij `files:scan`. Zonder `--install-apps` 51 van de 81 apps aan. + VM's 980-982 en de sim-backups op OVH en sub6 opgeruimd. + **Naar 1.0.3 (besluit 2026-11-06):** lege mappen komen niet terug (3.982 op nc01, 2.076 daarvan in + `files`, verklaart vrijwel zeker de 2.370 "removed" bij `files:scan`). De backup legt alleen + bestanden vast (`walk()`), dus oudere snapshots hebben ze niet. Voor een DR acceptabel, wel + fixen; issue #27. Daarna kleine volledige simulaties; een DR van 1 TB is hiervoor niet nodig. + **Salvage vóór de release getest (besluit 2026-11-06):** nieuwe modus `salvage-files` in + `full-restore-roundtrip.sh`, ook in CI: één pack beschadigd en één weg. Eerst stopt de restore, + daarna `--salvage --force` volgens de gids. Vond één fout: bij een ontbrekend bestand wees de + melding niet naar `--salvage` (opgelost). Stand 2026-11-04 (`polish`, gepusht): cron-waarschuwing + tick vanaf de pagina + voortgang van de db-fase; `--salvage` voor `backup:restore` en `backup:restore:full`; S3 vooruitlezen (8 ranges tegelijk, met `SaveAs` naar de spool); SMB in leesacties van 8 MiB diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md index df72ae5..06a238f 100644 --- a/docs/DISASTER-RECOVERY.md +++ b/docs/DISASTER-RECOVERY.md @@ -140,6 +140,9 @@ The command then ends with exit code 2 instead of 0. The same option exists for - From Nextcloud All-in-One to a regular server: also check the cache settings (`memcache.*`, `redis`), which point at AIO's containers. - Log in, open a few files, a calendar and a contact. +- Empty folders do not come back: a backup only records files (to be fixed in 1.0.3, see + [issue #27](https://github.com/dillardblom/ng_backup/issues/27)). `occ files:scan` then removes + them from the file cache, so it reports them as removed. - Start background jobs again (cron), as in step 2 in reverse. **10. Point your domain at the new server** and check that clients reconnect. diff --git a/lib/Command/Restore.php b/lib/Command/Restore.php index e07a8d0..f22cd19 100644 --- a/lib/Command/Restore.php +++ b/lib/Command/Restore.php @@ -67,7 +67,7 @@ HELP); }, $skipped); } catch (\Throwable $e) { $output->writeln('' . $e->getMessage() . ''); - if ($skipped === null && ($e instanceof \OCA\NgBackup\Repository\RepositoryException || $e instanceof \OCA\NgBackup\Crypto\CryptoException)) { + if ($skipped === null && self::backupLooksDamaged($e)) { $output->writeln('The data of a file on the location is damaged or missing. Run the same command with --salvage to restore everything else and list what could not be restored.'); } return 1; @@ -76,6 +76,13 @@ HELP); return self::reportSkipped($skipped, $output); } + /** Damaged data, or a file that is gone from the location: what --salvage is for. */ + public static function backupLooksDamaged(\Throwable $e): bool { + return $e instanceof \OCA\NgBackup\Repository\RepositoryException + || $e instanceof \OCA\NgBackup\Crypto\CryptoException + || ($e instanceof \OCA\NgBackup\Backend\BackendException && str_starts_with($e->getMessage(), 'Not found')); + } + /** * The files --salvage left out; exit code 2 when there are any. * diff --git a/lib/Command/RestoreFull.php b/lib/Command/RestoreFull.php index 920077e..6f2310e 100644 --- a/lib/Command/RestoreFull.php +++ b/lib/Command/RestoreFull.php @@ -111,7 +111,7 @@ HELP); $output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.'); } $output->writeln('If occ then says that Nextcloud needs an upgrade (the database is already the old server\'s), run occ upgrade first.'); - if (!$input->getOption('salvage') && ($e instanceof \OCA\NgBackup\Repository\RepositoryException || $e instanceof \OCA\NgBackup\Crypto\CryptoException)) { + if (!$input->getOption('salvage') && Restore::backupLooksDamaged($e)) { $output->writeln('The backup itself looks damaged here. --salvage restores everything that can be read and lists the rest.'); } return 1; diff --git a/tests/integration/full-restore-roundtrip.sh b/tests/integration/full-restore-roundtrip.sh index bcefbf3..b05f767 100755 --- a/tests/integration/full-restore-roundtrip.sh +++ b/tests/integration/full-restore-roundtrip.sh @@ -2,8 +2,11 @@ # Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a # SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup # location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7. -# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp|salvage|to-pgsql|to-mysql|to-sqlite] +# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp|salvage|salvage-files|to-pgsql|to-mysql|to-sqlite] # e.g. full-restore-roundtrip.sh 35-apache pgsql +# With "salvage-files" one pack is damaged and another is gone from the location. The restore must +# stop and point at --salvage; run again as the guide says, it restores the database and every file +# that can be read, and lists the rest (exit 2). NGB_FRT_DATA_MB sets the amount of data (default 200). # With "sftp" the location is an SFTP server instead of a shared directory, and the origin has a # file spread over several packs: the restore reads them ahead on connections of its own. # With "to-" the restore instance runs that database whatever the origin runs (a MySQL or @@ -36,7 +39,7 @@ cleanup() { docker network rm "$NET" >/dev/null 2>&1 || true docker run --rm -v "$SHARED:/target" alpine sh -c 'rm -rf /target/*' >/dev/null 2>&1 || true rmdir "$SHARED" 2>/dev/null || true - rm -f /tmp/ngb-frt-kit.json /tmp/ngb-frt-pass + rm -f /tmp/ngb-frt-kit.json /tmp/ngb-frt-pass /tmp/ngb-frt-sums [ -n "${LEGACY_DIR:-}" ] && rm -rf "$LEGACY_DIR" return 0 } @@ -105,6 +108,17 @@ if [ "$MODE" = sftp ]; then docker exec ngb-frt-origin sh -c "head -c 90000000 /dev/urandom > /var/www/html/data/frtest/files/Docs/large.bin && chown www-data:www-data /var/www/html/data/frtest/files/Docs/large.bin" LARGE_SUM=$(docker exec ngb-frt-origin sha256sum /var/www/html/data/frtest/files/Docs/large.bin | cut -d' ' -f1) fi +if [ "$MODE" = salvage-files ]; then + # Half in large files that fill packs of their own, half in files of 1 MiB; incompressible + DATA_MB=${NGB_FRT_DATA_MB:-200} + docker exec -e DATA_MB="$DATA_MB" ngb-frt-origin sh -c ' + cd /var/www/html/data/frtest/files && mkdir -p Large Small/a Small/b + i=0; while [ $((i * 50)) -lt $((DATA_MB / 2)) ]; do head -c 52428800 /dev/urandom > Large/f$i.bin; i=$((i + 1)); done + i=0; while [ $i -lt $((DATA_MB / 2)) ]; do head -c 1048576 /dev/urandom > Small/$([ $((i % 2)) = 0 ] && echo a || echo b)/f$i.bin; i=$((i + 1)); done + chown -R www-data:www-data . && find . -type f -print0 | xargs -0 sha256sum > /tmp/frt-sums' + docker cp ngb-frt-origin:/tmp/frt-sums /tmp/ngb-frt-sums + ok "origin has $DATA_MB MiB of test data, $(wc -l < /tmp/ngb-frt-sums) files" +fi run_on ngb-frt-origin files:scan frtest >/dev/null # SQL on an instance's own database (pgsql/mysql only) @@ -143,6 +157,18 @@ if [ "$MODE" = salvage ]; then || fail "damage the database dump" ok "database dump of the snapshot damaged on the location" fi +if [ "$MODE" = salvage-files ]; then + # The two largest packs hold user files for sure: bytes flipped in one, the other gone + docker exec -u www-data ngb-frt-origin php -r ' + $packs = glob("/mnt/shared/repo/packs/*/*"); + usort($packs, fn ($a, $b) => filesize($b) <=> filesize($a)); + count($packs) >= 3 or exit(1); + $h = fopen($packs[0], "r+b"); + for ($o = 4096; $o < filesize($packs[0]); $o += 1048576) { fseek($h, $o); $b = fread($h, 16); fseek($h, $o); fwrite($h, ~$b); } + fclose($h); + unlink($packs[1]);' || fail "damage the packs" + ok "one pack damaged and one pack gone on the location" +fi docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass @@ -196,6 +222,58 @@ if [ "$MODE" = salvage ]; then exit 0 fi +if [ "$MODE" = salvage-files ]; then + set +e + OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction 2>&1); CODE=$? + set -e + [ "$CODE" = 1 ] || { printf '%s\n' "$OUT" | tail -20; fail "a restore with damaged data did not stop (exit $CODE)"; } + printf '%s\n' "$OUT" | grep -- "--salvage" >/dev/null || { printf '%s\n' "$OUT" | tail -20; fail "the stopped restore does not point at --salvage"; } + run_on ngb-frt-restore status 2>&1 | grep "maintenance: true" >/dev/null || fail "maintenance mode is off after a restore that stopped halfway" + ok "damaged data: the restore stops, keeps maintenance mode on and points at --salvage" + + # As the guide and the message say: maintenance mode off and again, now with --salvage + run_on ngb-frt-restore maintenance:mode --off >/dev/null || fail "maintenance mode off" + set +e + OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction --salvage --force 2>&1); CODE=$? + set -e + [ "$CODE" = 2 ] || { printf '%s\n' "$OUT" | tail -30; fail "restore:full --salvage exited with $CODE, not 2"; } + printf '%s\n' "$OUT" | grep "could not be restored" >/dev/null || { printf '%s\n' "$OUT" | tail -30; fail "the damaged files are not listed"; } + printf '%s\n' "$OUT" | grep "ngb-salvage-" >/dev/null && { printf '%s\n' "$OUT" | tail -30; fail "the database was given up, only files are damaged"; } + docker cp /tmp/ngb-frt-sums ngb-frt-restore:/tmp/frt-sums + printf '%s\n' "$OUT" | docker exec -i ngb-frt-restore sh -c 'cat > /tmp/frt-out' + # Every file: listed and not there, or there and bit-identical + CHECK=$(docker exec ngb-frt-restore php -r ' + $root = "/var/www/html/data/frtest/files/"; $listed = []; $reasons = []; + foreach (file("/tmp/frt-out", FILE_IGNORE_NEW_LINES) as $l) { + if (preg_match("#^ (?:data/)?frtest/files/(.+?): (.*)$#", $l, $m)) { $listed[$m[1]] = true; $reasons[] = $m[2]; } + } + $same = 0; $gone = 0; $bad = []; + foreach (file("/tmp/frt-sums", FILE_IGNORE_NEW_LINES) as $l) { + [$sum, $p] = preg_split("/\s+/", $l, 2); $p = substr($p, 2); + if (isset($listed[$p])) { file_exists($root . $p) ? $bad[] = "listed but there: $p" : $gone++; continue; } + if (!file_exists($root . $p)) { $bad[] = "not listed and not there: $p"; continue; } + hash_file("sha256", $root . $p) === $sum ? $same++ : $bad[] = "different: $p"; + } + $temps = glob($root . "*/*.ngb-restore-*") ?: []; + foreach ($temps as $t) { $bad[] = "temporary left: $t"; } + echo "same=$same gone=$gone bad=", count($bad), "\n", implode("\n", array_slice($bad, 0, 10)), "\n"; + echo "reasons: ", implode(" | ", array_unique(array_map(fn ($r) => substr(preg_replace("#[0-9a-f]{16,}#", "", $r), 0, 70), $reasons))), "\n";') + printf '%s\n' "$CHECK" + printf '%s\n' "$CHECK" | grep -q " bad=0" || fail "files after the salvage are not as listed" + printf '%s\n' "$CHECK" | grep -qE "same=[1-9]" || fail "nothing was restored" + printf '%s\n' "$CHECK" | grep -qE "gone=[1-9]" || fail "no user file was hit by the damage" + ok "with --salvage: every file listed and absent, or there and identical" + run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the salvage" + run_on ngb-frt-restore user:list | grep frtest >/dev/null || fail "the backed-up user is missing: the database was not restored" + [ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt 2>/dev/null)" = "hello disaster recovery" ] \ + || printf '%s\n' "$OUT" | grep "Docs/note.txt" >/dev/null || fail "note.txt is neither restored nor listed" + run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after the salvage" + run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after the salvage" + ok "the database and config are the old server's, the instance works" + echo "FULL RESTORE ROUNDTRIP PASS ($DB, salvage-files)" + exit 0 +fi + RESTORE_OPTS=(--no-interaction --install-apps) if [ "$LEGACY" = legacy ] && [ "$DB" != sqlite ]; then # Without --skip-missing-tables an older backup must be refused, naming the app's tables