From 8e5c8ccda45296c8ce9a13ed4367bce9088cc151 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Sun, 18 Oct 2026 23:44:00 +0200 Subject: [PATCH] Full restore: skip the ID counters of tables left out with --skip-missing-tables; test a backup made by rc.1; version 0.9.9 --- .github/workflows/ci.yml | 4 +++ CHANGELOG.md | 6 ++++ appinfo/info.xml | 2 +- lib/Db/DbDumper.php | 9 +++-- tests/integration/full-restore-roundtrip.sh | 38 +++++++++++++++++---- 5 files changed, 50 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8df09c..dbc09a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,3 +108,7 @@ jobs: run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql - name: Full restore round trip on MariaDB, with Mail (foreign keys) and a disabled app run: tests/integration/full-restore-roundtrip.sh 35-apache mysql + - name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on PostgreSQL + run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql legacy + - name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on MariaDB + run: tests/integration/full-restore-roundtrip.sh 35-apache mysql legacy diff --git a/CHANGELOG.md b/CHANGELOG.md index 3409abf..0244488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ instead carried by the app description, the git tag, the release, and this file. ## [Unreleased] +### Fixed +- A full restore of a backup made before 1.0.0-rc.2 with `--skip-missing-tables` failed on + PostgreSQL with "relation ... does not exist": the left-out tables were still looked up to set + their ID counters. It now skips them. A new test restores a backup made by 1.0.0-rc.1 on + PostgreSQL and MariaDB. + ## [1.0.0-rc.2] - 2026-10-18 Second release candidate for 1.0.0; the app version is 0.9.8. Fixes from a full disaster recovery diff --git a/appinfo/info.xml b/appinfo/info.xml index efe6cc8..8763946 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -41,7 +41,7 @@ welcome: use the "Share your experience" link on the admin page. Developed and sponsored by [Ensembia](https://www.ensembia.com), which also offers paid help with restoring a backup after a crash.]]> - 0.9.8 + 0.9.9 agpl Dillard Blom NgBackup diff --git a/lib/Db/DbDumper.php b/lib/Db/DbDumper.php index 716b63e..605fb8e 100644 --- a/lib/Db/DbDumper.php +++ b/lib/Db/DbDumper.php @@ -216,7 +216,7 @@ final class DbDumper { } $report['restored'][] = $table; } - $this->restoreSequences($manifest['sequences'] ?? []); + $this->restoreSequences($manifest['sequences'] ?? [], $report['missing']); $this->resumeForeignKeys($foreignKeys); $this->db->commit(); // Foreign keys of the tables created above, now their rows are in. Not inside the @@ -573,11 +573,16 @@ final class DbDumper { return $name; } - private function restoreSequences(array $sequences): void { + /** @param list $leftOut tables not restored (--skip-missing-tables): they do not exist here */ + private function restoreSequences(array $sequences, array $leftOut = []): void { if (!$this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) { return; } + $leftOut = array_flip($leftOut); foreach ($sequences as $info) { + if (isset($leftOut[$info['table']])) { + continue; + } $seq = $this->db->fetchOne('SELECT pg_get_serial_sequence(quote_ident(?), ?)', [$info['table'], $info['column']]); if ($seq === false || $seq === null) { continue; diff --git a/tests/integration/full-restore-roundtrip.sh b/tests/integration/full-restore-roundtrip.sh index e9c3657..a26e755 100755 --- a/tests/integration/full-restore-roundtrip.sh +++ b/tests/integration/full-restore-roundtrip.sh @@ -2,15 +2,20 @@ # Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a # SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup # location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7. -# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] +# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy] # e.g. full-restore-roundtrip.sh 35-apache pgsql +# With "legacy" the origin runs NG Backup 1.0.0-rc.1 (0.9.7), whose backups do not record the +# structure of their tables: the restore must then leave the tables of the disabled app out +# with --skip-missing-tables, as for a real backup made before 1.0.0-rc.2. # With pgsql or mysql the origin also has what broke real disaster recoveries: the Mail app with a synced # account (foreign keys between its tables) and Deck installed and then switched off (tables of # an app that is not enabled). The restore instance has neither; --install-apps adds Mail. set -euo pipefail IMAGE="nextcloud:${1:-35-apache}" DB="${2:-sqlite}" +LEGACY="${3:-}" APP=$(cd "$(dirname "$0")/../.." && pwd) +ORIGIN_APP="$APP" NET=ngb-frt-net SHARED=$(mktemp -d) chmod 777 "$SHARED" @@ -23,9 +28,17 @@ cleanup() { docker run --rm -v "$SHARED:/target" alpine sh -c 'rm -rf /target/*' >/dev/null 2>&1 || true rmdir "$SHARED" 2>/dev/null || true rm -f /tmp/ngb-frt-kit.json /tmp/ngb-frt-pass + [ -n "${LEGACY_DIR:-}" ] && rm -rf "$LEGACY_DIR" + return 0 } trap cleanup EXIT +if [ "$LEGACY" = legacy ]; then + LEGACY_DIR=$(mktemp -d) + curl -fsSL https://github.com/dillardblom/ng_backup/releases/download/v1.0.0-rc.1/ng_backup-0.9.7.tar.gz | tar -xz -C "$LEGACY_DIR" + ORIGIN_APP="$LEGACY_DIR/ng_backup" +fi + run_on() { docker exec -u www-data "$1" php occ "${@:2}"; } # Database settings for an instance: SQLite inside the container, or its own PostgreSQL container @@ -50,11 +63,11 @@ ORIGIN_DB=$(db_env origin) # shellcheck disable=SC2086 docker run -d --name ngb-frt-origin --network "$NET" $ORIGIN_DB \ -e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \ - -v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null + -v "$ORIGIN_APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null for i in $(seq 1 90); do run_on ngb-frt-origin status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done docker exec ngb-frt-origin chown www-data:www-data /var/www/html/custom_apps run_on ngb-frt-origin app:enable ng_backup >/dev/null -ok "origin instance up ($DB)" +ok "origin instance up ($DB${LEGACY:+, NG Backup 0.9.7})" docker exec ngb-frt-origin sh -c "printf 'full restore roundtrip passphrase 2026\n' > /tmp/pass" run_on ngb-frt-origin backup:key:init --passphrase-file=/tmp/pass --label=CI >/dev/null || fail "key init" @@ -118,7 +131,16 @@ run_on ngb-frt-restore backup:target:add offsite local -a null::null -o datadir= run_on ngb-frt-restore backup:list offsite | grep -q "$SNAP" || fail "the origin's snapshot is not listed after connecting the location" ok "recovery kit imported, existing repository adopted, snapshot visible" -OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction --install-apps 2>&1) || { +RESTORE_OPTS=(--no-interaction --install-apps) +if [ "$LEGACY" = legacy ] && [ "$DB" != sqlite ]; then + # Without --skip-missing-tables an older backup must be refused, naming the app's tables + REFUSED=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" "${RESTORE_OPTS[@]}" 2>&1) \ + && fail "an older backup with tables of unknown structure was restored without --skip-missing-tables" + printf '%s\n' "$REFUSED" | grep -q "oc_deck_boards" || { printf '%s\n' "$REFUSED" | tail -20; fail "the refusal does not name the missing tables"; } + ok "older backup refused without --skip-missing-tables, missing tables named" + RESTORE_OPTS+=(--skip-missing-tables) +fi +OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" "${RESTORE_OPTS[@]}" 2>&1) || { printf '%s\n' "$OUT" | tail -40 docker exec ngb-frt-restore sh -c 'tail -c 6000 /var/www/html/data/nextcloud.log' || true fail "restore:full" @@ -134,7 +156,11 @@ docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.tx run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore" ok "database, files and ng_backup's own state are all intact and usable after the restore" -if [ "$DB" != sqlite ]; then +if [ "$DB" != sqlite ] && [ "$LEGACY" = legacy ]; then + [ "$(sql restore "select count(*) from information_schema.tables where table_name = 'oc_deck_boards'")" = 0 ] || fail "a table of unknown structure was created" + [ "$(sql restore "select count(*) from oc_mail_accounts where drafts_mailbox_id is not null")" = 1 ] || fail "the mail account was not restored" + ok "older backup: tables of the disabled app left out, Mail restored" +elif [ "$DB" != sqlite ]; then [ "$(sql restore "select count(*) from information_schema.tables where table_name = 'oc_deck_boards'")" = 1 ] || fail "the tables of the disabled app were not created" [ "$(sql restore "select count(*) from oc_mail_accounts where drafts_mailbox_id is not null")" = 1 ] || fail "the mail account was not restored" [ "$(sql restore "select count(*) from information_schema.table_constraints where constraint_type = 'FOREIGN KEY' and table_name like 'oc_mail_%'")" -gt 0 ] \ @@ -146,4 +172,4 @@ fi run_on ngb-frt-restore backup:run offsite -l post-restore-check >/dev/null || fail "a new backup after the restore fails" ok "the restored instance can run a new backup of its own" -echo "FULL RESTORE ROUNDTRIP PASS ($DB)" +echo "FULL RESTORE ROUNDTRIP PASS ($DB${LEGACY:+, legacy})"