From ac57e31c9300d9369d0061d328f358bd50b19121 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Tue, 20 Oct 2026 23:04:53 +0200 Subject: [PATCH] Full restore onto PostgreSQL: counters of restored tables go after their data when the backup records none (MySQL, SQLite); test MariaDB onto PostgreSQL --- .github/workflows/ci.yml | 2 ++ CHANGELOG.md | 5 ++++ lib/Db/DbDumper.php | 27 ++++++++++++++++++--- tests/integration/full-restore-roundtrip.sh | 17 +++++++++++-- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 549c2a5..3f701f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,3 +114,5 @@ jobs: run: tests/integration/full-restore-roundtrip.sh 35-apache mysql legacy - name: Full restore over SFTP on PostgreSQL, reading packs ahead on several connections run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql sftp + - name: Full restore of a MariaDB backup onto PostgreSQL (ID counters after the data) + run: tests/integration/full-restore-roundtrip.sh 35-apache mysql to-pgsql diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c1094..650cbfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,11 @@ stopped after 13 hours with "Blob not in index"; the backup itself was fine. dropped) and emptied the index. Reading the index again now only adds, an empty listing of a location that has an index is an error instead of an empty index, and a pack that could not be read is read once more before the restore gives up. +- After a full restore of a MySQL/MariaDB or SQLite backup onto PostgreSQL, new files, users or + activities could not be added ("duplicate key", "File entry could not be inserted"): such + backups record no ID counters, and they stayed at 1. Every counter of a restored table now goes + after its data. Found in a disaster recovery simulation of 138 GB from MariaDB to PostgreSQL, + where all 15,772 files and every count were identical. - An SFTP connection that a Hetzner Storage Box dropped without phpseclib noticing is now recognised and replaced: the check asks the server and counts its answer. - The extra connection a full restore reads ahead on is replaced when it breaks (up to 20 times), diff --git a/lib/Db/DbDumper.php b/lib/Db/DbDumper.php index 605fb8e..f99da8f 100644 --- a/lib/Db/DbDumper.php +++ b/lib/Db/DbDumper.php @@ -216,7 +216,7 @@ final class DbDumper { } $report['restored'][] = $table; } - $this->restoreSequences($manifest['sequences'] ?? [], $report['missing']); + $this->restoreSequences($manifest['sequences'] ?? [], $report['missing'], $report['restored']); $this->resumeForeignKeys($foreignKeys); $this->db->commit(); // Foreign keys of the tables created above, now their rows are in. Not inside the @@ -573,16 +573,21 @@ final class DbDumper { return $name; } - /** @param list $leftOut tables not restored (--skip-missing-tables): they do not exist here */ - private function restoreSequences(array $sequences, array $leftOut = []): void { + /** + * @param list $leftOut tables not restored (--skip-missing-tables): they do not exist here + * @param list $restored tables restored: their other counters go after their data + */ + private function restoreSequences(array $sequences, array $leftOut = [], array $restored = []): void { if (!$this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) { return; } $leftOut = array_flip($leftOut); + $set = []; foreach ($sequences as $info) { if (isset($leftOut[$info['table']])) { continue; } + $set[$info['table'] . '.' . $info['column']] = true; $seq = $this->db->fetchOne('SELECT pg_get_serial_sequence(quote_ident(?), ?)', [$info['table'], $info['column']]); if ($seq === false || $seq === null) { continue; @@ -597,5 +602,21 @@ final class DbDumper { [\Doctrine\DBAL\ParameterType::STRING, \Doctrine\DBAL\ParameterType::INTEGER, \Doctrine\DBAL\ParameterType::BOOLEAN]); } } + // A backup from MySQL/MariaDB or SQLite records no counters (those derive them from the + // data): every other counter of a restored table goes after its data, or new rows (a new + // file, an activity) would collide with restored ones. + $restored = array_flip($restored); + $rows = $this->db->fetchAllAssociative( + "SELECT c.table_name, c.column_name, pg_get_serial_sequence(quote_ident(c.table_name), c.column_name) AS seq + FROM information_schema.columns c + WHERE c.table_schema = current_schema() AND pg_get_serial_sequence(quote_ident(c.table_name), c.column_name) IS NOT NULL"); + foreach ($rows as $r) { + if (!isset($restored[$r['table_name']]) || isset($set[$r['table_name'] . '.' . $r['column_name']])) { + continue; + } + $max = (int)$this->db->fetchOne(sprintf('SELECT COALESCE(MAX(%s), 0) FROM %s', + $this->db->quoteIdentifier($r['column_name']), $this->db->quoteIdentifier($r['table_name']))); + $this->db->executeQuery('SELECT setval(CAST(? AS regclass), ?, false)', [$r['seq'], $max + 1]); + } } } diff --git a/tests/integration/full-restore-roundtrip.sh b/tests/integration/full-restore-roundtrip.sh index 7b65ed7..2b3061f 100755 --- a/tests/integration/full-restore-roundtrip.sh +++ b/tests/integration/full-restore-roundtrip.sh @@ -2,10 +2,12 @@ # Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a # SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup # location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7. -# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp] +# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp|to-pgsql] # e.g. full-restore-roundtrip.sh 35-apache pgsql # With "sftp" the location is an SFTP server instead of a shared directory, and the origin has a # file spread over several packs: the restore reads them ahead on connections of its own. +# With "to-pgsql" the restore instance runs PostgreSQL whatever the origin runs (MySQL and SQLite +# backups record no ID counters; after the restore new rows must not collide with restored ones). # With "legacy" the origin runs NG Backup 1.0.0-rc.1 (0.9.7), whose backups do not record the # structure of their tables: the restore must then leave the tables of the disabled app out # with --skip-missing-tables, as for a real backup made before 1.0.0-rc.2. @@ -127,6 +129,8 @@ ok "test data backed up, snapshot $SNAP" docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass +ORIGIN_DBTYPE=$DB +[ "$MODE" = to-pgsql ] && DB=pgsql RESTORE_DB=$(db_env restore) # shellcheck disable=SC2086 docker run -d --name ngb-frt-restore --network "$NET" $RESTORE_DB \ @@ -170,6 +174,15 @@ run_on ngb-frt-restore user:list | grep -q frtest || fail "the backed-up user is docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.txt || fail "restored file is missing" [ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt)" = "hello disaster recovery" ] || fail "restored file content differs" run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore" +# New rows after the restore: a new file (file cache) and a new user must not collide with +# restored rows (ID counters after the data). +docker exec ngb-frt-restore sh -c "echo new > /var/www/html/data/frtest/files/Docs/after-restore.txt && chown www-data /var/www/html/data/frtest/files/Docs/after-restore.txt" +run_on ngb-frt-restore files:scan frtest >/dev/null || fail "a new file cannot be added after the restore" +if [ "$DB" != sqlite ]; then + [ "$(sql restore "select count(*) from oc_filecache where path = 'files/Docs/after-restore.txt'")" = 1 ] \ + || fail "a new file did not make it into the file cache after the restore (ID counters?)" +fi +docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-restore php occ user:add --password-from-env frnew >/dev/null || fail "a new user cannot be added after the restore" ok "database, files and ng_backup's own state are all intact and usable after the restore" if [ "$DB" != sqlite ] && [ "$LEGACY" = legacy ]; then @@ -197,4 +210,4 @@ fi run_on ngb-frt-restore backup:run offsite -l post-restore-check >/dev/null || fail "a new backup after the restore fails" ok "the restored instance can run a new backup of its own" -echo "FULL RESTORE ROUNDTRIP PASS ($DB${MODE:+, $MODE})" +echo "FULL RESTORE ROUNDTRIP PASS ($ORIGIN_DBTYPE${MODE:+, $MODE})"