Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Full restore onto PostgreSQL: counters of restored tables go after their data when the backup records none (MySQL, SQLite); test MariaDB onto PostgreSQL

Browse Source
This commit is contained in:
Dillard Blom
2026-10-20 23:04:53 +02:00
parent e461c48056
commit ac57e31c93
4 changed files with 46 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+2
View File
@@ -114,3 +114,5 @@ jobs:
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql legacy
- name: Full restore over SFTP on PostgreSQL, reading packs ahead on several connections
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql sftp
- name: Full restore of a MariaDB backup onto PostgreSQL (ID counters after the data)
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql to-pgsql
CHANGELOG.md
+5
View File
@@ -18,6 +18,11 @@ stopped after 13 hours with "Blob not in index"; the backup itself was fine.
dropped) and emptied the index. Reading the index again now only adds, an empty listing of a
location that has an index is an error instead of an empty index, and a pack that could not be
read is read once more before the restore gives up.
- After a full restore of a MySQL/MariaDB or SQLite backup onto PostgreSQL, new files, users or
activities could not be added ("duplicate key", "File entry could not be inserted"): such
backups record no ID counters, and they stayed at 1. Every counter of a restored table now goes
after its data. Found in a disaster recovery simulation of 138 GB from MariaDB to PostgreSQL,
where all 15,772 files and every count were identical.
- An SFTP connection that a Hetzner Storage Box dropped without phpseclib noticing is now
recognised and replaced: the check asks the server and counts its answer.
- The extra connection a full restore reads ahead on is replaced when it breaks (up to 20 times),
lib/Db/DbDumper.php
+24 -3
View File
@@ -216,7 +216,7 @@ final class DbDumper {
}
$report['restored'][] = $table;
}
$this->restoreSequences($manifest['sequences'] ?? [], $report['missing']);
$this->restoreSequences($manifest['sequences'] ?? [], $report['missing'], $report['restored']);
$this->resumeForeignKeys($foreignKeys);
$this->db->commit();
// Foreign keys of the tables created above, now their rows are in. Not inside the
@@ -573,16 +573,21 @@ final class DbDumper {
return $name;
}
/** @param list<string> $leftOut tables not restored (--skip-missing-tables): they do not exist here */
private function restoreSequences(array $sequences, array $leftOut = []): void {
/**
* @param list<string> $leftOut tables not restored (--skip-missing-tables): they do not exist here
* @param list<string> $restored tables restored: their other counters go after their data
*/
private function restoreSequences(array $sequences, array $leftOut = [], array $restored = []): void {
if (!$this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
return;
}
$leftOut = array_flip($leftOut);
$set = [];
foreach ($sequences as $info) {
if (isset($leftOut[$info['table']])) {
continue;
}
$set[$info['table'] . '.' . $info['column']] = true;
$seq = $this->db->fetchOne('SELECT pg_get_serial_sequence(quote_ident(?), ?)', [$info['table'], $info['column']]);
if ($seq === false || $seq === null) {
continue;
@@ -597,5 +602,21 @@ final class DbDumper {
[\Doctrine\DBAL\ParameterType::STRING, \Doctrine\DBAL\ParameterType::INTEGER, \Doctrine\DBAL\ParameterType::BOOLEAN]);
}
}
// A backup from MySQL/MariaDB or SQLite records no counters (those derive them from the
// data): every other counter of a restored table goes after its data, or new rows (a new
// file, an activity) would collide with restored ones.
$restored = array_flip($restored);
$rows = $this->db->fetchAllAssociative(
"SELECT c.table_name, c.column_name, pg_get_serial_sequence(quote_ident(c.table_name), c.column_name) AS seq
FROM information_schema.columns c
WHERE c.table_schema = current_schema() AND pg_get_serial_sequence(quote_ident(c.table_name), c.column_name) IS NOT NULL");
foreach ($rows as $r) {
if (!isset($restored[$r['table_name']]) || isset($set[$r['table_name'] . '.' . $r['column_name']])) {
continue;
}
$max = (int)$this->db->fetchOne(sprintf('SELECT COALESCE(MAX(%s), 0) FROM %s',
$this->db->quoteIdentifier($r['column_name']), $this->db->quoteIdentifier($r['table_name'])));
$this->db->executeQuery('SELECT setval(CAST(? AS regclass), ?, false)', [$r['seq'], $max + 1]);
}
}
}
tests/integration/full-restore-roundtrip.sh
+15 -2
View File
@@ -2,10 +2,12 @@
# Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a
# SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup
# location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7.
# Usage: full-restore-roundtrip.sh <nextcloud-image-tag> [sqlite|pgsql|mysql] [legacy|sftp]
# Usage: full-restore-roundtrip.sh <nextcloud-image-tag> [sqlite|pgsql|mysql] [legacy|sftp|to-pgsql]
# e.g. full-restore-roundtrip.sh 35-apache pgsql
# With "sftp" the location is an SFTP server instead of a shared directory, and the origin has a
# file spread over several packs: the restore reads them ahead on connections of its own.
# With "to-pgsql" the restore instance runs PostgreSQL whatever the origin runs (MySQL and SQLite
# backups record no ID counters; after the restore new rows must not collide with restored ones).
# With "legacy" the origin runs NG Backup 1.0.0-rc.1 (0.9.7), whose backups do not record the
# structure of their tables: the restore must then leave the tables of the disabled app out
# with --skip-missing-tables, as for a real backup made before 1.0.0-rc.2.
@@ -127,6 +129,8 @@ ok "test data backed up, snapshot $SNAP"
docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json
docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass
ORIGIN_DBTYPE=$DB
[ "$MODE" = to-pgsql ] && DB=pgsql
RESTORE_DB=$(db_env restore)
# shellcheck disable=SC2086
docker run -d --name ngb-frt-restore --network "$NET" $RESTORE_DB \
@@ -170,6 +174,15 @@ run_on ngb-frt-restore user:list | grep -q frtest || fail "the backed-up user is
docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.txt || fail "restored file is missing"
[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt)" = "hello disaster recovery" ] || fail "restored file content differs"
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore"
# New rows after the restore: a new file (file cache) and a new user must not collide with
# restored rows (ID counters after the data).
docker exec ngb-frt-restore sh -c "echo new > /var/www/html/data/frtest/files/Docs/after-restore.txt && chown www-data /var/www/html/data/frtest/files/Docs/after-restore.txt"
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "a new file cannot be added after the restore"
if [ "$DB" != sqlite ]; then
[ "$(sql restore "select count(*) from oc_filecache where path = 'files/Docs/after-restore.txt'")" = 1 ] \
|| fail "a new file did not make it into the file cache after the restore (ID counters?)"
fi
docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-restore php occ user:add --password-from-env frnew >/dev/null || fail "a new user cannot be added after the restore"
ok "database, files and ng_backup's own state are all intact and usable after the restore"
if [ "$DB" != sqlite ] && [ "$LEGACY" = legacy ]; then
@@ -197,4 +210,4 @@ fi
run_on ngb-frt-restore backup:run offsite -l post-restore-check >/dev/null || fail "a new backup after the restore fails"
ok "the restored instance can run a new backup of its own"
echo "FULL RESTORE ROUNDTRIP PASS ($DB${MODE:+, $MODE})"
echo "FULL RESTORE ROUNDTRIP PASS ($ORIGIN_DBTYPE${MODE:+, $MODE})"