diff --git a/CHANGELOG.md b/CHANGELOG.md index b454a3c..84ccb29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ stopped after 13 hours with "Blob not in index"; the backup itself was fine. backups record no ID counters, and they stayed at 1. Every counter of a restored table now goes after its data. Found in a disaster recovery simulation of 138 GB from MariaDB to PostgreSQL, where all 15,772 files and every count were identical. +- A full restore now runs in maintenance mode, switched off again at the end (also after an + error): a background job, a page visit or a user writing into the instance meanwhile could + collide with restored rows ("duplicate key") or be lost. - Adding a location whose repository config could not be read, while backup data is there, no longer creates a new repository over it: it stops with an explanation. A snapshot that cannot be taken over when a location is added is now logged instead of left out silently. diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md index d1f48a1..acef877 100644 --- a/docs/DISASTER-RECOVERY.md +++ b/docs/DISASTER-RECOVERY.md @@ -94,7 +94,8 @@ tables that do not exist here, install the apps they belong to and run it again, out with `--skip-missing-tables`. **8. Let the restore run.** It restores the database, then the data directory, and takes hours for -a large backup. If it stops halfway, fix the cause and run the same command again with `--force`. +a large backup. Nextcloud is in maintenance mode meanwhile, so nobody and no background job +writes into it; the restore switches it off again at the end, also after an error. If it stops halfway, fix the cause and run the same command again with `--force`. If occ then says that Nextcloud or one of the apps requires an upgrade, the database is already the old server's: run `occ upgrade` first, then the same command with `--force`. diff --git a/lib/Command/RestoreFull.php b/lib/Command/RestoreFull.php index 9774e7a..e8867c1 100644 --- a/lib/Command/RestoreFull.php +++ b/lib/Command/RestoreFull.php @@ -88,6 +88,7 @@ HELP); $this->installApp($app, $output); } } + $output->writeln('Nextcloud is in maintenance mode while the restore runs (switched off again at the end, also after an error).'); $result = $this->restoreFull->restoreFull($target, $snapshot, $skipMissingTables); } catch (\Throwable $e) { $output->writeln('' . $e->getMessage() . ''); diff --git a/lib/Service/RestoreFullService.php b/lib/Service/RestoreFullService.php index 9a0e279..4b036dd 100644 --- a/lib/Service/RestoreFullService.php +++ b/lib/Service/RestoreFullService.php @@ -30,6 +30,8 @@ final class RestoreFullService { private const CONNECTION_KEYS = [ 'dbtype', 'dbhost', 'dbport', 'dbname', 'dbuser', 'dbpassword', 'dbpersistent', 'dbtableprefix', 'dbdriveroptions', 'datadirectory', + // set by restoreFull() itself for the length of the restore + 'maintenance', ]; public function __construct( @@ -91,6 +93,25 @@ final class RestoreFullService { if (!$check['ok']) { throw new \RuntimeException($this->describeBlockers($check)); } + // Maintenance mode for the whole restore: background jobs (cron.php stops in maintenance + // mode), page visits (AJAX cron) and users must not write into tables or the data + // directory while they are being replaced; a row written in between would collide with a + // restored one or be silently lost. Switched off again at the end, also after an error: + // occ only offers app commands outside maintenance mode, and the guide's next step is + // to run the same restore again with --force. + $wasInMaintenance = $this->config->getSystemValueBool('maintenance', false); + $this->config->setSystemValue('maintenance', true); + try { + return $this->restoreInMaintenance($target, $snapshotId, $skipMissingTables, $check); + } finally { + if (!$wasInMaintenance) { + $this->config->setSystemValue('maintenance', false); + } + } + } + + /** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list} */ + private function restoreInMaintenance(Target $target, string $snapshotId, bool $skipMissingTables, array $check): array { return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $skipMissingTables, $check) { $beat = LeaseService::throttled($refresh); $repo = $this->targets->repository($target); diff --git a/tests/integration/full-restore-roundtrip.sh b/tests/integration/full-restore-roundtrip.sh index 13bcb28..f15e311 100755 --- a/tests/integration/full-restore-roundtrip.sh +++ b/tests/integration/full-restore-roundtrip.sh @@ -172,7 +172,9 @@ OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" "${RESTORE_OPTS docker exec ngb-frt-restore sh -c 'tail -c 6000 /var/www/html/data/nextcloud.log' || true fail "restore:full" } -ok "backup:restore:full completed" +printf '%s\n' "$OUT" | grep -q "maintenance mode while the restore runs" || fail "the restore did not say it runs in maintenance mode" +run_on ngb-frt-restore status 2>&1 | grep -q "maintenance: false" || fail "maintenance mode is still on after the restore" +ok "backup:restore:full completed, in maintenance mode and out of it again" # Everything below must work in a FRESH occ process (a new PHP process per docker exec), since # earlier bugs here only showed up after config.php's secret was rewritten mid-restore.