From bf48d339e4beec43fe0b562cf2a6e7725de94aa8 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Thu, 5 Nov 2026 08:35:28 +0100 Subject: [PATCH] Remove the read-ahead folder of a killed restore with what it kept; changelog and DR guide: a stopped full restore goes on where it was --- CHANGELOG.md | 15 ++++++++++++--- docs/DISASTER-RECOVERY.md | 4 +++- lib/Repository/Repository.php | 12 +++++++++++- tests/Unit/Repository/ReadAheadTest.php | 5 ++++- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9802dfe..fecf689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,9 +35,18 @@ shown under 1.0.0. The git tag of a release candidate goes after the date. since 1.0.0-rc.4. Over SMB the packs are read in 8 MiB reads instead of 1 MiB: over a line with a 130 ms round trip, reading was four times faster in a test (18 instead of 4.5 MiB/s). - Over SMB the index files, which a new server reads in `backup:target:add`, take fewer round - trips: from Canada to a Hetzner Storage Box (130 ms), - 0.66 instead of 1.05 s per file, so about 5 instead of 8 hours for the 27,000 index files of a - 1 TB repository. + trips: from Canada to a Hetzner Storage Box (130 ms), 0.66 instead of 1.05 s per file, so about + 5 instead of 8 hours for the 27,000 index files of a 1 TB repository. +- `occ backup:restore:full` that runs again after it stopped (`--force`) goes on where it was: data + files already in place with the size and modification time of the backup are left, and only + the rest is read from the location. Before, a stopped restore of 1 TB started over. + +### Fixed +- A full restore over SMB could hang at 100% CPU when the connection to the server dropped: a + failed read was counted as data. It is now read again over a new connection. Found in a + disaster recovery test of 1 TB from Canada to a Hetzner Storage Box, after 392 GiB. +- The read-ahead folder of a restore that was killed while it wrote is removed by the next run; + it kept one file of up to 8 MiB. - `occ backup:target:add` only estimates the time left for reading the index once enough of it is in. The first file also carries the time to connect, which made it say "about 642 min left" for something done within a minute. diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md index a71ac28..df72ae5 100644 --- a/docs/DISASTER-RECOVERY.md +++ b/docs/DISASTER-RECOVERY.md @@ -103,7 +103,9 @@ mode on yourself before, it stays on: switch it off with `occ maintenance:mode - If it stops halfway, maintenance mode stays on (the instance may be partly restored). Fix the cause, then switch maintenance mode off and run the restore again with `--force`, in one line (the message prints it): `occ maintenance:mode --off && occ backup:restore:full --force`. -Leave out `--install-apps` this time, so that the instance is open only for the seconds until the +It goes on where it stopped: data files that are already in place with the size and modification +time of the backup are left, so only the rest comes from the location again (since 1.0.2; before, +it started over). The database is restored again in full, which takes minutes. Leave out `--install-apps` this time, so that the instance is open only for the seconds until the restore has switched maintenance mode on again; apps the first run could not install are listed at the end of the restore, install those afterwards. If occ then says that Nextcloud or one of the apps requires an upgrade, the database is already diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index 954e56e..44be374 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -253,9 +253,19 @@ final class Repository { if (!$this->backend instanceof IReadsAhead || !$this->backend->canReadAhead()) { return null; } - // what a restore that was stopped left behind: empty, its files were unlinked when opened + // what a restore that was stopped left behind: its files are unlinked once opened, but one + // that was being written when the process was killed stays (8 MiB, DR test 2026-11-05) $stale = glob(rtrim($target, '/') . '/.ngb-restore-spool-*', GLOB_ONLYDIR); foreach ($stale === false ? [] : $stale as $dir) { + if (is_link($dir)) { + continue; + } + $files = glob($dir . '/*'); + foreach ($files === false ? [] : $files as $file) { + if (is_file($file) && !is_link($file)) { + @unlink($file); + } + } @rmdir($dir); } $plan = new ReadAheadPlan(rtrim($target, '/') . '/.ngb-restore-spool-' . bin2hex(random_bytes(4))); diff --git a/tests/Unit/Repository/ReadAheadTest.php b/tests/Unit/Repository/ReadAheadTest.php index 2e9ef34..8e42e94 100644 --- a/tests/Unit/Repository/ReadAheadTest.php +++ b/tests/Unit/Repository/ReadAheadTest.php @@ -75,11 +75,14 @@ class ReadAheadTest extends TestCase { $this->assertSame(2, $backend->snapshotReads, 'the restore itself read the snapshot again'); } - public function testTheEmptySpoolOfAStoppedRestoreIsRemoved(): void { + public function testTheSpoolOfAStoppedRestoreIsRemoved(): void { $target = $this->tempDir(); mkdir("$target/.ngb-restore-spool-0badc0de", 0700); // a restore that was killed + mkdir("$target/.ngb-restore-spool-8badf00d", 0700); // ... while it wrote a range + file_put_contents("$target/.ngb-restore-spool-8badf00d/smb-01234567", str_repeat('x', 4096)); $this->assertRestored($this->restore(new ReadAheadBackend(new LocalBackend($this->root)), '', $target)); + $this->assertSame([], glob("$target/.ngb-restore-spool-*")); } public function testARangeThatCouldNotBeReadAheadIsReadTheUsualWay(): void {