Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Empty folders in backup and restore, a full disk stops a restore, progress in the Bin

Browse Source 
- Record folders without files in the meta record of a snapshot and restore them (#27)
- Errors writing to the restore directory stop the restore, also with --salvage
- Resume: check the folder before skipping, clean temporaries in folders that are complete
- Clean only read-ahead folders of our own; refresh the lease on skipped files
- Drop tables a failed database restore created; keep --salvage from calling a restored database fresh
- Retry a pack that fails to open on the storage (SMB)
- Without cron, running backups take turns from the open page
- Bin: read in parts with a counter and progress bar, show read errors, refresh when a run ends
This commit is contained in:
Dillard Blom
2026-11-06 15:50:02 +01:00
parent ce40053cec
commit b14b16c66c
22 changed files with 693 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Unit/Repository/CatalogTest.php
+20
View File
@@ -50,6 +50,26 @@ class CatalogTest extends TestCase {
$this->assertSame(4, $again->catalogGeneration());
}
public function testTheTrashIsReadInPartsThatTogetherAreAll(): void {
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
for ($i = 0; $i < 3; $i++) {
$repo->forget($repo->backupDirectory($this->src)['snapshot']);
}
$again = $this->open();
$seen = [];
// A deadline already past: one entry per request, as on a slow location
for ($request = 0; count($seen) < 3 && $request < 5; $request++) {
$page = $again->trashPage(count($seen), 0.0);
$this->assertSame(3, $page['total']);
$this->assertCount(1, $page['items']);
$seen += $page['items'];
}
$this->assertSame(array_keys($again->trash()), array_intersect(array_keys($again->trash()), array_keys($seen)));
$this->assertCount(3, $seen);
$this->assertSame(['total' => 3, 'items' => []], $again->trashPage(3, 0.0));
$this->assertCount(3, $again->trashPage(0, microtime(true) + 60)['items']);
}
public function testRollbackIsDetected(): void {
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
$repo->backupDirectory($this->src);
tests/Unit/Repository/EmptyDirsTest.php
+127
View File
@@ -0,0 +1,127 @@
<?php
declare(strict_types=1);
namespace OCA\NgBackup\Tests\Unit\Repository;
use OCA\NgBackup\Backend\LocalBackend;
use OCA\NgBackup\Crypto\KeyRing;
use OCA\NgBackup\Job\BackupRun;
use OCA\NgBackup\Repository\Repository;
use OCA\NgBackup\Tests\Unit\TempDirTrait;
use PHPUnit\Framework\TestCase;
/** Folders without files are recorded in the snapshot's meta record and come back on a restore. */
class EmptyDirsTest extends TestCase {
use TempDirTrait;
private LocalBackend $backend;
private KeyRing $keys;
private string $src;
protected function setUp(): void {
$this->backend = new LocalBackend($this->tempDir());
$this->keys = KeyRing::generate();
Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
$this->src = $this->tempDir();
mkdir("$this->src/data/alice/files/Docs", 0700, true);
mkdir("$this->src/data/alice/files/Hollow/Nested", 0700, true);
mkdir("$this->src/data/skipped/inner", 0700, true);
file_put_contents("$this->src/data/alice/files/Docs/note.txt", "hello\n");
touch("$this->src/data/alice/files/Hollow/Nested", 1700000000);
}
private function repo(): Repository {
return Repository::openWithKey($this->backend, $this->keys);
}
/** @param list<string> $exclude */
private function backup(array $exclude = [], float $seconds = 30.0): string {
$state = BackupRun::start(['data' => "$this->src/data"], null, '', $exclude);
for ($i = 0; $i < 500 && $state['phase'] !== 'commit'; $i++) {
$state = json_decode(json_encode(BackupRun::step($this->repo(), $state, microtime(true) + $seconds)), true);
}
$this->assertSame('commit', $state['phase']);
$this->repo()->recordSnapshot($state['snapshot']);
return $state['snapshot'];
}
public function testOnlyTheDeepestEmptyFolderIsRecordedAndFilesStayAsTheyAre(): void {
$snapshot = $this->backup();
$paths = array_column($this->repo()->emptyDirs($snapshot), 0);
$this->assertSame(['data/alice/files/Hollow/Nested', 'data/skipped/inner'], $paths);
$this->assertSame(['data/alice/files/Docs/note.txt'], array_column(iterator_to_array($this->repo()->entries($snapshot), false), 'p'));
$this->assertSame(2, $this->repo()->snapshotMeta($snapshot)['stats']['emptyDirs']);
}
public function testAFolderWithOnlyExcludedContentCountsAsEmpty(): void {
$snapshot = $this->backup(['data/skipped/inner']);
$this->assertContains('data/skipped', array_column($this->repo()->emptyDirs($snapshot), 0));
$this->assertNotContains('data/skipped/inner', array_column($this->repo()->emptyDirs($snapshot), 0));
}
public function testAResumedWalkRecordsEveryFolderOnce(): void {
for ($i = 0; $i < 30; $i++) {
mkdir("$this->src/data/many/d$i", 0700, true);
file_put_contents("$this->src/data/many/f$i", 'x');
}
$snapshot = $this->backup([], 0.0); // a step per entry
$paths = array_column($this->repo()->emptyDirs($snapshot), 0);
$this->assertCount(32, $paths);
$this->assertSame(array_unique($paths), $paths);
}
public function testRestoreBringsThemBackWithTheirTime(): void {
$snapshot = $this->backup();
$target = $this->tempDir();
$this->assertSame(1, $this->repo()->restore($snapshot, $target));
$this->assertDirectoryExists("$target/data/alice/files/Hollow/Nested");
clearstatcache();
$this->assertSame(1700000000, filemtime("$target/data/alice/files/Hollow/Nested"));
$this->assertDirectoryExists("$target/data/skipped/inner");
}
public function testRestoreOfAPrefixStripsAndLeavesOthersOut(): void {
$snapshot = $this->backup();
$target = $this->tempDir();
$this->repo()->restore($snapshot, $target, 'data/alice/', true);
$this->assertDirectoryExists("$target/files/Hollow/Nested");
$this->assertDirectoryDoesNotExist("$target/skipped");
$this->assertDirectoryDoesNotExist("$target/inner");
}
public function testRunningAgainLeavesAFolderThatIsThereAlone(): void {
$snapshot = $this->backup();
$target = $this->tempDir();
$this->repo()->restore($snapshot, $target);
touch("$target/data/alice/files/Hollow/Nested", 1800000000);
file_put_contents("$target/data/alice/files/Hollow/Nested/added", 'kept');
touch("$target/data/alice/files/Hollow/Nested", 1800000000);
$this->repo()->restore($snapshot, $target, keepUnchanged: true);
clearstatcache();
$this->assertSame(1800000000, filemtime("$target/data/alice/files/Hollow/Nested"));
$this->assertFileExists("$target/data/alice/files/Hollow/Nested/added");
}
public function testALinkOutOfTheTargetIsRefused(): void {
$snapshot = $this->backup();
$target = $this->tempDir();
$outside = $this->tempDir();
mkdir("$target/data/alice/files", 0700, true);
symlink($outside, "$target/data/alice/files/Hollow");
try {
$this->repo()->restore($snapshot, $target);
$this->fail('restored through a link');
} catch (\OCA\NgBackup\Repository\RestoreTargetException $e) {
$this->assertStringContainsString('outside', $e->getMessage());
}
$this->assertSame([], glob("$outside/*"));
}
public function testASnapshotWithoutThemHasNone(): void {
$src = $this->tempDir();
file_put_contents("$src/a", 'x');
$snapshot = $this->repo()->backupDirectory($src)['snapshot'];
$this->assertSame([], $this->repo()->emptyDirs($snapshot));
}
}
tests/Unit/Repository/RestoreTargetTest.php
+119
View File
@@ -0,0 +1,119 @@
<?php
declare(strict_types=1);
namespace OCA\NgBackup\Tests\Unit\Repository;
use OCA\NgBackup\Backend\LocalBackend;
use OCA\NgBackup\Command\Restore;
use OCA\NgBackup\Crypto\KeyRing;
use OCA\NgBackup\Repository\Repository;
use OCA\NgBackup\Repository\RestoreTargetException;
use OCA\NgBackup\Tests\Unit\TempDirTrait;
use PHPUnit\Framework\TestCase;
/**
* What goes wrong where a restore writes to is not damage in the backup: it stops the restore, also
* with --salvage, and the message does not point at --salvage.
*/
class RestoreTargetTest extends TestCase {
use TempDirTrait;
private LocalBackend $backend;
private KeyRing $keys;
private string $snapshot;
protected function setUp(): void {
$this->backend = new LocalBackend($this->tempDir());
$this->keys = KeyRing::generate();
$src = $this->tempDir();
mkdir("$src/dir");
file_put_contents("$src/dir/a.bin", random_bytes(6 * 1048576));
file_put_contents("$src/dir/b.bin", random_bytes(6 * 1048576));
file_put_contents("$src/dir/c.txt", 'small');
$this->snapshot = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'))->backupDirectory($src)['snapshot'];
}
private function repo(): Repository {
return Repository::openWithKey($this->backend, $this->keys);
}
public function testAShortWriteIsATargetError(): void {
if (!is_writable('/dev/full')) {
$this->markTestSkipped('needs /dev/full');
}
$out = fopen('/dev/full', 'wb');
stream_set_write_buffer($out, 0);
$this->expectException(RestoreTargetException::class);
Repository::writeAll($out, 'data', 'dir/a.bin');
}
public function testATargetErrorDoesNotLookLikeDamage(): void {
$this->assertFalse(Restore::backupLooksDamaged(new RestoreTargetException('Cannot write')));
}
/**
* A real full disk: run with NGB_TEST_SMALL_DISK set to an empty, writable tmpfs of a few MiB
* (docker run --tmpfs /small:size=8m,uid=...), with and without --salvage.
*
* @dataProvider salvageOrNot
*/
public function testAFullDiskStopsTheRestoreAndLeavesNoTemporary(bool $salvage): void {
$disk = getenv('NGB_TEST_SMALL_DISK');
if ($disk === false || $disk === '') {
$this->markTestSkipped('needs NGB_TEST_SMALL_DISK');
}
$target = $disk . '/' . bin2hex(random_bytes(4));
$skipped = $salvage ? [] : null;
try {
$this->repo()->restore($this->snapshot, $target, skipped: $skipped);
$this->fail('12 MiB fitted on the small disk');
} catch (RestoreTargetException $e) {
$this->assertStringContainsString('disk full', $e->getMessage());
} finally {
$left = glob("$target/dir/*.ngb-restore-*");
foreach (glob("$target/dir/*") ?: [] as $f) {
unlink($f);
}
@rmdir("$target/dir");
@rmdir($target);
}
$this->assertSame([], $left);
if ($salvage) {
$this->assertSame([], $skipped, 'nothing is listed as damaged');
}
}
/** @return array<string, array{bool}> */
public static function salvageOrNot(): array {
return ['without --salvage' => [false], 'with --salvage' => [true]];
}
public function testResumeChecksTheFolderBeforeSkipping(): void {
$target = $this->tempDir();
$this->repo()->restore($this->snapshot, $target);
$outside = $this->tempDir();
rename("$target/dir", "$outside/dir");
symlink("$outside/dir", "$target/dir");
$this->expectException(RestoreTargetException::class);
$this->repo()->restore($this->snapshot, $target, keepUnchanged: true);
}
public function testResumeRemovesTemporariesWhereEverythingIsInPlace(): void {
$target = $this->tempDir();
$this->repo()->restore($this->snapshot, $target);
file_put_contents("$target/dir/a.bin.ngb-restore-0123456789ab", 'left by a stopped restore');
$this->repo()->restore($this->snapshot, $target, keepUnchanged: true);
$this->assertFileDoesNotExist("$target/dir/a.bin.ngb-restore-0123456789ab");
}
public function testHeartbeatRunsForFilesThatAreSkipped(): void {
$target = $this->tempDir();
$this->repo()->restore($this->snapshot, $target);
$beats = 0;
$this->repo()->restore($this->snapshot, $target, '', false, function () use (&$beats): void {
$beats++;
}, keepUnchanged: true);
$this->assertSame(3, $beats);
}
}