Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 17 Wiki Activity

M2: cursor-based resumable runs with merge-join against the parent

Browse Source 
TreeWalker walks one or more roots in a stable segment-wise order with
exclusions and resumes after a cursor; ParentCursor merge-joins the
previous snapshot in the same order. BackupRun no longer keeps a file
list or the parent's entries in memory or in its state.
This commit is contained in:
Dillard Blom
2026-10-02 00:46:31 +02:00
parent 506854d597
commit 4a28bf8790
7 changed files with 344 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/.phpunit.result.cache
+1 -1
View File
@@ -1 +1 @@
{"version":2,"defects":[],"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.115,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.074,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.003,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.262,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":4.044,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":6.03,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.997,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.012,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.223,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.307,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.14,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.281,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.327,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.618,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.669}}
{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.075,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.094,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.004,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.276,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.498,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.8,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":2.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.016,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.136,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.049,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.087,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.064,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.095,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.24,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.802,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.006,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005}}
tests/Unit/Job/BackupRunTest.php
+36 -6
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace OCA\NgBackup\Tests\Unit\Job;
use OCA\NgBackup\Backend\LocalBackend;
use OCA\NgBackup\Crypto\KeyRing;
use OCA\NgBackup\Job\BackupRun;
use OCA\NgBackup\Repository\Repository;
use OCA\NgBackup\Tests\Unit\TempDirTrait;
@@ -21,13 +22,14 @@ class BackupRunTest extends TestCase {
}
file_put_contents("$src/big.bin", random_bytes(13 * 1048576));
$backend = new LocalBackend($this->tempDir());
Repository::init($backend, 'pw');
$keys = KeyRing::generate();
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
$state = BackupRun::start($src);
$steps = 0;
while ($state['phase'] !== 'done' && $steps < 500) {
// Fresh repository object per step, state through JSON: as separate processes would.
$repo = Repository::open($backend, 'pw');
$repo = Repository::openWithKey($backend, $keys);
$state = json_decode(json_encode(BackupRun::step($repo, $state, microtime(true) + 0.05)), true);
$steps++;
}
@@ -35,7 +37,7 @@ class BackupRunTest extends TestCase {
$this->assertGreaterThan(2, $steps, 'the run was actually split into steps');
$target = $this->tempDir();
Repository::open($backend, 'pw')->restore($state['snapshot'], $target);
Repository::openWithKey($backend, $keys)->restore($state['snapshot'], $target);
$this->assertSame(self::hashTree($src), self::hashTree($target));
}
@@ -43,12 +45,13 @@ class BackupRunTest extends TestCase {
$src = $this->tempDir();
file_put_contents("$src/big.bin", random_bytes(20 * 1048576));
$backend = new LocalBackend($this->tempDir());
Repository::init($backend, 'pw');
$keys = KeyRing::generate();
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
$state = BackupRun::start($src);
$mutated = false;
for ($i = 0; $i < 500 && $state['phase'] !== 'done'; $i++) {
$state = BackupRun::step(Repository::open($backend, 'pw'), $state, microtime(true) + 0.01);
$state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 0.01);
if (!$mutated && $state['cur'] !== null && $state['cur']['offset'] > 0) {
$fh = fopen("$src/big.bin", 'r+b');
fwrite($fh, random_bytes(1024)); // same size, new content at the start
@@ -60,7 +63,34 @@ class BackupRunTest extends TestCase {
$this->assertTrue($mutated);
$this->assertGreaterThanOrEqual(1, $state['stats']['retried'] ?? 0);
$target = $this->tempDir();
Repository::open($backend, 'pw')->restore($state['snapshot'], $target);
Repository::openWithKey($backend, $keys)->restore($state['snapshot'], $target);
$this->assertSame(self::hashTree($src), self::hashTree($target));
}
public function testIncrementalRunInTinyStepsReadsNothing(): void {
$src = $this->tempDir();
for ($i = 0; $i < 200; $i++) {
@mkdir("$src/d" . ($i % 7), 0700, true);
file_put_contents("$src/d" . ($i % 7) . "/f$i", random_bytes(3000));
}
$backend = new LocalBackend($this->tempDir());
$keys = KeyRing::generate();
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
$run = function (?string $parent) use ($backend, $src, $keys): array {
$state = BackupRun::start(['data' => $src], $parent);
for ($i = 0; $i < 2000 && $state['phase'] !== 'done'; $i++) {
$state = json_decode(json_encode(BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true))), true);
}
return $state;
};
$first = $run(null);
$second = $run($first['snapshot']);
$this->assertSame('done', $second['phase']);
$this->assertGreaterThan(150, $second["steps"], "one file per step");
$this->assertSame(0, $second['stats']['read'], 'merge-join with the parent across steps reuses every file');
$this->assertSame(200, $second['stats']['reused']);
$target = $this->tempDir();
Repository::openWithKey($backend, $keys)->restore($second['snapshot'], $target);
$this->assertSame(self::hashTree($src), self::hashTree("$target/data"));
}
}
tests/Unit/Job/TreeWalkerTest.php
+49
View File
@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
namespace OCA\NgBackup\Tests\Unit\Job;
use OCA\NgBackup\Job\TreeWalker;
use OCA\NgBackup\Tests\Unit\TempDirTrait;
use PHPUnit\Framework\TestCase;
class TreeWalkerTest extends TestCase {
use TempDirTrait;
private function tree(): string {
$d = $this->tempDir();
foreach (['a.txt', 'a/x', 'a/b/y', 'a-b', 'b', 'skip/me', 'Z'] as $f) {
@mkdir(dirname("$d/$f"), 0700, true);
file_put_contents("$d/$f", $f);
}
return $d;
}
public function testOrderMatchesCompare(): void {
$paths = array_keys(iterator_to_array((new TreeWalker(['' => $this->tree()]))->walk()));
$sorted = $paths;
usort($sorted, [TreeWalker::class, 'compare']);
$this->assertSame($sorted, $paths);
$this->assertSame(['Z', 'a', 'a-b', 'a.txt', 'b', 'skip'], array_values(array_unique(array_map(fn ($p) => explode('/', $p)[0], $paths))));
}
public function testResumeAfterCursorYieldsTheRest(): void {
$w = new TreeWalker(['' => $this->tree()]);
$all = array_keys(iterator_to_array($w->walk()));
foreach ($all as $i => $cursor) {
$this->assertSame(array_slice($all, $i + 1), array_keys(iterator_to_array($w->walk($cursor))), "after $cursor");
}
}
public function testExcludeAndMultipleRoots(): void {
$d = $this->tree();
$c = $this->tempDir();
file_put_contents("$c/config.php", '<?php');
$paths = array_keys(iterator_to_array((new TreeWalker(['config' => $c, 'data' => $d], ['data/skip', 'data/b']))->walk()));
$this->assertContains('config/config.php', $paths);
$this->assertContains('data/a/b/y', $paths);
$this->assertNotContains('data/skip/me', $paths);
$this->assertNotContains('data/b', $paths);
}
}