From 4a28bf879081b08fdb323a5d3dd946d34d949bd7 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Fri, 2 Oct 2026 00:46:31 +0200 Subject: [PATCH] M2: cursor-based resumable runs with merge-join against the parent TreeWalker walks one or more roots in a stable segment-wise order with exclusions and resumes after a cursor; ParentCursor merge-joins the previous snapshot in the same order. BackupRun no longer keeps a file list or the parent's entries in memory or in its state. --- lib/Job/BackupRun.php | 180 ++++++++++++++---------------- lib/Job/ParentCursor.php | 57 ++++++++++ lib/Job/TreeWalker.php | 102 +++++++++++++++++ lib/Repository/Repository.php | 17 +++ tests/.phpunit.result.cache | 2 +- tests/Unit/Job/BackupRunTest.php | 42 ++++++- tests/Unit/Job/TreeWalkerTest.php | 49 ++++++++ 7 files changed, 344 insertions(+), 105 deletions(-) create mode 100644 lib/Job/ParentCursor.php create mode 100644 lib/Job/TreeWalker.php create mode 100644 tests/Unit/Job/TreeWalkerTest.php diff --git a/lib/Job/BackupRun.php b/lib/Job/BackupRun.php index 03ce2d4..ca928b5 100644 --- a/lib/Job/BackupRun.php +++ b/lib/Job/BackupRun.php @@ -10,158 +10,142 @@ namespace OCA\NgBackup\Job; use OCA\NgBackup\Repository\Repository; /** - * A backup run as a resumable state machine, for environments with short PHP time limits - * (webcron/AJAX cron on hosted Nextcloud). Each step works until its deadline, then uploads - * what it has (pack + tree segment) and returns the new state. Even a single large file is - * resumed at the blob where the previous step stopped. + * A file backup as a resumable state machine, for environments with short PHP time limits. + * Each step works until its deadline, uploads what it has (pack + tree segment) and returns + * the new state; a large file is resumed at the next blob. * - * The state is a plain array (in the app: a database row). It is only saved after a step - * completes; if a process is killed mid-step, the next step redoes that work. Blobs uploaded by - * the killed step are not referenced and are removed later by pruning; nothing is corrupted, - * because the snapshot is written last. + * The state holds no file list: the position is a cursor in TreeWalker order, and the previous + * snapshot is merge-joined through ParentCursor, so memory and state size stay small however + * many files there are. The snapshot is written last; a killed step is simply redone. */ final class BackupRun { - public static function start(string $source, ?string $parent = null, string $label = ''): array { + /** + * @param string|array $roots one directory, or name => directory (walked in order) + * @param list $exclude logical path prefixes to skip + * @param array $meta extra fields for the snapshot meta record + */ + public static function start(string|array $roots, ?string $parent = null, string $label = '', array $exclude = [], array $meta = []): array { + $roots = is_string($roots) ? ['' => rtrim($roots, '/')] : $roots; + uksort($roots, fn ($a, $b) => TreeWalker::compare((string)$a, (string)$b)); // walk order = compare order return [ 'snapshot' => bin2hex(random_bytes(16)), - 'source' => rtrim($source, '/'), + 'roots' => $roots, + 'exclude' => $exclude, 'parent' => $parent, 'label' => $label, + 'meta' => $meta, 'started' => gmdate('c'), - 'phase' => 'scan', - 'files' => [], - 'pos' => 0, + 'phase' => 'files', + 'after' => null, + 'parentPos' => 0, 'cur' => null, 'segments' => [], 'steps' => 0, - 'stats' => ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0], + 'stats' => ['files' => 0, 'bytes' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0], ]; } /** Run one step until $deadline (microtime(true)); returns the new state. */ public static function step(Repository $repo, array $state, float $deadline): array { $state['steps']++; - if ($state['phase'] === 'scan') { - $state['files'] = self::scan($state['source']); - $state['phase'] = 'files'; - // Scanning can take a while on big trees; continue in the same step if time is left. - } - if ($state['phase'] === 'files') { - $previous = $state['parent'] !== null ? $repo->snapshotEntries($state['parent']) : []; + $walker = new TreeWalker($state['roots'], $state['exclude']); + $parent = new ParentCursor($repo, $state['parent'], $state['parentPos']); $stats = $state['stats']; $lines = []; - $total = count($state['files']); - while ($state['pos'] < $total) { - $relative = $state['files'][$state['pos']]; - $path = $state['source'] . '/' . $relative; - clearstatcache(true, $path); - $size = @filesize($path); - $mtime = @filemtime($path); - if ($size === false) { - // Deleted since the scan: skip. - $state['pos']++; - $state['cur'] = null; - continue; + $finishedWalk = true; + + foreach ($walker->walk($state['after']) as $logical => $path) { + $version = Repository::fileVersion($path); + if ($version === 'missing') { + continue; // deleted while walking } - $prev = $previous[$relative] ?? null; + [$size, $mtime] = array_map('intval', explode(':', $version, 3)); + $prev = $parent->find($logical); + if ($state['cur'] === null && $prev !== null && $prev['s'] === $size && $prev['m'] === $mtime) { - $lines[] = self::line($relative, $size, $mtime, $prev['b']); - $stats['files']++; + $blobs = $prev['b']; $stats['reused']++; - $state['pos']++; - continue; + } else { + $cur = $state['cur']; + if ($cur === null || $cur['path'] !== $logical || $cur['version'] !== $version) { + if ($cur !== null && $cur['path'] === $logical) { + $stats['retried'] = ($stats['retried'] ?? 0) + 1; // changed since the previous step + } + $cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version]; + } + $fh = fopen($path, 'rb'); + if ($fh === false) { + $stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1; + $state['cur'] = null; + continue; + } + fseek($fh, $cur['offset']); + $complete = true; + while (($data = self::readFull($fh, Repository::BLOB_SIZE)) !== '') { + $cur['blobs'][] = $repo->storeData($data, $stats); + $cur['offset'] += strlen($data); + if (microtime(true) >= $deadline && !feof($fh)) { + $complete = false; + break; + } + } + fclose($fh); + if (Repository::fileVersion($path) !== $cur['version']) { + // Changed while reading: read it again from the start (now or next step). + $stats['retried'] = ($stats['retried'] ?? 0) + 1; + $state['cur'] = null; + $finishedWalk = false; + break; + } + if (!$complete) { + $state['cur'] = $cur; + $finishedWalk = false; + break; + } + $blobs = $cur['blobs']; + $state['cur'] = null; + $stats['read']++; } - $version = Repository::fileVersion($path); - $cur = $state['cur']; - if ($cur === null || $cur['version'] !== $version) { - // New file, or it changed since the previous step: (re)start reading it. - if ($cur !== null) { - $stats['retried'] = ($stats['retried'] ?? 0) + 1; - } - $cur = ['path' => $relative, 'offset' => 0, 'blobs' => [], 'version' => $version]; - } - $fh = fopen($path, 'rb'); - fseek($fh, $cur['offset']); - $finished = true; - while (($data = self::readFull($fh, Repository::BLOB_SIZE)) !== '') { - $cur['blobs'][] = $repo->storeData($data, $stats); - $cur['offset'] += strlen($data); - if (microtime(true) >= $deadline && !feof($fh)) { - $finished = false; - break; - } - } - fclose($fh); - if (Repository::fileVersion($path) !== $cur['version']) { - // Changed while reading: start this file again (in this or the next step). - $stats['retried'] = ($stats['retried'] ?? 0) + 1; - $state['cur'] = null; - if (microtime(true) >= $deadline) { - break; - } - continue; - } - if (!$finished) { - $state['cur'] = $cur; - break; - } - $lines[] = self::line($relative, $size, $mtime, $cur['blobs']); + $lines[] = json_encode(['p' => $logical, 's' => $size, 'm' => $mtime, 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); $stats['files']++; - $stats['read']++; - $state['cur'] = null; - $state['pos']++; + $stats['bytes'] += $size; + $state['after'] = $logical; if (microtime(true) >= $deadline) { + $finishedWalk = false; break; } } - // Persist this step's tree lines as a blob stream segment and upload the open pack. if ($lines !== []) { $writer = $repo->blobWriter($stats); $writer->write(implode("\n", $lines) . "\n"); $state['segments'][] = $writer->finish(); } $repo->flushPacks(); + $state['parentPos'] = $parent->position(); $state['stats'] = $stats; - if ($state['pos'] >= $total) { + if ($finishedWalk) { $state['phase'] = 'finish'; } } - if ($state['phase'] === 'finish' && microtime(true) < $deadline + 5) { + if ($state['phase'] === 'finish') { $segments = $state['segments']; $repo->writeSnapshot($state['snapshot'], - ['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'], 'source' => $state['source']], + ['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'], + 'roots' => array_keys($state['roots']), 'stats' => $state['stats']] + $state['meta'], (function () use ($repo, $segments) { foreach ($segments as $blobs) { yield from $repo->readLines($blobs); } })()); $state['phase'] = 'done'; - $state['files'] = []; } return $state; } - /** @return list */ - private static function scan(string $root): array { - $files = []; - $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS)); - foreach ($it as $file) { - if ($file->isFile()) { - $files[] = substr($file->getPathname(), strlen($root) + 1); - } - } - sort($files, SORT_STRING); - return $files; - } - - private static function line(string $path, int $size, int $mtime, array $blobs): string { - return json_encode(['p' => $path, 's' => $size, 'm' => $mtime, 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); - } - private static function readFull($fh, int $length): string { $buffer = ''; while (strlen($buffer) < $length && !feof($fh)) { diff --git a/lib/Job/ParentCursor.php b/lib/Job/ParentCursor.php new file mode 100644 index 0000000..4394863 --- /dev/null +++ b/lib/Job/ParentCursor.php @@ -0,0 +1,57 @@ +entries = $repo->entries($snapshotId); + $this->current = $this->entries->valid() ? $this->entries->current() : null; + while ($this->consumed < $skip && $this->current !== null) { + $this->advance(); + } + } + + /** @return array{p:string, s:int, m:int, b:list}|null */ + public function find(string $path): ?array { + while ($this->current !== null) { + $c = TreeWalker::compare($this->current['p'], $path); + if ($c === 0) { + return $this->current; + } + if ($c > 0) { + return null; + } + $this->advance(); + } + return null; + } + + public function position(): int { + return $this->consumed; + } + + private function advance(): void { + $this->entries->next(); + $this->consumed++; + $this->current = $this->entries->valid() ? $this->entries->current() : null; + } +} diff --git a/lib/Job/TreeWalker.php b/lib/Job/TreeWalker.php new file mode 100644 index 0000000..67ad91a --- /dev/null +++ b/lib/Job/TreeWalker.php @@ -0,0 +1,102 @@ +/" for files. + * + * The same order is used for comparing paths (compare()), so a walk can resume after a cursor + * and be merge-joined with the previous snapshot without holding any list in memory. + */ +final class TreeWalker { + /** + * @param array $roots name => absolute directory, walked in this order + * @param list $exclude path prefixes ("/") that are skipped entirely + */ + public function __construct( + private array $roots, + private array $exclude = [], + ) { + } + + /** @return \Generator logical path => absolute path, strictly after $after */ + public function walk(?string $after = null): \Generator { + foreach ($this->roots as $name => $dir) { + $name = (string)$name; + if ($name !== '' && $after !== null && self::compare($name, self::firstSegment($after)) < 0) { + continue; // whole root lies before the cursor + } + yield from $this->walkDir(rtrim($dir, '/'), $name, $after); + } + } + + /** Order used everywhere: segment by segment, byte-wise. */ + public static function compare(string $a, string $b): int { + $sa = explode('/', $a); + $sb = explode('/', $b); + $n = min(count($sa), count($sb)); + for ($i = 0; $i < $n; $i++) { + $c = strcmp($sa[$i], $sb[$i]); + if ($c !== 0) { + return $c < 0 ? -1 : 1; + } + } + return count($sa) <=> count($sb); + } + + private function walkDir(string $abs, string $logical, ?string $after): \Generator { + foreach ($this->exclude as $prefix) { + if ($logical !== '' && ($logical === $prefix || str_starts_with($logical, $prefix . '/'))) { + return; + } + } + $names = @scandir($abs); + if ($names === false) { + return; + } + $names = array_values(array_diff($names, ['.', '..'])); + sort($names, SORT_STRING); + foreach ($names as $name) { + $childLogical = $logical === '' ? $name : $logical . '/' . $name; + $childAbs = $abs . '/' . $name; + if (is_link($childAbs)) { + continue; // never follow symlinks out of the data directory + } + if (is_dir($childAbs)) { + // Skip a directory entirely if everything in it sorts before the cursor. + if ($after !== null && self::compare($childLogical, $after) < 0 && !self::isPrefix($childLogical, $after)) { + continue; + } + yield from $this->walkDir($childAbs, $childLogical, $after); + } elseif (is_file($childAbs)) { + if ($after !== null && self::compare($childLogical, $after) <= 0) { + continue; + } + $excluded = false; + foreach ($this->exclude as $prefix) { + if ($childLogical === $prefix) { + $excluded = true; + break; + } + } + if (!$excluded) { + yield $childLogical => $childAbs; + } + } + } + } + + private static function isPrefix(string $dir, string $path): bool { + return str_starts_with($path, $dir . '/'); + } + + private static function firstSegment(string $path): string { + return explode('/', $path, 2)[0]; + } +} diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index 3d9fe4d..7684f7e 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -342,6 +342,23 @@ final class Repository { return $map; } + /** @return \Generator}> entries of a snapshot, in stored order */ + public function entries(string $snapshotId): \Generator { + return $this->streamEntries($snapshotId); + } + + /** The meta record (first line) of a snapshot. */ + public function snapshotMeta(string $snapshotId): array { + $plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); + $fh = $this->backend->get('snapshots/' . $snapshotId); + $this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId); + fclose($fh); + rewind($plain); + $meta = json_decode((string)fgets($plain), true, 512, JSON_THROW_ON_ERROR); + fclose($plain); + return $meta; + } + /** @return \Generator}> */ private function streamEntries(string $snapshotId): \Generator { $plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); diff --git a/tests/.phpunit.result.cache b/tests/.phpunit.result.cache index 29ad6fc..34c318e 100644 --- a/tests/.phpunit.result.cache +++ b/tests/.phpunit.result.cache @@ -1 +1 @@ -{"version":2,"defects":[],"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.115,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.074,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.003,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.262,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":4.044,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":6.03,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.997,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.012,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.223,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.307,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.14,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.281,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.327,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.618,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.669}} \ No newline at end of file +{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.075,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.094,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.004,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.276,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.498,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.8,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":2.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.016,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.136,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.049,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.087,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.064,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.095,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.24,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.802,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.006,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005}} \ No newline at end of file diff --git a/tests/Unit/Job/BackupRunTest.php b/tests/Unit/Job/BackupRunTest.php index 34dfa66..9df3667 100644 --- a/tests/Unit/Job/BackupRunTest.php +++ b/tests/Unit/Job/BackupRunTest.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace OCA\NgBackup\Tests\Unit\Job; use OCA\NgBackup\Backend\LocalBackend; +use OCA\NgBackup\Crypto\KeyRing; use OCA\NgBackup\Job\BackupRun; use OCA\NgBackup\Repository\Repository; use OCA\NgBackup\Tests\Unit\TempDirTrait; @@ -21,13 +22,14 @@ class BackupRunTest extends TestCase { } file_put_contents("$src/big.bin", random_bytes(13 * 1048576)); $backend = new LocalBackend($this->tempDir()); - Repository::init($backend, 'pw'); + $keys = KeyRing::generate(); + Repository::initWithKey($backend, $keys, $keys->wrap('pw')); $state = BackupRun::start($src); $steps = 0; while ($state['phase'] !== 'done' && $steps < 500) { // Fresh repository object per step, state through JSON: as separate processes would. - $repo = Repository::open($backend, 'pw'); + $repo = Repository::openWithKey($backend, $keys); $state = json_decode(json_encode(BackupRun::step($repo, $state, microtime(true) + 0.05)), true); $steps++; } @@ -35,7 +37,7 @@ class BackupRunTest extends TestCase { $this->assertGreaterThan(2, $steps, 'the run was actually split into steps'); $target = $this->tempDir(); - Repository::open($backend, 'pw')->restore($state['snapshot'], $target); + Repository::openWithKey($backend, $keys)->restore($state['snapshot'], $target); $this->assertSame(self::hashTree($src), self::hashTree($target)); } @@ -43,12 +45,13 @@ class BackupRunTest extends TestCase { $src = $this->tempDir(); file_put_contents("$src/big.bin", random_bytes(20 * 1048576)); $backend = new LocalBackend($this->tempDir()); - Repository::init($backend, 'pw'); + $keys = KeyRing::generate(); + Repository::initWithKey($backend, $keys, $keys->wrap('pw')); $state = BackupRun::start($src); $mutated = false; for ($i = 0; $i < 500 && $state['phase'] !== 'done'; $i++) { - $state = BackupRun::step(Repository::open($backend, 'pw'), $state, microtime(true) + 0.01); + $state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 0.01); if (!$mutated && $state['cur'] !== null && $state['cur']['offset'] > 0) { $fh = fopen("$src/big.bin", 'r+b'); fwrite($fh, random_bytes(1024)); // same size, new content at the start @@ -60,7 +63,34 @@ class BackupRunTest extends TestCase { $this->assertTrue($mutated); $this->assertGreaterThanOrEqual(1, $state['stats']['retried'] ?? 0); $target = $this->tempDir(); - Repository::open($backend, 'pw')->restore($state['snapshot'], $target); + Repository::openWithKey($backend, $keys)->restore($state['snapshot'], $target); $this->assertSame(self::hashTree($src), self::hashTree($target)); } + + public function testIncrementalRunInTinyStepsReadsNothing(): void { + $src = $this->tempDir(); + for ($i = 0; $i < 200; $i++) { + @mkdir("$src/d" . ($i % 7), 0700, true); + file_put_contents("$src/d" . ($i % 7) . "/f$i", random_bytes(3000)); + } + $backend = new LocalBackend($this->tempDir()); + $keys = KeyRing::generate(); + Repository::initWithKey($backend, $keys, $keys->wrap('pw')); + $run = function (?string $parent) use ($backend, $src, $keys): array { + $state = BackupRun::start(['data' => $src], $parent); + for ($i = 0; $i < 2000 && $state['phase'] !== 'done'; $i++) { + $state = json_decode(json_encode(BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true))), true); + } + return $state; + }; + $first = $run(null); + $second = $run($first['snapshot']); + $this->assertSame('done', $second['phase']); + $this->assertGreaterThan(150, $second["steps"], "one file per step"); + $this->assertSame(0, $second['stats']['read'], 'merge-join with the parent across steps reuses every file'); + $this->assertSame(200, $second['stats']['reused']); + $target = $this->tempDir(); + Repository::openWithKey($backend, $keys)->restore($second['snapshot'], $target); + $this->assertSame(self::hashTree($src), self::hashTree("$target/data")); + } } diff --git a/tests/Unit/Job/TreeWalkerTest.php b/tests/Unit/Job/TreeWalkerTest.php new file mode 100644 index 0000000..189b187 --- /dev/null +++ b/tests/Unit/Job/TreeWalkerTest.php @@ -0,0 +1,49 @@ +tempDir(); + foreach (['a.txt', 'a/x', 'a/b/y', 'a-b', 'b', 'skip/me', 'Z'] as $f) { + @mkdir(dirname("$d/$f"), 0700, true); + file_put_contents("$d/$f", $f); + } + return $d; + } + + public function testOrderMatchesCompare(): void { + $paths = array_keys(iterator_to_array((new TreeWalker(['' => $this->tree()]))->walk())); + $sorted = $paths; + usort($sorted, [TreeWalker::class, 'compare']); + $this->assertSame($sorted, $paths); + $this->assertSame(['Z', 'a', 'a-b', 'a.txt', 'b', 'skip'], array_values(array_unique(array_map(fn ($p) => explode('/', $p)[0], $paths)))); + } + + public function testResumeAfterCursorYieldsTheRest(): void { + $w = new TreeWalker(['' => $this->tree()]); + $all = array_keys(iterator_to_array($w->walk())); + foreach ($all as $i => $cursor) { + $this->assertSame(array_slice($all, $i + 1), array_keys(iterator_to_array($w->walk($cursor))), "after $cursor"); + } + } + + public function testExcludeAndMultipleRoots(): void { + $d = $this->tree(); + $c = $this->tempDir(); + file_put_contents("$c/config.php", ' $c, 'data' => $d], ['data/skip', 'data/b']))->walk())); + $this->assertContains('config/config.php', $paths); + $this->assertContains('data/a/b/y', $paths); + $this->assertNotContains('data/skip/me', $paths); + $this->assertNotContains('data/b', $paths); + } +}