M2: cursor-based resumable runs with merge-join against the parent
TreeWalker walks one or more roots in a stable segment-wise order with exclusions and resumes after a cursor; ParentCursor merge-joins the previous snapshot in the same order. BackupRun no longer keeps a file list or the parent's entries in memory or in its state.
This commit is contained in:
+82
-98
@@ -10,158 +10,142 @@ namespace OCA\NgBackup\Job;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
|
||||
/**
|
||||
* A backup run as a resumable state machine, for environments with short PHP time limits
|
||||
* (webcron/AJAX cron on hosted Nextcloud). Each step works until its deadline, then uploads
|
||||
* what it has (pack + tree segment) and returns the new state. Even a single large file is
|
||||
* resumed at the blob where the previous step stopped.
|
||||
* A file backup as a resumable state machine, for environments with short PHP time limits.
|
||||
* Each step works until its deadline, uploads what it has (pack + tree segment) and returns
|
||||
* the new state; a large file is resumed at the next blob.
|
||||
*
|
||||
* The state is a plain array (in the app: a database row). It is only saved after a step
|
||||
* completes; if a process is killed mid-step, the next step redoes that work. Blobs uploaded by
|
||||
* the killed step are not referenced and are removed later by pruning; nothing is corrupted,
|
||||
* because the snapshot is written last.
|
||||
* The state holds no file list: the position is a cursor in TreeWalker order, and the previous
|
||||
* snapshot is merge-joined through ParentCursor, so memory and state size stay small however
|
||||
* many files there are. The snapshot is written last; a killed step is simply redone.
|
||||
*/
|
||||
final class BackupRun {
|
||||
public static function start(string $source, ?string $parent = null, string $label = ''): array {
|
||||
/**
|
||||
* @param string|array<string, string> $roots one directory, or name => directory (walked in order)
|
||||
* @param list<string> $exclude logical path prefixes to skip
|
||||
* @param array<string, mixed> $meta extra fields for the snapshot meta record
|
||||
*/
|
||||
public static function start(string|array $roots, ?string $parent = null, string $label = '', array $exclude = [], array $meta = []): array {
|
||||
$roots = is_string($roots) ? ['' => rtrim($roots, '/')] : $roots;
|
||||
uksort($roots, fn ($a, $b) => TreeWalker::compare((string)$a, (string)$b)); // walk order = compare order
|
||||
return [
|
||||
'snapshot' => bin2hex(random_bytes(16)),
|
||||
'source' => rtrim($source, '/'),
|
||||
'roots' => $roots,
|
||||
'exclude' => $exclude,
|
||||
'parent' => $parent,
|
||||
'label' => $label,
|
||||
'meta' => $meta,
|
||||
'started' => gmdate('c'),
|
||||
'phase' => 'scan',
|
||||
'files' => [],
|
||||
'pos' => 0,
|
||||
'phase' => 'files',
|
||||
'after' => null,
|
||||
'parentPos' => 0,
|
||||
'cur' => null,
|
||||
'segments' => [],
|
||||
'steps' => 0,
|
||||
'stats' => ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0],
|
||||
'stats' => ['files' => 0, 'bytes' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0],
|
||||
];
|
||||
}
|
||||
|
||||
/** Run one step until $deadline (microtime(true)); returns the new state. */
|
||||
public static function step(Repository $repo, array $state, float $deadline): array {
|
||||
$state['steps']++;
|
||||
if ($state['phase'] === 'scan') {
|
||||
$state['files'] = self::scan($state['source']);
|
||||
$state['phase'] = 'files';
|
||||
// Scanning can take a while on big trees; continue in the same step if time is left.
|
||||
}
|
||||
|
||||
if ($state['phase'] === 'files') {
|
||||
$previous = $state['parent'] !== null ? $repo->snapshotEntries($state['parent']) : [];
|
||||
$walker = new TreeWalker($state['roots'], $state['exclude']);
|
||||
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
||||
$stats = $state['stats'];
|
||||
$lines = [];
|
||||
$total = count($state['files']);
|
||||
while ($state['pos'] < $total) {
|
||||
$relative = $state['files'][$state['pos']];
|
||||
$path = $state['source'] . '/' . $relative;
|
||||
clearstatcache(true, $path);
|
||||
$size = @filesize($path);
|
||||
$mtime = @filemtime($path);
|
||||
if ($size === false) {
|
||||
// Deleted since the scan: skip.
|
||||
$state['pos']++;
|
||||
$state['cur'] = null;
|
||||
continue;
|
||||
$finishedWalk = true;
|
||||
|
||||
foreach ($walker->walk($state['after']) as $logical => $path) {
|
||||
$version = Repository::fileVersion($path);
|
||||
if ($version === 'missing') {
|
||||
continue; // deleted while walking
|
||||
}
|
||||
$prev = $previous[$relative] ?? null;
|
||||
[$size, $mtime] = array_map('intval', explode(':', $version, 3));
|
||||
$prev = $parent->find($logical);
|
||||
|
||||
if ($state['cur'] === null && $prev !== null && $prev['s'] === $size && $prev['m'] === $mtime) {
|
||||
$lines[] = self::line($relative, $size, $mtime, $prev['b']);
|
||||
$stats['files']++;
|
||||
$blobs = $prev['b'];
|
||||
$stats['reused']++;
|
||||
$state['pos']++;
|
||||
continue;
|
||||
} else {
|
||||
$cur = $state['cur'];
|
||||
if ($cur === null || $cur['path'] !== $logical || $cur['version'] !== $version) {
|
||||
if ($cur !== null && $cur['path'] === $logical) {
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1; // changed since the previous step
|
||||
}
|
||||
$cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
||||
}
|
||||
$fh = fopen($path, 'rb');
|
||||
if ($fh === false) {
|
||||
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
||||
$state['cur'] = null;
|
||||
continue;
|
||||
}
|
||||
fseek($fh, $cur['offset']);
|
||||
$complete = true;
|
||||
while (($data = self::readFull($fh, Repository::BLOB_SIZE)) !== '') {
|
||||
$cur['blobs'][] = $repo->storeData($data, $stats);
|
||||
$cur['offset'] += strlen($data);
|
||||
if (microtime(true) >= $deadline && !feof($fh)) {
|
||||
$complete = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
fclose($fh);
|
||||
if (Repository::fileVersion($path) !== $cur['version']) {
|
||||
// Changed while reading: read it again from the start (now or next step).
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||
$state['cur'] = null;
|
||||
$finishedWalk = false;
|
||||
break;
|
||||
}
|
||||
if (!$complete) {
|
||||
$state['cur'] = $cur;
|
||||
$finishedWalk = false;
|
||||
break;
|
||||
}
|
||||
$blobs = $cur['blobs'];
|
||||
$state['cur'] = null;
|
||||
$stats['read']++;
|
||||
}
|
||||
|
||||
$version = Repository::fileVersion($path);
|
||||
$cur = $state['cur'];
|
||||
if ($cur === null || $cur['version'] !== $version) {
|
||||
// New file, or it changed since the previous step: (re)start reading it.
|
||||
if ($cur !== null) {
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||
}
|
||||
$cur = ['path' => $relative, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
||||
}
|
||||
$fh = fopen($path, 'rb');
|
||||
fseek($fh, $cur['offset']);
|
||||
$finished = true;
|
||||
while (($data = self::readFull($fh, Repository::BLOB_SIZE)) !== '') {
|
||||
$cur['blobs'][] = $repo->storeData($data, $stats);
|
||||
$cur['offset'] += strlen($data);
|
||||
if (microtime(true) >= $deadline && !feof($fh)) {
|
||||
$finished = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
fclose($fh);
|
||||
if (Repository::fileVersion($path) !== $cur['version']) {
|
||||
// Changed while reading: start this file again (in this or the next step).
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||
$state['cur'] = null;
|
||||
if (microtime(true) >= $deadline) {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!$finished) {
|
||||
$state['cur'] = $cur;
|
||||
break;
|
||||
}
|
||||
$lines[] = self::line($relative, $size, $mtime, $cur['blobs']);
|
||||
$lines[] = json_encode(['p' => $logical, 's' => $size, 'm' => $mtime, 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||
$stats['files']++;
|
||||
$stats['read']++;
|
||||
$state['cur'] = null;
|
||||
$state['pos']++;
|
||||
$stats['bytes'] += $size;
|
||||
$state['after'] = $logical;
|
||||
if (microtime(true) >= $deadline) {
|
||||
$finishedWalk = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Persist this step's tree lines as a blob stream segment and upload the open pack.
|
||||
if ($lines !== []) {
|
||||
$writer = $repo->blobWriter($stats);
|
||||
$writer->write(implode("\n", $lines) . "\n");
|
||||
$state['segments'][] = $writer->finish();
|
||||
}
|
||||
$repo->flushPacks();
|
||||
$state['parentPos'] = $parent->position();
|
||||
$state['stats'] = $stats;
|
||||
if ($state['pos'] >= $total) {
|
||||
if ($finishedWalk) {
|
||||
$state['phase'] = 'finish';
|
||||
}
|
||||
}
|
||||
|
||||
if ($state['phase'] === 'finish' && microtime(true) < $deadline + 5) {
|
||||
if ($state['phase'] === 'finish') {
|
||||
$segments = $state['segments'];
|
||||
$repo->writeSnapshot($state['snapshot'],
|
||||
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'], 'source' => $state['source']],
|
||||
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
|
||||
'roots' => array_keys($state['roots']), 'stats' => $state['stats']] + $state['meta'],
|
||||
(function () use ($repo, $segments) {
|
||||
foreach ($segments as $blobs) {
|
||||
yield from $repo->readLines($blobs);
|
||||
}
|
||||
})());
|
||||
$state['phase'] = 'done';
|
||||
$state['files'] = [];
|
||||
}
|
||||
return $state;
|
||||
}
|
||||
|
||||
/** @return list<string> */
|
||||
private static function scan(string $root): array {
|
||||
$files = [];
|
||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
|
||||
foreach ($it as $file) {
|
||||
if ($file->isFile()) {
|
||||
$files[] = substr($file->getPathname(), strlen($root) + 1);
|
||||
}
|
||||
}
|
||||
sort($files, SORT_STRING);
|
||||
return $files;
|
||||
}
|
||||
|
||||
private static function line(string $path, int $size, int $mtime, array $blobs): string {
|
||||
return json_encode(['p' => $path, 's' => $size, 'm' => $mtime, 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||
}
|
||||
|
||||
private static function readFull($fh, int $length): string {
|
||||
$buffer = '';
|
||||
while (strlen($buffer) < $length && !feof($fh)) {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Job;
|
||||
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
|
||||
/**
|
||||
* Reads the previous snapshot's entries in walk order and answers "what did the parent have
|
||||
* at this path?" for increasing paths (merge-join). Only the current entry is in memory; the
|
||||
* number of entries consumed is part of the resumable state.
|
||||
*/
|
||||
final class ParentCursor {
|
||||
private ?\Generator $entries = null;
|
||||
private ?array $current = null;
|
||||
private int $consumed = 0;
|
||||
|
||||
public function __construct(Repository $repo, ?string $snapshotId, int $skip = 0) {
|
||||
if ($snapshotId === null) {
|
||||
return;
|
||||
}
|
||||
$this->entries = $repo->entries($snapshotId);
|
||||
$this->current = $this->entries->valid() ? $this->entries->current() : null;
|
||||
while ($this->consumed < $skip && $this->current !== null) {
|
||||
$this->advance();
|
||||
}
|
||||
}
|
||||
|
||||
/** @return array{p:string, s:int, m:int, b:list<string>}|null */
|
||||
public function find(string $path): ?array {
|
||||
while ($this->current !== null) {
|
||||
$c = TreeWalker::compare($this->current['p'], $path);
|
||||
if ($c === 0) {
|
||||
return $this->current;
|
||||
}
|
||||
if ($c > 0) {
|
||||
return null;
|
||||
}
|
||||
$this->advance();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public function position(): int {
|
||||
return $this->consumed;
|
||||
}
|
||||
|
||||
private function advance(): void {
|
||||
$this->entries->next();
|
||||
$this->consumed++;
|
||||
$this->current = $this->entries->valid() ? $this->entries->current() : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Job;
|
||||
|
||||
/**
|
||||
* Walks one or more source roots in a stable depth-first order (entries of a directory sorted
|
||||
* by name, children right after their directory), yielding "<root>/<relative path>" for files.
|
||||
*
|
||||
* The same order is used for comparing paths (compare()), so a walk can resume after a cursor
|
||||
* and be merge-joined with the previous snapshot without holding any list in memory.
|
||||
*/
|
||||
final class TreeWalker {
|
||||
/**
|
||||
* @param array<string, string> $roots name => absolute directory, walked in this order
|
||||
* @param list<string> $exclude path prefixes ("<root>/<relative>") that are skipped entirely
|
||||
*/
|
||||
public function __construct(
|
||||
private array $roots,
|
||||
private array $exclude = [],
|
||||
) {
|
||||
}
|
||||
|
||||
/** @return \Generator<string, string> logical path => absolute path, strictly after $after */
|
||||
public function walk(?string $after = null): \Generator {
|
||||
foreach ($this->roots as $name => $dir) {
|
||||
$name = (string)$name;
|
||||
if ($name !== '' && $after !== null && self::compare($name, self::firstSegment($after)) < 0) {
|
||||
continue; // whole root lies before the cursor
|
||||
}
|
||||
yield from $this->walkDir(rtrim($dir, '/'), $name, $after);
|
||||
}
|
||||
}
|
||||
|
||||
/** Order used everywhere: segment by segment, byte-wise. */
|
||||
public static function compare(string $a, string $b): int {
|
||||
$sa = explode('/', $a);
|
||||
$sb = explode('/', $b);
|
||||
$n = min(count($sa), count($sb));
|
||||
for ($i = 0; $i < $n; $i++) {
|
||||
$c = strcmp($sa[$i], $sb[$i]);
|
||||
if ($c !== 0) {
|
||||
return $c < 0 ? -1 : 1;
|
||||
}
|
||||
}
|
||||
return count($sa) <=> count($sb);
|
||||
}
|
||||
|
||||
private function walkDir(string $abs, string $logical, ?string $after): \Generator {
|
||||
foreach ($this->exclude as $prefix) {
|
||||
if ($logical !== '' && ($logical === $prefix || str_starts_with($logical, $prefix . '/'))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
$names = @scandir($abs);
|
||||
if ($names === false) {
|
||||
return;
|
||||
}
|
||||
$names = array_values(array_diff($names, ['.', '..']));
|
||||
sort($names, SORT_STRING);
|
||||
foreach ($names as $name) {
|
||||
$childLogical = $logical === '' ? $name : $logical . '/' . $name;
|
||||
$childAbs = $abs . '/' . $name;
|
||||
if (is_link($childAbs)) {
|
||||
continue; // never follow symlinks out of the data directory
|
||||
}
|
||||
if (is_dir($childAbs)) {
|
||||
// Skip a directory entirely if everything in it sorts before the cursor.
|
||||
if ($after !== null && self::compare($childLogical, $after) < 0 && !self::isPrefix($childLogical, $after)) {
|
||||
continue;
|
||||
}
|
||||
yield from $this->walkDir($childAbs, $childLogical, $after);
|
||||
} elseif (is_file($childAbs)) {
|
||||
if ($after !== null && self::compare($childLogical, $after) <= 0) {
|
||||
continue;
|
||||
}
|
||||
$excluded = false;
|
||||
foreach ($this->exclude as $prefix) {
|
||||
if ($childLogical === $prefix) {
|
||||
$excluded = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!$excluded) {
|
||||
yield $childLogical => $childAbs;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static function isPrefix(string $dir, string $path): bool {
|
||||
return str_starts_with($path, $dir . '/');
|
||||
}
|
||||
|
||||
private static function firstSegment(string $path): string {
|
||||
return explode('/', $path, 2)[0];
|
||||
}
|
||||
}
|
||||
@@ -342,6 +342,23 @@ final class Repository {
|
||||
return $map;
|
||||
}
|
||||
|
||||
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> entries of a snapshot, in stored order */
|
||||
public function entries(string $snapshotId): \Generator {
|
||||
return $this->streamEntries($snapshotId);
|
||||
}
|
||||
|
||||
/** The meta record (first line) of a snapshot. */
|
||||
public function snapshotMeta(string $snapshotId): array {
|
||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
||||
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
||||
fclose($fh);
|
||||
rewind($plain);
|
||||
$meta = json_decode((string)fgets($plain), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($plain);
|
||||
return $meta;
|
||||
}
|
||||
|
||||
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
||||
private function streamEntries(string $snapshotId): \Generator {
|
||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"version":2,"defects":[],"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.115,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.074,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.003,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.262,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":4.044,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":6.03,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.997,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.012,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.223,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.307,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.14,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.281,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.327,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.618,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.669}}
|
||||
{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.075,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.094,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.004,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.276,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.498,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.8,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":2.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.016,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.136,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.049,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.087,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.064,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.095,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.24,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.802,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.006,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005}}
|
||||
@@ -5,6 +5,7 @@ declare(strict_types=1);
|
||||
namespace OCA\NgBackup\Tests\Unit\Job;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Job\BackupRun;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
@@ -21,13 +22,14 @@ class BackupRunTest extends TestCase {
|
||||
}
|
||||
file_put_contents("$src/big.bin", random_bytes(13 * 1048576));
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
Repository::init($backend, 'pw');
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
|
||||
$state = BackupRun::start($src);
|
||||
$steps = 0;
|
||||
while ($state['phase'] !== 'done' && $steps < 500) {
|
||||
// Fresh repository object per step, state through JSON: as separate processes would.
|
||||
$repo = Repository::open($backend, 'pw');
|
||||
$repo = Repository::openWithKey($backend, $keys);
|
||||
$state = json_decode(json_encode(BackupRun::step($repo, $state, microtime(true) + 0.05)), true);
|
||||
$steps++;
|
||||
}
|
||||
@@ -35,7 +37,7 @@ class BackupRunTest extends TestCase {
|
||||
$this->assertGreaterThan(2, $steps, 'the run was actually split into steps');
|
||||
|
||||
$target = $this->tempDir();
|
||||
Repository::open($backend, 'pw')->restore($state['snapshot'], $target);
|
||||
Repository::openWithKey($backend, $keys)->restore($state['snapshot'], $target);
|
||||
$this->assertSame(self::hashTree($src), self::hashTree($target));
|
||||
}
|
||||
|
||||
@@ -43,12 +45,13 @@ class BackupRunTest extends TestCase {
|
||||
$src = $this->tempDir();
|
||||
file_put_contents("$src/big.bin", random_bytes(20 * 1048576));
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
Repository::init($backend, 'pw');
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
|
||||
$state = BackupRun::start($src);
|
||||
$mutated = false;
|
||||
for ($i = 0; $i < 500 && $state['phase'] !== 'done'; $i++) {
|
||||
$state = BackupRun::step(Repository::open($backend, 'pw'), $state, microtime(true) + 0.01);
|
||||
$state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 0.01);
|
||||
if (!$mutated && $state['cur'] !== null && $state['cur']['offset'] > 0) {
|
||||
$fh = fopen("$src/big.bin", 'r+b');
|
||||
fwrite($fh, random_bytes(1024)); // same size, new content at the start
|
||||
@@ -60,7 +63,34 @@ class BackupRunTest extends TestCase {
|
||||
$this->assertTrue($mutated);
|
||||
$this->assertGreaterThanOrEqual(1, $state['stats']['retried'] ?? 0);
|
||||
$target = $this->tempDir();
|
||||
Repository::open($backend, 'pw')->restore($state['snapshot'], $target);
|
||||
Repository::openWithKey($backend, $keys)->restore($state['snapshot'], $target);
|
||||
$this->assertSame(self::hashTree($src), self::hashTree($target));
|
||||
}
|
||||
|
||||
public function testIncrementalRunInTinyStepsReadsNothing(): void {
|
||||
$src = $this->tempDir();
|
||||
for ($i = 0; $i < 200; $i++) {
|
||||
@mkdir("$src/d" . ($i % 7), 0700, true);
|
||||
file_put_contents("$src/d" . ($i % 7) . "/f$i", random_bytes(3000));
|
||||
}
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
$run = function (?string $parent) use ($backend, $src, $keys): array {
|
||||
$state = BackupRun::start(['data' => $src], $parent);
|
||||
for ($i = 0; $i < 2000 && $state['phase'] !== 'done'; $i++) {
|
||||
$state = json_decode(json_encode(BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true))), true);
|
||||
}
|
||||
return $state;
|
||||
};
|
||||
$first = $run(null);
|
||||
$second = $run($first['snapshot']);
|
||||
$this->assertSame('done', $second['phase']);
|
||||
$this->assertGreaterThan(150, $second["steps"], "one file per step");
|
||||
$this->assertSame(0, $second['stats']['read'], 'merge-join with the parent across steps reuses every file');
|
||||
$this->assertSame(200, $second['stats']['reused']);
|
||||
$target = $this->tempDir();
|
||||
Repository::openWithKey($backend, $keys)->restore($second['snapshot'], $target);
|
||||
$this->assertSame(self::hashTree($src), self::hashTree("$target/data"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Job;
|
||||
|
||||
use OCA\NgBackup\Job\TreeWalker;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class TreeWalkerTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private function tree(): string {
|
||||
$d = $this->tempDir();
|
||||
foreach (['a.txt', 'a/x', 'a/b/y', 'a-b', 'b', 'skip/me', 'Z'] as $f) {
|
||||
@mkdir(dirname("$d/$f"), 0700, true);
|
||||
file_put_contents("$d/$f", $f);
|
||||
}
|
||||
return $d;
|
||||
}
|
||||
|
||||
public function testOrderMatchesCompare(): void {
|
||||
$paths = array_keys(iterator_to_array((new TreeWalker(['' => $this->tree()]))->walk()));
|
||||
$sorted = $paths;
|
||||
usort($sorted, [TreeWalker::class, 'compare']);
|
||||
$this->assertSame($sorted, $paths);
|
||||
$this->assertSame(['Z', 'a', 'a-b', 'a.txt', 'b', 'skip'], array_values(array_unique(array_map(fn ($p) => explode('/', $p)[0], $paths))));
|
||||
}
|
||||
|
||||
public function testResumeAfterCursorYieldsTheRest(): void {
|
||||
$w = new TreeWalker(['' => $this->tree()]);
|
||||
$all = array_keys(iterator_to_array($w->walk()));
|
||||
foreach ($all as $i => $cursor) {
|
||||
$this->assertSame(array_slice($all, $i + 1), array_keys(iterator_to_array($w->walk($cursor))), "after $cursor");
|
||||
}
|
||||
}
|
||||
|
||||
public function testExcludeAndMultipleRoots(): void {
|
||||
$d = $this->tree();
|
||||
$c = $this->tempDir();
|
||||
file_put_contents("$c/config.php", '<?php');
|
||||
$paths = array_keys(iterator_to_array((new TreeWalker(['config' => $c, 'data' => $d], ['data/skip', 'data/b']))->walk()));
|
||||
$this->assertContains('config/config.php', $paths);
|
||||
$this->assertContains('data/a/b/y', $paths);
|
||||
$this->assertNotContains('data/skip/me', $paths);
|
||||
$this->assertNotContains('data/b', $paths);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user