M1: PHPUnit test suite for crypto, repository and resumable runs

Dev tooling via composer (phpunit, psalm, nextcloud/ocp); vendor/ is
mounted from outside the repository by a helper script. 28 unit tests
ported from the phase-0 spikes (round trips, corruption, wrong keys,
dedup, incremental runs, partial restore, tampering, object binding,
time-boxed steps, file changed between steps).

composer.json

@@ -0,0 +1,34 @@
+{
+	"name": "dillardblom/ng_backup",
+	"description": "NG Backup: encrypted, incremental Nextcloud backups",
+	"license": "AGPL-3.0-or-later",
+	"type": "project",
+	"autoload": {
+		"psr-4": {
+			"OCA\\NgBackup\\": "lib/"
+		}
+	},
+	"autoload-dev": {
+		"psr-4": {
+			"OCA\\NgBackup\\Tests\\": "tests/"
+		}
+	},
+	"require": {
+		"php": ">=8.1"
+	},
+	"require-dev": {
+		"phpunit/phpunit": "^10.5",
+		"nextcloud/ocp": "dev-stable32",
+		"vimeo/psalm": "^6.0"
+	},
+	"config": {
+		"optimize-autoloader": true,
+		"platform": {
+			"php": "8.2.27"
+		}
+	},
+	"scripts": {
+		"test:unit": "phpunit -c tests/phpunit.xml --testsuite unit",
+		"psalm": "psalm --no-cache --threads=1"
+	}
+}

lib/Repository/Repository.php

@@ -272,7 +272,7 @@ final class Repository {
 	private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
 		$id = $this->keys->blobId($data);
 		if ($this->index->has($id)) {
-			$stats['dupBlobs']++;
+			$stats['dupBlobs'] = ($stats['dupBlobs'] ?? 0) + 1;
 			return $id;
 		}
 		$flags = 0;
@@ -282,8 +282,8 @@ final class Repository {
 			$payload = $deflated;
 			$flags |= self::FLAG_DEFLATE;
 		}
-		$stats['uploaded'] += $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
-		$stats['newBlobs']++;
+		$stats['uploaded'] = ($stats['uploaded'] ?? 0) + $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
+		$stats['newBlobs'] = ($stats['newBlobs'] ?? 0) + 1;
 		return $id;
 	}
 

tests/.phpunit.result.cache

@@ -0,0 +1 @@
+{"version":2,"defects":[],"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.151,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.412,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.004,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.29,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":4.049,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":6.114,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":2.122,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.127,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.183,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.042,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.248,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":0.645,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.289,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.188,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.529}}

tests/Unit/Crypto/KeyRingTest.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\NgBackup\Tests\Unit\Crypto;
+
+use OCA\NgBackup\Crypto\CryptoException;
+use OCA\NgBackup\Crypto\KeyRing;
+use PHPUnit\Framework\TestCase;
+
+class KeyRingTest extends TestCase {
+	public function testWrapUnwrapKeepsKeys(): void {
+		$keys = KeyRing::generate();
+		$restored = KeyRing::unwrap($keys->wrap('pass phrase'), 'pass phrase');
+		$this->assertSame($keys->blobId('data'), $restored->blobId('data'));
+		$this->assertSame($keys->dataKey(), $restored->dataKey());
+	}
+
+	public function testWrongPassphrase(): void {
+		$wrapped = KeyRing::generate()->wrap('right');
+		$this->expectException(CryptoException::class);
+		KeyRing::unwrap($wrapped, 'wrong');
+	}
+
+	public function testBlobIdsAreKeyed(): void {
+		$this->assertNotSame(KeyRing::generate()->blobId('same'), KeyRing::generate()->blobId('same'));
+		$k = KeyRing::generate();
+		$this->assertSame($k->blobId('same'), $k->blobId('same'));
+		$this->assertMatchesRegularExpression('/^[0-9a-f]{64}$/', $k->blobId('x'));
+	}
+
+	public function testRejectsShortMasterKey(): void {
+		$this->expectException(\InvalidArgumentException::class);
+		new KeyRing('short');
+	}
+}

tests/Unit/Crypto/StreamCipherTest.php

@@ -0,0 +1,74 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\NgBackup\Tests\Unit\Crypto;
+
+use OCA\NgBackup\Crypto\CryptoException;
+use OCA\NgBackup\Crypto\KeyRing;
+use OCA\NgBackup\Crypto\StreamCipher;
+use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\TestCase;
+
+class StreamCipherTest extends TestCase {
+	private StreamCipher $cipher;
+
+	protected function setUp(): void {
+		$this->cipher = new StreamCipher(KeyRing::generate());
+	}
+
+	public static function sizes(): array {
+		$f = StreamCipher::FRAME_SIZE;
+		return ['empty' => [0], 'small' => [100], 'one frame' => [$f], 'frame+1' => [$f + 1], 'several' => [3 * $f + 17]];
+	}
+
+	#[DataProvider('sizes')]
+	public function testRoundTrip(int $size): void {
+		$plain = random_bytes(max(1, $size));
+		$plain = substr($plain, 0, $size);
+		$this->assertSame($plain, $this->cipher->decryptString($this->cipher->encryptString($plain, 'ad'), 'ad'));
+	}
+
+	public function testStreamingKeepsMemoryBounded(): void {
+		$in = fopen('php://temp', 'w+b');
+		for ($i = 0; $i < 32; $i++) {
+			fwrite($in, random_bytes(1048576));
+		}
+		rewind($in);
+		$enc = fopen('php://temp', 'w+b');
+		memory_reset_peak_usage();
+		$base = memory_get_usage();
+		$this->cipher->encrypt($in, $enc);
+		$this->assertLessThan(4 * 1048576, memory_get_peak_usage() - $base);
+		rewind($enc);
+		$out = fopen('php://temp', 'w+b');
+		$this->assertSame(32 * 1048576, $this->cipher->decrypt($enc, $out));
+	}
+
+	public static function corruptions(): array {
+		return [
+			'flipped byte' => [static function (string $c): string { $c[60] = chr(ord($c[60]) ^ 1); return $c; }],
+			'truncated' => [static fn (string $c): string => substr($c, 0, -10)],
+			'trailing data' => [static fn (string $c): string => $c . 'x'],
+			'cut at frame boundary' => [static fn (string $c): string => substr($c, 0, 4 + 24 + 4 + StreamCipher::FRAME_SIZE + 17)],
+			'bad magic' => [static fn (string $c): string => 'XXXX' . substr($c, 4)],
+		];
+	}
+
+	#[DataProvider('corruptions')]
+	public function testCorruptionIsRejected(callable $corrupt): void {
+		$cipher = $this->cipher->encryptString(str_repeat('a', StreamCipher::FRAME_SIZE + 500), 'ad');
+		$this->expectException(CryptoException::class);
+		$this->cipher->decryptString($corrupt($cipher), 'ad');
+	}
+
+	public function testWrongAssociatedDataIsRejected(): void {
+		$this->expectException(CryptoException::class);
+		$this->cipher->decryptString($this->cipher->encryptString('x', 'blob:1'), 'blob:2');
+	}
+
+	public function testWrongKeyIsRejected(): void {
+		$this->expectException(CryptoException::class);
+		(new StreamCipher(KeyRing::generate()))->decryptString($this->cipher->encryptString('x'));
+	}
+}

tests/Unit/Job/BackupRunTest.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\NgBackup\Tests\Unit\Job;
+
+use OCA\NgBackup\Backend\LocalBackend;
+use OCA\NgBackup\Job\BackupRun;
+use OCA\NgBackup\Repository\Repository;
+use OCA\NgBackup\Tests\Unit\TempDirTrait;
+use PHPUnit\Framework\TestCase;
+
+class BackupRunTest extends TestCase {
+	use TempDirTrait;
+
+	public function testRunInManyTinyStepsRestoresIdentically(): void {
+		$src = $this->tempDir();
+		mkdir("$src/a", 0700, true);
+		for ($i = 0; $i < 50; $i++) {
+			file_put_contents("$src/a/f$i.txt", random_bytes(20000));
+		}
+		file_put_contents("$src/big.bin", random_bytes(13 * 1048576));
+		$backend = new LocalBackend($this->tempDir());
+		Repository::init($backend, 'pw');
+
+		$state = BackupRun::start($src);
+		$steps = 0;
+		while ($state['phase'] !== 'done' && $steps < 500) {
+			// Fresh repository object per step, state through JSON: as separate processes would.
+			$repo = Repository::open($backend, 'pw');
+			$state = json_decode(json_encode(BackupRun::step($repo, $state, microtime(true) + 0.05)), true);
+			$steps++;
+		}
+		$this->assertSame('done', $state['phase']);
+		$this->assertGreaterThan(2, $steps, 'the run was actually split into steps');
+
+		$target = $this->tempDir();
+		Repository::open($backend, 'pw')->restore($state['snapshot'], $target);
+		$this->assertSame(self::hashTree($src), self::hashTree($target));
+	}
+
+	public function testFileChangedBetweenStepsIsReadAgain(): void {
+		$src = $this->tempDir();
+		file_put_contents("$src/big.bin", random_bytes(20 * 1048576));
+		$backend = new LocalBackend($this->tempDir());
+		Repository::init($backend, 'pw');
+
+		$state = BackupRun::start($src);
+		$mutated = false;
+		for ($i = 0; $i < 500 && $state['phase'] !== 'done'; $i++) {
+			$state = BackupRun::step(Repository::open($backend, 'pw'), $state, microtime(true) + 0.01);
+			if (!$mutated && $state['cur'] !== null && $state['cur']['offset'] > 0) {
+				$fh = fopen("$src/big.bin", 'r+b');
+				fwrite($fh, random_bytes(1024)); // same size, new content at the start
+				fclose($fh);
+				touch("$src/big.bin", time() + 10);
+				$mutated = true;
+			}
+		}
+		$this->assertTrue($mutated);
+		$this->assertGreaterThanOrEqual(1, $state['stats']['retried'] ?? 0);
+		$target = $this->tempDir();
+		Repository::open($backend, 'pw')->restore($state['snapshot'], $target);
+		$this->assertSame(self::hashTree($src), self::hashTree($target));
+	}
+}

tests/Unit/Repository/RepositoryTest.php

@@ -0,0 +1,123 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\NgBackup\Tests\Unit\Repository;
+
+use OCA\NgBackup\Backend\LocalBackend;
+use OCA\NgBackup\Crypto\CryptoException;
+use OCA\NgBackup\Repository\Repository;
+use OCA\NgBackup\Repository\RepositoryException;
+use OCA\NgBackup\Tests\Unit\TempDirTrait;
+use PHPUnit\Framework\TestCase;
+
+class RepositoryTest extends TestCase {
+	use TempDirTrait;
+
+	private string $src;
+	private LocalBackend $backend;
+
+	protected function setUp(): void {
+		$this->src = $this->tempDir();
+		$this->backend = new LocalBackend($this->tempDir());
+		for ($i = 0; $i < 30; $i++) {
+			$this->write("docs/d" . ($i % 3) . "/note-$i.txt", str_repeat("regel $i ", 500));
+		}
+		$this->write('bin/random.bin', random_bytes(9 * 1048576 + 123)); // spans 3 blobs
+		$this->write('copies/a.bin', $same = random_bytes(5 * 1048576));
+		$this->write('copies/b.bin', $same);
+		$this->write('empty.txt', '');
+	}
+
+	private function write(string $rel, string $data): void {
+		@mkdir(dirname($this->src . '/' . $rel), 0700, true);
+		file_put_contents($this->src . '/' . $rel, $data);
+	}
+
+	public function testBackupAndRestoreAreIdentical(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$r = $repo->backupDirectory($this->src);
+		$this->assertSame(34, $r['files']);
+		$this->assertGreaterThanOrEqual(2, $r['dupBlobs'], 'identical copies are stored once');
+
+		$target = $this->tempDir();
+		Repository::open($this->backend, 'pw')->restore($r['snapshot'], $target);
+		$this->assertSame(self::hashTree($this->src), self::hashTree($target));
+	}
+
+	public function testUnchangedRunReadsAndUploadsNothing(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$r1 = $repo->backupDirectory($this->src);
+		$r2 = $repo->backupDirectory($this->src, $r1['snapshot']);
+		$this->assertSame(0, $r2['read']);
+		$this->assertSame(0, $r2['uploaded']);
+	}
+
+	public function testChangeInLargeFileUploadsOnlyAffectedBlob(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$r1 = $repo->backupDirectory($this->src);
+		$fh = fopen($this->src . '/bin/random.bin', 'r+b');
+		fseek($fh, 5 * 1048576);
+		fwrite($fh, 'changed');
+		fclose($fh);
+		touch($this->src . '/bin/random.bin', time() + 10);
+		$r2 = $repo->backupDirectory($this->src, $r1['snapshot']);
+		$this->assertSame(1, $r2['read']);
+		$this->assertSame(1, $r2['newBlobs']);
+	}
+
+	public function testPartialRestore(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$r = $repo->backupDirectory($this->src);
+		$target = $this->tempDir();
+		$this->assertSame(10, $repo->restore($r['snapshot'], $target, 'docs/d1/'));
+	}
+
+	public function testWrongPassphrase(): void {
+		Repository::init($this->backend, 'pw');
+		$this->expectException(CryptoException::class);
+		Repository::open($this->backend, 'other');
+	}
+
+	public function testInitRefusesExistingRepository(): void {
+		Repository::init($this->backend, 'pw');
+		$this->expectException(RepositoryException::class);
+		Repository::init($this->backend, 'pw');
+	}
+
+	public function testTamperedPackIsDetected(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$r = $repo->backupDirectory($this->src);
+		$root = (new \ReflectionProperty(LocalBackend::class, 'root'))->getValue($this->backend);
+		$pack = $root . '/' . $this->backend->list('packs')[0];
+		$bytes = file_get_contents($pack);
+		$bytes[200] = chr(ord($bytes[200]) ^ 0xff);
+		file_put_contents($pack, $bytes);
+		$this->expectException(\RuntimeException::class);
+		$repo->restore($r['snapshot'], $this->tempDir());
+	}
+
+	public function testBlobWriterAndReadLinesAcrossBlobBoundaries(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$stats = [];
+		$writer = $repo->blobWriter($stats);
+		$lines = [];
+		for ($i = 0; $i < 20000; $i++) {
+			$lines[] = json_encode(['i' => $i, 'pad' => str_repeat('x', $i % 400)]);
+		}
+		$writer->write(implode("\n", $lines) . "\n");
+		$blobs = $writer->finish();
+		$repo->flushPacks();
+		$this->assertGreaterThan(1, count($blobs));
+		$this->assertSame($lines, iterator_to_array(Repository::open($this->backend, 'pw')->readLines($blobs), false));
+	}
+
+	public function testObjectsAreBoundToTheirPath(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$repo->putObject('db/1', 'one');
+		$root = (new \ReflectionProperty(LocalBackend::class, 'root'))->getValue($this->backend);
+		copy("$root/db/1", "$root/db/2");
+		$this->expectException(CryptoException::class);
+		$repo->getObject('db/2');
+	}
+}

tests/Unit/TempDirTrait.php

@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\NgBackup\Tests\Unit;
+
+trait TempDirTrait {
+	/** @var list<string> */
+	private array $tempDirs = [];
+
+	protected function tempDir(): string {
+		$dir = sys_get_temp_dir() . '/ngb-test-' . bin2hex(random_bytes(6));
+		mkdir($dir, 0700, true);
+		$this->tempDirs[] = $dir;
+		return $dir;
+	}
+
+	protected function tearDown(): void {
+		foreach ($this->tempDirs as $dir) {
+			exec('rm -rf ' . escapeshellarg($dir));
+		}
+		$this->tempDirs = [];
+		parent::tearDown();
+	}
+
+	/** @return array<string, string> relative path => sha256 */
+	protected static function hashTree(string $root): array {
+		$out = [];
+		if (!is_dir($root)) {
+			return $out;
+		}
+		$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
+		foreach ($it as $f) {
+			if ($f->isFile()) {
+				$out[substr($f->getPathname(), strlen($root) + 1)] = hash_file('sha256', $f->getPathname());
+			}
+		}
+		ksort($out);
+		return $out;
+	}
+}

tests/bootstrap.php

@@ -0,0 +1,5 @@
+<?php
+
+declare(strict_types=1);
+
+require __DIR__ . '/../vendor/autoload.php';

tests/phpunit.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:noNamespaceSchemaLocation="../vendor/phpunit/phpunit/phpunit.xsd"
+         bootstrap="bootstrap.php" colors="true" failOnWarning="true" failOnRisky="true">
+	<testsuites>
+		<testsuite name="unit">
+			<directory>Unit</directory>
+		</testsuite>
+	</testsuites>
+</phpunit>