diff --git a/lib/Backend/BackendException.php b/lib/Backend/BackendException.php new file mode 100644 index 0000000..0255d04 --- /dev/null +++ b/lib/Backend/BackendException.php @@ -0,0 +1,11 @@ + object paths below $prefix (recursive) */ + public function list(string $prefix): array; + + public function delete(string $path): void; +} diff --git a/lib/Backend/LocalBackend.php b/lib/Backend/LocalBackend.php new file mode 100644 index 0000000..17960bb --- /dev/null +++ b/lib/Backend/LocalBackend.php @@ -0,0 +1,102 @@ +root = rtrim($root, '/'); + if (!is_dir($this->root) && !mkdir($this->root, 0700, true) && !is_dir($this->root)) { + throw new BackendException('Cannot create ' . $this->root); + } + } + + public function put(string $path, $stream): void { + $target = $this->abs($path); + $dir = dirname($target); + if (!is_dir($dir) && !mkdir($dir, 0700, true) && !is_dir($dir)) { + throw new BackendException('Cannot create ' . $dir); + } + // Write to a temporary name and rename, so a crash never leaves a partial object. + $tmp = $target . '.part-' . bin2hex(random_bytes(4)); + $out = fopen($tmp, 'wb'); + if ($out === false) { + throw new BackendException('Cannot write ' . $path); + } + try { + if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) { + throw new BackendException('Write failed for ' . $path); + } + } finally { + fclose($out); + } + if (!rename($tmp, $target)) { + @unlink($tmp); + throw new BackendException('Cannot finalise ' . $path); + } + } + + public function get(string $path) { + $fh = @fopen($this->abs($path), 'rb'); + if ($fh === false) { + throw new BackendException('Not found: ' . $path); + } + return $fh; + } + + public function getRange(string $path, int $offset, int $length): string { + $fh = $this->get($path); + try { + if (fseek($fh, $offset) !== 0) { + throw new BackendException('Seek failed in ' . $path); + } + $data = ''; + while (strlen($data) < $length && !feof($fh)) { + $data .= (string)fread($fh, $length - strlen($data)); + } + if (strlen($data) !== $length) { + throw new BackendException('Short read in ' . $path); + } + return $data; + } finally { + fclose($fh); + } + } + + public function exists(string $path): bool { + return is_file($this->abs($path)); + } + + public function list(string $prefix): array { + $base = $this->abs($prefix); + if (!is_dir($base)) { + return []; + } + $result = []; + $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS)); + foreach ($it as $file) { + if ($file->isFile() && !str_contains($file->getFilename(), '.part-')) { + $result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/'); + } + } + sort($result); + return $result; + } + + public function delete(string $path): void { + @unlink($this->abs($path)); + } + + private function abs(string $path): string { + if (str_contains($path, '..') || str_starts_with($path, '/')) { + throw new BackendException('Invalid path: ' . $path); + } + return $this->root . '/' . $path; + } +} diff --git a/lib/Repository/BlobIndex.php b/lib/Repository/BlobIndex.php new file mode 100644 index 0000000..bccacf3 --- /dev/null +++ b/lib/Repository/BlobIndex.php @@ -0,0 +1,74 @@ +), written right after the pack, so an interrupted run leaves a consistent + * repository and the next run can reuse what was already uploaded. + * + * Phase 0 keeps the whole index in memory. For large repositories this becomes a table in the + * Nextcloud database (a cache that can always be rebuilt from index/ and the pack headers). + */ +final class BlobIndex { + /** @var array id => [pack, offset, length, raw, flags] */ + private array $blobs = []; + /** @var array blobs added in this run but whose pack is not uploaded yet */ + private array $pending = []; + + public function __construct( + private IBackend $backend, + private StreamCipher $cipher, + ) { + } + + public function load(): void { + foreach ($this->backend->list('index') as $path) { + $packId = basename($path); + $fh = $this->backend->get($path); + $entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR); + fclose($fh); + foreach ($entries as $e) { + $this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']]; + } + } + } + + public function has(string $id): bool { + return isset($this->blobs[$id]) || isset($this->pending[$id]); + } + + /** @return array{0:string, 1:int, 2:int, 3:int, 4:int} */ + public function get(string $id): array { + return $this->blobs[$id] ?? throw new RepositoryException('Blob not in index: ' . $id); + } + + public function addPending(string $id): void { + $this->pending[$id] = true; + } + + /** @param list $entries */ + public function addPack(string $packId, array $entries): void { + $stream = fopen('php://memory', 'w+b'); + fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId)); + rewind($stream); + $this->backend->put('index/' . $packId, $stream); + fclose($stream); + foreach ($entries as $e) { + $this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']]; + unset($this->pending[$e['id']]); + } + } + + public function count(): int { + return count($this->blobs); + } +} diff --git a/lib/Repository/PackWriter.php b/lib/Repository/PackWriter.php new file mode 100644 index 0000000..08f142e --- /dev/null +++ b/lib/Repository/PackWriter.php @@ -0,0 +1,79 @@ + */ + private array $entries = []; + + public function __construct( + private IBackend $backend, + private StreamCipher $cipher, + private BlobIndex $index, + private int $targetSize = 32 * 1048576, + ) { + $this->reset(); + } + + /** Add a blob that is not yet in the repository. Returns the number of bytes added to the pack. */ + public function add(string $id, string $encrypted, int $rawLength, int $flags): int { + fwrite($this->buffer, $encrypted); + $this->entries[] = ['id' => $id, 'offset' => $this->size, 'length' => strlen($encrypted), 'raw' => $rawLength, 'flags' => $flags]; + $this->size += strlen($encrypted); + // Make the blob known immediately, so a later duplicate in the same run is not stored twice. + $this->index->addPending($id); + if ($this->size >= $this->targetSize) { + $this->flush(); + } + return strlen($encrypted); + } + + /** Upload the current pack (if any) and record it in the index. */ + public function flush(): void { + if ($this->entries === []) { + return; + } + $packId = bin2hex(random_bytes(16)); + $header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId); + fwrite($this->buffer, $header . pack('N', strlen($header))); + rewind($this->buffer); + $path = 'packs/' . substr($packId, 0, 2) . '/' . $packId; + $this->backend->put($path, $this->buffer); + $this->index->addPack($packId, $this->entries); + $this->reset(); + } + + public function pendingBytes(): int { + return $this->size; + } + + private function reset(): void { + if (isset($this->buffer)) { + fclose($this->buffer); + } + $this->buffer = fopen('php://temp/maxmemory:' . ($this->targetSize + 8 * 1048576), 'w+b'); + $this->size = 0; + $this->entries = []; + } +} diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php new file mode 100644 index 0000000..223f7e5 --- /dev/null +++ b/lib/Repository/Repository.php @@ -0,0 +1,237 @@ + encrypted blobs + encrypted pack header + * index/ encrypted list of the blobs in that pack + * snapshots/ encrypted JSON lines: one meta line, then one line per file + * + * Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in + * memory at a time. A blob that already exists (same keyed hash) is not stored again. + */ +final class Repository { + public const FORMAT = 1; + public const BLOB_SIZE = 4 * 1048576; + private const FLAG_DEFLATE = 1; + + private StreamCipher $cipher; + private BlobIndex $index; + + private function __construct( + private IBackend $backend, + private KeyRing $keys, + private string $repositoryId, + ) { + $this->cipher = new StreamCipher($keys); + $this->index = new BlobIndex($backend, $this->cipher); + } + + public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self { + if ($backend->exists('config')) { + throw new RepositoryException('A repository already exists at this location'); + } + $keys = KeyRing::generate(); + $id = bin2hex(random_bytes(16)); + $config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $keys->wrap($passphrase)]; + self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); + return new self($backend, $keys, $id); + } + + public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self { + try { + $fh = $backend->get('config'); + } catch (BackendException) { + throw new RepositoryException('No repository at this location'); + } + $config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR); + fclose($fh); + if (($config['format'] ?? 0) !== self::FORMAT) { + throw new RepositoryException('Unsupported repository format'); + } + $repo = new self($backend, KeyRing::unwrap($config['key'], $passphrase), $config['id']); + $repo->index->load(); + return $repo; + } + + /** + * Back up a local directory tree. Files whose size and mtime match the parent snapshot are not + * read again; their blob list is reused. + * + * @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int} + */ + public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array { + $source = rtrim($source, '/'); + $previous = $parent !== null ? $this->loadEntries($parent) : []; + $packs = new PackWriter($this->backend, $this->cipher, $this->index); + $stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0]; + + $tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); + $snapshotId = bin2hex(random_bytes(16)); + fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n"); + + foreach ($this->walk($source) as $relative => $info) { + $stats['files']++; + $prev = $previous[$relative] ?? null; + if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) { + $blobs = $prev['b']; + $stats['reused']++; + } else { + $blobs = []; + $fh = fopen($info->getPathname(), 'rb'); + while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') { + $blobs[] = $this->storeBlob($data, $packs, $stats); + } + fclose($fh); + $stats['read']++; + } + fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n"); + } + $packs->flush(); + + // The snapshot is written last: it only references blobs whose packs are already stored. + rewind($tree); + $encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); + $this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId); + rewind($encrypted); + $this->backend->put('snapshots/' . $snapshotId, $encrypted); + fclose($tree); + fclose($encrypted); + + return ['snapshot' => $snapshotId] + $stats; + } + + /** Restore a snapshot (or only paths starting with $prefix) into a local directory. */ + public function restore(string $snapshotId, string $target, string $prefix = ''): int { + $count = 0; + foreach ($this->streamEntries($snapshotId) as $entry) { + if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) { + continue; + } + $dest = rtrim($target, '/') . '/' . $entry['p']; + if (!is_dir(dirname($dest))) { + mkdir(dirname($dest), 0700, true); + } + // Write next to the destination and rename after all blobs verified: at most one file + // is ever present twice. + $tmp = $dest . '.ngb-restore'; + $out = fopen($tmp, 'wb'); + foreach ($entry['b'] as $id) { + fwrite($out, $this->loadBlob($id)); + } + fclose($out); + if (filesize($tmp) !== $entry['s']) { + unlink($tmp); + throw new RepositoryException('Size mismatch restoring ' . $entry['p']); + } + rename($tmp, $dest); + touch($dest, $entry['m']); + $count++; + } + return $count; + } + + /** @return list */ + public function listSnapshots(): array { + return array_map('basename', $this->backend->list('snapshots')); + } + + public function blobCount(): int { + return $this->index->count(); + } + + private function storeBlob(string $data, PackWriter $packs, array &$stats): string { + $id = $this->keys->blobId($data); + if ($this->index->has($id)) { + $stats['dupBlobs']++; + return $id; + } + $flags = 0; + $payload = $data; + $deflated = gzdeflate($data, 6); + if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) { + $payload = $deflated; + $flags |= self::FLAG_DEFLATE; + } + $stats['uploaded'] += $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags); + $stats['newBlobs']++; + return $id; + } + + private function loadBlob(string $id): string { + [$pack, $offset, $length, $raw, $flags] = $this->index->get($id); + $encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length); + $payload = $this->cipher->decryptString($encrypted, 'blob:' . $id); + $data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload; + if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) { + throw new RepositoryException('Blob verification failed: ' . $id); + } + return $data; + } + + /** @return array}> */ + private function loadEntries(string $snapshotId): array { + $map = []; + foreach ($this->streamEntries($snapshotId) as $e) { + $map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']]; + } + return $map; + } + + /** @return \Generator}> */ + private function streamEntries(string $snapshotId): \Generator { + $plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); + $fh = $this->backend->get('snapshots/' . $snapshotId); + $this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId); + fclose($fh); + rewind($plain); + fgets($plain); // meta line + while (($line = fgets($plain)) !== false) { + yield json_decode($line, true, 512, JSON_THROW_ON_ERROR); + } + fclose($plain); + } + + /** @return \Generator relative path => file, in a stable order */ + private function walk(string $root): \Generator { + $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS)); + $files = []; + foreach ($it as $file) { + if ($file->isFile()) { + $files[substr($file->getPathname(), strlen($root) + 1)] = $file; + } + } + ksort($files, SORT_STRING); + yield from $files; + } + + private static function putString(IBackend $backend, string $path, string $data): void { + $s = fopen('php://memory', 'w+b'); + fwrite($s, $data); + rewind($s); + $backend->put($path, $s); + fclose($s); + } + + private static function readFull($fh, int $length): string { + $buffer = ''; + while (strlen($buffer) < $length && !feof($fh)) { + $buffer .= (string)fread($fh, $length - strlen($buffer)); + } + return $buffer; + } +} diff --git a/lib/Repository/RepositoryException.php b/lib/Repository/RepositoryException.php new file mode 100644 index 0000000..72bc823 --- /dev/null +++ b/lib/Repository/RepositoryException.php @@ -0,0 +1,11 @@ +isFile()) { + $out[substr($f->getPathname(), strlen($root) + 1)] = hash_file('sha256', $f->getPathname()); + } + } + ksort($out); + return $out; +}; +$du = static function (string $dir): int { + $total = 0; + $it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS)); + foreach ($it as $f) { + $total += $f->isFile() ? $f->getSize() : 0; + } + return $total; +}; +$mb = static fn (int $b): string => sprintf('%.1f MiB', $b / 1048576); + +// Source tree. +for ($i = 0; $i < 200; $i++) { + $write(sprintf('%s/docs/%02d/note-%03d.txt', $src, $i % 10, $i), str_repeat("Regel $i van een tekstbestand met herhaling. ", 1200)); +} +for ($i = 0; $i < 20; $i++) { + $randomFile("$src/photos/img-$i.jpg", 1 + $i % 5); +} +$randomFile("$src/video/big.mp4", 300); +$randomFile("$src/copies/a.bin", 10); +copy("$src/copies/a.bin", "$src/copies/b.bin"); +$sourceBytes = $du($src); +$state1 = $hashTree($src); +printf("source: %d files, %s\n", count($state1), $mb($sourceBytes)); + +$backend = new LocalBackend($repoDir); +$repo = Repository::init($backend, 'spike passphrase'); + +// Run 1: full. +memory_reset_peak_usage(); +$t = microtime(true); +$r1 = $repo->backupDirectory($src, null, 'run 1'); +printf("run 1: %.1fs, peak %s, %s\n", microtime(true) - $t, $mb(memory_get_peak_usage()), json_encode(array_diff_key($r1, ['snapshot' => 1]))); +$repoAfter1 = $du($repoDir); +printf("repository after run 1: %s (%.0f%% of source)\n", $mb($repoAfter1), $repoAfter1 / $sourceBytes * 100); +$check('run 1: identical copies stored once (dedup within run)', $r1['dupBlobs'] >= 3); +$check('run 1: repository smaller than source (compression + dedup)', $repoAfter1 < $sourceBytes); +$check('run 1: peak memory < 64 MiB with a 300 MiB file', memory_get_peak_usage() < 64 * 1048576); + +// Run 2: nothing changed. +$t = microtime(true); +$r2 = $repo->backupDirectory($src, $r1['snapshot'], 'run 2'); +printf("run 2: %.2fs, %s\n", microtime(true) - $t, json_encode(array_diff_key($r2, ['snapshot' => 1]))); +$check('run 2: no file read, nothing uploaded', $r2['read'] === 0 && $r2['uploaded'] === 0); + +// Changes: append to a note, overwrite 1 MiB in the middle of the big file, add, delete. +sleep(1); // mtime resolution +$write("$src/docs/03/note-003.txt", "Toegevoegde regel.\n", true); +$fh = fopen("$src/video/big.mp4", 'r+b'); +fseek($fh, 150 * 1048576); +fwrite($fh, random_bytes(1048576)); +fclose($fh); +$randomFile("$src/photos/new.jpg", 2); +unlink("$src/photos/img-0.jpg"); +$state3 = $hashTree($src); + +$t = microtime(true); +$r3 = $repo->backupDirectory($src, $r2['snapshot'], 'run 3'); +printf("run 3: %.1fs, %s\n", microtime(true) - $t, json_encode(array_diff_key($r3, ['snapshot' => 1]))); +$check('run 3: only changed/new files read (3)', $r3['read'] === 3); +$check('run 3: big file change costs ~1 blob, not 300 MiB', $r3['uploaded'] < 12 * 1048576); + +// Restore with a fresh process view of the repository (reopen with the passphrase). +$repo = Repository::open($backend, 'spike passphrase'); +$check('3 snapshots listed', count($repo->listSnapshots()) === 3); + +memory_reset_peak_usage(); +$repo->restore($r1['snapshot'], "$work/restore1"); +$check('restore run 1: all files identical to state 1', $hashTree("$work/restore1") === $state1); +printf("restore peak memory: %s\n", $mb(memory_get_peak_usage())); +$repo->restore($r3['snapshot'], "$work/restore3"); +$check('restore run 3: all files identical to state 3', $hashTree("$work/restore3") === $state3); +$n = $repo->restore($r3['snapshot'], "$work/restore-docs", 'docs/03/'); +$check('partial restore of one folder (20 files)', $n === 20); + +// Wrong passphrase. +try { + Repository::open($backend, 'wrong'); + $check('wrong passphrase rejected', false); +} catch (CryptoException) { + $check('wrong passphrase rejected', true); +} + +// Tamper with one pack: restore must fail loudly, not produce wrong data. +$packs = $backend->list('packs'); +$victim = "$repoDir/" . $packs[0]; +$bytes = file_get_contents($victim); +$bytes[1000] = chr(ord($bytes[1000]) ^ 0xff); +file_put_contents($victim, $bytes); +try { + $repo->restore($r1['snapshot'], "$work/restore-bad"); + $check('tampered pack detected on restore', false); +} catch (CryptoException|RepositoryException $e) { + $check('tampered pack detected on restore', true); +} + +exec('rm -rf ' . escapeshellarg($work)); +echo $ok ? "ALL PASS\n" : "FAILURES\n"; +exit($ok ? 0 : 1);