Phase 0: repository format with packs, dedup and incremental runs (spike B)

Content-addressed repository on an IBackend (local implementation):
4 MiB blobs, deflate when it helps, each blob encrypted with its id as
associated data, ~32 MiB packs with an encrypted header, one index file
per pack, snapshots as encrypted JSON lines written last.

Spike B: 390 MiB tree, peak 54 MiB; unchanged run reads nothing; a 1 MiB
change in a 300 MiB file uploads one blob; full and partial restore are
bit-identical; tampering is detected.
This commit is contained in:
2026-10-01 18:18:38 +02:00
parent fee189aec3
commit 4fce893389
8 changed files with 690 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Backend;
class BackendException extends \RuntimeException {
}
+30
View File
@@ -0,0 +1,30 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Backend;
/**
* Minimal object store used by the repository. Paths are relative, '/'-separated.
* Implementations: a local directory (spike, tests), and adapters over files_external storages.
*/
interface IBackend {
/** Store an object; the stream is read to EOF. Must not leave a partial object behind on failure. */
public function put(string $path, $stream): void;
/** @return resource readable stream of the whole object */
public function get(string $path);
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
public function getRange(string $path, int $offset, int $length): string;
public function exists(string $path): bool;
/** @return list<string> object paths below $prefix (recursive) */
public function list(string $prefix): array;
public function delete(string $path): void;
}
+102
View File
@@ -0,0 +1,102 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Backend;
final class LocalBackend implements IBackend {
public function __construct(
private string $root,
) {
$this->root = rtrim($root, '/');
if (!is_dir($this->root) && !mkdir($this->root, 0700, true) && !is_dir($this->root)) {
throw new BackendException('Cannot create ' . $this->root);
}
}
public function put(string $path, $stream): void {
$target = $this->abs($path);
$dir = dirname($target);
if (!is_dir($dir) && !mkdir($dir, 0700, true) && !is_dir($dir)) {
throw new BackendException('Cannot create ' . $dir);
}
// Write to a temporary name and rename, so a crash never leaves a partial object.
$tmp = $target . '.part-' . bin2hex(random_bytes(4));
$out = fopen($tmp, 'wb');
if ($out === false) {
throw new BackendException('Cannot write ' . $path);
}
try {
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
throw new BackendException('Write failed for ' . $path);
}
} finally {
fclose($out);
}
if (!rename($tmp, $target)) {
@unlink($tmp);
throw new BackendException('Cannot finalise ' . $path);
}
}
public function get(string $path) {
$fh = @fopen($this->abs($path), 'rb');
if ($fh === false) {
throw new BackendException('Not found: ' . $path);
}
return $fh;
}
public function getRange(string $path, int $offset, int $length): string {
$fh = $this->get($path);
try {
if (fseek($fh, $offset) !== 0) {
throw new BackendException('Seek failed in ' . $path);
}
$data = '';
while (strlen($data) < $length && !feof($fh)) {
$data .= (string)fread($fh, $length - strlen($data));
}
if (strlen($data) !== $length) {
throw new BackendException('Short read in ' . $path);
}
return $data;
} finally {
fclose($fh);
}
}
public function exists(string $path): bool {
return is_file($this->abs($path));
}
public function list(string $prefix): array {
$base = $this->abs($prefix);
if (!is_dir($base)) {
return [];
}
$result = [];
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
foreach ($it as $file) {
if ($file->isFile() && !str_contains($file->getFilename(), '.part-')) {
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
}
}
sort($result);
return $result;
}
public function delete(string $path): void {
@unlink($this->abs($path));
}
private function abs(string $path): string {
if (str_contains($path, '..') || str_starts_with($path, '/')) {
throw new BackendException('Invalid path: ' . $path);
}
return $this->root . '/' . $path;
}
}
+74
View File
@@ -0,0 +1,74 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Repository;
use OCA\NgBackup\Backend\IBackend;
use OCA\NgBackup\Crypto\StreamCipher;
/**
* Which blob lives in which pack. Every uploaded pack gets its own small encrypted index file
* (index/<packId>), written right after the pack, so an interrupted run leaves a consistent
* repository and the next run can reuse what was already uploaded.
*
* Phase 0 keeps the whole index in memory. For large repositories this becomes a table in the
* Nextcloud database (a cache that can always be rebuilt from index/ and the pack headers).
*/
final class BlobIndex {
/** @var array<string, array{0:string, 1:int, 2:int, 3:int, 4:int}> id => [pack, offset, length, raw, flags] */
private array $blobs = [];
/** @var array<string, true> blobs added in this run but whose pack is not uploaded yet */
private array $pending = [];
public function __construct(
private IBackend $backend,
private StreamCipher $cipher,
) {
}
public function load(): void {
foreach ($this->backend->list('index') as $path) {
$packId = basename($path);
$fh = $this->backend->get($path);
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
fclose($fh);
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
}
}
}
public function has(string $id): bool {
return isset($this->blobs[$id]) || isset($this->pending[$id]);
}
/** @return array{0:string, 1:int, 2:int, 3:int, 4:int} */
public function get(string $id): array {
return $this->blobs[$id] ?? throw new RepositoryException('Blob not in index: ' . $id);
}
public function addPending(string $id): void {
$this->pending[$id] = true;
}
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
public function addPack(string $packId, array $entries): void {
$stream = fopen('php://memory', 'w+b');
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
rewind($stream);
$this->backend->put('index/' . $packId, $stream);
fclose($stream);
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
unset($this->pending[$e['id']]);
}
}
public function count(): int {
return count($this->blobs);
}
}
+79
View File
@@ -0,0 +1,79 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Repository;
use OCA\NgBackup\Backend\IBackend;
use OCA\NgBackup\Crypto\StreamCipher;
/**
* Collects encrypted blobs into a pack in memory and uploads it when full.
*
* Pack layout: blob* | encrypted header | uint32 big-endian header length
* The header lists (id, offset, length, rawLength, flags) of every blob, so a pack can be
* re-indexed from the pack alone. Each blob is encrypted separately with its id as associated
* data, so single blobs can be fetched with a range read and cannot be swapped.
*
* Only one pack is held at a time (php://temp keeps it in memory up to the limit).
*/
final class PackWriter {
/** @var resource */
private $buffer;
private int $size = 0;
/** @var list<array{id:string, offset:int, length:int, raw:int, flags:int}> */
private array $entries = [];
public function __construct(
private IBackend $backend,
private StreamCipher $cipher,
private BlobIndex $index,
private int $targetSize = 32 * 1048576,
) {
$this->reset();
}
/** Add a blob that is not yet in the repository. Returns the number of bytes added to the pack. */
public function add(string $id, string $encrypted, int $rawLength, int $flags): int {
fwrite($this->buffer, $encrypted);
$this->entries[] = ['id' => $id, 'offset' => $this->size, 'length' => strlen($encrypted), 'raw' => $rawLength, 'flags' => $flags];
$this->size += strlen($encrypted);
// Make the blob known immediately, so a later duplicate in the same run is not stored twice.
$this->index->addPending($id);
if ($this->size >= $this->targetSize) {
$this->flush();
}
return strlen($encrypted);
}
/** Upload the current pack (if any) and record it in the index. */
public function flush(): void {
if ($this->entries === []) {
return;
}
$packId = bin2hex(random_bytes(16));
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
fwrite($this->buffer, $header . pack('N', strlen($header)));
rewind($this->buffer);
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
$this->backend->put($path, $this->buffer);
$this->index->addPack($packId, $this->entries);
$this->reset();
}
public function pendingBytes(): int {
return $this->size;
}
private function reset(): void {
if (isset($this->buffer)) {
fclose($this->buffer);
}
$this->buffer = fopen('php://temp/maxmemory:' . ($this->targetSize + 8 * 1048576), 'w+b');
$this->size = 0;
$this->entries = [];
}
}
+237
View File
@@ -0,0 +1,237 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Repository;
use OCA\NgBackup\Backend\BackendException;
use OCA\NgBackup\Backend\IBackend;
use OCA\NgBackup\Crypto\KeyRing;
use OCA\NgBackup\Crypto\StreamCipher;
/**
* Content-addressed, encrypted, deduplicating repository.
*
* Layout on the backend:
* config plaintext JSON: format version, repository id, wrapped master key
* packs/xx/<packId> encrypted blobs + encrypted pack header
* index/<packId> encrypted list of the blobs in that pack
* snapshots/<snapId> encrypted JSON lines: one meta line, then one line per file
*
* Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in
* memory at a time. A blob that already exists (same keyed hash) is not stored again.
*/
final class Repository {
public const FORMAT = 1;
public const BLOB_SIZE = 4 * 1048576;
private const FLAG_DEFLATE = 1;
private StreamCipher $cipher;
private BlobIndex $index;
private function __construct(
private IBackend $backend,
private KeyRing $keys,
private string $repositoryId,
) {
$this->cipher = new StreamCipher($keys);
$this->index = new BlobIndex($backend, $this->cipher);
}
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
if ($backend->exists('config')) {
throw new RepositoryException('A repository already exists at this location');
}
$keys = KeyRing::generate();
$id = bin2hex(random_bytes(16));
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $keys->wrap($passphrase)];
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
return new self($backend, $keys, $id);
}
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
try {
$fh = $backend->get('config');
} catch (BackendException) {
throw new RepositoryException('No repository at this location');
}
$config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR);
fclose($fh);
if (($config['format'] ?? 0) !== self::FORMAT) {
throw new RepositoryException('Unsupported repository format');
}
$repo = new self($backend, KeyRing::unwrap($config['key'], $passphrase), $config['id']);
$repo->index->load();
return $repo;
}
/**
* Back up a local directory tree. Files whose size and mtime match the parent snapshot are not
* read again; their blob list is reused.
*
* @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}
*/
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
$source = rtrim($source, '/');
$previous = $parent !== null ? $this->loadEntries($parent) : [];
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
$snapshotId = bin2hex(random_bytes(16));
fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n");
foreach ($this->walk($source) as $relative => $info) {
$stats['files']++;
$prev = $previous[$relative] ?? null;
if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) {
$blobs = $prev['b'];
$stats['reused']++;
} else {
$blobs = [];
$fh = fopen($info->getPathname(), 'rb');
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
$blobs[] = $this->storeBlob($data, $packs, $stats);
}
fclose($fh);
$stats['read']++;
}
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
}
$packs->flush();
// The snapshot is written last: it only references blobs whose packs are already stored.
rewind($tree);
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
rewind($encrypted);
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
fclose($tree);
fclose($encrypted);
return ['snapshot' => $snapshotId] + $stats;
}
/** Restore a snapshot (or only paths starting with $prefix) into a local directory. */
public function restore(string $snapshotId, string $target, string $prefix = ''): int {
$count = 0;
foreach ($this->streamEntries($snapshotId) as $entry) {
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
continue;
}
$dest = rtrim($target, '/') . '/' . $entry['p'];
if (!is_dir(dirname($dest))) {
mkdir(dirname($dest), 0700, true);
}
// Write next to the destination and rename after all blobs verified: at most one file
// is ever present twice.
$tmp = $dest . '.ngb-restore';
$out = fopen($tmp, 'wb');
foreach ($entry['b'] as $id) {
fwrite($out, $this->loadBlob($id));
}
fclose($out);
if (filesize($tmp) !== $entry['s']) {
unlink($tmp);
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
}
rename($tmp, $dest);
touch($dest, $entry['m']);
$count++;
}
return $count;
}
/** @return list<string> */
public function listSnapshots(): array {
return array_map('basename', $this->backend->list('snapshots'));
}
public function blobCount(): int {
return $this->index->count();
}
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
$id = $this->keys->blobId($data);
if ($this->index->has($id)) {
$stats['dupBlobs']++;
return $id;
}
$flags = 0;
$payload = $data;
$deflated = gzdeflate($data, 6);
if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
$payload = $deflated;
$flags |= self::FLAG_DEFLATE;
}
$stats['uploaded'] += $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
$stats['newBlobs']++;
return $id;
}
private function loadBlob(string $id): string {
[$pack, $offset, $length, $raw, $flags] = $this->index->get($id);
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
throw new RepositoryException('Blob verification failed: ' . $id);
}
return $data;
}
/** @return array<string, array{s:int, m:int, b:list<string>}> */
private function loadEntries(string $snapshotId): array {
$map = [];
foreach ($this->streamEntries($snapshotId) as $e) {
$map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']];
}
return $map;
}
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
private function streamEntries(string $snapshotId): \Generator {
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
$fh = $this->backend->get('snapshots/' . $snapshotId);
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
fclose($fh);
rewind($plain);
fgets($plain); // meta line
while (($line = fgets($plain)) !== false) {
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
}
fclose($plain);
}
/** @return \Generator<string, \SplFileInfo> relative path => file, in a stable order */
private function walk(string $root): \Generator {
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
$files = [];
foreach ($it as $file) {
if ($file->isFile()) {
$files[substr($file->getPathname(), strlen($root) + 1)] = $file;
}
}
ksort($files, SORT_STRING);
yield from $files;
}
private static function putString(IBackend $backend, string $path, string $data): void {
$s = fopen('php://memory', 'w+b');
fwrite($s, $data);
rewind($s);
$backend->put($path, $s);
fclose($s);
}
private static function readFull($fh, int $length): string {
$buffer = '';
while (strlen($buffer) < $length && !feof($fh)) {
$buffer .= (string)fread($fh, $length - strlen($buffer));
}
return $buffer;
}
}
+11
View File
@@ -0,0 +1,11 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Repository;
class RepositoryException extends \RuntimeException {
}