Phase 0: repository format with packs, dedup and incremental runs (spike B)
Content-addressed repository on an IBackend (local implementation): 4 MiB blobs, deflate when it helps, each blob encrypted with its id as associated data, ~32 MiB packs with an encrypted header, one index file per pack, snapshots as encrypted JSON lines written last. Spike B: 390 MiB tree, peak 54 MiB; unchanged run reads nothing; a 1 MiB change in a 300 MiB file uploads one blob; full and partial restore are bit-identical; tampering is detected.
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
/**
|
||||
* Which blob lives in which pack. Every uploaded pack gets its own small encrypted index file
|
||||
* (index/<packId>), written right after the pack, so an interrupted run leaves a consistent
|
||||
* repository and the next run can reuse what was already uploaded.
|
||||
*
|
||||
* Phase 0 keeps the whole index in memory. For large repositories this becomes a table in the
|
||||
* Nextcloud database (a cache that can always be rebuilt from index/ and the pack headers).
|
||||
*/
|
||||
final class BlobIndex {
|
||||
/** @var array<string, array{0:string, 1:int, 2:int, 3:int, 4:int}> id => [pack, offset, length, raw, flags] */
|
||||
private array $blobs = [];
|
||||
/** @var array<string, true> blobs added in this run but whose pack is not uploaded yet */
|
||||
private array $pending = [];
|
||||
|
||||
public function __construct(
|
||||
private IBackend $backend,
|
||||
private StreamCipher $cipher,
|
||||
) {
|
||||
}
|
||||
|
||||
public function load(): void {
|
||||
foreach ($this->backend->list('index') as $path) {
|
||||
$packId = basename($path);
|
||||
$fh = $this->backend->get($path);
|
||||
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($fh);
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function has(string $id): bool {
|
||||
return isset($this->blobs[$id]) || isset($this->pending[$id]);
|
||||
}
|
||||
|
||||
/** @return array{0:string, 1:int, 2:int, 3:int, 4:int} */
|
||||
public function get(string $id): array {
|
||||
return $this->blobs[$id] ?? throw new RepositoryException('Blob not in index: ' . $id);
|
||||
}
|
||||
|
||||
public function addPending(string $id): void {
|
||||
$this->pending[$id] = true;
|
||||
}
|
||||
|
||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||
public function addPack(string $packId, array $entries): void {
|
||||
$stream = fopen('php://memory', 'w+b');
|
||||
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
||||
rewind($stream);
|
||||
$this->backend->put('index/' . $packId, $stream);
|
||||
fclose($stream);
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
unset($this->pending[$e['id']]);
|
||||
}
|
||||
}
|
||||
|
||||
public function count(): int {
|
||||
return count($this->blobs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
/**
|
||||
* Collects encrypted blobs into a pack in memory and uploads it when full.
|
||||
*
|
||||
* Pack layout: blob* | encrypted header | uint32 big-endian header length
|
||||
* The header lists (id, offset, length, rawLength, flags) of every blob, so a pack can be
|
||||
* re-indexed from the pack alone. Each blob is encrypted separately with its id as associated
|
||||
* data, so single blobs can be fetched with a range read and cannot be swapped.
|
||||
*
|
||||
* Only one pack is held at a time (php://temp keeps it in memory up to the limit).
|
||||
*/
|
||||
final class PackWriter {
|
||||
/** @var resource */
|
||||
private $buffer;
|
||||
private int $size = 0;
|
||||
/** @var list<array{id:string, offset:int, length:int, raw:int, flags:int}> */
|
||||
private array $entries = [];
|
||||
|
||||
public function __construct(
|
||||
private IBackend $backend,
|
||||
private StreamCipher $cipher,
|
||||
private BlobIndex $index,
|
||||
private int $targetSize = 32 * 1048576,
|
||||
) {
|
||||
$this->reset();
|
||||
}
|
||||
|
||||
/** Add a blob that is not yet in the repository. Returns the number of bytes added to the pack. */
|
||||
public function add(string $id, string $encrypted, int $rawLength, int $flags): int {
|
||||
fwrite($this->buffer, $encrypted);
|
||||
$this->entries[] = ['id' => $id, 'offset' => $this->size, 'length' => strlen($encrypted), 'raw' => $rawLength, 'flags' => $flags];
|
||||
$this->size += strlen($encrypted);
|
||||
// Make the blob known immediately, so a later duplicate in the same run is not stored twice.
|
||||
$this->index->addPending($id);
|
||||
if ($this->size >= $this->targetSize) {
|
||||
$this->flush();
|
||||
}
|
||||
return strlen($encrypted);
|
||||
}
|
||||
|
||||
/** Upload the current pack (if any) and record it in the index. */
|
||||
public function flush(): void {
|
||||
if ($this->entries === []) {
|
||||
return;
|
||||
}
|
||||
$packId = bin2hex(random_bytes(16));
|
||||
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
||||
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
||||
rewind($this->buffer);
|
||||
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
||||
$this->backend->put($path, $this->buffer);
|
||||
$this->index->addPack($packId, $this->entries);
|
||||
$this->reset();
|
||||
}
|
||||
|
||||
public function pendingBytes(): int {
|
||||
return $this->size;
|
||||
}
|
||||
|
||||
private function reset(): void {
|
||||
if (isset($this->buffer)) {
|
||||
fclose($this->buffer);
|
||||
}
|
||||
$this->buffer = fopen('php://temp/maxmemory:' . ($this->targetSize + 8 * 1048576), 'w+b');
|
||||
$this->size = 0;
|
||||
$this->entries = [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
/**
|
||||
* Content-addressed, encrypted, deduplicating repository.
|
||||
*
|
||||
* Layout on the backend:
|
||||
* config plaintext JSON: format version, repository id, wrapped master key
|
||||
* packs/xx/<packId> encrypted blobs + encrypted pack header
|
||||
* index/<packId> encrypted list of the blobs in that pack
|
||||
* snapshots/<snapId> encrypted JSON lines: one meta line, then one line per file
|
||||
*
|
||||
* Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in
|
||||
* memory at a time. A blob that already exists (same keyed hash) is not stored again.
|
||||
*/
|
||||
final class Repository {
|
||||
public const FORMAT = 1;
|
||||
public const BLOB_SIZE = 4 * 1048576;
|
||||
private const FLAG_DEFLATE = 1;
|
||||
|
||||
private StreamCipher $cipher;
|
||||
private BlobIndex $index;
|
||||
|
||||
private function __construct(
|
||||
private IBackend $backend,
|
||||
private KeyRing $keys,
|
||||
private string $repositoryId,
|
||||
) {
|
||||
$this->cipher = new StreamCipher($keys);
|
||||
$this->index = new BlobIndex($backend, $this->cipher);
|
||||
}
|
||||
|
||||
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||
if ($backend->exists('config')) {
|
||||
throw new RepositoryException('A repository already exists at this location');
|
||||
}
|
||||
$keys = KeyRing::generate();
|
||||
$id = bin2hex(random_bytes(16));
|
||||
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $keys->wrap($passphrase)];
|
||||
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
||||
return new self($backend, $keys, $id);
|
||||
}
|
||||
|
||||
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||
try {
|
||||
$fh = $backend->get('config');
|
||||
} catch (BackendException) {
|
||||
throw new RepositoryException('No repository at this location');
|
||||
}
|
||||
$config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($fh);
|
||||
if (($config['format'] ?? 0) !== self::FORMAT) {
|
||||
throw new RepositoryException('Unsupported repository format');
|
||||
}
|
||||
$repo = new self($backend, KeyRing::unwrap($config['key'], $passphrase), $config['id']);
|
||||
$repo->index->load();
|
||||
return $repo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Back up a local directory tree. Files whose size and mtime match the parent snapshot are not
|
||||
* read again; their blob list is reused.
|
||||
*
|
||||
* @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}
|
||||
*/
|
||||
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
||||
$source = rtrim($source, '/');
|
||||
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
||||
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
|
||||
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
||||
|
||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$snapshotId = bin2hex(random_bytes(16));
|
||||
fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n");
|
||||
|
||||
foreach ($this->walk($source) as $relative => $info) {
|
||||
$stats['files']++;
|
||||
$prev = $previous[$relative] ?? null;
|
||||
if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) {
|
||||
$blobs = $prev['b'];
|
||||
$stats['reused']++;
|
||||
} else {
|
||||
$blobs = [];
|
||||
$fh = fopen($info->getPathname(), 'rb');
|
||||
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
||||
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
||||
}
|
||||
fclose($fh);
|
||||
$stats['read']++;
|
||||
}
|
||||
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
|
||||
}
|
||||
$packs->flush();
|
||||
|
||||
// The snapshot is written last: it only references blobs whose packs are already stored.
|
||||
rewind($tree);
|
||||
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
|
||||
rewind($encrypted);
|
||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||
fclose($tree);
|
||||
fclose($encrypted);
|
||||
|
||||
return ['snapshot' => $snapshotId] + $stats;
|
||||
}
|
||||
|
||||
/** Restore a snapshot (or only paths starting with $prefix) into a local directory. */
|
||||
public function restore(string $snapshotId, string $target, string $prefix = ''): int {
|
||||
$count = 0;
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
$dest = rtrim($target, '/') . '/' . $entry['p'];
|
||||
if (!is_dir(dirname($dest))) {
|
||||
mkdir(dirname($dest), 0700, true);
|
||||
}
|
||||
// Write next to the destination and rename after all blobs verified: at most one file
|
||||
// is ever present twice.
|
||||
$tmp = $dest . '.ngb-restore';
|
||||
$out = fopen($tmp, 'wb');
|
||||
foreach ($entry['b'] as $id) {
|
||||
fwrite($out, $this->loadBlob($id));
|
||||
}
|
||||
fclose($out);
|
||||
if (filesize($tmp) !== $entry['s']) {
|
||||
unlink($tmp);
|
||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||
}
|
||||
rename($tmp, $dest);
|
||||
touch($dest, $entry['m']);
|
||||
$count++;
|
||||
}
|
||||
return $count;
|
||||
}
|
||||
|
||||
/** @return list<string> */
|
||||
public function listSnapshots(): array {
|
||||
return array_map('basename', $this->backend->list('snapshots'));
|
||||
}
|
||||
|
||||
public function blobCount(): int {
|
||||
return $this->index->count();
|
||||
}
|
||||
|
||||
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
|
||||
$id = $this->keys->blobId($data);
|
||||
if ($this->index->has($id)) {
|
||||
$stats['dupBlobs']++;
|
||||
return $id;
|
||||
}
|
||||
$flags = 0;
|
||||
$payload = $data;
|
||||
$deflated = gzdeflate($data, 6);
|
||||
if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
|
||||
$payload = $deflated;
|
||||
$flags |= self::FLAG_DEFLATE;
|
||||
}
|
||||
$stats['uploaded'] += $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
|
||||
$stats['newBlobs']++;
|
||||
return $id;
|
||||
}
|
||||
|
||||
private function loadBlob(string $id): string {
|
||||
[$pack, $offset, $length, $raw, $flags] = $this->index->get($id);
|
||||
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
||||
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
||||
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
||||
throw new RepositoryException('Blob verification failed: ' . $id);
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
||||
private function loadEntries(string $snapshotId): array {
|
||||
$map = [];
|
||||
foreach ($this->streamEntries($snapshotId) as $e) {
|
||||
$map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']];
|
||||
}
|
||||
return $map;
|
||||
}
|
||||
|
||||
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
||||
private function streamEntries(string $snapshotId): \Generator {
|
||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
||||
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
||||
fclose($fh);
|
||||
rewind($plain);
|
||||
fgets($plain); // meta line
|
||||
while (($line = fgets($plain)) !== false) {
|
||||
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
fclose($plain);
|
||||
}
|
||||
|
||||
/** @return \Generator<string, \SplFileInfo> relative path => file, in a stable order */
|
||||
private function walk(string $root): \Generator {
|
||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
|
||||
$files = [];
|
||||
foreach ($it as $file) {
|
||||
if ($file->isFile()) {
|
||||
$files[substr($file->getPathname(), strlen($root) + 1)] = $file;
|
||||
}
|
||||
}
|
||||
ksort($files, SORT_STRING);
|
||||
yield from $files;
|
||||
}
|
||||
|
||||
private static function putString(IBackend $backend, string $path, string $data): void {
|
||||
$s = fopen('php://memory', 'w+b');
|
||||
fwrite($s, $data);
|
||||
rewind($s);
|
||||
$backend->put($path, $s);
|
||||
fclose($s);
|
||||
}
|
||||
|
||||
private static function readFull($fh, int $length): string {
|
||||
$buffer = '';
|
||||
while (strlen($buffer) < $length && !feof($fh)) {
|
||||
$buffer .= (string)fread($fh, $length - strlen($buffer));
|
||||
}
|
||||
return $buffer;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
class RepositoryException extends \RuntimeException {
|
||||
}
|
||||
Reference in New Issue
Block a user