Phase 0: repository format with packs, dedup and incremental runs (spike B)
Content-addressed repository on an IBackend (local implementation): 4 MiB blobs, deflate when it helps, each blob encrypted with its id as associated data, ~32 MiB packs with an encrypted header, one index file per pack, snapshots as encrypted JSON lines written last. Spike B: 390 MiB tree, peak 54 MiB; unchanged run reads nothing; a 1 MiB change in a 300 MiB file uploads one blob; full and partial restore are bit-identical; tampering is detected.
This commit is contained in:
@@ -0,0 +1,237 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
/**
|
||||
* Content-addressed, encrypted, deduplicating repository.
|
||||
*
|
||||
* Layout on the backend:
|
||||
* config plaintext JSON: format version, repository id, wrapped master key
|
||||
* packs/xx/<packId> encrypted blobs + encrypted pack header
|
||||
* index/<packId> encrypted list of the blobs in that pack
|
||||
* snapshots/<snapId> encrypted JSON lines: one meta line, then one line per file
|
||||
*
|
||||
* Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in
|
||||
* memory at a time. A blob that already exists (same keyed hash) is not stored again.
|
||||
*/
|
||||
final class Repository {
|
||||
public const FORMAT = 1;
|
||||
public const BLOB_SIZE = 4 * 1048576;
|
||||
private const FLAG_DEFLATE = 1;
|
||||
|
||||
private StreamCipher $cipher;
|
||||
private BlobIndex $index;
|
||||
|
||||
private function __construct(
|
||||
private IBackend $backend,
|
||||
private KeyRing $keys,
|
||||
private string $repositoryId,
|
||||
) {
|
||||
$this->cipher = new StreamCipher($keys);
|
||||
$this->index = new BlobIndex($backend, $this->cipher);
|
||||
}
|
||||
|
||||
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||
if ($backend->exists('config')) {
|
||||
throw new RepositoryException('A repository already exists at this location');
|
||||
}
|
||||
$keys = KeyRing::generate();
|
||||
$id = bin2hex(random_bytes(16));
|
||||
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $keys->wrap($passphrase)];
|
||||
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
||||
return new self($backend, $keys, $id);
|
||||
}
|
||||
|
||||
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||
try {
|
||||
$fh = $backend->get('config');
|
||||
} catch (BackendException) {
|
||||
throw new RepositoryException('No repository at this location');
|
||||
}
|
||||
$config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($fh);
|
||||
if (($config['format'] ?? 0) !== self::FORMAT) {
|
||||
throw new RepositoryException('Unsupported repository format');
|
||||
}
|
||||
$repo = new self($backend, KeyRing::unwrap($config['key'], $passphrase), $config['id']);
|
||||
$repo->index->load();
|
||||
return $repo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Back up a local directory tree. Files whose size and mtime match the parent snapshot are not
|
||||
* read again; their blob list is reused.
|
||||
*
|
||||
* @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}
|
||||
*/
|
||||
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
||||
$source = rtrim($source, '/');
|
||||
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
||||
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
|
||||
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
||||
|
||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$snapshotId = bin2hex(random_bytes(16));
|
||||
fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n");
|
||||
|
||||
foreach ($this->walk($source) as $relative => $info) {
|
||||
$stats['files']++;
|
||||
$prev = $previous[$relative] ?? null;
|
||||
if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) {
|
||||
$blobs = $prev['b'];
|
||||
$stats['reused']++;
|
||||
} else {
|
||||
$blobs = [];
|
||||
$fh = fopen($info->getPathname(), 'rb');
|
||||
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
||||
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
||||
}
|
||||
fclose($fh);
|
||||
$stats['read']++;
|
||||
}
|
||||
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
|
||||
}
|
||||
$packs->flush();
|
||||
|
||||
// The snapshot is written last: it only references blobs whose packs are already stored.
|
||||
rewind($tree);
|
||||
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
|
||||
rewind($encrypted);
|
||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||
fclose($tree);
|
||||
fclose($encrypted);
|
||||
|
||||
return ['snapshot' => $snapshotId] + $stats;
|
||||
}
|
||||
|
||||
/** Restore a snapshot (or only paths starting with $prefix) into a local directory. */
|
||||
public function restore(string $snapshotId, string $target, string $prefix = ''): int {
|
||||
$count = 0;
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
$dest = rtrim($target, '/') . '/' . $entry['p'];
|
||||
if (!is_dir(dirname($dest))) {
|
||||
mkdir(dirname($dest), 0700, true);
|
||||
}
|
||||
// Write next to the destination and rename after all blobs verified: at most one file
|
||||
// is ever present twice.
|
||||
$tmp = $dest . '.ngb-restore';
|
||||
$out = fopen($tmp, 'wb');
|
||||
foreach ($entry['b'] as $id) {
|
||||
fwrite($out, $this->loadBlob($id));
|
||||
}
|
||||
fclose($out);
|
||||
if (filesize($tmp) !== $entry['s']) {
|
||||
unlink($tmp);
|
||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||
}
|
||||
rename($tmp, $dest);
|
||||
touch($dest, $entry['m']);
|
||||
$count++;
|
||||
}
|
||||
return $count;
|
||||
}
|
||||
|
||||
/** @return list<string> */
|
||||
public function listSnapshots(): array {
|
||||
return array_map('basename', $this->backend->list('snapshots'));
|
||||
}
|
||||
|
||||
public function blobCount(): int {
|
||||
return $this->index->count();
|
||||
}
|
||||
|
||||
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
|
||||
$id = $this->keys->blobId($data);
|
||||
if ($this->index->has($id)) {
|
||||
$stats['dupBlobs']++;
|
||||
return $id;
|
||||
}
|
||||
$flags = 0;
|
||||
$payload = $data;
|
||||
$deflated = gzdeflate($data, 6);
|
||||
if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
|
||||
$payload = $deflated;
|
||||
$flags |= self::FLAG_DEFLATE;
|
||||
}
|
||||
$stats['uploaded'] += $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
|
||||
$stats['newBlobs']++;
|
||||
return $id;
|
||||
}
|
||||
|
||||
private function loadBlob(string $id): string {
|
||||
[$pack, $offset, $length, $raw, $flags] = $this->index->get($id);
|
||||
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
||||
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
||||
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
||||
throw new RepositoryException('Blob verification failed: ' . $id);
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
||||
private function loadEntries(string $snapshotId): array {
|
||||
$map = [];
|
||||
foreach ($this->streamEntries($snapshotId) as $e) {
|
||||
$map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']];
|
||||
}
|
||||
return $map;
|
||||
}
|
||||
|
||||
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
||||
private function streamEntries(string $snapshotId): \Generator {
|
||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
||||
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
||||
fclose($fh);
|
||||
rewind($plain);
|
||||
fgets($plain); // meta line
|
||||
while (($line = fgets($plain)) !== false) {
|
||||
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
fclose($plain);
|
||||
}
|
||||
|
||||
/** @return \Generator<string, \SplFileInfo> relative path => file, in a stable order */
|
||||
private function walk(string $root): \Generator {
|
||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
|
||||
$files = [];
|
||||
foreach ($it as $file) {
|
||||
if ($file->isFile()) {
|
||||
$files[substr($file->getPathname(), strlen($root) + 1)] = $file;
|
||||
}
|
||||
}
|
||||
ksort($files, SORT_STRING);
|
||||
yield from $files;
|
||||
}
|
||||
|
||||
private static function putString(IBackend $backend, string $path, string $data): void {
|
||||
$s = fopen('php://memory', 'w+b');
|
||||
fwrite($s, $data);
|
||||
rewind($s);
|
||||
$backend->put($path, $s);
|
||||
fclose($s);
|
||||
}
|
||||
|
||||
private static function readFull($fh, int $length): string {
|
||||
$buffer = '';
|
||||
while (strlen($buffer) < $length && !feof($fh)) {
|
||||
$buffer .= (string)fread($fh, $length - strlen($buffer));
|
||||
}
|
||||
return $buffer;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user