Phase 0: repository format with packs, dedup and incremental runs (spike B)
Content-addressed repository on an IBackend (local implementation): 4 MiB blobs, deflate when it helps, each blob encrypted with its id as associated data, ~32 MiB packs with an encrypted header, one index file per pack, snapshots as encrypted JSON lines written last. Spike B: 390 MiB tree, peak 54 MiB; unchanged run reads nothing; a 1 MiB change in a 300 MiB file uploads one blob; full and partial restore are bit-identical; tampering is detected.
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
class BackendException extends \RuntimeException {
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
/**
|
||||
* Minimal object store used by the repository. Paths are relative, '/'-separated.
|
||||
* Implementations: a local directory (spike, tests), and adapters over files_external storages.
|
||||
*/
|
||||
interface IBackend {
|
||||
/** Store an object; the stream is read to EOF. Must not leave a partial object behind on failure. */
|
||||
public function put(string $path, $stream): void;
|
||||
|
||||
/** @return resource readable stream of the whole object */
|
||||
public function get(string $path);
|
||||
|
||||
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
|
||||
public function getRange(string $path, int $offset, int $length): string;
|
||||
|
||||
public function exists(string $path): bool;
|
||||
|
||||
/** @return list<string> object paths below $prefix (recursive) */
|
||||
public function list(string $prefix): array;
|
||||
|
||||
public function delete(string $path): void;
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
final class LocalBackend implements IBackend {
|
||||
public function __construct(
|
||||
private string $root,
|
||||
) {
|
||||
$this->root = rtrim($root, '/');
|
||||
if (!is_dir($this->root) && !mkdir($this->root, 0700, true) && !is_dir($this->root)) {
|
||||
throw new BackendException('Cannot create ' . $this->root);
|
||||
}
|
||||
}
|
||||
|
||||
public function put(string $path, $stream): void {
|
||||
$target = $this->abs($path);
|
||||
$dir = dirname($target);
|
||||
if (!is_dir($dir) && !mkdir($dir, 0700, true) && !is_dir($dir)) {
|
||||
throw new BackendException('Cannot create ' . $dir);
|
||||
}
|
||||
// Write to a temporary name and rename, so a crash never leaves a partial object.
|
||||
$tmp = $target . '.part-' . bin2hex(random_bytes(4));
|
||||
$out = fopen($tmp, 'wb');
|
||||
if ($out === false) {
|
||||
throw new BackendException('Cannot write ' . $path);
|
||||
}
|
||||
try {
|
||||
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
|
||||
throw new BackendException('Write failed for ' . $path);
|
||||
}
|
||||
} finally {
|
||||
fclose($out);
|
||||
}
|
||||
if (!rename($tmp, $target)) {
|
||||
@unlink($tmp);
|
||||
throw new BackendException('Cannot finalise ' . $path);
|
||||
}
|
||||
}
|
||||
|
||||
public function get(string $path) {
|
||||
$fh = @fopen($this->abs($path), 'rb');
|
||||
if ($fh === false) {
|
||||
throw new BackendException('Not found: ' . $path);
|
||||
}
|
||||
return $fh;
|
||||
}
|
||||
|
||||
public function getRange(string $path, int $offset, int $length): string {
|
||||
$fh = $this->get($path);
|
||||
try {
|
||||
if (fseek($fh, $offset) !== 0) {
|
||||
throw new BackendException('Seek failed in ' . $path);
|
||||
}
|
||||
$data = '';
|
||||
while (strlen($data) < $length && !feof($fh)) {
|
||||
$data .= (string)fread($fh, $length - strlen($data));
|
||||
}
|
||||
if (strlen($data) !== $length) {
|
||||
throw new BackendException('Short read in ' . $path);
|
||||
}
|
||||
return $data;
|
||||
} finally {
|
||||
fclose($fh);
|
||||
}
|
||||
}
|
||||
|
||||
public function exists(string $path): bool {
|
||||
return is_file($this->abs($path));
|
||||
}
|
||||
|
||||
public function list(string $prefix): array {
|
||||
$base = $this->abs($prefix);
|
||||
if (!is_dir($base)) {
|
||||
return [];
|
||||
}
|
||||
$result = [];
|
||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
|
||||
foreach ($it as $file) {
|
||||
if ($file->isFile() && !str_contains($file->getFilename(), '.part-')) {
|
||||
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
|
||||
}
|
||||
}
|
||||
sort($result);
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
@unlink($this->abs($path));
|
||||
}
|
||||
|
||||
private function abs(string $path): string {
|
||||
if (str_contains($path, '..') || str_starts_with($path, '/')) {
|
||||
throw new BackendException('Invalid path: ' . $path);
|
||||
}
|
||||
return $this->root . '/' . $path;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
/**
|
||||
* Which blob lives in which pack. Every uploaded pack gets its own small encrypted index file
|
||||
* (index/<packId>), written right after the pack, so an interrupted run leaves a consistent
|
||||
* repository and the next run can reuse what was already uploaded.
|
||||
*
|
||||
* Phase 0 keeps the whole index in memory. For large repositories this becomes a table in the
|
||||
* Nextcloud database (a cache that can always be rebuilt from index/ and the pack headers).
|
||||
*/
|
||||
final class BlobIndex {
|
||||
/** @var array<string, array{0:string, 1:int, 2:int, 3:int, 4:int}> id => [pack, offset, length, raw, flags] */
|
||||
private array $blobs = [];
|
||||
/** @var array<string, true> blobs added in this run but whose pack is not uploaded yet */
|
||||
private array $pending = [];
|
||||
|
||||
public function __construct(
|
||||
private IBackend $backend,
|
||||
private StreamCipher $cipher,
|
||||
) {
|
||||
}
|
||||
|
||||
public function load(): void {
|
||||
foreach ($this->backend->list('index') as $path) {
|
||||
$packId = basename($path);
|
||||
$fh = $this->backend->get($path);
|
||||
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($fh);
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function has(string $id): bool {
|
||||
return isset($this->blobs[$id]) || isset($this->pending[$id]);
|
||||
}
|
||||
|
||||
/** @return array{0:string, 1:int, 2:int, 3:int, 4:int} */
|
||||
public function get(string $id): array {
|
||||
return $this->blobs[$id] ?? throw new RepositoryException('Blob not in index: ' . $id);
|
||||
}
|
||||
|
||||
public function addPending(string $id): void {
|
||||
$this->pending[$id] = true;
|
||||
}
|
||||
|
||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||
public function addPack(string $packId, array $entries): void {
|
||||
$stream = fopen('php://memory', 'w+b');
|
||||
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
||||
rewind($stream);
|
||||
$this->backend->put('index/' . $packId, $stream);
|
||||
fclose($stream);
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
unset($this->pending[$e['id']]);
|
||||
}
|
||||
}
|
||||
|
||||
public function count(): int {
|
||||
return count($this->blobs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
/**
|
||||
* Collects encrypted blobs into a pack in memory and uploads it when full.
|
||||
*
|
||||
* Pack layout: blob* | encrypted header | uint32 big-endian header length
|
||||
* The header lists (id, offset, length, rawLength, flags) of every blob, so a pack can be
|
||||
* re-indexed from the pack alone. Each blob is encrypted separately with its id as associated
|
||||
* data, so single blobs can be fetched with a range read and cannot be swapped.
|
||||
*
|
||||
* Only one pack is held at a time (php://temp keeps it in memory up to the limit).
|
||||
*/
|
||||
final class PackWriter {
|
||||
/** @var resource */
|
||||
private $buffer;
|
||||
private int $size = 0;
|
||||
/** @var list<array{id:string, offset:int, length:int, raw:int, flags:int}> */
|
||||
private array $entries = [];
|
||||
|
||||
public function __construct(
|
||||
private IBackend $backend,
|
||||
private StreamCipher $cipher,
|
||||
private BlobIndex $index,
|
||||
private int $targetSize = 32 * 1048576,
|
||||
) {
|
||||
$this->reset();
|
||||
}
|
||||
|
||||
/** Add a blob that is not yet in the repository. Returns the number of bytes added to the pack. */
|
||||
public function add(string $id, string $encrypted, int $rawLength, int $flags): int {
|
||||
fwrite($this->buffer, $encrypted);
|
||||
$this->entries[] = ['id' => $id, 'offset' => $this->size, 'length' => strlen($encrypted), 'raw' => $rawLength, 'flags' => $flags];
|
||||
$this->size += strlen($encrypted);
|
||||
// Make the blob known immediately, so a later duplicate in the same run is not stored twice.
|
||||
$this->index->addPending($id);
|
||||
if ($this->size >= $this->targetSize) {
|
||||
$this->flush();
|
||||
}
|
||||
return strlen($encrypted);
|
||||
}
|
||||
|
||||
/** Upload the current pack (if any) and record it in the index. */
|
||||
public function flush(): void {
|
||||
if ($this->entries === []) {
|
||||
return;
|
||||
}
|
||||
$packId = bin2hex(random_bytes(16));
|
||||
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
||||
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
||||
rewind($this->buffer);
|
||||
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
||||
$this->backend->put($path, $this->buffer);
|
||||
$this->index->addPack($packId, $this->entries);
|
||||
$this->reset();
|
||||
}
|
||||
|
||||
public function pendingBytes(): int {
|
||||
return $this->size;
|
||||
}
|
||||
|
||||
private function reset(): void {
|
||||
if (isset($this->buffer)) {
|
||||
fclose($this->buffer);
|
||||
}
|
||||
$this->buffer = fopen('php://temp/maxmemory:' . ($this->targetSize + 8 * 1048576), 'w+b');
|
||||
$this->size = 0;
|
||||
$this->entries = [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
/**
|
||||
* Content-addressed, encrypted, deduplicating repository.
|
||||
*
|
||||
* Layout on the backend:
|
||||
* config plaintext JSON: format version, repository id, wrapped master key
|
||||
* packs/xx/<packId> encrypted blobs + encrypted pack header
|
||||
* index/<packId> encrypted list of the blobs in that pack
|
||||
* snapshots/<snapId> encrypted JSON lines: one meta line, then one line per file
|
||||
*
|
||||
* Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in
|
||||
* memory at a time. A blob that already exists (same keyed hash) is not stored again.
|
||||
*/
|
||||
final class Repository {
|
||||
public const FORMAT = 1;
|
||||
public const BLOB_SIZE = 4 * 1048576;
|
||||
private const FLAG_DEFLATE = 1;
|
||||
|
||||
private StreamCipher $cipher;
|
||||
private BlobIndex $index;
|
||||
|
||||
private function __construct(
|
||||
private IBackend $backend,
|
||||
private KeyRing $keys,
|
||||
private string $repositoryId,
|
||||
) {
|
||||
$this->cipher = new StreamCipher($keys);
|
||||
$this->index = new BlobIndex($backend, $this->cipher);
|
||||
}
|
||||
|
||||
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||
if ($backend->exists('config')) {
|
||||
throw new RepositoryException('A repository already exists at this location');
|
||||
}
|
||||
$keys = KeyRing::generate();
|
||||
$id = bin2hex(random_bytes(16));
|
||||
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $keys->wrap($passphrase)];
|
||||
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
||||
return new self($backend, $keys, $id);
|
||||
}
|
||||
|
||||
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||
try {
|
||||
$fh = $backend->get('config');
|
||||
} catch (BackendException) {
|
||||
throw new RepositoryException('No repository at this location');
|
||||
}
|
||||
$config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($fh);
|
||||
if (($config['format'] ?? 0) !== self::FORMAT) {
|
||||
throw new RepositoryException('Unsupported repository format');
|
||||
}
|
||||
$repo = new self($backend, KeyRing::unwrap($config['key'], $passphrase), $config['id']);
|
||||
$repo->index->load();
|
||||
return $repo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Back up a local directory tree. Files whose size and mtime match the parent snapshot are not
|
||||
* read again; their blob list is reused.
|
||||
*
|
||||
* @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}
|
||||
*/
|
||||
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
||||
$source = rtrim($source, '/');
|
||||
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
||||
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
|
||||
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
||||
|
||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$snapshotId = bin2hex(random_bytes(16));
|
||||
fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n");
|
||||
|
||||
foreach ($this->walk($source) as $relative => $info) {
|
||||
$stats['files']++;
|
||||
$prev = $previous[$relative] ?? null;
|
||||
if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) {
|
||||
$blobs = $prev['b'];
|
||||
$stats['reused']++;
|
||||
} else {
|
||||
$blobs = [];
|
||||
$fh = fopen($info->getPathname(), 'rb');
|
||||
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
||||
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
||||
}
|
||||
fclose($fh);
|
||||
$stats['read']++;
|
||||
}
|
||||
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
|
||||
}
|
||||
$packs->flush();
|
||||
|
||||
// The snapshot is written last: it only references blobs whose packs are already stored.
|
||||
rewind($tree);
|
||||
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
|
||||
rewind($encrypted);
|
||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||
fclose($tree);
|
||||
fclose($encrypted);
|
||||
|
||||
return ['snapshot' => $snapshotId] + $stats;
|
||||
}
|
||||
|
||||
/** Restore a snapshot (or only paths starting with $prefix) into a local directory. */
|
||||
public function restore(string $snapshotId, string $target, string $prefix = ''): int {
|
||||
$count = 0;
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
$dest = rtrim($target, '/') . '/' . $entry['p'];
|
||||
if (!is_dir(dirname($dest))) {
|
||||
mkdir(dirname($dest), 0700, true);
|
||||
}
|
||||
// Write next to the destination and rename after all blobs verified: at most one file
|
||||
// is ever present twice.
|
||||
$tmp = $dest . '.ngb-restore';
|
||||
$out = fopen($tmp, 'wb');
|
||||
foreach ($entry['b'] as $id) {
|
||||
fwrite($out, $this->loadBlob($id));
|
||||
}
|
||||
fclose($out);
|
||||
if (filesize($tmp) !== $entry['s']) {
|
||||
unlink($tmp);
|
||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||
}
|
||||
rename($tmp, $dest);
|
||||
touch($dest, $entry['m']);
|
||||
$count++;
|
||||
}
|
||||
return $count;
|
||||
}
|
||||
|
||||
/** @return list<string> */
|
||||
public function listSnapshots(): array {
|
||||
return array_map('basename', $this->backend->list('snapshots'));
|
||||
}
|
||||
|
||||
public function blobCount(): int {
|
||||
return $this->index->count();
|
||||
}
|
||||
|
||||
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
|
||||
$id = $this->keys->blobId($data);
|
||||
if ($this->index->has($id)) {
|
||||
$stats['dupBlobs']++;
|
||||
return $id;
|
||||
}
|
||||
$flags = 0;
|
||||
$payload = $data;
|
||||
$deflated = gzdeflate($data, 6);
|
||||
if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
|
||||
$payload = $deflated;
|
||||
$flags |= self::FLAG_DEFLATE;
|
||||
}
|
||||
$stats['uploaded'] += $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
|
||||
$stats['newBlobs']++;
|
||||
return $id;
|
||||
}
|
||||
|
||||
private function loadBlob(string $id): string {
|
||||
[$pack, $offset, $length, $raw, $flags] = $this->index->get($id);
|
||||
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
||||
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
||||
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
||||
throw new RepositoryException('Blob verification failed: ' . $id);
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
||||
private function loadEntries(string $snapshotId): array {
|
||||
$map = [];
|
||||
foreach ($this->streamEntries($snapshotId) as $e) {
|
||||
$map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']];
|
||||
}
|
||||
return $map;
|
||||
}
|
||||
|
||||
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
||||
private function streamEntries(string $snapshotId): \Generator {
|
||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
||||
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
||||
fclose($fh);
|
||||
rewind($plain);
|
||||
fgets($plain); // meta line
|
||||
while (($line = fgets($plain)) !== false) {
|
||||
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
fclose($plain);
|
||||
}
|
||||
|
||||
/** @return \Generator<string, \SplFileInfo> relative path => file, in a stable order */
|
||||
private function walk(string $root): \Generator {
|
||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
|
||||
$files = [];
|
||||
foreach ($it as $file) {
|
||||
if ($file->isFile()) {
|
||||
$files[substr($file->getPathname(), strlen($root) + 1)] = $file;
|
||||
}
|
||||
}
|
||||
ksort($files, SORT_STRING);
|
||||
yield from $files;
|
||||
}
|
||||
|
||||
private static function putString(IBackend $backend, string $path, string $data): void {
|
||||
$s = fopen('php://memory', 'w+b');
|
||||
fwrite($s, $data);
|
||||
rewind($s);
|
||||
$backend->put($path, $s);
|
||||
fclose($s);
|
||||
}
|
||||
|
||||
private static function readFull($fh, int $length): string {
|
||||
$buffer = '';
|
||||
while (strlen($buffer) < $length && !feof($fh)) {
|
||||
$buffer .= (string)fread($fh, $length - strlen($buffer));
|
||||
}
|
||||
return $buffer;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
class RepositoryException extends \RuntimeException {
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// Spike B: repository format, deduplication, incremental runs, restore, tamper detection.
|
||||
// Usage: php -d memory_limit=128M spikes/02-repository.php [workdir]
|
||||
|
||||
require __DIR__ . '/bootstrap.php';
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\CryptoException;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Repository\RepositoryException;
|
||||
|
||||
$work = rtrim($argv[1] ?? sys_get_temp_dir(), '/') . '/ngb-spike-b';
|
||||
exec('rm -rf ' . escapeshellarg($work));
|
||||
$src = "$work/src";
|
||||
$repoDir = "$work/repo";
|
||||
mkdir($src, 0700, true);
|
||||
|
||||
$ok = true;
|
||||
$check = static function (string $name, bool $pass) use (&$ok): void {
|
||||
printf("%-62s %s\n", $name, $pass ? 'PASS' : 'FAIL');
|
||||
$ok = $ok && $pass;
|
||||
};
|
||||
$write = static function (string $path, string $data, bool $append = false): void {
|
||||
@mkdir(dirname($path), 0700, true);
|
||||
file_put_contents($path, $data, $append ? FILE_APPEND : 0);
|
||||
};
|
||||
$randomFile = static function (string $path, int $mib): void {
|
||||
@mkdir(dirname($path), 0700, true);
|
||||
$fh = fopen($path, 'wb');
|
||||
for ($i = 0; $i < $mib; $i++) {
|
||||
fwrite($fh, random_bytes(1048576));
|
||||
}
|
||||
fclose($fh);
|
||||
};
|
||||
$hashTree = static function (string $root): array {
|
||||
$out = [];
|
||||
$it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root, FilesystemIterator::SKIP_DOTS));
|
||||
foreach ($it as $f) {
|
||||
if ($f->isFile()) {
|
||||
$out[substr($f->getPathname(), strlen($root) + 1)] = hash_file('sha256', $f->getPathname());
|
||||
}
|
||||
}
|
||||
ksort($out);
|
||||
return $out;
|
||||
};
|
||||
$du = static function (string $dir): int {
|
||||
$total = 0;
|
||||
$it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS));
|
||||
foreach ($it as $f) {
|
||||
$total += $f->isFile() ? $f->getSize() : 0;
|
||||
}
|
||||
return $total;
|
||||
};
|
||||
$mb = static fn (int $b): string => sprintf('%.1f MiB', $b / 1048576);
|
||||
|
||||
// Source tree.
|
||||
for ($i = 0; $i < 200; $i++) {
|
||||
$write(sprintf('%s/docs/%02d/note-%03d.txt', $src, $i % 10, $i), str_repeat("Regel $i van een tekstbestand met herhaling. ", 1200));
|
||||
}
|
||||
for ($i = 0; $i < 20; $i++) {
|
||||
$randomFile("$src/photos/img-$i.jpg", 1 + $i % 5);
|
||||
}
|
||||
$randomFile("$src/video/big.mp4", 300);
|
||||
$randomFile("$src/copies/a.bin", 10);
|
||||
copy("$src/copies/a.bin", "$src/copies/b.bin");
|
||||
$sourceBytes = $du($src);
|
||||
$state1 = $hashTree($src);
|
||||
printf("source: %d files, %s\n", count($state1), $mb($sourceBytes));
|
||||
|
||||
$backend = new LocalBackend($repoDir);
|
||||
$repo = Repository::init($backend, 'spike passphrase');
|
||||
|
||||
// Run 1: full.
|
||||
memory_reset_peak_usage();
|
||||
$t = microtime(true);
|
||||
$r1 = $repo->backupDirectory($src, null, 'run 1');
|
||||
printf("run 1: %.1fs, peak %s, %s\n", microtime(true) - $t, $mb(memory_get_peak_usage()), json_encode(array_diff_key($r1, ['snapshot' => 1])));
|
||||
$repoAfter1 = $du($repoDir);
|
||||
printf("repository after run 1: %s (%.0f%% of source)\n", $mb($repoAfter1), $repoAfter1 / $sourceBytes * 100);
|
||||
$check('run 1: identical copies stored once (dedup within run)', $r1['dupBlobs'] >= 3);
|
||||
$check('run 1: repository smaller than source (compression + dedup)', $repoAfter1 < $sourceBytes);
|
||||
$check('run 1: peak memory < 64 MiB with a 300 MiB file', memory_get_peak_usage() < 64 * 1048576);
|
||||
|
||||
// Run 2: nothing changed.
|
||||
$t = microtime(true);
|
||||
$r2 = $repo->backupDirectory($src, $r1['snapshot'], 'run 2');
|
||||
printf("run 2: %.2fs, %s\n", microtime(true) - $t, json_encode(array_diff_key($r2, ['snapshot' => 1])));
|
||||
$check('run 2: no file read, nothing uploaded', $r2['read'] === 0 && $r2['uploaded'] === 0);
|
||||
|
||||
// Changes: append to a note, overwrite 1 MiB in the middle of the big file, add, delete.
|
||||
sleep(1); // mtime resolution
|
||||
$write("$src/docs/03/note-003.txt", "Toegevoegde regel.\n", true);
|
||||
$fh = fopen("$src/video/big.mp4", 'r+b');
|
||||
fseek($fh, 150 * 1048576);
|
||||
fwrite($fh, random_bytes(1048576));
|
||||
fclose($fh);
|
||||
$randomFile("$src/photos/new.jpg", 2);
|
||||
unlink("$src/photos/img-0.jpg");
|
||||
$state3 = $hashTree($src);
|
||||
|
||||
$t = microtime(true);
|
||||
$r3 = $repo->backupDirectory($src, $r2['snapshot'], 'run 3');
|
||||
printf("run 3: %.1fs, %s\n", microtime(true) - $t, json_encode(array_diff_key($r3, ['snapshot' => 1])));
|
||||
$check('run 3: only changed/new files read (3)', $r3['read'] === 3);
|
||||
$check('run 3: big file change costs ~1 blob, not 300 MiB', $r3['uploaded'] < 12 * 1048576);
|
||||
|
||||
// Restore with a fresh process view of the repository (reopen with the passphrase).
|
||||
$repo = Repository::open($backend, 'spike passphrase');
|
||||
$check('3 snapshots listed', count($repo->listSnapshots()) === 3);
|
||||
|
||||
memory_reset_peak_usage();
|
||||
$repo->restore($r1['snapshot'], "$work/restore1");
|
||||
$check('restore run 1: all files identical to state 1', $hashTree("$work/restore1") === $state1);
|
||||
printf("restore peak memory: %s\n", $mb(memory_get_peak_usage()));
|
||||
$repo->restore($r3['snapshot'], "$work/restore3");
|
||||
$check('restore run 3: all files identical to state 3', $hashTree("$work/restore3") === $state3);
|
||||
$n = $repo->restore($r3['snapshot'], "$work/restore-docs", 'docs/03/');
|
||||
$check('partial restore of one folder (20 files)', $n === 20);
|
||||
|
||||
// Wrong passphrase.
|
||||
try {
|
||||
Repository::open($backend, 'wrong');
|
||||
$check('wrong passphrase rejected', false);
|
||||
} catch (CryptoException) {
|
||||
$check('wrong passphrase rejected', true);
|
||||
}
|
||||
|
||||
// Tamper with one pack: restore must fail loudly, not produce wrong data.
|
||||
$packs = $backend->list('packs');
|
||||
$victim = "$repoDir/" . $packs[0];
|
||||
$bytes = file_get_contents($victim);
|
||||
$bytes[1000] = chr(ord($bytes[1000]) ^ 0xff);
|
||||
file_put_contents($victim, $bytes);
|
||||
try {
|
||||
$repo->restore($r1['snapshot'], "$work/restore-bad");
|
||||
$check('tampered pack detected on restore', false);
|
||||
} catch (CryptoException|RepositoryException $e) {
|
||||
$check('tampered pack detected on restore', true);
|
||||
}
|
||||
|
||||
exec('rm -rf ' . escapeshellarg($work));
|
||||
echo $ok ? "ALL PASS\n" : "FAILURES\n";
|
||||
exit($ok ? 0 : 1);
|
||||
Reference in New Issue
Block a user