Phase 0: repository format with packs, dedup and incremental runs (spike B)
Content-addressed repository on an IBackend (local implementation): 4 MiB blobs, deflate when it helps, each blob encrypted with its id as associated data, ~32 MiB packs with an encrypted header, one index file per pack, snapshots as encrypted JSON lines written last. Spike B: 390 MiB tree, peak 54 MiB; unchanged run reads nothing; a 1 MiB change in a 300 MiB file uploads one blob; full and partial restore are bit-identical; tampering is detected.
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
class BackendException extends \RuntimeException {
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
/**
|
||||
* Minimal object store used by the repository. Paths are relative, '/'-separated.
|
||||
* Implementations: a local directory (spike, tests), and adapters over files_external storages.
|
||||
*/
|
||||
interface IBackend {
|
||||
/** Store an object; the stream is read to EOF. Must not leave a partial object behind on failure. */
|
||||
public function put(string $path, $stream): void;
|
||||
|
||||
/** @return resource readable stream of the whole object */
|
||||
public function get(string $path);
|
||||
|
||||
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
|
||||
public function getRange(string $path, int $offset, int $length): string;
|
||||
|
||||
public function exists(string $path): bool;
|
||||
|
||||
/** @return list<string> object paths below $prefix (recursive) */
|
||||
public function list(string $prefix): array;
|
||||
|
||||
public function delete(string $path): void;
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
final class LocalBackend implements IBackend {
|
||||
public function __construct(
|
||||
private string $root,
|
||||
) {
|
||||
$this->root = rtrim($root, '/');
|
||||
if (!is_dir($this->root) && !mkdir($this->root, 0700, true) && !is_dir($this->root)) {
|
||||
throw new BackendException('Cannot create ' . $this->root);
|
||||
}
|
||||
}
|
||||
|
||||
public function put(string $path, $stream): void {
|
||||
$target = $this->abs($path);
|
||||
$dir = dirname($target);
|
||||
if (!is_dir($dir) && !mkdir($dir, 0700, true) && !is_dir($dir)) {
|
||||
throw new BackendException('Cannot create ' . $dir);
|
||||
}
|
||||
// Write to a temporary name and rename, so a crash never leaves a partial object.
|
||||
$tmp = $target . '.part-' . bin2hex(random_bytes(4));
|
||||
$out = fopen($tmp, 'wb');
|
||||
if ($out === false) {
|
||||
throw new BackendException('Cannot write ' . $path);
|
||||
}
|
||||
try {
|
||||
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
|
||||
throw new BackendException('Write failed for ' . $path);
|
||||
}
|
||||
} finally {
|
||||
fclose($out);
|
||||
}
|
||||
if (!rename($tmp, $target)) {
|
||||
@unlink($tmp);
|
||||
throw new BackendException('Cannot finalise ' . $path);
|
||||
}
|
||||
}
|
||||
|
||||
public function get(string $path) {
|
||||
$fh = @fopen($this->abs($path), 'rb');
|
||||
if ($fh === false) {
|
||||
throw new BackendException('Not found: ' . $path);
|
||||
}
|
||||
return $fh;
|
||||
}
|
||||
|
||||
public function getRange(string $path, int $offset, int $length): string {
|
||||
$fh = $this->get($path);
|
||||
try {
|
||||
if (fseek($fh, $offset) !== 0) {
|
||||
throw new BackendException('Seek failed in ' . $path);
|
||||
}
|
||||
$data = '';
|
||||
while (strlen($data) < $length && !feof($fh)) {
|
||||
$data .= (string)fread($fh, $length - strlen($data));
|
||||
}
|
||||
if (strlen($data) !== $length) {
|
||||
throw new BackendException('Short read in ' . $path);
|
||||
}
|
||||
return $data;
|
||||
} finally {
|
||||
fclose($fh);
|
||||
}
|
||||
}
|
||||
|
||||
public function exists(string $path): bool {
|
||||
return is_file($this->abs($path));
|
||||
}
|
||||
|
||||
public function list(string $prefix): array {
|
||||
$base = $this->abs($prefix);
|
||||
if (!is_dir($base)) {
|
||||
return [];
|
||||
}
|
||||
$result = [];
|
||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
|
||||
foreach ($it as $file) {
|
||||
if ($file->isFile() && !str_contains($file->getFilename(), '.part-')) {
|
||||
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
|
||||
}
|
||||
}
|
||||
sort($result);
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
@unlink($this->abs($path));
|
||||
}
|
||||
|
||||
private function abs(string $path): string {
|
||||
if (str_contains($path, '..') || str_starts_with($path, '/')) {
|
||||
throw new BackendException('Invalid path: ' . $path);
|
||||
}
|
||||
return $this->root . '/' . $path;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user