Files
dillard 6bba0b59fc Repository: list flat folders without a type check per entry
Opening a repository lists index/, which holds one file per pack.
StorageBackend::list() checked every entry with is_dir(), and on a
remote storage each check is a round trip. Over SFTP to a Hetzner Storage
Box (33 ms) with a 60 GiB repository, occ backup:run (30 s steps) spent
nearly every step on these requests and uploaded 45 KiB/s; with this
change it uploads 5.8 MB/s. The cost grows with the number of packs.

index/, snapshots/, catalog/ and trash/info/ only ever hold objects:
list them with the new IBackend::listFiles(), one listing without
per-entry checks.
2026-10-08 18:28:29 +02:00

142 lines
5.8 KiB
PHP

<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Repository;
use OCA\NgBackup\Backend\IBackend;
use OCA\NgBackup\Crypto\StreamCipher;
/**
* Chained catalog of a repository's state, against rollback and hidden snapshots.
*
* Every change to the set of snapshots or trash entries writes catalog/<generation>: the
* encrypted list of snapshot and trash ids plus the hash of the previous generation. The server
* keeps the newest generation it has seen (ICatalogAnchor). On open, the newest catalog on the
* location must be at least the anchored generation and lead back to it through the chain;
* snapshots listed in the catalog must exist. Otherwise the location was rolled back or
* tampered with, and the repository is refused.
*/
final class Catalog {
private const KEEP = 200; // generations kept on the location (enough to verify long gaps)
public int $gen = 0;
/** @var array<string, true> */
public array $snapshots = [];
/** @var array<string, true> */
public array $trash = [];
/** @var array<string, true> manifest path => true, for a per-user export (users/<uid>/<exportId>) */
public array $userExports = [];
public string $hash = '';
public function __construct(
private IBackend $backend,
private StreamCipher $cipher,
private string $repositoryId,
private ?ICatalogAnchor $anchor,
) {
}
/** Load and verify the newest catalog; create the first one for repositories that have none. */
public function load(callable $listSnapshots, callable $listTrash, callable $listUserExports): void {
$gens = $this->generations();
if ($gens === []) {
// First use (new or pre-catalog repository): adopt the current state as generation 1.
$anchored = $this->anchor?->get($this->repositoryId);
if ($anchored !== null) {
throw new RepositoryException('The catalog of this repository has disappeared from the location (possible tampering)');
}
$this->snapshots = array_fill_keys($listSnapshots(), true);
$this->trash = array_fill_keys($listTrash(), true);
$this->userExports = array_fill_keys($listUserExports(), true);
$this->write();
return;
}
$head = max($gens);
[$data, $hash] = $this->read($head);
$anchored = $this->anchor?->get($this->repositoryId);
if ($anchored !== null) {
if ($head < $anchored['gen']) {
throw new RepositoryException(sprintf('The location shows catalog generation %d, but this server has seen %d: the backups were rolled back', $head, $anchored['gen']));
}
// Walk back to the anchored generation, checking every link.
[$cur, $curHash] = [$data, $hash];
for ($g = $head; $g > $anchored['gen']; $g--) {
[$prev, $prevHash] = $this->read($g - 1);
if (!hash_equals($prevHash, (string)$cur['prev'])) {
throw new RepositoryException("Catalog chain broken at generation $g (possible tampering)");
}
[$cur, $curHash] = [$prev, $prevHash];
}
if (!hash_equals($anchored['hash'], $curHash)) {
throw new RepositoryException('The catalog does not match what this server saw before (possible tampering)');
}
}
$this->gen = $head;
$this->hash = $hash;
$this->snapshots = array_fill_keys($data['snapshots'], true);
$this->trash = array_fill_keys($data['trash'], true);
// Catalogs written before user exports existed have no such key: default to empty rather
// than adopting the location's current users/ listing (that would skip verification).
$this->userExports = array_fill_keys($data['userExports'] ?? [], true);
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
}
/** Write the next generation after the in-memory state was changed. */
public function write(): void {
$next = $this->gen + 1;
// Two writers starting from the same generation would otherwise overwrite each other
// silently and break the chain for whichever server anchored the lost write.
if ($this->backend->exists($this->path($next))) {
throw new RepositoryException("Catalog generation $next was written by another process at the same time; try again");
}
$plain = json_encode(['repository' => $this->repositoryId, 'gen' => $next, 'time' => gmdate('c'), 'prev' => $this->hash,
'snapshots' => array_keys($this->snapshots), 'trash' => array_keys($this->trash),
'userExports' => array_keys($this->userExports)], JSON_THROW_ON_ERROR);
$encrypted = $this->cipher->encryptString($plain, $this->ad($next));
$stream = fopen('php://memory', 'w+b');
fwrite($stream, $encrypted);
rewind($stream);
$this->backend->put($this->path($next), $stream);
$this->gen = $next;
$this->hash = hash('sha256', $encrypted);
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
$old = $next - self::KEEP;
if ($old > 0) {
$this->backend->delete($this->path($old));
}
}
/** @return array{0: array, 1: string} decoded catalog and the hash of its encrypted bytes */
private function read(int $gen): array {
try {
$fh = $this->backend->get($this->path($gen));
} catch (\Throwable) {
throw new RepositoryException("Catalog generation $gen is missing (possible tampering)");
}
$encrypted = (string)stream_get_contents($fh);
fclose($fh);
$data = json_decode($this->cipher->decryptString($encrypted, $this->ad($gen)), true, 512, JSON_THROW_ON_ERROR);
if (($data['repository'] ?? '') !== $this->repositoryId || ($data['gen'] ?? 0) !== $gen) {
throw new RepositoryException("Catalog generation $gen belongs to another repository or position");
}
return [$data, hash('sha256', $encrypted)];
}
/** @return list<int> */
private function generations(): array {
return array_map(fn ($p) => (int)basename($p), $this->backend->listFiles('catalog'));
}
private function path(int $gen): string {
return sprintf('catalog/%010d', $gen);
}
private function ad(int $gen): string {
return 'catalog:' . $this->repositoryId . ':' . $gen;
}
}