From 62e365350fcea8bb92c70a45ae1c73fbfd127f7e Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Fri, 2 Oct 2026 11:30:42 +0200 Subject: [PATCH] M5: chained catalog against rollback and hidden snapshots Every change to the set of snapshots or trash entries writes an encrypted catalog generation bound to the repository and its position, with the hash of the previous generation. The server anchors the newest generation it has seen in its app config; on open, the location must show at least that generation and chain back to it, and every snapshot listed must exist. A rolled-back or tampered location is refused (the run fails and administrators are notified). Opening with the passphrase on a new server accepts the newest valid chain. occ backup:target:trust-current accepts a legitimately older state (e.g. restored from the provider's snapshot) after typing the location name; audited and announced to all administrators. Tests: chain advances, rollback, hidden snapshot, swapped generation, disaster recovery without anchor. Tested on NC34 by deleting the newest catalog generation on the SFTP location. --- appinfo/info.xml | 3 +- lib/Command/TargetTrustCurrent.php | 57 ++++++++++++ lib/Db/AppConfigCatalogAnchor.php | 34 +++++++ lib/Notification/Notifier.php | 1 + lib/Repository/Catalog.php | 129 ++++++++++++++++++++++++++ lib/Repository/ICatalogAnchor.php | 20 ++++ lib/Repository/Repository.php | 62 +++++++++++-- lib/Service/TargetService.php | 7 +- tests/Unit/Repository/CatalogTest.php | 90 ++++++++++++++++++ 9 files changed, 392 insertions(+), 11 deletions(-) create mode 100644 lib/Command/TargetTrustCurrent.php create mode 100644 lib/Db/AppConfigCatalogAnchor.php create mode 100644 lib/Repository/Catalog.php create mode 100644 lib/Repository/ICatalogAnchor.php create mode 100644 tests/Unit/Repository/CatalogTest.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 12484c4..1bbef8c 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -5,7 +5,7 @@ NG Backup Encrypted, incremental backups of your Nextcloud with restore from the web interface - 0.0.13 + 0.0.15 agpl Dillard Blom NgBackup @@ -35,6 +35,7 @@ OCA\NgBackup\Command\Prune OCA\NgBackup\Command\Retention OCA\NgBackup\Command\TargetAppendOnly + OCA\NgBackup\Command\TargetTrustCurrent OCA\NgBackup\Command\Trash OCA\NgBackup\Command\TrashRestore OCA\NgBackup\Command\KeySlots diff --git a/lib/Command/TargetTrustCurrent.php b/lib/Command/TargetTrustCurrent.php new file mode 100644 index 0000000..3618d6c --- /dev/null +++ b/lib/Command/TargetTrustCurrent.php @@ -0,0 +1,57 @@ +setName('backup:target:trust-current') + ->setDescription('Accept the current state of a location that was refused as rolled back') + ->addArgument('target', InputArgument::REQUIRED, 'Location name or id') + ->setHelp("NG Backup refuses a location whose backup catalog is older than what this server saw\n" + . "before, because an attacker could have restored an old state. If you restored the location\n" + . "yourself (e.g. from the provider's snapshot), accept it with this command."); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $t = $this->targets->get($input->getArgument('target')); + $output->writeln('Only continue if you know why the location shows an older state.'); + $answer = (string)$this->getHelper('question')->ask($input, $output, new Question('Type the location name to accept its current state: ')); + if ($answer !== $t->getName()) { + $output->writeln('Not confirmed.'); + return 1; + } + $this->appConfig->deleteKey('ng_backup', 'catalog_' . preg_replace('/[^a-f0-9]/', '', (string)$t->getRepositoryId())); + $repo = $this->targets->repository($t); // re-anchors on the newest valid catalog + $this->alerts->securityEvent('catalog_trusted', ['target' => $t->getName(), 'generation' => (string)$repo->catalogGeneration()]); + $output->writeln(sprintf('Accepted generation %d of %s (%d snapshots).', $repo->catalogGeneration(), $t->getName(), count($repo->listSnapshots()))); + return 0; + } +} diff --git a/lib/Db/AppConfigCatalogAnchor.php b/lib/Db/AppConfigCatalogAnchor.php new file mode 100644 index 0000000..a7ad178 --- /dev/null +++ b/lib/Db/AppConfigCatalogAnchor.php @@ -0,0 +1,34 @@ +appConfig->getValueString(Application::APP_ID, $this->key($repositoryId), '', true); + $data = $raw === '' ? null : json_decode($raw, true); + return is_array($data) && isset($data['gen'], $data['hash']) ? ['gen' => (int)$data['gen'], 'hash' => (string)$data['hash']] : null; + } + + public function set(string $repositoryId, int $gen, string $hash): void { + $this->appConfig->setValueString(Application::APP_ID, $this->key($repositoryId), json_encode(['gen' => $gen, 'hash' => $hash]), true); + } + + private function key(string $repositoryId): string { + return 'catalog_' . preg_replace('/[^a-f0-9]/', '', $repositoryId); + } +} diff --git a/lib/Notification/Notifier.php b/lib/Notification/Notifier.php index 906e821..2a47c45 100644 --- a/lib/Notification/Notifier.php +++ b/lib/Notification/Notifier.php @@ -55,6 +55,7 @@ class Notifier implements INotifier { 'retention_changed' => $l->t('%1$s changed the backup retention to %2$s', [$who, $p['policy'] ?? '']), 'snapshots_forgotten' => $l->t('%1$s moved %2$s snapshots of %3$s to the trash; they are deleted permanently after %4$s unless restored', [$who, $p['count'] ?? '', $p['target'] ?? '', $p['until'] ?? '']), 'snapshot_untrashed' => $l->t('%1$s restored snapshot %2$s of %3$s from the trash', [$who, $p['snapshot'] ?? '', $p['target'] ?? '']), + 'catalog_trusted' => $l->t('%1$s accepted an older state of backup location %2$s (catalog generation %3$s)', [$who, $p['target'] ?? '', $p['generation'] ?? '']), 'slot_added' => $l->t('%1$s added backup passphrase slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']), 'slot_replaced' => $l->t('%1$s set a new passphrase for backup key slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']), 'slot_removed' => $l->t('%1$s removed backup passphrase slot %2$s; download and confirm the new recovery kit', [$who, $p['slot'] ?? '']), diff --git a/lib/Repository/Catalog.php b/lib/Repository/Catalog.php new file mode 100644 index 0000000..56ec8fd --- /dev/null +++ b/lib/Repository/Catalog.php @@ -0,0 +1,129 @@ +: the + * encrypted list of snapshot and trash ids plus the hash of the previous generation. The server + * keeps the newest generation it has seen (ICatalogAnchor). On open, the newest catalog on the + * location must be at least the anchored generation and lead back to it through the chain; + * snapshots listed in the catalog must exist. Otherwise the location was rolled back or + * tampered with, and the repository is refused. + */ +final class Catalog { + private const KEEP = 200; // generations kept on the location (enough to verify long gaps) + + public int $gen = 0; + /** @var array */ + public array $snapshots = []; + /** @var array */ + public array $trash = []; + public string $hash = ''; + + public function __construct( + private IBackend $backend, + private StreamCipher $cipher, + private string $repositoryId, + private ?ICatalogAnchor $anchor, + ) { + } + + /** Load and verify the newest catalog; create the first one for repositories that have none. */ + public function load(callable $listSnapshots, callable $listTrash): void { + $gens = $this->generations(); + if ($gens === []) { + // First use (new or pre-catalog repository): adopt the current state as generation 1. + $anchored = $this->anchor?->get($this->repositoryId); + if ($anchored !== null) { + throw new RepositoryException('The catalog of this repository has disappeared from the location (possible tampering)'); + } + $this->snapshots = array_fill_keys($listSnapshots(), true); + $this->trash = array_fill_keys($listTrash(), true); + $this->write(); + return; + } + $head = max($gens); + [$data, $hash] = $this->read($head); + $anchored = $this->anchor?->get($this->repositoryId); + if ($anchored !== null) { + if ($head < $anchored['gen']) { + throw new RepositoryException(sprintf('The location shows catalog generation %d, but this server has seen %d: the backups were rolled back', $head, $anchored['gen'])); + } + // Walk back to the anchored generation, checking every link. + [$cur, $curHash] = [$data, $hash]; + for ($g = $head; $g > $anchored['gen']; $g--) { + [$prev, $prevHash] = $this->read($g - 1); + if (!hash_equals($prevHash, (string)$cur['prev'])) { + throw new RepositoryException("Catalog chain broken at generation $g (possible tampering)"); + } + [$cur, $curHash] = [$prev, $prevHash]; + } + if (!hash_equals($anchored['hash'], $curHash)) { + throw new RepositoryException('The catalog does not match what this server saw before (possible tampering)'); + } + } + $this->gen = $head; + $this->hash = $hash; + $this->snapshots = array_fill_keys($data['snapshots'], true); + $this->trash = array_fill_keys($data['trash'], true); + $this->anchor?->set($this->repositoryId, $this->gen, $this->hash); + } + + /** Write the next generation after the in-memory state was changed. */ + public function write(): void { + $next = $this->gen + 1; + $plain = json_encode(['repository' => $this->repositoryId, 'gen' => $next, 'time' => gmdate('c'), 'prev' => $this->hash, + 'snapshots' => array_keys($this->snapshots), 'trash' => array_keys($this->trash)], JSON_THROW_ON_ERROR); + $encrypted = $this->cipher->encryptString($plain, $this->ad($next)); + $stream = fopen('php://memory', 'w+b'); + fwrite($stream, $encrypted); + rewind($stream); + $this->backend->put($this->path($next), $stream); + $this->gen = $next; + $this->hash = hash('sha256', $encrypted); + $this->anchor?->set($this->repositoryId, $this->gen, $this->hash); + $old = $next - self::KEEP; + if ($old > 0) { + $this->backend->delete($this->path($old)); + } + } + + /** @return array{0: array, 1: string} decoded catalog and the hash of its encrypted bytes */ + private function read(int $gen): array { + try { + $fh = $this->backend->get($this->path($gen)); + } catch (\Throwable) { + throw new RepositoryException("Catalog generation $gen is missing (possible tampering)"); + } + $encrypted = (string)stream_get_contents($fh); + fclose($fh); + $data = json_decode($this->cipher->decryptString($encrypted, $this->ad($gen)), true, 512, JSON_THROW_ON_ERROR); + if (($data['repository'] ?? '') !== $this->repositoryId || ($data['gen'] ?? 0) !== $gen) { + throw new RepositoryException("Catalog generation $gen belongs to another repository or position"); + } + return [$data, hash('sha256', $encrypted)]; + } + + /** @return list */ + private function generations(): array { + return array_map(fn ($p) => (int)basename($p), $this->backend->list('catalog')); + } + + private function path(int $gen): string { + return sprintf('catalog/%010d', $gen); + } + + private function ad(int $gen): string { + return 'catalog:' . $this->repositoryId . ':' . $gen; + } +} diff --git a/lib/Repository/ICatalogAnchor.php b/lib/Repository/ICatalogAnchor.php new file mode 100644 index 0000000..4c49e5e --- /dev/null +++ b/lib/Repository/ICatalogAnchor.php @@ -0,0 +1,20 @@ +exists('config')) { throw new RepositoryException('A repository already exists at this location'); } @@ -60,11 +61,13 @@ final class Repository { $config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey, 'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))]; self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); - return new self($backend, $keys, $id); + $repo = new self($backend, $keys, $id); + $repo->loadCatalog($anchor); + return $repo; } /** Open with the master key the server holds (scheduled backups, no passphrase needed). */ - public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null): self { + public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null): self { $config = self::readConfig($backend); try { $ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id']; @@ -79,9 +82,22 @@ final class Repository { $repo->index->setCache($cache, $config['id']); } $repo->index->load(); + $repo->loadCatalog($anchor); return $repo; } + private function loadCatalog(?ICatalogAnchor $anchor): void { + $this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor); + $this->catalog->load( + fn () => array_map('basename', $this->backend->list('snapshots')), + fn () => array_map('basename', $this->backend->list('trash/info'))); + } + + /** Current catalog generation (for status and tests). */ + public function catalogGeneration(): int { + return $this->catalog->gen; + } + /** Index files downloaded when this repository was opened (0 when the local cache was complete). */ public function indexDownloads(): int { return $this->index->downloaded; @@ -116,6 +132,7 @@ final class Repository { $config = self::readConfig($backend); $repo = new self($backend, KeyRing::unwrapAny($config['key'], $passphrase), $config['id']); $repo->index->load(); + $repo->loadCatalog(null); // disaster recovery on a new server: no anchor yet, trust the newest valid chain return $repo; } @@ -156,6 +173,8 @@ final class Repository { rewind($encrypted); $this->backend->put('snapshots/' . $snapshotId, $encrypted); fclose($tree); + $this->catalog->snapshots[$snapshotId] = true; + $this->catalog->write(); return ['snapshot' => $snapshotId] + $stats; } @@ -263,6 +282,8 @@ final class Repository { fclose($tree); rewind($encrypted); $this->backend->put('snapshots/' . $snapshotId, $encrypted); + $this->catalog->snapshots[$snapshotId] = true; + $this->catalog->write(); } /** @return array}> */ @@ -321,6 +342,9 @@ final class Repository { $this->backend->put('trash/snapshots/' . $snapshotId, $this->backend->get('snapshots/' . $snapshotId)); $this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null, 'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR)); + unset($this->catalog->snapshots[$snapshotId]); + $this->catalog->trash[$snapshotId] = true; + $this->catalog->write(); if ($dbInTrash !== null) { $this->backend->delete($meta['db']); } @@ -335,6 +359,9 @@ final class Repository { $this->backend->delete($info['dbInTrash']); } $this->backend->put('snapshots/' . $snapshotId, $this->backend->get('trash/snapshots/' . $snapshotId)); + $this->catalog->snapshots[$snapshotId] = true; + unset($this->catalog->trash[$snapshotId]); + $this->catalog->write(); $this->backend->delete('trash/snapshots/' . $snapshotId); $this->backend->delete('trash/info/' . $snapshotId); } @@ -342,8 +369,12 @@ final class Repository { /** @return array */ public function trash(): array { $out = []; - foreach ($this->backend->list('trash/info') as $path) { - $out[basename($path)] = $this->trashInfo(basename($path)); + foreach (array_keys($this->catalog->trash) as $id) { + try { + $out[$id] = $this->trashInfo($id); + } catch (\Throwable) { + throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)"); + } } return $out; } @@ -360,8 +391,12 @@ final class Repository { } $this->backend->delete('trash/snapshots/' . $id); $this->backend->delete('trash/info/' . $id); + unset($this->catalog->trash[$id]); $purged[] = $id; } + if ($purged !== []) { + $this->catalog->write(); + } return $purged; } @@ -470,9 +505,22 @@ final class Repository { return $stats; } - /** @return list */ + /** + * Snapshots according to the verified catalog. A snapshot the catalog lists but the location + * no longer has means it was deleted or hidden behind NG Backup's back. + * + * @return list + */ public function listSnapshots(): array { - return array_map('basename', $this->backend->list('snapshots')); + $present = array_flip(array_map('basename', $this->backend->list('snapshots'))); + $listed = array_keys($this->catalog->snapshots); + foreach ($listed as $id) { + if (!isset($present[$id])) { + throw new RepositoryException("Snapshot $id is missing on the location (deleted or hidden outside NG Backup)"); + } + } + sort($listed); + return $listed; } public function blobCount(): int { diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php index 2a3e499..889a209 100644 --- a/lib/Service/TargetService.php +++ b/lib/Service/TargetService.php @@ -29,6 +29,7 @@ class TargetService { private ICrypto $crypto, private IAppManager $appManager, private \OCA\NgBackup\Db\DbIndexCache $indexCache, + private \OCA\NgBackup\Db\AppConfigCatalogAnchor $anchor, ) { } @@ -56,10 +57,10 @@ class TargetService { $keys = $this->keys->keyRing(); $storageBackend = $this->buildBackend($backend, $auth, $options, $basePath); if ($storageBackend->exists('config')) { - $repo = Repository::openWithKey($storageBackend, $keys); + $repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor); $created = false; } else { - $repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey()); + $repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor); $created = true; } @@ -104,7 +105,7 @@ class TargetService { } public function repository(Target $target): Repository { - $repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache); + $repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache, $this->anchor); if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) { throw new \RuntimeException('The target now contains a different repository than when it was added'); } diff --git a/tests/Unit/Repository/CatalogTest.php b/tests/Unit/Repository/CatalogTest.php new file mode 100644 index 0000000..c287c09 --- /dev/null +++ b/tests/Unit/Repository/CatalogTest.php @@ -0,0 +1,90 @@ +root = $this->tempDir(); + $this->backend = new LocalBackend($this->root); + $this->keys = KeyRing::generate(); + $this->anchor = new class implements ICatalogAnchor { + public array $data = []; + public function get(string $r): ?array { return $this->data[$r] ?? null; } + public function set(string $r, int $g, string $h): void { $this->data[$r] = ['gen' => $g, 'hash' => $h]; } + }; + $this->src = $this->tempDir(); + file_put_contents("$this->src/a", random_bytes(100000)); + } + + private function open(): Repository { + return Repository::openWithKey($this->backend, $this->keys, null, $this->anchor); + } + + public function testNormalUseAdvancesTheChain(): void { + $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor); + $s1 = $repo->backupDirectory($this->src); + file_put_contents("$this->src/b", 'x'); + $s2 = $repo->backupDirectory($this->src, $s1['snapshot']); + $repo->forget($s1['snapshot']); + $again = $this->open(); + $this->assertSame([$s2['snapshot']], $again->listSnapshots()); + $this->assertArrayHasKey($s1['snapshot'], $again->trash()); + $this->assertSame(4, $again->catalogGeneration()); + } + + public function testRollbackIsDetected(): void { + $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor); + $repo->backupDirectory($this->src); + $repo->backupDirectory($this->src); + // The location "loses" the newest generation and presents an older, validly encrypted state. + $catalogs = $this->backend->list('catalog'); + unlink($this->root . '/' . end($catalogs)); + $this->expectException(RepositoryException::class); + $this->expectExceptionMessage('rolled back'); + $this->open(); + } + + public function testHiddenSnapshotIsDetected(): void { + $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor); + $s = $repo->backupDirectory($this->src); + unlink($this->root . '/snapshots/' . $s['snapshot']); + $this->expectException(RepositoryException::class); + $this->open()->listSnapshots(); + } + + public function testSwappedGenerationIsRejected(): void { + $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor); + $repo->backupDirectory($this->src); + $repo->backupDirectory($this->src); + $c = $this->backend->list('catalog'); + // Put generation 2's bytes where generation 3 should be. + copy($this->root . '/' . $c[1], $this->root . '/' . $c[2]); + $this->expectException(\RuntimeException::class); + $this->open(); + } + + public function testWithoutAnchorTheNewestChainIsAccepted(): void { + $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw')); + $s = $repo->backupDirectory($this->src); + // Disaster recovery on a new server: opened with the passphrase, no anchor. + $this->assertSame([$s['snapshot']], Repository::open($this->backend, 'pw')->listSnapshots()); + } +}