diff --git a/appinfo/info.xml b/appinfo/info.xml
index 99cc4e4..90b064f 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -51,6 +51,7 @@ relying on this for anything.]]>
OCA\NgBackup\Command\RestoreUser
OCA\NgBackup\Command\KeyImport
OCA\NgBackup\Command\RestoreFull
+ OCA\NgBackup\Command\Verify
OCA\NgBackup\Settings\Admin
diff --git a/lib/Command/Verify.php b/lib/Command/Verify.php
new file mode 100644
index 0000000..bae03fb
--- /dev/null
+++ b/lib/Command/Verify.php
@@ -0,0 +1,71 @@
+setName('backup:verify')
+ ->setDescription('Checksum audit of a snapshot: every blob it needs must be in the (freshly re-checked) index and its pack present')
+ ->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
+ ->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
+ ->addOption('deep', null, InputOption::VALUE_NONE, 'Also download and decrypt every blob (verifies its authentication tag and content hash, not just that it is listed); much slower')
+ ->setHelp(<<<'HELP'
+Without --deep this re-downloads and re-authenticates the index (never trusting the local cache)
+and confirms every blob the snapshot needs is listed and its pack is present -- fast, no blob
+content is read, so it cannot by itself catch a truncated or bit-rotted pack. With --deep every
+blob is also downloaded and decrypted, which fails loudly on exactly that even if the index still
+lists it -- the same verification an actual restore would do, run here deliberately instead of
+waiting to find out during one.
+
+Run this periodically (cron, picking a sample of recent snapshots) rather than only right before
+you'd need to rely on a restore.
+
+Examples:
+ occ backup:verify offsite 537bf7e9c1a4...
+ occ backup:verify offsite 537bf7e9c1a4... --deep
+HELP);
+ }
+
+ protected function execute(InputInterface $input, OutputInterface $output): int {
+ try {
+ $target = $this->targets->get($input->getArgument('target'));
+ $result = $this->verify->verify($target, (string)$input->getArgument('snapshot'), (bool)$input->getOption('deep'));
+ } catch (\Throwable $e) {
+ $output->writeln('' . $e->getMessage() . '');
+ return 1;
+ }
+ $output->writeln(sprintf('%d files, %d blobs checked%s.', $result['filesChecked'], $result['blobsChecked'],
+ $input->getOption('deep') ? sprintf(' (%.1f MiB decrypted)', $result['bytesChecked'] / 1048576) : ''));
+ if ($result['missing'] !== []) {
+ $output->writeln('' . count($result['missing']) . ' blob(s) missing from the index: ' . implode(', ', $result['missing']));
+ }
+ if ($result['failed'] !== []) {
+ $output->writeln('' . count($result['failed']) . ' blob(s) failed verification (corrupted or tampered): ' . implode(', ', $result['failed']));
+ }
+ if ($result['missing'] === [] && $result['failed'] === []) {
+ $output->writeln('OK');
+ return 0;
+ }
+ return 1;
+ }
+}
diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php
index ad65baf..2d83758 100644
--- a/lib/Repository/Repository.php
+++ b/lib/Repository/Repository.php
@@ -662,6 +662,83 @@ final class Repository {
return $this->streamEntries($snapshotId);
}
+ /**
+ * Checksum audit of a snapshot: every blob it references (files and the database dump, when
+ * the snapshot has one) must be in the index, freshly re-downloaded and re-authenticated from
+ * the location rather than trusted from the local cache (same reasoning as prune()), and its
+ * pack must still be present. With $deep, each blob is also downloaded and decrypted, which
+ * verifies its AEAD authentication tag and its content hash against the blob id (loadBlob()
+ * already does both on every real read -- this just does it for everything in the snapshot,
+ * deliberately, rather than relying on it to come up during a later restore). A blob
+ * referenced by more than one file is only checked once; a missing database manifest is a
+ * failure, not something to silently skip over.
+ *
+ * @param ?callable $heartbeat called after every blob (shallow) or every blob read (deep);
+ * callers use it to keep a time-boxed lease alive during a large audit
+ * @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list, failed:list}
+ */
+ public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
+ $heartbeat ??= static function (): void {
+ };
+ $this->index->load(true);
+ $stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => []];
+ $seen = [];
+ $packSeen = []; // pack id => exists on the backend; avoids one exists() call per blob
+ foreach ($this->streamEntries($snapshotId) as $entry) {
+ $stats['filesChecked']++;
+ foreach ($entry['b'] as $id) {
+ if (isset($seen[$id])) {
+ continue;
+ }
+ $seen[$id] = true;
+ $this->verifyBlob($id, $deep, $stats, $packSeen);
+ $heartbeat();
+ }
+ }
+ $meta = $this->snapshotMeta($snapshotId);
+ if (isset($meta['db']) && is_string($meta['db'])) {
+ if (!$this->backend->exists($meta['db'])) {
+ $stats['missing'][] = $meta['db'];
+ } else {
+ $dbManifest = json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
+ foreach ($dbManifest['tables'] as $info) {
+ foreach ($info['blobs'] as $id) {
+ if (isset($seen[$id])) {
+ continue;
+ }
+ $seen[$id] = true;
+ $this->verifyBlob($id, $deep, $stats, $packSeen);
+ $heartbeat();
+ }
+ }
+ }
+ }
+ return $stats;
+ }
+
+ /** @param array $packSeen */
+ private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen): void {
+ if (!$this->index->has($id)) {
+ $stats['missing'][] = $id;
+ return;
+ }
+ [$pack] = $this->index->get($id);
+ $packExists = $packSeen[$pack] ??= $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
+ if (!$packExists) {
+ $stats['missing'][] = $id;
+ return;
+ }
+ $stats['blobsChecked']++;
+ if (!$deep) {
+ return;
+ }
+ try {
+ $stats['bytesChecked'] += strlen($this->loadBlob($id));
+ } catch (\Throwable) {
+ $stats['failed'][] = $id;
+ }
+ }
+
/** The meta record (first line) of a snapshot. */
public function snapshotMeta(string $snapshotId): array {
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
diff --git a/lib/Service/VerifyService.php b/lib/Service/VerifyService.php
new file mode 100644
index 0000000..ef3d893
--- /dev/null
+++ b/lib/Service/VerifyService.php
@@ -0,0 +1,39 @@
+, failed:list} */
+ public function verify(Target $target, string $snapshotId, bool $deep = false): array {
+ $ttl = $deep ? 600 : 300;
+ try {
+ return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, function (callable $refresh) use ($target, $snapshotId, $deep, $ttl) {
+ $last = time();
+ $heartbeat = function () use ($refresh, $ttl, &$last): void {
+ if (time() - $last >= intdiv($ttl, 2)) {
+ $refresh();
+ $last = time();
+ }
+ };
+ return $this->targets->repository($target)->verify($snapshotId, $deep, $heartbeat);
+ });
+ } catch (LockedException) {
+ throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
+ }
+ }
+}
diff --git a/tests/Unit/Repository/VerifyTest.php b/tests/Unit/Repository/VerifyTest.php
new file mode 100644
index 0000000..7b9ed77
--- /dev/null
+++ b/tests/Unit/Repository/VerifyTest.php
@@ -0,0 +1,95 @@
+root = $this->tempDir();
+ $this->backend = new LocalBackend($this->root);
+ $this->keys = KeyRing::generate();
+ $this->src = $this->tempDir();
+ file_put_contents("$this->src/a", random_bytes(100000));
+ file_put_contents("$this->src/b", random_bytes(100000));
+ }
+
+ private function open(): Repository {
+ return Repository::openWithKey($this->backend, $this->keys, null, null);
+ }
+
+ public function testShallowVerifyPassesOnAHealthyRepository(): void {
+ $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
+ $s = $repo->backupDirectory($this->src);
+
+ $result = $this->open()->verify($s['snapshot']);
+
+ $this->assertSame(2, $result['filesChecked']);
+ $this->assertGreaterThanOrEqual(1, $result['blobsChecked']);
+ $this->assertSame(0, $result['bytesChecked']); // not downloaded without --deep
+ $this->assertSame([], $result['missing']);
+ $this->assertSame([], $result['failed']);
+ }
+
+ public function testDeepVerifyDownloadsAndCountsBytes(): void {
+ $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
+ $s = $repo->backupDirectory($this->src);
+
+ $result = $this->open()->verify($s['snapshot'], deep: true);
+
+ $this->assertSame(200000, $result['bytesChecked']);
+ $this->assertSame([], $result['missing']);
+ $this->assertSame([], $result['failed']);
+ }
+
+ public function testShallowVerifyDetectsAMissingPack(): void {
+ $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
+ $s = $repo->backupDirectory($this->src);
+ foreach ($this->backend->list('packs') as $pack) {
+ $this->backend->delete($pack);
+ }
+
+ $result = $this->open()->verify($s['snapshot']);
+
+ $this->assertNotSame([], $result['missing']);
+ }
+
+ public function testVerifyFailsOnAMissingDatabaseManifest(): void {
+ $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
+ $repo->writeSnapshot('snap-no-db', ['time' => gmdate('c'), 'db' => 'db/does-not-exist'], []);
+
+ $result = $this->open()->verify('snap-no-db');
+
+ $this->assertSame(['db/does-not-exist'], $result['missing']);
+ }
+
+ public function testDeepVerifyDetectsATamperedPack(): void {
+ $repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
+ $s = $repo->backupDirectory($this->src);
+ $packs = $this->backend->list('packs');
+ $this->assertNotEmpty($packs);
+ $path = $this->root . '/' . $packs[0];
+ $bytes = file_get_contents($path);
+ // Flip a byte well inside the file, away from any header, so decryption's own
+ // authentication tag catches it (this must fail loudly, not just "do nothing").
+ $bytes[(int)(strlen($bytes) / 2)] = chr(~ord($bytes[(int)(strlen($bytes) / 2)]) & 0xFF);
+ file_put_contents($path, $bytes);
+
+ $result = $this->open()->verify($s['snapshot'], deep: true);
+
+ $this->assertNotSame([], $result['failed']);
+ }
+}
diff --git a/tests/integration/roundtrip.sh b/tests/integration/roundtrip.sh
index b94ce2a..3b76b80 100755
--- a/tests/integration/roundtrip.sh
+++ b/tests/integration/roundtrip.sh
@@ -49,6 +49,11 @@ SUM_AFTER=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xa
$X sh -c "ls $DATA/$U/files_trashbin/files/ | grep -q '^extra.txt'" || fail "extra file not in trash"
ok "restore new-folder and replace: files identical, extra file in the trash"
+# Checksum audit: shallow (index presence) and deep (download, decrypt, verify every blob).
+occ backup:verify ci "$SNAP" | grep -q "^OK$" || fail "shallow verify"
+occ backup:verify ci "$SNAP" --deep | grep -q "^OK$" || fail "deep verify"
+ok "shallow and deep checksum audit"
+
# Database dump/restore round trip (all tables, checksums, sequences on PostgreSQL).
$X php -d memory_limit=512M "$APPS/ng_backup/spikes/03-db-dump.php" /tmp/ngb-rt-db | grep "ALL PASS" >/dev/null || fail "database round trip"
ok "database dump and restore"