From 11ac30635a1cd5cd6a5826b68cd178c68041a849 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Sun, 18 Oct 2026 22:14:45 +0200 Subject: [PATCH] target:add: show progress and time left while reading the index of an existing repository --- CHANGELOG.md | 3 +- docs/DISASTER-RECOVERY.md | 5 +-- lib/Command/TargetAdd.php | 40 ++++++++++++++++++++++-- lib/Repository/Repository.php | 5 +-- lib/Service/TargetService.php | 5 +-- tests/Unit/Repository/IndexCacheTest.php | 18 +++++++++++ 6 files changed, 66 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96323af..3018ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,8 @@ All-in-One onto a plain Nextcloud with PostgreSQL). restore commits. - `occ backup:target:add` without a key pointed to `backup:key:init`, which makes a new key that cannot read existing backups. It now points to `backup:key:import` for a restore, and says that - connecting to a location with backups can take a while. + connecting to a location with backups can take a while. While it reads the index of a location + with backups, it shows how far it is and about how long it still takes. - A restore stopped with "Not found: packs/..." when the location was cleaned up while it ran, for example by the weekly cleanup of the original server during a recovery test on a second server: blobs that were moved into new packs were still looked for in the old ones. A restore diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md index 93b1717..fd7bdf1 100644 --- a/docs/DISASTER-RECOVERY.md +++ b/docs/DISASTER-RECOVERY.md @@ -59,8 +59,9 @@ Import it; you type the passphrase yourself: It shows the key fingerprint: it must match the one of your old server (on the recovery kit). -**5. Add the backup location** with the same details as before. This can take a while on a large -backup (the index is read first) and shows nothing until it is done. +**5. Add the backup location** with the same details as before. On a large backup this first reads +the index and shows how far it is, for example +`Reading the index: 6,913/26,791 files (25%), about 12 min left`. occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=... -o root=... --path=... diff --git a/lib/Command/TargetAdd.php b/lib/Command/TargetAdd.php index d91368c..b56d250 100644 --- a/lib/Command/TargetAdd.php +++ b/lib/Command/TargetAdd.php @@ -64,11 +64,15 @@ HELP); return 1; } // Adopting an existing repository reads its whole index first, which takes a while on a - // large backup or a slow line; say so instead of looking stuck. - $output->writeln('Connecting to the location; with existing backups this reads their index first and can take a while ...'); + // large backup or a slow line; show how far it is instead of looking stuck. + $output->writeln('Connecting to the location; with existing backups this reads their index first ...'); try { - $r = $this->targets->add($input->getArgument('name'), $input->getArgument('backend'), $input->getOption('auth'), $options, (string)$input->getOption('path')); + $r = $this->targets->add($input->getArgument('name'), $input->getArgument('backend'), $input->getOption('auth'), $options, + (string)$input->getOption('path'), self::progress($output)); } catch (\Throwable $e) { + if ($output->isDecorated()) { + $output->write("\r\033[K"); // clear an unfinished progress line + } $output->writeln('' . $e->getMessage() . ''); return 1; } @@ -76,4 +80,34 @@ HELP); $r['target']->getRepositoryId(), $r['created'] ? 'created' : 'found and adopted')); return 0; } + + /** + * Progress of reading the index: on a terminal one line that is updated in place, otherwise + * (a log, a pipe) a line every 30 seconds. The time left is estimated from the rate so far. + * + * @return callable(int, int): void + */ + private static function progress(OutputInterface $output): callable { + $start = microtime(true); + $shown = 0.0; + $inPlace = $output->isDecorated(); + return static function (int $done, int $total) use ($output, $start, &$shown, $inPlace): void { + $now = microtime(true); + if ($done < $total && $now - $shown < ($inPlace ? 1 : 30)) { + return; + } + $shown = $now; + $line = sprintf('Reading the index: %s/%s files (%d%%)', number_format($done), number_format($total), intdiv(100 * $done, max(1, $total))); + $elapsed = $now - $start; + if ($done < $total && $done > 0 && $elapsed >= 5) { + $left = (int)ceil($elapsed / (float)$done * (float)($total - $done) / 60.0); + $line .= $left <= 1 ? ', less than a minute left' : sprintf(', about %d min left', $left); + } + if ($inPlace) { + $output->write("\r\033[K" . $line . ($done >= $total ? "\n" : '')); + } else { + $output->writeln($line); + } + }; + } } diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index fdc6be5..83b4abc 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -81,7 +81,8 @@ final class Repository { } /** Open with the master key the server holds (scheduled backups, no passphrase needed). */ - public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null): self { + /** @param ?callable(int, int): void $progress index files downloaded so far, and in all (see BlobIndex::load()) */ + public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null, ?callable $progress = null): self { $config = self::readConfig($backend); try { $ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id']; @@ -95,7 +96,7 @@ final class Repository { if ($cache !== null) { $repo->index->setCache($cache, $config['id']); } - $repo->index->load(); + $repo->index->load(false, $progress); $repo->loadCatalog($anchor); return $repo; } diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php index 80d602e..b0e914e 100644 --- a/lib/Service/TargetService.php +++ b/lib/Service/TargetService.php @@ -48,15 +48,16 @@ final class TargetService { * installation's key), then save it. * * @param array $options + * @param ?callable(int, int): void $progress reading an existing repository's index: files done, and in all * @return array{target: Target, created: bool} */ - public function add(string $name, string $backend, string $auth, array $options, string $basePath): array { + public function add(string $name, string $backend, string $auth, array $options, string $basePath, ?callable $progress = null): array { $this->validateName($name); $this->requireUsableBackend($backend); $keys = $this->keys->keyRing(); $storageBackend = $this->buildBackend($backend, $auth, $options, $basePath); if ($storageBackend->exists('config')) { - $repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor); + $repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor, $progress); $created = false; } else { $repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor); diff --git a/tests/Unit/Repository/IndexCacheTest.php b/tests/Unit/Repository/IndexCacheTest.php index 3383d63..ff2b112 100644 --- a/tests/Unit/Repository/IndexCacheTest.php +++ b/tests/Unit/Repository/IndexCacheTest.php @@ -54,6 +54,24 @@ class IndexCacheTest extends TestCase { $this->assertCount(12, self::hashTree($target)); } + /** Connecting a new server to a location with backups reports how far reading the index is. */ + public function testOpeningReportsIndexProgress(): void { + $src = $this->tempDir(); + for ($i = 0; $i < 6; $i++) { + file_put_contents("$src/f$i", random_bytes(3 * 1048576)); + } + $backend = new LocalBackend($this->tempDir()); + $keys = KeyRing::generate(); + Repository::initWithKey($backend, $keys, $keys->wrap('pw'))->backupDirectory($src); + $files = count($backend->list('index')); + $seen = []; + Repository::openWithKey($backend, $keys, null, null, function (int $done, int $total) use (&$seen): void { + $seen[] = [$done, $total]; + }); + $this->assertCount($files, $seen); + $this->assertSame([$files, $files], end($seen)); + } + /** The database dump reads in a read-only transaction: a pack that fills up then must not write the cache yet. */ public function testHeldCacheWritesWaitForRelease(): void { $cache = new class implements IIndexCache {