diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96323af..3018ccf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,7 +38,8 @@ All-in-One onto a plain Nextcloud with PostgreSQL).
restore commits.
- `occ backup:target:add` without a key pointed to `backup:key:init`, which makes a new key that
cannot read existing backups. It now points to `backup:key:import` for a restore, and says that
- connecting to a location with backups can take a while.
+ connecting to a location with backups can take a while. While it reads the index of a location
+ with backups, it shows how far it is and about how long it still takes.
- A restore stopped with "Not found: packs/..." when the location was cleaned up while it ran,
for example by the weekly cleanup of the original server during a recovery test on a second
server: blobs that were moved into new packs were still looked for in the old ones. A restore
diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md
index 93b1717..fd7bdf1 100644
--- a/docs/DISASTER-RECOVERY.md
+++ b/docs/DISASTER-RECOVERY.md
@@ -59,8 +59,9 @@ Import it; you type the passphrase yourself:
It shows the key fingerprint: it must match the one of your old server (on the recovery kit).
-**5. Add the backup location** with the same details as before. This can take a while on a large
-backup (the index is read first) and shows nothing until it is done.
+**5. Add the backup location** with the same details as before. On a large backup this first reads
+the index and shows how far it is, for example
+`Reading the index: 6,913/26,791 files (25%), about 12 min left`.
occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=... -o root=... --path=...
diff --git a/lib/Command/TargetAdd.php b/lib/Command/TargetAdd.php
index d91368c..b56d250 100644
--- a/lib/Command/TargetAdd.php
+++ b/lib/Command/TargetAdd.php
@@ -64,11 +64,15 @@ HELP);
return 1;
}
// Adopting an existing repository reads its whole index first, which takes a while on a
- // large backup or a slow line; say so instead of looking stuck.
- $output->writeln('Connecting to the location; with existing backups this reads their index first and can take a while ...');
+ // large backup or a slow line; show how far it is instead of looking stuck.
+ $output->writeln('Connecting to the location; with existing backups this reads their index first ...');
try {
- $r = $this->targets->add($input->getArgument('name'), $input->getArgument('backend'), $input->getOption('auth'), $options, (string)$input->getOption('path'));
+ $r = $this->targets->add($input->getArgument('name'), $input->getArgument('backend'), $input->getOption('auth'), $options,
+ (string)$input->getOption('path'), self::progress($output));
} catch (\Throwable $e) {
+ if ($output->isDecorated()) {
+ $output->write("\r\033[K"); // clear an unfinished progress line
+ }
$output->writeln('' . $e->getMessage() . '');
return 1;
}
@@ -76,4 +80,34 @@ HELP);
$r['target']->getRepositoryId(), $r['created'] ? 'created' : 'found and adopted'));
return 0;
}
+
+ /**
+ * Progress of reading the index: on a terminal one line that is updated in place, otherwise
+ * (a log, a pipe) a line every 30 seconds. The time left is estimated from the rate so far.
+ *
+ * @return callable(int, int): void
+ */
+ private static function progress(OutputInterface $output): callable {
+ $start = microtime(true);
+ $shown = 0.0;
+ $inPlace = $output->isDecorated();
+ return static function (int $done, int $total) use ($output, $start, &$shown, $inPlace): void {
+ $now = microtime(true);
+ if ($done < $total && $now - $shown < ($inPlace ? 1 : 30)) {
+ return;
+ }
+ $shown = $now;
+ $line = sprintf('Reading the index: %s/%s files (%d%%)', number_format($done), number_format($total), intdiv(100 * $done, max(1, $total)));
+ $elapsed = $now - $start;
+ if ($done < $total && $done > 0 && $elapsed >= 5) {
+ $left = (int)ceil($elapsed / (float)$done * (float)($total - $done) / 60.0);
+ $line .= $left <= 1 ? ', less than a minute left' : sprintf(', about %d min left', $left);
+ }
+ if ($inPlace) {
+ $output->write("\r\033[K" . $line . ($done >= $total ? "\n" : ''));
+ } else {
+ $output->writeln($line);
+ }
+ };
+ }
}
diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php
index fdc6be5..83b4abc 100644
--- a/lib/Repository/Repository.php
+++ b/lib/Repository/Repository.php
@@ -81,7 +81,8 @@ final class Repository {
}
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
- public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null): self {
+ /** @param ?callable(int, int): void $progress index files downloaded so far, and in all (see BlobIndex::load()) */
+ public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null, ?callable $progress = null): self {
$config = self::readConfig($backend);
try {
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
@@ -95,7 +96,7 @@ final class Repository {
if ($cache !== null) {
$repo->index->setCache($cache, $config['id']);
}
- $repo->index->load();
+ $repo->index->load(false, $progress);
$repo->loadCatalog($anchor);
return $repo;
}
diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php
index 80d602e..b0e914e 100644
--- a/lib/Service/TargetService.php
+++ b/lib/Service/TargetService.php
@@ -48,15 +48,16 @@ final class TargetService {
* installation's key), then save it.
*
* @param array $options
+ * @param ?callable(int, int): void $progress reading an existing repository's index: files done, and in all
* @return array{target: Target, created: bool}
*/
- public function add(string $name, string $backend, string $auth, array $options, string $basePath): array {
+ public function add(string $name, string $backend, string $auth, array $options, string $basePath, ?callable $progress = null): array {
$this->validateName($name);
$this->requireUsableBackend($backend);
$keys = $this->keys->keyRing();
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath);
if ($storageBackend->exists('config')) {
- $repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor);
+ $repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor, $progress);
$created = false;
} else {
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor);
diff --git a/tests/Unit/Repository/IndexCacheTest.php b/tests/Unit/Repository/IndexCacheTest.php
index 3383d63..ff2b112 100644
--- a/tests/Unit/Repository/IndexCacheTest.php
+++ b/tests/Unit/Repository/IndexCacheTest.php
@@ -54,6 +54,24 @@ class IndexCacheTest extends TestCase {
$this->assertCount(12, self::hashTree($target));
}
+ /** Connecting a new server to a location with backups reports how far reading the index is. */
+ public function testOpeningReportsIndexProgress(): void {
+ $src = $this->tempDir();
+ for ($i = 0; $i < 6; $i++) {
+ file_put_contents("$src/f$i", random_bytes(3 * 1048576));
+ }
+ $backend = new LocalBackend($this->tempDir());
+ $keys = KeyRing::generate();
+ Repository::initWithKey($backend, $keys, $keys->wrap('pw'))->backupDirectory($src);
+ $files = count($backend->list('index'));
+ $seen = [];
+ Repository::openWithKey($backend, $keys, null, null, function (int $done, int $total) use (&$seen): void {
+ $seen[] = [$done, $total];
+ });
+ $this->assertCount($files, $seen);
+ $this->assertSame([$files, $files], end($seen));
+ }
+
/** The database dump reads in a read-only transaction: a pack that fills up then must not write the cache yet. */
public function testHeldCacheWritesWaitForRelease(): void {
$cache = new class implements IIndexCache {