Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

target:add: show progress and time left while reading the index of an existing repository

Browse Source
This commit is contained in:
Dillard Blom
2026-10-18 22:14:45 +02:00
parent 9e1378c6ab
commit 11ac30635a
6 changed files with 66 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2 -1
View File
@@ -38,7 +38,8 @@ All-in-One onto a plain Nextcloud with PostgreSQL).
restore commits.
- `occ backup:target:add` without a key pointed to `backup:key:init`, which makes a new key that
cannot read existing backups. It now points to `backup:key:import` for a restore, and says that
connecting to a location with backups can take a while.
connecting to a location with backups can take a while. While it reads the index of a location
with backups, it shows how far it is and about how long it still takes.
- A restore stopped with "Not found: packs/..." when the location was cleaned up while it ran,
for example by the weekly cleanup of the original server during a recovery test on a second
server: blobs that were moved into new packs were still looked for in the old ones. A restore
docs/DISASTER-RECOVERY.md
+3 -2
View File
@@ -59,8 +59,9 @@ Import it; you type the passphrase yourself:
It shows the key fingerprint: it must match the one of your old server (on the recovery kit).
**5. Add the backup location** with the same details as before. This can take a while on a large
backup (the index is read first) and shows nothing until it is done.
**5. Add the backup location** with the same details as before. On a large backup this first reads
the index and shows how far it is, for example
`Reading the index: 6,913/26,791 files (25%), about 12 min left`.
occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=... -o root=... --path=...
lib/Command/TargetAdd.php
+37 -3
View File
@@ -64,11 +64,15 @@ HELP);
return 1;
}
// Adopting an existing repository reads its whole index first, which takes a while on a
// large backup or a slow line; say so instead of looking stuck.
$output->writeln('Connecting to the location; with existing backups this reads their index first and can take a while ...');
// large backup or a slow line; show how far it is instead of looking stuck.
$output->writeln('Connecting to the location; with existing backups this reads their index first ...');
try {
$r = $this->targets->add($input->getArgument('name'), $input->getArgument('backend'), $input->getOption('auth'), $options, (string)$input->getOption('path'));
$r = $this->targets->add($input->getArgument('name'), $input->getArgument('backend'), $input->getOption('auth'), $options,
(string)$input->getOption('path'), self::progress($output));
} catch (\Throwable $e) {
if ($output->isDecorated()) {
$output->write("\r\033[K"); // clear an unfinished progress line
}
$output->writeln('<error>' . $e->getMessage() . '</error>');
return 1;
}
@@ -76,4 +80,34 @@ HELP);
$r['target']->getRepositoryId(), $r['created'] ? 'created' : 'found and adopted'));
return 0;
}
/**
* Progress of reading the index: on a terminal one line that is updated in place, otherwise
* (a log, a pipe) a line every 30 seconds. The time left is estimated from the rate so far.
*
* @return callable(int, int): void
*/
private static function progress(OutputInterface $output): callable {
$start = microtime(true);
$shown = 0.0;
$inPlace = $output->isDecorated();
return static function (int $done, int $total) use ($output, $start, &$shown, $inPlace): void {
$now = microtime(true);
if ($done < $total && $now - $shown < ($inPlace ? 1 : 30)) {
return;
}
$shown = $now;
$line = sprintf('Reading the index: %s/%s files (%d%%)', number_format($done), number_format($total), intdiv(100 * $done, max(1, $total)));
$elapsed = $now - $start;
if ($done < $total && $done > 0 && $elapsed >= 5) {
$left = (int)ceil($elapsed / (float)$done * (float)($total - $done) / 60.0);
$line .= $left <= 1 ? ', less than a minute left' : sprintf(', about %d min left', $left);
}
if ($inPlace) {
$output->write("\r\033[K" . $line . ($done >= $total ? "\n" : ''));
} else {
$output->writeln($line);
}
};
}
}
lib/Repository/Repository.php
+3 -2
View File
@@ -81,7 +81,8 @@ final class Repository {
}
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null): self {
/** @param ?callable(int, int): void $progress index files downloaded so far, and in all (see BlobIndex::load()) */
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null, ?callable $progress = null): self {
$config = self::readConfig($backend);
try {
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
@@ -95,7 +96,7 @@ final class Repository {
if ($cache !== null) {
$repo->index->setCache($cache, $config['id']);
}
$repo->index->load();
$repo->index->load(false, $progress);
$repo->loadCatalog($anchor);
return $repo;
}
lib/Service/TargetService.php
+3 -2
View File
@@ -48,15 +48,16 @@ final class TargetService {
* installation's key), then save it.
*
* @param array<string, mixed> $options
* @param ?callable(int, int): void $progress reading an existing repository's index: files done, and in all
* @return array{target: Target, created: bool}
*/
public function add(string $name, string $backend, string $auth, array $options, string $basePath): array {
public function add(string $name, string $backend, string $auth, array $options, string $basePath, ?callable $progress = null): array {
$this->validateName($name);
$this->requireUsableBackend($backend);
$keys = $this->keys->keyRing();
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath);
if ($storageBackend->exists('config')) {
$repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor);
$repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor, $progress);
$created = false;
} else {
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor);
tests/Unit/Repository/IndexCacheTest.php
+18
View File
@@ -54,6 +54,24 @@ class IndexCacheTest extends TestCase {
$this->assertCount(12, self::hashTree($target));
}
/** Connecting a new server to a location with backups reports how far reading the index is. */
public function testOpeningReportsIndexProgress(): void {
$src = $this->tempDir();
for ($i = 0; $i < 6; $i++) {
file_put_contents("$src/f$i", random_bytes(3 * 1048576));
}
$backend = new LocalBackend($this->tempDir());
$keys = KeyRing::generate();
Repository::initWithKey($backend, $keys, $keys->wrap('pw'))->backupDirectory($src);
$files = count($backend->list('index'));
$seen = [];
Repository::openWithKey($backend, $keys, null, null, function (int $done, int $total) use (&$seen): void {
$seen[] = [$done, $total];
});
$this->assertCount($files, $seen);
$this->assertSame([$files, $files], end($seen));
}
/** The database dump reads in a read-only transaction: a pack that fills up then must not write the cache yet. */
public function testHeldCacheWritesWaitForRelease(): void {
$cache = new class implements IIndexCache {