Fast S3 verify and snapshot commit; shorter runs list; durations with days

On S3, index files and the tree segments read back when a snapshot is
written are fetched 16 at a time through the storage's S3 client
(IBackend::getRanges(), getMany()). Blobs that follow each other in a pack
are read as one range on every location. The snapshot is written in a step
of its own and the run shows "commit" meanwhile. On a 1 TB production
repository the check went from about 4 hours to 3.5 minutes.

backup:status and the web interface list the runs of the last 7 days,
running ones and at least the newest 5; --all / "Show older runs" list
all. Durations of a day or more show the days.

CHANGELOG.md

@@ -7,6 +7,26 @@ the app store accepts a `-beta` suffix, but Nextcloud servers then only offer th
 beta/daily update channel, which would hide the app from most admins. The stability stage is
 instead carried by the app description, the git tag, the release, and this file.
 
+## [Unreleased]
+
+### Changed
+- The list of backup runs (web interface and `occ backup:status`) shows the runs of the last 7
+  days, those still running, and at least the 5 newest; a failed run stays visible for that week.
+  `occ backup:status --all` and "Show older runs" in the web interface list all of them.
+- `occ backup:status` and `occ backup:verify` show durations of a day or more with the days
+  ("1d 12:10:13" instead of "12:10:13").
+
+### Fixed
+- Checking a backup on an S3 location (`occ backup:verify`) took hours on a large repository:
+  the index files were downloaded one at a time, at about two a second. They are now downloaded
+  16 at a time: on a production instance (27,585 index files, 1 TB) the check went from about
+  4 hours to 3.5 minutes.
+- Writing the snapshot at the end of a long backup to S3 took a quarter of an hour without any
+  progress shown: it read back the file list of every step one at a time. These reads are now
+  done 16 at a time on S3 (96 reads: 82 seconds before, 2 now), blobs that follow each other in
+  a pack are read at once on every location, the snapshot is written in a step of its own, and
+  the run shows "commit" while it does so.
+
 ## [0.9.6-beta.7] - 2026-10-14
 
 Bugfix release for SMB locations, with `occ backup:cancel` to stop a backup that does not get

README.md

@@ -194,7 +194,7 @@ On the backlog; none of these block the beta or 1.0.0, but know them before you
 | `backup:key:init`, `backup:key:kit`, `backup:key:confirm` | create the key, recovery kit, confirmation |
 | `backup:key:slots`, `backup:key:slot:add/replace/remove` | passphrase slots |
 | `backup:target:backends/add/edit/list/remove/append-only` | backup locations |
-| `backup:run`, `backup:status`, `backup:list` | back up, status, restore points |
+| `backup:run`, `backup:status`, `backup:list` | back up, status (runs of the last 7 days; `--all` for all), restore points |
 | `backup:cancel <run>` | stop a running backup |
 | `backup:browse`, `backup:restore` | browse and restore |
 | `backup:retention`, `backup:prune`, `backup:trash`, `backup:trash:restore` | retention, cleanup, trash |

lib/Backend/GetsOneByOne.php

@@ -7,7 +7,7 @@ declare(strict_types=1);
 
 namespace OCA\NgBackup\Backend;
 
-/** IBackend::getMany() for a backend where reading files one after the other costs nothing extra. */
+/** IBackend::getMany() and getRanges() for a backend where reading one after the other costs nothing extra. */
 trait GetsOneByOne {
 	/**
 	 * @param list<string> $paths
@@ -27,4 +27,14 @@ trait GetsOneByOne {
 			yield $path => $data;
 		}
 	}
+
+	/**
+	 * @param list<array{0:string, 1:int, 2:int}> $ranges
+	 * @return \Generator<int, string>
+	 */
+	public function getRanges(array $ranges): \Generator {
+		foreach ($ranges as $i => [$path, $offset, $length]) {
+			yield $i => $this->getRange($path, $offset, $length);
+		}
+	}
 }

lib/Backend/IBackend.php

@@ -35,6 +35,17 @@ interface IBackend {
 	/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
 	public function getRange(string $path, int $offset, int $length): string;
 
+	/**
+	 * Many byte ranges, in the order given (blobs spread over many packs, such as the tree
+	 * segments of a backup). A backend where every request costs a round trip can fetch them
+	 * several at a time.
+	 *
+	 * @param list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] each
+	 * @return \Generator<int, string> contents by position in $ranges
+	 * @throws BackendException for a range that cannot be read
+	 */
+	public function getRanges(array $ranges): \Generator;
+
 	public function exists(string $path): bool;
 
 	/**

lib/Backend/StorageBackend.php

@@ -19,6 +19,13 @@ final class StorageBackend implements IBackend {
 	private bool $atomicPut;
 	/** SFTP: read through SftpReadStream, which keeps several requests in flight. */
 	private bool $sftpReads;
+	/** S3: many small reads go straight to the storage's S3 client, several at a time. */
+	private bool $s3Reads;
+	/** S3 requests in flight at once for getMany() and getRanges(). */
+	public const S3_CONCURRENCY = 16;
+	/** Most ranges, and bytes, getRanges() holds in memory at once. */
+	private const RANGES_LOT = 64;
+	private const RANGES_LOT_BYTES = 64 * 1048576;
 	/** Seconds without a request after which an SFTP connection is checked before it is used. */
 	public const IDLE_CHECK = 60;
 	private float $lastUsed;
@@ -40,6 +47,7 @@ final class StorageBackend implements IBackend {
 		self::validate($this->base, true);
 		$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
 		$this->sftpReads = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\SFTP::class);
+		$this->s3Reads = $storage instanceof \OCA\Files_External\Lib\Storage\AmazonS3;
 		if ($createBase && $this->base !== '') {
 			$this->mkdirs($this->base);
 		}
@@ -116,11 +124,19 @@ final class StorageBackend implements IBackend {
 		return $fh;
 	}
 
-	/** Over SFTP several at a time (SftpReadStream::readMany()); elsewhere one after the other. */
+	/**
+	 * Over SFTP and S3 several at a time (SftpReadStream::readMany(), s3Fetch()); elsewhere one
+	 * after the other.
+	 */
 	public function getMany(array $paths): \Generator {
-		foreach (array_chunk($paths, 8 * SftpReadStream::BATCH) as $chunk) {
+		$size = $this->s3Reads ? 16 * self::S3_CONCURRENCY : 8 * SftpReadStream::BATCH;
+		foreach (array_chunk($paths, $size) as $chunk) {
 			$fast = null;
-			if ($this->sftpReads) {
+			if ($this->s3Reads) {
+				$this->idleCheck();
+				$fast = $this->s3Fetch(array_map(fn (string $p): array => [$p, null], $chunk));
+				$fast = $fast === null ? null : array_combine($chunk, array_map(fn (int $i): ?string => $fast[$i] ?? null, array_keys($chunk)));
+			} elseif ($this->sftpReads) {
 				$abs = array_map(fn (string $p): string => $this->abs($p), $chunk);
 				$fast = $this->call(fn () => SftpReadStream::readMany($this->storage, $abs));
 				$fast = $fast === null ? null : array_combine($chunk, array_map(fn (string $a): ?string => $fast[$a] ?? null, $abs));
@@ -145,6 +161,88 @@ final class StorageBackend implements IBackend {
 		}
 	}
 
+	/** Over S3 several at a time (s3Fetch()); elsewhere one after the other. */
+	public function getRanges(array $ranges): \Generator {
+		$lot = [];
+		$bytes = 0;
+		foreach ($ranges as $i => $range) {
+			if ($lot !== [] && (count($lot) >= self::RANGES_LOT || $bytes + $range[2] > self::RANGES_LOT_BYTES)) {
+				yield from $this->rangesLot($lot);
+				$lot = [];
+				$bytes = 0;
+			}
+			$lot[$i] = $range;
+			$bytes += $range[2];
+		}
+		if ($lot !== []) {
+			yield from $this->rangesLot($lot);
+		}
+	}
+
+	/**
+	 * @param array<int, array{0:string, 1:int, 2:int}> $lot
+	 * @return \Generator<int, string>
+	 */
+	private function rangesLot(array $lot): \Generator {
+		$fast = null;
+		if ($this->s3Reads) {
+			$this->idleCheck();
+			$fast = $this->s3Fetch(array_map(fn (array $r): array => [$r[0], $r[1] . '-' . ($r[1] + $r[2] - 1)], $lot));
+		}
+		foreach ($lot as $i => [$path, $offset, $length]) {
+			$data = $fast[$i] ?? null;
+			// not read in the batch, or not all of it: the usual way, which also reports why
+			yield $i => ($data !== null && strlen($data) === $length) ? $data : $this->getRange($path, $offset, $length);
+		}
+	}
+
+	/**
+	 * Read objects (or byte ranges of them, "first-last") straight through the S3 client of
+	 * files_external's S3 storage, S3_CONCURRENCY at a time. Its own reads open a new HTTPS
+	 * connection per object, after a HEAD request: one after the other that is about two reads a
+	 * second, which made checking the index of a large repository take hours. What fails here
+	 * is left out of the result, for the caller to read the usual way (which reports the error).
+	 *
+	 * @param array<int, array{0:string, 1:?string}> $requests [path, range or null] by key
+	 * @return array<int, string>|null contents by key; null when the storage is not as expected
+	 * @psalm-suppress UndefinedClass the AWS SDK comes with Nextcloud, not with this app
+	 */
+	private function s3Fetch(array $requests): ?array {
+		try {
+			/** @psalm-suppress UndefinedMethod, UndefinedThisPropertyFetch */
+			[$client, $bucket, $sse] = (function (): array {
+				$sse = method_exists($this, 'getServerSideEncryptionParameters') ? $this->getServerSideEncryptionParameters()
+					: (method_exists($this, 'getSSECParameters') ? $this->getSSECParameters() : []);
+				return [$this->getConnection(), $this->getBucket(), $sse];
+			})->call($this->storage);
+			if (!$client instanceof \Aws\S3\S3Client || !is_string($bucket)) {
+				return null;
+			}
+			$results = [];
+			$keys = array_keys($requests); // the pool numbers results 0, 1, ...: mapped back below
+			$commands = (function () use ($requests, $client, $bucket, $sse): \Generator {
+				foreach (array_values($requests) as $n => [$path, $range]) {
+					$args = ['Bucket' => $bucket, 'Key' => trim($this->abs($path), '/')] + $sse;
+					if ($range !== null) {
+						$args['Range'] = 'bytes=' . $range;
+					}
+					yield $n => $client->getCommand('GetObject', $args);
+				}
+			})();
+			(new \Aws\CommandPool($client, $commands, [
+				'concurrency' => self::S3_CONCURRENCY,
+				'fulfilled' => function (mixed $result, int $n) use (&$results, $keys): void {
+					$results[$keys[$n]] = (string)$result['Body'];
+				},
+				'rejected' => static function (): void {
+				},
+			]))->promise()->wait();
+			return $results;
+		} catch (\Throwable) {
+			return null;
+		}
+	}
+
 	public function getRange(string $path, int $offset, int $length): string {
 		return $this->call(fn () => $this->readRange($path, $offset, $length));
 	}

lib/Command/Status.php

@@ -14,9 +14,13 @@ use OCA\NgBackup\Service\VerifyService;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Helper\Table;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 final class Status extends Command {
+	/** Runs of the last days that are listed by default (failed ones stay visible that long). */
+	public const RECENT_DAYS = 7;
+
 	public function __construct(
 		private KeyService $keys,
 		private TargetService $targets,
@@ -27,7 +31,8 @@ final class Status extends Command {
 	}
 
 	protected function configure(): void {
-		$this->setName('backup:status')->setDescription('Show the backup key, locations and recent runs');
+		$this->setName('backup:status')->setDescription('Show the backup key, locations and recent runs')
+			->addOption('all', null, InputOption::VALUE_NONE, 'List all runs, not only those of the last ' . self::RECENT_DAYS . ' days');
 	}
 
 	protected function execute(InputInterface $input, OutputInterface $output): int {
@@ -44,16 +49,22 @@ final class Status extends Command {
 		}
 		$output->writeln('Locations: ' . ($names === [] ? 'none' : implode(', ', $names)));
 		$rows = [];
-		foreach ($this->runs->findRecent(10) as $r) {
+		$all = (bool)$input->getOption('all');
+		$shown = $this->runs->findShown($all ? null : time() - self::RECENT_DAYS * 86400);
+		foreach ($shown as $r) {
 			$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
 			$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . self::percent($r) . ')' : ''),
-				date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
+				date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? \OCA\NgBackup\Format::duration($r->getFinishedAt() - $r->getStartedAt()) : '',
 				$s['files'] ?? '', isset($s['bytes']) ? self::size((int)$s['bytes']) : '',
 				isset($s['uploaded']) ? self::size((int)$s['uploaded']) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
 		}
 		if ($rows !== []) {
 			(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'size', 'uploaded', 'error'])->setRows($rows)->render();
 		}
+		$older = $all ? 0 : $this->runs->countAll() - count($shown);
+		if ($older > 0) {
+			$output->writeln(sprintf('%d older run%s not shown (--all)', $older, $older === 1 ? '' : 's'));
+		}
 		foreach ($this->verify->progress() as $targetId => $state) {
 			$output->writeln(sprintf('Verify of %s on %s: %s', mb_strimwidth((string)($state['what'] ?? ''), 0, 40, '…'),
 				$names[$targetId] ?? '#' . $targetId, VerifyService::describe($state)));

lib/Controller/AdminApiController.php

@@ -51,14 +51,16 @@ final class AdminApiController extends Controller {
 		parent::__construct(Application::APP_ID, $request);
 	}
 
-	public function status(): JSONResponse {
+	/** @param bool $allRuns all runs instead of those of the last days (Status::RECENT_DAYS) */
+	public function status(bool $allRuns = false): JSONResponse {
 		$targets = [];
 		foreach ($this->targets->list() as $t) {
 			$targets[] = ['id' => $t->getId(), 'name' => $t->getName(), 'backend' => $t->getBackend(), 'path' => $t->getBasePath(),
 				'repositoryId' => $t->getRepositoryId(), 'createdAt' => $t->getCreatedAt(), 'appendOnly' => (bool)$t->getAppendOnly(),
 				'maxBytes' => $t->getMaxBytes(), 'storedBytes' => $this->alerts->lastStoredBytes($t)];
 		}
-		$runs = array_map(fn (Run $r) => $this->runJson($r), $this->runs->findRecent(15));
+		$shown = $this->runs->findShown($allRuns ? null : time() - \OCA\NgBackup\Command\Status::RECENT_DAYS * 86400);
+		$runs = array_map(fn (Run $r) => $this->runJson($r), $shown);
 		return new JSONResponse([
 			'key' => ['initialized' => $this->keys->isInitialized(), 'fingerprint' => $this->keys->fingerprint(),
 				'confirmation' => $this->keys->recoveryKitConfirmation(), 'everConfirmed' => $this->keys->isInitialized() && $this->keys->everConfirmed(),
@@ -68,6 +70,8 @@ final class AdminApiController extends Controller {
 			'confirmationPhrase' => KeyConfirm::CONFIRMATION,
 			'targets' => $targets,
 			'runs' => $runs,
+			'olderRuns' => $allRuns ? 0 : $this->runs->countAll() - count($shown),
+			'recentDays' => \OCA\NgBackup\Command\Status::RECENT_DAYS,
 			'schedule' => $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', ''),
 		]);
 	}

lib/Db/RunMapper.php

@@ -73,6 +73,40 @@ final class RunMapper extends QBMapper {
 		return $this->findEntities($qb);
 	}
 
+	/**
+	 * The runs to list, newest first: those started at or after $since, those still running, and
+	 * at least the newest $atLeast, so the list is never empty after a quiet week. All runs when
+	 * $since is null.
+	 *
+	 * @return list<Run>
+	 */
+	public function findShown(?int $since, int $atLeast = 5): array {
+		$qb = $this->db->getQueryBuilder();
+		$qb->select('*')->from($this->getTableName())->orderBy('id', 'DESC');
+		if ($since === null) {
+			return $this->findEntities($qb);
+		}
+		$qb->where($qb->expr()->orX(
+			$qb->expr()->gte('started_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT)),
+			$qb->expr()->eq('status', $qb->createNamedParameter(Run::RUNNING)),
+		));
+		$runs = [];
+		foreach ([...$this->findEntities($qb), ...$this->findRecent($atLeast)] as $r) {
+			$runs[$r->getId()] = $r;
+		}
+		krsort($runs);
+		return array_values($runs);
+	}
+
+	public function countAll(): int {
+		$qb = $this->db->getQueryBuilder();
+		$qb->select($qb->func()->count('*', 'n'))->from($this->getTableName());
+		$result = $qb->executeQuery();
+		$n = (int)$result->fetchOne();
+		$result->closeCursor();
+		return $n;
+	}
+
 	/** @return list<Run> runs of $kind started at or after $since, newest first */
 	public function startedSince(int $targetId, string $kind, int $since): array {
 		$qb = $this->db->getQueryBuilder();

lib/Format.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+// SPDX-FileCopyrightText: 2026 Dillard Blom
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+namespace OCA\NgBackup;
+
+final class Format {
+	/** "12:10:13", or "1d 12:10:13" from a day on (a long first backup takes days). */
+	public static function duration(int $seconds): string {
+		$seconds = max(0, $seconds);
+		$days = intdiv($seconds, 86400);
+		$time = gmdate('H:i:s', $seconds % 86400);
+		return $days > 0 ? $days . 'd ' . $time : $time;
+	}
+}

lib/Job/BackupRun.php

@@ -140,21 +140,21 @@ final class BackupRun {
 			$state['parentPos'] = $parent->position();
 			$state['stats'] = $stats;
 			if ($finishedWalk) {
+				// The snapshot is written in a step of its own, with its own time: after a long
+				// run it reads back thousands of segments, and the run reports that it got here.
 				$state['phase'] = 'finish';
 			}
+			return $state;
 		}
 
 		if ($state['phase'] === 'finish') {
-			$segments = $state['segments'];
+			// Every segment ends with a newline, so they read as one stream: fetched together,
+			// several at a time where the location allows it.
 			$repo->putSnapshot($state['snapshot'],
 				['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
 					'roots' => array_keys($state['roots']), 'stats' => $state['stats'],
 					'warnings' => self::warnings($state)] + $state['meta'],
-				(function () use ($repo, $segments) {
+				$repo->readLines(array_values(array_merge(...$state['segments']))));
-					foreach ($segments as $blobs) {
-						yield from $repo->readLines($blobs);
-					}
-				})());
 			$state['phase'] = 'commit';
 		}
 		return $state;

lib/Repository/PackReaderGuard.php

@@ -38,6 +38,11 @@ final class PackReaderGuard implements IBackend {
 		return $this->inner->getRange($path, $offset, $length);
 	}
 
+	public function getRanges(array $ranges): \Generator {
+		$this->reader->close();
+		return $this->inner->getRanges($ranges);
+	}
+
 	public function exists(string $path): bool {
 		$this->reader->close();
 		return $this->inner->exists($path);

lib/Repository/Repository.php

@@ -30,6 +30,8 @@ final class Repository {
 	private const FLAG_DEFLATE = 1;
 	/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
 	private const VERIFY_READ_ATTEMPTS = 3;
+	/** Largest read that loadBlobs() makes of blobs that follow each other in a pack. */
+	private const MERGED_READ_MAX = 16 * 1048576;
 
 	private StreamCipher $cipher;
 	private BlobIndex $index;
@@ -288,8 +290,8 @@ final class Repository {
 	 */
 	public function readLines(array $blobIds): \Generator {
 		$carry = '';
-		foreach ($blobIds as $id) {
+		foreach ($this->loadBlobs($blobIds) as $blob) {
-			$data = $carry . $this->loadBlob($id);
+			$data = $carry . $blob;
 			$lines = explode("\n", $data);
 			$carry = array_pop($lines);
 			yield from $lines;
@@ -711,6 +713,50 @@ final class Repository {
 		return $id;
 	}
 
+	/**
+	 * Blobs spread over many packs (the tree segments of a run, one per step), through
+	 * IBackend::getRanges(): on a remote location one at a time is a round trip each, which for
+	 * a long run made writing its snapshot take a quarter of an hour. Blobs that follow each
+	 * other in a pack are read as one range.
+	 *
+	 * @param list<string> $ids
+	 * @return \Generator<int, string> verified blob contents, in order
+	 */
+	private function loadBlobs(array $ids): \Generator {
+		/** @var list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] of each read */
+		$ranges = [];
+		/** @var list<list<array{0:string, 1:int, 2:int}>> $parts per read: [blob id, offset in the read, length] */
+		$parts = [];
+		// the read being extended
+		$curPath = null;
+		$curOffset = 0;
+		$curLength = 0;
+		$curParts = [];
+		foreach ($ids as $id) {
+			[$pack, $offset, $length] = $this->index->get($id);
+			$path = 'packs/' . substr($pack, 0, 2) . '/' . $pack;
+			if ($curPath === $path && $curOffset + $curLength === $offset && $curLength + $length <= self::MERGED_READ_MAX) {
+				$curParts[] = [$id, $curLength, $length];
+				$curLength += $length;
+				continue;
+			}
+			if ($curPath !== null) {
+				$ranges[] = [$curPath, $curOffset, $curLength];
+				$parts[] = $curParts;
+			}
+			[$curPath, $curOffset, $curLength, $curParts] = [$path, $offset, $length, [[$id, 0, $length]]];
+		}
+		if ($curPath !== null) {
+			$ranges[] = [$curPath, $curOffset, $curLength];
+			$parts[] = $curParts;
+		}
+		foreach ($this->backend->getRanges($ranges) as $i => $data) {
+			foreach ($parts[$i] as [$id, $at, $length]) {
+				yield $this->decodeBlob($id, substr($data, $at, $length));
+			}
+		}
+	}
+
 	private function loadBlob(string $id): string {
 		[$pack, $offset, $length] = $this->index->get($id);
 		return $this->decodeBlob($id, $this->packReader->read($pack, $offset, $length));

lib/Service/BackupService.php

@@ -225,6 +225,9 @@ final class BackupService {
 			} else {
 				$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
 				$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
+				if (in_array($state['files']['phase'], ['finish', 'commit'], true)) {
+					$run->setPhase('commit'); // all files read: writing and recording the snapshot
+				}
 				if ($state['files']['phase'] === 'commit') {
 					// Catalog write under the catalog lock, from a repository opened inside it: a per-user
 					// export or restore may have written the catalog since this step opened $repo.

lib/Service/VerifyService.php

@@ -64,7 +64,7 @@ final class VerifyService {
 				$age < 120 ? $age . ' s' : intdiv($age, 60) . ' min',
 				$age > 600 ? ' (no progress for a while: the process may have stopped)' : '');
 		}
-		$took = gmdate('H:i:s', max(0, (int)($s['finished'] ?? 0) - (int)($s['started'] ?? 0)));
+		$took = \OCA\NgBackup\Format::duration((int)($s['finished'] ?? 0) - (int)($s['started'] ?? 0));
 		return match ($s['status'] ?? '') {
 			'ok' => sprintf('%s OK on %s (took %s): %s files, %s blobs%s', $kind, date('Y-m-d H:i', (int)$s['finished']), $took,
 				number_format((int)($s['files'] ?? 0)), number_format((int)($s['blobs'] ?? 0)), $deep ? ', ' . $size((int)($s['bytes'] ?? 0)) . ' decrypted' : ''),

psalm-baseline.xml

@@ -218,16 +218,11 @@
     </PropertyNotSetInConstructor>
   </file>
   <file src="lib/Job/BackupRun.php">
-    <ArgumentTypeCoercion>
-      <code><![CDATA[$blobs]]></code>
-    </ArgumentTypeCoercion>
     <MissingParamType>
       <code><![CDATA[$fh]]></code>
     </MissingParamType>
     <PossiblyUndefinedArrayOffset>
       <code><![CDATA[$mtime]]></code>
-      <code><![CDATA[$state['phase']]]></code>
-      <code><![CDATA[$state['stats']]]></code>
     </PossiblyUndefinedArrayOffset>
   </file>
   <file src="lib/Job/ParentCursor.php">

src/AdminSettings.vue

@@ -22,7 +22,7 @@
 			</template>
 			<template v-else-if="status.key.everConfirmed">
 				<template v-if="status.targets.length">
-					<Backups :status="status" @changed="refresh" />
+					<Backups :status="status" :all-runs="allRuns" @changed="refresh" @show-all-runs="showAllRuns" />
 					<RestoreBrowser :status="status" @changed="refresh" />
 				</template>
 				<NcNoteCard v-else type="info">{{ t('ng_backup', 'Add a backup location under "Settings" first.') }}</NcNoteCard>
@@ -46,14 +46,20 @@ import Locations from './components/Locations.vue'
 import RestoreBrowser from './components/RestoreBrowser.vue'
 
 const status = ref(null)
+const allRuns = ref(false) // runs older than the last days too
 const error = ref('')
 const tab = ref('settings')
 let tabInitialized = false
 let timer = null
 
+function showAllRuns(all) {
+	allRuns.value = all
+	refresh()
+}
+
 async function refresh() {
 	try {
-		status.value = await api.status()
+		status.value = await api.status(allRuns.value)
 		error.value = ''
 		// Default to the daily-use tab once the key is set up; stay on Settings otherwise.
 		// Only decided on first load, so switching tabs ourselves isn't undone by polling.

src/api.js

@@ -16,7 +16,7 @@ async function call(method, path, data) {
 }
 
 export default {
-	status: () => call('get', '/status'),
+	status: (allRuns = false) => call('get', '/status', allRuns ? { allRuns: true } : undefined),
 	initKey: (passphrase, deleteDelayDays, label) => call('post', '/key', { passphrase, deleteDelayDays, label }),
 	addSlot: (passphrase, label) => call('post', '/key/slots', { passphrase, label }),
 	replaceSlot: (slot, passphrase, label) => call('put', `/key/slots/${slot}`, { passphrase, label }),

src/components/Backups.vue

@@ -58,12 +58,22 @@
 				</tbody>
 			</table>
 		</div>
+		<p v-if="allRuns || status.olderRuns" class="ngb-muted">
+			<template v-if="allRuns">
+				{{ t('ng_backup', 'All runs.') }}
+				<NcButton variant="tertiary" @click="emit('show-all-runs', false)">{{ n('ng_backup', 'Only the last day', 'Only the last %n days', status.recentDays) }}</NcButton>
+			</template>
+			<template v-else>
+				{{ n('ng_backup', 'Runs of the last day.', 'Runs of the last %n days.', status.recentDays) }}
+				<NcButton variant="tertiary" @click="emit('show-all-runs', true)">{{ n('ng_backup', 'Show %n older run', 'Show %n older runs', status.olderRuns) }}</NcButton>
+			</template>
+		</p>
 	</NcSettingsSection>
 </template>
 
 <script setup>
 import { ref, watch } from 'vue'
-import { t } from '@nextcloud/l10n'
+import { n, t } from '@nextcloud/l10n'
 import NcButton from '@nextcloud/vue/components/NcButton'
 import NcNoteCard from '@nextcloud/vue/components/NcNoteCard'
 import NcSettingsSection from '@nextcloud/vue/components/NcSettingsSection'
@@ -73,8 +83,8 @@ import { formatSize } from '../format.js'
 import { confirmPassword } from '@nextcloud/password-confirmation'
 import { reactive } from 'vue'
 
-const props = defineProps({ status: { type: Object, required: true } })
+const props = defineProps({ status: { type: Object, required: true }, allRuns: { type: Boolean, default: false } })
-const emit = defineEmits(['changed'])
+const emit = defineEmits(['changed', 'show-all-runs'])
 const schedule = ref(props.status.schedule)
 const error = ref('')
 watch(() => props.status.schedule, (s) => { schedule.value = s })

tests/Unit/FormatTest.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\NgBackup\Tests\Unit;
+
+use OCA\NgBackup\Format;
+use PHPUnit\Framework\TestCase;
+
+class FormatTest extends TestCase {
+	public function testDurationShowsDaysFromOneDayOn(): void {
+		$this->assertSame('00:00:00', Format::duration(-5));
+		$this->assertSame('12:10:13', Format::duration(12 * 3600 + 10 * 60 + 13));
+		$this->assertSame('1d 00:00:00', Format::duration(86400));
+		$this->assertSame('1d 12:10:13', Format::duration(86400 + 12 * 3600 + 10 * 60 + 13));
+	}
+}

tests/Unit/Job/BackupRunTest.php

@@ -42,6 +42,61 @@ class BackupRunTest extends TestCase {
 		$this->assertSame(self::hashTree($src), self::hashTree($target));
 	}
 
+	public function testTheSnapshotIsWrittenInAStepOfItsOwnFromAllSegmentsAtOnce(): void {
+		$src = $this->tempDir();
+		for ($i = 0; $i < 60; $i++) {
+			file_put_contents("$src/f$i.txt", random_bytes(200000));
+		}
+		$backend = new RangeCountingBackend(new LocalBackend($this->tempDir()));
+		$keys = KeyRing::generate();
+		Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
+
+		$state = BackupRun::start($src);
+		for ($i = 0; $i < 500 && $state['phase'] === 'files'; $i++) {
+			$state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 0.01);
+		}
+		$this->assertSame('finish', $state['phase'], 'the step that read the last file does not write the snapshot');
+		$this->assertGreaterThan(2, count($state['segments']));
+		$backend->batches = [];
+
+		$state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 0.01);
+
+		$this->assertSame('commit', $state['phase']);
+		// One batch; the blobs of a segment follow each other in its pack and are one read.
+		$this->assertSame([count($state['segments'])], $backend->batches);
+		Repository::openWithKey($backend, $keys)->recordSnapshot($state['snapshot']);
+		$target = $this->tempDir();
+		Repository::openWithKey($backend, $keys)->restore($state['snapshot'], $target);
+		$this->assertSame(self::hashTree($src), self::hashTree($target));
+	}
+
+	public function testBlobsThatFollowEachOtherInAPackAreReadAsOneRange(): void {
+		$backend = new RangeCountingBackend(new LocalBackend($this->tempDir()));
+		$keys = KeyRing::generate();
+		$repo = Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
+		$stats = [];
+		$segments = [];
+		$expected = [];
+		for ($s = 0; $s < 3; $s++) {
+			$writer = new \OCA\NgBackup\Repository\BlobWriter(fn (string $data): string => $repo->storeData($data, $stats), 1000);
+			$lines = [];
+			for ($i = 0; $i < 100; $i++) {
+				$lines[] = json_encode(['segment' => $s, 'line' => $i, 'pad' => bin2hex(random_bytes(20))]);
+			}
+			$writer->write(implode("\n", $lines) . "\n");
+			$segments[] = $writer->finish();
+			$repo->flushPacks(); // one pack per segment, as one per step
+			array_push($expected, ...$lines);
+		}
+		$this->assertGreaterThan(3, count(array_merge(...$segments)));
+		$backend->batches = [];
+
+		$read = iterator_to_array(Repository::openWithKey($backend, $keys)->readLines(array_merge(...$segments)), false);
+
+		$this->assertSame($expected, array_values(array_filter($read, fn (string $l): bool => $l !== '')));
+		$this->assertSame([3], $backend->batches, 'one read per segment, all in one batch');
+	}
+
 	public function testFileChangedBetweenStepsIsReadAgain(): void {
 		$src = $this->tempDir();
 		file_put_contents("$src/big.bin", random_bytes(20 * 1048576));
@@ -100,3 +155,54 @@ class BackupRunTest extends TestCase {
 		$this->assertSame(self::hashTree($src), self::hashTree("$target/data"));
 	}
 }
+
+final class RangeCountingBackend implements \OCA\NgBackup\Backend\IBackend {
+	use \OCA\NgBackup\Backend\GetsOneByOne {
+		getRanges as private oneByOne;
+	}
+
+	/** @var list<int> number of ranges per getRanges() call */
+	public array $batches = [];
+
+	public function __construct(
+		private \OCA\NgBackup\Backend\IBackend $inner,
+	) {
+	}
+
+	public function getRanges(array $ranges): \Generator {
+		$this->batches[] = count($ranges);
+		return $this->oneByOne($ranges);
+	}
+
+	public function put(string $path, $stream): void {
+		$this->inner->put($path, $stream);
+	}
+
+	public function get(string $path) {
+		return $this->inner->get($path);
+	}
+
+	public function getRange(string $path, int $offset, int $length): string {
+		return $this->inner->getRange($path, $offset, $length);
+	}
+
+	public function exists(string $path): bool {
+		return $this->inner->exists($path);
+	}
+
+	public function move(string $from, string $to): void {
+		$this->inner->move($from, $to);
+	}
+
+	public function list(string $prefix): array {
+		return $this->inner->list($prefix);
+	}
+
+	public function listFiles(string $dir): array {
+		return $this->inner->listFiles($dir);
+	}
+
+	public function delete(string $path): void {
+		$this->inner->delete($path);
+	}
+}