Backups: show the progress of the files phase

A run only showed "running (files)", which says nothing during a backup
of a terabyte. The files phase now records an expected size on its first
step: the previous snapshot's size, or for a first backup the users'
files, versions and trash from Nextcloud's file cache (approximate, shown
with a ~). The web interface and occ backup:status show the share of it
done so far, including a large file that is still being read; it stays
below 100% until the run finishes.

CHANGELOG.md

@@ -10,6 +10,10 @@ instead carried by the app description, the git tag, the release, and this file.
 ## [Unreleased]
 
 ### Added
+- Progress of a running backup in the web interface and `occ backup:status`, e.g.
+  "running (files) 68%". The total is the size of the previous snapshot; for a first backup it
+  is estimated from Nextcloud's file cache (users' files, versions and trash) and shown as
+  "~68%".
 - Edit a backup location (web interface and `occ backup:target:edit`): change the password, host,
   folder, connection type or name without losing its history. Before this the only option was a
   new location, which starts an empty repository and, with a retention lock at the provider,

lib/Command/Status.php

@@ -44,7 +44,7 @@ final class Status extends Command {
 		$rows = [];
 		foreach ($this->runs->findRecent(10) as $r) {
 			$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
-			$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . ')' : ''),
+			$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . self::percent($r) . ')' : ''),
 				date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
 				$s['files'] ?? '', isset($s['uploaded']) ? sprintf('%.1f MiB', $s['uploaded'] / 1048576) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
 		}
@@ -53,4 +53,10 @@ final class Status extends Command {
 		}
 		return 0;
 	}
+
+	/** " 68%", or " ~68%" when estimated from the file cache; empty without an estimate */
+	private static function percent(\OCA\NgBackup\Db\Run $r): string {
+		$p = \OCA\NgBackup\Job\BackupRun::progressOf($r);
+		return $p === null ? '' : sprintf(' %s%d%%', $p['approximate'] ? '~' : '', (int)floor($p['fraction'] * 100.0));
+	}
 }

lib/Controller/AdminApiController.php

@@ -305,6 +305,6 @@ final class AdminApiController extends Controller {
 		return ['id' => $r->getId(), 'targetId' => $r->getTargetId(), 'kind' => $r->getKind(), 'status' => $r->getStatus(),
 			'phase' => $r->getPhase(), 'snapshotId' => $r->getSnapshotId(), 'snapshot' => $snapshot, 'startedAt' => $r->getStartedAt(),
 			'updatedAt' => $r->getUpdatedAt(), 'finishedAt' => $r->getFinishedAt(), 'error' => $r->getError(),
-			'stats' => json_decode($r->getStats() ?? 'null', true)];
+			'stats' => json_decode($r->getStats() ?? 'null', true), 'progress' => \OCA\NgBackup\Job\BackupRun::progressOf($r)];
 	}
 }

lib/Job/BackupRun.php

@@ -160,6 +160,38 @@ final class BackupRun {
 		return $state;
 	}
 
+	/**
+	 * Progress of the files phase as a fraction (at most 0.99 until it is done), or null without an
+	 * estimate. Bytes of a large file that is still being read count too.
+	 *
+	 * @return array{fraction: float, approximate: bool}|null
+	 */
+	public static function progress(array $state): ?array {
+		$estimate = $state['estimate'] ?? null;
+		if (($state['phase'] ?? null) !== 'files' || !is_array($estimate) || ($estimate['bytes'] ?? 0) <= 0) {
+			return null;
+		}
+		$done = (int)($state['stats']['bytes'] ?? 0) + (int)($state['cur']['offset'] ?? 0);
+		return ['fraction' => min(0.99, $done / (int)$estimate['bytes']), 'approximate' => (bool)($estimate['approximate'] ?? true)];
+	}
+
+	/**
+	 * progress() of a run, while it is in its files phase.
+	 *
+	 * @return array{fraction: float, approximate: bool}|null
+	 */
+	public static function progressOf(\OCA\NgBackup\Db\Run $run): ?array {
+		if ($run->getStatus() !== \OCA\NgBackup\Db\Run::RUNNING || $run->getPhase() !== 'files') {
+			return null;
+		}
+		try {
+			$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
+		} catch (\JsonException) {
+			return null;
+		}
+		return is_array($state['files'] ?? null) ? self::progress($state['files']) : null;
+	}
+
 	/**
 	 * A file or directory that could not be read: it is missing from this snapshot. Remembered (first
 	 * 1000, deduplicated across steps) so the run can report it instead of silently succeeding.

lib/Service/BackupService.php

@@ -17,6 +17,7 @@ use OCA\NgBackup\Job\BackupRun;
 use OCA\NgBackup\Repository\RepositoryException;
 use OCP\App\IAppManager;
 use OCP\IConfig;
+use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\IDBConnection;
 use OCP\Lock\ILockingProvider;
 use OCP\Lock\LockedException;
@@ -68,6 +69,46 @@ final class BackupService {
 		}
 	}
 
+	/**
+	 * The expected size of the files phase, for the progress shown while it runs: the size of the
+	 * previous snapshot, or for a first backup the users' files, versions and trash as Nextcloud's
+	 * file cache knows them (approximate: other data in the data directory is not counted).
+	 *
+	 * @return array{bytes: int, approximate: bool}
+	 */
+	private function estimateBytes(Run $run, array $files): array {
+		if (($files['parent'] ?? null) !== null) {
+			try {
+				return ['bytes' => $this->snapshots->findOne($run->getTargetId(), (string)$files['parent'])->getBytes() ?? 0, 'approximate' => false];
+			} catch (\Throwable) {
+			}
+		}
+		try {
+			$qb = $this->db->getQueryBuilder();
+			$qb->select('numeric_id')->from('storages')
+				->where($qb->expr()->like('id', $qb->createNamedParameter('home::%')))
+				->orWhere($qb->expr()->like('id', $qb->createNamedParameter('object::user:%')));
+			$result = $qb->executeQuery();
+			$storages = [];
+			while (($id = $result->fetchOne()) !== false) {
+				$storages[] = (int)$id;
+			}
+			$result->closeCursor();
+			$bytes = 0;
+			foreach (array_chunk($storages, 500) as $chunk) {
+				$qb = $this->db->getQueryBuilder();
+				$qb->select($qb->func()->sum('size'))->from('filecache')
+					->where($qb->expr()->in('storage', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
+					->andWhere($qb->expr()->in('path_hash', $qb->createNamedParameter(array_map('md5', ['files', 'files_versions', 'files_trashbin']), IQueryBuilder::PARAM_STR_ARRAY)))
+					->andWhere($qb->expr()->gt('size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
+				$bytes += (int)$qb->executeQuery()->fetchOne();
+			}
+			return ['bytes' => $bytes, 'approximate' => true];
+		} catch (\Throwable) {
+			return ['bytes' => 0, 'approximate' => true];
+		}
+	}
+
 	private function createRun(Target $target, string $label): Run {
 		$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
 		if ($running !== []) {
@@ -147,6 +188,7 @@ final class BackupService {
 					$state['dump'] = $dump;
 				}
 			} else {
+				$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
 				$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
 				if ($state['files']['phase'] === 'commit') {
 					// Catalog write under the catalog lock, from a repository opened inside it: a per-user

src/components/Backups.vue

@@ -49,7 +49,7 @@
 						<td>{{ targetName(run.targetId) }}</td>
 						<td>{{ run.kind }}</td>
 						<td>{{ run.snapshot ? new Date(run.snapshot.createdAt * 1000).toLocaleString() + (run.snapshot.label ? ' – ' + run.snapshot.label : '') : '' }}</td>
-						<td :class="'ngb-' + run.status">{{ run.status }}{{ run.status === 'running' ? ' (' + run.phase + ')' : '' }}{{ run.error ? ': ' + run.error : '' }}</td>
+						<td :class="'ngb-' + run.status">{{ run.status }}{{ run.status === 'running' ? ' (' + run.phase + percent(run) + ')' : '' }}{{ run.error ? ': ' + run.error : '' }}</td>
 						<td>{{ run.stats?.files?.files ?? run.stats?.restored ?? '' }}</td>
 						<td>{{ run.stats?.files ? formatSize(run.stats.files.uploaded) : '' }}</td>
 					</tr>
@@ -101,6 +101,8 @@ async function untrash(target, item) {
 	}
 }
 
+// " 68%", or " ~68%" when the total is estimated from Nextcloud's file cache (first backup)
+const percent = (run) => run.progress ? ' ' + (run.progress.approximate ? '~' : '') + Math.floor(run.progress.fraction * 100) + '%' : ''
 const isRunning = (targetId) => props.status.runs.some(r => r.targetId === targetId && r.kind === 'full' && r.status === 'running')
 const targetName = (id) => props.status.targets.find(tg => tg.id === id)?.name ?? '#' + id
 

tests/Unit/Job/BackupProgressTest.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\NgBackup\Tests\Unit\Job;
+
+use OCA\NgBackup\Job\BackupRun;
+use PHPUnit\Framework\TestCase;
+
+class BackupProgressTest extends TestCase {
+	public function testCountsFinishedFilesAndTheFileBeingRead(): void {
+		$state = ['phase' => 'files', 'estimate' => ['bytes' => 1000, 'approximate' => false],
+			'stats' => ['bytes' => 500], 'cur' => ['offset' => 180]];
+		$this->assertSame(['fraction' => 0.68, 'approximate' => false], BackupRun::progress($state));
+	}
+
+	public function testStaysBelowHundredPercentWhenTheEstimateIsTooLow(): void {
+		$state = ['phase' => 'files', 'estimate' => ['bytes' => 1000, 'approximate' => true], 'stats' => ['bytes' => 5000], 'cur' => null];
+		$this->assertSame(['fraction' => 0.99, 'approximate' => true], BackupRun::progress($state));
+	}
+
+	public function testNoProgressWithoutEstimateOrOutsideTheFilesPhase(): void {
+		$this->assertNull(BackupRun::progress(['phase' => 'files', 'stats' => ['bytes' => 1]]));
+		$this->assertNull(BackupRun::progress(['phase' => 'files', 'estimate' => ['bytes' => 0, 'approximate' => true], 'stats' => ['bytes' => 1]]));
+		$this->assertNull(BackupRun::progress(['phase' => 'commit', 'estimate' => ['bytes' => 10, 'approximate' => false], 'stats' => ['bytes' => 1]]));
+	}
+}