Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Fast S3 verify and snapshot commit; shorter runs list; durations with days

Browse Source 
On S3, index files and the tree segments read back when a snapshot is
written are fetched 16 at a time through the storage's S3 client
(IBackend::getRanges(), getMany()). Blobs that follow each other in a pack
are read as one range on every location. The snapshot is written in a step
of its own and the run shows "commit" meanwhile. On a 1 TB production
repository the check went from about 4 hours to 3.5 minutes.

backup:status and the web interface list the runs of the last 7 days,
running ones and at least the newest 5; --all / "Show older runs" list
all. Durations of a day or more show the days.
This commit is contained in:
Dillard Blom
2026-10-14 17:12:41 +02:00
parent 9a3bb99d12
commit ca003c88ba
20 changed files with 424 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Backend/GetsOneByOne.php
+11 -1
View File
@@ -7,7 +7,7 @@ declare(strict_types=1);
namespace OCA\NgBackup\Backend;
/** IBackend::getMany() for a backend where reading files one after the other costs nothing extra. */
/** IBackend::getMany() and getRanges() for a backend where reading one after the other costs nothing extra. */
trait GetsOneByOne {
/**
* @param list<string> $paths
@@ -27,4 +27,14 @@ trait GetsOneByOne {
yield $path => $data;
}
}
/**
* @param list<array{0:string, 1:int, 2:int}> $ranges
* @return \Generator<int, string>
*/
public function getRanges(array $ranges): \Generator {
foreach ($ranges as $i => [$path, $offset, $length]) {
yield $i => $this->getRange($path, $offset, $length);
}
}
}
lib/Backend/IBackend.php
+11
View File
@@ -35,6 +35,17 @@ interface IBackend {
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
public function getRange(string $path, int $offset, int $length): string;
/**
* Many byte ranges, in the order given (blobs spread over many packs, such as the tree
* segments of a backup). A backend where every request costs a round trip can fetch them
* several at a time.
*
* @param list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] each
* @return \Generator<int, string> contents by position in $ranges
* @throws BackendException for a range that cannot be read
*/
public function getRanges(array $ranges): \Generator;
public function exists(string $path): bool;
/**
lib/Backend/StorageBackend.php
+101 -3
View File
@@ -19,6 +19,13 @@ final class StorageBackend implements IBackend {
private bool $atomicPut;
/** SFTP: read through SftpReadStream, which keeps several requests in flight. */
private bool $sftpReads;
/** S3: many small reads go straight to the storage's S3 client, several at a time. */
private bool $s3Reads;
/** S3 requests in flight at once for getMany() and getRanges(). */
public const S3_CONCURRENCY = 16;
/** Most ranges, and bytes, getRanges() holds in memory at once. */
private const RANGES_LOT = 64;
private const RANGES_LOT_BYTES = 64 * 1048576;
/** Seconds without a request after which an SFTP connection is checked before it is used. */
public const IDLE_CHECK = 60;
private float $lastUsed;
@@ -40,6 +47,7 @@ final class StorageBackend implements IBackend {
self::validate($this->base, true);
$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
$this->sftpReads = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\SFTP::class);
$this->s3Reads = $storage instanceof \OCA\Files_External\Lib\Storage\AmazonS3;
if ($createBase && $this->base !== '') {
$this->mkdirs($this->base);
}
@@ -116,11 +124,19 @@ final class StorageBackend implements IBackend {
return $fh;
}
/** Over SFTP several at a time (SftpReadStream::readMany()); elsewhere one after the other. */
/**
* Over SFTP and S3 several at a time (SftpReadStream::readMany(), s3Fetch()); elsewhere one
* after the other.
*/
public function getMany(array $paths): \Generator {
foreach (array_chunk($paths, 8 * SftpReadStream::BATCH) as $chunk) {
$size = $this->s3Reads ? 16 * self::S3_CONCURRENCY : 8 * SftpReadStream::BATCH;
foreach (array_chunk($paths, $size) as $chunk) {
$fast = null;
if ($this->sftpReads) {
if ($this->s3Reads) {
$this->idleCheck();
$fast = $this->s3Fetch(array_map(fn (string $p): array => [$p, null], $chunk));
$fast = $fast === null ? null : array_combine($chunk, array_map(fn (int $i): ?string => $fast[$i] ?? null, array_keys($chunk)));
} elseif ($this->sftpReads) {
$abs = array_map(fn (string $p): string => $this->abs($p), $chunk);
$fast = $this->call(fn () => SftpReadStream::readMany($this->storage, $abs));
$fast = $fast === null ? null : array_combine($chunk, array_map(fn (string $a): ?string => $fast[$a] ?? null, $abs));
@@ -145,6 +161,88 @@ final class StorageBackend implements IBackend {
}
}
/** Over S3 several at a time (s3Fetch()); elsewhere one after the other. */
public function getRanges(array $ranges): \Generator {
$lot = [];
$bytes = 0;
foreach ($ranges as $i => $range) {
if ($lot !== [] && (count($lot) >= self::RANGES_LOT || $bytes + $range[2] > self::RANGES_LOT_BYTES)) {
yield from $this->rangesLot($lot);
$lot = [];
$bytes = 0;
}
$lot[$i] = $range;
$bytes += $range[2];
}
if ($lot !== []) {
yield from $this->rangesLot($lot);
}
}
/**
* @param array<int, array{0:string, 1:int, 2:int}> $lot
* @return \Generator<int, string>
*/
private function rangesLot(array $lot): \Generator {
$fast = null;
if ($this->s3Reads) {
$this->idleCheck();
$fast = $this->s3Fetch(array_map(fn (array $r): array => [$r[0], $r[1] . '-' . ($r[1] + $r[2] - 1)], $lot));
}
foreach ($lot as $i => [$path, $offset, $length]) {
$data = $fast[$i] ?? null;
// not read in the batch, or not all of it: the usual way, which also reports why
yield $i => ($data !== null && strlen($data) === $length) ? $data : $this->getRange($path, $offset, $length);
}
}
/**
* Read objects (or byte ranges of them, "first-last") straight through the S3 client of
* files_external's S3 storage, S3_CONCURRENCY at a time. Its own reads open a new HTTPS
* connection per object, after a HEAD request: one after the other that is about two reads a
* second, which made checking the index of a large repository take hours. What fails here
* is left out of the result, for the caller to read the usual way (which reports the error).
*
* @param array<int, array{0:string, 1:?string}> $requests [path, range or null] by key
* @return array<int, string>|null contents by key; null when the storage is not as expected
* @psalm-suppress UndefinedClass the AWS SDK comes with Nextcloud, not with this app
*/
private function s3Fetch(array $requests): ?array {
try {
/** @psalm-suppress UndefinedMethod, UndefinedThisPropertyFetch */
[$client, $bucket, $sse] = (function (): array {
$sse = method_exists($this, 'getServerSideEncryptionParameters') ? $this->getServerSideEncryptionParameters()
: (method_exists($this, 'getSSECParameters') ? $this->getSSECParameters() : []);
return [$this->getConnection(), $this->getBucket(), $sse];
})->call($this->storage);
if (!$client instanceof \Aws\S3\S3Client || !is_string($bucket)) {
return null;
}
$results = [];
$keys = array_keys($requests); // the pool numbers results 0, 1, ...: mapped back below
$commands = (function () use ($requests, $client, $bucket, $sse): \Generator {
foreach (array_values($requests) as $n => [$path, $range]) {
$args = ['Bucket' => $bucket, 'Key' => trim($this->abs($path), '/')] + $sse;
if ($range !== null) {
$args['Range'] = 'bytes=' . $range;
}
yield $n => $client->getCommand('GetObject', $args);
}
})();
(new \Aws\CommandPool($client, $commands, [
'concurrency' => self::S3_CONCURRENCY,
'fulfilled' => function (mixed $result, int $n) use (&$results, $keys): void {
$results[$keys[$n]] = (string)$result['Body'];
},
'rejected' => static function (): void {
},
]))->promise()->wait();
return $results;
} catch (\Throwable) {
return null;
}
}
public function getRange(string $path, int $offset, int $length): string {
return $this->call(fn () => $this->readRange($path, $offset, $length));
}
lib/Command/Status.php
+14 -3
View File
@@ -14,9 +14,13 @@ use OCA\NgBackup\Service\VerifyService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
final class Status extends Command {
/** Runs of the last days that are listed by default (failed ones stay visible that long). */
public const RECENT_DAYS = 7;
public function __construct(
private KeyService $keys,
private TargetService $targets,
@@ -27,7 +31,8 @@ final class Status extends Command {
}
protected function configure(): void {
$this->setName('backup:status')->setDescription('Show the backup key, locations and recent runs');
$this->setName('backup:status')->setDescription('Show the backup key, locations and recent runs')
->addOption('all', null, InputOption::VALUE_NONE, 'List all runs, not only those of the last ' . self::RECENT_DAYS . ' days');
}
protected function execute(InputInterface $input, OutputInterface $output): int {
@@ -44,16 +49,22 @@ final class Status extends Command {
}
$output->writeln('Locations: ' . ($names === [] ? 'none' : implode(', ', $names)));
$rows = [];
foreach ($this->runs->findRecent(10) as $r) {
$all = (bool)$input->getOption('all');
$shown = $this->runs->findShown($all ? null : time() - self::RECENT_DAYS * 86400);
foreach ($shown as $r) {
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . self::percent($r) . ')' : ''),
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? \OCA\NgBackup\Format::duration($r->getFinishedAt() - $r->getStartedAt()) : '',
$s['files'] ?? '', isset($s['bytes']) ? self::size((int)$s['bytes']) : '',
isset($s['uploaded']) ? self::size((int)$s['uploaded']) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
}
if ($rows !== []) {
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'size', 'uploaded', 'error'])->setRows($rows)->render();
}
$older = $all ? 0 : $this->runs->countAll() - count($shown);
if ($older > 0) {
$output->writeln(sprintf('%d older run%s not shown (--all)', $older, $older === 1 ? '' : 's'));
}
foreach ($this->verify->progress() as $targetId => $state) {
$output->writeln(sprintf('Verify of %s on %s: %s', mb_strimwidth((string)($state['what'] ?? ''), 0, 40, '…'),
$names[$targetId] ?? '#' . $targetId, VerifyService::describe($state)));
lib/Controller/AdminApiController.php
+6 -2
View File
@@ -51,14 +51,16 @@ final class AdminApiController extends Controller {
parent::__construct(Application::APP_ID, $request);
}
public function status(): JSONResponse {
/** @param bool $allRuns all runs instead of those of the last days (Status::RECENT_DAYS) */
public function status(bool $allRuns = false): JSONResponse {
$targets = [];
foreach ($this->targets->list() as $t) {
$targets[] = ['id' => $t->getId(), 'name' => $t->getName(), 'backend' => $t->getBackend(), 'path' => $t->getBasePath(),
'repositoryId' => $t->getRepositoryId(), 'createdAt' => $t->getCreatedAt(), 'appendOnly' => (bool)$t->getAppendOnly(),
'maxBytes' => $t->getMaxBytes(), 'storedBytes' => $this->alerts->lastStoredBytes($t)];
}
$runs = array_map(fn (Run $r) => $this->runJson($r), $this->runs->findRecent(15));
$shown = $this->runs->findShown($allRuns ? null : time() - \OCA\NgBackup\Command\Status::RECENT_DAYS * 86400);
$runs = array_map(fn (Run $r) => $this->runJson($r), $shown);
return new JSONResponse([
'key' => ['initialized' => $this->keys->isInitialized(), 'fingerprint' => $this->keys->fingerprint(),
'confirmation' => $this->keys->recoveryKitConfirmation(), 'everConfirmed' => $this->keys->isInitialized() && $this->keys->everConfirmed(),
@@ -68,6 +70,8 @@ final class AdminApiController extends Controller {
'confirmationPhrase' => KeyConfirm::CONFIRMATION,
'targets' => $targets,
'runs' => $runs,
'olderRuns' => $allRuns ? 0 : $this->runs->countAll() - count($shown),
'recentDays' => \OCA\NgBackup\Command\Status::RECENT_DAYS,
'schedule' => $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', ''),
]);
}
lib/Db/RunMapper.php
+34
View File
@@ -73,6 +73,40 @@ final class RunMapper extends QBMapper {
return $this->findEntities($qb);
}
/**
* The runs to list, newest first: those started at or after $since, those still running, and
* at least the newest $atLeast, so the list is never empty after a quiet week. All runs when
* $since is null.
*
* @return list<Run>
*/
public function findShown(?int $since, int $atLeast = 5): array {
$qb = $this->db->getQueryBuilder();
$qb->select('*')->from($this->getTableName())->orderBy('id', 'DESC');
if ($since === null) {
return $this->findEntities($qb);
}
$qb->where($qb->expr()->orX(
$qb->expr()->gte('started_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT)),
$qb->expr()->eq('status', $qb->createNamedParameter(Run::RUNNING)),
));
$runs = [];
foreach ([...$this->findEntities($qb), ...$this->findRecent($atLeast)] as $r) {
$runs[$r->getId()] = $r;
}
krsort($runs);
return array_values($runs);
}
public function countAll(): int {
$qb = $this->db->getQueryBuilder();
$qb->select($qb->func()->count('*', 'n'))->from($this->getTableName());
$result = $qb->executeQuery();
$n = (int)$result->fetchOne();
$result->closeCursor();
return $n;
}
/** @return list<Run> runs of $kind started at or after $since, newest first */
public function startedSince(int $targetId, string $kind, int $since): array {
$qb = $this->db->getQueryBuilder();
lib/Format.php
+18
View File
@@ -0,0 +1,18 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup;
final class Format {
/** "12:10:13", or "1d 12:10:13" from a day on (a long first backup takes days). */
public static function duration(int $seconds): string {
$seconds = max(0, $seconds);
$days = intdiv($seconds, 86400);
$time = gmdate('H:i:s', $seconds % 86400);
return $days > 0 ? $days . 'd ' . $time : $time;
}
}
lib/Job/BackupRun.php
+6 -6
View File
@@ -140,21 +140,21 @@ final class BackupRun {
$state['parentPos'] = $parent->position();
$state['stats'] = $stats;
if ($finishedWalk) {
// The snapshot is written in a step of its own, with its own time: after a long
// run it reads back thousands of segments, and the run reports that it got here.
$state['phase'] = 'finish';
}
return $state;
}
if ($state['phase'] === 'finish') {
$segments = $state['segments'];
// Every segment ends with a newline, so they read as one stream: fetched together,
// several at a time where the location allows it.
$repo->putSnapshot($state['snapshot'],
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
'roots' => array_keys($state['roots']), 'stats' => $state['stats'],
'warnings' => self::warnings($state)] + $state['meta'],
(function () use ($repo, $segments) {
foreach ($segments as $blobs) {
yield from $repo->readLines($blobs);
}
})());
$repo->readLines(array_values(array_merge(...$state['segments']))));
$state['phase'] = 'commit';
}
return $state;
lib/Repository/PackReaderGuard.php
+5
View File
@@ -38,6 +38,11 @@ final class PackReaderGuard implements IBackend {
return $this->inner->getRange($path, $offset, $length);
}
public function getRanges(array $ranges): \Generator {
$this->reader->close();
return $this->inner->getRanges($ranges);
}
public function exists(string $path): bool {
$this->reader->close();
return $this->inner->exists($path);
lib/Repository/Repository.php
+48 -2
View File
@@ -30,6 +30,8 @@ final class Repository {
private const FLAG_DEFLATE = 1;
/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
private const VERIFY_READ_ATTEMPTS = 3;
/** Largest read that loadBlobs() makes of blobs that follow each other in a pack. */
private const MERGED_READ_MAX = 16 * 1048576;
private StreamCipher $cipher;
private BlobIndex $index;
@@ -288,8 +290,8 @@ final class Repository {
*/
public function readLines(array $blobIds): \Generator {
$carry = '';
foreach ($blobIds as $id) {
$data = $carry . $this->loadBlob($id);
foreach ($this->loadBlobs($blobIds) as $blob) {
$data = $carry . $blob;
$lines = explode("\n", $data);
$carry = array_pop($lines);
yield from $lines;
@@ -711,6 +713,50 @@ final class Repository {
return $id;
}
/**
* Blobs spread over many packs (the tree segments of a run, one per step), through
* IBackend::getRanges(): on a remote location one at a time is a round trip each, which for
* a long run made writing its snapshot take a quarter of an hour. Blobs that follow each
* other in a pack are read as one range.
*
* @param list<string> $ids
* @return \Generator<int, string> verified blob contents, in order
*/
private function loadBlobs(array $ids): \Generator {
/** @var list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] of each read */
$ranges = [];
/** @var list<list<array{0:string, 1:int, 2:int}>> $parts per read: [blob id, offset in the read, length] */
$parts = [];
// the read being extended
$curPath = null;
$curOffset = 0;
$curLength = 0;
$curParts = [];
foreach ($ids as $id) {
[$pack, $offset, $length] = $this->index->get($id);
$path = 'packs/' . substr($pack, 0, 2) . '/' . $pack;
if ($curPath === $path && $curOffset + $curLength === $offset && $curLength + $length <= self::MERGED_READ_MAX) {
$curParts[] = [$id, $curLength, $length];
$curLength += $length;
continue;
}
if ($curPath !== null) {
$ranges[] = [$curPath, $curOffset, $curLength];
$parts[] = $curParts;
}
[$curPath, $curOffset, $curLength, $curParts] = [$path, $offset, $length, [[$id, 0, $length]]];
}
if ($curPath !== null) {
$ranges[] = [$curPath, $curOffset, $curLength];
$parts[] = $curParts;
}
foreach ($this->backend->getRanges($ranges) as $i => $data) {
foreach ($parts[$i] as [$id, $at, $length]) {
yield $this->decodeBlob($id, substr($data, $at, $length));
}
}
}
private function loadBlob(string $id): string {
[$pack, $offset, $length] = $this->index->get($id);
return $this->decodeBlob($id, $this->packReader->read($pack, $offset, $length));
lib/Service/BackupService.php
+3
View File
@@ -225,6 +225,9 @@ final class BackupService {
} else {
$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
if (in_array($state['files']['phase'], ['finish', 'commit'], true)) {
$run->setPhase('commit'); // all files read: writing and recording the snapshot
}
if ($state['files']['phase'] === 'commit') {
// Catalog write under the catalog lock, from a repository opened inside it: a per-user
// export or restore may have written the catalog since this step opened $repo.
lib/Service/VerifyService.php
+1 -1
View File
@@ -64,7 +64,7 @@ final class VerifyService {
$age < 120 ? $age . ' s' : intdiv($age, 60) . ' min',
$age > 600 ? ' (no progress for a while: the process may have stopped)' : '');
}
$took = gmdate('H:i:s', max(0, (int)($s['finished'] ?? 0) - (int)($s['started'] ?? 0)));
$took = \OCA\NgBackup\Format::duration((int)($s['finished'] ?? 0) - (int)($s['started'] ?? 0));
return match ($s['status'] ?? '') {
'ok' => sprintf('%s OK on %s (took %s): %s files, %s blobs%s', $kind, date('Y-m-d H:i', (int)$s['finished']), $took,
number_format((int)($s['files'] ?? 0)), number_format((int)($s['blobs'] ?? 0)), $deep ? ', ' . $size((int)($s['bytes'] ?? 0)) . ' decrypted' : ''),