dillard
ca003c88ba
On S3, index files and the tree segments read back when a snapshot is written are fetched 16 at a time through the storage's S3 client (IBackend::getRanges(), getMany()). Blobs that follow each other in a pack are read as one range on every location. The snapshot is written in a step of its own and the run shows "commit" meanwhile. On a 1 TB production repository the check went from about 4 hours to 3.5 minutes. backup:status and the web interface list the runs of the last 7 days, running ones and at least the newest 5; --all / "Show older runs" list all. Durations of a day or more show the days.
41 lines
970 B
PHP
41 lines
970 B
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
namespace OCA\NgBackup\Backend;
|
|
|
|
/** IBackend::getMany() and getRanges() for a backend where reading one after the other costs nothing extra. */
|
|
trait GetsOneByOne {
|
|
/**
|
|
* @param list<string> $paths
|
|
* @return \Generator<string, string>
|
|
*/
|
|
public function getMany(array $paths): \Generator {
|
|
foreach ($paths as $path) {
|
|
$fh = $this->get($path);
|
|
try {
|
|
$data = stream_get_contents($fh);
|
|
} finally {
|
|
fclose($fh);
|
|
}
|
|
if ($data === false) {
|
|
throw new BackendException('Read failed for ' . $path);
|
|
}
|
|
yield $path => $data;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param list<array{0:string, 1:int, 2:int}> $ranges
|
|
* @return \Generator<int, string>
|
|
*/
|
|
public function getRanges(array $ranges): \Generator {
|
|
foreach ($ranges as $i => [$path, $offset, $length]) {
|
|
yield $i => $this->getRange($path, $offset, $length);
|
|
}
|
|
}
|
|
}
|