Repository: list flat folders without a type check per entry
Opening a repository lists index/, which holds one file per pack. StorageBackend::list() checked every entry with is_dir(), and on a remote storage each check is a round trip. Over SFTP to a Hetzner Storage Box (33 ms) with a 60 GiB repository, occ backup:run (30 s steps) spent nearly every step on these requests and uploaded 45 KiB/s; with this change it uploads 5.8 MB/s. The cost grows with the number of packs. index/, snapshots/, catalog/ and trash/info/ only ever hold objects: list them with the new IBackend::listFiles(), one listing without per-entry checks.
This commit is contained in:
@@ -37,5 +37,15 @@ interface IBackend {
|
||||
/** @return list<string> object paths below $prefix (recursive) */
|
||||
public function list(string $prefix): array;
|
||||
|
||||
/**
|
||||
* Object paths directly in $dir, for a folder that only holds objects (index/, snapshots/,
|
||||
* catalog/, trash/info/). One listing, without checking the type of every entry: on a remote
|
||||
* storage list() costs a round trip per entry, which for index/ (one file per pack) grows
|
||||
* with the size of the repository.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public function listFiles(string $dir): array;
|
||||
|
||||
public function delete(string $path): void;
|
||||
}
|
||||
|
||||
@@ -107,6 +107,22 @@ final class LocalBackend implements IBackend {
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
$base = $this->abs($dir);
|
||||
if (!is_dir($base)) {
|
||||
return [];
|
||||
}
|
||||
$result = [];
|
||||
$names = scandir($base);
|
||||
foreach ($names === false ? [] : $names as $name) {
|
||||
if ($name !== '.' && $name !== '..' && !StorageBackend::isTemporary($name)) {
|
||||
$result[] = trim($dir, '/') . '/' . $name;
|
||||
}
|
||||
}
|
||||
sort($result);
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
@unlink($this->abs($path));
|
||||
}
|
||||
|
||||
@@ -133,6 +133,27 @@ final class StorageBackend implements IBackend {
|
||||
}
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
$abs = $this->abs($dir);
|
||||
if (!$this->storage->is_dir($abs)) {
|
||||
return []; // not created yet
|
||||
}
|
||||
$dh = $this->storage->opendir($abs);
|
||||
if ($dh === false) {
|
||||
return [];
|
||||
}
|
||||
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
||||
$result = [];
|
||||
while (($name = readdir($dh)) !== false) {
|
||||
if ($name !== '.' && $name !== '..' && !self::isTemporary($name)) {
|
||||
$result[] = substr($abs . '/' . $name, $cut);
|
||||
}
|
||||
}
|
||||
closedir($dh);
|
||||
sort($result);
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function listInto(string $dir, array &$result): void {
|
||||
if (!$this->storage->is_dir($dir)) {
|
||||
return;
|
||||
|
||||
@@ -92,7 +92,7 @@ final class BlobIndex {
|
||||
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
|
||||
$present = [];
|
||||
$this->downloaded = 0;
|
||||
foreach ($this->backend->list('index') as $path) {
|
||||
foreach ($this->backend->listFiles('index') as $path) {
|
||||
$packId = basename($path);
|
||||
$present[$packId] = true;
|
||||
if (isset($cached[$packId])) {
|
||||
|
||||
@@ -128,7 +128,7 @@ final class Catalog {
|
||||
|
||||
/** @return list<int> */
|
||||
private function generations(): array {
|
||||
return array_map(fn ($p) => (int)basename($p), $this->backend->list('catalog'));
|
||||
return array_map(fn ($p) => (int)basename($p), $this->backend->listFiles('catalog'));
|
||||
}
|
||||
|
||||
private function path(int $gen): string {
|
||||
|
||||
@@ -92,8 +92,8 @@ final class Repository {
|
||||
private function loadCatalog(?ICatalogAnchor $anchor): void {
|
||||
$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
|
||||
$this->catalog->load(
|
||||
fn () => array_map('basename', $this->backend->list('snapshots')),
|
||||
fn () => array_map('basename', $this->backend->list('trash/info')),
|
||||
fn () => array_map('basename', $this->backend->listFiles('snapshots')),
|
||||
fn () => array_map('basename', $this->backend->listFiles('trash/info')),
|
||||
fn () => $this->backend->list('users'));
|
||||
}
|
||||
|
||||
@@ -643,7 +643,7 @@ final class Repository {
|
||||
* @return list<string>
|
||||
*/
|
||||
public function listSnapshots(): array {
|
||||
$present = array_flip(array_map('basename', $this->backend->list('snapshots')));
|
||||
$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
|
||||
$listed = array_keys($this->catalog->snapshots);
|
||||
foreach ($listed as $id) {
|
||||
if (!isset($present[$id])) {
|
||||
|
||||
Reference in New Issue
Block a user