Repository: list flat folders without a type check per entry

Opening a repository lists index/, which holds one file per pack.
StorageBackend::list() checked every entry with is_dir(), and on a
remote storage each check is a round trip. Over SFTP to a Hetzner Storage
Box (33 ms) with a 60 GiB repository, occ backup:run (30 s steps) spent
nearly every step on these requests and uploaded 45 KiB/s; with this
change it uploads 5.8 MB/s. The cost grows with the number of packs.

index/, snapshots/, catalog/ and trash/info/ only ever hold objects:
list them with the new IBackend::listFiles(), one listing without
per-entry checks.
This commit is contained in:
2026-10-08 18:28:29 +02:00
parent 62766e1e71
commit 6bba0b59fc
9 changed files with 113 additions and 5 deletions
+10
View File
@@ -37,5 +37,15 @@ interface IBackend {
/** @return list<string> object paths below $prefix (recursive) */
public function list(string $prefix): array;
/**
* Object paths directly in $dir, for a folder that only holds objects (index/, snapshots/,
* catalog/, trash/info/). One listing, without checking the type of every entry: on a remote
* storage list() costs a round trip per entry, which for index/ (one file per pack) grows
* with the size of the repository.
*
* @return list<string>
*/
public function listFiles(string $dir): array;
public function delete(string $path): void;
}
+16
View File
@@ -107,6 +107,22 @@ final class LocalBackend implements IBackend {
return $result;
}
public function listFiles(string $dir): array {
$base = $this->abs($dir);
if (!is_dir($base)) {
return [];
}
$result = [];
$names = scandir($base);
foreach ($names === false ? [] : $names as $name) {
if ($name !== '.' && $name !== '..' && !StorageBackend::isTemporary($name)) {
$result[] = trim($dir, '/') . '/' . $name;
}
}
sort($result);
return $result;
}
public function delete(string $path): void {
@unlink($this->abs($path));
}
+21
View File
@@ -133,6 +133,27 @@ final class StorageBackend implements IBackend {
}
}
public function listFiles(string $dir): array {
$abs = $this->abs($dir);
if (!$this->storage->is_dir($abs)) {
return []; // not created yet
}
$dh = $this->storage->opendir($abs);
if ($dh === false) {
return [];
}
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
$result = [];
while (($name = readdir($dh)) !== false) {
if ($name !== '.' && $name !== '..' && !self::isTemporary($name)) {
$result[] = substr($abs . '/' . $name, $cut);
}
}
closedir($dh);
sort($result);
return $result;
}
private function listInto(string $dir, array &$result): void {
if (!$this->storage->is_dir($dir)) {
return;