Repository: list flat folders without a type check per entry

Opening a repository lists index/, which holds one file per pack.
StorageBackend::list() checked every entry with is_dir(), and on a
remote storage each check is a round trip. Over SFTP to a Hetzner Storage
Box (33 ms) with a 60 GiB repository, occ backup:run (30 s steps) spent
nearly every step on these requests and uploaded 45 KiB/s; with this
change it uploads 5.8 MB/s. The cost grows with the number of packs.

index/, snapshots/, catalog/ and trash/info/ only ever hold objects:
list them with the new IBackend::listFiles(), one listing without
per-entry checks.
This commit is contained in:
2026-10-08 18:28:29 +02:00
parent 62766e1e71
commit 6bba0b59fc
9 changed files with 113 additions and 5 deletions
@@ -0,0 +1,47 @@
<?php
declare(strict_types=1);
namespace OCA\NgBackup\Tests\Unit\Backend;
use OCA\NgBackup\Backend\StorageBackend;
use OCA\NgBackup\Tests\Unit\TempDirTrait;
use OCP\Files\Storage\IStorage;
use PHPUnit\Framework\TestCase;
/** On a remote storage every is_dir() is a round trip: listing index/ must not cost one per pack. */
class StorageBackendListTest extends TestCase {
use TempDirTrait;
public function testListFilesChecksOnlyTheFolderItself(): void {
$dir = $this->tempDir();
foreach (['b1', 'a2', 'c3.part-0123abcd'] as $name) {
touch("$dir/$name");
}
$isDirCalls = 0;
$storage = $this->createMock(IStorage::class);
$storage->method('instanceOfStorage')->willReturn(false);
$storage->method('is_dir')->willReturnCallback(function (string $path) use (&$isDirCalls): bool {
$isDirCalls++;
return $path === 'repo' || $path === 'repo/index';
});
$storage->method('opendir')->willReturnCallback(fn (string $path) => $path === 'repo/index' ? opendir($dir) : false);
$backend = new StorageBackend($storage, 'repo');
$isDirCalls = 0; // the constructor checks the base folder
$this->assertSame(['index/a2', 'index/b1'], $backend->listFiles('index'));
$this->assertSame(1, $isDirCalls);
$isDirCalls = 0;
$this->assertSame(['index/a2', 'index/b1'], $backend->list('index'));
$this->assertSame(4, $isDirCalls, 'list() checks every entry: the reason listFiles() exists');
}
public function testListFilesOfAMissingFolderIsEmpty(): void {
$storage = $this->createMock(IStorage::class);
$storage->method('instanceOfStorage')->willReturn(false);
$storage->method('is_dir')->willReturnCallback(fn (string $path): bool => $path === 'repo');
$storage->expects($this->never())->method('opendir');
$this->assertSame([], (new StorageBackend($storage, 'repo'))->listFiles('snapshots'));
}
}
+4
View File
@@ -166,6 +166,10 @@ final class FlakyRangeBackend implements IBackend {
return $this->inner->list($prefix);
}
public function listFiles(string $dir): array {
return $this->inner->listFiles($dir);
}
public function delete(string $path): void {
$this->inner->delete($path);
}
+10
View File
@@ -3,3 +3,13 @@
declare(strict_types=1);
require __DIR__ . '/../vendor/autoload.php';
// nextcloud/ocp ships the public interfaces without an autoloader; tests that mock them need one.
spl_autoload_register(static function (string $class): void {
if (str_starts_with($class, 'OCP\\')) {
$file = __DIR__ . '/../vendor/nextcloud/ocp/' . str_replace('\\', '/', $class) . '.php';
if (is_file($file)) {
require $file;
}
}
});