dillard
ccbbf01c54
A restore read one pack at a time over one SFTP connection, which over a long line is bound by the round trip and by what one TCP connection gets: about 2-3 MiB/s from a Hetzner Storage Box to Canada, with PHP mostly waiting. Repository::restore() now first lists the ranges of packs the restore needs, in order (ReadAheadPlan), and the location reads them ahead into unlinked files in a spool folder in the target (IReadsAhead). Over SFTP, SftpRangeFetcher does that on four connections of its own, 16 requests in flight each, taking the answers of whichever connection has one. A range that fails, a damaged copy, or a blob that moved since the plan is read the usual way, which reports what is wrong. Other locations restore as before. On a test line with 120 ms RTT and 0.2% loss, a 1.6 GiB restore went from 5.1 to 17.8 MiB/s, all files identical.
104 lines
3.7 KiB
PHP
104 lines
3.7 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace OCA\NgBackup\Tests\Unit\Backend;
|
|
|
|
use OCA\NgBackup\Backend\SftpRangeFetcher;
|
|
use OCA\NgBackup\Backend\SftpReadStream;
|
|
use OCA\NgBackup\Backend\StorageBackend;
|
|
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
|
use OCP\Files\Storage\IStorage;
|
|
use PHPUnit\Framework\TestCase;
|
|
|
|
require_once __DIR__ . '/SftpReadStreamTest.php'; // FakeSftpConnection
|
|
require_once __DIR__ . '/StorageBackendReconnectTest.php'; // SftpLikeStorage
|
|
|
|
/** StorageBackend::readAhead(): over SFTP on connections of its own, built with the reconnect function. */
|
|
class StorageBackendReadAheadTest extends TestCase {
|
|
use TempDirTrait;
|
|
|
|
/** @var list<FakeSftpConnection> */
|
|
private array $connections = [];
|
|
/** @var array<string, string> */
|
|
private array $files = [];
|
|
/** New storages after this many fail to connect. */
|
|
private ?int $connectLimit = null;
|
|
|
|
protected function setUp(): void {
|
|
SftpReadStream::reset();
|
|
for ($i = 0; $i < 6; $i++) {
|
|
$this->files["/home/base/repo/packs/p$i"] = random_bytes(1048576 + $i);
|
|
}
|
|
}
|
|
|
|
private function storage(): IStorage {
|
|
if ($this->connectLimit !== null && count($this->connections) >= $this->connectLimit) {
|
|
throw new \RuntimeException('Too many connections');
|
|
}
|
|
$sftp = new FakeSftpConnection($this->files);
|
|
$this->connections[] = $sftp;
|
|
$storage = $this->createMock(SftpLikeRootStorage::class);
|
|
$storage->method('instanceOfStorage')->willReturnCallback(fn (string $class): bool => $class === \OCA\Files_External\Lib\Storage\SFTP::class);
|
|
$storage->method('getConnection')->willReturn($sftp);
|
|
$storage->method('getRoot')->willReturn('/home/');
|
|
$storage->method('is_dir')->willReturn(true);
|
|
return $storage;
|
|
}
|
|
|
|
/** @return list<array{0:string, 1:int, 2:int}> */
|
|
private static function ranges(): array {
|
|
return array_map(fn (int $i): array => ["packs/p$i", 7, 1048576 - 7], range(0, 5));
|
|
}
|
|
|
|
public function testReadsAheadOnNewConnectionsOnly(): void {
|
|
$backend = new StorageBackend($this->storage(), 'base/repo', false, fn (): IStorage => $this->storage());
|
|
$this->assertTrue($backend->canReadAhead());
|
|
|
|
$result = [];
|
|
foreach ($backend->readAhead(self::ranges(), $this->tempDir()) as $i => $fh) {
|
|
$result[$i] = sha1(stream_get_contents($fh));
|
|
}
|
|
|
|
$expected = array_map(fn (int $i): string => sha1(substr($this->files["/home/base/repo/packs/p$i"], 7, 1048576 - 7)), range(0, 5));
|
|
$this->assertSame($expected, $result);
|
|
$this->assertCount(1 + SftpRangeFetcher::CONNECTIONS, $this->connections);
|
|
$this->assertSame(0, $this->connections[0]->sent, 'the storage\'s own connection stays free');
|
|
foreach (array_slice($this->connections, 1) as $c) {
|
|
$this->assertTrue($c->disconnected);
|
|
}
|
|
}
|
|
|
|
public function testGoesOnWithTheConnectionsTheServerAllows(): void {
|
|
$this->connectLimit = 3;
|
|
$backend = new StorageBackend($this->storage(), 'base/repo', false, fn (): IStorage => $this->storage());
|
|
|
|
$count = 0;
|
|
foreach ($backend->readAhead(self::ranges(), $this->tempDir()) as $fh) {
|
|
$this->assertIsResource($fh);
|
|
$count++;
|
|
}
|
|
|
|
$this->assertSame(6, $count);
|
|
$this->assertCount(3, $this->connections);
|
|
}
|
|
|
|
public function testNoNewConnectionMeansNoReadingAhead(): void {
|
|
$this->connectLimit = 1;
|
|
$backend = new StorageBackend($this->storage(), 'base/repo', false, fn (): IStorage => $this->storage());
|
|
|
|
$this->assertNull($backend->readAhead(self::ranges(), $this->tempDir()));
|
|
}
|
|
|
|
public function testWithoutAWayToConnectAgainItDoesNotReadAhead(): void {
|
|
$backend = new StorageBackend($this->storage(), 'base/repo', false, null);
|
|
|
|
$this->assertFalse($backend->canReadAhead());
|
|
$this->assertNull($backend->readAhead(self::ranges(), $this->tempDir()));
|
|
}
|
|
}
|
|
|
|
interface SftpLikeRootStorage extends SftpLikeStorage {
|
|
public function getRoot(): string;
|
|
}
|