SftpReadStream: fall back to the storage's own stream instead of failing a read
This commit is contained in:
+4
-1
@@ -18,7 +18,10 @@ instead carried by the app description, the git tag, the release, and this file.
|
||||
- Reading from an SFTP location waited a full round trip for every 256 KiB, whatever the line
|
||||
speed. Reads now keep up to 8 requests in flight. Restoring 1.97 GB in 500 files from a Hetzner
|
||||
Storage Box (47 ms away) took 694 s before the fix above, 499 s with it and 142 s with both
|
||||
(2.8 → 3.9 → 13.9 MB/s); a full 51 GB restore ran at 13.0 MB/s.
|
||||
(2.8 → 3.9 → 13.9 MB/s); a full 51 GB restore ran at 13.0 MB/s. When a server error
|
||||
interrupts such a read, it goes on from the same position with Nextcloud's own SFTP stream;
|
||||
after three failing files in a row, or on a Nextcloud whose SFTP library lacks what this needs,
|
||||
only Nextcloud's own stream is used.
|
||||
- README: a WebDAV location on a slow upload needs a higher `davstorage.request_timeout`.
|
||||
|
||||
## [0.9.3-beta.4] - 2026-10-08
|
||||
|
||||
+183
-33
@@ -17,7 +17,10 @@ namespace OCA\NgBackup\Backend;
|
||||
*
|
||||
* Talks to the storage's phpseclib connection through the same internal methods files_external's
|
||||
* read stream uses (_send_sftp_packet, _get_sftp_packet, _realpath); open() returns null when the
|
||||
* connection does not have them, and StorageBackend then falls back to the storage's own stream.
|
||||
* connection does not offer them (phpseclib 3 does not), and StorageBackend then falls back to the
|
||||
* storage's own stream. A restore must rather be slow than fail: when a read fails halfway, the
|
||||
* stream goes on from the same position with the storage's own stream. After MAX_FAILURES streams
|
||||
* in a row failed, open() returns null for the rest of the process; one that works resets the count.
|
||||
* Like that stream, it owns the connection while open: no other request may go over the same
|
||||
* connection until it is closed (PackReaderGuard takes care of that for the repository).
|
||||
*/
|
||||
@@ -47,6 +50,11 @@ final class SftpReadStream {
|
||||
*/
|
||||
private static ?\WeakMap $chunks = null;
|
||||
private static int $requestId = 0x4e470000;
|
||||
/** Streams in a row that failed; at MAX_FAILURES only the storage's own stream is used. */
|
||||
private static int $failures = 0;
|
||||
public const MAX_FAILURES = 3;
|
||||
/** Set by stream_open() when the connection failed rather than the file. */
|
||||
private static bool $openBroke = false;
|
||||
|
||||
/** @var resource|null set by PHP for stream wrappers */
|
||||
public $context;
|
||||
@@ -58,6 +66,11 @@ final class SftpReadStream {
|
||||
* @psalm-suppress PropertyNotSetInConstructor
|
||||
*/
|
||||
private object $sftp;
|
||||
/** The storage and the path on it, to fall back to the storage's own stream. */
|
||||
private ?object $storage = null;
|
||||
private string $file = '';
|
||||
/** @var resource|null the storage's own stream, once this one has failed */
|
||||
private $fallback = null;
|
||||
private string $handle = '';
|
||||
/** Position of the reader: the first byte of $buffer. */
|
||||
private int $pos = 0;
|
||||
@@ -68,7 +81,10 @@ final class SftpReadStream {
|
||||
private array $inFlight = [];
|
||||
private int $depth = 1;
|
||||
private bool $eof = false;
|
||||
/** This stream gave up on its own requests. */
|
||||
private bool $failed = false;
|
||||
/** The connection is in an unknown state: nothing more is sent over it or read from it. */
|
||||
private bool $broken = false;
|
||||
/** Length of the last short answer, until the next answer tells whether it was the end of the file. */
|
||||
private ?int $shortAnswer = null;
|
||||
|
||||
@@ -76,28 +92,52 @@ final class SftpReadStream {
|
||||
* @param object $storage files_external's SFTP storage (getConnection(), getRoot())
|
||||
* @return resource|false|null the stream, false when the file cannot be opened, null when the
|
||||
* connection does not offer what this stream needs
|
||||
* @throws BackendException when the connection failed after the OPEN request went out: the
|
||||
* storage's own stream must not be tried on it, as it would take a
|
||||
* late answer to that request for its own
|
||||
*/
|
||||
public static function open(object $storage, string $path) {
|
||||
if (!method_exists($storage, 'getConnection') || !method_exists($storage, 'getRoot')) {
|
||||
if (self::$failures >= self::MAX_FAILURES) {
|
||||
return null;
|
||||
}
|
||||
$sftp = $storage->getConnection();
|
||||
if (!is_object($sftp) || !method_exists($sftp, '_send_sftp_packet') || !method_exists($sftp, '_get_sftp_packet')
|
||||
|| !method_exists($sftp, '_realpath') || !property_exists($sftp, 'packet_type')) {
|
||||
try {
|
||||
if (!is_callable([$storage, 'getConnection']) || !is_callable([$storage, 'getRoot'])) {
|
||||
return null;
|
||||
}
|
||||
$sftp = $storage->getConnection();
|
||||
// is_callable and get_object_vars from here only see what is public: phpseclib 3 has
|
||||
// methods of these names neither public nor at all.
|
||||
if (!is_object($sftp) || !is_callable([$sftp, '_send_sftp_packet']) || !is_callable([$sftp, '_get_sftp_packet'])
|
||||
|| !is_callable([$sftp, '_realpath']) || !array_key_exists('packet_type', get_object_vars($sftp))) {
|
||||
return null;
|
||||
}
|
||||
if (!in_array(self::PROTOCOL, stream_get_wrappers(), true)) {
|
||||
stream_wrapper_register(self::PROTOCOL, self::class);
|
||||
}
|
||||
$root = rtrim((string)$storage->getRoot(), '/');
|
||||
$context = stream_context_create([self::PROTOCOL => [
|
||||
'session' => $sftp,
|
||||
'path' => $root . '/' . ltrim($path, '/'),
|
||||
'storage' => $storage,
|
||||
'file' => $path,
|
||||
]]);
|
||||
self::$openBroke = false;
|
||||
$fh = @fopen(self::PROTOCOL . '://file', 'r', false, $context);
|
||||
} catch (\Throwable) {
|
||||
// nothing was sent yet
|
||||
self::$failures++;
|
||||
return null;
|
||||
}
|
||||
if (!in_array(self::PROTOCOL, stream_get_wrappers(), true)) {
|
||||
stream_wrapper_register(self::PROTOCOL, self::class);
|
||||
if ($fh === false && self::openBroke()) {
|
||||
throw new BackendException('SFTP connection failed while opening ' . $path);
|
||||
}
|
||||
$root = rtrim((string)$storage->getRoot(), '/');
|
||||
$context = stream_context_create([self::PROTOCOL => ['session' => $sftp, 'path' => $root . '/' . ltrim($path, '/')]]);
|
||||
$fh = @fopen(self::PROTOCOL . '://file', 'r', false, $context);
|
||||
return $fh === false ? false : $fh;
|
||||
return $fh;
|
||||
}
|
||||
|
||||
/** Test hook: forget the learned READ sizes. */
|
||||
/** Test hook: forget the learned READ sizes and earlier failures. */
|
||||
public static function reset(): void {
|
||||
self::$chunks = null;
|
||||
self::$failures = 0;
|
||||
}
|
||||
|
||||
public function stream_open(string $url, string $mode, int $options, ?string &$openedPath): bool {
|
||||
@@ -106,26 +146,51 @@ final class SftpReadStream {
|
||||
return false;
|
||||
}
|
||||
$this->sftp = $options['session'];
|
||||
$remote = $this->sftp->_realpath((string)$options['path']);
|
||||
if (!is_string($remote)) {
|
||||
return false;
|
||||
$storage = $options['storage'] ?? null;
|
||||
$this->storage = is_object($storage) ? $storage : null;
|
||||
$this->file = (string)($options['file'] ?? '');
|
||||
try {
|
||||
$remote = $this->sftp->_realpath((string)$options['path']);
|
||||
if (!is_string($remote)) {
|
||||
return false;
|
||||
}
|
||||
$id = self::nextId();
|
||||
if (!$this->sftp->_send_sftp_packet(self::FXP_OPEN, pack('Na*N2', strlen($remote), $remote, self::FXF_READ, 0), $id)) {
|
||||
return $this->connectionFailed();
|
||||
}
|
||||
$response = $this->sftp->_get_sftp_packet($id);
|
||||
} catch (\Throwable) {
|
||||
return $this->connectionFailed();
|
||||
}
|
||||
$id = self::nextId();
|
||||
if (!$this->sftp->_send_sftp_packet(self::FXP_OPEN, pack('Na*N2', strlen($remote), $remote, self::FXF_READ, 0), $id)) {
|
||||
return false;
|
||||
if (!is_string($response)) {
|
||||
return $this->connectionFailed();
|
||||
}
|
||||
$response = $this->sftp->_get_sftp_packet($id);
|
||||
if ($this->sftp->packet_type !== self::FXP_HANDLE || !is_string($response)) {
|
||||
return false;
|
||||
if ($this->sftp->packet_type !== self::FXP_HANDLE) {
|
||||
return false; // an answer, but no handle: the file cannot be opened
|
||||
}
|
||||
$this->handle = substr($response, 4);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Not a missing file but a connection that does not behave as expected. */
|
||||
private function connectionFailed(): bool {
|
||||
self::$failures++;
|
||||
self::$openBroke = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Whether the last stream_open() failed on the connection; a call, as fopen() sets it. */
|
||||
private static function openBroke(): bool {
|
||||
return self::$openBroke;
|
||||
}
|
||||
|
||||
public function stream_read(int $count): string|false {
|
||||
if ($this->fallback !== null) {
|
||||
return fread($this->fallback, $count);
|
||||
}
|
||||
while (strlen($this->buffer) < $count && !$this->eof) {
|
||||
if (!$this->receive()) {
|
||||
return false;
|
||||
return $this->useFallback() ? $this->stream_read($count) : false;
|
||||
}
|
||||
}
|
||||
$data = substr($this->buffer, 0, $count);
|
||||
@@ -135,22 +200,36 @@ final class SftpReadStream {
|
||||
}
|
||||
|
||||
public function stream_eof(): bool {
|
||||
if ($this->fallback !== null) {
|
||||
return feof($this->fallback);
|
||||
}
|
||||
return $this->eof && $this->buffer === '';
|
||||
}
|
||||
|
||||
public function stream_tell(): int {
|
||||
if ($this->fallback !== null) {
|
||||
$pos = ftell($this->fallback);
|
||||
return $pos === false ? $this->pos : $pos;
|
||||
}
|
||||
return $this->pos;
|
||||
}
|
||||
|
||||
public function stream_seek(int $offset, int $whence = SEEK_SET): bool {
|
||||
if ($this->fallback !== null) {
|
||||
return fseek($this->fallback, $offset, $whence) === 0;
|
||||
}
|
||||
if ($whence === SEEK_CUR) {
|
||||
$offset += $this->pos;
|
||||
} elseif ($whence !== SEEK_SET) {
|
||||
return false; // the size is never asked for, to save a round trip
|
||||
}
|
||||
if ($offset < 0 || $this->failed) {
|
||||
if ($offset < 0) {
|
||||
return false;
|
||||
}
|
||||
if ($this->failed) {
|
||||
$this->pos = $offset;
|
||||
return $this->useFallback();
|
||||
}
|
||||
if ($offset >= $this->pos && $offset - $this->pos <= strlen($this->buffer)) {
|
||||
$this->buffer = substr($this->buffer, $offset - $this->pos);
|
||||
} else {
|
||||
@@ -171,11 +250,59 @@ final class SftpReadStream {
|
||||
}
|
||||
|
||||
public function stream_close(): void {
|
||||
if (!$this->failed) {
|
||||
self::$failures = 0;
|
||||
}
|
||||
$fallback = $this->fallback;
|
||||
$this->fallback = null;
|
||||
if ($fallback !== null) {
|
||||
fclose($fallback);
|
||||
}
|
||||
$this->closeHandle();
|
||||
}
|
||||
|
||||
/**
|
||||
* Go on with the storage's own stream from the current position, after a server error on this
|
||||
* one. The data still buffered is dropped and read again. False after a connection error, or
|
||||
* when that stream cannot be opened or positioned: then the read fails as it would have
|
||||
* without this class. Reopening by path is safe for a repository: packs and index files are
|
||||
* never rewritten, and every object is authenticated when it is decrypted.
|
||||
*/
|
||||
private function useFallback(): bool {
|
||||
$this->closeHandle();
|
||||
$this->buffer = '';
|
||||
// files_external's stream takes whatever answer comes next on the connection: after a
|
||||
// connection error, answers to this stream's requests may still be on their way.
|
||||
if ($this->broken || $this->storage === null || !is_callable([$this->storage, 'fopen'])) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
$fh = $this->storage->fopen($this->file, 'r');
|
||||
if (!is_resource($fh)) {
|
||||
return false;
|
||||
}
|
||||
if ($this->pos > 0 && fseek($fh, $this->pos) !== 0) {
|
||||
fclose($fh);
|
||||
return false;
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
return false;
|
||||
}
|
||||
$this->fallback = $fh;
|
||||
return true;
|
||||
}
|
||||
|
||||
private function closeHandle(): void {
|
||||
$this->drain();
|
||||
if ($this->handle !== '') {
|
||||
$id = self::nextId();
|
||||
if ($this->sftp->_send_sftp_packet(self::FXP_CLOSE, pack('Na*', strlen($this->handle), $this->handle), $id)) {
|
||||
$this->sftp->_get_sftp_packet($id);
|
||||
if ($this->handle !== '' && !$this->broken) {
|
||||
try {
|
||||
$id = self::nextId();
|
||||
if (!$this->sftp->_send_sftp_packet(self::FXP_CLOSE, pack('Na*', strlen($this->handle), $this->handle), $id)
|
||||
|| !is_string($this->sftp->_get_sftp_packet($id))) {
|
||||
$this->broken = true;
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
$this->broken = true;
|
||||
}
|
||||
}
|
||||
$this->handle = '';
|
||||
@@ -186,11 +313,21 @@ final class SftpReadStream {
|
||||
if ($this->failed) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return $this->receiveOne();
|
||||
} catch (\Throwable) {
|
||||
$this->broken = true;
|
||||
return $this->fail();
|
||||
}
|
||||
}
|
||||
|
||||
private function receiveOne(): bool {
|
||||
while (count($this->inFlight) < $this->depth) {
|
||||
$id = self::nextId();
|
||||
$length = self::chunk($this->sftp);
|
||||
$packet = pack('Na*N3', strlen($this->handle), $this->handle, intdiv($this->next, 4294967296), $this->next % 4294967296, $length);
|
||||
if (!$this->sftp->_send_sftp_packet(self::FXP_READ, $packet, $id)) {
|
||||
$this->broken = true;
|
||||
return $this->fail();
|
||||
}
|
||||
$this->inFlight[] = [$id, $this->next, $length];
|
||||
@@ -199,10 +336,11 @@ final class SftpReadStream {
|
||||
[$id, $offset, $length] = array_shift($this->inFlight);
|
||||
$response = $this->sftp->_get_sftp_packet($id);
|
||||
if (!is_string($response)) {
|
||||
$this->broken = true;
|
||||
return $this->fail();
|
||||
}
|
||||
if ($this->sftp->packet_type === self::FXP_DATA) {
|
||||
$data = substr($response, 4);
|
||||
$data = substr($response, 4);
|
||||
if ($data === '') {
|
||||
return $this->fail();
|
||||
}
|
||||
@@ -236,18 +374,30 @@ final class SftpReadStream {
|
||||
|
||||
private function fail(): bool {
|
||||
$this->drain();
|
||||
if (!$this->failed) {
|
||||
self::$failures++;
|
||||
}
|
||||
$this->failed = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Read and drop the answers still on their way, so the connection is free again. */
|
||||
private function drain(): void {
|
||||
foreach ($this->inFlight as [$id]) {
|
||||
if (!$this->failed) {
|
||||
$this->sftp->_get_sftp_packet($id);
|
||||
}
|
||||
}
|
||||
$inFlight = $this->inFlight;
|
||||
$this->inFlight = [];
|
||||
if ($this->broken) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
foreach ($inFlight as [$id]) {
|
||||
if (!is_string($this->sftp->_get_sftp_packet($id))) {
|
||||
$this->broken = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
$this->broken = true;
|
||||
}
|
||||
}
|
||||
|
||||
private static function chunk(object $sftp): int {
|
||||
|
||||
@@ -87,10 +87,8 @@ final class StorageBackend implements IBackend {
|
||||
if (is_resource($fh)) {
|
||||
return $fh;
|
||||
}
|
||||
if ($fh === null) {
|
||||
$this->sftpReads = false;
|
||||
}
|
||||
// false: not opened, which need not mean missing; the storage's own stream decides
|
||||
// null: not supported or failing for now; false: not opened, which need not mean
|
||||
// missing. Either way the storage's own stream decides.
|
||||
}
|
||||
$fh = $this->storage->fopen($this->abs($path), 'r');
|
||||
if ($fh === false) {
|
||||
|
||||
@@ -4,6 +4,7 @@ declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Backend;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\SftpReadStream;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
@@ -102,24 +103,132 @@ class SftpReadStreamTest extends TestCase {
|
||||
$this->assertSame(0, $sftp->pending());
|
||||
}
|
||||
|
||||
public function testAServerErrorFailsTheReadAndLeavesTheConnectionUsable(): void {
|
||||
$sftp = new FakeSftpConnection(['/home/f' => random_bytes(3 * 1048576), '/home/g' => 'second'], failAt: 1048576);
|
||||
$fh = SftpReadStream::open(new FakeSftpStorage($sftp), 'f');
|
||||
$read = '';
|
||||
while (!feof($fh)) {
|
||||
$chunk = fread($fh, 65536);
|
||||
if ($chunk === false || $chunk === '') {
|
||||
break;
|
||||
}
|
||||
$read .= $chunk;
|
||||
}
|
||||
$this->assertLessThan(3 * 1048576, strlen($read));
|
||||
fclose($fh);
|
||||
$this->assertSame(0, $sftp->pending());
|
||||
public function testAServerErrorGoesOnWithTheStoragesOwnStreamFromTheSamePosition(): void {
|
||||
$data = random_bytes(3 * 1048576);
|
||||
$sftp = new FakeSftpConnection(['/home/f' => $data], failAt: 1048576);
|
||||
$storage = new FakeSftpStorage($sftp);
|
||||
$fh = SftpReadStream::open($storage, 'f');
|
||||
|
||||
$fh = SftpReadStream::open(new FakeSftpStorage($sftp), 'g');
|
||||
$this->assertSame('second', stream_get_contents($fh));
|
||||
$this->assertSame($data, $this->readAll($fh));
|
||||
$this->assertSame(['f'], $storage->fopens);
|
||||
$this->assertSame(strlen($data), ftell($fh));
|
||||
fclose($fh);
|
||||
$this->assertSame(0, $sftp->pending(), 'the connection is clean for the storage\'s own stream');
|
||||
$this->assertSame([], $sftp->openHandles, 'the failed handle is closed');
|
||||
|
||||
// One failure is no reason to give up on the fast reads.
|
||||
$fh = SftpReadStream::open($storage, 'f');
|
||||
$this->assertIsResource($fh);
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
public function testOnlyFailuresInARowSwitchTheFastReadsOff(): void {
|
||||
$data = random_bytes(2 * 1048576);
|
||||
$failing = new FakeSftpStorage(new FakeSftpConnection(['/home/f' => $data], failAt: 1048576));
|
||||
$working = new FakeSftpStorage(new FakeSftpConnection(['/home/f' => $data]));
|
||||
for ($i = 1; $i < SftpReadStream::MAX_FAILURES; $i++) {
|
||||
$fh = SftpReadStream::open($failing, 'f');
|
||||
$this->assertSame($data, $this->readAll($fh));
|
||||
fclose($fh);
|
||||
}
|
||||
$fh = SftpReadStream::open($working, 'f'); // a stream that works resets the count
|
||||
$this->assertSame($data, $this->readAll($fh));
|
||||
fclose($fh);
|
||||
for ($i = 1; $i <= SftpReadStream::MAX_FAILURES; $i++) {
|
||||
$fh = SftpReadStream::open($failing, 'f');
|
||||
$this->assertIsResource($fh, "failure $i");
|
||||
$this->assertSame($data, $this->readAll($fh));
|
||||
fclose($fh);
|
||||
}
|
||||
$this->assertNull(SftpReadStream::open($working, 'f'));
|
||||
$this->assertSame(SftpReadStream::MAX_FAILURES * 2 - 1, count($failing->fopens), 'every failed stream still gave all its data');
|
||||
}
|
||||
|
||||
public function testAServerErrorAfterASeekGoesOnFromTheSeekedPosition(): void {
|
||||
$data = random_bytes(3 * 1048576);
|
||||
$sftp = new FakeSftpConnection(['/home/f' => $data], failAt: 2500000);
|
||||
$storage = new FakeSftpStorage($sftp);
|
||||
$fh = SftpReadStream::open($storage, 'f');
|
||||
$this->assertSame(0, fseek($fh, 2400000));
|
||||
$this->assertSame(substr($data, 2400000), $this->readAll($fh));
|
||||
$this->assertSame(0, fseek($fh, 100)); // seeks go to the storage's stream now
|
||||
$this->assertSame(substr($data, 100, 1000), $this->readExactly($fh, 1000));
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
public function testAConnectionErrorDoesNotFallBackOnTheSameConnection(): void {
|
||||
// After an exception, answers may still be on their way; files_external's stream would
|
||||
// take them for its own.
|
||||
$sftp = new FakeSftpConnection(['/home/f' => random_bytes(3 * 1048576)], throwAt: 1048576);
|
||||
$storage = new FakeSftpStorage($sftp);
|
||||
$fh = SftpReadStream::open($storage, 'f');
|
||||
$this->assertLessThan(3 * 1048576, strlen($this->readAll($fh)));
|
||||
fclose($fh);
|
||||
$this->assertSame([], $storage->fopens);
|
||||
}
|
||||
|
||||
public function testALostConnectionDoesNotFallBackEither(): void {
|
||||
$sftp = new FakeSftpConnection(['/home/f' => random_bytes(3 * 1048576)], loseAt: 1048576);
|
||||
$storage = new FakeSftpStorage($sftp);
|
||||
$fh = SftpReadStream::open($storage, 'f');
|
||||
$this->assertLessThan(3 * 1048576, strlen($this->readAll($fh)));
|
||||
fclose($fh);
|
||||
$this->assertSame([], $storage->fopens);
|
||||
}
|
||||
|
||||
public function testWhenTheStoragesOwnStreamFailsTooTheReadFails(): void {
|
||||
$sftp = new FakeSftpConnection(['/home/f' => random_bytes(3 * 1048576)], failAt: 1048576);
|
||||
$storage = new FakeSftpStorage($sftp, fopenFails: true);
|
||||
$fh = SftpReadStream::open($storage, 'f');
|
||||
$this->assertLessThan(3 * 1048576, strlen($this->readAll($fh)));
|
||||
fclose($fh);
|
||||
$this->assertSame(['f'], $storage->fopens);
|
||||
}
|
||||
|
||||
public function testAConnectionErrorWhileOpeningIsAnErrorNotAMissingFile(): void {
|
||||
// The OPEN request went out: the storage's own stream must not be tried on this connection.
|
||||
$storage = new FakeSftpStorage(new FakeSftpConnection(['/home/f' => 'x'], throwAt: -1));
|
||||
try {
|
||||
SftpReadStream::open($storage, 'f');
|
||||
$this->fail('expected a BackendException');
|
||||
} catch (BackendException $e) {
|
||||
$this->assertStringContainsString('connection', $e->getMessage());
|
||||
}
|
||||
$lost = new FakeSftpStorage(new FakeSftpConnection(['/home/f' => 'x'], loseAt: -1));
|
||||
$this->expectException(BackendException::class);
|
||||
SftpReadStream::open($lost, 'f');
|
||||
}
|
||||
|
||||
public function testAConnectionErrorWhileOpeningDoesNotStopTheNextFile(): void {
|
||||
try {
|
||||
SftpReadStream::open(new FakeSftpStorage(new FakeSftpConnection(['/home/f' => 'x'], throwAt: -1)), 'f');
|
||||
} catch (BackendException) {
|
||||
}
|
||||
$fh = SftpReadStream::open(new FakeSftpStorage(new FakeSftpConnection(['/home/f' => 'x'])), 'f');
|
||||
$this->assertSame('x', stream_get_contents($fh));
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
public function testAFailedCloseDoesNotFallBackOnTheSameConnection(): void {
|
||||
$sftp = new FakeSftpConnection(['/home/f' => random_bytes(3 * 1048576)], failAt: 1048576, closeFails: true);
|
||||
$storage = new FakeSftpStorage($sftp);
|
||||
$fh = SftpReadStream::open($storage, 'f');
|
||||
$this->assertLessThan(3 * 1048576, strlen($this->readAll($fh)));
|
||||
fclose($fh);
|
||||
$this->assertSame([], $storage->fopens);
|
||||
}
|
||||
|
||||
public function testAStorageThatThrowsIsNotUsed(): void {
|
||||
$storage = new class {
|
||||
public function getRoot(): string {
|
||||
return '/';
|
||||
}
|
||||
|
||||
public function getConnection(): object {
|
||||
throw new \RuntimeException('Login failed');
|
||||
}
|
||||
};
|
||||
$this->assertNull(SftpReadStream::open($storage, 'f'));
|
||||
}
|
||||
|
||||
public function testAConnectionWithoutTheNeededMethodsIsNotUsed(): void {
|
||||
@@ -136,6 +245,90 @@ class SftpReadStreamTest extends TestCase {
|
||||
$this->assertNull(SftpReadStream::open(new \stdClass(), 'f'));
|
||||
}
|
||||
|
||||
public function testAPhpseclib3ConnectionIsNotUsed(): void {
|
||||
// phpseclib 3 has no _-prefixed methods; its packet_type and packet methods are private.
|
||||
$v3 = new class {
|
||||
private int $packet_type = -1;
|
||||
|
||||
private function _send_sftp_packet(): bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
private function _get_sftp_packet(): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
public function _realpath(string $path): string {
|
||||
return $path;
|
||||
}
|
||||
|
||||
public function realpath(string $path): string {
|
||||
return $path . (string)$this->packet_type . $this->_send_sftp_packet() . $this->_get_sftp_packet();
|
||||
}
|
||||
};
|
||||
$storage = new class($v3) {
|
||||
public function __construct(
|
||||
private object $connection,
|
||||
) {
|
||||
}
|
||||
|
||||
public function getRoot(): string {
|
||||
return '/';
|
||||
}
|
||||
|
||||
public function getConnection(): object {
|
||||
return $this->connection;
|
||||
}
|
||||
};
|
||||
$this->assertNull(SftpReadStream::open($storage, 'f'));
|
||||
}
|
||||
|
||||
public function testAPrivatePacketTypeIsNotUsed(): void {
|
||||
$conn = new class {
|
||||
private int $packet_type = -1;
|
||||
|
||||
public function _send_sftp_packet(): bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
public function _get_sftp_packet(): string {
|
||||
return (string)$this->packet_type;
|
||||
}
|
||||
|
||||
public function _realpath(string $path): string {
|
||||
return $path;
|
||||
}
|
||||
};
|
||||
$storage = new class($conn) {
|
||||
public function __construct(
|
||||
private object $connection,
|
||||
) {
|
||||
}
|
||||
|
||||
public function getRoot(): string {
|
||||
return '/';
|
||||
}
|
||||
|
||||
public function getConnection(): object {
|
||||
return $this->connection;
|
||||
}
|
||||
};
|
||||
$this->assertNull(SftpReadStream::open($storage, 'f'));
|
||||
}
|
||||
|
||||
/** @param resource $fh */
|
||||
private function readAll($fh): string {
|
||||
$data = '';
|
||||
while (!feof($fh)) {
|
||||
$chunk = fread($fh, 65536);
|
||||
if ($chunk === false || $chunk === '') {
|
||||
break;
|
||||
}
|
||||
$data .= $chunk;
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/** @param resource $fh */
|
||||
private function readExactly($fh, int $length): string {
|
||||
$data = '';
|
||||
@@ -147,11 +340,28 @@ class SftpReadStreamTest extends TestCase {
|
||||
}
|
||||
|
||||
final class FakeSftpStorage {
|
||||
/** @var list<string> paths opened through the storage's own stream */
|
||||
public array $fopens = [];
|
||||
|
||||
public function __construct(
|
||||
private FakeSftpConnection $connection,
|
||||
private bool $fopenFails = false,
|
||||
) {
|
||||
}
|
||||
|
||||
/** files_external's own read stream: here the file from memory, seekable. */
|
||||
public function fopen(string $path, string $mode) {
|
||||
$this->fopens[] = $path;
|
||||
$data = $this->connection->file('/home/' . $path);
|
||||
if ($this->fopenFails || $data === null) {
|
||||
return false;
|
||||
}
|
||||
$fh = fopen('php://memory', 'r+');
|
||||
fwrite($fh, $data);
|
||||
rewind($fh);
|
||||
return $fh;
|
||||
}
|
||||
|
||||
public function getRoot(): string {
|
||||
return '/home/';
|
||||
}
|
||||
@@ -178,9 +388,18 @@ final class FakeSftpConnection {
|
||||
private array $files,
|
||||
private int $cap = 261120,
|
||||
private ?int $failAt = null,
|
||||
/** READ at or past this offset throws; -1: OPEN throws */
|
||||
private ?int $throwAt = null,
|
||||
/** READ at or past this offset finds the connection gone (phpseclib returns false); -1: OPEN */
|
||||
private ?int $loseAt = null,
|
||||
private bool $closeFails = false,
|
||||
) {
|
||||
}
|
||||
|
||||
public function file(string $path): ?string {
|
||||
return $this->files[$path] ?? null;
|
||||
}
|
||||
|
||||
public function pending(): int {
|
||||
return count($this->answers);
|
||||
}
|
||||
@@ -192,6 +411,12 @@ final class FakeSftpConnection {
|
||||
public function _send_sftp_packet(int $type, string $data, int $id = 1): bool {
|
||||
switch ($type) {
|
||||
case 3: // OPEN
|
||||
if ($this->throwAt === -1) {
|
||||
throw new \RuntimeException('Connection closed by server');
|
||||
}
|
||||
if ($this->loseAt === -1) {
|
||||
return false;
|
||||
}
|
||||
$path = substr($data, 4, unpack('N', $data)[1]);
|
||||
if (!isset($this->files[$path])) {
|
||||
$this->answers[$id] = [101, pack('N', 2)];
|
||||
@@ -202,6 +427,9 @@ final class FakeSftpConnection {
|
||||
$this->answers[$id] = [102, pack('Na*', strlen($handle), $handle)];
|
||||
break;
|
||||
case 4: // CLOSE
|
||||
if ($this->closeFails) {
|
||||
return false;
|
||||
}
|
||||
$handle = substr($data, 4, unpack('N', $data)[1]);
|
||||
unset($this->openHandles[$handle]);
|
||||
$this->answers[$id] = [101, pack('N', 0)];
|
||||
@@ -212,6 +440,13 @@ final class FakeSftpConnection {
|
||||
['hi' => $hi, 'lo' => $lo, 'n' => $n] = unpack('Nhi/Nlo/Nn', substr($data, 4 + $len));
|
||||
$offset = $hi * 4294967296 + $lo;
|
||||
$this->reads[] = $n;
|
||||
if ($this->throwAt !== null && $this->throwAt >= 0 && $offset >= $this->throwAt) {
|
||||
throw new \RuntimeException('Connection closed by server');
|
||||
}
|
||||
if ($this->loseAt !== null && $this->loseAt >= 0 && $offset >= $this->loseAt) {
|
||||
$this->answers = [];
|
||||
return false;
|
||||
}
|
||||
$file = $this->files[$this->openHandles[$handle]];
|
||||
if ($this->failAt !== null && $offset >= $this->failAt) {
|
||||
$this->answers[$id] = [101, pack('N', 4)]; // FX_FAILURE
|
||||
|
||||
Reference in New Issue
Block a user