Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

SFTP: replace a dropped connection; write the index cache in one transaction

Browse Source 
A Hetzner Storage Box drops an SFTP connection that carried a lot of data
and then stayed quiet for a few minutes; files_external keeps using the dead
client, on which every file looks missing. StorageBackend now checks the
connection after a minute without requests, and replaces a connection that
is dead after a failed request (once per request; put, move and delete are
not repeated). SftpReadStream closes a connection it left in an unknown state.

The verify cache refresh took more than 6 minutes on PostgreSQL for 27,547
packs, one autocommitted insert per pack: putMany()/replaceAll() write in one
transaction. After a verify the cache holds the index files as they are, so a
blob in two packs stays cached for both.
This commit is contained in:
Dillard Blom
2026-10-12 01:58:54 +02:00
parent d659322e60
commit 9c06ab2841
9 changed files with 442 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Backend/SftpReadStream.php
+11 -5
View File
@@ -362,6 +362,8 @@ final class SftpReadStream {
self::$failures++;
self::$openBroke = true;
self::$openCause = $cause . self::connectionState($this->sftp);
// unknown state: closed, so that nothing else uses it (and the backend can build a new one)
self::disconnect($this->sftp);
return false;
}
@@ -514,6 +516,9 @@ final class SftpReadStream {
} catch (\Throwable) {
$this->broken = true;
}
if ($this->broken) {
self::disconnect($this->sftp);
}
}
$this->handle = '';
}
@@ -595,19 +600,20 @@ final class SftpReadStream {
private function drain(): void {
$inFlight = $this->inFlight;
$this->inFlight = [];
if ($this->broken) {
return;
}
try {
foreach ($inFlight as [$id]) {
foreach ($this->broken ? [] : $inFlight as [$id]) {
if (!is_string($this->sftp->_get_sftp_packet($id))) {
$this->broken = true;
return;
break;
}
}
} catch (\Throwable) {
$this->broken = true;
}
if ($this->broken) {
// unknown state: closed, so that nothing else uses it (and the backend can build a new one)
self::disconnect($this->sftp);
}
}
private static function chunk(object $sftp): int {
lib/Backend/StorageBackend.php
+127 -4
View File
@@ -19,12 +19,23 @@ final class StorageBackend implements IBackend {
private bool $atomicPut;
/** SFTP: read through SftpReadStream, which keeps several requests in flight. */
private bool $sftpReads;
/** Seconds without a request after which an SFTP connection is checked before it is used. */
public const IDLE_CHECK = 60;
private float $lastUsed;
/** Times the storage was built anew after its connection died (for tests and diagnostics). */
public int $reconnects = 0;
/**
* @param (\Closure():IStorage)|null $reconnect builds the same storage anew, with a new
* connection
*/
public function __construct(
private IStorage $storage,
private string $base = '',
bool $createBase = true,
private ?\Closure $reconnect = null,
) {
$this->lastUsed = microtime(true);
$this->base = trim($base, '/');
self::validate($this->base, true);
$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
@@ -40,6 +51,8 @@ final class StorageBackend implements IBackend {
}
public function put(string $path, $stream): void {
// not repeated on a new connection: the stream is (partly) read
$this->idleCheck();
$expected = self::remaining($stream);
$target = $this->abs($path);
$this->mkdirs(dirname($target));
@@ -83,6 +96,11 @@ final class StorageBackend implements IBackend {
}
public function get(string $path) {
return $this->call(fn () => $this->open($path));
}
/** @return resource */
private function open(string $path) {
if ($this->sftpReads) {
$fh = SftpReadStream::open($this->storage, $this->abs($path));
if (is_resource($fh)) {
@@ -104,7 +122,7 @@ final class StorageBackend implements IBackend {
$fast = null;
if ($this->sftpReads) {
$abs = array_map(fn (string $p): string => $this->abs($p), $chunk);
$fast = SftpReadStream::readMany($this->storage, $abs);
$fast = $this->call(fn () => SftpReadStream::readMany($this->storage, $abs));
$fast = $fast === null ? null : array_combine($chunk, array_map(fn (string $a): ?string => $fast[$a] ?? null, $abs));
}
foreach ($chunk as $path) {
@@ -128,6 +146,10 @@ final class StorageBackend implements IBackend {
}
public function getRange(string $path, int $offset, int $length): string {
return $this->call(fn () => $this->readRange($path, $offset, $length));
}
private function readRange(string $path, int $offset, int $length): string {
$fh = $this->get($path);
try {
if (fseek($fh, $offset) !== 0) {
@@ -160,22 +182,30 @@ final class StorageBackend implements IBackend {
}
public function exists(string $path): bool {
return $this->storage->file_exists($this->abs($path));
$abs = $this->abs($path);
return $this->call(fn () => $this->storage->file_exists($abs), fn (bool $found) => !$found);
}
public function list(string $prefix): array {
$result = [];
$this->listInto($this->abs($prefix), $result);
$abs = $this->abs($prefix);
$result = $this->call(function () use ($abs): array {
$result = [];
$this->listInto($abs, $result);
return $result;
}, fn (array $found) => $found === []);
sort($result);
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
return array_map(fn ($p) => substr($p, $cut), $result);
}
public function delete(string $path): void {
$this->idleCheck();
$this->storage->unlink($this->abs($path));
}
public function move(string $from, string $to): void {
// not repeated on a new connection: the server may have done it before the connection died
$this->idleCheck();
$target = $this->abs($to);
$this->mkdirs(dirname($target));
if (!$this->storage->rename($this->abs($from), $target)) {
@@ -185,6 +215,11 @@ final class StorageBackend implements IBackend {
public function listFiles(string $dir): array {
$abs = $this->abs($dir);
return $this->call(fn () => $this->filesIn($dir, $abs), fn (array $found) => $found === []);
}
/** @return list<string> */
private function filesIn(string $dir, string $abs): array {
if (!$this->storage->is_dir($abs)) {
return []; // not created yet
}
@@ -205,6 +240,94 @@ final class StorageBackend implements IBackend {
return $result;
}
/**
* Run $op; when it throws, or $failed says its result may come from a dead connection, and
* the SFTP connection turns out to be dead, run it once more on a new connection.
*
* @template T
* @param \Closure():T $op
* @param (\Closure(T):bool)|null $failed
* @return T
*/
private function call(\Closure $op, ?\Closure $failed = null): mixed {
$this->idleCheck();
try {
$result = $op();
} catch (\Throwable $e) {
if (!$this->reopenIfDead()) {
throw $e;
}
return $op();
}
if ($failed !== null && $failed($result) && $this->reopenIfDead()) {
return $op();
}
return $result;
}
/**
* A Hetzner Storage Box drops an SFTP connection that carried a lot of data and then stayed
* quiet for a few minutes, while phpseclib still takes it for open; files_external never
* builds a new one, and every request on it fails as if the file were missing. So after
* such a pause the connection is checked (one round trip) before it is used again.
*/
private function idleCheck(): void {
$now = microtime(true);
if ($this->reconnect !== null && $this->sftpReads && $now - $this->lastUsed >= self::IDLE_CHECK
&& $this->alive(true) === false) {
$this->reopen();
}
$this->lastUsed = microtime(true);
}
private function reopenIfDead(): bool {
if ($this->reconnect === null || !$this->sftpReads || $this->alive(false) !== false) {
return false;
}
$this->reopen();
return true;
}
/**
* Whether the storage's SFTP connection is alive; null when it cannot tell.
* $ask: send a request instead of trusting what phpseclib saw last.
*/
private function alive(bool $ask): ?bool {
try {
if (!is_callable([$this->storage, 'getConnection'])) {
return null;
}
$sftp = $this->storage->getConnection();
if (!is_object($sftp) || !is_callable([$sftp, 'isConnected'])) {
return null;
}
if (!$sftp->isConnected()) {
return false;
}
if (!$ask || !is_callable([$sftp, 'stat'])) {
return true;
}
if (is_callable([$sftp, 'clearStatCache'])) {
$sftp->clearStatCache(); // or stat() answers from memory
}
$sftp->stat('.');
return (bool)$sftp->isConnected();
} catch (\Throwable) {
return false;
}
}
private function reopen(): void {
/** @var \Closure():IStorage $reconnect */
$reconnect = $this->reconnect;
$this->storage = $reconnect();
$this->reconnects++;
$this->lastUsed = microtime(true);
if (function_exists('OCP\Log\logger')) {
\OCP\Log\logger('ng_backup')->info('The SFTP connection to the backup location was lost; reconnected');
}
}
private function listInto(string $dir, array &$result): void {
if (!$this->storage->is_dir($dir)) {
return;