Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Full restore: an empty listing never empties the index; read a failed pack once more; recognise and replace SFTP connections a Storage Box dropped quietly; version 0.9.11

Browse Source
This commit is contained in:
Dillard Blom
2026-10-19 21:53:05 +02:00
parent 0accd5a3f2
commit c37b26ea38
9 changed files with 324 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Backend/SftpRangeFetcher.php
+31 -2
View File
@@ -54,6 +54,9 @@ final class SftpRangeFetcher {
private const FXF_READ = 1;
private const FX_OK = 0;
/** Connections that broke and are replaced, at most, in one restore. */
public const MAX_RECONNECTS = 20;
private const OPEN = 0;
private const READ = 1;
private const CLOSE = 2;
@@ -65,6 +68,8 @@ final class SftpRangeFetcher {
/** @var array<int, resource|null> ranges read (or failed) and not yet given to the caller */
private array $done = [];
private int $requestId = 0x4e520000;
/** Connections replaced so far (see MAX_RECONNECTS). */
public int $reconnects = 0;
/**
* @param array<int, object> $connections phpseclib 2 connections with their SFTP channel open
@@ -75,6 +80,7 @@ final class SftpRangeFetcher {
private string $root,
private array $ranges,
private string $spool,
private ?\Closure $connect = null,
) {
foreach (array_keys($connections) as $c) {
$this->chunks[$c] = self::CHUNK;
@@ -86,10 +92,12 @@ final class SftpRangeFetcher {
*
* @param list<object> $connections
* @param list<array{0:string, 1:int, 2:int}> $ranges
* @param (\Closure():?object)|null $connect a new connection in place of one that broke
* (null when it cannot be made)
* @return \Generator<int, resource|null>
*/
public static function fetch(array $connections, string $root, array $ranges, string $spool): \Generator {
$fetcher = new self($connections, rtrim($root, '/'), $ranges, $spool);
public static function fetch(array $connections, string $root, array $ranges, string $spool, ?\Closure $connect = null): \Generator {
$fetcher = new self($connections, rtrim($root, '/'), $ranges, $spool, $connect);
try {
yield from $fetcher->run();
} finally {
@@ -319,6 +327,27 @@ final class SftpRangeFetcher {
}
self::disconnect($this->connections[$c]);
unset($this->connections[$c]);
// A long restore outlives connections (a Storage Box drops one now and then): without a
// new one, the rest would be read the slow way.
if ($this->connect === null || $this->reconnects >= self::MAX_RECONNECTS) {
return;
}
$this->reconnects++;
try {
$new = ($this->connect)();
} catch (\Throwable) {
$new = null;
}
if (function_exists('OCP\Log\logger')) {
\OCP\Log\logger('ng_backup')->info($new === null
? 'Reading ahead: an SFTP connection broke and no new one could be made'
: 'Reading ahead: an SFTP connection broke; reading on with a new one');
}
if ($new !== null) {
$key = max(array_merge(array_keys($this->chunks), [-1])) + 1;
$this->connections[$key] = $new;
$this->chunks[$key] = self::CHUNK;
}
}
/** @return resource|null a new file in the spool, already unlinked: nothing is left behind if the process dies */
lib/Backend/StorageBackend.php
+16 -5
View File
@@ -282,10 +282,19 @@ final class StorageBackend implements IBackend, IReadsAhead {
if ($connections === []) {
return null;
}
return (static function () use ($storages, $connections, $root, $abs, $spool): \Generator {
// in place of a connection that broke: a new storage, kept like the others
$connect = static function () use ($reconnect, &$storages): ?object {
$storage = $reconnect();
$sftp = SftpReadStream::connection($storage);
if ($sftp !== null) {
$storages[] = $storage;
}
return $sftp;
};
return (static function () use (&$storages, $connections, $root, $abs, $spool, $connect): \Generator {
// the storages hold their connections: kept until the fetcher is done
try {
yield from SftpRangeFetcher::fetch($connections, $root, $abs, $spool);
yield from SftpRangeFetcher::fetch($connections, $root, $abs, $spool, $connect);
} finally {
unset($storages);
}
@@ -478,8 +487,9 @@ final class StorageBackend implements IBackend, IReadsAhead {
$this->lastUsed = microtime(true);
}
/** After a failed request: ask the server (one round trip), as phpseclib may not know it is gone. */
private function reopenIfDead(): bool {
if ($this->reconnect === null || !$this->sftpReads || $this->alive(false) !== false) {
if ($this->reconnect === null || !$this->sftpReads || $this->alive(true) !== false) {
return false;
}
$this->reopen();
@@ -508,8 +518,9 @@ final class StorageBackend implements IBackend, IReadsAhead {
if (is_callable([$sftp, 'clearStatCache'])) {
$sftp->clearStatCache(); // or stat() answers from memory
}
$sftp->stat('.');
return (bool)$sftp->isConnected();
// A Storage Box drops a quiet connection without phpseclib noticing: it still says
// connected, and every request fails. So the answer counts, not what it says after.
return $sftp->stat('.') !== false && (bool)$sftp->isConnected();
} catch (\Throwable) {
return false;
}
lib/Repository/BlobIndex.php
+39
View File
@@ -7,6 +7,7 @@ declare(strict_types=1);
namespace OCA\NgBackup\Repository;
use OCA\NgBackup\Backend\BackendException;
use OCA\NgBackup\Backend\IBackend;
use OCA\NgBackup\Crypto\StreamCipher;
@@ -104,6 +105,11 @@ final class BlobIndex {
$fetch[] = $path;
}
}
if ($present === [] && $cached !== []) {
// A repository never loses all its index files: an empty listing is a connection that
// failed quietly. Taken as is, it would empty the index and the cache.
throw new BackendException('The location listed no index files, while ' . count($cached) . ' are known: not trusting that listing');
}
// The rest is downloaded several at a time where the location allows it: on a large
// repository these are tens of thousands of small files. getMany() keeps their order, so
// the cached packs listed before each one are added first.
@@ -192,6 +198,39 @@ final class BlobIndex {
}
/** @return array<string, list<array{id:string, offset:int, length:int}>> packId => blobs currently located in that pack */
/**
* Add the index files that appeared on the location since load(), such as those of packs
* another server's cleanup wrote while a restore runs. Only adds and updates: a blob is
* never dropped, as a listing that misses files (a connection that failed) must not make a
* restore lose blobs it knows. A blob in a new pack takes that pack's place.
*/
public function refresh(): void {
$known = array_flip(array_map(fn (array $b): string => $b[0], $this->blobs));
$fetch = [];
$listed = 0;
foreach ($this->backend->listFiles('index') as $path) {
$listed++;
if (!isset($known[basename($path)])) {
$fetch[] = $path;
}
}
if ($listed === 0 && $this->blobs !== []) {
throw new BackendException('The location listed no index files, while ' . count($known) . ' are known: not trusting that listing');
}
$toCache = [];
foreach ($this->backend->getMany($fetch) as $path => $data) {
$packId = basename($path);
/** @var list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries authenticated: written by NG Backup */
$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
$this->downloaded++;
$this->add($packId, $entries);
$toCache[$packId] = $entries;
}
if ($toCache !== []) {
$this->cache?->putMany($this->repositoryId, $toCache);
}
}
public function packs(): array {
$packs = [];
foreach ($this->blobs as $id => [$pack, $offset, $length]) {
lib/Repository/Repository.php
+15 -17
View File
@@ -841,9 +841,9 @@ final class Repository {
$done++;
}
}
} catch (BackendException $e) {
// A pack moved (see loadBlob()): the rest one blob at a time, from the fresh index.
$this->refreshIndex($e);
} catch (BackendException) {
// A pack moved, or the read failed for a while (see loadBlob()): the rest one blob at a
// time, each read once more after a failure.
foreach (array_slice($ids, $done) as $id) {
yield $this->loadBlob($id);
}
@@ -872,27 +872,25 @@ final class Repository {
$encrypted = $this->packReader->read($pack, $offset, $length);
} catch (BackendException $e) {
// Another server's cleanup may have repacked this blob into a new pack since the index
// was loaded (a disaster recovery test while the original server still runs). Reload
// the index from the location; read again only if the blob has moved.
$this->refreshIndex($e);
// was loaded (a disaster recovery test while the original server still runs): add the
// new index files, and read the blob where it is now. Or the read failed for a while
// (a connection that died; the backend has replaced it by now): then read it once
// more where it was. A second failure is the real one.
try {
$this->index->refresh();
} catch (\Throwable) {
// the location could not be listed either: read again where the blob was
}
[$moved, $offset, $length] = $this->index->get($id);
if ($moved === $pack) {
try {
$encrypted = $this->packReader->read($moved, $offset, $length);
} catch (BackendException) {
throw $e;
}
$encrypted = $this->packReader->read($moved, $offset, $length);
}
return $this->decodeBlob($id, $encrypted);
}
/** Reload the index from the location after a failed pack read; rethrows that failure if it cannot. */
private function refreshIndex(BackendException $cause): void {
try {
$this->index->load();
} catch (\Throwable) {
throw $cause;
}
}
/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
private function decodeBlob(string $id, string $encrypted): string {
[, , , $raw, $flags] = $this->index->get($id);