Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

SFTP: open the channel before the first fast read; backup step: survive a takeover by another process

Browse Source
This commit is contained in:
Dillard Blom
2026-10-11 11:45:27 +02:00
parent 54750e9742
commit 894258f836
5 changed files with 84 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+14
View File
@@ -7,6 +7,20 @@ the app store accepts a `-beta` suffix, but Nextcloud servers then only offer th
beta/daily update channel, which would hide the app from most admins. The stability stage is
instead carried by the app description, the git tag, the release, and this file.
## [Unreleased]
### Fixed
- 0.9.4 could not open a repository at the root of an SFTP location (empty path): its faster
SFTP reads sent their first request before phpseclib had opened the SFTP channel, which only
happens on the first SFTP call after login. Every backup then failed with "No repository at
this location". The channel is now opened first.
- A long first backup could end as failed although its snapshot was complete: when its final
step took longer than the step's lease (150 s with `occ backup:run`), the background job took
the run over, finished it a second time and failed on the snapshot it had already recorded,
overwriting the result. The lease now lasts at least 15 minutes, a step no longer overwrites a
run that another process wrote in the meantime, and recording a snapshot that is already
recorded is no longer an error.
## [0.9.4-beta.5] - 2026-10-09
Bugfix release: faster restores.
lib/Backend/SftpReadStream.php
+7 -1
View File
@@ -108,7 +108,13 @@ final class SftpReadStream {
// is_callable and get_object_vars from here only see what is public: phpseclib 3 has
// methods of these names neither public nor at all.
if (!is_object($sftp) || !is_callable([$sftp, '_send_sftp_packet']) || !is_callable([$sftp, '_get_sftp_packet'])
|| !is_callable([$sftp, '_realpath']) || !array_key_exists('packet_type', get_object_vars($sftp))) {
|| !is_callable([$sftp, '_realpath']) || !is_callable([$sftp, 'pwd'])
|| !array_key_exists('packet_type', get_object_vars($sftp))) {
return null;
}
// phpseclib opens the SFTP channel on its first SFTP call, not at login: on a fresh
// connection a packet sent directly would go nowhere. pwd() opens it like any call does.
if ($sftp->pwd() === false) {
return null;
}
if (!in_array(self::PROTOCOL, stream_get_wrappers(), true)) {
lib/Service/BackupService.php
+29 -2
View File
@@ -15,6 +15,7 @@ use OCA\NgBackup\Db\SnapshotMapper;
use OCA\NgBackup\Db\Target;
use OCA\NgBackup\Job\BackupRun;
use OCA\NgBackup\Repository\RepositoryException;
use OCP\AppFramework\Db\DoesNotExistException;
use OCP\App\IAppManager;
use OCP\IConfig;
use OCP\IDBConnection;
@@ -105,7 +106,10 @@ final class BackupService {
*/
public function step(Run $run, float $deadline): Run {
// Leases expire on their own: a step killed by a time limit or OOM blocks nothing for long.
$ttl = (int)min(3600, max(120, $deadline - microtime(true) + 120));
// A step can run well past its deadline (the commit of a large snapshot writes its tree and
// index), so the lease holds at least 15 minutes; should it still expire and another process
// take the run over, the checks below keep the two from overwriting each other.
$ttl = (int)min(3600, max(900, $deadline - microtime(true) + 600));
try {
$runLease = $this->leases->acquire('run/' . $run->getId(), LeaseService::EXCLUSIVE, $ttl);
} catch (LockedException) {
@@ -123,6 +127,7 @@ final class BackupService {
if ($run->getStatus() !== Run::RUNNING || $run->getKind() !== self::KIND_FULL) {
return $run;
}
$loadedAt = $run->getUpdatedAt();
$target = $this->targets->get((string)$run->getTargetId());
$repo = $this->targets->repository($target);
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
@@ -179,8 +184,16 @@ final class BackupService {
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
$run->setUpdatedAt(time());
return $this->runs->update($run);
$current = $this->takenOver($run->getId(), $loadedAt);
return $current ?? $this->runs->update($run);
} catch (\Throwable $e) {
// Another process that took the run over (after this step's lease expired) may have
// finished it in the meantime: its result stands, this step's error does not.
$current = isset($loadedAt) ? $this->takenOver($run->getId(), $loadedAt) : null;
if ($current !== null) {
$this->logger->warning('NG Backup run ' . $run->getId() . ': a step that another process had taken over ended with: ' . $e->getMessage());
return $current;
}
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
$run->setStatus(Run::FAILED);
$run->setError($e->getMessage());
@@ -198,6 +211,15 @@ final class BackupService {
}
}
/**
* The run as stored, when another process has written it since this step loaded it (updated_at
* $loadedAt); null when it is still this step's to write.
*/
private function takenOver(int $runId, int $loadedAt): ?Run {
$current = $this->runs->find($runId);
return $current->getUpdatedAt() !== $loadedAt || $current->getStatus() !== Run::RUNNING ? $current : null;
}
/** Run to completion in this process (occ, system cron), reporting progress. */
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
while ($run->getStatus() === Run::RUNNING) {
@@ -224,6 +246,11 @@ final class BackupService {
}
private function recordSnapshot(Run $run, array $files): void {
try {
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
return; // already recorded, by an earlier attempt at this step
} catch (DoesNotExistException) {
}
$s = new Snapshot();
$s->setTargetId($run->getTargetId());
$s->setSnapshotId($files['snapshot']);
psalm-baseline.xml
+1 -1
View File
@@ -395,7 +395,7 @@
</file>
<file src="lib/Service/BackupService.php">
<InvalidOperand>
<code><![CDATA[$deadline - microtime(true) + 120]]></code>
<code><![CDATA[$deadline - microtime(true) + 600]]></code>
</InvalidOperand>
<UndefinedClass>
<code><![CDATA[\OC]]></code>
tests/Unit/Backend/SftpReadStreamTest.php
+33
View File
@@ -231,6 +231,22 @@ class SftpReadStreamTest extends TestCase {
$this->assertNull(SftpReadStream::open($storage, 'f'));
}
public function testAFreshConnectionHasItsSftpChannelOpenedFirst(): void {
// phpseclib 2 logs in at connect time but opens the SFTP channel on the first SFTP call.
$sftp = new FakeSftpConnection(['/home/config' => 'repository config']);
$this->assertFalse($sftp->initialised);
$fh = SftpReadStream::open(new FakeSftpStorage($sftp), 'config');
$this->assertIsResource($fh);
$this->assertSame('repository config', stream_get_contents($fh));
fclose($fh);
}
public function testAChannelThatCannotBeOpenedLeavesItToTheStoragesOwnStream(): void {
$sftp = new FakeSftpConnection(['/home/config' => 'x'], channelFails: true);
$this->assertNull(SftpReadStream::open(new FakeSftpStorage($sftp), 'config'));
$this->assertSame(0, $sftp->sent, 'nothing may be sent on a connection without an SFTP channel');
}
public function testAConnectionWithoutTheNeededMethodsIsNotUsed(): void {
$storage = new class {
public function getRoot(): string {
@@ -379,6 +395,10 @@ final class FakeSftpConnection {
public array $reads = [];
/** @var array<string, string> */
public array $openHandles = [];
/** Whether the SFTP channel is open: phpseclib 2 opens it on the first SFTP call. */
public bool $initialised = false;
/** Packets sent */
public int $sent = 0;
/** @var array<int, array{int, string}> answers by request id, in the order the server sent them */
private array $answers = [];
private int $handles = 0;
@@ -393,9 +413,18 @@ final class FakeSftpConnection {
/** READ at or past this offset finds the connection gone (phpseclib returns false); -1: OPEN */
private ?int $loseAt = null,
private bool $closeFails = false,
private bool $channelFails = false,
) {
}
public function pwd(): string|false {
if ($this->channelFails) {
return false;
}
$this->initialised = true;
return '/home';
}
public function file(string $path): ?string {
return $this->files[$path] ?? null;
}
@@ -409,6 +438,10 @@ final class FakeSftpConnection {
}
public function _send_sftp_packet(int $type, string $data, int $id = 1): bool {
$this->sent++;
if (!$this->initialised) {
return false; // like phpseclib 2: no SFTP channel yet
}
switch ($type) {
case 3: // OPEN
if ($this->throwAt === -1) {