Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

SFTP: replace a dropped connection; write the index cache in one transaction

Browse Source 
A Hetzner Storage Box drops an SFTP connection that carried a lot of data
and then stayed quiet for a few minutes; files_external keeps using the dead
client, on which every file looks missing. StorageBackend now checks the
connection after a minute without requests, and replaces a connection that
is dead after a failed request (once per request; put, move and delete are
not repeated). SftpReadStream closes a connection it left in an unknown state.

The verify cache refresh took more than 6 minutes on PostgreSQL for 27,547
packs, one autocommitted insert per pack: putMany()/replaceAll() write in one
transaction. After a verify the cache holds the index files as they are, so a
blob in two packs stays cached for both.
This commit is contained in:
Dillard Blom
2026-10-12 01:58:54 +02:00
parent d659322e60
commit 9c06ab2841
9 changed files with 442 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Unit/Repository/IndexCacheTest.php
+38
View File
@@ -20,6 +20,8 @@ class IndexCacheTest extends TestCase {
public function all(string $r): array { return $this->data[$r] ?? []; }
public function put(string $r, string $p, array $e): void { $this->data[$r][$p] = $e; }
public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
public function putMany(string $r, array $packs): void { foreach ($packs as $p => $e) { $this->put($r, (string)$p, $e); } }
public function replaceAll(string $r, array $packs): void { $this->data[$r] = []; $this->putMany($r, $packs); }
};
$src = $this->tempDir();
for ($i = 0; $i < 12; $i++) {
@@ -65,6 +67,8 @@ class IndexCacheTest extends TestCase {
$this->data[$r][$p] = $e;
}
public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
public function putMany(string $r, array $packs): void { foreach ($packs as $p => $e) { $this->put($r, (string)$p, $e); } }
public function replaceAll(string $r, array $packs): void { $this->data[$r] = []; $this->putMany($r, $packs); }
};
$backend = new LocalBackend($this->tempDir());
$keys = KeyRing::generate();
@@ -88,4 +92,38 @@ class IndexCacheTest extends TestCase {
$this->assertCount(count($backend->list('index')), array_merge(...array_values($cache->data)));
$this->assertSame(0, Repository::openWithKey($backend, $keys, $cache)->indexDownloads());
}
/** After a check the cache holds every index file as it is, also a blob that is in two packs. */
public function testACheckCachesTheIndexFilesAsTheyAre(): void {
$cache = new class implements IIndexCache {
public array $data = [];
public int $replaced = 0;
public function all(string $r): array { return $this->data[$r] ?? []; }
public function put(string $r, string $p, array $e): void { $this->data[$r][$p] = $e; }
public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
public function putMany(string $r, array $packs): void { foreach ($packs as $p => $e) { $this->put($r, (string)$p, $e); } }
public function replaceAll(string $r, array $packs): void { $this->replaced++; $this->data[$r] = $packs; }
};
$backend = new LocalBackend($this->tempDir());
$cipher = new \OCA\NgBackup\Crypto\StreamCipher(KeyRing::generate());
$index = new \OCA\NgBackup\Repository\BlobIndex($backend, $cipher);
$index->setCache($cache, 'r');
$blob = static fn (string $id, int $offset): array => ['id' => $id, 'offset' => $offset, 'length' => 10, 'raw' => 8, 'flags' => 0];
$index->addPack('pack1', [$blob('a', 0), $blob('shared', 10)]);
$index->addPack('pack2', [$blob('shared', 0), $blob('b', 10)]);
$cache->data = ['r' => ['gone' => [$blob('x', 0)]]];
$index->load(true);
$this->assertSame(1, $cache->replaced);
$this->assertSame(['pack1', 'pack2'], array_keys($cache->data['r']));
$this->assertCount(2, $cache->data['r']['pack1'], 'the shared blob stays in the row of its first pack');
// the second pack goes: the shared blob is still found in the first
$index->removePack('pack2');
$again = new \OCA\NgBackup\Repository\BlobIndex($backend, $cipher);
$again->setCache($cache, 'r');
$again->load();
$this->assertSame(0, $again->downloaded);
$this->assertSame('pack1', $again->get('shared')[0]);
}
}