M2: local cache of pack indexes (ngb_index)
Opening a repository lists index/ once and downloads only index files that are not cached; packs written by this server are cached on write, stale cache rows are dropped. Each resumable step therefore no longer downloads every index file from the location.
This commit is contained in:
+1
-1
@@ -5,7 +5,7 @@
|
||||
<name>NG Backup</name>
|
||||
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
||||
<description><![CDATA[Work in progress, not ready for use.]]></description>
|
||||
<version>0.0.2</version>
|
||||
<version>0.0.3</version>
|
||||
<licence>agpl</licence>
|
||||
<author>Dillard Blom</author>
|
||||
<namespace>NgBackup</namespace>
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Db;
|
||||
|
||||
use OCA\NgBackup\Repository\IIndexCache;
|
||||
use OCP\DB\Exception;
|
||||
use OCP\IDBConnection;
|
||||
|
||||
class DbIndexCache implements IIndexCache {
|
||||
public function __construct(
|
||||
private IDBConnection $db,
|
||||
) {
|
||||
}
|
||||
|
||||
public function all(string $repositoryId): array {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->select('pack_id', 'entries')->from('ngb_index')->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId)));
|
||||
$result = $qb->executeQuery();
|
||||
$out = [];
|
||||
while ($row = $result->fetch()) {
|
||||
$out[$row['pack_id']] = json_decode($row['entries'], true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
$result->closeCursor();
|
||||
return $out;
|
||||
}
|
||||
|
||||
public function put(string $repositoryId, string $packId, array $entries): void {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->insert('ngb_index')->values([
|
||||
'repository_id' => $qb->createNamedParameter($repositoryId),
|
||||
'pack_id' => $qb->createNamedParameter($packId),
|
||||
'entries' => $qb->createNamedParameter(json_encode($entries, JSON_THROW_ON_ERROR)),
|
||||
]);
|
||||
try {
|
||||
$qb->executeStatement();
|
||||
} catch (Exception $e) {
|
||||
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function remove(string $repositoryId, array $packIds): void {
|
||||
foreach (array_chunk($packIds, 500) as $chunk) {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->delete('ngb_index')
|
||||
->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId)))
|
||||
->andWhere($qb->expr()->in('pack_id', $qb->createNamedParameter($chunk, \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_STR_ARRAY)));
|
||||
$qb->executeStatement();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Migration;
|
||||
|
||||
use Closure;
|
||||
use OCP\DB\ISchemaWrapper;
|
||||
use OCP\DB\Types;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\SimpleMigrationStep;
|
||||
|
||||
/** Local cache of the per-pack index files, so opening a repository does not download them all. */
|
||||
class Version000002Date20261002010000 extends SimpleMigrationStep {
|
||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||
/** @var ISchemaWrapper $schema */
|
||||
$schema = $schemaClosure();
|
||||
if (!$schema->hasTable('ngb_index')) {
|
||||
$t = $schema->createTable('ngb_index');
|
||||
$t->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'unsigned' => true]);
|
||||
$t->addColumn('repository_id', Types::STRING, ['notnull' => true, 'length' => 32]);
|
||||
$t->addColumn('pack_id', Types::STRING, ['notnull' => true, 'length' => 32]);
|
||||
// Plain JSON list of blob entries. Blob ids are keyed hashes and reveal no content.
|
||||
$t->addColumn('entries', Types::TEXT, ['notnull' => true]);
|
||||
$t->setPrimaryKey(['id']);
|
||||
$t->addUniqueIndex(['repository_id', 'pack_id'], 'ngb_index_repo_pack');
|
||||
}
|
||||
return $schema;
|
||||
}
|
||||
}
|
||||
@@ -24,22 +24,51 @@ final class BlobIndex {
|
||||
/** @var array<string, true> blobs added in this run but whose pack is not uploaded yet */
|
||||
private array $pending = [];
|
||||
|
||||
private ?IIndexCache $cache = null;
|
||||
private string $repositoryId = '';
|
||||
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
|
||||
public int $downloaded = 0;
|
||||
|
||||
public function __construct(
|
||||
private IBackend $backend,
|
||||
private StreamCipher $cipher,
|
||||
) {
|
||||
}
|
||||
|
||||
public function setCache(IIndexCache $cache, string $repositoryId): void {
|
||||
$this->cache = $cache;
|
||||
$this->repositoryId = $repositoryId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the index: one listing of index/, entries from the local cache where available,
|
||||
* and only the index files that are not cached yet are downloaded. Cached packs that no
|
||||
* longer exist on the backend are dropped from the cache.
|
||||
*/
|
||||
public function load(): void {
|
||||
$cached = $this->cache?->all($this->repositoryId) ?? [];
|
||||
$present = [];
|
||||
$this->downloaded = 0;
|
||||
foreach ($this->backend->list('index') as $path) {
|
||||
$packId = basename($path);
|
||||
$fh = $this->backend->get($path);
|
||||
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($fh);
|
||||
$present[$packId] = true;
|
||||
if (isset($cached[$packId])) {
|
||||
$entries = $cached[$packId];
|
||||
} else {
|
||||
$fh = $this->backend->get($path);
|
||||
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
||||
fclose($fh);
|
||||
$this->downloaded++;
|
||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||
}
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
}
|
||||
}
|
||||
$stale = array_keys(array_diff_key($cached, $present));
|
||||
if ($stale !== []) {
|
||||
$this->cache?->remove($this->repositoryId, $stale);
|
||||
}
|
||||
}
|
||||
|
||||
public function has(string $id): bool {
|
||||
@@ -68,6 +97,7 @@ final class BlobIndex {
|
||||
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
||||
rewind($stream);
|
||||
$this->backend->put('index/' . $packId, $stream);
|
||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
unset($this->pending[$e['id']]);
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
/** Optional local cache of pack index entries (the repository's index/ files stay authoritative). */
|
||||
interface IIndexCache {
|
||||
/** @return array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> packId => entries */
|
||||
public function all(string $repositoryId): array;
|
||||
|
||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||
public function put(string $repositoryId, string $packId, array $entries): void;
|
||||
|
||||
/** @param list<string> $packIds */
|
||||
public function remove(string $repositoryId, array $packIds): void;
|
||||
}
|
||||
@@ -64,7 +64,7 @@ final class Repository {
|
||||
}
|
||||
|
||||
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
||||
public static function openWithKey(IBackend $backend, KeyRing $keys): self {
|
||||
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null): self {
|
||||
$config = self::readConfig($backend);
|
||||
try {
|
||||
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
||||
@@ -75,10 +75,18 @@ final class Repository {
|
||||
throw new RepositoryException('This repository was created with a different key');
|
||||
}
|
||||
$repo = new self($backend, $keys, $config['id']);
|
||||
if ($cache !== null) {
|
||||
$repo->index->setCache($cache, $config['id']);
|
||||
}
|
||||
$repo->index->load();
|
||||
return $repo;
|
||||
}
|
||||
|
||||
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
||||
public function indexDownloads(): int {
|
||||
return $this->index->downloaded;
|
||||
}
|
||||
|
||||
private static function readConfig(IBackend $backend): array {
|
||||
try {
|
||||
$fh = $backend->get('config');
|
||||
|
||||
@@ -28,6 +28,7 @@ class TargetService {
|
||||
private KeyService $keys,
|
||||
private ICrypto $crypto,
|
||||
private IAppManager $appManager,
|
||||
private \OCA\NgBackup\Db\DbIndexCache $indexCache,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -98,7 +99,7 @@ class TargetService {
|
||||
}
|
||||
|
||||
public function repository(Target $target): Repository {
|
||||
$repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing());
|
||||
$repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache);
|
||||
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
||||
throw new \RuntimeException('The target now contains a different repository than when it was added');
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.068,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.175,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.003,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.234,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.39,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.724,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.985,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.045,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.175,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.101,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.038,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.024,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.06,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.378,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.984,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.069}}
|
||||
{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.308,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.19,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.24,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.371,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.836,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.941,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":0.989,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.126,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.068,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.097,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.078,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.166,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.189,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.529,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":3.034,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.004,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\IndexCacheTest::testSecondOpenDownloadsNoIndexFiles":2.497}}
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\IIndexCache;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class IndexCacheTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
public function testSecondOpenDownloadsNoIndexFiles(): void {
|
||||
$cache = new class implements IIndexCache {
|
||||
public array $data = [];
|
||||
public function all(string $r): array { return $this->data[$r] ?? []; }
|
||||
public function put(string $r, string $p, array $e): void { $this->data[$r][$p] = $e; }
|
||||
public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
|
||||
};
|
||||
$src = $this->tempDir();
|
||||
for ($i = 0; $i < 12; $i++) {
|
||||
file_put_contents("$src/f$i", random_bytes(3 * 1048576));
|
||||
}
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
|
||||
$repo = Repository::openWithKey($backend, $keys, $cache);
|
||||
$r1 = $repo->backupDirectory($src);
|
||||
$this->assertGreaterThan(0, count($backend->list('index')));
|
||||
|
||||
$again = Repository::openWithKey($backend, $keys, $cache);
|
||||
$this->assertSame(0, $again->indexDownloads(), 'everything came from the cache');
|
||||
|
||||
file_put_contents("$src/new", random_bytes(2 * 1048576));
|
||||
$again->backupDirectory($src, $r1['snapshot']);
|
||||
$third = Repository::openWithKey($backend, $keys, $cache);
|
||||
$this->assertSame(0, $third->indexDownloads(), 'packs written by this server are cached on write');
|
||||
|
||||
// A cache that misses entries (e.g. another server wrote packs) fetches only those.
|
||||
$repoId = array_key_first($cache->data);
|
||||
array_pop($cache->data[$repoId]);
|
||||
$fourth = Repository::openWithKey($backend, $keys, $cache);
|
||||
$this->assertSame(1, $fourth->indexDownloads());
|
||||
|
||||
$target = $this->tempDir();
|
||||
$fourth->restore($r1['snapshot'], $target);
|
||||
$this->assertCount(12, self::hashTree($target));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user