From 370f3c55faa9f0f31f40861a4be361bf5d0c01da Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Fri, 2 Oct 2026 00:55:46 +0200 Subject: [PATCH] M2: local cache of pack indexes (ngb_index) Opening a repository lists index/ once and downloads only index files that are not cached; packs written by this server are cached on write, stale cache rows are dropped. Each resumable step therefore no longer downloads every index file from the location. --- appinfo/info.xml | 2 +- lib/Db/DbIndexCache.php | 57 +++++++++++++++++++ .../Version000002Date20261002010000.php | 33 +++++++++++ lib/Repository/BlobIndex.php | 36 +++++++++++- lib/Repository/IIndexCache.php | 20 +++++++ lib/Repository/Repository.php | 10 +++- lib/Service/TargetService.php | 3 +- tests/.phpunit.result.cache | 2 +- tests/Unit/Repository/IndexCacheTest.php | 54 ++++++++++++++++++ 9 files changed, 210 insertions(+), 7 deletions(-) create mode 100644 lib/Db/DbIndexCache.php create mode 100644 lib/Migration/Version000002Date20261002010000.php create mode 100644 lib/Repository/IIndexCache.php create mode 100644 tests/Unit/Repository/IndexCacheTest.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 5a6f1ee..a13b017 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -5,7 +5,7 @@ NG Backup Encrypted, incremental backups of your Nextcloud with restore from the web interface - 0.0.2 + 0.0.3 agpl Dillard Blom NgBackup diff --git a/lib/Db/DbIndexCache.php b/lib/Db/DbIndexCache.php new file mode 100644 index 0000000..bb61dab --- /dev/null +++ b/lib/Db/DbIndexCache.php @@ -0,0 +1,57 @@ +db->getQueryBuilder(); + $qb->select('pack_id', 'entries')->from('ngb_index')->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId))); + $result = $qb->executeQuery(); + $out = []; + while ($row = $result->fetch()) { + $out[$row['pack_id']] = json_decode($row['entries'], true, 512, JSON_THROW_ON_ERROR); + } + $result->closeCursor(); + return $out; + } + + public function put(string $repositoryId, string $packId, array $entries): void { + $qb = $this->db->getQueryBuilder(); + $qb->insert('ngb_index')->values([ + 'repository_id' => $qb->createNamedParameter($repositoryId), + 'pack_id' => $qb->createNamedParameter($packId), + 'entries' => $qb->createNamedParameter(json_encode($entries, JSON_THROW_ON_ERROR)), + ]); + try { + $qb->executeStatement(); + } catch (Exception $e) { + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + } + } + + public function remove(string $repositoryId, array $packIds): void { + foreach (array_chunk($packIds, 500) as $chunk) { + $qb = $this->db->getQueryBuilder(); + $qb->delete('ngb_index') + ->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId))) + ->andWhere($qb->expr()->in('pack_id', $qb->createNamedParameter($chunk, \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_STR_ARRAY))); + $qb->executeStatement(); + } + } +} diff --git a/lib/Migration/Version000002Date20261002010000.php b/lib/Migration/Version000002Date20261002010000.php new file mode 100644 index 0000000..2a6cbab --- /dev/null +++ b/lib/Migration/Version000002Date20261002010000.php @@ -0,0 +1,33 @@ +hasTable('ngb_index')) { + $t = $schema->createTable('ngb_index'); + $t->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'unsigned' => true]); + $t->addColumn('repository_id', Types::STRING, ['notnull' => true, 'length' => 32]); + $t->addColumn('pack_id', Types::STRING, ['notnull' => true, 'length' => 32]); + // Plain JSON list of blob entries. Blob ids are keyed hashes and reveal no content. + $t->addColumn('entries', Types::TEXT, ['notnull' => true]); + $t->setPrimaryKey(['id']); + $t->addUniqueIndex(['repository_id', 'pack_id'], 'ngb_index_repo_pack'); + } + return $schema; + } +} diff --git a/lib/Repository/BlobIndex.php b/lib/Repository/BlobIndex.php index f52a134..a5c3169 100644 --- a/lib/Repository/BlobIndex.php +++ b/lib/Repository/BlobIndex.php @@ -24,22 +24,51 @@ final class BlobIndex { /** @var array blobs added in this run but whose pack is not uploaded yet */ private array $pending = []; + private ?IIndexCache $cache = null; + private string $repositoryId = ''; + /** @var int index files downloaded by the last load() (for tests and diagnostics) */ + public int $downloaded = 0; + public function __construct( private IBackend $backend, private StreamCipher $cipher, ) { } + public function setCache(IIndexCache $cache, string $repositoryId): void { + $this->cache = $cache; + $this->repositoryId = $repositoryId; + } + + /** + * Load the index: one listing of index/, entries from the local cache where available, + * and only the index files that are not cached yet are downloaded. Cached packs that no + * longer exist on the backend are dropped from the cache. + */ public function load(): void { + $cached = $this->cache?->all($this->repositoryId) ?? []; + $present = []; + $this->downloaded = 0; foreach ($this->backend->list('index') as $path) { $packId = basename($path); - $fh = $this->backend->get($path); - $entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR); - fclose($fh); + $present[$packId] = true; + if (isset($cached[$packId])) { + $entries = $cached[$packId]; + } else { + $fh = $this->backend->get($path); + $entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR); + fclose($fh); + $this->downloaded++; + $this->cache?->put($this->repositoryId, $packId, $entries); + } foreach ($entries as $e) { $this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']]; } } + $stale = array_keys(array_diff_key($cached, $present)); + if ($stale !== []) { + $this->cache?->remove($this->repositoryId, $stale); + } } public function has(string $id): bool { @@ -68,6 +97,7 @@ final class BlobIndex { fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId)); rewind($stream); $this->backend->put('index/' . $packId, $stream); + $this->cache?->put($this->repositoryId, $packId, $entries); foreach ($entries as $e) { $this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']]; unset($this->pending[$e['id']]); diff --git a/lib/Repository/IIndexCache.php b/lib/Repository/IIndexCache.php new file mode 100644 index 0000000..1a7cff0 --- /dev/null +++ b/lib/Repository/IIndexCache.php @@ -0,0 +1,20 @@ +> packId => entries */ + public function all(string $repositoryId): array; + + /** @param list $entries */ + public function put(string $repositoryId, string $packId, array $entries): void; + + /** @param list $packIds */ + public function remove(string $repositoryId, array $packIds): void; +} diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index 7684f7e..4c12d84 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -64,7 +64,7 @@ final class Repository { } /** Open with the master key the server holds (scheduled backups, no passphrase needed). */ - public static function openWithKey(IBackend $backend, KeyRing $keys): self { + public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null): self { $config = self::readConfig($backend); try { $ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id']; @@ -75,10 +75,18 @@ final class Repository { throw new RepositoryException('This repository was created with a different key'); } $repo = new self($backend, $keys, $config['id']); + if ($cache !== null) { + $repo->index->setCache($cache, $config['id']); + } $repo->index->load(); return $repo; } + /** Index files downloaded when this repository was opened (0 when the local cache was complete). */ + public function indexDownloads(): int { + return $this->index->downloaded; + } + private static function readConfig(IBackend $backend): array { try { $fh = $backend->get('config'); diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php index 8c0d4d5..ed301ee 100644 --- a/lib/Service/TargetService.php +++ b/lib/Service/TargetService.php @@ -28,6 +28,7 @@ class TargetService { private KeyService $keys, private ICrypto $crypto, private IAppManager $appManager, + private \OCA\NgBackup\Db\DbIndexCache $indexCache, ) { } @@ -98,7 +99,7 @@ class TargetService { } public function repository(Target $target): Repository { - $repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing()); + $repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache); if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) { throw new \RuntimeException('The target now contains a different repository than when it was added'); } diff --git a/tests/.phpunit.result.cache b/tests/.phpunit.result.cache index e9fa417..b5f6c65 100644 --- a/tests/.phpunit.result.cache +++ b/tests/.phpunit.result.cache @@ -1 +1 @@ -{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.068,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.175,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.003,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.234,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.39,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.724,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.985,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.045,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.175,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.101,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.038,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.024,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.06,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.378,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.984,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.069}} \ No newline at end of file +{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.308,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.19,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.24,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.371,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.836,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.941,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":0.989,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.126,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.068,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.097,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.078,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.166,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.189,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.529,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":3.034,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.004,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\IndexCacheTest::testSecondOpenDownloadsNoIndexFiles":2.497}} \ No newline at end of file diff --git a/tests/Unit/Repository/IndexCacheTest.php b/tests/Unit/Repository/IndexCacheTest.php new file mode 100644 index 0000000..c2b0c5b --- /dev/null +++ b/tests/Unit/Repository/IndexCacheTest.php @@ -0,0 +1,54 @@ +data[$r] ?? []; } + public function put(string $r, string $p, array $e): void { $this->data[$r][$p] = $e; } + public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } } + }; + $src = $this->tempDir(); + for ($i = 0; $i < 12; $i++) { + file_put_contents("$src/f$i", random_bytes(3 * 1048576)); + } + $backend = new LocalBackend($this->tempDir()); + $keys = KeyRing::generate(); + Repository::initWithKey($backend, $keys, $keys->wrap('pw')); + + $repo = Repository::openWithKey($backend, $keys, $cache); + $r1 = $repo->backupDirectory($src); + $this->assertGreaterThan(0, count($backend->list('index'))); + + $again = Repository::openWithKey($backend, $keys, $cache); + $this->assertSame(0, $again->indexDownloads(), 'everything came from the cache'); + + file_put_contents("$src/new", random_bytes(2 * 1048576)); + $again->backupDirectory($src, $r1['snapshot']); + $third = Repository::openWithKey($backend, $keys, $cache); + $this->assertSame(0, $third->indexDownloads(), 'packs written by this server are cached on write'); + + // A cache that misses entries (e.g. another server wrote packs) fetches only those. + $repoId = array_key_first($cache->data); + array_pop($cache->data[$repoId]); + $fourth = Repository::openWithKey($backend, $keys, $cache); + $this->assertSame(1, $fourth->indexDownloads()); + + $target = $this->tempDir(); + $fourth->restore($r1['snapshot'], $target); + $this->assertCount(12, self::hashTree($target)); + } +}