diff --git a/appinfo/info.xml b/appinfo/info.xml
index 5a6f1ee..a13b017 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -5,7 +5,7 @@
NG Backup
Encrypted, incremental backups of your Nextcloud with restore from the web interface
- 0.0.2
+ 0.0.3
agpl
Dillard Blom
NgBackup
diff --git a/lib/Db/DbIndexCache.php b/lib/Db/DbIndexCache.php
new file mode 100644
index 0000000..bb61dab
--- /dev/null
+++ b/lib/Db/DbIndexCache.php
@@ -0,0 +1,57 @@
+db->getQueryBuilder();
+ $qb->select('pack_id', 'entries')->from('ngb_index')->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId)));
+ $result = $qb->executeQuery();
+ $out = [];
+ while ($row = $result->fetch()) {
+ $out[$row['pack_id']] = json_decode($row['entries'], true, 512, JSON_THROW_ON_ERROR);
+ }
+ $result->closeCursor();
+ return $out;
+ }
+
+ public function put(string $repositoryId, string $packId, array $entries): void {
+ $qb = $this->db->getQueryBuilder();
+ $qb->insert('ngb_index')->values([
+ 'repository_id' => $qb->createNamedParameter($repositoryId),
+ 'pack_id' => $qb->createNamedParameter($packId),
+ 'entries' => $qb->createNamedParameter(json_encode($entries, JSON_THROW_ON_ERROR)),
+ ]);
+ try {
+ $qb->executeStatement();
+ } catch (Exception $e) {
+ if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
+ throw $e;
+ }
+ }
+ }
+
+ public function remove(string $repositoryId, array $packIds): void {
+ foreach (array_chunk($packIds, 500) as $chunk) {
+ $qb = $this->db->getQueryBuilder();
+ $qb->delete('ngb_index')
+ ->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId)))
+ ->andWhere($qb->expr()->in('pack_id', $qb->createNamedParameter($chunk, \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_STR_ARRAY)));
+ $qb->executeStatement();
+ }
+ }
+}
diff --git a/lib/Migration/Version000002Date20261002010000.php b/lib/Migration/Version000002Date20261002010000.php
new file mode 100644
index 0000000..2a6cbab
--- /dev/null
+++ b/lib/Migration/Version000002Date20261002010000.php
@@ -0,0 +1,33 @@
+hasTable('ngb_index')) {
+ $t = $schema->createTable('ngb_index');
+ $t->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'unsigned' => true]);
+ $t->addColumn('repository_id', Types::STRING, ['notnull' => true, 'length' => 32]);
+ $t->addColumn('pack_id', Types::STRING, ['notnull' => true, 'length' => 32]);
+ // Plain JSON list of blob entries. Blob ids are keyed hashes and reveal no content.
+ $t->addColumn('entries', Types::TEXT, ['notnull' => true]);
+ $t->setPrimaryKey(['id']);
+ $t->addUniqueIndex(['repository_id', 'pack_id'], 'ngb_index_repo_pack');
+ }
+ return $schema;
+ }
+}
diff --git a/lib/Repository/BlobIndex.php b/lib/Repository/BlobIndex.php
index f52a134..a5c3169 100644
--- a/lib/Repository/BlobIndex.php
+++ b/lib/Repository/BlobIndex.php
@@ -24,22 +24,51 @@ final class BlobIndex {
/** @var array blobs added in this run but whose pack is not uploaded yet */
private array $pending = [];
+ private ?IIndexCache $cache = null;
+ private string $repositoryId = '';
+ /** @var int index files downloaded by the last load() (for tests and diagnostics) */
+ public int $downloaded = 0;
+
public function __construct(
private IBackend $backend,
private StreamCipher $cipher,
) {
}
+ public function setCache(IIndexCache $cache, string $repositoryId): void {
+ $this->cache = $cache;
+ $this->repositoryId = $repositoryId;
+ }
+
+ /**
+ * Load the index: one listing of index/, entries from the local cache where available,
+ * and only the index files that are not cached yet are downloaded. Cached packs that no
+ * longer exist on the backend are dropped from the cache.
+ */
public function load(): void {
+ $cached = $this->cache?->all($this->repositoryId) ?? [];
+ $present = [];
+ $this->downloaded = 0;
foreach ($this->backend->list('index') as $path) {
$packId = basename($path);
- $fh = $this->backend->get($path);
- $entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
- fclose($fh);
+ $present[$packId] = true;
+ if (isset($cached[$packId])) {
+ $entries = $cached[$packId];
+ } else {
+ $fh = $this->backend->get($path);
+ $entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
+ fclose($fh);
+ $this->downloaded++;
+ $this->cache?->put($this->repositoryId, $packId, $entries);
+ }
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
}
}
+ $stale = array_keys(array_diff_key($cached, $present));
+ if ($stale !== []) {
+ $this->cache?->remove($this->repositoryId, $stale);
+ }
}
public function has(string $id): bool {
@@ -68,6 +97,7 @@ final class BlobIndex {
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
rewind($stream);
$this->backend->put('index/' . $packId, $stream);
+ $this->cache?->put($this->repositoryId, $packId, $entries);
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
unset($this->pending[$e['id']]);
diff --git a/lib/Repository/IIndexCache.php b/lib/Repository/IIndexCache.php
new file mode 100644
index 0000000..1a7cff0
--- /dev/null
+++ b/lib/Repository/IIndexCache.php
@@ -0,0 +1,20 @@
+> packId => entries */
+ public function all(string $repositoryId): array;
+
+ /** @param list $entries */
+ public function put(string $repositoryId, string $packId, array $entries): void;
+
+ /** @param list $packIds */
+ public function remove(string $repositoryId, array $packIds): void;
+}
diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php
index 7684f7e..4c12d84 100644
--- a/lib/Repository/Repository.php
+++ b/lib/Repository/Repository.php
@@ -64,7 +64,7 @@ final class Repository {
}
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
- public static function openWithKey(IBackend $backend, KeyRing $keys): self {
+ public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null): self {
$config = self::readConfig($backend);
try {
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
@@ -75,10 +75,18 @@ final class Repository {
throw new RepositoryException('This repository was created with a different key');
}
$repo = new self($backend, $keys, $config['id']);
+ if ($cache !== null) {
+ $repo->index->setCache($cache, $config['id']);
+ }
$repo->index->load();
return $repo;
}
+ /** Index files downloaded when this repository was opened (0 when the local cache was complete). */
+ public function indexDownloads(): int {
+ return $this->index->downloaded;
+ }
+
private static function readConfig(IBackend $backend): array {
try {
$fh = $backend->get('config');
diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php
index 8c0d4d5..ed301ee 100644
--- a/lib/Service/TargetService.php
+++ b/lib/Service/TargetService.php
@@ -28,6 +28,7 @@ class TargetService {
private KeyService $keys,
private ICrypto $crypto,
private IAppManager $appManager,
+ private \OCA\NgBackup\Db\DbIndexCache $indexCache,
) {
}
@@ -98,7 +99,7 @@ class TargetService {
}
public function repository(Target $target): Repository {
- $repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing());
+ $repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache);
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
throw new \RuntimeException('The target now contains a different repository than when it was added');
}
diff --git a/tests/.phpunit.result.cache b/tests/.phpunit.result.cache
index e9fa417..b5f6c65 100644
--- a/tests/.phpunit.result.cache
+++ b/tests/.phpunit.result.cache
@@ -1 +1 @@
-{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.068,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.175,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.003,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.234,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.39,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.724,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.985,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.045,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.175,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.101,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.038,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.024,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.06,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.378,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.984,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.069}}
\ No newline at end of file
+{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.308,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.19,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.24,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.371,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.836,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.941,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":0.989,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.126,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.068,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.097,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.078,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.166,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.189,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.529,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":3.034,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.004,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\IndexCacheTest::testSecondOpenDownloadsNoIndexFiles":2.497}}
\ No newline at end of file
diff --git a/tests/Unit/Repository/IndexCacheTest.php b/tests/Unit/Repository/IndexCacheTest.php
new file mode 100644
index 0000000..c2b0c5b
--- /dev/null
+++ b/tests/Unit/Repository/IndexCacheTest.php
@@ -0,0 +1,54 @@
+data[$r] ?? []; }
+ public function put(string $r, string $p, array $e): void { $this->data[$r][$p] = $e; }
+ public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
+ };
+ $src = $this->tempDir();
+ for ($i = 0; $i < 12; $i++) {
+ file_put_contents("$src/f$i", random_bytes(3 * 1048576));
+ }
+ $backend = new LocalBackend($this->tempDir());
+ $keys = KeyRing::generate();
+ Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
+
+ $repo = Repository::openWithKey($backend, $keys, $cache);
+ $r1 = $repo->backupDirectory($src);
+ $this->assertGreaterThan(0, count($backend->list('index')));
+
+ $again = Repository::openWithKey($backend, $keys, $cache);
+ $this->assertSame(0, $again->indexDownloads(), 'everything came from the cache');
+
+ file_put_contents("$src/new", random_bytes(2 * 1048576));
+ $again->backupDirectory($src, $r1['snapshot']);
+ $third = Repository::openWithKey($backend, $keys, $cache);
+ $this->assertSame(0, $third->indexDownloads(), 'packs written by this server are cached on write');
+
+ // A cache that misses entries (e.g. another server wrote packs) fetches only those.
+ $repoId = array_key_first($cache->data);
+ array_pop($cache->data[$repoId]);
+ $fourth = Repository::openWithKey($backend, $keys, $cache);
+ $this->assertSame(1, $fourth->indexDownloads());
+
+ $target = $this->tempDir();
+ $fourth->restore($r1['snapshot'], $target);
+ $this->assertCount(12, self::hashTree($target));
+ }
+}