From d504067cd89ad4f5884b1bb1818cd8127fe29db1 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Mon, 19 Oct 2026 22:13:26 +0200 Subject: [PATCH] Index: an empty listing is an error in checks and cleanups too; refresh() fetches only index files it has not loaded; full-restore CI on Nextcloud 35.0.0 --- .github/workflows/ci.yml | 14 ++++++------ lib/Repository/BlobIndex.php | 25 ++++++++++++++-------- tests/Unit/Repository/IndexListingTest.php | 21 ++++++++++++++++++ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 549c2a5..3546369 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,8 @@ jobs: - name: Per-user backup/restore round trip on Nextcloud ${{ matrix.nextcloud }} run: tests/integration/user-roundtrip.sh "docker exec -u www-data ngbu" /var/www/html/data + # 35.0.0, not 35: the app store lists apps for a new point release only some time after it + # is out, and this test installs Mail and Deck from it. integration-full-restore: # Disaster recovery (occ backup:restore:full) on a SECOND, fresh Nextcloud instance that # never saw the first one, sharing only the backup location. Does not depend on the database @@ -103,14 +105,14 @@ jobs: with: persist-credentials: false - name: Full restore round trip - run: tests/integration/full-restore-roundtrip.sh 35-apache + run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache - name: Full restore round trip on PostgreSQL, with Mail (foreign keys) and a disabled app - run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql + run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache pgsql - name: Full restore round trip on MariaDB, with Mail (foreign keys) and a disabled app - run: tests/integration/full-restore-roundtrip.sh 35-apache mysql + run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache mysql - name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on PostgreSQL - run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql legacy + run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache pgsql legacy - name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on MariaDB - run: tests/integration/full-restore-roundtrip.sh 35-apache mysql legacy + run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache mysql legacy - name: Full restore over SFTP on PostgreSQL, reading packs ahead on several connections - run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql sftp + run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache pgsql sftp diff --git a/lib/Repository/BlobIndex.php b/lib/Repository/BlobIndex.php index fac0fa2..32ca142 100644 --- a/lib/Repository/BlobIndex.php +++ b/lib/Repository/BlobIndex.php @@ -33,6 +33,8 @@ final class BlobIndex { private const CACHE_LOT = 256; /** @var int index files downloaded by the last load() (for tests and diagnostics) */ public int $downloaded = 0; + /** @var array packs whose index file is loaded (refresh() fetches only others) */ + private array $loaded = []; public function __construct( private IBackend $backend, @@ -91,6 +93,7 @@ final class BlobIndex { * longer exist on the backend are dropped from the cache. */ public function load(bool $verify = false, ?callable $progress = null): void { + $hadIndex = $this->blobs !== []; $this->blobs = []; $cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []); $present = []; @@ -105,11 +108,13 @@ final class BlobIndex { $fetch[] = $path; } } - if ($present === [] && $cached !== []) { - // A repository never loses all its index files: an empty listing is a connection that - // failed quietly. Taken as is, it would empty the index and the cache. - throw new BackendException('The location listed no index files, while ' . count($cached) . ' are known: not trusting that listing'); + // A repository never loses all its index files: an empty listing is a connection that + // failed quietly. Taken as is, it would empty the index and the cache (or make a check + // report every blob missing, or a cleanup wipe the cache). + if ($present === [] && ($hadIndex || $cached !== [] || ($verify && ($this->cache?->all($this->repositoryId) ?? []) !== []))) { + throw new BackendException('The location listed no index files, while some are known: not trusting that listing'); } + $this->loaded = $present; // The rest is downloaded several at a time where the location allows it: on a large // repository these are tens of thousands of small files. getMany() keeps their order, so // the cached packs listed before each one are added first. @@ -191,13 +196,13 @@ final class BlobIndex { rewind($stream); $this->backend->put('index/' . $packId, $stream); $this->cachePut($packId, $entries); + $this->loaded[$packId] = true; foreach ($entries as $e) { $this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']]; unset($this->pending[$e['id']]); } } - /** @return array> packId => blobs currently located in that pack */ /** * Add the index files that appeared on the location since load(), such as those of packs * another server's cleanup wrote while a restore runs. Only adds and updates: a blob is @@ -205,17 +210,16 @@ final class BlobIndex { * restore lose blobs it knows. A blob in a new pack takes that pack's place. */ public function refresh(): void { - $known = array_flip(array_map(fn (array $b): string => $b[0], $this->blobs)); $fetch = []; $listed = 0; foreach ($this->backend->listFiles('index') as $path) { $listed++; - if (!isset($known[basename($path)])) { + if (!isset($this->loaded[basename($path)])) { $fetch[] = $path; } } - if ($listed === 0 && $this->blobs !== []) { - throw new BackendException('The location listed no index files, while ' . count($known) . ' are known: not trusting that listing'); + if ($listed === 0 && $this->loaded !== []) { + throw new BackendException('The location listed no index files, while ' . count($this->loaded) . ' are known: not trusting that listing'); } $toCache = []; foreach ($this->backend->getMany($fetch) as $path => $data) { @@ -224,6 +228,7 @@ final class BlobIndex { $entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR); $this->downloaded++; $this->add($packId, $entries); + $this->loaded[$packId] = true; $toCache[$packId] = $entries; } if ($toCache !== []) { @@ -231,6 +236,7 @@ final class BlobIndex { } } + /** @return array> packId => blobs currently located in that pack */ public function packs(): array { $packs = []; foreach ($this->blobs as $id => [$pack, $offset, $length]) { @@ -243,6 +249,7 @@ final class BlobIndex { public function removePack(string $packId): void { $this->backend->delete('index/' . $packId); $this->cacheRemove([$packId]); + unset($this->loaded[$packId]); foreach ($this->blobs as $id => $loc) { if ($loc[0] === $packId) { unset($this->blobs[$id]); diff --git a/tests/Unit/Repository/IndexListingTest.php b/tests/Unit/Repository/IndexListingTest.php index d1eab6b..b014fb4 100644 --- a/tests/Unit/Repository/IndexListingTest.php +++ b/tests/Unit/Repository/IndexListingTest.php @@ -65,6 +65,27 @@ class IndexListingTest extends TestCase { } } + public function testACheckWithAnEmptyListingIsAnErrorNotEveryBlobMissing(): void { + $backend = new FlakyBackend(new LocalBackend($this->root)); + $repo = Repository::openWithKey($backend, $this->keys); + $backend->emptyIndexListings = 1; + + $this->expectException(BackendException::class); + $this->expectExceptionMessage('listed no index files'); + $repo->verify($this->snapshot); + } + + public function testRefreshingFetchesOnlyIndexFilesItHasNotLoaded(): void { + $backend = new FlakyBackend(new LocalBackend($this->root)); + $repo = Repository::openWithKey($backend, $this->keys); + $index = (new \ReflectionProperty($repo, 'index'))->getValue($repo); + $index->downloaded = 0; + + $index->refresh(); + + $this->assertSame(0, $index->downloaded, 'every index file is loaded already, also those whose blobs are elsewhere too'); + } + public function testOpeningWithAnEmptyListingKeepsTheCachedIndex(): void { $cache = new class implements IIndexCache { public array $data = [];