Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Index: an empty listing is an error in checks and cleanups too; refresh() fetches only index files it has not loaded; full-restore CI on Nextcloud 35.0.0

Browse Source
This commit is contained in:
Dillard Blom
2026-10-19 22:13:26 +02:00
parent c37b26ea38
commit d504067cd8
3 changed files with 45 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+8 -6
View File
@@ -93,6 +93,8 @@ jobs:
- name: Per-user backup/restore round trip on Nextcloud ${{ matrix.nextcloud }}
run: tests/integration/user-roundtrip.sh "docker exec -u www-data ngbu" /var/www/html/data
# 35.0.0, not 35: the app store lists apps for a new point release only some time after it
# is out, and this test installs Mail and Deck from it.
integration-full-restore:
# Disaster recovery (occ backup:restore:full) on a SECOND, fresh Nextcloud instance that
# never saw the first one, sharing only the backup location. Does not depend on the database
@@ -103,14 +105,14 @@ jobs:
with:
persist-credentials: false
- name: Full restore round trip
run: tests/integration/full-restore-roundtrip.sh 35-apache
run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache
- name: Full restore round trip on PostgreSQL, with Mail (foreign keys) and a disabled app
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql
run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache pgsql
- name: Full restore round trip on MariaDB, with Mail (foreign keys) and a disabled app
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql
run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache mysql
- name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on PostgreSQL
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql legacy
run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache pgsql legacy
- name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on MariaDB
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql legacy
run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache mysql legacy
- name: Full restore over SFTP on PostgreSQL, reading packs ahead on several connections
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql sftp
run: tests/integration/full-restore-roundtrip.sh 35.0.0-apache pgsql sftp
lib/Repository/BlobIndex.php
+16 -9
View File
@@ -33,6 +33,8 @@ final class BlobIndex {
private const CACHE_LOT = 256;
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
public int $downloaded = 0;
/** @var array<string, true> packs whose index file is loaded (refresh() fetches only others) */
private array $loaded = [];
public function __construct(
private IBackend $backend,
@@ -91,6 +93,7 @@ final class BlobIndex {
* longer exist on the backend are dropped from the cache.
*/
public function load(bool $verify = false, ?callable $progress = null): void {
$hadIndex = $this->blobs !== [];
$this->blobs = [];
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
$present = [];
@@ -105,11 +108,13 @@ final class BlobIndex {
$fetch[] = $path;
}
}
if ($present === [] && $cached !== []) {
// A repository never loses all its index files: an empty listing is a connection that
// failed quietly. Taken as is, it would empty the index and the cache.
throw new BackendException('The location listed no index files, while ' . count($cached) . ' are known: not trusting that listing');
// A repository never loses all its index files: an empty listing is a connection that
// failed quietly. Taken as is, it would empty the index and the cache (or make a check
// report every blob missing, or a cleanup wipe the cache).
if ($present === [] && ($hadIndex || $cached !== [] || ($verify && ($this->cache?->all($this->repositoryId) ?? []) !== []))) {
throw new BackendException('The location listed no index files, while some are known: not trusting that listing');
}
$this->loaded = $present;
// The rest is downloaded several at a time where the location allows it: on a large
// repository these are tens of thousands of small files. getMany() keeps their order, so
// the cached packs listed before each one are added first.
@@ -191,13 +196,13 @@ final class BlobIndex {
rewind($stream);
$this->backend->put('index/' . $packId, $stream);
$this->cachePut($packId, $entries);
$this->loaded[$packId] = true;
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
unset($this->pending[$e['id']]);
}
}
/** @return array<string, list<array{id:string, offset:int, length:int}>> packId => blobs currently located in that pack */
/**
* Add the index files that appeared on the location since load(), such as those of packs
* another server's cleanup wrote while a restore runs. Only adds and updates: a blob is
@@ -205,17 +210,16 @@ final class BlobIndex {
* restore lose blobs it knows. A blob in a new pack takes that pack's place.
*/
public function refresh(): void {
$known = array_flip(array_map(fn (array $b): string => $b[0], $this->blobs));
$fetch = [];
$listed = 0;
foreach ($this->backend->listFiles('index') as $path) {
$listed++;
if (!isset($known[basename($path)])) {
if (!isset($this->loaded[basename($path)])) {
$fetch[] = $path;
}
}
if ($listed === 0 && $this->blobs !== []) {
throw new BackendException('The location listed no index files, while ' . count($known) . ' are known: not trusting that listing');
if ($listed === 0 && $this->loaded !== []) {
throw new BackendException('The location listed no index files, while ' . count($this->loaded) . ' are known: not trusting that listing');
}
$toCache = [];
foreach ($this->backend->getMany($fetch) as $path => $data) {
@@ -224,6 +228,7 @@ final class BlobIndex {
$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
$this->downloaded++;
$this->add($packId, $entries);
$this->loaded[$packId] = true;
$toCache[$packId] = $entries;
}
if ($toCache !== []) {
@@ -231,6 +236,7 @@ final class BlobIndex {
}
}
/** @return array<string, list<array{id:string, offset:int, length:int}>> packId => blobs currently located in that pack */
public function packs(): array {
$packs = [];
foreach ($this->blobs as $id => [$pack, $offset, $length]) {
@@ -243,6 +249,7 @@ final class BlobIndex {
public function removePack(string $packId): void {
$this->backend->delete('index/' . $packId);
$this->cacheRemove([$packId]);
unset($this->loaded[$packId]);
foreach ($this->blobs as $id => $loc) {
if ($loc[0] === $packId) {
unset($this->blobs[$id]);
tests/Unit/Repository/IndexListingTest.php
+21
View File
@@ -65,6 +65,27 @@ class IndexListingTest extends TestCase {
}
}
public function testACheckWithAnEmptyListingIsAnErrorNotEveryBlobMissing(): void {
$backend = new FlakyBackend(new LocalBackend($this->root));
$repo = Repository::openWithKey($backend, $this->keys);
$backend->emptyIndexListings = 1;
$this->expectException(BackendException::class);
$this->expectExceptionMessage('listed no index files');
$repo->verify($this->snapshot);
}
public function testRefreshingFetchesOnlyIndexFilesItHasNotLoaded(): void {
$backend = new FlakyBackend(new LocalBackend($this->root));
$repo = Repository::openWithKey($backend, $this->keys);
$index = (new \ReflectionProperty($repo, 'index'))->getValue($repo);
$index->downloaded = 0;
$index->refresh();
$this->assertSame(0, $index->downloaded, 'every index file is loaded already, also those whose blobs are elsewhere too');
}
public function testOpeningWithAnEmptyListingKeepsTheCachedIndex(): void {
$cache = new class implements IIndexCache {
public array $data = [];