Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Merge branch 'main' into polish

Browse Source 
# Conflicts:
#	CHANGELOG.md
This commit is contained in:
Dillard Blom
2026-10-08 21:19:34 +02:00
parent a01aa16ccd 4fa7cbf6ef
commit 65f14b4380
14 changed files with 276 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+14
View File
@@ -26,6 +26,19 @@ instead carried by the app description, the git tag, the release, and this file.
- App store listing: the homepage link points to the GitHub repository instead of back to the
app store page itself.
## [0.9.3-beta.4] - 2026-10-08
Bugfix release.
### Fixed
- A backup failed with "cannot execute INSERT in a read-only transaction" on PostgreSQL as soon
as the database dump filled a whole pack (32 MiB): the pack's index cache row was written
inside the read-only dump transaction. On other databases that row was rolled back silently.
- Backups to a remote location slowed down as the repository grew: every backup step listed the
repository's index with a round trip per file. Over SFTP to a Hetzner Storage Box a 60 GiB
repository already lost a large part of each step to it; backing up continuously
(`occ backup:run`) went from 45 KiB/s to 5.8 MB/s with the fix.
## [0.9.2-beta.3] - 2026-10-08
Bugfix release.
@@ -141,6 +154,7 @@ and not implemented yet.
- See the "Storage backend experiences" section of the README for provider-specific quirks found
so far (e.g. Hetzner Storage Box sub-accounts).
[0.9.3-beta.4]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.3-beta.4
[0.9.2-beta.3]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.2-beta.3
[0.9.1-beta.2]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.1-beta.2
[0.9.0-beta.1]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.0-beta.1
README.md
+1 -1
View File
@@ -7,7 +7,7 @@ installations and hosted Nextcloud. (Choosing SMB as a storage backend is the on
see "Storage backend experiences" below — that requirement comes from files_external, not from
ng_backup, and applies to any Nextcloud app that writes to an SMB share.)
**Status: beta (0.9.2).** Tested thoroughly, usable alongside your current backup, but not yet a
**Status: beta (0.9.3).** Tested thoroughly, usable alongside your current backup, but not yet a
drop-in replacement for it. See "Known limitations" below, `PLAN.md` for the design and
`docs/HOWTO.md` for a step-by-step guide from setup to a tested restore.
appinfo/info.xml
+1 -1
View File
@@ -9,7 +9,7 @@ directory to S3, SFTP, WebDAV, FTP, SMB or a local path, with restore of files a
single users (via user_migration) and of the whole server onto a fresh installation. Tested
thoroughly and usable alongside your current backup, but not yet a drop-in replacement for it.
See the README for known limitations and the recovery kit before relying on it.]]></description>
<version>0.9.2</version>
<version>0.9.3</version>
<licence>agpl</licence>
<author mail="dillard.blom@ensembia.com">Dillard Blom</author>
<namespace>NgBackup</namespace>
lib/Backend/IBackend.php
+10
View File
@@ -37,5 +37,15 @@ interface IBackend {
/** @return list<string> object paths below $prefix (recursive) */
public function list(string $prefix): array;
/**
* Object paths directly in $dir, for a folder that only holds objects (index/, snapshots/,
* catalog/, trash/info/). One listing, without checking the type of every entry: on a remote
* storage list() costs a round trip per entry, which for index/ (one file per pack) grows
* with the size of the repository.
*
* @return list<string>
*/
public function listFiles(string $dir): array;
public function delete(string $path): void;
}
lib/Backend/LocalBackend.php
+19
View File
@@ -107,6 +107,25 @@ final class LocalBackend implements IBackend {
return $result;
}
public function listFiles(string $dir): array {
$base = $this->abs($dir);
if (!is_dir($base)) {
return [];
}
$result = [];
$names = scandir($base);
if ($names === false) {
throw new BackendException('Cannot list ' . $dir);
}
foreach ($names as $name) {
if ($name !== '.' && $name !== '..' && !StorageBackend::isTemporary($name)) {
$result[] = trim($dir, '/') . '/' . $name;
}
}
sort($result);
return $result;
}
public function delete(string $path): void {
@unlink($this->abs($path));
}
lib/Backend/StorageBackend.php
+22
View File
@@ -143,6 +143,28 @@ final class StorageBackend implements IBackend {
}
}
public function listFiles(string $dir): array {
$abs = $this->abs($dir);
if (!$this->storage->is_dir($abs)) {
return []; // not created yet
}
$dh = $this->storage->opendir($abs);
if ($dh === false) {
// An existing folder read as empty would look like a new repository.
throw new BackendException('Cannot list ' . $dir);
}
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
$result = [];
while (($name = readdir($dh)) !== false) {
if ($name !== '.' && $name !== '..' && !self::isTemporary($name)) {
$result[] = substr($abs . '/' . $name, $cut);
}
}
closedir($dh);
sort($result);
return $result;
}
private function listInto(string $dir, array &$result): void {
if (!$this->storage->is_dir($dir)) {
return;
lib/Db/DbDumper.php
+43 -36
View File
@@ -72,50 +72,57 @@ final class DbDumper {
public function dumpStep(Repository $repo, array $state, float $deadline, array &$stats): array {
$state['steps']++;
$sm = $this->db->createSchemaManager();
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
$this->db->beginTransaction();
// A pack that fills up during the dump is uploaded right away; its index cache row must
// wait until the read-only transaction is over.
$repo->holdIndexCache();
try {
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
$this->db->executeStatement('SET TRANSACTION READ ONLY');
}
while ($state['i'] < count($state['tables'])) {
$table = $state['tables'][$state['i']];
$schema = $sm->introspectTable($table);
$writer = $repo->blobWriter($stats);
$hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256');
if ($state['after'] === null && $state['segments'] === []) {
$writer->write($this->header($schema) . "\n");
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
$this->db->beginTransaction();
try {
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
$this->db->executeStatement('SET TRANSACTION READ ONLY');
}
$complete = true;
foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) {
$writer->write($line . "\n");
hash_update($hash, $line . "\n");
$state['rows']++;
$state['after'] = $key;
if (microtime(true) >= $deadline) {
$complete = false;
while ($state['i'] < count($state['tables'])) {
$table = $state['tables'][$state['i']];
$schema = $sm->introspectTable($table);
$writer = $repo->blobWriter($stats);
$hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256');
if ($state['after'] === null && $state['segments'] === []) {
$writer->write($this->header($schema) . "\n");
}
$complete = true;
foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) {
$writer->write($line . "\n");
hash_update($hash, $line . "\n");
$state['rows']++;
$state['after'] = $key;
if (microtime(true) >= $deadline) {
$complete = false;
break;
}
}
$state['segments'][] = $writer->finish();
$state['bytes'] += $writer->bytes();
if (!$complete) {
$state['hash'] = base64_encode(serialize($hash));
break;
}
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
break;
}
}
$state['segments'][] = $writer->finish();
$state['bytes'] += $writer->bytes();
if (!$complete) {
$state['hash'] = base64_encode(serialize($hash));
break;
if ($state['i'] >= count($state['tables'])) {
$state['sequences'] = $this->sequenceStates($state['tables']);
$state['done'] = true;
}
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
break;
}
}
if ($state['i'] >= count($state['tables'])) {
$state['sequences'] = $this->sequenceStates($state['tables']);
$state['done'] = true;
} finally {
$this->db->rollBack();
}
} finally {
$this->db->rollBack();
$repo->releaseIndexCache();
}
$repo->flushPacks();
return $state;
lib/Repository/BlobIndex.php
+45 -3
View File
@@ -25,6 +25,8 @@ final class BlobIndex {
private array $pending = [];
private ?IIndexCache $cache = null;
/** @var list<\Closure():void>|null cache writes held back by holdCache(), or null when writing directly */
private ?array $heldCacheWrites = null;
private string $repositoryId = '';
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
public int $downloaded = 0;
@@ -40,6 +42,46 @@ final class BlobIndex {
$this->repositoryId = $repositoryId;
}
/**
* Hold back cache writes until releaseCache(). The database dump reads inside a read-only
* transaction on the same connection that the cache writes to: a write there fails on
* PostgreSQL and would be rolled back with the dump transaction elsewhere.
*/
public function holdCache(): void {
$this->heldCacheWrites ??= [];
}
/** Write the cache changes held back since holdCache(). */
public function releaseCache(): void {
$writes = $this->heldCacheWrites ?? [];
$this->heldCacheWrites = null;
foreach ($writes as $write) {
$write();
}
}
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
private function cachePut(string $packId, array $entries): void {
if ($this->heldCacheWrites !== null) {
$this->heldCacheWrites[] = function () use ($packId, $entries): void {
$this->cache?->put($this->repositoryId, $packId, $entries);
};
return;
}
$this->cache?->put($this->repositoryId, $packId, $entries);
}
/** @param list<string> $packIds */
private function cacheRemove(array $packIds): void {
if ($this->heldCacheWrites !== null) {
$this->heldCacheWrites[] = function () use ($packIds): void {
$this->cache?->remove($this->repositoryId, $packIds);
};
return;
}
$this->cache?->remove($this->repositoryId, $packIds);
}
/**
* Load the index: one listing of index/, entries from the local cache where available,
* and only the index files that are not cached yet are downloaded. Cached packs that no
@@ -50,7 +92,7 @@ final class BlobIndex {
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
$present = [];
$this->downloaded = 0;
foreach ($this->backend->list('index') as $path) {
foreach ($this->backend->listFiles('index') as $path) {
$packId = basename($path);
$present[$packId] = true;
if (isset($cached[$packId])) {
@@ -112,7 +154,7 @@ final class BlobIndex {
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
rewind($stream);
$this->backend->put('index/' . $packId, $stream);
$this->cache?->put($this->repositoryId, $packId, $entries);
$this->cachePut($packId, $entries);
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
unset($this->pending[$e['id']]);
@@ -131,7 +173,7 @@ final class BlobIndex {
/** Forget a pack: delete its index file and cache row. Blobs that now live elsewhere are kept. */
public function removePack(string $packId): void {
$this->backend->delete('index/' . $packId);
$this->cache?->remove($this->repositoryId, [$packId]);
$this->cacheRemove([$packId]);
foreach ($this->blobs as $id => $loc) {
if ($loc[0] === $packId) {
unset($this->blobs[$id]);
lib/Repository/Catalog.php
+1 -1
View File
@@ -128,7 +128,7 @@ final class Catalog {
/** @return list<int> */
private function generations(): array {
return array_map(fn ($p) => (int)basename($p), $this->backend->list('catalog'));
return array_map(fn ($p) => (int)basename($p), $this->backend->listFiles('catalog'));
}
private function path(int $gen): string {
lib/Repository/Repository.php
+12 -3
View File
@@ -92,8 +92,8 @@ final class Repository {
private function loadCatalog(?ICatalogAnchor $anchor): void {
$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
$this->catalog->load(
fn () => array_map('basename', $this->backend->list('snapshots')),
fn () => array_map('basename', $this->backend->list('trash/info')),
fn () => array_map('basename', $this->backend->listFiles('snapshots')),
fn () => array_map('basename', $this->backend->listFiles('trash/info')),
fn () => $this->backend->list('users'));
}
@@ -262,6 +262,15 @@ final class Repository {
});
}
/** See BlobIndex::holdCache(): no database writes until releaseIndexCache(). */
public function holdIndexCache(): void {
$this->index->holdCache();
}
public function releaseIndexCache(): void {
$this->index->releaseCache();
}
public function flushPacks(): void {
$this->runPacks?->flush();
}
@@ -634,7 +643,7 @@ final class Repository {
* @return list<string>
*/
public function listSnapshots(): array {
$present = array_flip(array_map('basename', $this->backend->list('snapshots')));
$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
$listed = array_keys($this->catalog->snapshots);
foreach ($listed as $id) {
if (!isset($present[$id])) {
tests/Unit/Backend/StorageBackendListTest.php
+57
View File
@@ -0,0 +1,57 @@
<?php
declare(strict_types=1);
namespace OCA\NgBackup\Tests\Unit\Backend;
use OCA\NgBackup\Backend\BackendException;
use OCA\NgBackup\Backend\StorageBackend;
use OCA\NgBackup\Tests\Unit\TempDirTrait;
use OCP\Files\Storage\IStorage;
use PHPUnit\Framework\TestCase;
/** On a remote storage every is_dir() is a round trip: listing index/ must not cost one per pack. */
class StorageBackendListTest extends TestCase {
use TempDirTrait;
public function testListFilesChecksOnlyTheFolderItself(): void {
$dir = $this->tempDir();
foreach (['b1', 'a2', 'c3.part-0123abcd'] as $name) {
touch("$dir/$name");
}
$isDirCalls = 0;
$storage = $this->createMock(IStorage::class);
$storage->method('instanceOfStorage')->willReturn(false);
$storage->method('is_dir')->willReturnCallback(function (string $path) use (&$isDirCalls): bool {
$isDirCalls++;
return $path === 'repo' || $path === 'repo/index';
});
$storage->method('opendir')->willReturnCallback(fn (string $path) => $path === 'repo/index' ? opendir($dir) : false);
$backend = new StorageBackend($storage, 'repo');
$isDirCalls = 0; // the constructor checks the base folder
$this->assertSame(['index/a2', 'index/b1'], $backend->listFiles('index'));
$this->assertSame(1, $isDirCalls);
$isDirCalls = 0;
$this->assertSame(['index/a2', 'index/b1'], $backend->list('index'));
$this->assertSame(4, $isDirCalls, 'list() checks every entry: the reason listFiles() exists');
}
public function testListFilesOfAMissingFolderIsEmpty(): void {
$storage = $this->createMock(IStorage::class);
$storage->method('instanceOfStorage')->willReturn(false);
$storage->method('is_dir')->willReturnCallback(fn (string $path): bool => $path === 'repo');
$storage->expects($this->never())->method('opendir');
$this->assertSame([], (new StorageBackend($storage, 'repo'))->listFiles('snapshots'));
}
public function testAnUnreadableFolderIsAnErrorNotEmpty(): void {
$storage = $this->createMock(IStorage::class);
$storage->method('instanceOfStorage')->willReturn(false);
$storage->method('is_dir')->willReturn(true);
$storage->method('opendir')->willReturn(false);
$this->expectException(BackendException::class);
(new StorageBackend($storage, 'repo'))->listFiles('catalog');
}
}
tests/Unit/Repository/IndexCacheTest.php
+37
View File
@@ -51,4 +51,41 @@ class IndexCacheTest extends TestCase {
$fourth->restore($r1['snapshot'], $target);
$this->assertCount(12, self::hashTree($target));
}
/** The database dump reads in a read-only transaction: a pack that fills up then must not write the cache yet. */
public function testHeldCacheWritesWaitForRelease(): void {
$cache = new class implements IIndexCache {
public array $data = [];
public bool $readOnly = false;
public function all(string $r): array { return $this->data[$r] ?? []; }
public function put(string $r, string $p, array $e): void {
if ($this->readOnly) {
throw new \RuntimeException('cannot execute INSERT in a read-only transaction');
}
$this->data[$r][$p] = $e;
}
public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
};
$backend = new LocalBackend($this->tempDir());
$keys = KeyRing::generate();
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
$repo = Repository::openWithKey($backend, $keys, $cache);
$stats = [];
$repo->holdIndexCache();
$cache->readOnly = true;
$writer = $repo->blobWriter($stats);
for ($i = 0; $i < 40; $i++) {
$writer->write(random_bytes(1048576)); // more than one 32 MiB pack
}
$writer->finish();
$this->assertGreaterThan(0, count($backend->list('index')), 'a full pack was uploaded during the hold');
$this->assertSame([], $cache->data);
$cache->readOnly = false;
$repo->releaseIndexCache();
$repo->flushPacks();
$this->assertCount(count($backend->list('index')), array_merge(...array_values($cache->data)));
$this->assertSame(0, Repository::openWithKey($backend, $keys, $cache)->indexDownloads());
}
}
tests/Unit/Repository/VerifyTest.php
+4
View File
@@ -166,6 +166,10 @@ final class FlakyRangeBackend implements IBackend {
return $this->inner->list($prefix);
}
public function listFiles(string $dir): array {
return $this->inner->listFiles($dir);
}
public function delete(string $path): void {
$this->inner->delete($path);
}
tests/bootstrap.php
+10
View File
@@ -3,3 +3,13 @@
declare(strict_types=1);
require __DIR__ . '/../vendor/autoload.php';
// nextcloud/ocp ships the public interfaces without an autoloader; tests that mock them need one.
spl_autoload_register(static function (string $class): void {
if (str_starts_with($class, 'OCP\\')) {
$file = __DIR__ . '/../vendor/nextcloud/ocp/' . str_replace('\\', '/', $class) . '.php';
if (is_file($file)) {
require $file;
}
}
});