Phase 0: logical DB dump/restore without external tools (spike C)

DbDumper reads every prefixed table through the Nextcloud connection in
one REPEATABLE READ (read-only) transaction, with keyset pagination on
the primary key, and streams each table as JSON lines into its own blob
stream (BlobWriter), so unchanged tables are deduplicated. Restore
replaces table contents in one transaction and resets PostgreSQL
sequences.

Spike C (NC34, PostgreSQL, standard image without pg_dump/pgsql ext):
300k-row table with bytea: dump 6.3s / 50 MiB peak, 94 MiB JSON -> 20 MiB
stored; unchanged second dump stores nothing; restore makes every table
checksum-identical and all 110 sequences continue above the maximum.
This commit is contained in:
2026-10-01 18:26:27 +02:00
parent 1839997972
commit 915e534ab2
4 changed files with 443 additions and 0 deletions
+49
View File
@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Repository;
/**
* Write side of a blob stream: data written here is cut into blobs as it arrives and handed
* to the repository right away. Nothing is buffered beyond one blob, so a database dump or any
* other generated stream never needs a temporary file.
*/
final class BlobWriter {
private string $buffer = '';
/** @var list<string> */
private array $blobs = [];
private int $bytes = 0;
/** @param \Closure(string): string $store stores one blob, returns its id */
public function __construct(
private \Closure $store,
private int $blobSize = Repository::BLOB_SIZE,
) {
}
public function write(string $data): void {
$this->buffer .= $data;
$this->bytes += strlen($data);
while (strlen($this->buffer) >= $this->blobSize) {
$this->blobs[] = ($this->store)(substr($this->buffer, 0, $this->blobSize));
$this->buffer = substr($this->buffer, $this->blobSize);
}
}
/** @return list<string> blob ids, in order */
public function finish(): array {
if ($this->buffer !== '') {
$this->blobs[] = ($this->store)($this->buffer);
$this->buffer = '';
}
return $this->blobs;
}
public function bytes(): int {
return $this->bytes;
}
}
+48
View File
@@ -31,6 +31,7 @@ final class Repository {
private StreamCipher $cipher;
private BlobIndex $index;
private ?PackWriter $runPacks = null;
private function __construct(
private IBackend $backend,
@@ -145,6 +146,53 @@ final class Repository {
return $count;
}
/**
* Start writing a generated stream (e.g. a table dump) as blobs. Call flushPacks() when the
* run is done, so the last pack is uploaded before anything references it.
*/
public function blobWriter(array &$stats): BlobWriter {
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
$packs = $this->runPacks;
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
return $this->storeBlob($data, $packs, $stats);
});
}
public function flushPacks(): void {
$this->runPacks?->flush();
}
/**
* Read a blob stream back as lines (lines may span blob boundaries).
*
* @param list<string> $blobIds
* @return \Generator<string>
*/
public function readLines(array $blobIds): \Generator {
$carry = '';
foreach ($blobIds as $id) {
$data = $carry . $this->loadBlob($id);
$lines = explode("\n", $data);
$carry = array_pop($lines);
yield from $lines;
}
if ($carry !== '') {
yield $carry;
}
}
/** Small encrypted objects (manifests). */
public function putObject(string $path, string $data): void {
self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
}
public function getObject(string $path): string {
$fh = $this->backend->get($path);
$data = $this->cipher->decryptString((string)stream_get_contents($fh), 'object:' . $path);
fclose($fh);
return $data;
}
/** @return list<string> */
public function listSnapshots(): array {
return array_map('basename', $this->backend->list('snapshots'));