Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 17 Wiki Activity

M5: resumable database dump and faster database restore

Browse Source 
- DbDumper::dumpStep(): tables are dumped until the step's deadline in one
  REPEATABLE READ transaction and resumed at the last primary key in the
  next step; the running table checksum is carried over. A dump that fits
  in one step is fully consistent, otherwise the snapshot records
  dbConsistent=false. BackupService uses it, so the database phase no
  longer needs one long request (hosted Nextcloud, web cron)
- restore inserts 100 rows per statement and leaves tables whose checksum
  already matches the dump untouched
- spike C: 30-step dump gives the same checksums as the one-shot dump;
  restore rewrote only the 3 damaged tables of 147; 300k-row table
  restored in 14.6 s instead of 46 s
This commit is contained in:
Dillard Blom
2026-10-02 11:34:59 +02:00
parent 08b916cac8
commit 56b14a15b0
3 changed files with 154 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Db/DbDumper.php
+121 -45
View File
@@ -31,6 +31,8 @@ use OCA\NgBackup\Repository\Repository;
*/ */
final class DbDumper { final class DbDumper {
private const BATCH = 2000; private const BATCH = 2000;
/** Rows per INSERT on restore; keeps the number of bound parameters well below database limits. */
private const INSERT_ROWS = 100;
/** Tables without a primary key are read in one query; refuse that above this size. */ /** Tables without a primary key are read in one query; refuse that above this size. */
private const MAX_ROWS_WITHOUT_PK = 50000; private const MAX_ROWS_WITHOUT_PK = 50000;
@@ -41,39 +43,87 @@ final class DbDumper {
} }
/** /**
* @return array{tables: array<string, array{blobs: list<string>, rows: int, bytes: int, sha256: string}>, provider: string} * Dump everything in one go (one transaction, fully consistent).
*
* @return array{tables: array<string, array{blobs: list<string>, rows: int, bytes: int, sha256: string}>, sequences: array, provider: string, consistent: bool}
*/ */
public function dump(Repository $repo, array &$stats): array { public function dump(Repository $repo, array &$stats): array {
$sm = $this->db->createSchemaManager(); $state = $this->startDump();
$tables = array_values(array_filter($sm->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix))); while (!$state['done']) {
sort($tables); $state = $this->dumpStep($repo, $state, PHP_FLOAT_MAX, $stats);
}
return $this->manifest($state);
}
/** State for a resumable dump (see dumpStep()). */
public function startDump(): array {
$tables = array_values(array_filter($this->db->createSchemaManager()->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix)));
sort($tables);
return ['tables' => $tables, 'i' => 0, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null,
'result' => [], 'sequences' => [], 'steps' => 0, 'done' => false];
}
/**
* Dump tables until $deadline, inside one REPEATABLE READ transaction for this step. A table
* that does not fit is resumed in the next step at the last primary key written. If the whole
* dump fits in one step it is fully consistent; otherwise each step is consistent on its own
* ('consistent' => false in the manifest).
*/
public function dumpStep(Repository $repo, array $state, float $deadline, array &$stats): array {
$state['steps']++;
$sm = $this->db->createSchemaManager();
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ); $this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
$this->db->beginTransaction(); $this->db->beginTransaction();
try { try {
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) { if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
$this->db->executeStatement('SET TRANSACTION READ ONLY'); $this->db->executeStatement('SET TRANSACTION READ ONLY');
} }
$result = []; while ($state['i'] < count($state['tables'])) {
foreach ($tables as $table) { $table = $state['tables'][$state['i']];
$schema = $sm->introspectTable($table);
$writer = $repo->blobWriter($stats); $writer = $repo->blobWriter($stats);
$hash = hash_init('sha256'); $hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256');
$rows = 0; if ($state['after'] === null && $state['segments'] === []) {
foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) { $writer->write($this->header($schema) . "\n");
}
$complete = true;
foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) {
$writer->write($line . "\n"); $writer->write($line . "\n");
if ($kind === 'row') { hash_update($hash, $line . "\n");
hash_update($hash, $line . "\n"); $state['rows']++;
$rows++; $state['after'] = $key;
if (microtime(true) >= $deadline) {
$complete = false;
break;
} }
} }
$result[$table] = ['blobs' => $writer->finish(), 'rows' => $rows, 'bytes' => $writer->bytes(), 'sha256' => hash_final($hash)]; $state['segments'][] = $writer->finish();
$state['bytes'] += $writer->bytes();
if (!$complete) {
$state['hash'] = base64_encode(serialize($hash));
break;
}
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
break;
}
}
if ($state['i'] >= count($state['tables'])) {
$state['sequences'] = $this->sequenceStates($state['tables']);
$state['done'] = true;
} }
$sequences = $this->sequenceStates($tables);
} finally { } finally {
$this->db->rollBack(); $this->db->rollBack();
} }
$repo->flushPacks(); $repo->flushPacks();
return ['tables' => $result, 'sequences' => $sequences, 'provider' => get_class($this->db->getDatabasePlatform())]; return $state;
}
public function manifest(array $state): array {
return ['tables' => $state['result'], 'sequences' => $state['sequences'],
'provider' => get_class($this->db->getDatabasePlatform()), 'consistent' => $state['steps'] === 1];
} }
/** /**
@@ -82,16 +132,23 @@ final class DbDumper {
* *
* @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest * @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest
*/ */
public function restore(Repository $repo, array $manifest): int { public function restore(Repository $repo, array $manifest, ?array &$report = null): int {
$sm = $this->db->createSchemaManager(); $sm = $this->db->createSchemaManager();
$existing = array_flip($sm->listTableNames()); $existing = array_flip($sm->listTableNames());
$total = 0; $total = 0;
$report = ['skipped' => [], 'restored' => []];
$this->db->beginTransaction(); $this->db->beginTransaction();
try { try {
foreach ($manifest['tables'] as $table => $info) { foreach ($manifest['tables'] as $table => $info) {
if (!isset($existing[$table])) { if (!isset($existing[$table])) {
throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)"); throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)");
} }
// Unchanged since the backup: leave it alone (much faster than delete + insert).
$live = $this->checksum($table);
if ($live['rows'] === $info['rows'] && hash_equals($info['sha256'], $live['sha256'])) {
$report['skipped'][] = $table;
continue;
}
$this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($table)); $this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($table));
$columns = null; $columns = null;
$binary = []; $binary = [];
@@ -112,17 +169,18 @@ final class DbDumper {
if (is_array($value) && isset($value['$b'])) { if (is_array($value) && isset($value['$b'])) {
$value = base64_decode($value['$b'], true); // text that was not valid UTF-8 $value = base64_decode($value['$b'], true); // text that was not valid UTF-8
} }
$row[$col] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value; $row[] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value;
} }
$batch[] = $row; $batch[] = $row;
if (count($batch) >= 500) { if (count($batch) >= self::INSERT_ROWS) {
$total += $this->insertBatch($table, $batch, $binary); $total += $this->insertBatch($table, $columns, $batch, $binary);
$batch = []; $batch = [];
} }
} }
if ($batch !== []) { if ($batch !== []) {
$total += $this->insertBatch($table, $batch, $binary); $total += $this->insertBatch($table, $columns, $batch, $binary);
} }
$report['restored'][] = $table;
} }
$this->restoreSequences($manifest['sequences'] ?? []); $this->restoreSequences($manifest['sequences'] ?? []);
$this->db->commit(); $this->db->commit();
@@ -135,20 +193,22 @@ final class DbDumper {
/** Same checksum as dump(): sha256 over the row lines in primary-key order. */ /** Same checksum as dump(): sha256 over the row lines in primary-key order. */
public function checksum(string $table): array { public function checksum(string $table): array {
$sm = $this->db->createSchemaManager();
$hash = hash_init('sha256'); $hash = hash_init('sha256');
$rows = 0; $rows = 0;
foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) { foreach ($this->rowLines($this->db->createSchemaManager()->introspectTable($table)) as [, $line]) {
if ($kind === 'row') { hash_update($hash, $line . "\n");
hash_update($hash, $line . "\n"); $rows++;
$rows++;
}
} }
return ['rows' => $rows, 'sha256' => hash_final($hash)]; return ['rows' => $rows, 'sha256' => hash_final($hash)];
} }
/** @return \Generator<string, string> 'header' => line, then 'row' => line ... */ private function header(\Doctrine\DBAL\Schema\Table $table): string {
private function tableLines(\Doctrine\DBAL\Schema\Table $table): \Generator { [$columns, $binary, $pk] = $this->describe($table);
return json_encode(['table' => $table->getName(), 'columns' => $columns, 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR);
}
/** @return array{0: list<string>, 1: list<string>, 2: list<string>} columns, binary columns, primary key */
private function describe(\Doctrine\DBAL\Schema\Table $table): array {
$columns = array_values(array_map(fn ($c) => $c->getName(), $table->getColumns())); $columns = array_values(array_map(fn ($c) => $c->getName(), $table->getColumns()));
$binary = []; $binary = [];
foreach ($table->getColumns() as $c) { foreach ($table->getColumns() as $c) {
@@ -156,25 +216,34 @@ final class DbDumper {
$binary[] = $c->getName(); $binary[] = $c->getName();
} }
} }
$pk = $table->getPrimaryKey()?->getColumns() ?? []; return [$columns, $binary, $table->getPrimaryKey()?->getColumns() ?? []];
yield 'header' => json_encode(['table' => $table->getName(), 'columns' => array_values($columns), 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR); }
/**
* Rows in primary-key order, after the key $after (resume), as [key, JSON line].
*
* @return \Generator<array{0: list<mixed>|null, 1: string}>
*/
private function rowLines(\Doctrine\DBAL\Schema\Table $table, ?array $after = null): \Generator {
[$columns, $binary, $pk] = $this->describe($table);
$binaryFlip = array_flip($binary); $binaryFlip = array_flip($binary);
$q = fn (string $c) => $this->db->quoteIdentifier($c); $q = fn (string $c) => $this->db->quoteIdentifier($c);
$select = 'SELECT ' . implode(', ', array_map($q, $columns)) . ' FROM ' . $q($table->getName()); $select = 'SELECT ' . implode(', ', array_map($q, $columns)) . ' FROM ' . $q($table->getName());
if ($pk === []) { if ($pk === []) {
// Nextcloud requires primary keys; small odd tables are read in one query. // Nextcloud requires primary keys; small odd tables are read in one query (not resumable).
$count = (int)$this->db->fetchOne('SELECT COUNT(*) FROM ' . $q($table->getName())); $count = (int)$this->db->fetchOne('SELECT COUNT(*) FROM ' . $q($table->getName()));
if ($count > self::MAX_ROWS_WITHOUT_PK) { if ($count > self::MAX_ROWS_WITHOUT_PK) {
throw new \RuntimeException("Table {$table->getName()} has no primary key and $count rows; cannot dump it with bounded memory"); throw new \RuntimeException("Table {$table->getName()} has no primary key and $count rows; cannot dump it with bounded memory");
} }
$batches = [$this->db->fetchAllNumeric($select)]; $batches = [$this->db->fetchAllNumeric($select)];
$pkIdx = [];
} else { } else {
$batches = $this->keysetBatches($select, $pk, $columns); $batches = $this->keysetBatches($select, $pk, $columns, $after);
$pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk);
} }
foreach ($batches as $rows) { foreach ($batches as $rows) {
foreach ($rows as $row) { foreach ($rows as $row) {
$key = $pkIdx === [] ? null : array_map(fn ($i) => $row[$i], $pkIdx);
foreach ($columns as $i => $col) { foreach ($columns as $i => $col) {
if (is_resource($row[$i])) { if (is_resource($row[$i])) {
$row[$i] = stream_get_contents($row[$i]); $row[$i] = stream_get_contents($row[$i]);
@@ -189,23 +258,23 @@ final class DbDumper {
$row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]); $row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]);
} }
} }
yield 'row' => json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR); yield [$key, json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR)];
} }
} }
} }
/** @return \Generator<list<list<mixed>>> */ /** @return \Generator<list<list<mixed>>> */
private function keysetBatches(string $select, array $pk, array $columns): \Generator { private function keysetBatches(string $select, array $pk, array $columns, ?array $after): \Generator {
$q = fn (string $c) => $this->db->quoteIdentifier($c); $q = fn (string $c) => $this->db->quoteIdentifier($c);
$order = ' ORDER BY ' . implode(', ', array_map($q, $pk)); $order = ' ORDER BY ' . implode(', ', array_map($q, $pk));
$pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk); $cols = array_map(fn ($c) => $q($c), $pk);
$last = null; $last = $after;
while (true) { while (true) {
$sql = $select; $sql = $select;
$params = []; $params = [];
if ($last !== null) { if ($last !== null) {
// Row-value comparison (a, b) > (?, ?), supported by PostgreSQL, MySQL/MariaDB and SQLite. // Row-value comparison (a, b) > (?, ?), supported by PostgreSQL, MySQL/MariaDB and SQLite.
$sql .= ' WHERE (' . implode(', ', array_map($q, $pk)) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')'; $sql .= ' WHERE (' . implode(', ', $cols) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')';
$params = $last; $params = $last;
} }
$rows = $this->db->fetchAllNumeric($sql . $order . ' LIMIT ' . self::BATCH, $params); $rows = $this->db->fetchAllNumeric($sql . $order . ' LIMIT ' . self::BATCH, $params);
@@ -213,22 +282,29 @@ final class DbDumper {
return; return;
} }
yield $rows; yield $rows;
$end = end($rows);
$last = array_map(fn ($i) => $end[$i], $pkIdx);
if (count($rows) < self::BATCH) { if (count($rows) < self::BATCH) {
return; return;
} }
$end = end($rows);
$last = array_map(fn ($c) => $end[array_search($c, $columns, true)], $pk);
} }
} }
private function insertBatch(string $table, array $rows, array $binary): int { /** One INSERT with several rows (portable multi-row VALUES). */
private function insertBatch(string $table, array $columns, array $rows, array $binary): int {
$q = fn (string $c) => $this->db->quoteIdentifier($c);
$placeholders = '(' . implode(', ', array_fill(0, count($columns), '?')) . ')';
$sql = 'INSERT INTO ' . $q($table) . ' (' . implode(', ', array_map($q, $columns)) . ') VALUES '
. implode(', ', array_fill(0, count($rows), $placeholders));
$params = [];
$types = [];
foreach ($rows as $row) { foreach ($rows as $row) {
$types = []; foreach ($columns as $i => $col) {
foreach ($row as $col => $_) { $params[] = $row[$i];
$types[$col] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : \Doctrine\DBAL\ParameterType::STRING; $types[] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : ($row[$i] === null ? \Doctrine\DBAL\ParameterType::NULL : \Doctrine\DBAL\ParameterType::STRING);
} }
$this->db->insert($this->db->quoteIdentifier($table), array_combine(array_map(fn ($c) => $this->db->quoteIdentifier($c), array_keys($row)), $row), array_values($types));
} }
$this->db->executeStatement($sql, $params, $types);
return count($rows); return count($rows);
} }
lib/Service/BackupService.php
+18 -8
View File
@@ -127,14 +127,24 @@ class BackupService {
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR); $state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
if ($run->getPhase() === 'db') { if ($run->getPhase() === 'db') {
@set_time_limit(0); // the dump is one transaction; it cannot be split across requests yet $dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_'));
$dbStats = []; $dbStats = $state['dbStats'] ?? [];
$manifest = (new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_')))->dump($repo, $dbStats); // With system cron the whole dump usually fits in one step (one transaction, fully
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR)); // consistent). Under web cron it continues per step; the snapshot then says so.
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot']; $dump = $dumper->dumpStep($repo, $state['dump'] ?? $dumper->startDump(), $deadline, $dbStats);
$state['dbStats'] = $dbStats + ['tables' => count($manifest['tables']), $state['dbStats'] = $dbStats;
'rows' => array_sum(array_column($manifest['tables'], 'rows'))]; if ($dump['done']) {
$run->setPhase('files'); $manifest = $dumper->manifest($dump);
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
$state['files']['meta']['dbConsistent'] = $manifest['consistent'];
$state['dbStats'] += ['tables' => count($manifest['tables']), 'rows' => array_sum(array_column($manifest['tables'], 'rows')),
'steps' => $dump['steps'], 'consistent' => $manifest['consistent']];
unset($state['dump']);
$run->setPhase('files');
} else {
$state['dump'] = $dump;
}
} else { } else {
$state['files'] = BackupRun::step($repo, $state['files'], $deadline); $state['files'] = BackupRun::step($repo, $state['files'], $deadline);
if ($state['files']['phase'] === 'done') { if ($state['files']['phase'] === 'done') {
spikes/03-db-dump.php
+15 -2
View File
@@ -71,6 +71,17 @@ $dumpBytes = array_sum(array_column($manifest['tables'], 'bytes'));
printf("dump size (plain JSON lines): %.1f MiB\n", $dumpBytes / 1048576); printf("dump size (plain JSON lines): %.1f MiB\n", $dumpBytes / 1048576);
$check('checksums in dump match live tables', array_map(fn ($x) => $x['sha256'], $manifest['tables']) === array_map(fn ($x) => $x['sha256'], $before)); $check('checksums in dump match live tables', array_map(fn ($x) => $x['sha256'], $manifest['tables']) === array_map(fn ($x) => $x['sha256'], $before));
// The same dump in many tiny steps (as under web cron) must give identical table checksums.
$stepped = $dumper->startDump();
$stepStats = [];
while (!$stepped['done']) {
$stepped = $dumper->dumpStep($repo, $stepped, microtime(true) + 0.05, $stepStats);
}
$sm = $dumper->manifest($stepped);
printf("stepped dump: %d steps, consistent=%s\n", $stepped['steps'], json_encode($sm['consistent']));
$check('stepped dump: same checksums as the one-shot dump', array_map(fn ($x) => $x['sha256'], $sm['tables']) === array_map(fn ($x) => $x['sha256'], $manifest['tables']));
$check('stepped dump is marked as not fully consistent', $stepped['steps'] === 1 || $sm['consistent'] === false);
// Second dump without changes: blobs are reused. // Second dump without changes: blobs are reused.
$stats2 = ['newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0]; $stats2 = ['newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
$dumper->dump($repo, $stats2); $dumper->dump($repo, $stats2);
@@ -90,8 +101,10 @@ memory_reset_peak_usage();
$t = microtime(true); $t = microtime(true);
$repo2 = Repository::open(new LocalBackend($repoDir), 'spike passphrase'); $repo2 = Repository::open(new LocalBackend($repoDir), 'spike passphrase');
$m = json_decode($repo2->getObject('db/1'), true, 512, JSON_THROW_ON_ERROR); $m = json_decode($repo2->getObject('db/1'), true, 512, JSON_THROW_ON_ERROR);
$rows = $dumper->restore($repo2, $m); $report = [];
printf("restore: %d rows in %.1fs, peak %.1f MiB\n", $rows, microtime(true) - $t, memory_get_peak_usage() / 1048576); $rows = $dumper->restore($repo2, $m, $report);
printf("restore: %d rows in %.1fs, peak %.1f MiB, %d tables restored, %d unchanged skipped\n", $rows, microtime(true) - $t, memory_get_peak_usage() / 1048576, count($report['restored']), count($report['skipped']));
$check('restore only rewrote the damaged tables', count($report['restored']) === count($changed));
$mismatch = []; $mismatch = [];
foreach ($tables as $tb) { foreach ($tables as $tb) {