From f98487a060fcdc87f11e587a4b037c4be104800b Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Fri, 2 Oct 2026 11:34:59 +0200 Subject: [PATCH] M5: resumable database dump and faster database restore - DbDumper::dumpStep(): tables are dumped until the step's deadline in one REPEATABLE READ transaction and resumed at the last primary key in the next step; the running table checksum is carried over. A dump that fits in one step is fully consistent, otherwise the snapshot records dbConsistent=false. BackupService uses it, so the database phase no longer needs one long request (hosted Nextcloud, web cron) - restore inserts 100 rows per statement and leaves tables whose checksum already matches the dump untouched - spike C: 30-step dump gives the same checksums as the one-shot dump; restore rewrote only the 3 damaged tables of 147; 300k-row table restored in 14.6 s instead of 46 s --- lib/Db/DbDumper.php | 166 +++++++++++++++++++++++++--------- lib/Service/BackupService.php | 26 ++++-- spikes/03-db-dump.php | 17 +++- 3 files changed, 154 insertions(+), 55 deletions(-) diff --git a/lib/Db/DbDumper.php b/lib/Db/DbDumper.php index 035c19d..d1fb64c 100644 --- a/lib/Db/DbDumper.php +++ b/lib/Db/DbDumper.php @@ -31,6 +31,8 @@ use OCA\NgBackup\Repository\Repository; */ final class DbDumper { private const BATCH = 2000; + /** Rows per INSERT on restore; keeps the number of bound parameters well below database limits. */ + private const INSERT_ROWS = 100; /** Tables without a primary key are read in one query; refuse that above this size. */ private const MAX_ROWS_WITHOUT_PK = 50000; @@ -41,39 +43,87 @@ final class DbDumper { } /** - * @return array{tables: array, rows: int, bytes: int, sha256: string}>, provider: string} + * Dump everything in one go (one transaction, fully consistent). + * + * @return array{tables: array, rows: int, bytes: int, sha256: string}>, sequences: array, provider: string, consistent: bool} */ public function dump(Repository $repo, array &$stats): array { - $sm = $this->db->createSchemaManager(); - $tables = array_values(array_filter($sm->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix))); - sort($tables); + $state = $this->startDump(); + while (!$state['done']) { + $state = $this->dumpStep($repo, $state, PHP_FLOAT_MAX, $stats); + } + return $this->manifest($state); + } + /** State for a resumable dump (see dumpStep()). */ + public function startDump(): array { + $tables = array_values(array_filter($this->db->createSchemaManager()->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix))); + sort($tables); + return ['tables' => $tables, 'i' => 0, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null, + 'result' => [], 'sequences' => [], 'steps' => 0, 'done' => false]; + } + + /** + * Dump tables until $deadline, inside one REPEATABLE READ transaction for this step. A table + * that does not fit is resumed in the next step at the last primary key written. If the whole + * dump fits in one step it is fully consistent; otherwise each step is consistent on its own + * ('consistent' => false in the manifest). + */ + public function dumpStep(Repository $repo, array $state, float $deadline, array &$stats): array { + $state['steps']++; + $sm = $this->db->createSchemaManager(); $this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ); $this->db->beginTransaction(); try { if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) { $this->db->executeStatement('SET TRANSACTION READ ONLY'); } - $result = []; - foreach ($tables as $table) { + while ($state['i'] < count($state['tables'])) { + $table = $state['tables'][$state['i']]; + $schema = $sm->introspectTable($table); $writer = $repo->blobWriter($stats); - $hash = hash_init('sha256'); - $rows = 0; - foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) { + $hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256'); + if ($state['after'] === null && $state['segments'] === []) { + $writer->write($this->header($schema) . "\n"); + } + $complete = true; + foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) { $writer->write($line . "\n"); - if ($kind === 'row') { - hash_update($hash, $line . "\n"); - $rows++; + hash_update($hash, $line . "\n"); + $state['rows']++; + $state['after'] = $key; + if (microtime(true) >= $deadline) { + $complete = false; + break; } } - $result[$table] = ['blobs' => $writer->finish(), 'rows' => $rows, 'bytes' => $writer->bytes(), 'sha256' => hash_final($hash)]; + $state['segments'][] = $writer->finish(); + $state['bytes'] += $writer->bytes(); + if (!$complete) { + $state['hash'] = base64_encode(serialize($hash)); + break; + } + $state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'], + 'bytes' => $state['bytes'], 'sha256' => hash_final($hash)]; + $state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]); + if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) { + break; + } + } + if ($state['i'] >= count($state['tables'])) { + $state['sequences'] = $this->sequenceStates($state['tables']); + $state['done'] = true; } - $sequences = $this->sequenceStates($tables); } finally { $this->db->rollBack(); } $repo->flushPacks(); - return ['tables' => $result, 'sequences' => $sequences, 'provider' => get_class($this->db->getDatabasePlatform())]; + return $state; + } + + public function manifest(array $state): array { + return ['tables' => $state['result'], 'sequences' => $state['sequences'], + 'provider' => get_class($this->db->getDatabasePlatform()), 'consistent' => $state['steps'] === 1]; } /** @@ -82,16 +132,23 @@ final class DbDumper { * * @param array{tables: array, rows: int}>} $manifest */ - public function restore(Repository $repo, array $manifest): int { + public function restore(Repository $repo, array $manifest, ?array &$report = null): int { $sm = $this->db->createSchemaManager(); $existing = array_flip($sm->listTableNames()); $total = 0; + $report = ['skipped' => [], 'restored' => []]; $this->db->beginTransaction(); try { foreach ($manifest['tables'] as $table => $info) { if (!isset($existing[$table])) { throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)"); } + // Unchanged since the backup: leave it alone (much faster than delete + insert). + $live = $this->checksum($table); + if ($live['rows'] === $info['rows'] && hash_equals($info['sha256'], $live['sha256'])) { + $report['skipped'][] = $table; + continue; + } $this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($table)); $columns = null; $binary = []; @@ -112,17 +169,18 @@ final class DbDumper { if (is_array($value) && isset($value['$b'])) { $value = base64_decode($value['$b'], true); // text that was not valid UTF-8 } - $row[$col] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value; + $row[] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value; } $batch[] = $row; - if (count($batch) >= 500) { - $total += $this->insertBatch($table, $batch, $binary); + if (count($batch) >= self::INSERT_ROWS) { + $total += $this->insertBatch($table, $columns, $batch, $binary); $batch = []; } } if ($batch !== []) { - $total += $this->insertBatch($table, $batch, $binary); + $total += $this->insertBatch($table, $columns, $batch, $binary); } + $report['restored'][] = $table; } $this->restoreSequences($manifest['sequences'] ?? []); $this->db->commit(); @@ -135,20 +193,22 @@ final class DbDumper { /** Same checksum as dump(): sha256 over the row lines in primary-key order. */ public function checksum(string $table): array { - $sm = $this->db->createSchemaManager(); $hash = hash_init('sha256'); $rows = 0; - foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) { - if ($kind === 'row') { - hash_update($hash, $line . "\n"); - $rows++; - } + foreach ($this->rowLines($this->db->createSchemaManager()->introspectTable($table)) as [, $line]) { + hash_update($hash, $line . "\n"); + $rows++; } return ['rows' => $rows, 'sha256' => hash_final($hash)]; } - /** @return \Generator 'header' => line, then 'row' => line ... */ - private function tableLines(\Doctrine\DBAL\Schema\Table $table): \Generator { + private function header(\Doctrine\DBAL\Schema\Table $table): string { + [$columns, $binary, $pk] = $this->describe($table); + return json_encode(['table' => $table->getName(), 'columns' => $columns, 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR); + } + + /** @return array{0: list, 1: list, 2: list} columns, binary columns, primary key */ + private function describe(\Doctrine\DBAL\Schema\Table $table): array { $columns = array_values(array_map(fn ($c) => $c->getName(), $table->getColumns())); $binary = []; foreach ($table->getColumns() as $c) { @@ -156,25 +216,34 @@ final class DbDumper { $binary[] = $c->getName(); } } - $pk = $table->getPrimaryKey()?->getColumns() ?? []; - yield 'header' => json_encode(['table' => $table->getName(), 'columns' => array_values($columns), 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR); + return [$columns, $binary, $table->getPrimaryKey()?->getColumns() ?? []]; + } + /** + * Rows in primary-key order, after the key $after (resume), as [key, JSON line]. + * + * @return \Generator|null, 1: string}> + */ + private function rowLines(\Doctrine\DBAL\Schema\Table $table, ?array $after = null): \Generator { + [$columns, $binary, $pk] = $this->describe($table); $binaryFlip = array_flip($binary); $q = fn (string $c) => $this->db->quoteIdentifier($c); $select = 'SELECT ' . implode(', ', array_map($q, $columns)) . ' FROM ' . $q($table->getName()); - if ($pk === []) { - // Nextcloud requires primary keys; small odd tables are read in one query. + // Nextcloud requires primary keys; small odd tables are read in one query (not resumable). $count = (int)$this->db->fetchOne('SELECT COUNT(*) FROM ' . $q($table->getName())); if ($count > self::MAX_ROWS_WITHOUT_PK) { throw new \RuntimeException("Table {$table->getName()} has no primary key and $count rows; cannot dump it with bounded memory"); } $batches = [$this->db->fetchAllNumeric($select)]; + $pkIdx = []; } else { - $batches = $this->keysetBatches($select, $pk, $columns); + $batches = $this->keysetBatches($select, $pk, $columns, $after); + $pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk); } foreach ($batches as $rows) { foreach ($rows as $row) { + $key = $pkIdx === [] ? null : array_map(fn ($i) => $row[$i], $pkIdx); foreach ($columns as $i => $col) { if (is_resource($row[$i])) { $row[$i] = stream_get_contents($row[$i]); @@ -189,23 +258,23 @@ final class DbDumper { $row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]); } } - yield 'row' => json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR); + yield [$key, json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR)]; } } } /** @return \Generator>> */ - private function keysetBatches(string $select, array $pk, array $columns): \Generator { + private function keysetBatches(string $select, array $pk, array $columns, ?array $after): \Generator { $q = fn (string $c) => $this->db->quoteIdentifier($c); $order = ' ORDER BY ' . implode(', ', array_map($q, $pk)); - $pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk); - $last = null; + $cols = array_map(fn ($c) => $q($c), $pk); + $last = $after; while (true) { $sql = $select; $params = []; if ($last !== null) { // Row-value comparison (a, b) > (?, ?), supported by PostgreSQL, MySQL/MariaDB and SQLite. - $sql .= ' WHERE (' . implode(', ', array_map($q, $pk)) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')'; + $sql .= ' WHERE (' . implode(', ', $cols) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')'; $params = $last; } $rows = $this->db->fetchAllNumeric($sql . $order . ' LIMIT ' . self::BATCH, $params); @@ -213,22 +282,29 @@ final class DbDumper { return; } yield $rows; - $end = end($rows); - $last = array_map(fn ($i) => $end[$i], $pkIdx); if (count($rows) < self::BATCH) { return; } + $end = end($rows); + $last = array_map(fn ($c) => $end[array_search($c, $columns, true)], $pk); } } - private function insertBatch(string $table, array $rows, array $binary): int { + /** One INSERT with several rows (portable multi-row VALUES). */ + private function insertBatch(string $table, array $columns, array $rows, array $binary): int { + $q = fn (string $c) => $this->db->quoteIdentifier($c); + $placeholders = '(' . implode(', ', array_fill(0, count($columns), '?')) . ')'; + $sql = 'INSERT INTO ' . $q($table) . ' (' . implode(', ', array_map($q, $columns)) . ') VALUES ' + . implode(', ', array_fill(0, count($rows), $placeholders)); + $params = []; + $types = []; foreach ($rows as $row) { - $types = []; - foreach ($row as $col => $_) { - $types[$col] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : \Doctrine\DBAL\ParameterType::STRING; + foreach ($columns as $i => $col) { + $params[] = $row[$i]; + $types[] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : ($row[$i] === null ? \Doctrine\DBAL\ParameterType::NULL : \Doctrine\DBAL\ParameterType::STRING); } - $this->db->insert($this->db->quoteIdentifier($table), array_combine(array_map(fn ($c) => $this->db->quoteIdentifier($c), array_keys($row)), $row), array_values($types)); } + $this->db->executeStatement($sql, $params, $types); return count($rows); } diff --git a/lib/Service/BackupService.php b/lib/Service/BackupService.php index a118fae..dcf42da 100644 --- a/lib/Service/BackupService.php +++ b/lib/Service/BackupService.php @@ -127,14 +127,24 @@ class BackupService { $state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR); if ($run->getPhase() === 'db') { - @set_time_limit(0); // the dump is one transaction; it cannot be split across requests yet - $dbStats = []; - $manifest = (new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_')))->dump($repo, $dbStats); - $repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR)); - $state['files']['meta']['db'] = 'db/' . $state['files']['snapshot']; - $state['dbStats'] = $dbStats + ['tables' => count($manifest['tables']), - 'rows' => array_sum(array_column($manifest['tables'], 'rows'))]; - $run->setPhase('files'); + $dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_')); + $dbStats = $state['dbStats'] ?? []; + // With system cron the whole dump usually fits in one step (one transaction, fully + // consistent). Under web cron it continues per step; the snapshot then says so. + $dump = $dumper->dumpStep($repo, $state['dump'] ?? $dumper->startDump(), $deadline, $dbStats); + $state['dbStats'] = $dbStats; + if ($dump['done']) { + $manifest = $dumper->manifest($dump); + $repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR)); + $state['files']['meta']['db'] = 'db/' . $state['files']['snapshot']; + $state['files']['meta']['dbConsistent'] = $manifest['consistent']; + $state['dbStats'] += ['tables' => count($manifest['tables']), 'rows' => array_sum(array_column($manifest['tables'], 'rows')), + 'steps' => $dump['steps'], 'consistent' => $manifest['consistent']]; + unset($state['dump']); + $run->setPhase('files'); + } else { + $state['dump'] = $dump; + } } else { $state['files'] = BackupRun::step($repo, $state['files'], $deadline); if ($state['files']['phase'] === 'done') { diff --git a/spikes/03-db-dump.php b/spikes/03-db-dump.php index e636790..bd9fbbc 100644 --- a/spikes/03-db-dump.php +++ b/spikes/03-db-dump.php @@ -71,6 +71,17 @@ $dumpBytes = array_sum(array_column($manifest['tables'], 'bytes')); printf("dump size (plain JSON lines): %.1f MiB\n", $dumpBytes / 1048576); $check('checksums in dump match live tables', array_map(fn ($x) => $x['sha256'], $manifest['tables']) === array_map(fn ($x) => $x['sha256'], $before)); +// The same dump in many tiny steps (as under web cron) must give identical table checksums. +$stepped = $dumper->startDump(); +$stepStats = []; +while (!$stepped['done']) { + $stepped = $dumper->dumpStep($repo, $stepped, microtime(true) + 0.05, $stepStats); +} +$sm = $dumper->manifest($stepped); +printf("stepped dump: %d steps, consistent=%s\n", $stepped['steps'], json_encode($sm['consistent'])); +$check('stepped dump: same checksums as the one-shot dump', array_map(fn ($x) => $x['sha256'], $sm['tables']) === array_map(fn ($x) => $x['sha256'], $manifest['tables'])); +$check('stepped dump is marked as not fully consistent', $stepped['steps'] === 1 || $sm['consistent'] === false); + // Second dump without changes: blobs are reused. $stats2 = ['newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0]; $dumper->dump($repo, $stats2); @@ -90,8 +101,10 @@ memory_reset_peak_usage(); $t = microtime(true); $repo2 = Repository::open(new LocalBackend($repoDir), 'spike passphrase'); $m = json_decode($repo2->getObject('db/1'), true, 512, JSON_THROW_ON_ERROR); -$rows = $dumper->restore($repo2, $m); -printf("restore: %d rows in %.1fs, peak %.1f MiB\n", $rows, microtime(true) - $t, memory_get_peak_usage() / 1048576); +$report = []; +$rows = $dumper->restore($repo2, $m, $report); +printf("restore: %d rows in %.1fs, peak %.1f MiB, %d tables restored, %d unchanged skipped\n", $rows, microtime(true) - $t, memory_get_peak_usage() / 1048576, count($report['restored']), count($report['skipped'])); +$check('restore only rewrote the damaged tables', count($report['restored']) === count($changed)); $mismatch = []; foreach ($tables as $tb) {