From dfe1fe9aad9a928bdd91bf7b4017ffe6d42b8151 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Wed, 4 Nov 2026 04:08:20 +0100 Subject: [PATCH] --salvage for backup:restore and backup:restore:full: restore what can be read from a damaged backup A file whose data cannot be read or decrypted is left out and listed, and the command exits with code 2. When the database cannot be restored (its dump is damaged, or it does not fit this installation), backup:restore:full leaves the fresh database as it is and puts the data directory and config of the backup in ngb-salvage- in the data directory. Without --salvage a restore of damaged data stops as before and points at the option. --- .github/workflows/ci.yml | 2 + docs/DISASTER-RECOVERY.md | 17 ++ lib/Command/Restore.php | 31 +++- lib/Command/RestoreFull.php | 32 +++- lib/Repository/Repository.php | 42 +++-- lib/Service/RestoreFullService.php | 173 ++++++++++++++------ lib/Service/RestoreService.php | 28 +++- tests/Unit/Repository/SalvageTest.php | 88 ++++++++++ tests/integration/full-restore-roundtrip.sh | 34 +++- 9 files changed, 369 insertions(+), 78 deletions(-) create mode 100644 tests/Unit/Repository/SalvageTest.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60254cb..4cd6278 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,3 +138,5 @@ jobs: run: tests/integration/full-restore-roundtrip.sh 35-apache mysql to-sqlite - name: Full restore of a SQLite backup onto PostgreSQL run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite to-pgsql + - name: Full restore with a damaged database dump (--salvage) + run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite salvage diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md index 86e35fc..a71ac28 100644 --- a/docs/DISASTER-RECOVERY.md +++ b/docs/DISASTER-RECOVERY.md @@ -109,6 +109,23 @@ the end of the restore, install those afterwards. If occ then says that Nextcloud or one of the apps requires an upgrade, the database is already the old server's: run `occ upgrade` first, then the same command with `--force`. +**If the backup itself is damaged** (the restore stops with "Decryption failed", "Blob +verification failed" or a file that is missing on the location), try an older restore point +first. If none is better, `--salvage` restores whatever can be read: + + occ backup:restore:full offsite --install-apps --salvage + +- Files that cannot be read are left out and listed at the end; everything else is restored. +- If the database cannot be restored (its dump is damaged, or it does not fit this + installation), the database stays as the fresh installation made it, and the data directory + and config of the backup go to a folder of their own in the data directory, + `ngb-salvage-`. Each user's files are in `data//files` there: create the users + again and copy their files back by hand (then `occ files:scan --all`). Calendars, contacts and + shares are in the database and cannot be recovered this way. + +The command then ends with exit code 2 instead of 0. The same option exists for +`occ backup:restore`, to restore single folders. + **9. After the restore.** These worked in our tests: occ upgrade diff --git a/lib/Command/Restore.php b/lib/Command/Restore.php index 6ae877b..e07a8d0 100644 --- a/lib/Command/Restore.php +++ b/lib/Command/Restore.php @@ -31,6 +31,7 @@ final class Restore extends Command { ->addArgument('path', InputArgument::REQUIRED, 'Path in the snapshot (see backup:browse)') ->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'new-folder, merge or replace (user files)', RestoreService::MODE_NEW_FOLDER) ->addOption('to-directory', 'd', InputOption::VALUE_REQUIRED, 'Write to this local directory instead (any path, raw files)') + ->addOption('salvage', null, InputOption::VALUE_NONE, 'Leave out a file that cannot be read or decrypted, list it at the end and go on, instead of stopping') ->setHelp(<<<'HELP' User files (data//files/...) are restored through Nextcloud, so versions, trash and the file cache stay correct. Modes: @@ -42,30 +43,52 @@ Examples: occ backup:restore offsite 537bf7e9... data/alice/files/Documents occ backup:restore offsite 537bf7e9... data/alice/files/Documents/plan.odt --mode=merge occ backup:restore offsite 537bf7e9... config --to-directory=/root/restored-config +With --salvage, a file whose data on the location is damaged or missing is left out and listed +at the end; everything else is restored. The command then exits with code 2. HELP); } protected function execute(InputInterface $input, OutputInterface $output): int { + $skipped = $input->getOption('salvage') ? [] : null; try { $target = $this->targets->get($input->getArgument('target')); $snapshot = $input->getArgument('snapshot'); $path = trim((string)$input->getArgument('path'), '/'); // CLI convenience; the service itself only accepts canonical paths if ($dir = $input->getOption('to-directory')) { - $n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir); + $n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir, false, null, $skipped); $output->writeln("Restored $n files to $dir."); - return 0; + return self::reportSkipped($skipped, $output); } $r = $this->restore->restoreUserFiles($target, $snapshot, $path, (string)$input->getOption('mode'), function (int $done, int $total, string $name) use ($output) { if ($output->isVerbose() || $done === $total || $done % 100 === 0) { $output->writeln(" $done/$total $name"); } - }); + }, $skipped); } catch (\Throwable $e) { $output->writeln('' . $e->getMessage() . ''); + if ($skipped === null && ($e instanceof \OCA\NgBackup\Repository\RepositoryException || $e instanceof \OCA\NgBackup\Crypto\CryptoException)) { + $output->writeln('The data of a file on the location is damaged or missing. Run the same command with --salvage to restore everything else and list what could not be restored.'); + } return 1; } $output->writeln(sprintf('Restored %d files to %s%s.', $r['restored'], $r['target'], $r['trashed'] ? ", moved {$r['trashed']} extra files to the trash" : '')); - return 0; + return self::reportSkipped($skipped, $output); + } + + /** + * The files --salvage left out; exit code 2 when there are any. + * + * @param array|null $skipped + */ + public static function reportSkipped(?array $skipped, OutputInterface $output): int { + if ($skipped === null || $skipped === []) { + return 0; + } + $output->writeln(sprintf('%d file%s could not be restored (damaged or missing on the location):', count($skipped), count($skipped) === 1 ? '' : 's')); + foreach ($skipped as $path => $reason) { + $output->writeln(" $path: $reason"); + } + return 2; } } diff --git a/lib/Command/RestoreFull.php b/lib/Command/RestoreFull.php index 6c804c6..920077e 100644 --- a/lib/Command/RestoreFull.php +++ b/lib/Command/RestoreFull.php @@ -36,6 +36,7 @@ final class RestoreFull extends Command { ->addOption('force', null, InputOption::VALUE_NONE, 'Also on an installation that already has more than one user, and without asking') ->addOption('install-apps', null, InputOption::VALUE_NONE, 'First install (from the app store) or enable the apps that were enabled at backup time and are missing here') ->addOption('skip-missing-tables', null, InputOption::VALUE_NONE, 'Older backups only: leave out tables that do not exist here and whose structure the backup does not record') + ->addOption('salvage', null, InputOption::VALUE_NONE, 'Restore what can be restored: leave out files that cannot be read, and when the database cannot be restored, put the files in a folder of their own') ->setHelp(<<<'HELP' For a FRESH Nextcloud installation only, same major version as the backup, right after occ maintenance:install, occ backup:key:import and occ backup:target:add -- before anyone else @@ -55,6 +56,12 @@ here. Other files under config/ (if any) are not restored. Afterwards: check trusted_domains and overwritehost/overwriteprotocol match this server, and occ maintenance:repair. +When the backup is damaged: --salvage restores what can be restored. Files whose data cannot be +read or decrypted are left out and listed. When the database cannot be restored (its dump is +damaged, or it does not fit this installation), the database stays as occ maintenance:install +made it, and the data directory and config of the backup go to ngb-salvage- in the +data directory instead: from there, the files can be put back by hand. The command then exits +with code 2. Example: occ backup:restore:full offsite 537bf7e9c1a4... @@ -82,29 +89,41 @@ HELP); $target = $this->targets->get($input->getArgument('target')); $snapshot = (string)$input->getArgument('snapshot'); $skipMissingTables = (bool)$input->getOption('skip-missing-tables'); + $salvage = (bool)$input->getOption('salvage'); if ($input->getOption('install-apps')) { - $check = $this->restoreFull->checkCompatibility($target, $snapshot, $skipMissingTables); + $check = $this->restoreFull->checkCompatibility($target, $snapshot, $skipMissingTables, $salvage); foreach ($check['missingApps'] as $app) { $this->installApp($app, $output); } } $output->writeln('Nextcloud is in maintenance mode while the restore runs; it is switched off when the restore completes.'); - $result = $this->restoreFull->restoreFull($target, $snapshot, $skipMissingTables); + $result = $this->restoreFull->restoreFull($target, $snapshot, $skipMissingTables, $salvage); } catch (\Throwable $e) { $output->writeln('' . $e->getMessage() . ''); if (\OCP\Server::get(\OCP\IConfig::class)->getSystemValueBool('maintenance', false)) { $output->writeln('The restore had started: the instance may be partly restored, and maintenance mode stays on so that nobody uses it like that.'); $output->writeln('Fix the cause, then switch maintenance mode off and run the same command again with --force, in one line:'); $again = 'occ backup:restore:full ' . escapeshellarg((string)$input->getArgument('target')) . ' ' . escapeshellarg((string)$input->getArgument('snapshot')) - . ($input->getOption('skip-missing-tables') ? ' --skip-missing-tables' : '') . ' --force --no-interaction'; + . ($input->getOption('skip-missing-tables') ? ' --skip-missing-tables' : '') . ($input->getOption('salvage') ? ' --salvage' : '') . ' --force --no-interaction'; $output->writeln(' occ maintenance:mode --off && ' . $again); $output->writeln('(Without --install-apps, so the instance is open only for the seconds until the restore starts again. Apps the first run could not install are listed at the end: install those afterwards.)'); } else { $output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.'); } $output->writeln('If occ then says that Nextcloud needs an upgrade (the database is already the old server\'s), run occ upgrade first.'); + if (!$input->getOption('salvage') && ($e instanceof \OCA\NgBackup\Repository\RepositoryException || $e instanceof \OCA\NgBackup\Crypto\CryptoException)) { + $output->writeln('The backup itself looks damaged here. --salvage restores everything that can be read and lists the rest.'); + } return 1; } + if ($result['dbError'] !== null) { + $output->writeln('Salvage: ' . $result['dbError'] . ''); + $output->writeln(sprintf('The database stays as occ maintenance:install made it. The data directory and config of the backup (%d data files) are in:', $result['dataFiles'])); + $output->writeln(' ' . $result['salvageDir']); + $output->writeln('Each user\'s files are in data//files there. Put back what is needed by hand, for example into the files of new users (then occ files:scan).'); + Restore::reportSkipped($result['skipped'], $output); + return 2; + } $output->writeln(sprintf( 'Database restored (%d tables, %d unchanged), config merged (%d keys), %d data files restored.', count($result['dbReport']['restored']), @@ -135,7 +154,12 @@ HELP); $output->writeln('Maintenance mode was on before the restore and stays on: switch it off with occ maintenance:mode --off when the instance is ready.'); } $output->writeln('Next: check trusted_domains and overwritehost/overwriteprotocol match this server, then occ maintenance:repair.'); - return 0; + $code = 0; + if ($result['configError'] !== null) { + $output->writeln('Salvage: config.php of the backup was not merged (' . $result['configError'] . '). Set at least \'instanceid\', \'passwordsalt\' and \'secret\' to the old server\'s values if you have them, or data encrypted with them (server-side encryption, stored passwords) cannot be read.'); + $code = 2; + } + return max($code, Restore::reportSkipped($result['skipped'], $output)); } /** Enable an app that is here but switched off, or install it from the app store. */ diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index e87a173..f2c81d9 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -221,14 +221,17 @@ final class Repository { * "$target/" (disaster recovery: restoring e.g. "data/" or * "config/" directly into the real data/config directory, not a subdirectory * named after the prefix). Ignored when $prefix is '' (nothing to strip). + * @param array|null $skipped salvage: an array to collect the files that could + * not be read or decrypted (path => reason), which are then left out instead of + * stopping the restore; null (default) stops at the first such file */ - public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null): int { + public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null): int { if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) { throw new RepositoryException('Invalid path prefix'); } $plan = $this->readAhead($snapshotId, $prefix, $target); try { - return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan); + return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan, $skipped); } finally { $plan?->close(); if ($plan !== null) { @@ -285,7 +288,8 @@ final class Repository { return $plan; } - private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan): int { + /** @param array|null $skipped see restore() */ + private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan, ?array &$skipped): int { $count = 0; $cleaned = []; foreach ($this->streamEntries($snapshotId) as $entry) { @@ -317,16 +321,30 @@ final class Repository { if ($out === false) { throw new RepositoryException('Cannot create a temporary file for ' . $entry['p']); } - foreach ($entry['b'] as $id) { - fwrite($out, $this->restoreBlob($id, $plan)); - if ($heartbeat !== null) { - $heartbeat(); + try { + foreach ($entry['b'] as $id) { + fwrite($out, $this->restoreBlob($id, $plan)); + if ($heartbeat !== null) { + $heartbeat(); + } } - } - fclose($out); - if (filesize($tmp) !== $entry['s']) { - unlink($tmp); - throw new RepositoryException('Size mismatch restoring ' . $entry['p']); + fclose($out); + clearstatcache(true, $tmp); + if (filesize($tmp) !== $entry['s']) { + throw new RepositoryException('Size mismatch restoring ' . $entry['p']); + } + } catch (RepositoryException|BackendException|\OCA\NgBackup\Crypto\CryptoException $e) { + // What the location gave back for this file is missing or damaged. Writing to the + // target (below) is not part of this: a full disk must still stop the restore. + if (is_resource($out)) { + fclose($out); + } + @unlink($tmp); + if ($skipped === null) { + throw $e; + } + $skipped[$entry['p']] = $e->getMessage(); + continue; } if (!rename($tmp, $dest)) { @unlink($tmp); diff --git a/lib/Service/RestoreFullService.php b/lib/Service/RestoreFullService.php index 2a1978e..0aa3dd0 100644 --- a/lib/Service/RestoreFullService.php +++ b/lib/Service/RestoreFullService.php @@ -58,9 +58,12 @@ final class RestoreFullService { * - tables in the backup that do not exist here: a backup made from 1.0.0-rc.2 on records * their structure and the restore creates them, an older one cannot (missingTables). * - * @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list, missingTables:list} + * With $salvage, a database dump that cannot be read does not block: dbUnreadable says why, and + * the restore then salvages the files instead (see restoreFull()). + * + * @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list, missingTables:list, dbUnreadable?:string} */ - public function checkCompatibility(Target $target, string $snapshotId, bool $skipMissingTables = false): array { + public function checkCompatibility(Target $target, string $snapshotId, bool $skipMissingTables = false, bool $salvage = false): array { $repo = $this->targets->repository($target); $meta = $repo->snapshotMeta($snapshotId); $currentMajor = explode('.', $this->config->getSystemValueString('version'))[0]; @@ -69,8 +72,19 @@ final class RestoreFullService { $missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled), self::ENVIRONMENT_APPS)); $missingTables = []; $prefixMismatch = null; + $dbUnreadable = null; + $manifest = null; if (isset($meta['db'])) { - $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); + try { + $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); + } catch (\Throwable $e) { + if (!$salvage) { + throw $e; + } + $dbUnreadable = 'the database dump cannot be read: ' . $e->getMessage(); + } + } + if (is_array($manifest)) { $prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_'); $sourcePrefix = DbDumper::dumpPrefix($manifest); if ($sourcePrefix !== null && $sourcePrefix !== $prefix) { @@ -87,11 +101,21 @@ final class RestoreFullService { 'missingApps' => $missingApps, 'missingTables' => $prefixMismatch === null ? $missingTables : [], 'prefixMismatch' => $prefixMismatch, - ]; + ] + ($dbUnreadable !== null ? ['dbUnreadable' => $dbUnreadable] : []); } - /** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list, maintenanceLeftOn:bool} */ - public function restoreFull(Target $target, string $snapshotId, bool $skipMissingTables = false): array { + /** + * With $salvage, restore whatever can be restored instead of stopping: + * - a file that cannot be read or decrypted is left out and listed (skipped); + * - when the database cannot be restored here (its dump is unreadable or damaged, or it does + * not fit this installation), the fresh database stays as it is, and the data directory and + * config of the backup go to a folder of their own (salvageDir) instead: the files are what + * matters most, and putting Nextcloud back together from there is work by hand; + * - a config.php that cannot be read is not merged (configError). + * + * @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list, maintenanceLeftOn:bool, skipped:array, dbError:?string, salvageDir:?string, configError:?string} + */ + public function restoreFull(Target $target, string $snapshotId, bool $skipMissingTables = false, bool $salvage = false): array { // Maintenance mode for the whole restore: background jobs (cron.php stops in maintenance // mode), page visits (AJAX cron) and users must not write into tables or the data // directory while they are being replaced; a row written in between would collide with a @@ -114,17 +138,21 @@ final class RestoreFullService { } }; try { - $check = $this->checkCompatibility($target, $snapshotId, $skipMissingTables); + $check = $this->checkCompatibility($target, $snapshotId, $skipMissingTables, $salvage); } catch (\Throwable $e) { $reopen(); throw $e; } if (!$check['ok']) { - $reopen(); - throw new \RuntimeException($this->describeBlockers($check)); + if (!$salvage) { + $reopen(); + throw new \RuntimeException($this->describeBlockers($check)); + } + // The database cannot go into this installation; the files can. + $check['dbUnreadable'] = 'the database cannot be restored here: ' . $this->describeBlockers($check); } $appConfig->setValueInt(\OCA\NgBackup\AppInfo\Application::APP_ID, self::INTERRUPTED_KEY, time()); - $result = $this->restoreInMaintenance($target, $snapshotId, $skipMissingTables, $check); + $result = $this->restoreInMaintenance($target, $snapshotId, $skipMissingTables, $check, $salvage); $appConfig->deleteKey(\OCA\NgBackup\AppInfo\Application::APP_ID, self::INTERRUPTED_KEY); if (!$wasInMaintenance) { $this->config->setSystemValue('maintenance', false); @@ -132,55 +160,56 @@ final class RestoreFullService { return $result + ['maintenanceLeftOn' => $wasInMaintenance]; } - /** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list} */ - private function restoreInMaintenance(Target $target, string $snapshotId, bool $skipMissingTables, array $check): array { - return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $skipMissingTables, $check) { + /** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list, skipped:array, dbError:?string, salvageDir:?string, configError:?string} */ + private function restoreInMaintenance(Target $target, string $snapshotId, bool $skipMissingTables, array $check, bool $salvage): array { + return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $skipMissingTables, $check, $salvage) { $beat = LeaseService::throttled($refresh); $repo = $this->targets->repository($target); $meta = $repo->snapshotMeta($snapshotId); $dbReport = null; $manifest = []; - if (isset($meta['db'])) { - $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); - $prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_'); - // ng_backup's own tables (and its own rows in the shared appconfig table) must - // survive this restore untouched: they describe the backup/key/target that is - // actively performing it right now, on THIS installation, not the one that was - // backed up. Overwriting them mid-restore would pull the rug from under it. - foreach (array_keys($manifest['tables']) as $table) { - /** @var string $table */ - if (str_starts_with($table, $prefix . 'ngb_')) { - unset($manifest['tables'][$table]); - } - } - // Sequences are keyed separately from tables; excluding a table does not exclude - // its sequence, which would otherwise rewind a retained ngb_* table's sequence - // below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()). - foreach (array_keys($manifest['sequences'] ?? []) as $seq) { - /** @var string $seq */ - if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) { - unset($manifest['sequences'][$seq]); - } - } - $ownAppConfig = $this->preserveOwnAppConfig(); - $dumper = new DbDumper($this->db->getInner(), $prefix); + $dbError = $check['dbUnreadable'] ?? null; + $skipped = $salvage ? [] : null; + $dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/'); + if ($dbError === null) { try { - $dumper->restore($repo, $manifest, $dbReport, $beat, $skipMissingTables); - } finally { - $this->restoreOwnAppConfig($ownAppConfig); + $this->restoreDatabase($repo, $meta, $manifest, $dbReport, $beat, $skipMissingTables); + } catch (\Throwable $e) { + if (!$salvage) { + throw $e; + } + // DbDumper::restore() rolled its transaction back: the database is the fresh one. + $dbError = 'the database could not be restored: ' . $e->getMessage(); + $dbReport = null; } } + if ($dbError !== null) { + $salvageDir = $dataDir . '/ngb-salvage-' . substr($snapshotId, 0, 12); + $dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $salvageDir . '/data', stripPrefix: true, heartbeat: $beat, skipped: $skipped); + $this->restore->restoreToDirectory($target, $snapshotId, 'config', $salvageDir . '/config', stripPrefix: true, heartbeat: $beat, skipped: $skipped); + return ['dbReport' => ['skipped' => [], 'restored' => [], 'created' => [], 'missing' => []], 'dataFiles' => $dataFiles, 'configKeys' => 0, + 'dbConsistent' => true, 'missingApps' => $check['missingApps'], 'skipped' => $skipped ?? [], 'dbError' => $dbError, + 'salvageDir' => $salvageDir, 'configError' => null]; + } // Read (but do not yet apply) the backed-up config.php before touching the data // directory: once applied, 'secret' may differ from what ng_backup's own Target // options were encrypted with (see restoreFull()'s docblock), and every further // repository access through $this->targets would fail to decrypt them. - /** @var array $backupConfig */ - $backupConfig = $this->fetchBackupConfig($target, $snapshotId); + $configError = null; + try { + /** @var array $backupConfig */ + $backupConfig = $this->fetchBackupConfig($target, $snapshotId, $salvage); + } catch (\Throwable $e) { + if (!$salvage) { + throw $e; + } + $configError = $e->getMessage(); + $backupConfig = []; + } - $dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/'); - $dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat); + $dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat, skipped: $skipped); // Last step: nothing after this may need $this->targets->repository($target) again. if ($backupConfig !== []) { @@ -190,15 +219,63 @@ final class RestoreFullService { } return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => [], 'created' => [], 'missing' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig), - 'dbConsistent' => (bool)($manifest['consistent'] ?? true), 'missingApps' => $check['missingApps']]; + 'dbConsistent' => (bool)($manifest['consistent'] ?? true), 'missingApps' => $check['missingApps'], + 'skipped' => $skipped ?? [], 'dbError' => null, 'salvageDir' => null, 'configError' => $configError]; }); } - /** @return array config.php's $CONFIG from the backup, minus the connection keys */ - private function fetchBackupConfig(Target $target, string $snapshotId): array { + /** + * Replace the database with the snapshot's dump (one transaction, see DbDumper::restore()). + * + * @param array $meta the snapshot's meta record + * @param array $manifest set to the dump's manifest + */ + private function restoreDatabase(\OCA\NgBackup\Repository\Repository $repo, array $meta, array &$manifest, ?array &$dbReport, callable $beat, bool $skipMissingTables): void { + if (!isset($meta['db'])) { + return; + } + $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); + $prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_'); + // ng_backup's own tables (and its own rows in the shared appconfig table) must + // survive this restore untouched: they describe the backup/key/target that is + // actively performing it right now, on THIS installation, not the one that was + // backed up. Overwriting them mid-restore would pull the rug from under it. + foreach (array_keys($manifest['tables']) as $table) { + /** @var string $table */ + if (str_starts_with($table, $prefix . 'ngb_')) { + unset($manifest['tables'][$table]); + } + } + // Sequences are keyed separately from tables; excluding a table does not exclude + // its sequence, which would otherwise rewind a retained ngb_* table's sequence + // below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()). + foreach (array_keys($manifest['sequences'] ?? []) as $seq) { + /** @var string $seq */ + if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) { + unset($manifest['sequences'][$seq]); + } + } + $ownAppConfig = $this->preserveOwnAppConfig(); + $dumper = new DbDumper($this->db->getInner(), $prefix); + try { + $dumper->restore($repo, $manifest, $dbReport, $beat, $skipMissingTables); + } finally { + $this->restoreOwnAppConfig($ownAppConfig); + } + } + + /** + * @return array config.php's $CONFIG from the backup, minus the connection keys + * @throws \RuntimeException with $salvage, when config.php itself cannot be read + */ + private function fetchBackupConfig(Target $target, string $snapshotId, bool $salvage = false): array { $tmp = sys_get_temp_dir() . '/ngb-restore-config-' . bin2hex(random_bytes(6)); try { - $this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true); + $skipped = $salvage ? [] : null; + $this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true, skipped: $skipped); + if (isset($skipped['config/config.php'])) { + throw new \RuntimeException('config.php cannot be read: ' . $skipped['config/config.php']); + } $path = $tmp . '/config.php'; if (!is_file($path)) { return []; // nothing to merge; the live config.php stays as occ maintenance:install made it diff --git a/lib/Service/RestoreService.php b/lib/Service/RestoreService.php index fc3e276..0bc238c 100644 --- a/lib/Service/RestoreService.php +++ b/lib/Service/RestoreService.php @@ -84,8 +84,8 @@ final class RestoreService { * * @return array{restored:int, trashed:int, target:string} */ - public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array { - return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress) { + public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null, ?array &$skipped = null): array { + return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress, &$skipped) { $last = time(); // Keep the lease alive while files are being written. $beat = function (int $done, int $total, string $name) use ($refresh, $progress, &$last): void { @@ -97,11 +97,12 @@ final class RestoreService { $progress($done, $total, $name); } }; - return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat); + return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat, $skipped); }); } - private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress): array { + /** @param array|null $skipped see Repository::restore() */ + private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress, ?array &$skipped): array { if (!Repository::isSafePath($logicalPath) || !preg_match('#^data/([^/]+)/files(?:/(.*))?$#', $logicalPath, $m)) { throw new \InvalidArgumentException('Expected a path like data//files/'); } @@ -139,7 +140,16 @@ final class RestoreService { $this->cleaned = []; $restored = 0; foreach ($names as $relPath => $entry) { - $this->writeFile($repo, $dest, (string)$relPath, $entry); + try { + $this->writeFile($repo, $dest, (string)$relPath, $entry); + } catch (\OCA\NgBackup\Repository\RepositoryException|\OCA\NgBackup\Backend\BackendException|\OCA\NgBackup\Crypto\CryptoException $e) { + // Damaged or missing on the location (writeFile removed its temporary) + if ($skipped === null) { + throw $e; + } + $skipped[$entry['p']] = $e->getMessage(); + continue; + } $restored++; if ($progress !== null) { $progress($restored, count($names), (string)$relPath); @@ -160,14 +170,14 @@ final class RestoreService { * Repository::restore()); used to restore "data/" or "config/" straight into the * real data/config directory for occ backup:restore:full. */ - public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null): int { + public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null): int { if ($prefix !== '' && !Repository::isSafePath($prefix)) { throw new \InvalidArgumentException('Invalid path: ' . $prefix); } if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) { throw new \RuntimeException("Cannot create $directory"); } - return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat): int { + return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat, &$skipped): int { $own = LeaseService::throttled($refresh); $beat = static function () use ($own, $heartbeat): void { $own(); @@ -175,7 +185,7 @@ final class RestoreService { $heartbeat(); } }; - return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat); + return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat, $skipped); }, 3600); } @@ -203,7 +213,7 @@ final class RestoreService { throw new \RuntimeException('Write failed for ' . $relPath); } if ($written !== $entry['s']) { - throw new \RuntimeException("Size mismatch restoring $relPath"); + throw new \OCA\NgBackup\Repository\RepositoryException("Size mismatch restoring $relPath"); } if ($folder->nodeExists($name)) { $existing = $folder->get($name); diff --git a/tests/Unit/Repository/SalvageTest.php b/tests/Unit/Repository/SalvageTest.php new file mode 100644 index 0000000..f7dc458 --- /dev/null +++ b/tests/Unit/Repository/SalvageTest.php @@ -0,0 +1,88 @@ +root = $this->tempDir(); + $this->keys = KeyRing::generate(); + $this->src = $this->tempDir(); + mkdir("$this->src/docs"); + // Incompressible, 2 MiB each: every file is a blob of its own in one pack + foreach (['a', 'b', 'c'] as $name) { + file_put_contents("$this->src/docs/$name.bin", random_bytes(2 * 1048576)); + } + file_put_contents("$this->src/note.txt", "small\n"); + $repo = Repository::initWithKey(new LocalBackend($this->root), $this->keys, $this->keys->wrap('pw')); + $this->snapshot = $repo->backupDirectory($this->src)['snapshot']; + } + + /** Flip bytes in the middle of the (only) pack: inside one of the 2 MiB files. */ + private function damagePack(): void { + $packs = glob($this->root . '/packs/*/*'); + $this->assertCount(1, $packs); + $fh = fopen($packs[0], 'r+b'); + $middle = intdiv(filesize($packs[0]), 2); + fseek($fh, $middle); + $bytes = fread($fh, 16); + fseek($fh, $middle); + fwrite($fh, ~$bytes); + fclose($fh); + } + + public function testSalvageLeavesOutTheDamagedFileAndRestoresTheRest(): void { + $this->damagePack(); + $target = $this->tempDir(); + $skipped = []; + + $restored = Repository::openWithKey(new LocalBackend($this->root), $this->keys) + ->restore($this->snapshot, $target, '', false, null, $skipped); + + $this->assertSame(3, $restored); + $this->assertCount(1, $skipped); + $damaged = array_key_first($skipped); + $this->assertMatchesRegularExpression('#^docs/[abc]\.bin$#', $damaged); + $this->assertFileDoesNotExist("$target/$damaged"); + foreach (['docs/a.bin', 'docs/b.bin', 'docs/c.bin', 'note.txt'] as $file) { + if ($file !== $damaged) { + $this->assertFileEquals("$this->src/$file", "$target/$file"); + } + } + $this->assertSame([], glob("$target/docs/*.ngb-restore-*"), 'no temporary left of the damaged file'); + } + + public function testWithoutSalvageTheRestoreStopsAtTheDamagedFile(): void { + $this->damagePack(); + + $this->expectException(\Exception::class); + Repository::openWithKey(new LocalBackend($this->root), $this->keys)->restore($this->snapshot, $this->tempDir()); + } + + public function testSalvageOfAnUndamagedSnapshotSkipsNothing(): void { + $target = $this->tempDir(); + $skipped = []; + + $this->assertSame(4, Repository::openWithKey(new LocalBackend($this->root), $this->keys) + ->restore($this->snapshot, $target, '', false, null, $skipped)); + $this->assertSame([], $skipped); + } +} diff --git a/tests/integration/full-restore-roundtrip.sh b/tests/integration/full-restore-roundtrip.sh index 3cb3c81..bcefbf3 100755 --- a/tests/integration/full-restore-roundtrip.sh +++ b/tests/integration/full-restore-roundtrip.sh @@ -2,13 +2,15 @@ # Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a # SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup # location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7. -# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp|to-pgsql|to-mysql|to-sqlite] +# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp|salvage|to-pgsql|to-mysql|to-sqlite] # e.g. full-restore-roundtrip.sh 35-apache pgsql # With "sftp" the location is an SFTP server instead of a shared directory, and the origin has a # file spread over several packs: the restore reads them ahead on connections of its own. # With "to-" the restore instance runs that database whatever the origin runs (a MySQL or # SQLite backup records no ID counters; after the restore new rows must not collide with restored # ones, whatever the database on either side). +# With "salvage" the snapshot's database dump is damaged on the location: the restore must stop +# without changing anything, and with --salvage put the files in a folder of their own. # With "legacy" the origin runs NG Backup 1.0.0-rc.1 (0.9.7), whose backups do not record the # structure of their tables: the restore must then leave the tables of the disabled app out # with --skip-missing-tables, as for a real backup made before 1.0.0-rc.2. @@ -135,6 +137,12 @@ RUN_OUT=$(run_on ngb-frt-origin backup:run offsite -l disaster-test) SNAP=$(printf '%s\n' "$RUN_OUT" | head -1 | awk -F'snapshot ' '{print $2}' | tr -d ',') [ -n "$SNAP" ] || fail "no snapshot id from backup:run" ok "test data backed up, snapshot $SNAP" +if [ "$MODE" = salvage ]; then + # The database dump of this snapshot damaged on the location: a byte flipped in its manifest + docker exec -u www-data ngb-frt-origin php -r '$p = "/mnt/shared/repo/db/" . $argv[1]; $d = file_get_contents($p); $i = intdiv(strlen($d), 2); $d[$i] = ~$d[$i]; file_put_contents($p, $d);' "$SNAP" \ + || fail "damage the database dump" + ok "database dump of the snapshot damaged on the location" +fi docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass @@ -164,6 +172,30 @@ printf '%s\n' "$LIST_OUT" | grep -q "$SNAP" || { } ok "recovery kit imported, existing repository adopted, snapshot visible" +if [ "$MODE" = salvage ]; then + set +e + OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction 2>&1); CODE=$? + set -e + [ "$CODE" = 1 ] || { printf '%s\n' "$OUT" | tail -20; fail "a restore with a damaged database dump did not stop (exit $CODE)"; } + printf '%s\n' "$OUT" | grep -- "--salvage" >/dev/null || { printf '%s\n' "$OUT" | tail -20; fail "the stopped restore does not point at --salvage"; } + run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode stayed on after a restore that changed nothing" + ok "damaged database dump: the restore stops before changing anything and points at --salvage" + set +e + OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction --salvage 2>&1); CODE=$? + set -e + [ "$CODE" = 2 ] || { printf '%s\n' "$OUT" | tail -30; fail "restore:full --salvage exited with $CODE, not 2"; } + SALVAGE=/var/www/html/data/ngb-salvage-${SNAP:0:12} + printf '%s\n' "$OUT" | grep "$SALVAGE" >/dev/null || { printf '%s\n' "$OUT" | tail -30; fail "the output does not name the salvage folder"; } + [ "$(docker exec ngb-frt-restore cat "$SALVAGE/data/frtest/files/Docs/note.txt")" = "hello disaster recovery" ] || fail "the user's file is not in the salvage folder" + docker exec ngb-frt-restore test -f "$SALVAGE/config/config.php" || fail "config.php is not in the salvage folder" + run_on ngb-frt-restore user:list | grep frtest >/dev/null && fail "the fresh database was changed (the backed-up user is there)" + run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the salvage" + run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after the salvage" + ok "with --salvage: the fresh database untouched, data and config of the backup in $SALVAGE (exit 2)" + echo "FULL RESTORE ROUNDTRIP PASS ($DB, salvage)" + exit 0 +fi + RESTORE_OPTS=(--no-interaction --install-apps) if [ "$LEGACY" = legacy ] && [ "$DB" != sqlite ]; then # Without --skip-missing-tables an older backup must be refused, naming the app's tables