diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60254cb..4cd6278 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,3 +138,5 @@ jobs: run: tests/integration/full-restore-roundtrip.sh 35-apache mysql to-sqlite - name: Full restore of a SQLite backup onto PostgreSQL run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite to-pgsql + - name: Full restore with a damaged database dump (--salvage) + run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite salvage diff --git a/docs/DISASTER-RECOVERY.md b/docs/DISASTER-RECOVERY.md index 86e35fc..a71ac28 100644 --- a/docs/DISASTER-RECOVERY.md +++ b/docs/DISASTER-RECOVERY.md @@ -109,6 +109,23 @@ the end of the restore, install those afterwards. If occ then says that Nextcloud or one of the apps requires an upgrade, the database is already the old server's: run `occ upgrade` first, then the same command with `--force`. +**If the backup itself is damaged** (the restore stops with "Decryption failed", "Blob +verification failed" or a file that is missing on the location), try an older restore point +first. If none is better, `--salvage` restores whatever can be read: + + occ backup:restore:full offsite --install-apps --salvage + +- Files that cannot be read are left out and listed at the end; everything else is restored. +- If the database cannot be restored (its dump is damaged, or it does not fit this + installation), the database stays as the fresh installation made it, and the data directory + and config of the backup go to a folder of their own in the data directory, + `ngb-salvage-`. Each user's files are in `data//files` there: create the users + again and copy their files back by hand (then `occ files:scan --all`). Calendars, contacts and + shares are in the database and cannot be recovered this way. + +The command then ends with exit code 2 instead of 0. The same option exists for +`occ backup:restore`, to restore single folders. + **9. After the restore.** These worked in our tests: occ upgrade diff --git a/lib/Command/Restore.php b/lib/Command/Restore.php index 6ae877b..e07a8d0 100644 --- a/lib/Command/Restore.php +++ b/lib/Command/Restore.php @@ -31,6 +31,7 @@ final class Restore extends Command { ->addArgument('path', InputArgument::REQUIRED, 'Path in the snapshot (see backup:browse)') ->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'new-folder, merge or replace (user files)', RestoreService::MODE_NEW_FOLDER) ->addOption('to-directory', 'd', InputOption::VALUE_REQUIRED, 'Write to this local directory instead (any path, raw files)') + ->addOption('salvage', null, InputOption::VALUE_NONE, 'Leave out a file that cannot be read or decrypted, list it at the end and go on, instead of stopping') ->setHelp(<<<'HELP' User files (data//files/...) are restored through Nextcloud, so versions, trash and the file cache stay correct. Modes: @@ -42,30 +43,52 @@ Examples: occ backup:restore offsite 537bf7e9... data/alice/files/Documents occ backup:restore offsite 537bf7e9... data/alice/files/Documents/plan.odt --mode=merge occ backup:restore offsite 537bf7e9... config --to-directory=/root/restored-config +With --salvage, a file whose data on the location is damaged or missing is left out and listed +at the end; everything else is restored. The command then exits with code 2. HELP); } protected function execute(InputInterface $input, OutputInterface $output): int { + $skipped = $input->getOption('salvage') ? [] : null; try { $target = $this->targets->get($input->getArgument('target')); $snapshot = $input->getArgument('snapshot'); $path = trim((string)$input->getArgument('path'), '/'); // CLI convenience; the service itself only accepts canonical paths if ($dir = $input->getOption('to-directory')) { - $n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir); + $n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir, false, null, $skipped); $output->writeln("Restored $n files to $dir."); - return 0; + return self::reportSkipped($skipped, $output); } $r = $this->restore->restoreUserFiles($target, $snapshot, $path, (string)$input->getOption('mode'), function (int $done, int $total, string $name) use ($output) { if ($output->isVerbose() || $done === $total || $done % 100 === 0) { $output->writeln(" $done/$total $name"); } - }); + }, $skipped); } catch (\Throwable $e) { $output->writeln('' . $e->getMessage() . ''); + if ($skipped === null && ($e instanceof \OCA\NgBackup\Repository\RepositoryException || $e instanceof \OCA\NgBackup\Crypto\CryptoException)) { + $output->writeln('The data of a file on the location is damaged or missing. Run the same command with --salvage to restore everything else and list what could not be restored.'); + } return 1; } $output->writeln(sprintf('Restored %d files to %s%s.', $r['restored'], $r['target'], $r['trashed'] ? ", moved {$r['trashed']} extra files to the trash" : '')); - return 0; + return self::reportSkipped($skipped, $output); + } + + /** + * The files --salvage left out; exit code 2 when there are any. + * + * @param array|null $skipped + */ + public static function reportSkipped(?array $skipped, OutputInterface $output): int { + if ($skipped === null || $skipped === []) { + return 0; + } + $output->writeln(sprintf('%d file%s could not be restored (damaged or missing on the location):', count($skipped), count($skipped) === 1 ? '' : 's')); + foreach ($skipped as $path => $reason) { + $output->writeln(" $path: $reason"); + } + return 2; } } diff --git a/lib/Command/RestoreFull.php b/lib/Command/RestoreFull.php index 6c804c6..920077e 100644 --- a/lib/Command/RestoreFull.php +++ b/lib/Command/RestoreFull.php @@ -36,6 +36,7 @@ final class RestoreFull extends Command { ->addOption('force', null, InputOption::VALUE_NONE, 'Also on an installation that already has more than one user, and without asking') ->addOption('install-apps', null, InputOption::VALUE_NONE, 'First install (from the app store) or enable the apps that were enabled at backup time and are missing here') ->addOption('skip-missing-tables', null, InputOption::VALUE_NONE, 'Older backups only: leave out tables that do not exist here and whose structure the backup does not record') + ->addOption('salvage', null, InputOption::VALUE_NONE, 'Restore what can be restored: leave out files that cannot be read, and when the database cannot be restored, put the files in a folder of their own') ->setHelp(<<<'HELP' For a FRESH Nextcloud installation only, same major version as the backup, right after occ maintenance:install, occ backup:key:import and occ backup:target:add -- before anyone else @@ -55,6 +56,12 @@ here. Other files under config/ (if any) are not restored. Afterwards: check trusted_domains and overwritehost/overwriteprotocol match this server, and occ maintenance:repair. +When the backup is damaged: --salvage restores what can be restored. Files whose data cannot be +read or decrypted are left out and listed. When the database cannot be restored (its dump is +damaged, or it does not fit this installation), the database stays as occ maintenance:install +made it, and the data directory and config of the backup go to ngb-salvage- in the +data directory instead: from there, the files can be put back by hand. The command then exits +with code 2. Example: occ backup:restore:full offsite 537bf7e9c1a4... @@ -82,29 +89,41 @@ HELP); $target = $this->targets->get($input->getArgument('target')); $snapshot = (string)$input->getArgument('snapshot'); $skipMissingTables = (bool)$input->getOption('skip-missing-tables'); + $salvage = (bool)$input->getOption('salvage'); if ($input->getOption('install-apps')) { - $check = $this->restoreFull->checkCompatibility($target, $snapshot, $skipMissingTables); + $check = $this->restoreFull->checkCompatibility($target, $snapshot, $skipMissingTables, $salvage); foreach ($check['missingApps'] as $app) { $this->installApp($app, $output); } } $output->writeln('Nextcloud is in maintenance mode while the restore runs; it is switched off when the restore completes.'); - $result = $this->restoreFull->restoreFull($target, $snapshot, $skipMissingTables); + $result = $this->restoreFull->restoreFull($target, $snapshot, $skipMissingTables, $salvage); } catch (\Throwable $e) { $output->writeln('' . $e->getMessage() . ''); if (\OCP\Server::get(\OCP\IConfig::class)->getSystemValueBool('maintenance', false)) { $output->writeln('The restore had started: the instance may be partly restored, and maintenance mode stays on so that nobody uses it like that.'); $output->writeln('Fix the cause, then switch maintenance mode off and run the same command again with --force, in one line:'); $again = 'occ backup:restore:full ' . escapeshellarg((string)$input->getArgument('target')) . ' ' . escapeshellarg((string)$input->getArgument('snapshot')) - . ($input->getOption('skip-missing-tables') ? ' --skip-missing-tables' : '') . ' --force --no-interaction'; + . ($input->getOption('skip-missing-tables') ? ' --skip-missing-tables' : '') . ($input->getOption('salvage') ? ' --salvage' : '') . ' --force --no-interaction'; $output->writeln(' occ maintenance:mode --off && ' . $again); $output->writeln('(Without --install-apps, so the instance is open only for the seconds until the restore starts again. Apps the first run could not install are listed at the end: install those afterwards.)'); } else { $output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.'); } $output->writeln('If occ then says that Nextcloud needs an upgrade (the database is already the old server\'s), run occ upgrade first.'); + if (!$input->getOption('salvage') && ($e instanceof \OCA\NgBackup\Repository\RepositoryException || $e instanceof \OCA\NgBackup\Crypto\CryptoException)) { + $output->writeln('The backup itself looks damaged here. --salvage restores everything that can be read and lists the rest.'); + } return 1; } + if ($result['dbError'] !== null) { + $output->writeln('Salvage: ' . $result['dbError'] . ''); + $output->writeln(sprintf('The database stays as occ maintenance:install made it. The data directory and config of the backup (%d data files) are in:', $result['dataFiles'])); + $output->writeln(' ' . $result['salvageDir']); + $output->writeln('Each user\'s files are in data//files there. Put back what is needed by hand, for example into the files of new users (then occ files:scan).'); + Restore::reportSkipped($result['skipped'], $output); + return 2; + } $output->writeln(sprintf( 'Database restored (%d tables, %d unchanged), config merged (%d keys), %d data files restored.', count($result['dbReport']['restored']), @@ -135,7 +154,12 @@ HELP); $output->writeln('Maintenance mode was on before the restore and stays on: switch it off with occ maintenance:mode --off when the instance is ready.'); } $output->writeln('Next: check trusted_domains and overwritehost/overwriteprotocol match this server, then occ maintenance:repair.'); - return 0; + $code = 0; + if ($result['configError'] !== null) { + $output->writeln('Salvage: config.php of the backup was not merged (' . $result['configError'] . '). Set at least \'instanceid\', \'passwordsalt\' and \'secret\' to the old server\'s values if you have them, or data encrypted with them (server-side encryption, stored passwords) cannot be read.'); + $code = 2; + } + return max($code, Restore::reportSkipped($result['skipped'], $output)); } /** Enable an app that is here but switched off, or install it from the app store. */ diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index e87a173..f2c81d9 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -221,14 +221,17 @@ final class Repository { * "$target/" (disaster recovery: restoring e.g. "data/" or * "config/" directly into the real data/config directory, not a subdirectory * named after the prefix). Ignored when $prefix is '' (nothing to strip). + * @param array|null $skipped salvage: an array to collect the files that could + * not be read or decrypted (path => reason), which are then left out instead of + * stopping the restore; null (default) stops at the first such file */ - public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null): int { + public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null): int { if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) { throw new RepositoryException('Invalid path prefix'); } $plan = $this->readAhead($snapshotId, $prefix, $target); try { - return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan); + return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan, $skipped); } finally { $plan?->close(); if ($plan !== null) { @@ -285,7 +288,8 @@ final class Repository { return $plan; } - private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan): int { + /** @param array|null $skipped see restore() */ + private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan, ?array &$skipped): int { $count = 0; $cleaned = []; foreach ($this->streamEntries($snapshotId) as $entry) { @@ -317,16 +321,30 @@ final class Repository { if ($out === false) { throw new RepositoryException('Cannot create a temporary file for ' . $entry['p']); } - foreach ($entry['b'] as $id) { - fwrite($out, $this->restoreBlob($id, $plan)); - if ($heartbeat !== null) { - $heartbeat(); + try { + foreach ($entry['b'] as $id) { + fwrite($out, $this->restoreBlob($id, $plan)); + if ($heartbeat !== null) { + $heartbeat(); + } } - } - fclose($out); - if (filesize($tmp) !== $entry['s']) { - unlink($tmp); - throw new RepositoryException('Size mismatch restoring ' . $entry['p']); + fclose($out); + clearstatcache(true, $tmp); + if (filesize($tmp) !== $entry['s']) { + throw new RepositoryException('Size mismatch restoring ' . $entry['p']); + } + } catch (RepositoryException|BackendException|\OCA\NgBackup\Crypto\CryptoException $e) { + // What the location gave back for this file is missing or damaged. Writing to the + // target (below) is not part of this: a full disk must still stop the restore. + if (is_resource($out)) { + fclose($out); + } + @unlink($tmp); + if ($skipped === null) { + throw $e; + } + $skipped[$entry['p']] = $e->getMessage(); + continue; } if (!rename($tmp, $dest)) { @unlink($tmp); diff --git a/lib/Service/RestoreFullService.php b/lib/Service/RestoreFullService.php index 2a1978e..0aa3dd0 100644 --- a/lib/Service/RestoreFullService.php +++ b/lib/Service/RestoreFullService.php @@ -58,9 +58,12 @@ final class RestoreFullService { * - tables in the backup that do not exist here: a backup made from 1.0.0-rc.2 on records * their structure and the restore creates them, an older one cannot (missingTables). * - * @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list, missingTables:list} + * With $salvage, a database dump that cannot be read does not block: dbUnreadable says why, and + * the restore then salvages the files instead (see restoreFull()). + * + * @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list, missingTables:list, dbUnreadable?:string} */ - public function checkCompatibility(Target $target, string $snapshotId, bool $skipMissingTables = false): array { + public function checkCompatibility(Target $target, string $snapshotId, bool $skipMissingTables = false, bool $salvage = false): array { $repo = $this->targets->repository($target); $meta = $repo->snapshotMeta($snapshotId); $currentMajor = explode('.', $this->config->getSystemValueString('version'))[0]; @@ -69,8 +72,19 @@ final class RestoreFullService { $missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled), self::ENVIRONMENT_APPS)); $missingTables = []; $prefixMismatch = null; + $dbUnreadable = null; + $manifest = null; if (isset($meta['db'])) { - $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); + try { + $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); + } catch (\Throwable $e) { + if (!$salvage) { + throw $e; + } + $dbUnreadable = 'the database dump cannot be read: ' . $e->getMessage(); + } + } + if (is_array($manifest)) { $prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_'); $sourcePrefix = DbDumper::dumpPrefix($manifest); if ($sourcePrefix !== null && $sourcePrefix !== $prefix) { @@ -87,11 +101,21 @@ final class RestoreFullService { 'missingApps' => $missingApps, 'missingTables' => $prefixMismatch === null ? $missingTables : [], 'prefixMismatch' => $prefixMismatch, - ]; + ] + ($dbUnreadable !== null ? ['dbUnreadable' => $dbUnreadable] : []); } - /** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list, maintenanceLeftOn:bool} */ - public function restoreFull(Target $target, string $snapshotId, bool $skipMissingTables = false): array { + /** + * With $salvage, restore whatever can be restored instead of stopping: + * - a file that cannot be read or decrypted is left out and listed (skipped); + * - when the database cannot be restored here (its dump is unreadable or damaged, or it does + * not fit this installation), the fresh database stays as it is, and the data directory and + * config of the backup go to a folder of their own (salvageDir) instead: the files are what + * matters most, and putting Nextcloud back together from there is work by hand; + * - a config.php that cannot be read is not merged (configError). + * + * @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list, maintenanceLeftOn:bool, skipped:array, dbError:?string, salvageDir:?string, configError:?string} + */ + public function restoreFull(Target $target, string $snapshotId, bool $skipMissingTables = false, bool $salvage = false): array { // Maintenance mode for the whole restore: background jobs (cron.php stops in maintenance // mode), page visits (AJAX cron) and users must not write into tables or the data // directory while they are being replaced; a row written in between would collide with a @@ -114,17 +138,21 @@ final class RestoreFullService { } }; try { - $check = $this->checkCompatibility($target, $snapshotId, $skipMissingTables); + $check = $this->checkCompatibility($target, $snapshotId, $skipMissingTables, $salvage); } catch (\Throwable $e) { $reopen(); throw $e; } if (!$check['ok']) { - $reopen(); - throw new \RuntimeException($this->describeBlockers($check)); + if (!$salvage) { + $reopen(); + throw new \RuntimeException($this->describeBlockers($check)); + } + // The database cannot go into this installation; the files can. + $check['dbUnreadable'] = 'the database cannot be restored here: ' . $this->describeBlockers($check); } $appConfig->setValueInt(\OCA\NgBackup\AppInfo\Application::APP_ID, self::INTERRUPTED_KEY, time()); - $result = $this->restoreInMaintenance($target, $snapshotId, $skipMissingTables, $check); + $result = $this->restoreInMaintenance($target, $snapshotId, $skipMissingTables, $check, $salvage); $appConfig->deleteKey(\OCA\NgBackup\AppInfo\Application::APP_ID, self::INTERRUPTED_KEY); if (!$wasInMaintenance) { $this->config->setSystemValue('maintenance', false); @@ -132,55 +160,56 @@ final class RestoreFullService { return $result + ['maintenanceLeftOn' => $wasInMaintenance]; } - /** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list} */ - private function restoreInMaintenance(Target $target, string $snapshotId, bool $skipMissingTables, array $check): array { - return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $skipMissingTables, $check) { + /** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list, skipped:array, dbError:?string, salvageDir:?string, configError:?string} */ + private function restoreInMaintenance(Target $target, string $snapshotId, bool $skipMissingTables, array $check, bool $salvage): array { + return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $skipMissingTables, $check, $salvage) { $beat = LeaseService::throttled($refresh); $repo = $this->targets->repository($target); $meta = $repo->snapshotMeta($snapshotId); $dbReport = null; $manifest = []; - if (isset($meta['db'])) { - $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); - $prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_'); - // ng_backup's own tables (and its own rows in the shared appconfig table) must - // survive this restore untouched: they describe the backup/key/target that is - // actively performing it right now, on THIS installation, not the one that was - // backed up. Overwriting them mid-restore would pull the rug from under it. - foreach (array_keys($manifest['tables']) as $table) { - /** @var string $table */ - if (str_starts_with($table, $prefix . 'ngb_')) { - unset($manifest['tables'][$table]); - } - } - // Sequences are keyed separately from tables; excluding a table does not exclude - // its sequence, which would otherwise rewind a retained ngb_* table's sequence - // below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()). - foreach (array_keys($manifest['sequences'] ?? []) as $seq) { - /** @var string $seq */ - if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) { - unset($manifest['sequences'][$seq]); - } - } - $ownAppConfig = $this->preserveOwnAppConfig(); - $dumper = new DbDumper($this->db->getInner(), $prefix); + $dbError = $check['dbUnreadable'] ?? null; + $skipped = $salvage ? [] : null; + $dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/'); + if ($dbError === null) { try { - $dumper->restore($repo, $manifest, $dbReport, $beat, $skipMissingTables); - } finally { - $this->restoreOwnAppConfig($ownAppConfig); + $this->restoreDatabase($repo, $meta, $manifest, $dbReport, $beat, $skipMissingTables); + } catch (\Throwable $e) { + if (!$salvage) { + throw $e; + } + // DbDumper::restore() rolled its transaction back: the database is the fresh one. + $dbError = 'the database could not be restored: ' . $e->getMessage(); + $dbReport = null; } } + if ($dbError !== null) { + $salvageDir = $dataDir . '/ngb-salvage-' . substr($snapshotId, 0, 12); + $dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $salvageDir . '/data', stripPrefix: true, heartbeat: $beat, skipped: $skipped); + $this->restore->restoreToDirectory($target, $snapshotId, 'config', $salvageDir . '/config', stripPrefix: true, heartbeat: $beat, skipped: $skipped); + return ['dbReport' => ['skipped' => [], 'restored' => [], 'created' => [], 'missing' => []], 'dataFiles' => $dataFiles, 'configKeys' => 0, + 'dbConsistent' => true, 'missingApps' => $check['missingApps'], 'skipped' => $skipped ?? [], 'dbError' => $dbError, + 'salvageDir' => $salvageDir, 'configError' => null]; + } // Read (but do not yet apply) the backed-up config.php before touching the data // directory: once applied, 'secret' may differ from what ng_backup's own Target // options were encrypted with (see restoreFull()'s docblock), and every further // repository access through $this->targets would fail to decrypt them. - /** @var array $backupConfig */ - $backupConfig = $this->fetchBackupConfig($target, $snapshotId); + $configError = null; + try { + /** @var array $backupConfig */ + $backupConfig = $this->fetchBackupConfig($target, $snapshotId, $salvage); + } catch (\Throwable $e) { + if (!$salvage) { + throw $e; + } + $configError = $e->getMessage(); + $backupConfig = []; + } - $dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/'); - $dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat); + $dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat, skipped: $skipped); // Last step: nothing after this may need $this->targets->repository($target) again. if ($backupConfig !== []) { @@ -190,15 +219,63 @@ final class RestoreFullService { } return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => [], 'created' => [], 'missing' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig), - 'dbConsistent' => (bool)($manifest['consistent'] ?? true), 'missingApps' => $check['missingApps']]; + 'dbConsistent' => (bool)($manifest['consistent'] ?? true), 'missingApps' => $check['missingApps'], + 'skipped' => $skipped ?? [], 'dbError' => null, 'salvageDir' => null, 'configError' => $configError]; }); } - /** @return array config.php's $CONFIG from the backup, minus the connection keys */ - private function fetchBackupConfig(Target $target, string $snapshotId): array { + /** + * Replace the database with the snapshot's dump (one transaction, see DbDumper::restore()). + * + * @param array $meta the snapshot's meta record + * @param array $manifest set to the dump's manifest + */ + private function restoreDatabase(\OCA\NgBackup\Repository\Repository $repo, array $meta, array &$manifest, ?array &$dbReport, callable $beat, bool $skipMissingTables): void { + if (!isset($meta['db'])) { + return; + } + $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); + $prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_'); + // ng_backup's own tables (and its own rows in the shared appconfig table) must + // survive this restore untouched: they describe the backup/key/target that is + // actively performing it right now, on THIS installation, not the one that was + // backed up. Overwriting them mid-restore would pull the rug from under it. + foreach (array_keys($manifest['tables']) as $table) { + /** @var string $table */ + if (str_starts_with($table, $prefix . 'ngb_')) { + unset($manifest['tables'][$table]); + } + } + // Sequences are keyed separately from tables; excluding a table does not exclude + // its sequence, which would otherwise rewind a retained ngb_* table's sequence + // below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()). + foreach (array_keys($manifest['sequences'] ?? []) as $seq) { + /** @var string $seq */ + if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) { + unset($manifest['sequences'][$seq]); + } + } + $ownAppConfig = $this->preserveOwnAppConfig(); + $dumper = new DbDumper($this->db->getInner(), $prefix); + try { + $dumper->restore($repo, $manifest, $dbReport, $beat, $skipMissingTables); + } finally { + $this->restoreOwnAppConfig($ownAppConfig); + } + } + + /** + * @return array config.php's $CONFIG from the backup, minus the connection keys + * @throws \RuntimeException with $salvage, when config.php itself cannot be read + */ + private function fetchBackupConfig(Target $target, string $snapshotId, bool $salvage = false): array { $tmp = sys_get_temp_dir() . '/ngb-restore-config-' . bin2hex(random_bytes(6)); try { - $this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true); + $skipped = $salvage ? [] : null; + $this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true, skipped: $skipped); + if (isset($skipped['config/config.php'])) { + throw new \RuntimeException('config.php cannot be read: ' . $skipped['config/config.php']); + } $path = $tmp . '/config.php'; if (!is_file($path)) { return []; // nothing to merge; the live config.php stays as occ maintenance:install made it diff --git a/lib/Service/RestoreService.php b/lib/Service/RestoreService.php index fc3e276..0bc238c 100644 --- a/lib/Service/RestoreService.php +++ b/lib/Service/RestoreService.php @@ -84,8 +84,8 @@ final class RestoreService { * * @return array{restored:int, trashed:int, target:string} */ - public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array { - return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress) { + public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null, ?array &$skipped = null): array { + return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress, &$skipped) { $last = time(); // Keep the lease alive while files are being written. $beat = function (int $done, int $total, string $name) use ($refresh, $progress, &$last): void { @@ -97,11 +97,12 @@ final class RestoreService { $progress($done, $total, $name); } }; - return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat); + return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat, $skipped); }); } - private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress): array { + /** @param array|null $skipped see Repository::restore() */ + private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress, ?array &$skipped): array { if (!Repository::isSafePath($logicalPath) || !preg_match('#^data/([^/]+)/files(?:/(.*))?$#', $logicalPath, $m)) { throw new \InvalidArgumentException('Expected a path like data//files/'); } @@ -139,7 +140,16 @@ final class RestoreService { $this->cleaned = []; $restored = 0; foreach ($names as $relPath => $entry) { - $this->writeFile($repo, $dest, (string)$relPath, $entry); + try { + $this->writeFile($repo, $dest, (string)$relPath, $entry); + } catch (\OCA\NgBackup\Repository\RepositoryException|\OCA\NgBackup\Backend\BackendException|\OCA\NgBackup\Crypto\CryptoException $e) { + // Damaged or missing on the location (writeFile removed its temporary) + if ($skipped === null) { + throw $e; + } + $skipped[$entry['p']] = $e->getMessage(); + continue; + } $restored++; if ($progress !== null) { $progress($restored, count($names), (string)$relPath); @@ -160,14 +170,14 @@ final class RestoreService { * Repository::restore()); used to restore "data/" or "config/" straight into the * real data/config directory for occ backup:restore:full. */ - public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null): int { + public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null): int { if ($prefix !== '' && !Repository::isSafePath($prefix)) { throw new \InvalidArgumentException('Invalid path: ' . $prefix); } if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) { throw new \RuntimeException("Cannot create $directory"); } - return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat): int { + return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat, &$skipped): int { $own = LeaseService::throttled($refresh); $beat = static function () use ($own, $heartbeat): void { $own(); @@ -175,7 +185,7 @@ final class RestoreService { $heartbeat(); } }; - return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat); + return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat, $skipped); }, 3600); } @@ -203,7 +213,7 @@ final class RestoreService { throw new \RuntimeException('Write failed for ' . $relPath); } if ($written !== $entry['s']) { - throw new \RuntimeException("Size mismatch restoring $relPath"); + throw new \OCA\NgBackup\Repository\RepositoryException("Size mismatch restoring $relPath"); } if ($folder->nodeExists($name)) { $existing = $folder->get($name); diff --git a/tests/Unit/Repository/SalvageTest.php b/tests/Unit/Repository/SalvageTest.php new file mode 100644 index 0000000..f7dc458 --- /dev/null +++ b/tests/Unit/Repository/SalvageTest.php @@ -0,0 +1,88 @@ +root = $this->tempDir(); + $this->keys = KeyRing::generate(); + $this->src = $this->tempDir(); + mkdir("$this->src/docs"); + // Incompressible, 2 MiB each: every file is a blob of its own in one pack + foreach (['a', 'b', 'c'] as $name) { + file_put_contents("$this->src/docs/$name.bin", random_bytes(2 * 1048576)); + } + file_put_contents("$this->src/note.txt", "small\n"); + $repo = Repository::initWithKey(new LocalBackend($this->root), $this->keys, $this->keys->wrap('pw')); + $this->snapshot = $repo->backupDirectory($this->src)['snapshot']; + } + + /** Flip bytes in the middle of the (only) pack: inside one of the 2 MiB files. */ + private function damagePack(): void { + $packs = glob($this->root . '/packs/*/*'); + $this->assertCount(1, $packs); + $fh = fopen($packs[0], 'r+b'); + $middle = intdiv(filesize($packs[0]), 2); + fseek($fh, $middle); + $bytes = fread($fh, 16); + fseek($fh, $middle); + fwrite($fh, ~$bytes); + fclose($fh); + } + + public function testSalvageLeavesOutTheDamagedFileAndRestoresTheRest(): void { + $this->damagePack(); + $target = $this->tempDir(); + $skipped = []; + + $restored = Repository::openWithKey(new LocalBackend($this->root), $this->keys) + ->restore($this->snapshot, $target, '', false, null, $skipped); + + $this->assertSame(3, $restored); + $this->assertCount(1, $skipped); + $damaged = array_key_first($skipped); + $this->assertMatchesRegularExpression('#^docs/[abc]\.bin$#', $damaged); + $this->assertFileDoesNotExist("$target/$damaged"); + foreach (['docs/a.bin', 'docs/b.bin', 'docs/c.bin', 'note.txt'] as $file) { + if ($file !== $damaged) { + $this->assertFileEquals("$this->src/$file", "$target/$file"); + } + } + $this->assertSame([], glob("$target/docs/*.ngb-restore-*"), 'no temporary left of the damaged file'); + } + + public function testWithoutSalvageTheRestoreStopsAtTheDamagedFile(): void { + $this->damagePack(); + + $this->expectException(\Exception::class); + Repository::openWithKey(new LocalBackend($this->root), $this->keys)->restore($this->snapshot, $this->tempDir()); + } + + public function testSalvageOfAnUndamagedSnapshotSkipsNothing(): void { + $target = $this->tempDir(); + $skipped = []; + + $this->assertSame(4, Repository::openWithKey(new LocalBackend($this->root), $this->keys) + ->restore($this->snapshot, $target, '', false, null, $skipped)); + $this->assertSame([], $skipped); + } +} diff --git a/tests/integration/full-restore-roundtrip.sh b/tests/integration/full-restore-roundtrip.sh index 3cb3c81..bcefbf3 100755 --- a/tests/integration/full-restore-roundtrip.sh +++ b/tests/integration/full-restore-roundtrip.sh @@ -2,13 +2,15 @@ # Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a # SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup # location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7. -# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp|to-pgsql|to-mysql|to-sqlite] +# Usage: full-restore-roundtrip.sh [sqlite|pgsql|mysql] [legacy|sftp|salvage|to-pgsql|to-mysql|to-sqlite] # e.g. full-restore-roundtrip.sh 35-apache pgsql # With "sftp" the location is an SFTP server instead of a shared directory, and the origin has a # file spread over several packs: the restore reads them ahead on connections of its own. # With "to-" the restore instance runs that database whatever the origin runs (a MySQL or # SQLite backup records no ID counters; after the restore new rows must not collide with restored # ones, whatever the database on either side). +# With "salvage" the snapshot's database dump is damaged on the location: the restore must stop +# without changing anything, and with --salvage put the files in a folder of their own. # With "legacy" the origin runs NG Backup 1.0.0-rc.1 (0.9.7), whose backups do not record the # structure of their tables: the restore must then leave the tables of the disabled app out # with --skip-missing-tables, as for a real backup made before 1.0.0-rc.2. @@ -135,6 +137,12 @@ RUN_OUT=$(run_on ngb-frt-origin backup:run offsite -l disaster-test) SNAP=$(printf '%s\n' "$RUN_OUT" | head -1 | awk -F'snapshot ' '{print $2}' | tr -d ',') [ -n "$SNAP" ] || fail "no snapshot id from backup:run" ok "test data backed up, snapshot $SNAP" +if [ "$MODE" = salvage ]; then + # The database dump of this snapshot damaged on the location: a byte flipped in its manifest + docker exec -u www-data ngb-frt-origin php -r '$p = "/mnt/shared/repo/db/" . $argv[1]; $d = file_get_contents($p); $i = intdiv(strlen($d), 2); $d[$i] = ~$d[$i]; file_put_contents($p, $d);' "$SNAP" \ + || fail "damage the database dump" + ok "database dump of the snapshot damaged on the location" +fi docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass @@ -164,6 +172,30 @@ printf '%s\n' "$LIST_OUT" | grep -q "$SNAP" || { } ok "recovery kit imported, existing repository adopted, snapshot visible" +if [ "$MODE" = salvage ]; then + set +e + OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction 2>&1); CODE=$? + set -e + [ "$CODE" = 1 ] || { printf '%s\n' "$OUT" | tail -20; fail "a restore with a damaged database dump did not stop (exit $CODE)"; } + printf '%s\n' "$OUT" | grep -- "--salvage" >/dev/null || { printf '%s\n' "$OUT" | tail -20; fail "the stopped restore does not point at --salvage"; } + run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode stayed on after a restore that changed nothing" + ok "damaged database dump: the restore stops before changing anything and points at --salvage" + set +e + OUT=$(run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction --salvage 2>&1); CODE=$? + set -e + [ "$CODE" = 2 ] || { printf '%s\n' "$OUT" | tail -30; fail "restore:full --salvage exited with $CODE, not 2"; } + SALVAGE=/var/www/html/data/ngb-salvage-${SNAP:0:12} + printf '%s\n' "$OUT" | grep "$SALVAGE" >/dev/null || { printf '%s\n' "$OUT" | tail -30; fail "the output does not name the salvage folder"; } + [ "$(docker exec ngb-frt-restore cat "$SALVAGE/data/frtest/files/Docs/note.txt")" = "hello disaster recovery" ] || fail "the user's file is not in the salvage folder" + docker exec ngb-frt-restore test -f "$SALVAGE/config/config.php" || fail "config.php is not in the salvage folder" + run_on ngb-frt-restore user:list | grep frtest >/dev/null && fail "the fresh database was changed (the backed-up user is there)" + run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the salvage" + run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after the salvage" + ok "with --salvage: the fresh database untouched, data and config of the backup in $SALVAGE (exit 2)" + echo "FULL RESTORE ROUNDTRIP PASS ($DB, salvage)" + exit 0 +fi + RESTORE_OPTS=(--no-interaction --install-apps) if [ "$LEGACY" = legacy ] && [ "$DB" != sqlite ]; then # Without --skip-missing-tables an older backup must be refused, naming the app's tables