diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a37530..a216970 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,3 +77,13 @@ jobs: docker exec ngbu chown www-data:www-data /var/www/html/custom_apps - name: Per-user backup/restore round trip run: tests/integration/user-roundtrip.sh "docker exec -u www-data ngbu" /var/www/html/data + + integration-full-restore: + # Disaster recovery (occ backup:restore:full) on a SECOND, fresh Nextcloud instance that + # never saw the first one, sharing only the backup location. Does not depend on the database + # beyond what the main integration job already covers, so this runs once, on one NC version. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Full restore round trip + run: tests/integration/full-restore-roundtrip.sh 35-apache diff --git a/PLAN.md b/PLAN.md index de67187..34876e3 100644 --- a/PLAN.md +++ b/PLAN.md @@ -338,6 +338,18 @@ een backup-app die niet betrouwbaar is, is erger dan geen backup-app. Volgorde: fase 1 grondig testen, reviewen en debuggen; dan nadenken over 0.1.0-alpha.1. Een minimale volledige restore (`occ backup:restore:full` op een verse installatie) komt vóór de beta. +**Roadmap-indeling fase 2 (besluit 2026-10-05):** naar v1.2.0 toewerken met in fase 2: +1. `backup:restore:full` (disaster recovery, verse installatie) — **in uitvoering** +2. FTP/SMB-ondersteuning +3. Verificatie/checksum-audit-commando +4. Retentie/prune voor user-exports +5. Automatische user-export inpluggen in het reguliere schema (of expliciet besluit dat het on-demand blijft) +9. SMB/NFS intern testen op de eigen OMV-instantie + +Doorgeschoven naar **v1.3.0** (fase 3, niet meer fase 2): in-place volledige restore +(eigen restore-modus i.p.v. onderhoudsmodus) en zelfbediening voor gebruikers (eigen bestanden +terugzetten vanuit de Files-app). + **Let op (ontdekt 2026-10-05):** Nextcloud's eigen `info.xsd` staat voor `` alleen `[0-9]+(\.[0-9]+){0,2}` toe — geen `-alpha`/`-beta`/`-rc`-achtervoegsel. De stabiliteitsaanduiding uit deze tabel gaat dus niet in `info.xml` (dat blijft gewoon oplopend, bijv. `0.1.0`), maar in de diff --git a/appinfo/info.xml b/appinfo/info.xml index 348c967..99cc4e4 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -49,6 +49,8 @@ relying on this for anything.]]> OCA\NgBackup\Command\KeySlotRemove OCA\NgBackup\Command\BackupUser OCA\NgBackup\Command\RestoreUser + OCA\NgBackup\Command\KeyImport + OCA\NgBackup\Command\RestoreFull OCA\NgBackup\Settings\Admin diff --git a/lib/Command/KeyImport.php b/lib/Command/KeyImport.php new file mode 100644 index 0000000..6f9dd02 --- /dev/null +++ b/lib/Command/KeyImport.php @@ -0,0 +1,77 @@ +setName('backup:key:import') + ->setDescription('Disaster recovery: adopt the backup key from a recovery kit on a fresh installation') + ->addArgument('kit-file', InputArgument::REQUIRED, 'The recovery kit file (occ backup:key:kit on the original installation)') + ->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking') + ->setHelp(<<<'HELP' +For a fresh installation only (occ backup:key:init must not have been run here yet). Reads one +of the kit's passphrase slots to recover the original master key, so occ backup:target:add opens +the existing repository on the location instead of creating a new, empty one. + +Example: + occ backup:key:import /root/ng_backup-recovery-kit.json + occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=... + occ backup:restore:full offsite +HELP); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + if ($this->keys->isInitialized()) { + $output->writeln('A backup key already exists here (fingerprint ' . $this->keys->fingerprint() . '). occ backup:key:import is only for a fresh installation.'); + return 1; + } + $kitFile = (string)$input->getArgument('kit-file'); + $raw = @file_get_contents($kitFile); + if ($raw === false) { + $output->writeln("Cannot read $kitFile"); + return 1; + } + try { + $kit = json_decode($raw, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + $output->writeln('Not a valid recovery kit file: ' . $e->getMessage() . ''); + return 1; + } + $file = $input->getOption('passphrase-file'); + if ($file !== null) { + $pass = rtrim((string)@file_get_contents($file), "\r\n"); + } else { + $helper = $this->getHelper('question'); + $q = (new Question('Passphrase of one of the kit\'s slots: '))->setHidden(true)->setHiddenFallback(false); + $pass = (string)$helper->ask($input, $output, $q); + } + try { + $this->keys->importRecoveryKit($kit, $pass); + } catch (\Throwable $e) { + $output->writeln('' . $e->getMessage() . ''); + return 1; + } + $output->writeln('Backup key imported, fingerprint ' . $this->keys->fingerprint() . '.'); + $output->writeln('Next: connect the backup location with the same details as before (occ backup:target:add), then occ backup:list and occ backup:restore:full.'); + return 0; + } +} diff --git a/lib/Command/RestoreFull.php b/lib/Command/RestoreFull.php new file mode 100644 index 0000000..4487b54 --- /dev/null +++ b/lib/Command/RestoreFull.php @@ -0,0 +1,71 @@ +setName('backup:restore:full') + ->setDescription('Disaster recovery: restore the database, data directory and config on a fresh installation') + ->addArgument('target', InputArgument::REQUIRED, 'Location name or id') + ->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)') + ->setHelp(<<<'HELP' +For a FRESH Nextcloud installation only, same major version as the backup, right after +occ maintenance:install, occ backup:key:import and occ backup:target:add -- before anyone else +uses this instance. + +Every app that was enabled when the snapshot was taken must already be installed here (its +database tables must exist before the restore can write into them); the command lists what is +missing and refuses to continue rather than guess. + +Steps: the database is replaced (consistent, in one transaction); the data directory is restored +directly in place; config.php is merged in -- everything from the backup except the database +connection and data directory settings, which stay as occ maintenance:install configured them +here. Other files under config/ (if any) are not restored. + +Afterwards: check trusted_domains and overwritehost/overwriteprotocol match this server, and +occ maintenance:repair. + +Example: + occ backup:restore:full offsite 537bf7e9c1a4... +HELP); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + try { + $target = $this->targets->get($input->getArgument('target')); + $snapshot = (string)$input->getArgument('snapshot'); + $result = $this->restoreFull->restoreFull($target, $snapshot); + } catch (\Throwable $e) { + $output->writeln('' . $e->getMessage() . ''); + return 1; + } + $output->writeln(sprintf( + 'Database restored (%d tables, %d unchanged), config merged (%d keys), %d data files restored.', + count($result['dbReport']['restored']), + count($result['dbReport']['skipped']), + $result['configKeys'], + $result['dataFiles'], + )); + $output->writeln('Next: check trusted_domains and overwritehost/overwriteprotocol match this server, then occ maintenance:repair.'); + return 0; + } +} diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index 59d83bf..ad65baf 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -190,8 +190,15 @@ final class Repository { return ['snapshot' => $snapshotId] + $stats; } - /** Restore a snapshot (or only paths starting with $prefix) into a local directory. */ - public function restore(string $snapshotId, string $target, string $prefix = ''): int { + /** + * Restore a snapshot (or only paths starting with $prefix) into a local directory. + * + * @param bool $stripPrefix write "$target/" instead of + * "$target/" (disaster recovery: restoring e.g. "data/" or + * "config/" directly into the real data/config directory, not a subdirectory + * named after the prefix). Ignored when $prefix is '' (nothing to strip). + */ + public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false): int { if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) { throw new RepositoryException('Invalid path prefix'); } @@ -200,7 +207,8 @@ final class Repository { if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) { continue; } - $dest = rtrim($target, '/') . '/' . $entry['p']; + $relPath = ($stripPrefix && $prefix !== '') ? substr($entry['p'], strlen(rtrim($prefix, '/')) + 1) : $entry['p']; + $dest = rtrim($target, '/') . '/' . $relPath; if (!is_dir(dirname($dest))) { mkdir(dirname($dest), 0700, true); } diff --git a/lib/Service/KeyService.php b/lib/Service/KeyService.php index 06ac581..9538c79 100644 --- a/lib/Service/KeyService.php +++ b/lib/Service/KeyService.php @@ -71,6 +71,84 @@ final class KeyService { $this->cached = $keys; } + /** + * Disaster recovery on a fresh installation: adopt the master key from a recovery kit + * (occ backup:key:kit on the original installation) instead of generating a new one. + * Afterwards occ backup:target:add opens the existing repository on the location with this + * key, instead of creating a new, empty one. + * + * @param array $kit as produced by recoveryKit() + * @throws \OCA\NgBackup\Crypto\CryptoException wrong passphrase, or a damaged/foreign kit + */ + public function importRecoveryKit(array $kit, #[\SensitiveParameter] string $passphrase): void { + if ($this->isInitialized()) { + throw new \RuntimeException('A backup key already exists'); + } + if (!isset($kit['wrapped_key'])) { + throw new \InvalidArgumentException('Not a NG Backup recovery kit file'); + } + $keys = KeyRing::unwrapAny($kit['wrapped_key'], $passphrase); + $this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER, + $this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true); + $this->cached = $keys; + $this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode($kit['wrapped_key'], JSON_THROW_ON_ERROR), true, true); + $this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true); + $this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, self::DEFAULT_DELETE_DELAY, true); + $this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, (int)($kit['kit_version'] ?? 1), true); + // Having and successfully using a real kit file is stronger proof than the normal + // download-and-confirm flow it replaces. + $this->confirmRecoveryKit('cli-import'); + } + + /** + * The master key and the recovery kit code are stored encrypted with Nextcloud's own + * instance secret (ICrypto), not with themselves; the wrapped key is not (it is sealed with + * the admin's passphrase instead), but Nextcloud's own AppConfig additionally encrypts any + * "sensitive" value with the instance secret regardless, underneath whatever we store. So + * changing that secret (occ backup:restore:full merging config.php) would otherwise leave + * all three permanently undecryptable. Call $applySecretChange (which must be the only thing + * that changes the secret) through here instead: all three are decrypted first, then + * re-encrypted under whatever secret is active once $applySecretChange returns. + * + * Not crash-safe: a process kill between the delete and the re-save of one of these keys + * would leave it missing rather than merely stale. Disaster recovery is run once, by hand, + * right after occ maintenance:install; a dedicated two-phase write is not justified yet for + * that narrow a window. + */ + public function reencryptForSecretRotation(callable $applySecretChange): mixed { + $master = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true); + $wrapped = $this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '', true); + $kitCodeRaw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '', true); + $kitCode = $kitCodeRaw !== '' ? json_decode($kitCodeRaw, true, 512, JSON_THROW_ON_ERROR) : null; + $masterPlain = $master !== '' ? $this->crypto->decrypt($master) : null; + $kitCodePlain = $kitCode !== null && isset($kitCode['code']) ? $this->crypto->decrypt($kitCode['code']) : null; + + $result = $applySecretChange(); + $this->cached = null; // the master key is about to be re-saved; do not keep the pre-rotation copy + + // Nextcloud's own AppConfig additionally encrypts a "sensitive" value itself (tied to + // the same secret, underneath our own encrypt() above); setValueString() on an existing + // key reads the old value first to compare, which would try to decrypt it with the new + // secret and fail the same way. Delete first so there is nothing to compare against. + if ($masterPlain !== null) { + $this->appConfig->deleteKey(Application::APP_ID, self::KEY_MASTER); + $this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER, $this->crypto->encrypt($masterPlain), true, true); + } + if ($wrapped !== '') { + // Already plaintext JSON here: getValueString() above transparently undid + // AppConfig's own sensitive-value encryption; there is no encrypt() of our own to + // redo (wrapping is sealed with the passphrase, not the instance secret). + $this->appConfig->deleteKey(Application::APP_ID, self::KEY_WRAPPED); + $this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, $wrapped, true, true); + } + if ($kitCodePlain !== null) { + $kitCode['code'] = $this->crypto->encrypt($kitCodePlain); + $this->appConfig->deleteKey(Application::APP_ID, self::KEY_KIT_CODE); + $this->appConfig->setValueString(Application::APP_ID, self::KEY_KIT_CODE, json_encode($kitCode, JSON_THROW_ON_ERROR), true, true); + } + return $result; + } + public function keyRing(): KeyRing { if ($this->cached === null) { $enc = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true); diff --git a/lib/Service/RestoreFullService.php b/lib/Service/RestoreFullService.php new file mode 100644 index 0000000..8fc72b9 --- /dev/null +++ b/lib/Service/RestoreFullService.php @@ -0,0 +1,225 @@ +} + */ + public function checkCompatibility(Target $target, string $snapshotId): array { + $meta = $this->targets->repository($target)->snapshotMeta($snapshotId); + $currentMajor = explode('.', $this->config->getSystemValueString('version'))[0]; + $snapshotMajor = explode('.', (string)($meta['nextcloud'] ?? ''))[0]; + $enabled = array_flip($this->appManager->getEnabledApps()); + $missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled))); + return [ + 'ok' => $currentMajor === $snapshotMajor && $missingApps === [], + 'currentMajor' => $currentMajor, + 'snapshotMajor' => $snapshotMajor, + 'missingApps' => $missingApps, + ]; + } + + /** @return array{dbReport:array, dataFiles:int, configKeys:int} */ + public function restoreFull(Target $target, string $snapshotId): array { + $check = $this->checkCompatibility($target, $snapshotId); + if (!$check['ok']) { + throw new \RuntimeException($this->describeBlockers($check)); + } + return $this->withSharedLock($target, function () use ($target, $snapshotId) { + $repo = $this->targets->repository($target); + $meta = $repo->snapshotMeta($snapshotId); + + $dbReport = null; + if (isset($meta['db'])) { + $manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR); + $prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_'); + // ng_backup's own tables (and its own rows in the shared appconfig table) must + // survive this restore untouched: they describe the backup/key/target that is + // actively performing it right now, on THIS installation, not the one that was + // backed up. Overwriting them mid-restore would pull the rug from under it. + foreach (array_keys($manifest['tables']) as $table) { + /** @var string $table */ + if (str_starts_with($table, $prefix . 'ngb_')) { + unset($manifest['tables'][$table]); + } + } + // Sequences are keyed separately from tables; excluding a table does not exclude + // its sequence, which would otherwise rewind a retained ngb_* table's sequence + // below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()). + foreach (array_keys($manifest['sequences'] ?? []) as $seq) { + /** @var string $seq */ + if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) { + unset($manifest['sequences'][$seq]); + } + } + $ownAppConfig = $this->preserveOwnAppConfig(); + $dumper = new DbDumper($this->db->getInner(), $prefix); + try { + $dumper->restore($repo, $manifest, $dbReport); + } finally { + $this->restoreOwnAppConfig($ownAppConfig); + } + } + + // Read (but do not yet apply) the backed-up config.php before touching the data + // directory: once applied, 'secret' may differ from what ng_backup's own Target + // options were encrypted with (see restoreFull()'s docblock), and every further + // repository access through $this->targets would fail to decrypt them. + $backupConfig = $this->fetchBackupConfig($target, $snapshotId); + + $dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/'); + $dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true); + + // Last step: nothing after this may need $this->targets->repository($target) again. + if ($backupConfig !== []) { + $this->targets->reencryptOptionsAround( + fn (): mixed => $this->keys->reencryptForSecretRotation(fn (): mixed => $this->config->setSystemValues($backupConfig)), + ); + } + + return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig)]; + }); + } + + /** @return array config.php's $CONFIG from the backup, minus the connection keys */ + private function fetchBackupConfig(Target $target, string $snapshotId): array { + $tmp = sys_get_temp_dir() . '/ngb-restore-config-' . bin2hex(random_bytes(6)); + try { + $this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true); + $path = $tmp . '/config.php'; + if (!is_file($path)) { + return []; // nothing to merge; the live config.php stays as occ maintenance:install made it + } + $backupConfig = (static function () use ($path): array { + $CONFIG = []; + include $path; + return $CONFIG; + })(); + foreach (self::CONNECTION_KEYS as $key) { + unset($backupConfig[$key]); + } + return $backupConfig; + } finally { + $this->rmrf($tmp); + } + } + + /** @return list> raw appconfig rows (appid, configkey, configvalue, type, lazy) */ + private function preserveOwnAppConfig(): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('*')->from('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID))); + /** @var list> */ + return $qb->executeQuery()->fetchAll(); + } + + /** + * @param list> $rows raw appconfig rows, as returned by preserveOwnAppConfig() + * + * Runs in its own transaction (DbDumper::restore() already committed its own by the time + * this is called): not a full guarantee against a crash spanning both, but means this step + * itself cannot leave ng_backup's appconfig rows half deleted, half reinserted. + */ + private function restoreOwnAppConfig(array $rows): void { + $this->db->beginTransaction(); + try { + $qb = $this->db->getQueryBuilder(); + $qb->delete('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID))); + $qb->executeStatement(); + foreach ($rows as $row) { + $ins = $this->db->getQueryBuilder(); + $ins->insert('appconfig')->values([ + 'appid' => $ins->createNamedParameter($row['appid']), + 'configkey' => $ins->createNamedParameter($row['configkey']), + 'configvalue' => $ins->createNamedParameter($row['configvalue']), + 'type' => $ins->createNamedParameter($row['type'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT), + 'lazy' => $ins->createNamedParameter($row['lazy'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT), + ]); + $ins->executeStatement(); + } + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + throw $e; + } + } + + private function rmrf(string $dir): void { + if (!is_dir($dir)) { + return; + } + foreach (new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST) as $f) { + $f->isDir() ? rmdir($f->getPathname()) : unlink($f->getPathname()); + } + rmdir($dir); + } + + private function describeBlockers(array $check): string { + $parts = []; + if ($check['currentMajor'] !== $check['snapshotMajor']) { + $parts[] = "this installation is Nextcloud {$check['currentMajor']}, the backup is from {$check['snapshotMajor']} (same major version required)"; + } + if ($check['missingApps'] !== []) { + $parts[] = 'install these apps first (their tables must exist before the database restore can use them): ' + . implode(', ', array_map(fn (string $a) => "occ app:install $a", $check['missingApps'])); + } + return implode('; ', $parts); + } + + private function withSharedLock(Target $target, callable $fn): mixed { + try { + // No progress from DbDumper::restore()/Repository::restore() to refresh this lease + // against; a very large instance could in theory outrun this TTL (same limitation as + // RestoreService::restoreToDirectory(), see also UserBackupService). + return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 7200, $fn); + } catch (LockedException) { + throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later'); + } + } +} diff --git a/lib/Service/RestoreService.php b/lib/Service/RestoreService.php index c68c677..d45d7f4 100644 --- a/lib/Service/RestoreService.php +++ b/lib/Service/RestoreService.php @@ -149,15 +149,21 @@ final class RestoreService { return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())]; } - /** Raw restore of any snapshot path to a local directory (administrators, disaster recovery). */ - public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory): int { + /** + * Raw restore of any snapshot path to a local directory (administrators, disaster recovery). + * + * @param bool $stripPrefix write into $directory itself instead of $directory/$prefix (see + * Repository::restore()); used to restore "data/" or "config/" straight into the + * real data/config directory for occ backup:restore:full. + */ + public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false): int { if ($prefix !== '' && !Repository::isSafePath($prefix)) { throw new \InvalidArgumentException('Invalid path: ' . $prefix); } if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) { throw new \RuntimeException("Cannot create $directory"); } - return $this->withSharedLock($target, fn () => $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/'), 3600); + return $this->withSharedLock($target, fn () => $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix), 3600); } private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void { diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php index 67a81a9..3878feb 100644 --- a/lib/Service/TargetService.php +++ b/lib/Service/TargetService.php @@ -10,10 +10,13 @@ namespace OCA\NgBackup\Service; use OCA\NgBackup\Backend\ExternalStorageFactory; use OCA\NgBackup\Backend\IBackend; use OCA\NgBackup\Backend\StorageBackend; +use OCA\NgBackup\Db\Snapshot; +use OCA\NgBackup\Db\SnapshotMapper; use OCA\NgBackup\Db\Target; use OCA\NgBackup\Db\TargetMapper; use OCA\NgBackup\Repository\Repository; use OCP\App\IAppManager; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\Security\ICrypto; use OCP\Server; @@ -30,6 +33,7 @@ final class TargetService { private IAppManager $appManager, private \OCA\NgBackup\Db\DbIndexCache $indexCache, private \OCA\NgBackup\Db\AppConfigCatalogAnchor $anchor, + private SnapshotMapper $snapshots, ) { } @@ -78,7 +82,62 @@ final class TargetService { $target->setBasePath(trim($basePath, '/')); $target->setRepositoryId($repo->id()); $target->setCreatedAt(time()); - return ['target' => $this->mapper->insert($target), 'created' => $created]; + $target = $this->mapper->insert($target); + if (!$created) { + // Disaster recovery: occ backup:list reads this local cache, not the repository + // directly, so a snapshot made by another installation must be backfilled here too. + $this->adoptSnapshots($target, $repo); + } + return ['target' => $target, 'created' => $created]; + } + + /** + * A target's options are encrypted with Nextcloud's own instance secret (ICrypto), not + * ng_backup's key, so changing that secret (occ backup:restore:full merging config.php) + * would otherwise leave every target permanently undecryptable. Call $applySecretChange + * (which must be the only thing that changes the secret) through here instead: every + * target's options are decrypted first, then re-encrypted under whatever secret is active + * once $applySecretChange returns. + */ + public function reencryptOptionsAround(callable $applySecretChange): mixed { + $plain = []; + foreach ($this->mapper->findAll() as $target) { + $plain[$target->getId()] = $this->options($target); + } + $result = $applySecretChange(); + foreach ($this->mapper->findAll() as $target) { + if (!isset($plain[$target->getId()])) { + continue; // added by $applySecretChange itself; nothing to re-encrypt + } + $target->setOptions($this->crypto->encrypt(json_encode($plain[$target->getId()], JSON_THROW_ON_ERROR))); + $this->mapper->update($target); + } + return $result; + } + + private function adoptSnapshots(Target $target, Repository $repo): void { + foreach ($repo->listSnapshots() as $snapshotId) { + try { + $this->snapshots->findOne($target->getId(), $snapshotId); + continue; // already known locally + } catch (DoesNotExistException) { + } + // One unreadable/corrupt snapshot must not stop the rest of the location's + // snapshots from being backfilled, nor abort adding the target itself. + try { + $meta = $repo->snapshotMeta($snapshotId); + $s = new Snapshot(); + $s->setTargetId($target->getId()); + $s->setSnapshotId($snapshotId); + $s->setKind((string)($meta['kind'] ?? 'full')); + $s->setLabel(($meta['label'] ?? '') !== '' ? $meta['label'] : null); + $s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now'))); + $s->setFiles((int)($meta['stats']['files'] ?? 0)); + $s->setBytes((int)($meta['stats']['bytes'] ?? 0)); + $this->snapshots->insert($s); + } catch (\Throwable) { + } + } } /** @return list */ diff --git a/psalm-baseline.xml b/psalm-baseline.xml index 98da393..d874a89 100644 --- a/psalm-baseline.xml +++ b/psalm-baseline.xml @@ -49,6 +49,11 @@ + + + + + @@ -455,6 +460,17 @@ >, forgotten: list, purged: list, prune: array}]]> + + + + + + + + + + + diff --git a/tests/integration/full-restore-roundtrip.sh b/tests/integration/full-restore-roundtrip.sh new file mode 100755 index 0000000..c0d270c --- /dev/null +++ b/tests/integration/full-restore-roundtrip.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a +# SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup +# location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7. +# Usage: full-restore-roundtrip.sh +# e.g. full-restore-roundtrip.sh 35-apache +set -euo pipefail +IMAGE="nextcloud:${1:-35-apache}" +APP=$(cd "$(dirname "$0")/../.." && pwd) +NET=ngb-frt-net +SHARED=$(mktemp -d) +chmod 777 "$SHARED" +fail() { echo "FAIL: $*"; exit 1; } +ok() { echo "ok $*"; } + +cleanup() { + docker rm -fv ngb-frt-origin ngb-frt-restore >/dev/null 2>&1 || true + docker network rm "$NET" >/dev/null 2>&1 || true + docker run --rm -v "$SHARED:/target" alpine sh -c 'rm -rf /target/*' >/dev/null 2>&1 || true + rmdir "$SHARED" 2>/dev/null || true + rm -f /tmp/ngb-frt-kit.json /tmp/ngb-frt-pass +} +trap cleanup EXIT + +run_on() { docker exec -u www-data "$1" php occ "${@:2}"; } + +docker network create "$NET" >/dev/null +docker run -d --name ngb-frt-origin --network "$NET" -e SQLITE_DATABASE=nextcloud \ + -e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \ + -v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null +for i in $(seq 1 90); do run_on ngb-frt-origin status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done +docker exec ngb-frt-origin chown www-data:www-data /var/www/html/custom_apps +run_on ngb-frt-origin app:enable ng_backup >/dev/null +ok "origin instance up" + +docker exec ngb-frt-origin sh -c "printf 'full restore roundtrip passphrase 2026\n' > /tmp/pass" +run_on ngb-frt-origin backup:key:init --passphrase-file=/tmp/pass --label=CI >/dev/null || fail "key init" +run_on ngb-frt-origin backup:key:kit --output=/tmp/kit.json >/dev/null +CODE=$(run_on ngb-frt-origin backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];') +run_on ngb-frt-origin backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "confirm" +run_on ngb-frt-origin backup:target:add offsite local -a null::null -o datadir=/mnt/shared --path=repo >/dev/null || fail "target add" +ok "key, kit and location on the origin" + +NC_PASS=$(openssl rand -hex 16) +docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-origin php occ user:add --password-from-env frtest >/dev/null +docker exec ngb-frt-origin sh -c "mkdir -p /var/www/html/data/frtest/files/Docs && head -c 200000 /dev/urandom > /var/www/html/data/frtest/files/Docs/report.bin && echo 'hello disaster recovery' > /var/www/html/data/frtest/files/Docs/note.txt && chown -R www-data:www-data /var/www/html/data/frtest" +run_on ngb-frt-origin files:scan frtest >/dev/null +RUN_OUT=$(run_on ngb-frt-origin backup:run offsite -l disaster-test) +SNAP=$(printf '%s\n' "$RUN_OUT" | head -1 | awk -F'snapshot ' '{print $2}' | tr -d ',') +[ -n "$SNAP" ] || fail "no snapshot id from backup:run" +ok "test data backed up, snapshot $SNAP" + +docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json +docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass +docker run -d --name ngb-frt-restore --network "$NET" -e SQLITE_DATABASE=nextcloud \ + -e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \ + -v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null +for i in $(seq 1 90); do run_on ngb-frt-restore status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done +docker exec ngb-frt-restore chown www-data:www-data /var/www/html/custom_apps +run_on ngb-frt-restore app:enable ng_backup >/dev/null +docker cp /tmp/ngb-frt-kit.json ngb-frt-restore:/tmp/kit.json +docker exec ngb-frt-restore chmod 644 /tmp/kit.json +docker cp /tmp/ngb-frt-pass ngb-frt-restore:/tmp/pass +ok "fresh restore instance up" + +run_on ngb-frt-restore backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass >/dev/null || fail "key import" +run_on ngb-frt-restore backup:target:add offsite local -a null::null -o datadir=/mnt/shared --path=repo >/dev/null || fail "target add on the restore instance" +run_on ngb-frt-restore backup:list offsite | grep -q "$SNAP" || fail "the origin's snapshot is not listed after connecting the location" +ok "recovery kit imported, existing repository adopted, snapshot visible" + +run_on ngb-frt-restore backup:restore:full offsite "$SNAP" >/dev/null || fail "restore:full" +ok "backup:restore:full completed" + +# Everything below must work in a FRESH occ process (a new PHP process per docker exec), since +# earlier bugs here only showed up after config.php's secret was rewritten mid-restore. +run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after restore (own key/target undecryptable?)" +run_on ngb-frt-restore user:list | grep -q frtest || fail "the backed-up user is missing from the restored database" +docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.txt || fail "restored file is missing" +[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt)" = "hello disaster recovery" ] || fail "restored file content differs" +run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore" +ok "database, files and ng_backup's own state are all intact and usable after the restore" + +run_on ngb-frt-restore backup:run offsite -l post-restore-check >/dev/null || fail "a new backup after the restore fails" +ok "the restored instance can run a new backup of its own" + +echo "FULL RESTORE ROUNDTRIP PASS"