From c601a02b2b6ccbe2136304be73c577780bad7850 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Fri, 2 Oct 2026 00:58:18 +0200 Subject: [PATCH] M3: restore files and folders (occ backup:browse, backup:restore) User files are restored through the Files API so the file cache, versions and trash stay consistent: modes new-folder (default, into 'Restored /'), merge (overwritten files keep their old content as a version) and replace (extra files go to the trash). Each file is staged under a temporary name and moved into place only after all its blobs were verified. Any snapshot path can also be restored raw to a local directory (disaster recovery). Tested on NC34: all three modes restore checksum-identical files, the overwritten content is kept as a version, the extra file is in the trash, no temporary files remain. --- appinfo/info.xml | 4 +- lib/Command/Browse.php | 61 +++++++++ lib/Command/Restore.php | 71 +++++++++++ lib/Service/RestoreService.php | 221 +++++++++++++++++++++++++++++++++ tests/.phpunit.result.cache | 2 +- 5 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 lib/Command/Browse.php create mode 100644 lib/Command/Restore.php create mode 100644 lib/Service/RestoreService.php diff --git a/appinfo/info.xml b/appinfo/info.xml index a13b017..8574c5b 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -5,7 +5,7 @@ NG Backup Encrypted, incremental backups of your Nextcloud with restore from the web interface - 0.0.3 + 0.0.4 agpl Dillard Blom NgBackup @@ -30,5 +30,7 @@ OCA\NgBackup\Command\RunBackup OCA\NgBackup\Command\Status OCA\NgBackup\Command\ListSnapshots + OCA\NgBackup\Command\Browse + OCA\NgBackup\Command\Restore diff --git a/lib/Command/Browse.php b/lib/Command/Browse.php new file mode 100644 index 0000000..58a7a51 --- /dev/null +++ b/lib/Command/Browse.php @@ -0,0 +1,61 @@ +setName('backup:browse') + ->setDescription('Show folders and files in a snapshot') + ->addArgument('target', InputArgument::REQUIRED, 'Location name or id') + ->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)') + ->addArgument('path', InputArgument::OPTIONAL, 'Path in the snapshot, e.g. data/alice/files/Documents', '') + ->setHelp("Top level: config/ and data/. User files are under data//files/.\n\nExample:\n occ backup:browse offsite 537bf7e9... data/alice/files"); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + try { + $items = $this->restore->browse($this->targets->get($input->getArgument('target')), $input->getArgument('snapshot'), (string)$input->getArgument('path')); + } catch (\Throwable $e) { + $output->writeln('' . $e->getMessage() . ''); + return 1; + } + if ($items === []) { + $output->writeln('Nothing here.'); + return 0; + } + $rows = array_map(fn ($i) => [$i['type'] === 'dir' ? $i['name'] . '/' : $i['name'], $this->size($i['size']), + $i['type'] === 'dir' ? $i['files'] . ' files' : date('Y-m-d H:i', (int)$i['mtime'])], $items); + (new Table($output))->setHeaders(['name', 'size', 'files / modified'])->setRows($rows)->render(); + return 0; + } + + private function size(int $b): string { + foreach (['B', 'KiB', 'MiB', 'GiB', 'TiB'] as $u) { + if ($b < 1024 || $u === 'TiB') { + return $u === 'B' ? "$b B" : sprintf('%.1f %s', $b, $u); + } + $b /= 1024; + } + return ''; + } +} diff --git a/lib/Command/Restore.php b/lib/Command/Restore.php new file mode 100644 index 0000000..d4131ed --- /dev/null +++ b/lib/Command/Restore.php @@ -0,0 +1,71 @@ +setName('backup:restore') + ->setDescription('Restore files or folders from a snapshot') + ->addArgument('target', InputArgument::REQUIRED, 'Location name or id') + ->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)') + ->addArgument('path', InputArgument::REQUIRED, 'Path in the snapshot (see backup:browse)') + ->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'new-folder, merge or replace (user files)', RestoreService::MODE_NEW_FOLDER) + ->addOption('to-directory', 'd', InputOption::VALUE_REQUIRED, 'Write to this local directory instead (any path, raw files)') + ->setHelp(<<<'HELP' +User files (data//files/...) are restored through Nextcloud, so versions, trash and the +file cache stay correct. Modes: + new-folder (default) into "Restored /" in the user's files; nothing existing is touched + merge into the original place; overwritten files keep their old content as a version + replace like merge, and files that are not in the snapshot are moved to the trash + +Examples: + occ backup:restore offsite 537bf7e9... data/alice/files/Documents + occ backup:restore offsite 537bf7e9... data/alice/files/Documents/plan.odt --mode=merge + occ backup:restore offsite 537bf7e9... config --to-directory=/root/restored-config +HELP); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + try { + $target = $this->targets->get($input->getArgument('target')); + $snapshot = $input->getArgument('snapshot'); + $path = $input->getArgument('path'); + if ($dir = $input->getOption('to-directory')) { + $n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir); + $output->writeln("Restored $n files to $dir."); + return 0; + } + $r = $this->restore->restoreUserFiles($target, $snapshot, $path, (string)$input->getOption('mode'), + function (int $done, int $total, string $name) use ($output) { + if ($output->isVerbose() || $done === $total || $done % 100 === 0) { + $output->writeln(" $done/$total $name"); + } + }); + } catch (\Throwable $e) { + $output->writeln('' . $e->getMessage() . ''); + return 1; + } + $output->writeln(sprintf('Restored %d files to %s%s.', $r['restored'], $r['target'], $r['trashed'] ? ", moved {$r['trashed']} extra files to the trash" : '')); + return 0; + } +} diff --git a/lib/Service/RestoreService.php b/lib/Service/RestoreService.php new file mode 100644 index 0000000..9c44f92 --- /dev/null +++ b/lib/Service/RestoreService.php @@ -0,0 +1,221 @@ +/files/...) are written through the Nextcloud Files API, so the file + * cache, versions and trash stay consistent: an overwritten file becomes a version and a + * removed file goes to the trash, which makes a restore itself reversible. Every file is + * written to a temporary name first and only renamed into place after all its blobs were + * decrypted and verified. + * + * Modes for user files: + * - new-folder (default): into "Restored /", nothing existing is touched + * - merge: into the original place, existing files are overwritten (old content kept as a version) + * - replace: like merge, and files that are not in the snapshot are moved to the trash + */ +class RestoreService { + public const MODE_NEW_FOLDER = 'new-folder'; + public const MODE_MERGE = 'merge'; + public const MODE_REPLACE = 'replace'; + + public function __construct( + private TargetService $targets, + private IRootFolder $root, + ) { + } + + /** + * Direct children (files and folders) of a path in a snapshot. + * + * @return list + */ + public function browse(Target $target, string $snapshotId, string $path = ''): array { + $prefix = trim($path, '/'); + $prefix = $prefix === '' ? '' : $prefix . '/'; + $children = []; + foreach ($this->targets->repository($target)->entries($snapshotId) as $e) { + if ($prefix !== '' && !str_starts_with($e['p'], $prefix)) { + continue; + } + $rest = substr($e['p'], strlen($prefix)); + [$name, $more] = array_pad(explode('/', $rest, 2), 2, null); + $c = $children[$name] ?? ['name' => $name, 'type' => $more === null ? 'file' : 'dir', 'size' => 0, 'mtime' => null, 'files' => 0]; + $c['size'] += $e['s']; + $c['files']++; + if ($more === null) { + $c['mtime'] = $e['m']; + } + $children[$name] = $c; + } + ksort($children, SORT_STRING); + return array_values($children); + } + + /** + * Restore user files from "data//files[/]" into that user's Nextcloud files. + * + * @return array{restored:int, trashed:int, target:string} + */ + public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array { + if (!preg_match('#^data/([^/]+)/files(?:/(.*))?$#', trim($logicalPath, '/'), $m)) { + throw new \InvalidArgumentException('Expected a path like data//files/'); + } + [$uid, $rel] = [$m[1], trim($m[2] ?? '', '/')]; + $userFolder = $this->root->getUserFolder($uid); + $repo = $this->targets->repository($target); + $source = 'data/' . $uid . '/files' . ($rel === '' ? '' : '/' . $rel); + + $entries = []; + $single = false; + foreach ($repo->entries($snapshotId) as $e) { + if ($e['p'] === $source) { + $single = true; + $entries[''] = $e; + break; + } + if (str_starts_with($e['p'], $source . '/')) { + $entries[substr($e['p'], strlen($source) + 1)] = $e; + } + } + if ($entries === []) { + throw new NotFoundException("Nothing at $source in snapshot $snapshotId"); + } + + // Where to write. new-folder mirrors the path under "Restored " in the user's root. + $parentRel = $single ? (dirname($rel) === '.' ? '' : dirname($rel)) : $rel; + if ($mode === self::MODE_NEW_FOLDER) { + $parentRel = trim('Restored ' . date('Y-m-d H.i') . '/' . $parentRel, '/'); + } elseif ($mode !== self::MODE_MERGE && $mode !== self::MODE_REPLACE) { + throw new \InvalidArgumentException("Unknown mode $mode"); + } + $dest = $this->ensureFolder($userFolder, $parentRel); + $names = $single ? [basename($rel) => $entries['']] : $entries; + + $restored = 0; + foreach ($names as $relPath => $entry) { + $this->writeFile($repo, $dest, (string)$relPath, $entry); + $restored++; + if ($progress !== null) { + $progress($restored, count($names), (string)$relPath); + } + } + + $trashed = 0; + if ($mode === self::MODE_REPLACE && !$single) { + $trashed = $this->trashExtras($dest, '', array_flip(array_keys($names))); + } + return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())]; + } + + /** Raw restore of any snapshot path to a local directory (administrators, disaster recovery). */ + public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory): int { + if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) { + throw new \RuntimeException("Cannot create $directory"); + } + return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/'); + } + + private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void { + $dir = dirname($relPath); + $folder = $dir === '.' ? $dest : $this->ensureFolder($dest, $dir); + $name = basename($relPath); + // Stage under a temporary name; only rename into place after every blob was verified. + $tmpName = '.' . $name . '.ngb-restore-' . bin2hex(random_bytes(3)); + $tmp = $folder->newFile($tmpName); + try { + $out = $tmp->fopen('wb'); + $written = 0; + foreach ($repo->readBlobs($entry['b']) as $data) { // each blob is verified on read + if (fwrite($out, $data) !== strlen($data)) { + throw new \RuntimeException('Write failed for ' . $relPath); + } + $written += strlen($data); + } + if (!fclose($out)) { + throw new \RuntimeException('Write failed for ' . $relPath); + } + if ($written !== $entry['s']) { + throw new \RuntimeException("Size mismatch restoring $relPath"); + } + if ($folder->nodeExists($name)) { + $existing = $folder->get($name); + if (!$existing instanceof File) { + throw new \RuntimeException("$relPath exists and is not a file"); + } + // Overwrite through the Files API: the current content becomes a version. + $in = $tmp->fopen('rb'); + $existing->putContent($in); + if (is_resource($in)) { + fclose($in); + } + $tmp->delete(); + $existing->touch($entry['m']); + } else { + $tmp->move($folder->getPath() . '/' . $name); + $folder->get($name)->touch($entry['m']); + } + } catch (\Throwable $e) { + try { + $tmp->delete(); + } catch (\Throwable) { + } + throw $e; + } + } + + private function trashExtras(Folder $folder, string $prefix, array $keep): int { + $count = 0; + foreach ($folder->getDirectoryListing() as $node) { + $rel = $prefix === '' ? $node->getName() : $prefix . '/' . $node->getName(); + if ($node instanceof Folder) { + $hasKept = false; + foreach (array_keys($keep) as $k) { + if (str_starts_with((string)$k, $rel . '/')) { + $hasKept = true; + break; + } + } + if ($hasKept) { + $count += $this->trashExtras($node, $rel, $keep); + } else { + $node->delete(); // to the trash + $count++; + } + } elseif (!isset($keep[$rel])) { + $node->delete(); + $count++; + } + } + return $count; + } + + private function ensureFolder(Folder $base, string $path): Folder { + $path = trim($path, '/'); + if ($path === '' || $path === '.') { + return $base; + } + if ($base->nodeExists($path)) { + $node = $base->get($path); + if (!$node instanceof Folder) { + throw new \RuntimeException("$path exists and is not a folder"); + } + return $node; + } + return $base->newFolder($path); + } +} diff --git a/tests/.phpunit.result.cache b/tests/.phpunit.result.cache index b5f6c65..82bfc1b 100644 --- a/tests/.phpunit.result.cache +++ b/tests/.phpunit.result.cache @@ -1 +1 @@ -{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.308,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.19,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.002,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.24,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.371,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.836,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.941,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":0.989,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.126,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.068,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.097,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.078,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.166,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.189,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.529,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":3.034,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.004,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\IndexCacheTest::testSecondOpenDownloadsNoIndexFiles":2.497}} \ No newline at end of file +{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.129,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.126,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.005,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.004,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.004,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.007,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.334,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.428,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.902,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.951,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.018,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.184,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.001,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.027,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.056,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.074,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.258,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.514,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.848,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.006,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.008,"OCA\\NgBackup\\Tests\\Unit\\Repository\\IndexCacheTest::testSecondOpenDownloadsNoIndexFiles":2.618}} \ No newline at end of file