From 0b4246b1b3d4fa83bda009a84de379251e0a8d97 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Thu, 1 Oct 2026 18:36:57 +0200 Subject: [PATCH] Phase 0: per-user backup/restore through user_migration (spike E) RepositoryExportDestination / RepositoryImportSource implement the public OCP\UserMigration interfaces on top of the repository, so user_migration's migrators (calendar, contacts, trashbin, account, files) export straight into the encrypted repository. Spike E (NC34): export 0.8s, unchanged second export stores nothing; after deleting the user, import restores account, files (bit-identical), calendar event and contact. Findings: calendars/address books come back as 'migrated-*' with new object URIs; importing over an existing user duplicates contacts and adds a migrated birthday calendar; creating the user on import needs >128 MB (password_policy word list). --- lib/Repository/Repository.php | 10 ++ .../RepositoryExportDestination.php | 94 ++++++++++++ lib/UserMigration/RepositoryImportSource.php | 120 +++++++++++++++ spikes/05-user-migration.php | 145 ++++++++++++++++++ 4 files changed, 369 insertions(+) create mode 100644 lib/UserMigration/RepositoryExportDestination.php create mode 100644 lib/UserMigration/RepositoryImportSource.php create mode 100644 spikes/05-user-migration.php diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index ecab14d..d08e7aa 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -180,6 +180,16 @@ final class Repository { } } + /** + * @param list $blobIds + * @return \Generator verified blob contents, in order + */ + public function readBlobs(array $blobIds): \Generator { + foreach ($blobIds as $id) { + yield $this->loadBlob($id); + } + } + /** Small encrypted objects (manifests). */ public function putObject(string $path, string $data): void { self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path)); diff --git a/lib/UserMigration/RepositoryExportDestination.php b/lib/UserMigration/RepositoryExportDestination.php new file mode 100644 index 0000000..41d59db --- /dev/null +++ b/lib/UserMigration/RepositoryExportDestination.php @@ -0,0 +1,94 @@ +/) when close() is called. + */ +final class RepositoryExportDestination implements IExportDestination { + /** @var array, s?:int}> path => entry */ + private array $entries = []; + private array $versions = []; + private array $stats = ['newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0]; + private string $exportId; + + public function __construct( + private Repository $repo, + private string $uid, + ) { + $this->exportId = gmdate('Ymd\THis\Z') . '-' . bin2hex(random_bytes(4)); + } + + public function addFileContents(string $path, string $content): void { + $writer = $this->repo->blobWriter($this->stats); + $writer->write($content); + $this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => strlen($content)]; + } + + public function addFileAsStream(string $path, $stream): void { + $writer = $this->repo->blobWriter($this->stats); + while (!feof($stream)) { + $chunk = fread($stream, 1048576); + if ($chunk === false) { + break; + } + $writer->write($chunk); + } + $this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => $writer->bytes()]; + } + + public function copyFolder(Folder $folder, string $destinationPath, ?callable $nodeFilter = null): void { + $base = self::norm($destinationPath); + $this->entries[$base] = ['t' => 'd']; + foreach ($folder->getDirectoryListing() as $node) { + if ($nodeFilter !== null && !$nodeFilter($node)) { + continue; + } + $target = $base . '/' . $node->getName(); + if ($node instanceof Folder) { + $this->copyFolder($node, $target, $nodeFilter); + } elseif ($node instanceof File) { + $stream = $node->fopen('rb'); + if ($stream === false) { + throw new \RuntimeException('Cannot read ' . $node->getPath()); + } + $this->addFileAsStream($target, $stream); + fclose($stream); + } + } + } + + public function setMigratorVersions(array $versions): void { + $this->versions = $versions; + } + + public function close(): void { + $this->repo->flushPacks(); + $manifest = ['uid' => $this->uid, 'time' => gmdate('c'), 'versions' => $this->versions, 'entries' => $this->entries]; + $this->repo->putObject($this->manifestPath(), json_encode($manifest, JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR)); + } + + public function manifestPath(): string { + return 'users/' . $this->uid . '/' . $this->exportId; + } + + public function stats(): array { + return $this->stats; + } + + private static function norm(string $path): string { + return trim($path, '/'); + } +} diff --git a/lib/UserMigration/RepositoryImportSource.php b/lib/UserMigration/RepositoryImportSource.php new file mode 100644 index 0000000..4876fbd --- /dev/null +++ b/lib/UserMigration/RepositoryImportSource.php @@ -0,0 +1,120 @@ +manifest = json_decode($repo->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR); + } + + public function getFileContents(string $path): string { + $stream = $this->getFileAsStream($path); + $data = (string)stream_get_contents($stream); + fclose($stream); + return $data; + } + + public function getFileAsStream(string $path) { + $entry = $this->entry($path, 'f'); + // Phase 0: buffer in php://temp (memory up to 8 MiB, then a temp file for this one file). + $stream = fopen('php://temp/maxmemory:' . (8 * 1048576), 'w+b'); + foreach ($this->repo->readBlobs($entry['b']) as $data) { + fwrite($stream, $data); + } + rewind($stream); + return $stream; + } + + public function getFolderListing(string $path): array { + $prefix = trim($path, '/'); + $prefix = $prefix === '' ? '' : $prefix . '/'; + $names = []; + foreach (array_keys($this->manifest['entries']) as $p) { + if ($prefix === '' || str_starts_with($p, $prefix)) { + $rest = substr($p, strlen($prefix)); + if ($rest !== '' && !str_contains($rest, '/')) { + $names[] = $rest; + } + } + } + sort($names); + return $names; + } + + public function pathExists(string $path): bool { + $p = trim($path, '/'); + if (isset($this->manifest['entries'][$p])) { + return true; + } + foreach (array_keys($this->manifest['entries']) as $k) { + if (str_starts_with($k, $p . '/')) { + return true; + } + } + return false; + } + + public function copyToFolder(Folder $destination, string $sourcePath): void { + foreach ($this->getFolderListing($sourcePath) as $name) { + $path = trim($sourcePath, '/') . '/' . $name; + $entry = $this->manifest['entries'][$path] ?? ['t' => 'd']; + if ($entry['t'] === 'd') { + $sub = $destination->nodeExists($name) ? $destination->get($name) : $destination->newFolder($name); + if (!$sub instanceof Folder) { + throw new UserMigrationException("$path exists and is not a folder"); + } + $this->copyToFolder($sub, $path); + } else { + $stream = $this->getFileAsStream($path); + if ($destination->nodeExists($name)) { + $destination->get($name)->putContent($stream); + } else { + $destination->newFile($name, $stream); + } + if (is_resource($stream)) { + fclose($stream); + } + } + } + } + + public function getMigratorVersions(): array { + return $this->manifest['versions']; + } + + public function getMigratorVersion(string $migrator): ?int { + return $this->manifest['versions'][$migrator] ?? null; + } + + public function getOriginalUid(): string { + return $this->manifest['uid']; + } + + public function close(): void { + } + + /** @return array{t:string, b:list, s:int} */ + private function entry(string $path, string $type): array { + $entry = $this->manifest['entries'][trim($path, '/')] ?? null; + if ($entry === null || $entry['t'] !== $type) { + throw new UserMigrationException('Path not found in backup: ' . $path); + } + return $entry; + } +} diff --git a/spikes/05-user-migration.php b/spikes/05-user-migration.php new file mode 100644 index 0000000..73cff3c --- /dev/null +++ b/spikes/05-user-migration.php @@ -0,0 +1,145 @@ +get($uid)) { + $old->delete(); +} + +// 1. Test user with files, a calendar event, a contact and profile data. +$user = $users->createUser($uid, bin2hex(random_bytes(12))); +$user->setDisplayName('Alice Test'); +$user->setSystemEMailAddress('alice@example.com'); +$home = $root->getUserFolder($uid); +$docs = $home->newFolder('Projecten/2026'); +$docs->newFile('plan.txt', "Plan voor NG Backup\n"); +$big = fopen('php://temp', 'w+b'); +for ($i = 0; $i < 20; $i++) { + fwrite($big, random_bytes(1048576)); +} +rewind($big); +$home->newFile('Projecten/video.bin', $big); +$calId = $caldav->createCalendar($principal, 'werk', ['{DAV:}displayname' => 'Werk']); +$caldav->createCalendarObject($calId, 'ev1.ics', "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//ngb//spike//EN\r\nBEGIN:VEVENT\r\nUID:ngb-spike-1\r\nDTSTAMP:20261001T120000Z\r\nDTSTART:20261002T090000Z\r\nDTEND:20261002T100000Z\r\nSUMMARY:Overleg backup\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"); +$abId = $carddav->createAddressBook($principal, 'prive', ['{DAV:}displayname' => 'Privé']); +$carddav->createCard($abId, 'c1.vcf', "BEGIN:VCARD\r\nVERSION:3.0\r\nUID:ngb-spike-c1\r\nFN:Bob Voorbeeld\r\nEMAIL:bob@example.com\r\nEND:VCARD\r\n"); + +$state = static function () use ($users, $root, $caldav, $carddav, $uid, $principal): array { + $u = $users->get($uid); + // Read from the data directory: IRootFolder caches the folder of a deleted user. + $dir = \OCP\Server::get(\OCP\IConfig::class)->getSystemValueString('datadirectory') . '/' . $uid . '/files'; + $files = []; + if (is_dir($dir)) { + $it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS)); + foreach ($it as $f) { + if ($f->isFile()) { + $files[substr($f->getPathname(), strlen($dir) + 1)] = hash_file('sha256', $f->getPathname()); + } + } + } + ksort($files); + $events = []; + foreach ($caldav->getCalendarsForUser($principal) as $cal) { + foreach ($caldav->getCalendarObjects($cal['id']) as $o) { + $events[] = $cal['uri'] . '/' . $o['uri']; + } + } + $cards = []; + foreach ($carddav->getAddressBooksForUser($principal) as $ab) { + foreach ($carddav->getCards($ab['id']) as $c) { + $cards[] = $ab['uri'] . '/' . $c['uri']; + } + } + sort($events); + sort($cards); + return ['display' => $u?->getDisplayName(), 'email' => $u?->getSystemEMailAddress(), 'files' => $files, 'events' => $events, 'cards' => $cards]; +}; +$before = $state(); +printf("user %s: %d files, %d events, %d cards\n", $uid, count($before['files']), count($before['events']), count($before['cards'])); +printf("migrators: %s\n", implode(', ', array_map(fn ($m) => $m->getId(), $migration->getMigrators()))); + +// 2. Export into the repository. +$repoDir = '/tmp/ngb-e-repo'; +exec('rm -rf ' . $repoDir); +$repo = Repository::init(new LocalBackend($repoDir), 'spike passphrase'); +memory_reset_peak_usage(); +$t = microtime(true); +$dest = new RepositoryExportDestination($repo, $uid); +$migration->export($dest, $users->get($uid)); +printf("export: %.1fs, peak %.1f MiB, %s\n", microtime(true) - $t, memory_get_peak_usage() / 1048576, json_encode($dest->stats())); +$manifest = $dest->manifestPath(); +$check('export manifest written', in_array($manifest, array_map(fn ($p) => $p, (new LocalBackend($repoDir))->list('users')), true)); + +// Second export without changes: blobs are reused. +$dest2 = new RepositoryExportDestination($repo, $uid); +$migration->export($dest2, $users->get($uid)); +printf("second export: %s\n", json_encode($dest2->stats())); +$check('unchanged second export stores (almost) nothing', $dest2->stats()['uploaded'] < 0.05 * $dest->stats()['uploaded']); + +// 3. Delete the user and restore from the repository into a new account. +$users->get($uid)->delete(); +$check('user deleted', $users->get($uid) === null); +$repo = Repository::open(new LocalBackend($repoDir), 'spike passphrase'); +memory_reset_peak_usage(); +$t = microtime(true); +$migration->import(new RepositoryImportSource($repo, $manifest)); +printf("import: %.1fs, peak %.1f MiB\n", microtime(true) - $t, memory_get_peak_usage() / 1048576); +$after = $state(); +$check('restored: display name and e-mail', $after['display'] === $before['display'] && $after['email'] === $before['email']); +$check('restored: all files identical', $after['files'] === $before['files']); +$check('restored: calendar event', count($after['events']) === count($before['events'])); +$check('restored: contact', count($after['cards']) === count($before['cards'])); +if ($after !== $before) { + echo json_encode(['before' => $before, 'after' => $after], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n"; +} + +// 4. Restore over the existing (now restored) user: what happens? +try { + $migration->import(new RepositoryImportSource($repo, $manifest), $users->get($uid)); + $again = $state(); + printf("import over existing user: files %d -> %d, events %d -> %d, cards %d -> %d\n", + count($after['files']), count($again['files']), count($after['events']), count($again['events']), count($after['cards']), count($again['cards'])); + echo 'calendars now: ' . implode(', ', array_map(fn ($c) => $c['uri'], $caldav->getCalendarsForUser($principal))) . "\n"; +} catch (\Throwable $e) { + printf("import over existing user fails: %s: %s\n", get_class($e), $e->getMessage()); +} + +if (getenv("KEEP") !== "1") { + $users->get($uid)?->delete(); +} +exec('rm -rf ' . $repoDir); +echo $ok ? "ALL PASS\n" : "FAILURES\n"; +exit($ok ? 0 : 1);