From 101f954c5dfca7852877326a01c3905f8192d5a5 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Thu, 1 Oct 2026 18:39:26 +0200 Subject: [PATCH] Phase 0: resumable time-boxed backup steps (spike F) and results BackupRun is a resumable state machine: each step works until a deadline, uploads its pack and tree segment, and returns the state; large files are resumed at the next blob. Snapshot is written last. Spike F: 200 MiB file resumed across steps in separate processes; with 6 random kill -9 the run completes and restores bit-identically. spikes/RESULTS.md summarises phase 0 and the findings for the plan. --- lib/Job/BackupRun.php | 155 ++++++++++++++++++++++++++++++++++ lib/Repository/Repository.php | 33 ++++++++ spikes/06-resumable.php | 82 ++++++++++++++++++ spikes/RESULTS.md | 47 +++++++++++ 4 files changed, 317 insertions(+) create mode 100644 lib/Job/BackupRun.php create mode 100644 spikes/06-resumable.php create mode 100644 spikes/RESULTS.md diff --git a/lib/Job/BackupRun.php b/lib/Job/BackupRun.php new file mode 100644 index 0000000..3e2f642 --- /dev/null +++ b/lib/Job/BackupRun.php @@ -0,0 +1,155 @@ + bin2hex(random_bytes(16)), + 'source' => rtrim($source, '/'), + 'parent' => $parent, + 'label' => $label, + 'started' => gmdate('c'), + 'phase' => 'scan', + 'files' => [], + 'pos' => 0, + 'cur' => null, + 'segments' => [], + 'steps' => 0, + 'stats' => ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0], + ]; + } + + /** Run one step until $deadline (microtime(true)); returns the new state. */ + public static function step(Repository $repo, array $state, float $deadline): array { + $state['steps']++; + if ($state['phase'] === 'scan') { + $state['files'] = self::scan($state['source']); + $state['phase'] = 'files'; + // Scanning can take a while on big trees; continue in the same step if time is left. + } + + if ($state['phase'] === 'files') { + $previous = $state['parent'] !== null ? $repo->snapshotEntries($state['parent']) : []; + $stats = $state['stats']; + $lines = []; + $total = count($state['files']); + while ($state['pos'] < $total) { + $relative = $state['files'][$state['pos']]; + $path = $state['source'] . '/' . $relative; + clearstatcache(true, $path); + $size = @filesize($path); + $mtime = @filemtime($path); + if ($size === false) { + // Deleted since the scan: skip. + $state['pos']++; + $state['cur'] = null; + continue; + } + $prev = $previous[$relative] ?? null; + if ($state['cur'] === null && $prev !== null && $prev['s'] === $size && $prev['m'] === $mtime) { + $lines[] = self::line($relative, $size, $mtime, $prev['b']); + $stats['files']++; + $stats['reused']++; + $state['pos']++; + continue; + } + + $cur = $state['cur'] ?? ['path' => $relative, 'offset' => 0, 'blobs' => []]; + $fh = fopen($path, 'rb'); + fseek($fh, $cur['offset']); + $finished = true; + while (($data = self::readFull($fh, Repository::BLOB_SIZE)) !== '') { + $cur['blobs'][] = $repo->storeData($data, $stats); + $cur['offset'] += strlen($data); + if (microtime(true) >= $deadline && !feof($fh)) { + $finished = false; + break; + } + } + fclose($fh); + if (!$finished) { + $state['cur'] = $cur; + break; + } + $lines[] = self::line($relative, $size, $mtime, $cur['blobs']); + $stats['files']++; + $stats['read']++; + $state['cur'] = null; + $state['pos']++; + if (microtime(true) >= $deadline) { + break; + } + } + + // Persist this step's tree lines as a blob stream segment and upload the open pack. + if ($lines !== []) { + $writer = $repo->blobWriter($stats); + $writer->write(implode("\n", $lines) . "\n"); + $state['segments'][] = $writer->finish(); + } + $repo->flushPacks(); + $state['stats'] = $stats; + if ($state['pos'] >= $total) { + $state['phase'] = 'finish'; + } + } + + if ($state['phase'] === 'finish' && microtime(true) < $deadline + 5) { + $segments = $state['segments']; + $repo->writeSnapshot($state['snapshot'], + ['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'], 'source' => $state['source']], + (function () use ($repo, $segments) { + foreach ($segments as $blobs) { + yield from $repo->readLines($blobs); + } + })()); + $state['phase'] = 'done'; + $state['files'] = []; + } + return $state; + } + + /** @return list */ + private static function scan(string $root): array { + $files = []; + $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS)); + foreach ($it as $file) { + if ($file->isFile()) { + $files[] = substr($file->getPathname(), strlen($root) + 1); + } + } + sort($files, SORT_STRING); + return $files; + } + + private static function line(string $path, int $size, int $mtime, array $blobs): string { + return json_encode(['p' => $path, 's' => $size, 'm' => $mtime, 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); + } + + private static function readFull($fh, int $length): string { + $buffer = ''; + while (strlen($buffer) < $length && !feof($fh)) { + $buffer .= (string)fread($fh, $length - strlen($buffer)); + } + return $buffer; + } +} diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index d08e7aa..90f3b0e 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -180,6 +180,39 @@ final class Repository { } } + /** Store one blob of file data in the current run (deduplicated); returns its id. */ + public function storeData(string $data, array &$stats): string { + $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index); + return $this->storeBlob($data, $this->runPacks, $stats); + } + + /** + * Write a snapshot from a meta record and the tree lines (JSON, one per file). Must be called + * after flushPacks(), so the snapshot only references stored blobs. + * + * @param iterable $lines + */ + public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void { + $tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); + fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n"); + foreach ($lines as $line) { + if ($line !== '') { + fwrite($tree, $line . "\n"); + } + } + rewind($tree); + $encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); + $this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId); + fclose($tree); + rewind($encrypted); + $this->backend->put('snapshots/' . $snapshotId, $encrypted); + } + + /** @return array}> */ + public function snapshotEntries(string $snapshotId): array { + return $this->loadEntries($snapshotId); + } + /** * @param list $blobIds * @return \Generator verified blob contents, in order diff --git a/spikes/06-resumable.php b/spikes/06-resumable.php new file mode 100644 index 0000000..94e9d3e --- /dev/null +++ b/spikes/06-resumable.php @@ -0,0 +1,82 @@ + +// php 06-resumable.php step (repeat until it prints "done") +// php 06-resumable.php verify + +require __DIR__ . '/bootstrap.php'; + +use OCA\NgBackup\Backend\LocalBackend; +use OCA\NgBackup\Job\BackupRun; +use OCA\NgBackup\Repository\Repository; + +[$_, $mode, $work] = $argv + [null, null, '/tmp/ngb-f']; +$src = "$work/src"; +$stateFile = "$work/state.json"; +$pass = 'spike passphrase'; + +$hashTree = static function (string $root): array { + $out = []; + $it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root, FilesystemIterator::SKIP_DOTS)); + foreach ($it as $f) { + if ($f->isFile()) { + $out[substr($f->getPathname(), strlen($root) + 1)] = hash_file('sha256', $f->getPathname()); + } + } + ksort($out); + return $out; +}; + +switch ($mode) { + case 'prepare': + exec('rm -rf ' . escapeshellarg($work)); + mkdir("$src/docs", 0700, true); + for ($i = 0; $i < 400; $i++) { + file_put_contents(sprintf('%s/docs/n%03d.txt', $src, $i), random_bytes(30000) . str_repeat('x', 30000)); + } + $fh = fopen("$src/huge.bin", 'wb'); + for ($i = 0; $i < 200; $i++) { + fwrite($fh, random_bytes(1048576)); + } + fclose($fh); + Repository::init(new LocalBackend("$work/repo"), $pass); + file_put_contents("$work/expected.json", json_encode($hashTree($src))); + file_put_contents($stateFile, json_encode(BackupRun::start($src, null, 'resumable'))); + echo "prepared\n"; + break; + + case 'step': + $seconds = (float)($argv[3] ?? 1.5); + $state = json_decode(file_get_contents($stateFile), true); + if ($state['phase'] === 'done') { + echo "done\n"; + break; + } + $repo = Repository::open(new LocalBackend("$work/repo"), $pass); + $t = microtime(true); + $state = BackupRun::step($repo, $state, $t + $seconds); + // Save the state atomically: a kill before this point means the step is simply redone. + file_put_contents("$stateFile.tmp", json_encode($state)); + rename("$stateFile.tmp", $stateFile); + printf("step %d: %.1fs, phase %s, file %d/%d%s, peak %.0f MiB\n", $state['steps'], microtime(true) - $t, $state['phase'], + $state['pos'], max(count($state['files']), $state['pos']), + $state['cur'] ? sprintf(' (in %s at %d MiB)', $state['cur']['path'], $state['cur']['offset'] / 1048576) : '', + memory_get_peak_usage() / 1048576); + if ($state['phase'] === 'done') { + echo 'done ' . json_encode($state['stats']) . "\n"; + } + break; + + case 'verify': + $state = json_decode(file_get_contents($stateFile), true); + $repo = Repository::open(new LocalBackend("$work/repo"), $pass); + exec('rm -rf ' . escapeshellarg("$work/restore")); + $repo->restore($state['snapshot'], "$work/restore"); + $same = $hashTree("$work/restore") === json_decode(file_get_contents("$work/expected.json"), true); + printf("%-60s %s\n", 'restored tree identical after resumable run with kills', $same ? 'PASS' : 'FAIL'); + exec('rm -rf ' . escapeshellarg($work)); + exit($same ? 0 : 1); +} diff --git a/spikes/RESULTS.md b/spikes/RESULTS.md new file mode 100644 index 0000000..06143a5 --- /dev/null +++ b/spikes/RESULTS.md @@ -0,0 +1,47 @@ +# Phase 0 — results (2026-10-01) + +All spikes run on the standard `nextcloud:34-apache` image (PHP 8.5, sodium 1.0.18), without +`pg_dump`, `mysqldump`, the `pgsql`/`mysqli` extensions, rclone or a shell on the target. + +| Spike | What | Result | +|---|---|---| +| A `01-stream-crypto.php` | Streaming encryption (secretstream XChaCha20-Poly1305), key wrapping (Argon2id) | 1 GiB round trip with `memory_limit=64M`: peak < 2 MiB, 160 MiB/s encrypt, 122 MiB/s decrypt, 0.03 % overhead. Tampering, truncation (also at a frame boundary), trailing data, wrong key/AD/passphrase rejected. | +| B `02-repository.php` | Repository format: 4 MiB blobs, deflate, packs, index, snapshots, dedup, incremental, restore | 390 MiB tree, peak 54 MiB. Unchanged run: 0.06 s, nothing read or uploaded. 1 MiB change in a 300 MiB file: one blob uploaded. Full and partial restore bit-identical, restore peak 17 MiB. Tampered pack detected. | +| C `03-db-dump.php` | Logical DB dump/restore via the Nextcloud connection (PostgreSQL) | 300k-row table with bytea (110 MB): dump 6.3 s, peak 50 MiB, 94 MiB JSON → 20 MiB stored. Unchanged second dump stores nothing (per-table blob streams). Restore 46 s: every table checksum-identical, all 110 sequences correct. | +| D `04-external-storage.php` | Repository on files_external storages without a mount | S3 (SeaweedFS) and SFTP: 120 MiB backup ~6 s, unchanged run uploads nothing, restore bit-identical, no row in `external_mounts`, /tmp stayed at 12 KiB (no local copy). Also works with files_external disabled when its classes are loaded. | +| E `05-user-migration.php` | Per-user backup/restore through user_migration with own `IExportDestination`/`IImportSource` | Migrators: calendar, contacts, trashbin, account, files. Export 0.8 s, unchanged re-export stores nothing. After deleting the user, import restores account, files (bit-identical), event and contact. | +| F `06-resumable.php` | Time-boxed, resumable steps in separate processes | 200 MiB file resumed mid-file across steps; with 6 random `kill -9` the run still completes and restores bit-identically; peak 29–53 MiB per step. | + +## Findings that change or refine the plan + +1. **Restore over an existing user (user_migration)** is not clean: contacts are duplicated and a + `migrated-contact_birthdays` calendar is added; calendars/address books come back as + `migrated-*` with new object URIs (clients resync; shares/subscriptions of those calendars + break). → Per-user restore in phase 2 needs its own strategy: restore into a fresh account + (after deleting or renaming the existing one, with a safety export first), or restore + calendars/contacts at DAV level ourselves and use user_migration only for the rest. +2. **files_external without enabling it:** its classes can be loaded while the app is disabled + (`OC_App::loadApp`, private API). Choice to make: rely on that (nothing visible to anyone), or + require files_external enabled with user mounting off (public path, a bit more visible). +3. **Peak memory on S3/SFTP backups is 85–89 MiB** (local target 54 MiB): the files_external + upload path buffers about one extra pack. Fine within 512 MB; to look at in phase 1 + (smaller packs on hosted, or our own S3 multipart adapter). +4. **DB restore speed:** row-by-row INSERT (46 s for 300k rows). Phase 1: multi-row INSERT, + skip tables whose checksum matches the dump, disable/re-enable nothing (keep it portable). +5. **Fixed-size chunking:** a row change in a big table (e.g. `filecache`) or an insertion in a big + file shifts all following blobs. Phase 1: content-defined chunking (FastCDC) for files and dumps. +6. **Index and snapshot entries are held in memory** in phase 0. Phase 1: index as a cache table in + the Nextcloud database (rebuildable from `index/` and the pack headers), streaming comparison + with the parent snapshot. +7. **user creation on import needs > 128 MB** (password_policy word list) — document 512 MB as minimum. +8. **Snowflake ids:** since NC 33 some tables (e.g. `oc_jobs`) have no sequence; the dump/restore + handles that (no special case needed), the test just must not assume sequences everywhere. +9. **files_external S3 `writeStream()` closes the stream it is given** → `IBackend::put()` owns and + closes the stream (contract documented). + +## Not done in phase 0 + +- MySQL/MariaDB and SQLite runs of spike C (PostgreSQL only so far). +- Real hosted environment (webcron/AJAX cron, `max_execution_time`); spike F simulates it. +- Restore of the database while Nextcloud is running (spike C restored with Nextcloud idle). +- Schema introspection still uses the Doctrine connection behind `IDBConnection` (private API).