From c9f82a79e48447aa9ab0c644c2d459ca59fdfba7 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Tue, 6 Oct 2026 00:44:23 +0200 Subject: [PATCH] Phase 2: retention for per-user exports (occ backup:retention --user-last) PruneService::userExportRetention()/setUserExportRetention(): a single, app-wide "keep the newest N" setting (default 3, same default as the snapshot policy's "last"), deliberately simpler than the full daily/weekly/monthly RetentionPolicy -- per-user exports are on-demand only (occ backup:backup:user, see the fold-out task about whether they ever join the automatic schedule), so there is no regular cadence yet to bucket by day/week/month. PruneService::applyUserExports(): groups a location's user exports by uid (read from each manifest's own 'time' field, the same field snapshot retention already reads), keeps the newest N per user, forgets the rest. No trash, unlike a snapshot: these are not yet the primary safety net the deletion delay protects. Runs under the same exclusive lock as snapshot retention. occ backup:retention gained --user-last=N; occ backup:prune now also applies it for every target it processes, reporting counts the same way it already does for snapshots and blob packs. Verified live against the dev rig (four real exports across four users, confirmed exactly the expected ones were kept/forgotten at two different --user-last values) and added coverage to tests/integration/user-roundtrip.sh (an account's manual export plus the automatic safety-export a replace-mode restore makes of it, pruned down to --user-last=2). --- PLAN.md | 5 ++- lib/Command/Prune.php | 14 +++++++ lib/Command/Retention.php | 12 +++++- lib/Service/PruneService.php | 64 +++++++++++++++++++++++++++++ tests/integration/user-roundtrip.sh | 12 ++++++ 5 files changed, 105 insertions(+), 2 deletions(-) diff --git a/PLAN.md b/PLAN.md index 53ff3a6..3709d3d 100644 --- a/PLAN.md +++ b/PLAN.md @@ -346,7 +346,10 @@ minimale volledige restore (`occ backup:restore:full` op een verse installatie) SMB vereist de `smbclient` PECL-extensie (niet standaard in de Nextcloud-image, in CI nu on-the-fly gebouwd). NFS is geen eigen backend: lokaal mounten + `local`-backend erop richten. 3. Verificatie/checksum-audit-commando — **klaar** (2026-10-05) -4. Retentie/prune voor user-exports +4. Retentie/prune voor user-exports — **klaar** (2026-10-06): `occ backup:retention --user-last=N` + (standaard 3, per installatie, net als de bestaande snapshot-policy), toegepast door + `occ backup:prune` (geen prullenbak voor user-exports, in tegenstelling tot snapshots — on-demand, + niet het primaire vangnet). 5. Automatische user-export inpluggen in het reguliere schema (of expliciet besluit dat het on-demand blijft) 9. SMB/NFS intern testen op de eigen OMV-instantie — nog open (intern, niet in CI) diff --git a/lib/Command/Prune.php b/lib/Command/Prune.php index aaafc3e..6b5bf3b 100644 --- a/lib/Command/Prune.php +++ b/lib/Command/Prune.php @@ -37,6 +37,20 @@ final class Prune extends Command { $output->writeln('Policy: ' . json_encode($this->prune->policy()->toArray()) . ($dry ? ' (dry run)' : '')); $rc = 0; foreach ($targets as $t) { + // User exports first: Repository::prune()'s blob garbage collection (run as part of + // apply() below) scans the users/ manifests still on the location at that point, so + // forgetting one here lets its now-unused blobs be freed in this same pass. + try { + $ru = $this->prune->applyUserExports($t, $dry); + } catch (\Throwable $e) { + $output->writeln('' . $t->getName() . ' (user exports): ' . $e->getMessage() . ''); + $rc = 1; + continue; + } + if ($ru['forgotten'] !== []) { + $output->writeln(sprintf('%s: %s %d user export(s) for %d user(s)', + $t->getName(), $dry ? 'would forget' : 'forgot', count($ru['forgotten']), count($ru['kept']))); + } try { $r = $this->prune->apply($t, $dry); } catch (\Throwable $e) { diff --git a/lib/Command/Retention.php b/lib/Command/Retention.php index 9e205e5..68c60c6 100644 --- a/lib/Command/Retention.php +++ b/lib/Command/Retention.php @@ -28,7 +28,8 @@ final class Retention extends Command { ->addOption('daily', null, InputOption::VALUE_REQUIRED, 'Keep one per day for the last N days with a backup') ->addOption('weekly', null, InputOption::VALUE_REQUIRED, 'Keep one per week for N weeks') ->addOption('monthly', null, InputOption::VALUE_REQUIRED, 'Keep one per month for N months') - ->setHelp("Example: occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12"); + ->addOption('user-last', null, InputOption::VALUE_REQUIRED, 'Keep the newest N of each user\'s own exports (occ backup:backup:user)') + ->setHelp("Example: occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12 --user-last=3"); } protected function execute(InputInterface $input, OutputInterface $output): int { @@ -48,7 +49,16 @@ final class Retention extends Command { return 1; } } + if ($input->getOption('user-last') !== null) { + try { + $this->prune->setUserExportRetention((int)$input->getOption('user-last')); + } catch (\InvalidArgumentException $e) { + $output->writeln('' . $e->getMessage() . ''); + return 1; + } + } $output->writeln(sprintf('Keep the last %d, plus one per day for %d days, per week for %d weeks, per month for %d months.', $p['last'], $p['daily'], $p['weekly'], $p['monthly'])); + $output->writeln(sprintf('Keep the newest %d of each user\'s own exports.', $this->prune->userExportRetention())); return 0; } } diff --git a/lib/Service/PruneService.php b/lib/Service/PruneService.php index 2c78399..284de2c 100644 --- a/lib/Service/PruneService.php +++ b/lib/Service/PruneService.php @@ -111,6 +111,70 @@ final class PruneService { return self::lockKey($target) . '/catalog'; } + /** How many of each user's own exports (occ backup:backup:user) to keep, newest first. */ + public function userExportRetention(): int { + return $this->appConfig->getValueInt(Application::APP_ID, 'user_export_retention', 3); + } + + public function setUserExportRetention(int $keep): void { + if ($keep < 1) { + throw new \InvalidArgumentException('Keep at least the last export'); + } + $old = $this->userExportRetention(); + $this->appConfig->setValueInt(Application::APP_ID, 'user_export_retention', $keep); + if ($old !== $keep) { + $this->alerts->securityEvent('user_export_retention_changed', ['keep' => (string)$keep]); + } + } + + /** + * Forgets a user's older exports once more than userExportRetention() exist for them, + * newest first; no trash, unlike a full snapshot (occ backup:backup:user is on-demand, not + * yet the primary safety net the deletion delay protects). + * + * @return array{kept: array>, forgotten: list} + */ + public function applyUserExports(Target $target, bool $dryRun = false): array { + if ($target->getAppendOnly()) { + throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there'); + } + $lease = $this->lockExclusive($target, 1800); + try { + $repo = $this->targets->repository($target); + $keep = $this->userExportRetention(); + $byUser = []; + foreach ($repo->listUserExports() as $path) { + $uid = explode('/', $path)[1] ?? ''; + $manifest = json_decode($repo->getObject($path), true, 512, JSON_THROW_ON_ERROR); + $byUser[$uid][$path] = (int)strtotime((string)($manifest['time'] ?? '@0')); + } + $kept = []; + $forget = []; + foreach ($byUser as $uid => $times) { + arsort($times); + $i = 0; + foreach (array_keys($times) as $path) { + if ($i++ < $keep) { + $kept[$uid][] = $path; + } else { + $forget[] = $path; + } + } + } + if (!$dryRun) { + foreach ($forget as $path) { + $repo->forgetUserExport($path); + } + if ($forget !== []) { + $this->alerts->securityEvent('user_exports_forgotten', ['target' => $target->getName(), 'count' => (string)count($forget)], false); + } + } + return ['kept' => $kept, 'forgotten' => $forget]; + } finally { + $this->leases->release($lease); + } + } + public function policy(): RetentionPolicy { return RetentionPolicy::fromArray(json_decode($this->appConfig->getValueString(Application::APP_ID, 'retention', '{}'), true) ?: []); } diff --git a/tests/integration/user-roundtrip.sh b/tests/integration/user-roundtrip.sh index 69d3197..e6efe5a 100755 --- a/tests/integration/user-roundtrip.sh +++ b/tests/integration/user-roundtrip.sh @@ -54,6 +54,18 @@ SUM_AFTER=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xa [ "$SUM_BEFORE" = "$SUM_AFTER" ] || fail "files differ after a replace restore" ok "replace restores the original content under the same uid, byte-identical" +# Retention: $U now has the manual export above plus the automatic safety-export the replace +# restore made of it just before deleting it (4 total after these two more), then --user-last=2 +# must forget exactly the 2 oldest. +sleep 1.1 # exportId is time-based to the second; force each one to be distinct +occ backup:backup:user ci "$U" >/dev/null || fail "third backup:backup:user" +sleep 1.1 +occ backup:backup:user ci "$U" >/dev/null || fail "fourth backup:backup:user" +occ backup:retention --user-last=2 >/dev/null || fail "retention --user-last" +PRUNE_OUT=$(occ backup:prune ci) +echo "$PRUNE_OUT" | grep -q "forgot 2 user export(s) for 1 user(s)" || fail "prune did not forget the expected two user exports" +ok "user-export retention forgets exactly the exports over the keep count" + occ user:delete "$U" >/dev/null $X sh -c "rm -rf /tmp/ngb-urt-target /tmp/ngb-urt-pass" echo "USER ROUNDTRIP PASS"