M5: retention policy, prune with repacking, append-only locations

- RetentionPolicy (last/daily/weekly/monthly) and PruneService: forget
  snapshots the policy drops, delete packs nothing uses and repack mostly
  unused ones by copying encrypted blobs as-is; new packs are indexed
  before old ones are removed
- a location lock keeps prune and backups apart; append-only locations
  are never pruned (cleanup via the location's lifecycle rules)
- weekly prune from the background job; occ backup:prune [--dry-run],
  backup:retention, backup:target:append-only
- tests: forget+prune frees space and the remaining snapshot restores
  identically, repack of mixed packs, dry-run preview, retention buckets
This commit is contained in:
2026-10-02 02:21:48 +02:00
parent fabdfa6786
commit 38654317a2
16 changed files with 606 additions and 1 deletions
+3
View File
@@ -51,6 +51,9 @@ class BackupService {
if ($this->config->getSystemValue('objectstore', null) !== null) {
throw new \RuntimeException('Object storage as primary storage is not supported yet');
}
if ($this->locking->isLocked(PruneService::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE)) {
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is running; try again later');
}
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
if ($running !== []) {
return $running[0];
+88
View File
@@ -0,0 +1,88 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Service;
use OCA\NgBackup\AppInfo\Application;
use OCA\NgBackup\Db\RunMapper;
use OCA\NgBackup\Db\SnapshotMapper;
use OCA\NgBackup\Db\Target;
use OCP\IAppConfig;
use OCP\Lock\ILockingProvider;
use OCP\Lock\LockedException;
/**
* Applies the retention policy to a location: forgets snapshots the policy does not keep and
* prunes data no remaining snapshot uses. Never runs on append-only locations, never while a
* backup to the same location is running (the location lock is shared with BackupService).
*/
class PruneService {
public function __construct(
private TargetService $targets,
private RunMapper $runs,
private SnapshotMapper $snapshots,
private IAppConfig $appConfig,
private ILockingProvider $locking,
) {
}
public static function lockKey(Target $target): string {
return 'ng_backup/target/' . $target->getId();
}
public function policy(): RetentionPolicy {
return RetentionPolicy::fromArray(json_decode($this->appConfig->getValueString(Application::APP_ID, 'retention', '{}'), true) ?: []);
}
public function setPolicy(RetentionPolicy $policy): void {
if ($policy->last < 1) {
throw new \InvalidArgumentException('Keep at least the last snapshot');
}
$this->appConfig->setValueString(Application::APP_ID, 'retention', json_encode($policy->toArray(), JSON_THROW_ON_ERROR));
}
/**
* @return array{kept: array<string, list<string>>, forgotten: list<string>, prune: array}
*/
public function apply(Target $target, bool $dryRun = false): array {
if ($target->getAppendOnly()) {
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
}
if ($this->runs->findRunning($target->getId(), BackupService::KIND_FULL) !== []) {
throw new \RuntimeException('A backup to ' . $target->getName() . ' is running; try again later');
}
try {
$this->locking->acquireLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
} catch (LockedException) {
throw new \RuntimeException('Another backup or cleanup of ' . $target->getName() . ' is running');
}
try {
$repo = $this->targets->repository($target);
$times = [];
foreach ($repo->listSnapshots() as $id) {
$meta = $repo->snapshotMeta($id);
$times[$id] = (int)strtotime((string)($meta['time'] ?? '@0'));
}
$keep = $this->policy()->keep($times);
$forget = array_values(array_diff(array_keys($times), array_keys($keep)));
if (!$dryRun) {
foreach ($forget as $id) {
$repo->forget($id);
try {
$this->snapshots->delete($this->snapshots->findOne($target->getId(), $id));
} catch (\Throwable) {
}
}
}
// Also removes leftovers of interrupted runs. A dry run counts the snapshots it would forget as gone.
$prune = $repo->prune($dryRun, 0.5, $dryRun ? $forget : []);
return ['kept' => $keep, 'forgotten' => $forget, 'prune' => $prune];
} finally {
$this->locking->releaseLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
}
}
}
+60
View File
@@ -0,0 +1,60 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Service;
/**
* Which snapshots to keep: the newest $last, plus the newest snapshot of each of the most recent
* $daily days, $weekly ISO weeks and $monthly months that have a snapshot (like restic/borg).
*/
final class RetentionPolicy {
public function __construct(
public readonly int $last = 3,
public readonly int $daily = 7,
public readonly int $weekly = 4,
public readonly int $monthly = 6,
) {
}
public static function fromArray(array $a): self {
return new self((int)($a['last'] ?? 3), (int)($a['daily'] ?? 7), (int)($a['weekly'] ?? 4), (int)($a['monthly'] ?? 6));
}
public function toArray(): array {
return ['last' => $this->last, 'daily' => $this->daily, 'weekly' => $this->weekly, 'monthly' => $this->monthly];
}
/**
* @param array<string, int> $snapshots id => unix time
* @return array<string, list<string>> id => reasons it is kept (ids not in the result are removed)
*/
public function keep(array $snapshots): array {
arsort($snapshots);
$keep = [];
$i = 0;
foreach ($snapshots as $id => $time) {
if ($i++ < $this->last) {
$keep[$id][] = 'last';
}
}
foreach (['daily' => 'Y-m-d', 'weekly' => 'o-W', 'monthly' => 'Y-m'] as $kind => $format) {
$seen = [];
foreach ($snapshots as $id => $time) {
$bucket = date($format, $time);
if (isset($seen[$bucket])) {
continue;
}
if (count($seen) >= $this->$kind) {
break;
}
$seen[$bucket] = true;
$keep[$id][] = $kind;
}
}
return $keep;
}
}
+5
View File
@@ -89,6 +89,11 @@ class TargetService {
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
}
public function setAppendOnly(Target $target, bool $appendOnly): Target {
$target->setAppendOnly($appendOnly);
return $this->mapper->update($target);
}
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
public function remove(Target $target): void {
$this->mapper->delete($target);