M5: retention policy, prune with repacking, append-only locations
- RetentionPolicy (last/daily/weekly/monthly) and PruneService: forget snapshots the policy drops, delete packs nothing uses and repack mostly unused ones by copying encrypted blobs as-is; new packs are indexed before old ones are removed - a location lock keeps prune and backups apart; append-only locations are never pruned (cleanup via the location's lifecycle rules) - weekly prune from the background job; occ backup:prune [--dry-run], backup:retention, backup:target:append-only - tests: forget+prune frees space and the remaining snapshot restores identically, repack of mixed packs, dry-run preview, retention buckets
This commit is contained in:
@@ -51,6 +51,9 @@ class BackupService {
|
||||
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
||||
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
||||
}
|
||||
if ($this->locking->isLocked(PruneService::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE)) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is running; try again later');
|
||||
}
|
||||
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
||||
if ($running !== []) {
|
||||
return $running[0];
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\AppInfo\Application;
|
||||
use OCA\NgBackup\Db\RunMapper;
|
||||
use OCA\NgBackup\Db\SnapshotMapper;
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCP\IAppConfig;
|
||||
use OCP\Lock\ILockingProvider;
|
||||
use OCP\Lock\LockedException;
|
||||
|
||||
/**
|
||||
* Applies the retention policy to a location: forgets snapshots the policy does not keep and
|
||||
* prunes data no remaining snapshot uses. Never runs on append-only locations, never while a
|
||||
* backup to the same location is running (the location lock is shared with BackupService).
|
||||
*/
|
||||
class PruneService {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private RunMapper $runs,
|
||||
private SnapshotMapper $snapshots,
|
||||
private IAppConfig $appConfig,
|
||||
private ILockingProvider $locking,
|
||||
) {
|
||||
}
|
||||
|
||||
public static function lockKey(Target $target): string {
|
||||
return 'ng_backup/target/' . $target->getId();
|
||||
}
|
||||
|
||||
public function policy(): RetentionPolicy {
|
||||
return RetentionPolicy::fromArray(json_decode($this->appConfig->getValueString(Application::APP_ID, 'retention', '{}'), true) ?: []);
|
||||
}
|
||||
|
||||
public function setPolicy(RetentionPolicy $policy): void {
|
||||
if ($policy->last < 1) {
|
||||
throw new \InvalidArgumentException('Keep at least the last snapshot');
|
||||
}
|
||||
$this->appConfig->setValueString(Application::APP_ID, 'retention', json_encode($policy->toArray(), JSON_THROW_ON_ERROR));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{kept: array<string, list<string>>, forgotten: list<string>, prune: array}
|
||||
*/
|
||||
public function apply(Target $target, bool $dryRun = false): array {
|
||||
if ($target->getAppendOnly()) {
|
||||
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
|
||||
}
|
||||
if ($this->runs->findRunning($target->getId(), BackupService::KIND_FULL) !== []) {
|
||||
throw new \RuntimeException('A backup to ' . $target->getName() . ' is running; try again later');
|
||||
}
|
||||
try {
|
||||
$this->locking->acquireLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('Another backup or cleanup of ' . $target->getName() . ' is running');
|
||||
}
|
||||
try {
|
||||
$repo = $this->targets->repository($target);
|
||||
$times = [];
|
||||
foreach ($repo->listSnapshots() as $id) {
|
||||
$meta = $repo->snapshotMeta($id);
|
||||
$times[$id] = (int)strtotime((string)($meta['time'] ?? '@0'));
|
||||
}
|
||||
$keep = $this->policy()->keep($times);
|
||||
$forget = array_values(array_diff(array_keys($times), array_keys($keep)));
|
||||
if (!$dryRun) {
|
||||
foreach ($forget as $id) {
|
||||
$repo->forget($id);
|
||||
try {
|
||||
$this->snapshots->delete($this->snapshots->findOne($target->getId(), $id));
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also removes leftovers of interrupted runs. A dry run counts the snapshots it would forget as gone.
|
||||
$prune = $repo->prune($dryRun, 0.5, $dryRun ? $forget : []);
|
||||
return ['kept' => $keep, 'forgotten' => $forget, 'prune' => $prune];
|
||||
} finally {
|
||||
$this->locking->releaseLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
/**
|
||||
* Which snapshots to keep: the newest $last, plus the newest snapshot of each of the most recent
|
||||
* $daily days, $weekly ISO weeks and $monthly months that have a snapshot (like restic/borg).
|
||||
*/
|
||||
final class RetentionPolicy {
|
||||
public function __construct(
|
||||
public readonly int $last = 3,
|
||||
public readonly int $daily = 7,
|
||||
public readonly int $weekly = 4,
|
||||
public readonly int $monthly = 6,
|
||||
) {
|
||||
}
|
||||
|
||||
public static function fromArray(array $a): self {
|
||||
return new self((int)($a['last'] ?? 3), (int)($a['daily'] ?? 7), (int)($a['weekly'] ?? 4), (int)($a['monthly'] ?? 6));
|
||||
}
|
||||
|
||||
public function toArray(): array {
|
||||
return ['last' => $this->last, 'daily' => $this->daily, 'weekly' => $this->weekly, 'monthly' => $this->monthly];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, int> $snapshots id => unix time
|
||||
* @return array<string, list<string>> id => reasons it is kept (ids not in the result are removed)
|
||||
*/
|
||||
public function keep(array $snapshots): array {
|
||||
arsort($snapshots);
|
||||
$keep = [];
|
||||
$i = 0;
|
||||
foreach ($snapshots as $id => $time) {
|
||||
if ($i++ < $this->last) {
|
||||
$keep[$id][] = 'last';
|
||||
}
|
||||
}
|
||||
foreach (['daily' => 'Y-m-d', 'weekly' => 'o-W', 'monthly' => 'Y-m'] as $kind => $format) {
|
||||
$seen = [];
|
||||
foreach ($snapshots as $id => $time) {
|
||||
$bucket = date($format, $time);
|
||||
if (isset($seen[$bucket])) {
|
||||
continue;
|
||||
}
|
||||
if (count($seen) >= $this->$kind) {
|
||||
break;
|
||||
}
|
||||
$seen[$bucket] = true;
|
||||
$keep[$id][] = $kind;
|
||||
}
|
||||
}
|
||||
return $keep;
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,11 @@ class TargetService {
|
||||
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
||||
}
|
||||
|
||||
public function setAppendOnly(Target $target, bool $appendOnly): Target {
|
||||
$target->setAppendOnly($appendOnly);
|
||||
return $this->mapper->update($target);
|
||||
}
|
||||
|
||||
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
|
||||
public function remove(Target $target): void {
|
||||
$this->mapper->delete($target);
|
||||
|
||||
Reference in New Issue
Block a user