M5: retention policy, prune with repacking, append-only locations
- RetentionPolicy (last/daily/weekly/monthly) and PruneService: forget snapshots the policy drops, delete packs nothing uses and repack mostly unused ones by copying encrypted blobs as-is; new packs are indexed before old ones are removed - a location lock keeps prune and backups apart; append-only locations are never pruned (cleanup via the location's lifecycle rules) - weekly prune from the background job; occ backup:prune [--dry-run], backup:retention, backup:target:append-only - tests: forget+prune frees space and the remaining snapshot restores identically, repack of mixed packs, dry-run preview, retention buckets
This commit is contained in:
+4
-1
@@ -5,7 +5,7 @@
|
||||
<name>NG Backup</name>
|
||||
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
||||
<description><![CDATA[Work in progress, not ready for use.]]></description>
|
||||
<version>0.0.5</version>
|
||||
<version>0.0.6</version>
|
||||
<licence>agpl</licence>
|
||||
<author>Dillard Blom</author>
|
||||
<namespace>NgBackup</namespace>
|
||||
@@ -32,6 +32,9 @@
|
||||
<command>OCA\NgBackup\Command\ListSnapshots</command>
|
||||
<command>OCA\NgBackup\Command\Browse</command>
|
||||
<command>OCA\NgBackup\Command\Restore</command>
|
||||
<command>OCA\NgBackup\Command\Prune</command>
|
||||
<command>OCA\NgBackup\Command\Retention</command>
|
||||
<command>OCA\NgBackup\Command\TargetAppendOnly</command>
|
||||
</commands>
|
||||
<settings>
|
||||
<admin>OCA\NgBackup\Settings\Admin</admin>
|
||||
|
||||
@@ -26,6 +26,7 @@ class BackupJob extends TimedJob {
|
||||
public function __construct(
|
||||
ITimeFactory $time,
|
||||
private BackupService $backups,
|
||||
private \OCA\NgBackup\Service\PruneService $prune,
|
||||
private TargetService $targets,
|
||||
private RunMapper $runs,
|
||||
private IAppConfig $appConfig,
|
||||
@@ -45,6 +46,26 @@ class BackupJob extends TimedJob {
|
||||
$run = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
||||
}
|
||||
}
|
||||
$this->weeklyPrune();
|
||||
}
|
||||
|
||||
/** Once a week (after Sunday's backup), apply the retention policy to non-append-only locations. */
|
||||
private function weeklyPrune(): void {
|
||||
$lastWeek = $this->appConfig->getValueString(Application::APP_ID, 'last_prune_week', '');
|
||||
if (date('N') !== '7' || $lastWeek === date('o-W') || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
|
||||
return;
|
||||
}
|
||||
foreach ($this->targets->list() as $t) {
|
||||
if ($t->getAppendOnly()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
$this->prune->apply($t);
|
||||
} catch (\Throwable $e) {
|
||||
$this->logger->warning('NG Backup: cleanup of ' . $t->getName() . ' failed: ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', date('o-W'));
|
||||
}
|
||||
|
||||
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\PruneService;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class Prune extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private PruneService $prune,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:prune')
|
||||
->setDescription('Apply the retention policy: remove old snapshots and the data only they used')
|
||||
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all that are not append-only)')
|
||||
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Only show what would be removed');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$name = $input->getArgument('target');
|
||||
$targets = $name === null ? array_filter($this->targets->list(), fn ($t) => !$t->getAppendOnly()) : [$this->targets->get($name)];
|
||||
$dry = (bool)$input->getOption('dry-run');
|
||||
$output->writeln('Policy: ' . json_encode($this->prune->policy()->toArray()) . ($dry ? ' (dry run)' : ''));
|
||||
$rc = 0;
|
||||
foreach ($targets as $t) {
|
||||
try {
|
||||
$r = $this->prune->apply($t, $dry);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $t->getName() . ': ' . $e->getMessage() . '</error>');
|
||||
$rc = 1;
|
||||
continue;
|
||||
}
|
||||
$p = $r['prune'];
|
||||
$output->writeln(sprintf('<info>%s</info>: keep %d, %s %d snapshots; packs: %d, %s %d, repack %d, frees %.1f MiB',
|
||||
$t->getName(), count($r['kept']), $dry ? 'would forget' : 'forgot', count($r['forgotten']), $p['packs'],
|
||||
$dry ? 'would delete' : 'deleted', $p['deleted'], $p['repacked'], $p['freedBytes'] / 1048576));
|
||||
}
|
||||
return $rc;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\PruneService;
|
||||
use OCA\NgBackup\Service\RetentionPolicy;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class Retention extends Command {
|
||||
public function __construct(
|
||||
private PruneService $prune,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:retention')
|
||||
->setDescription('Show or set how many snapshots are kept')
|
||||
->addOption('last', null, InputOption::VALUE_REQUIRED, 'Always keep the newest N')
|
||||
->addOption('daily', null, InputOption::VALUE_REQUIRED, 'Keep one per day for the last N days with a backup')
|
||||
->addOption('weekly', null, InputOption::VALUE_REQUIRED, 'Keep one per week for N weeks')
|
||||
->addOption('monthly', null, InputOption::VALUE_REQUIRED, 'Keep one per month for N months')
|
||||
->setHelp("Example: occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12");
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$p = $this->prune->policy()->toArray();
|
||||
$changed = false;
|
||||
foreach (['last', 'daily', 'weekly', 'monthly'] as $k) {
|
||||
if ($input->getOption($k) !== null) {
|
||||
$p[$k] = max(0, (int)$input->getOption($k));
|
||||
$changed = true;
|
||||
}
|
||||
}
|
||||
if ($changed) {
|
||||
try {
|
||||
$this->prune->setPolicy(RetentionPolicy::fromArray($p));
|
||||
} catch (\InvalidArgumentException $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
$output->writeln(sprintf('Keep the last %d, plus one per day for %d days, per week for %d weeks, per month for %d months.', $p['last'], $p['daily'], $p['weekly'], $p['monthly']));
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class TargetAppendOnly extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:target:append-only')
|
||||
->setDescription('Make a location append-only (NG Backup never deletes there) or normal again')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('state', InputArgument::REQUIRED, 'on or off')
|
||||
->setHelp("Append-only protects backups if this server is compromised: combine it with credentials\nwithout delete rights and a lifecycle rule or object lock on the location itself.");
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$state = strtolower((string)$input->getArgument('state'));
|
||||
if (!in_array($state, ['on', 'off'], true)) {
|
||||
$output->writeln('<error>state must be on or off</error>');
|
||||
return 1;
|
||||
}
|
||||
$t = $this->targets->setAppendOnly($this->targets->get($input->getArgument('target')), $state === 'on');
|
||||
$output->writeln($t->getName() . ' is ' . ($t->getAppendOnly() ? 'append-only' : 'normal') . '.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,8 @@ use OCP\AppFramework\Db\Entity;
|
||||
* @method void setBasePath(string $v)
|
||||
* @method string|null getRepositoryId()
|
||||
* @method void setRepositoryId(?string $v)
|
||||
* @method bool|null getAppendOnly()
|
||||
* @method void setAppendOnly(bool $v)
|
||||
* @method int getCreatedAt()
|
||||
* @method void setCreatedAt(int $v)
|
||||
*/
|
||||
@@ -33,8 +35,10 @@ class Target extends Entity {
|
||||
protected string $basePath = '';
|
||||
protected ?string $repositoryId = null;
|
||||
protected int $createdAt = 0;
|
||||
protected ?bool $appendOnly = null;
|
||||
|
||||
public function __construct() {
|
||||
$this->addType('createdAt', 'integer');
|
||||
$this->addType('appendOnly', 'boolean');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Migration;
|
||||
|
||||
use Closure;
|
||||
use OCP\DB\ISchemaWrapper;
|
||||
use OCP\DB\Types;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\SimpleMigrationStep;
|
||||
|
||||
/** Append-only locations: NG Backup never deletes anything there (cleanup by the location's own lifecycle rules). */
|
||||
class Version000003Date20261002020000 extends SimpleMigrationStep {
|
||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||
/** @var ISchemaWrapper $schema */
|
||||
$schema = $schemaClosure();
|
||||
$t = $schema->getTable('ngb_targets');
|
||||
if (!$t->hasColumn('append_only')) {
|
||||
$t->addColumn('append_only', Types::BOOLEAN, ['notnull' => false, 'default' => false]);
|
||||
return $schema;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,26 @@ final class BlobIndex {
|
||||
}
|
||||
}
|
||||
|
||||
/** @return array<string, list<array{id:string, offset:int, length:int}>> packId => blobs currently located in that pack */
|
||||
public function packs(): array {
|
||||
$packs = [];
|
||||
foreach ($this->blobs as $id => [$pack, $offset, $length]) {
|
||||
$packs[$pack][] = ['id' => $id, 'offset' => $offset, 'length' => $length];
|
||||
}
|
||||
return $packs;
|
||||
}
|
||||
|
||||
/** Forget a pack: delete its index file and cache row. Blobs that now live elsewhere are kept. */
|
||||
public function removePack(string $packId): void {
|
||||
$this->backend->delete('index/' . $packId);
|
||||
$this->cache?->remove($this->repositoryId, [$packId]);
|
||||
foreach ($this->blobs as $id => $loc) {
|
||||
if ($loc[0] === $packId) {
|
||||
unset($this->blobs[$id]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function count(): int {
|
||||
return count($this->blobs);
|
||||
}
|
||||
|
||||
@@ -49,6 +49,16 @@ final class PackWriter {
|
||||
return strlen($encrypted);
|
||||
}
|
||||
|
||||
/** Add an already encrypted blob as-is (repacking: no decryption needed, the blob is bound to its id, not to a pack). */
|
||||
public function addRaw(string $id, string $encrypted, int $rawLength, int $flags): void {
|
||||
fwrite($this->buffer, $encrypted);
|
||||
$this->entries[] = ['id' => $id, 'offset' => $this->size, 'length' => strlen($encrypted), 'raw' => $rawLength, 'flags' => $flags];
|
||||
$this->size += strlen($encrypted);
|
||||
if ($this->size >= $this->targetSize) {
|
||||
$this->flush();
|
||||
}
|
||||
}
|
||||
|
||||
/** Upload the current pack (if any) and record it in the index. */
|
||||
public function flush(): void {
|
||||
if ($this->entries === []) {
|
||||
|
||||
@@ -273,6 +273,103 @@ final class Repository {
|
||||
return $data;
|
||||
}
|
||||
|
||||
/** Remove a snapshot (and its database dump). Its data is freed by the next prune(). */
|
||||
public function forget(string $snapshotId): void {
|
||||
try {
|
||||
$meta = $this->snapshotMeta($snapshotId);
|
||||
} catch (\Throwable) {
|
||||
$meta = [];
|
||||
}
|
||||
if (isset($meta['db']) && is_string($meta['db']) && str_starts_with($meta['db'], 'db/')) {
|
||||
$this->backend->delete($meta['db']);
|
||||
}
|
||||
$this->backend->delete('snapshots/' . $snapshotId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete packs no snapshot or user export uses any more, and repack packs where more than
|
||||
* $repackThreshold of the bytes are unused. Must not run while a backup to this repository
|
||||
* is in progress (its blobs are not referenced by a snapshot yet); callers hold the lock.
|
||||
*
|
||||
* @return array{packs:int, deleted:int, repacked:int, freedBytes:int, keptBlobs:int, unusedBlobs:int}
|
||||
*/
|
||||
public function prune(bool $dryRun = false, float $repackThreshold = 0.5, array $ignoreSnapshots = []): array {
|
||||
$used = [];
|
||||
$ignore = array_flip($ignoreSnapshots); // e.g. snapshots a dry run would forget first
|
||||
foreach ($this->listSnapshots() as $sid) {
|
||||
if (isset($ignore[$sid])) {
|
||||
continue;
|
||||
}
|
||||
foreach ($this->streamEntries($sid) as $e) {
|
||||
foreach ($e['b'] as $b) {
|
||||
$used[$b] = true;
|
||||
}
|
||||
}
|
||||
$meta = $this->snapshotMeta($sid);
|
||||
if (isset($meta['db']) && $this->backend->exists($meta['db'])) {
|
||||
foreach (json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
||||
foreach ($t['blobs'] as $b) {
|
||||
$used[$b] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($this->backend->list('users') as $path) {
|
||||
foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
|
||||
foreach ($e['b'] ?? [] as $b) {
|
||||
$used[$b] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$stats = ['packs' => 0, 'deleted' => 0, 'repacked' => 0, 'freedBytes' => 0, 'keptBlobs' => count($used), 'unusedBlobs' => 0];
|
||||
$packs = $this->index->packs();
|
||||
$stats['packs'] = count($packs);
|
||||
$writer = null;
|
||||
$toRemove = [];
|
||||
foreach ($packs as $packId => $blobs) {
|
||||
$total = 0;
|
||||
$unused = 0;
|
||||
foreach ($blobs as $b) {
|
||||
$total += $b['length'];
|
||||
if (!isset($used[$b['id']])) {
|
||||
$unused += $b['length'];
|
||||
$stats['unusedBlobs']++;
|
||||
}
|
||||
}
|
||||
if ($unused === 0 || ($unused < $total && $unused / $total <= $repackThreshold)) {
|
||||
continue;
|
||||
}
|
||||
$stats['freedBytes'] += $unused;
|
||||
if ($unused === $total) {
|
||||
$stats['deleted']++;
|
||||
} else {
|
||||
$stats['repacked']++;
|
||||
if (!$dryRun) {
|
||||
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
||||
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
||||
foreach ($blobs as $b) {
|
||||
if (isset($used[$b['id']])) {
|
||||
[, , , $raw, $flags] = $this->index->get($b['id']);
|
||||
$writer->addRaw($b['id'], $this->backend->getRange($path, $b['offset'], $b['length']), $raw, $flags);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$toRemove[] = $packId;
|
||||
}
|
||||
if ($dryRun) {
|
||||
return $stats;
|
||||
}
|
||||
// New packs and their index entries are durable before any old pack is removed.
|
||||
$writer?->flush();
|
||||
foreach ($toRemove as $packId) {
|
||||
$this->index->removePack($packId);
|
||||
$this->backend->delete('packs/' . substr($packId, 0, 2) . '/' . $packId);
|
||||
}
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/** @return list<string> */
|
||||
public function listSnapshots(): array {
|
||||
return array_map('basename', $this->backend->list('snapshots'));
|
||||
|
||||
@@ -51,6 +51,9 @@ class BackupService {
|
||||
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
||||
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
||||
}
|
||||
if ($this->locking->isLocked(PruneService::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE)) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is running; try again later');
|
||||
}
|
||||
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
||||
if ($running !== []) {
|
||||
return $running[0];
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\AppInfo\Application;
|
||||
use OCA\NgBackup\Db\RunMapper;
|
||||
use OCA\NgBackup\Db\SnapshotMapper;
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCP\IAppConfig;
|
||||
use OCP\Lock\ILockingProvider;
|
||||
use OCP\Lock\LockedException;
|
||||
|
||||
/**
|
||||
* Applies the retention policy to a location: forgets snapshots the policy does not keep and
|
||||
* prunes data no remaining snapshot uses. Never runs on append-only locations, never while a
|
||||
* backup to the same location is running (the location lock is shared with BackupService).
|
||||
*/
|
||||
class PruneService {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private RunMapper $runs,
|
||||
private SnapshotMapper $snapshots,
|
||||
private IAppConfig $appConfig,
|
||||
private ILockingProvider $locking,
|
||||
) {
|
||||
}
|
||||
|
||||
public static function lockKey(Target $target): string {
|
||||
return 'ng_backup/target/' . $target->getId();
|
||||
}
|
||||
|
||||
public function policy(): RetentionPolicy {
|
||||
return RetentionPolicy::fromArray(json_decode($this->appConfig->getValueString(Application::APP_ID, 'retention', '{}'), true) ?: []);
|
||||
}
|
||||
|
||||
public function setPolicy(RetentionPolicy $policy): void {
|
||||
if ($policy->last < 1) {
|
||||
throw new \InvalidArgumentException('Keep at least the last snapshot');
|
||||
}
|
||||
$this->appConfig->setValueString(Application::APP_ID, 'retention', json_encode($policy->toArray(), JSON_THROW_ON_ERROR));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{kept: array<string, list<string>>, forgotten: list<string>, prune: array}
|
||||
*/
|
||||
public function apply(Target $target, bool $dryRun = false): array {
|
||||
if ($target->getAppendOnly()) {
|
||||
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
|
||||
}
|
||||
if ($this->runs->findRunning($target->getId(), BackupService::KIND_FULL) !== []) {
|
||||
throw new \RuntimeException('A backup to ' . $target->getName() . ' is running; try again later');
|
||||
}
|
||||
try {
|
||||
$this->locking->acquireLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('Another backup or cleanup of ' . $target->getName() . ' is running');
|
||||
}
|
||||
try {
|
||||
$repo = $this->targets->repository($target);
|
||||
$times = [];
|
||||
foreach ($repo->listSnapshots() as $id) {
|
||||
$meta = $repo->snapshotMeta($id);
|
||||
$times[$id] = (int)strtotime((string)($meta['time'] ?? '@0'));
|
||||
}
|
||||
$keep = $this->policy()->keep($times);
|
||||
$forget = array_values(array_diff(array_keys($times), array_keys($keep)));
|
||||
if (!$dryRun) {
|
||||
foreach ($forget as $id) {
|
||||
$repo->forget($id);
|
||||
try {
|
||||
$this->snapshots->delete($this->snapshots->findOne($target->getId(), $id));
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also removes leftovers of interrupted runs. A dry run counts the snapshots it would forget as gone.
|
||||
$prune = $repo->prune($dryRun, 0.5, $dryRun ? $forget : []);
|
||||
return ['kept' => $keep, 'forgotten' => $forget, 'prune' => $prune];
|
||||
} finally {
|
||||
$this->locking->releaseLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
/**
|
||||
* Which snapshots to keep: the newest $last, plus the newest snapshot of each of the most recent
|
||||
* $daily days, $weekly ISO weeks and $monthly months that have a snapshot (like restic/borg).
|
||||
*/
|
||||
final class RetentionPolicy {
|
||||
public function __construct(
|
||||
public readonly int $last = 3,
|
||||
public readonly int $daily = 7,
|
||||
public readonly int $weekly = 4,
|
||||
public readonly int $monthly = 6,
|
||||
) {
|
||||
}
|
||||
|
||||
public static function fromArray(array $a): self {
|
||||
return new self((int)($a['last'] ?? 3), (int)($a['daily'] ?? 7), (int)($a['weekly'] ?? 4), (int)($a['monthly'] ?? 6));
|
||||
}
|
||||
|
||||
public function toArray(): array {
|
||||
return ['last' => $this->last, 'daily' => $this->daily, 'weekly' => $this->weekly, 'monthly' => $this->monthly];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, int> $snapshots id => unix time
|
||||
* @return array<string, list<string>> id => reasons it is kept (ids not in the result are removed)
|
||||
*/
|
||||
public function keep(array $snapshots): array {
|
||||
arsort($snapshots);
|
||||
$keep = [];
|
||||
$i = 0;
|
||||
foreach ($snapshots as $id => $time) {
|
||||
if ($i++ < $this->last) {
|
||||
$keep[$id][] = 'last';
|
||||
}
|
||||
}
|
||||
foreach (['daily' => 'Y-m-d', 'weekly' => 'o-W', 'monthly' => 'Y-m'] as $kind => $format) {
|
||||
$seen = [];
|
||||
foreach ($snapshots as $id => $time) {
|
||||
$bucket = date($format, $time);
|
||||
if (isset($seen[$bucket])) {
|
||||
continue;
|
||||
}
|
||||
if (count($seen) >= $this->$kind) {
|
||||
break;
|
||||
}
|
||||
$seen[$bucket] = true;
|
||||
$keep[$id][] = $kind;
|
||||
}
|
||||
}
|
||||
return $keep;
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,11 @@ class TargetService {
|
||||
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
||||
}
|
||||
|
||||
public function setAppendOnly(Target $target, bool $appendOnly): Target {
|
||||
$target->setAppendOnly($appendOnly);
|
||||
return $this->mapper->update($target);
|
||||
}
|
||||
|
||||
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
|
||||
public function remove(Target $target): void {
|
||||
$this->mapper->delete($target);
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class PruneTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private function du(LocalBackend $b): int {
|
||||
$root = (new \ReflectionProperty(LocalBackend::class, 'root'))->getValue($b);
|
||||
$total = 0;
|
||||
foreach ($b->list('packs') as $p) {
|
||||
$total += filesize("$root/$p");
|
||||
}
|
||||
return $total;
|
||||
}
|
||||
|
||||
public function testForgetAndPruneFreesSpaceAndKeepsTheRest(): void {
|
||||
$src = $this->tempDir();
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
$keys = KeyRing::generate();
|
||||
$repo = Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
|
||||
file_put_contents("$src/keep.bin", random_bytes(6 * 1048576));
|
||||
file_put_contents("$src/old.bin", random_bytes(40 * 1048576));
|
||||
$old = $repo->backupDirectory($src);
|
||||
unlink("$src/old.bin");
|
||||
file_put_contents("$src/new.bin", random_bytes(3 * 1048576));
|
||||
$new = $repo->backupDirectory($src, $old['snapshot']);
|
||||
$expected = self::hashTree($src);
|
||||
$before = $this->du($backend);
|
||||
|
||||
$dry = $repo->prune(true);
|
||||
$this->assertSame(0, $dry['freedBytes'], 'nothing is unused while both snapshots exist');
|
||||
$preview = $repo->prune(true, 0.5, [$old['snapshot']]);
|
||||
$this->assertGreaterThan(35 * 1048576, $preview['freedBytes'], 'dry run can preview forgetting a snapshot');
|
||||
|
||||
$repo->forget($old['snapshot']);
|
||||
$plan = $repo->prune(true);
|
||||
$this->assertSame($plan, $repo->prune(true, 0.5, []));
|
||||
$this->assertGreaterThan(35 * 1048576, $plan['freedBytes']);
|
||||
$this->assertSame($before, $this->du($backend), 'dry run changes nothing');
|
||||
|
||||
$result = $repo->prune();
|
||||
$this->assertGreaterThan(0, $result['deleted'] + $result['repacked']);
|
||||
$this->assertLessThan($before - 35 * 1048576, $this->du($backend));
|
||||
|
||||
$reopened = Repository::openWithKey($backend, $keys);
|
||||
$target = $this->tempDir();
|
||||
$reopened->restore($new['snapshot'], $target);
|
||||
$this->assertSame($expected, self::hashTree($target));
|
||||
$this->assertSame([$new['snapshot']], $reopened->listSnapshots());
|
||||
}
|
||||
|
||||
public function testRepackKeepsUsedBlobsOfMixedPacks(): void {
|
||||
$src = $this->tempDir();
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
$keys = KeyRing::generate();
|
||||
$repo = Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
// Small files share packs: deleting most of them leaves mixed packs to repack.
|
||||
for ($i = 0; $i < 40; $i++) {
|
||||
file_put_contents("$src/f$i", random_bytes(500000));
|
||||
}
|
||||
$a = $repo->backupDirectory($src);
|
||||
for ($i = 0; $i < 40; $i++) {
|
||||
if ($i % 5 !== 0) {
|
||||
unlink("$src/f$i");
|
||||
}
|
||||
}
|
||||
$b = $repo->backupDirectory($src, $a['snapshot']);
|
||||
$repo->forget($a['snapshot']);
|
||||
$r = $repo->prune(false, 0.5);
|
||||
$this->assertGreaterThan(0, $r['repacked']);
|
||||
$target = $this->tempDir();
|
||||
Repository::openWithKey($backend, $keys)->restore($b['snapshot'], $target);
|
||||
$this->assertSame(self::hashTree($src), self::hashTree($target));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Service;
|
||||
|
||||
use OCA\NgBackup\Service\RetentionPolicy;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class RetentionPolicyTest extends TestCase {
|
||||
public function testDailyWeeklyMonthly(): void {
|
||||
$snaps = [];
|
||||
$start = strtotime('2026-01-01 03:00:00');
|
||||
for ($d = 0; $d < 120; $d++) {
|
||||
$snaps["d$d"] = $start + $d * 86400; // one per day for 120 days
|
||||
$snaps["d$d-b"] = $start + $d * 86400 + 3600; // and a second one an hour later
|
||||
}
|
||||
$keep = (new RetentionPolicy(2, 7, 4, 3))->keep($snaps);
|
||||
$this->assertArrayHasKey('d119-b', $keep, 'newest is always kept');
|
||||
$this->assertArrayHasKey('d119', $keep, 'last=2');
|
||||
$this->assertContains('daily', $keep['d113-b']);
|
||||
$this->assertArrayNotHasKey('d113', $keep, 'only the newest of a day');
|
||||
$this->assertLessThanOrEqual(2 + 7 + 4 + 3, count($keep));
|
||||
$months = array_unique(array_map(fn ($id) => date('Y-m', $snaps[$id]), array_keys(array_filter($keep, fn ($r) => in_array('monthly', $r, true)))));
|
||||
$this->assertCount(3, $months);
|
||||
}
|
||||
|
||||
public function testKeepsEverythingWhenFewSnapshots(): void {
|
||||
$keep = (new RetentionPolicy())->keep(['a' => 100, 'b' => 200]);
|
||||
$this->assertSame(['a', 'b'], array_values(array_intersect(['a', 'b'], array_keys($keep))));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user