From 532b23482e57fad8a15e2cf71616a4b1631c7488 Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Fri, 2 Oct 2026 14:23:25 +0200 Subject: [PATCH] Storage limit per location S3 has no bucket quota, so NG Backup enforces one: before a pack is uploaded, the stored size plus the pack must stay under the location's limit, otherwise the backup stops with an error (and administrators are notified through the failed run). Repacking during prune is exempt, as it is how space is freed. occ backup:target:limit NAME [SIZE|none]. Also ignore credential files in the repository. --- .gitignore | 3 ++ appinfo/info.xml | 3 +- lib/Command/TargetLimit.php | 52 +++++++++++++++++++ lib/Db/Target.php | 4 ++ .../Version000004Date20261002030000.php | 28 ++++++++++ lib/Repository/BlobIndex.php | 9 ++++ lib/Repository/PackWriter.php | 7 +++ lib/Repository/QuotaExceededException.php | 11 ++++ lib/Repository/Repository.php | 17 ++++-- lib/Service/TargetService.php | 7 +++ tests/Unit/Repository/RepositoryTest.php | 12 +++++ 11 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 lib/Command/TargetLimit.php create mode 100644 lib/Migration/Version000004Date20261002030000.php create mode 100644 lib/Repository/QuotaExceededException.php diff --git a/.gitignore b/.gitignore index c644c26..f07a86f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ /build/ +cred-* +*credentials* /css/ /js/ /node_modules/ .phpunit.result.cache +*.secret /spikes/out/ /vendor/ diff --git a/appinfo/info.xml b/appinfo/info.xml index 1bbef8c..105ab81 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -5,7 +5,7 @@ NG Backup Encrypted, incremental backups of your Nextcloud with restore from the web interface - 0.0.15 + 0.0.16 agpl Dillard Blom NgBackup @@ -36,6 +36,7 @@ OCA\NgBackup\Command\Retention OCA\NgBackup\Command\TargetAppendOnly OCA\NgBackup\Command\TargetTrustCurrent + OCA\NgBackup\Command\TargetLimit OCA\NgBackup\Command\Trash OCA\NgBackup\Command\TrashRestore OCA\NgBackup\Command\KeySlots diff --git a/lib/Command/TargetLimit.php b/lib/Command/TargetLimit.php new file mode 100644 index 0000000..6e9a3bd --- /dev/null +++ b/lib/Command/TargetLimit.php @@ -0,0 +1,52 @@ +setName('backup:target:limit') + ->setDescription('Set a maximum size for the backups stored on a location (e.g. 100G), or "none"') + ->addArgument('target', InputArgument::REQUIRED, 'Location name or id') + ->addArgument('size', InputArgument::OPTIONAL, 'Size like 500M, 20G, 2T, or "none"; omit to show the current limit') + ->setHelp("A backup that would exceed the limit stops with an error and administrators are notified.\n" + . "Note: with versioning or Object Lock on the location, removed data keeps using space there\n" + . "until it expires; keep the limit below what you are willing to pay for."); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $t = $this->targets->get($input->getArgument('target')); + $size = $input->getArgument('size'); + if ($size !== null) { + if (strtolower($size) === 'none') { + $t = $this->targets->setMaxBytes($t, null); + } elseif (preg_match('/^(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?$/i', $size, $m)) { + $factor = ['' => 1, 'K' => 1024, 'M' => 1024 ** 2, 'G' => 1024 ** 3, 'T' => 1024 ** 4][strtoupper($m[2])]; + $t = $this->targets->setMaxBytes($t, (int)round((float)$m[1] * $factor)); + } else { + $output->writeln('Use a size like 500M, 20G or 2T, or "none".'); + return 1; + } + } + $used = $this->targets->repository($t)->storedBytes(); + $output->writeln(sprintf('%s: %.2f GiB stored, limit %s', $t->getName(), $used / 1073741824, + $t->getMaxBytes() === null ? 'none' : sprintf('%.2f GiB', $t->getMaxBytes() / 1073741824))); + return 0; + } +} diff --git a/lib/Db/Target.php b/lib/Db/Target.php index 0cab9d6..f3a316c 100644 --- a/lib/Db/Target.php +++ b/lib/Db/Target.php @@ -24,6 +24,8 @@ use OCP\AppFramework\Db\Entity; * @method void setRepositoryId(?string $v) * @method bool|null getAppendOnly() * @method void setAppendOnly(bool $v) + * @method int|null getMaxBytes() + * @method void setMaxBytes(?int $v) * @method int getCreatedAt() * @method void setCreatedAt(int $v) */ @@ -36,9 +38,11 @@ class Target extends Entity { protected ?string $repositoryId = null; protected int $createdAt = 0; protected ?bool $appendOnly = null; + protected ?int $maxBytes = null; public function __construct() { $this->addType('createdAt', 'integer'); $this->addType('appendOnly', 'boolean'); + $this->addType('maxBytes', 'integer'); } } diff --git a/lib/Migration/Version000004Date20261002030000.php b/lib/Migration/Version000004Date20261002030000.php new file mode 100644 index 0000000..a2e52a6 --- /dev/null +++ b/lib/Migration/Version000004Date20261002030000.php @@ -0,0 +1,28 @@ +getTable('ngb_targets'); + if (!$t->hasColumn('max_bytes')) { + $t->addColumn('max_bytes', Types::BIGINT, ['notnull' => false, 'unsigned' => true]); + return $schema; + } + return null; + } +} diff --git a/lib/Repository/BlobIndex.php b/lib/Repository/BlobIndex.php index ccaa4c0..cae45cf 100644 --- a/lib/Repository/BlobIndex.php +++ b/lib/Repository/BlobIndex.php @@ -139,6 +139,15 @@ final class BlobIndex { } } + /** Bytes of all blobs in the index (encrypted size, i.e. what is stored in packs). */ + public function storedBytes(): int { + $total = 0; + foreach ($this->blobs as [, , $length]) { + $total += $length; + } + return $total; + } + public function count(): int { return count($this->blobs); } diff --git a/lib/Repository/PackWriter.php b/lib/Repository/PackWriter.php index 7ad4d5e..8d8478b 100644 --- a/lib/Repository/PackWriter.php +++ b/lib/Repository/PackWriter.php @@ -32,6 +32,7 @@ final class PackWriter { private StreamCipher $cipher, private BlobIndex $index, private int $targetSize = 32 * 1048576, + private ?int $maxBytes = null, ) { $this->reset(); } @@ -64,6 +65,12 @@ final class PackWriter { if ($this->entries === []) { return; } + if ($this->maxBytes !== null && $this->index->storedBytes() + $this->size > $this->maxBytes) { + $this->index->dropPending(array_column($this->entries, 'id')); + $this->reset(); + throw new QuotaExceededException(sprintf('Storage limit of this location reached (%.1f GiB): backup stopped', + $this->maxBytes / 1073741824)); + } $packId = bin2hex(random_bytes(16)); $header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId); fwrite($this->buffer, $header . pack('N', strlen($header))); diff --git a/lib/Repository/QuotaExceededException.php b/lib/Repository/QuotaExceededException.php new file mode 100644 index 0000000..2e970af --- /dev/null +++ b/lib/Repository/QuotaExceededException.php @@ -0,0 +1,11 @@ + array_map('basename', $this->backend->list('trash/info'))); } + /** Refuse uploads that would make the stored data larger than this (null = no limit). */ + public function setMaxBytes(?int $maxBytes): void { + $this->maxBytes = $maxBytes; + } + + public function storedBytes(): int { + return $this->index->storedBytes(); + } + /** Current catalog generation (for status and tests). */ public function catalogGeneration(): int { return $this->catalog->gen; @@ -145,7 +155,7 @@ final class Repository { public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array { $source = rtrim($source, '/'); $previous = $parent !== null ? $this->loadEntries($parent) : []; - $packs = new PackWriter($this->backend, $this->cipher, $this->index); + $packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes); $stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0]; $tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); @@ -226,7 +236,7 @@ final class Repository { * run is done, so the last pack is uploaded before anything references it. */ public function blobWriter(array &$stats): BlobWriter { - $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index); + $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes); $packs = $this->runPacks; return new BlobWriter(function (string $data) use ($packs, &$stats): string { return $this->storeBlob($data, $packs, $stats); @@ -258,7 +268,7 @@ final class Repository { /** Store one blob of file data in the current run (deduplicated); returns its id. */ public function storeData(string $data, array &$stats): string { - $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index); + $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes); return $this->storeBlob($data, $this->runPacks, $stats); } @@ -481,6 +491,7 @@ final class Repository { } else { $stats['repacked']++; if (!$dryRun) { + // No size limit here: repacking is how space is freed. $writer ??= new PackWriter($this->backend, $this->cipher, $this->index); $path = 'packs/' . substr($packId, 0, 2) . '/' . $packId; foreach ($blobs as $b) { diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php index 7b0ffc3..e9e9d25 100644 --- a/lib/Service/TargetService.php +++ b/lib/Service/TargetService.php @@ -90,6 +90,12 @@ class TargetService { return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId); } + /** Size limit for the data stored on this location (null = none). */ + public function setMaxBytes(Target $target, ?int $maxBytes): Target { + $target->setMaxBytes($maxBytes); + return $this->mapper->update($target); + } + public function setAppendOnly(Target $target, bool $appendOnly): Target { $target->setAppendOnly($appendOnly); return $this->mapper->update($target); @@ -109,6 +115,7 @@ class TargetService { if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) { throw new \RuntimeException('The target now contains a different repository than when it was added'); } + $repo->setMaxBytes($target->getMaxBytes()); return $repo; } diff --git a/tests/Unit/Repository/RepositoryTest.php b/tests/Unit/Repository/RepositoryTest.php index 9d01f5b..c17e3b1 100644 --- a/tests/Unit/Repository/RepositoryTest.php +++ b/tests/Unit/Repository/RepositoryTest.php @@ -146,4 +146,16 @@ class RepositoryTest extends TestCase { $this->assertFileDoesNotExist(dirname($target, 2) . '/escaped.txt'); $this->assertFileDoesNotExist(dirname($target) . '/escaped.txt'); } + + public function testStorageLimitStopsTheBackup(): void { + $repo = Repository::init($this->backend, 'pw'); + $repo->setMaxBytes(6 * 1048576); // less than the test tree + try { + $repo->backupDirectory($this->src); + $this->fail('backup should hit the limit'); + } catch (\OCA\NgBackup\Repository\QuotaExceededException) { + } + $this->assertLessThanOrEqual(6 * 1048576, $repo->storedBytes()); + $this->assertSame([], $repo->listSnapshots(), 'no snapshot from an incomplete backup'); + } }