Storage limit per location
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled

S3 has no bucket quota, so NG Backup enforces one: before a pack is
uploaded, the stored size plus the pack must stay under the location's
limit, otherwise the backup stops with an error (and administrators are
notified through the failed run). Repacking during prune is exempt, as it
is how space is freed. occ backup:target:limit NAME [SIZE|none].

Also ignore credential files in the repository.
This commit is contained in:
2026-10-02 14:23:25 +02:00
parent 066873a818
commit 532b23482e
11 changed files with 149 additions and 4 deletions
+3
View File
@@ -1,7 +1,10 @@
/build/
cred-*
*credentials*
/css/
/js/
/node_modules/
.phpunit.result.cache
*.secret
/spikes/out/
/vendor/
+2 -1
View File
@@ -5,7 +5,7 @@
<name>NG Backup</name>
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
<description><![CDATA[Work in progress, not ready for use.]]></description>
<version>0.0.15</version>
<version>0.0.16</version>
<licence>agpl</licence>
<author>Dillard Blom</author>
<namespace>NgBackup</namespace>
@@ -36,6 +36,7 @@
<command>OCA\NgBackup\Command\Retention</command>
<command>OCA\NgBackup\Command\TargetAppendOnly</command>
<command>OCA\NgBackup\Command\TargetTrustCurrent</command>
<command>OCA\NgBackup\Command\TargetLimit</command>
<command>OCA\NgBackup\Command\Trash</command>
<command>OCA\NgBackup\Command\TrashRestore</command>
<command>OCA\NgBackup\Command\KeySlots</command>
+52
View File
@@ -0,0 +1,52 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Command;
use OCA\NgBackup\Service\TargetService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class TargetLimit extends Command {
public function __construct(
private TargetService $targets,
) {
parent::__construct();
}
protected function configure(): void {
$this->setName('backup:target:limit')
->setDescription('Set a maximum size for the backups stored on a location (e.g. 100G), or "none"')
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
->addArgument('size', InputArgument::OPTIONAL, 'Size like 500M, 20G, 2T, or "none"; omit to show the current limit')
->setHelp("A backup that would exceed the limit stops with an error and administrators are notified.\n"
. "Note: with versioning or Object Lock on the location, removed data keeps using space there\n"
. "until it expires; keep the limit below what you are willing to pay for.");
}
protected function execute(InputInterface $input, OutputInterface $output): int {
$t = $this->targets->get($input->getArgument('target'));
$size = $input->getArgument('size');
if ($size !== null) {
if (strtolower($size) === 'none') {
$t = $this->targets->setMaxBytes($t, null);
} elseif (preg_match('/^(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?$/i', $size, $m)) {
$factor = ['' => 1, 'K' => 1024, 'M' => 1024 ** 2, 'G' => 1024 ** 3, 'T' => 1024 ** 4][strtoupper($m[2])];
$t = $this->targets->setMaxBytes($t, (int)round((float)$m[1] * $factor));
} else {
$output->writeln('<error>Use a size like 500M, 20G or 2T, or "none".</error>');
return 1;
}
}
$used = $this->targets->repository($t)->storedBytes();
$output->writeln(sprintf('%s: %.2f GiB stored, limit %s', $t->getName(), $used / 1073741824,
$t->getMaxBytes() === null ? 'none' : sprintf('%.2f GiB', $t->getMaxBytes() / 1073741824)));
return 0;
}
}
+4
View File
@@ -24,6 +24,8 @@ use OCP\AppFramework\Db\Entity;
* @method void setRepositoryId(?string $v)
* @method bool|null getAppendOnly()
* @method void setAppendOnly(bool $v)
* @method int|null getMaxBytes()
* @method void setMaxBytes(?int $v)
* @method int getCreatedAt()
* @method void setCreatedAt(int $v)
*/
@@ -36,9 +38,11 @@ class Target extends Entity {
protected ?string $repositoryId = null;
protected int $createdAt = 0;
protected ?bool $appendOnly = null;
protected ?int $maxBytes = null;
public function __construct() {
$this->addType('createdAt', 'integer');
$this->addType('appendOnly', 'boolean');
$this->addType('maxBytes', 'integer');
}
}
@@ -0,0 +1,28 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Migration;
use Closure;
use OCP\DB\ISchemaWrapper;
use OCP\DB\Types;
use OCP\Migration\IOutput;
use OCP\Migration\SimpleMigrationStep;
/** Optional size limit per location (cost control; S3 has no bucket quota). */
class Version000004Date20261002030000 extends SimpleMigrationStep {
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
$t = $schema->getTable('ngb_targets');
if (!$t->hasColumn('max_bytes')) {
$t->addColumn('max_bytes', Types::BIGINT, ['notnull' => false, 'unsigned' => true]);
return $schema;
}
return null;
}
}
+9
View File
@@ -139,6 +139,15 @@ final class BlobIndex {
}
}
/** Bytes of all blobs in the index (encrypted size, i.e. what is stored in packs). */
public function storedBytes(): int {
$total = 0;
foreach ($this->blobs as [, , $length]) {
$total += $length;
}
return $total;
}
public function count(): int {
return count($this->blobs);
}
+7
View File
@@ -32,6 +32,7 @@ final class PackWriter {
private StreamCipher $cipher,
private BlobIndex $index,
private int $targetSize = 32 * 1048576,
private ?int $maxBytes = null,
) {
$this->reset();
}
@@ -64,6 +65,12 @@ final class PackWriter {
if ($this->entries === []) {
return;
}
if ($this->maxBytes !== null && $this->index->storedBytes() + $this->size > $this->maxBytes) {
$this->index->dropPending(array_column($this->entries, 'id'));
$this->reset();
throw new QuotaExceededException(sprintf('Storage limit of this location reached (%.1f GiB): backup stopped',
$this->maxBytes / 1073741824));
}
$packId = bin2hex(random_bytes(16));
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
fwrite($this->buffer, $header . pack('N', strlen($header)));
+11
View File
@@ -0,0 +1,11 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Repository;
class QuotaExceededException extends RepositoryException {
}
+14 -3
View File
@@ -32,6 +32,7 @@ final class Repository {
private StreamCipher $cipher;
private BlobIndex $index;
private ?PackWriter $runPacks = null;
private ?int $maxBytes = null;
private Catalog $catalog;
private function __construct(
@@ -93,6 +94,15 @@ final class Repository {
fn () => array_map('basename', $this->backend->list('trash/info')));
}
/** Refuse uploads that would make the stored data larger than this (null = no limit). */
public function setMaxBytes(?int $maxBytes): void {
$this->maxBytes = $maxBytes;
}
public function storedBytes(): int {
return $this->index->storedBytes();
}
/** Current catalog generation (for status and tests). */
public function catalogGeneration(): int {
return $this->catalog->gen;
@@ -145,7 +155,7 @@ final class Repository {
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
$source = rtrim($source, '/');
$previous = $parent !== null ? $this->loadEntries($parent) : [];
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
$packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
@@ -226,7 +236,7 @@ final class Repository {
* run is done, so the last pack is uploaded before anything references it.
*/
public function blobWriter(array &$stats): BlobWriter {
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
$packs = $this->runPacks;
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
return $this->storeBlob($data, $packs, $stats);
@@ -258,7 +268,7 @@ final class Repository {
/** Store one blob of file data in the current run (deduplicated); returns its id. */
public function storeData(string $data, array &$stats): string {
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
return $this->storeBlob($data, $this->runPacks, $stats);
}
@@ -481,6 +491,7 @@ final class Repository {
} else {
$stats['repacked']++;
if (!$dryRun) {
// No size limit here: repacking is how space is freed.
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
foreach ($blobs as $b) {
+7
View File
@@ -90,6 +90,12 @@ class TargetService {
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
}
/** Size limit for the data stored on this location (null = none). */
public function setMaxBytes(Target $target, ?int $maxBytes): Target {
$target->setMaxBytes($maxBytes);
return $this->mapper->update($target);
}
public function setAppendOnly(Target $target, bool $appendOnly): Target {
$target->setAppendOnly($appendOnly);
return $this->mapper->update($target);
@@ -109,6 +115,7 @@ class TargetService {
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
throw new \RuntimeException('The target now contains a different repository than when it was added');
}
$repo->setMaxBytes($target->getMaxBytes());
return $repo;
}
+12
View File
@@ -146,4 +146,16 @@ class RepositoryTest extends TestCase {
$this->assertFileDoesNotExist(dirname($target, 2) . '/escaped.txt');
$this->assertFileDoesNotExist(dirname($target) . '/escaped.txt');
}
public function testStorageLimitStopsTheBackup(): void {
$repo = Repository::init($this->backend, 'pw');
$repo->setMaxBytes(6 * 1048576); // less than the test tree
try {
$repo->backupDirectory($this->src);
$this->fail('backup should hit the limit');
} catch (\OCA\NgBackup\Repository\QuotaExceededException) {
}
$this->assertLessThanOrEqual(6 * 1048576, $repo->storedBytes());
$this->assertSame([], $repo->listSnapshots(), 'no snapshot from an incomplete backup');
}
}