Storage limit per location
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
S3 has no bucket quota, so NG Backup enforces one: before a pack is uploaded, the stored size plus the pack must stay under the location's limit, otherwise the backup stops with an error (and administrators are notified through the failed run). Repacking during prune is exempt, as it is how space is freed. occ backup:target:limit NAME [SIZE|none]. Also ignore credential files in the repository.
This commit is contained in:
@@ -1,7 +1,10 @@
|
|||||||
/build/
|
/build/
|
||||||
|
cred-*
|
||||||
|
*credentials*
|
||||||
/css/
|
/css/
|
||||||
/js/
|
/js/
|
||||||
/node_modules/
|
/node_modules/
|
||||||
.phpunit.result.cache
|
.phpunit.result.cache
|
||||||
|
*.secret
|
||||||
/spikes/out/
|
/spikes/out/
|
||||||
/vendor/
|
/vendor/
|
||||||
|
|||||||
+2
-1
@@ -5,7 +5,7 @@
|
|||||||
<name>NG Backup</name>
|
<name>NG Backup</name>
|
||||||
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
||||||
<description><![CDATA[Work in progress, not ready for use.]]></description>
|
<description><![CDATA[Work in progress, not ready for use.]]></description>
|
||||||
<version>0.0.15</version>
|
<version>0.0.16</version>
|
||||||
<licence>agpl</licence>
|
<licence>agpl</licence>
|
||||||
<author>Dillard Blom</author>
|
<author>Dillard Blom</author>
|
||||||
<namespace>NgBackup</namespace>
|
<namespace>NgBackup</namespace>
|
||||||
@@ -36,6 +36,7 @@
|
|||||||
<command>OCA\NgBackup\Command\Retention</command>
|
<command>OCA\NgBackup\Command\Retention</command>
|
||||||
<command>OCA\NgBackup\Command\TargetAppendOnly</command>
|
<command>OCA\NgBackup\Command\TargetAppendOnly</command>
|
||||||
<command>OCA\NgBackup\Command\TargetTrustCurrent</command>
|
<command>OCA\NgBackup\Command\TargetTrustCurrent</command>
|
||||||
|
<command>OCA\NgBackup\Command\TargetLimit</command>
|
||||||
<command>OCA\NgBackup\Command\Trash</command>
|
<command>OCA\NgBackup\Command\Trash</command>
|
||||||
<command>OCA\NgBackup\Command\TrashRestore</command>
|
<command>OCA\NgBackup\Command\TrashRestore</command>
|
||||||
<command>OCA\NgBackup\Command\KeySlots</command>
|
<command>OCA\NgBackup\Command\KeySlots</command>
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
class TargetLimit extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:target:limit')
|
||||||
|
->setDescription('Set a maximum size for the backups stored on a location (e.g. 100G), or "none"')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('size', InputArgument::OPTIONAL, 'Size like 500M, 20G, 2T, or "none"; omit to show the current limit')
|
||||||
|
->setHelp("A backup that would exceed the limit stops with an error and administrators are notified.\n"
|
||||||
|
. "Note: with versioning or Object Lock on the location, removed data keeps using space there\n"
|
||||||
|
. "until it expires; keep the limit below what you are willing to pay for.");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$t = $this->targets->get($input->getArgument('target'));
|
||||||
|
$size = $input->getArgument('size');
|
||||||
|
if ($size !== null) {
|
||||||
|
if (strtolower($size) === 'none') {
|
||||||
|
$t = $this->targets->setMaxBytes($t, null);
|
||||||
|
} elseif (preg_match('/^(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?$/i', $size, $m)) {
|
||||||
|
$factor = ['' => 1, 'K' => 1024, 'M' => 1024 ** 2, 'G' => 1024 ** 3, 'T' => 1024 ** 4][strtoupper($m[2])];
|
||||||
|
$t = $this->targets->setMaxBytes($t, (int)round((float)$m[1] * $factor));
|
||||||
|
} else {
|
||||||
|
$output->writeln('<error>Use a size like 500M, 20G or 2T, or "none".</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$used = $this->targets->repository($t)->storedBytes();
|
||||||
|
$output->writeln(sprintf('%s: %.2f GiB stored, limit %s', $t->getName(), $used / 1073741824,
|
||||||
|
$t->getMaxBytes() === null ? 'none' : sprintf('%.2f GiB', $t->getMaxBytes() / 1073741824)));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,6 +24,8 @@ use OCP\AppFramework\Db\Entity;
|
|||||||
* @method void setRepositoryId(?string $v)
|
* @method void setRepositoryId(?string $v)
|
||||||
* @method bool|null getAppendOnly()
|
* @method bool|null getAppendOnly()
|
||||||
* @method void setAppendOnly(bool $v)
|
* @method void setAppendOnly(bool $v)
|
||||||
|
* @method int|null getMaxBytes()
|
||||||
|
* @method void setMaxBytes(?int $v)
|
||||||
* @method int getCreatedAt()
|
* @method int getCreatedAt()
|
||||||
* @method void setCreatedAt(int $v)
|
* @method void setCreatedAt(int $v)
|
||||||
*/
|
*/
|
||||||
@@ -36,9 +38,11 @@ class Target extends Entity {
|
|||||||
protected ?string $repositoryId = null;
|
protected ?string $repositoryId = null;
|
||||||
protected int $createdAt = 0;
|
protected int $createdAt = 0;
|
||||||
protected ?bool $appendOnly = null;
|
protected ?bool $appendOnly = null;
|
||||||
|
protected ?int $maxBytes = null;
|
||||||
|
|
||||||
public function __construct() {
|
public function __construct() {
|
||||||
$this->addType('createdAt', 'integer');
|
$this->addType('createdAt', 'integer');
|
||||||
$this->addType('appendOnly', 'boolean');
|
$this->addType('appendOnly', 'boolean');
|
||||||
|
$this->addType('maxBytes', 'integer');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Migration;
|
||||||
|
|
||||||
|
use Closure;
|
||||||
|
use OCP\DB\ISchemaWrapper;
|
||||||
|
use OCP\DB\Types;
|
||||||
|
use OCP\Migration\IOutput;
|
||||||
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
|
/** Optional size limit per location (cost control; S3 has no bucket quota). */
|
||||||
|
class Version000004Date20261002030000 extends SimpleMigrationStep {
|
||||||
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
|
/** @var ISchemaWrapper $schema */
|
||||||
|
$schema = $schemaClosure();
|
||||||
|
$t = $schema->getTable('ngb_targets');
|
||||||
|
if (!$t->hasColumn('max_bytes')) {
|
||||||
|
$t->addColumn('max_bytes', Types::BIGINT, ['notnull' => false, 'unsigned' => true]);
|
||||||
|
return $schema;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -139,6 +139,15 @@ final class BlobIndex {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Bytes of all blobs in the index (encrypted size, i.e. what is stored in packs). */
|
||||||
|
public function storedBytes(): int {
|
||||||
|
$total = 0;
|
||||||
|
foreach ($this->blobs as [, , $length]) {
|
||||||
|
$total += $length;
|
||||||
|
}
|
||||||
|
return $total;
|
||||||
|
}
|
||||||
|
|
||||||
public function count(): int {
|
public function count(): int {
|
||||||
return count($this->blobs);
|
return count($this->blobs);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ final class PackWriter {
|
|||||||
private StreamCipher $cipher,
|
private StreamCipher $cipher,
|
||||||
private BlobIndex $index,
|
private BlobIndex $index,
|
||||||
private int $targetSize = 32 * 1048576,
|
private int $targetSize = 32 * 1048576,
|
||||||
|
private ?int $maxBytes = null,
|
||||||
) {
|
) {
|
||||||
$this->reset();
|
$this->reset();
|
||||||
}
|
}
|
||||||
@@ -64,6 +65,12 @@ final class PackWriter {
|
|||||||
if ($this->entries === []) {
|
if ($this->entries === []) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if ($this->maxBytes !== null && $this->index->storedBytes() + $this->size > $this->maxBytes) {
|
||||||
|
$this->index->dropPending(array_column($this->entries, 'id'));
|
||||||
|
$this->reset();
|
||||||
|
throw new QuotaExceededException(sprintf('Storage limit of this location reached (%.1f GiB): backup stopped',
|
||||||
|
$this->maxBytes / 1073741824));
|
||||||
|
}
|
||||||
$packId = bin2hex(random_bytes(16));
|
$packId = bin2hex(random_bytes(16));
|
||||||
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
||||||
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Repository;
|
||||||
|
|
||||||
|
class QuotaExceededException extends RepositoryException {
|
||||||
|
}
|
||||||
@@ -32,6 +32,7 @@ final class Repository {
|
|||||||
private StreamCipher $cipher;
|
private StreamCipher $cipher;
|
||||||
private BlobIndex $index;
|
private BlobIndex $index;
|
||||||
private ?PackWriter $runPacks = null;
|
private ?PackWriter $runPacks = null;
|
||||||
|
private ?int $maxBytes = null;
|
||||||
private Catalog $catalog;
|
private Catalog $catalog;
|
||||||
|
|
||||||
private function __construct(
|
private function __construct(
|
||||||
@@ -93,6 +94,15 @@ final class Repository {
|
|||||||
fn () => array_map('basename', $this->backend->list('trash/info')));
|
fn () => array_map('basename', $this->backend->list('trash/info')));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Refuse uploads that would make the stored data larger than this (null = no limit). */
|
||||||
|
public function setMaxBytes(?int $maxBytes): void {
|
||||||
|
$this->maxBytes = $maxBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function storedBytes(): int {
|
||||||
|
return $this->index->storedBytes();
|
||||||
|
}
|
||||||
|
|
||||||
/** Current catalog generation (for status and tests). */
|
/** Current catalog generation (for status and tests). */
|
||||||
public function catalogGeneration(): int {
|
public function catalogGeneration(): int {
|
||||||
return $this->catalog->gen;
|
return $this->catalog->gen;
|
||||||
@@ -145,7 +155,7 @@ final class Repository {
|
|||||||
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
||||||
$source = rtrim($source, '/');
|
$source = rtrim($source, '/');
|
||||||
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
||||||
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
|
$packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
||||||
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
||||||
|
|
||||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
@@ -226,7 +236,7 @@ final class Repository {
|
|||||||
* run is done, so the last pack is uploaded before anything references it.
|
* run is done, so the last pack is uploaded before anything references it.
|
||||||
*/
|
*/
|
||||||
public function blobWriter(array &$stats): BlobWriter {
|
public function blobWriter(array &$stats): BlobWriter {
|
||||||
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
||||||
$packs = $this->runPacks;
|
$packs = $this->runPacks;
|
||||||
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
||||||
return $this->storeBlob($data, $packs, $stats);
|
return $this->storeBlob($data, $packs, $stats);
|
||||||
@@ -258,7 +268,7 @@ final class Repository {
|
|||||||
|
|
||||||
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
||||||
public function storeData(string $data, array &$stats): string {
|
public function storeData(string $data, array &$stats): string {
|
||||||
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
||||||
return $this->storeBlob($data, $this->runPacks, $stats);
|
return $this->storeBlob($data, $this->runPacks, $stats);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -481,6 +491,7 @@ final class Repository {
|
|||||||
} else {
|
} else {
|
||||||
$stats['repacked']++;
|
$stats['repacked']++;
|
||||||
if (!$dryRun) {
|
if (!$dryRun) {
|
||||||
|
// No size limit here: repacking is how space is freed.
|
||||||
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
||||||
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
||||||
foreach ($blobs as $b) {
|
foreach ($blobs as $b) {
|
||||||
|
|||||||
@@ -90,6 +90,12 @@ class TargetService {
|
|||||||
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Size limit for the data stored on this location (null = none). */
|
||||||
|
public function setMaxBytes(Target $target, ?int $maxBytes): Target {
|
||||||
|
$target->setMaxBytes($maxBytes);
|
||||||
|
return $this->mapper->update($target);
|
||||||
|
}
|
||||||
|
|
||||||
public function setAppendOnly(Target $target, bool $appendOnly): Target {
|
public function setAppendOnly(Target $target, bool $appendOnly): Target {
|
||||||
$target->setAppendOnly($appendOnly);
|
$target->setAppendOnly($appendOnly);
|
||||||
return $this->mapper->update($target);
|
return $this->mapper->update($target);
|
||||||
@@ -109,6 +115,7 @@ class TargetService {
|
|||||||
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
||||||
throw new \RuntimeException('The target now contains a different repository than when it was added');
|
throw new \RuntimeException('The target now contains a different repository than when it was added');
|
||||||
}
|
}
|
||||||
|
$repo->setMaxBytes($target->getMaxBytes());
|
||||||
return $repo;
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -146,4 +146,16 @@ class RepositoryTest extends TestCase {
|
|||||||
$this->assertFileDoesNotExist(dirname($target, 2) . '/escaped.txt');
|
$this->assertFileDoesNotExist(dirname($target, 2) . '/escaped.txt');
|
||||||
$this->assertFileDoesNotExist(dirname($target) . '/escaped.txt');
|
$this->assertFileDoesNotExist(dirname($target) . '/escaped.txt');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function testStorageLimitStopsTheBackup(): void {
|
||||||
|
$repo = Repository::init($this->backend, 'pw');
|
||||||
|
$repo->setMaxBytes(6 * 1048576); // less than the test tree
|
||||||
|
try {
|
||||||
|
$repo->backupDirectory($this->src);
|
||||||
|
$this->fail('backup should hit the limit');
|
||||||
|
} catch (\OCA\NgBackup\Repository\QuotaExceededException) {
|
||||||
|
}
|
||||||
|
$this->assertLessThanOrEqual(6 * 1048576, $repo->storedBytes());
|
||||||
|
$this->assertSame([], $repo->listSnapshots(), 'no snapshot from an incomplete backup');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user