diff --git a/.gitignore b/.gitignore index c644c26..f07a86f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ /build/ +cred-* +*credentials* /css/ /js/ /node_modules/ .phpunit.result.cache +*.secret /spikes/out/ /vendor/ diff --git a/appinfo/info.xml b/appinfo/info.xml index 1bbef8c..105ab81 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -5,7 +5,7 @@ NG Backup Encrypted, incremental backups of your Nextcloud with restore from the web interface - 0.0.15 + 0.0.16 agpl Dillard Blom NgBackup @@ -36,6 +36,7 @@ OCA\NgBackup\Command\Retention OCA\NgBackup\Command\TargetAppendOnly OCA\NgBackup\Command\TargetTrustCurrent + OCA\NgBackup\Command\TargetLimit OCA\NgBackup\Command\Trash OCA\NgBackup\Command\TrashRestore OCA\NgBackup\Command\KeySlots diff --git a/lib/Command/TargetLimit.php b/lib/Command/TargetLimit.php new file mode 100644 index 0000000..6e9a3bd --- /dev/null +++ b/lib/Command/TargetLimit.php @@ -0,0 +1,52 @@ +setName('backup:target:limit') + ->setDescription('Set a maximum size for the backups stored on a location (e.g. 100G), or "none"') + ->addArgument('target', InputArgument::REQUIRED, 'Location name or id') + ->addArgument('size', InputArgument::OPTIONAL, 'Size like 500M, 20G, 2T, or "none"; omit to show the current limit') + ->setHelp("A backup that would exceed the limit stops with an error and administrators are notified.\n" + . "Note: with versioning or Object Lock on the location, removed data keeps using space there\n" + . "until it expires; keep the limit below what you are willing to pay for."); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $t = $this->targets->get($input->getArgument('target')); + $size = $input->getArgument('size'); + if ($size !== null) { + if (strtolower($size) === 'none') { + $t = $this->targets->setMaxBytes($t, null); + } elseif (preg_match('/^(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?$/i', $size, $m)) { + $factor = ['' => 1, 'K' => 1024, 'M' => 1024 ** 2, 'G' => 1024 ** 3, 'T' => 1024 ** 4][strtoupper($m[2])]; + $t = $this->targets->setMaxBytes($t, (int)round((float)$m[1] * $factor)); + } else { + $output->writeln('Use a size like 500M, 20G or 2T, or "none".'); + return 1; + } + } + $used = $this->targets->repository($t)->storedBytes(); + $output->writeln(sprintf('%s: %.2f GiB stored, limit %s', $t->getName(), $used / 1073741824, + $t->getMaxBytes() === null ? 'none' : sprintf('%.2f GiB', $t->getMaxBytes() / 1073741824))); + return 0; + } +} diff --git a/lib/Db/Target.php b/lib/Db/Target.php index 0cab9d6..f3a316c 100644 --- a/lib/Db/Target.php +++ b/lib/Db/Target.php @@ -24,6 +24,8 @@ use OCP\AppFramework\Db\Entity; * @method void setRepositoryId(?string $v) * @method bool|null getAppendOnly() * @method void setAppendOnly(bool $v) + * @method int|null getMaxBytes() + * @method void setMaxBytes(?int $v) * @method int getCreatedAt() * @method void setCreatedAt(int $v) */ @@ -36,9 +38,11 @@ class Target extends Entity { protected ?string $repositoryId = null; protected int $createdAt = 0; protected ?bool $appendOnly = null; + protected ?int $maxBytes = null; public function __construct() { $this->addType('createdAt', 'integer'); $this->addType('appendOnly', 'boolean'); + $this->addType('maxBytes', 'integer'); } } diff --git a/lib/Migration/Version000004Date20261002030000.php b/lib/Migration/Version000004Date20261002030000.php new file mode 100644 index 0000000..a2e52a6 --- /dev/null +++ b/lib/Migration/Version000004Date20261002030000.php @@ -0,0 +1,28 @@ +getTable('ngb_targets'); + if (!$t->hasColumn('max_bytes')) { + $t->addColumn('max_bytes', Types::BIGINT, ['notnull' => false, 'unsigned' => true]); + return $schema; + } + return null; + } +} diff --git a/lib/Repository/BlobIndex.php b/lib/Repository/BlobIndex.php index ccaa4c0..cae45cf 100644 --- a/lib/Repository/BlobIndex.php +++ b/lib/Repository/BlobIndex.php @@ -139,6 +139,15 @@ final class BlobIndex { } } + /** Bytes of all blobs in the index (encrypted size, i.e. what is stored in packs). */ + public function storedBytes(): int { + $total = 0; + foreach ($this->blobs as [, , $length]) { + $total += $length; + } + return $total; + } + public function count(): int { return count($this->blobs); } diff --git a/lib/Repository/PackWriter.php b/lib/Repository/PackWriter.php index 7ad4d5e..8d8478b 100644 --- a/lib/Repository/PackWriter.php +++ b/lib/Repository/PackWriter.php @@ -32,6 +32,7 @@ final class PackWriter { private StreamCipher $cipher, private BlobIndex $index, private int $targetSize = 32 * 1048576, + private ?int $maxBytes = null, ) { $this->reset(); } @@ -64,6 +65,12 @@ final class PackWriter { if ($this->entries === []) { return; } + if ($this->maxBytes !== null && $this->index->storedBytes() + $this->size > $this->maxBytes) { + $this->index->dropPending(array_column($this->entries, 'id')); + $this->reset(); + throw new QuotaExceededException(sprintf('Storage limit of this location reached (%.1f GiB): backup stopped', + $this->maxBytes / 1073741824)); + } $packId = bin2hex(random_bytes(16)); $header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId); fwrite($this->buffer, $header . pack('N', strlen($header))); diff --git a/lib/Repository/QuotaExceededException.php b/lib/Repository/QuotaExceededException.php new file mode 100644 index 0000000..2e970af --- /dev/null +++ b/lib/Repository/QuotaExceededException.php @@ -0,0 +1,11 @@ + array_map('basename', $this->backend->list('trash/info'))); } + /** Refuse uploads that would make the stored data larger than this (null = no limit). */ + public function setMaxBytes(?int $maxBytes): void { + $this->maxBytes = $maxBytes; + } + + public function storedBytes(): int { + return $this->index->storedBytes(); + } + /** Current catalog generation (for status and tests). */ public function catalogGeneration(): int { return $this->catalog->gen; @@ -145,7 +155,7 @@ final class Repository { public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array { $source = rtrim($source, '/'); $previous = $parent !== null ? $this->loadEntries($parent) : []; - $packs = new PackWriter($this->backend, $this->cipher, $this->index); + $packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes); $stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0]; $tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b'); @@ -226,7 +236,7 @@ final class Repository { * run is done, so the last pack is uploaded before anything references it. */ public function blobWriter(array &$stats): BlobWriter { - $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index); + $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes); $packs = $this->runPacks; return new BlobWriter(function (string $data) use ($packs, &$stats): string { return $this->storeBlob($data, $packs, $stats); @@ -258,7 +268,7 @@ final class Repository { /** Store one blob of file data in the current run (deduplicated); returns its id. */ public function storeData(string $data, array &$stats): string { - $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index); + $this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes); return $this->storeBlob($data, $this->runPacks, $stats); } @@ -481,6 +491,7 @@ final class Repository { } else { $stats['repacked']++; if (!$dryRun) { + // No size limit here: repacking is how space is freed. $writer ??= new PackWriter($this->backend, $this->cipher, $this->index); $path = 'packs/' . substr($packId, 0, 2) . '/' . $packId; foreach ($blobs as $b) { diff --git a/lib/Service/TargetService.php b/lib/Service/TargetService.php index 7b0ffc3..e9e9d25 100644 --- a/lib/Service/TargetService.php +++ b/lib/Service/TargetService.php @@ -90,6 +90,12 @@ class TargetService { return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId); } + /** Size limit for the data stored on this location (null = none). */ + public function setMaxBytes(Target $target, ?int $maxBytes): Target { + $target->setMaxBytes($maxBytes); + return $this->mapper->update($target); + } + public function setAppendOnly(Target $target, bool $appendOnly): Target { $target->setAppendOnly($appendOnly); return $this->mapper->update($target); @@ -109,6 +115,7 @@ class TargetService { if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) { throw new \RuntimeException('The target now contains a different repository than when it was added'); } + $repo->setMaxBytes($target->getMaxBytes()); return $repo; } diff --git a/tests/Unit/Repository/RepositoryTest.php b/tests/Unit/Repository/RepositoryTest.php index 9d01f5b..c17e3b1 100644 --- a/tests/Unit/Repository/RepositoryTest.php +++ b/tests/Unit/Repository/RepositoryTest.php @@ -146,4 +146,16 @@ class RepositoryTest extends TestCase { $this->assertFileDoesNotExist(dirname($target, 2) . '/escaped.txt'); $this->assertFileDoesNotExist(dirname($target) . '/escaped.txt'); } + + public function testStorageLimitStopsTheBackup(): void { + $repo = Repository::init($this->backend, 'pw'); + $repo->setMaxBytes(6 * 1048576); // less than the test tree + try { + $repo->backupDirectory($this->src); + $this->fail('backup should hit the limit'); + } catch (\OCA\NgBackup\Repository\QuotaExceededException) { + } + $this->assertLessThanOrEqual(6 * 1048576, $repo->storedBytes()); + $this->assertSame([], $repo->listSnapshots(), 'no snapshot from an incomplete backup'); + } }