From 144097d15911817f243daa8b3b0f74c6d5c8e48e Mon Sep 17 00:00:00 2001 From: Dillard Blom Date: Thu, 15 Oct 2026 20:59:04 +0200 Subject: [PATCH] Run the backup job at every cron run; skip deflate on compressed data --- CHANGELOG.md | 6 ++++++ lib/BackgroundJob/BackupJob.php | 6 +++++- lib/Repository/Repository.php | 23 ++++++++++++++++++++++- tests/Unit/Repository/RepositoryTest.php | 14 ++++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cee3c9b..1cd6e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,12 @@ instead carried by the app description, the git tag, the release, and this file. ("1d 12:10:13" instead of "12:10:13"). ### Fixed +- With system cron, a running backup moved forward only every other cron run: Nextcloud + skipped the background job when cron came a few seconds before its 5-minute interval was + over, so a backup was idle 6 of every 10 minutes. The job now runs at every cron run. +- Photos, video and other compressed data are no longer deflated only to throw the result away: + a quick test on three small slices of each block skips it. Compressing was the slowest part of + a backup (about 24 MiB/s). - Checking a backup on an S3 location (`occ backup:verify`) took hours on a large repository: the index files were downloaded one at a time, at about two a second. They are now downloaded 16 at a time: on a production instance (27,585 index files, 1 TB) the check went from about diff --git a/lib/BackgroundJob/BackupJob.php b/lib/BackgroundJob/BackupJob.php index cce6c89..4171fe4 100644 --- a/lib/BackgroundJob/BackupJob.php +++ b/lib/BackgroundJob/BackupJob.php @@ -34,7 +34,11 @@ final class BackupJob extends TimedJob { private LoggerInterface $logger, ) { parent::__construct($time); - $this->setInterval(300); + // Well below the usual 5-minute cron interval: the last run is stamped a moment after cron + // starts, so with 300 the next cron run came a few seconds too early and the job ran only + // every other time, idle for 6 of every 10 minutes during a backup. When nothing runs, the + // job only does a few cheap database checks. + $this->setInterval(60); $this->setTimeSensitivity(IJob::TIME_SENSITIVE); } diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index 94efd0d..9eb0cc7 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -28,6 +28,10 @@ final class Repository { public const FORMAT = 1; public const BLOB_SIZE = 4 * 1048576; private const FLAG_DEFLATE = 1; + /** Blobs from this size on get a quick compression test before the full compression. */ + private const DEFLATE_PROBE_FROM = 256 * 1024; + /** Bytes of each of the three slices (start, middle, end) in that test. */ + private const DEFLATE_PROBE_SLICE = 32 * 1024; /** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */ private const VERIFY_READ_ATTEMPTS = 3; /** Largest read that loadBlobs() makes of blobs that follow each other in a pack. */ @@ -703,7 +707,7 @@ final class Repository { } $flags = 0; $payload = $data; - $deflated = gzdeflate($data, 6); + $deflated = self::worthDeflating($data) ? gzdeflate($data, 6) : false; if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) { $payload = $deflated; $flags |= self::FLAG_DEFLATE; @@ -713,6 +717,23 @@ final class Repository { return $id; } + /** + * Photos, video and archives are compressed already: deflating them takes most of a backup's + * CPU time (about 24 MiB/s) only to be thrown away, as it saves less than the 5% that keeps + * it. A fast deflate of three small slices tells first; below 10% savings there, the full + * deflate is skipped. + */ + private static function worthDeflating(string $data): bool { + $n = strlen($data); + if ($n < self::DEFLATE_PROBE_FROM) { + return true; + } + $k = self::DEFLATE_PROBE_SLICE; + $probe = substr($data, 0, $k) . substr($data, intdiv($n - $k, 2), $k) . substr($data, $n - $k); + $deflated = gzdeflate($probe, 1); + return $deflated !== false && strlen($deflated) * 10 < strlen($probe) * 9; + } + /** * Blobs spread over many packs (the tree segments of a run, one per step), through * IBackend::getRanges(): on a remote location one at a time is a round trip each, which for diff --git a/tests/Unit/Repository/RepositoryTest.php b/tests/Unit/Repository/RepositoryTest.php index c17e3b1..b8710d6 100644 --- a/tests/Unit/Repository/RepositoryTest.php +++ b/tests/Unit/Repository/RepositoryTest.php @@ -66,6 +66,20 @@ class RepositoryTest extends TestCase { $this->assertSame(1, $r2['newBlobs']); } + public function testCompressedDataSkipsDeflateAndOtherDataIsStillCompressed(): void { + $repo = Repository::init($this->backend, 'pw'); + $text = str_repeat("a line of a log file, much like the one before\n", 30000); // ~1.4 MiB + $noise = random_bytes(1048576); // like a photo: deflate saves nothing + $stats = []; + $textId = $repo->storeData($text, $stats); + $textUploaded = $stats['uploaded']; + $noiseId = $repo->storeData($noise, $stats); + $repo->flushPacks(); + $this->assertLessThan(strlen($text) / 10, $textUploaded); + $this->assertLessThan(strlen($noise) + 1024, $stats['uploaded'] - $textUploaded); + $this->assertSame($text . $noise, implode('', iterator_to_array(Repository::open($this->backend, 'pw')->readBlobs([$textId, $noiseId]), false))); + } + public function testPartialRestore(): void { $repo = Repository::init($this->backend, 'pw'); $r = $repo->backupDirectory($this->src);