Run the backup job at every cron run; skip deflate on compressed data

CHANGELOG.md

@@ -21,6 +21,12 @@ instead carried by the app description, the git tag, the release, and this file.
   ("1d 12:10:13" instead of "12:10:13").
 
 ### Fixed
+- With system cron, a running backup moved forward only every other cron run: Nextcloud
+  skipped the background job when cron came a few seconds before its 5-minute interval was
+  over, so a backup was idle 6 of every 10 minutes. The job now runs at every cron run.
+- Photos, video and other compressed data are no longer deflated only to throw the result away:
+  a quick test on three small slices of each block skips it. Compressing was the slowest part of
+  a backup (about 24 MiB/s).
 - Checking a backup on an S3 location (`occ backup:verify`) took hours on a large repository:
   the index files were downloaded one at a time, at about two a second. They are now downloaded
   16 at a time: on a production instance (27,585 index files, 1 TB) the check went from about

lib/BackgroundJob/BackupJob.php

@@ -34,7 +34,11 @@ final class BackupJob extends TimedJob {
 		private LoggerInterface $logger,
 	) {
 		parent::__construct($time);
-		$this->setInterval(300);
+		// Well below the usual 5-minute cron interval: the last run is stamped a moment after cron
+		// starts, so with 300 the next cron run came a few seconds too early and the job ran only
+		// every other time, idle for 6 of every 10 minutes during a backup. When nothing runs, the
+		// job only does a few cheap database checks.
+		$this->setInterval(60);
 		$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
 	}
 

lib/Repository/Repository.php

@@ -28,6 +28,10 @@ final class Repository {
 	public const FORMAT = 1;
 	public const BLOB_SIZE = 4 * 1048576;
 	private const FLAG_DEFLATE = 1;
+	/** Blobs from this size on get a quick compression test before the full compression. */
+	private const DEFLATE_PROBE_FROM = 256 * 1024;
+	/** Bytes of each of the three slices (start, middle, end) in that test. */
+	private const DEFLATE_PROBE_SLICE = 32 * 1024;
 	/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
 	private const VERIFY_READ_ATTEMPTS = 3;
 	/** Largest read that loadBlobs() makes of blobs that follow each other in a pack. */
@@ -703,7 +707,7 @@ final class Repository {
 		}
 		$flags = 0;
 		$payload = $data;
-		$deflated = gzdeflate($data, 6);
+		$deflated = self::worthDeflating($data) ? gzdeflate($data, 6) : false;
 		if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
 			$payload = $deflated;
 			$flags |= self::FLAG_DEFLATE;
@@ -713,6 +717,23 @@ final class Repository {
 		return $id;
 	}
 
+	/**
+	 * Photos, video and archives are compressed already: deflating them takes most of a backup's
+	 * CPU time (about 24 MiB/s) only to be thrown away, as it saves less than the 5% that keeps
+	 * it. A fast deflate of three small slices tells first; below 10% savings there, the full
+	 * deflate is skipped.
+	 */
+	private static function worthDeflating(string $data): bool {
+		$n = strlen($data);
+		if ($n < self::DEFLATE_PROBE_FROM) {
+			return true;
+		}
+		$k = self::DEFLATE_PROBE_SLICE;
+		$probe = substr($data, 0, $k) . substr($data, intdiv($n - $k, 2), $k) . substr($data, $n - $k);
+		$deflated = gzdeflate($probe, 1);
+		return $deflated !== false && strlen($deflated) * 10 < strlen($probe) * 9;
+	}
+
 	/**
 	 * Blobs spread over many packs (the tree segments of a run, one per step), through
 	 * IBackend::getRanges(): on a remote location one at a time is a round trip each, which for

tests/Unit/Repository/RepositoryTest.php

@@ -66,6 +66,20 @@ class RepositoryTest extends TestCase {
 		$this->assertSame(1, $r2['newBlobs']);
 	}
 
+	public function testCompressedDataSkipsDeflateAndOtherDataIsStillCompressed(): void {
+		$repo = Repository::init($this->backend, 'pw');
+		$text = str_repeat("a line of a log file, much like the one before\n", 30000); // ~1.4 MiB
+		$noise = random_bytes(1048576); // like a photo: deflate saves nothing
+		$stats = [];
+		$textId = $repo->storeData($text, $stats);
+		$textUploaded = $stats['uploaded'];
+		$noiseId = $repo->storeData($noise, $stats);
+		$repo->flushPacks();
+		$this->assertLessThan(strlen($text) / 10, $textUploaded);
+		$this->assertLessThan(strlen($noise) + 1024, $stats['uploaded'] - $textUploaded);
+		$this->assertSame($text . $noise, implode('', iterator_to_array(Repository::open($this->backend, 'pw')->readBlobs([$textId, $noiseId]), false)));
+	}
+
 	public function testPartialRestore(): void {
 		$repo = Repository::init($this->backend, 'pw');
 		$r = $repo->backupDirectory($this->src);