diff --git a/CHANGELOG.md b/CHANGELOG.md index 122fbfd..2d2e4d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ shown under 1.0.0. The git tag of a release candidate goes after the date. - All folders get their original modification time back on a restore, not only the empty ones, from the first backup made with this version. An older version of NG Backup can still read new restore points. A restore through Nextcloud into a folder that is already there leaves its time alone. +- Heavy background jobs (Memories, Recognize, Preview Generator, full text search, the antivirus + background scan) pause while a backup runs, at most 6 hours in a row, then they get an hour. No app + is turned off, and a job that is running goes on. On by default; turn it off under Backups or with + `occ backup:pause-jobs off`. ### Changed - Locations are backed up one after the other: a backup that has to wait shows as queued. A restore diff --git a/PLAN.md b/PLAN.md index 175d94c..6a0e625 100644 --- a/PLAN.md +++ b/PLAN.md @@ -579,7 +579,11 @@ in README "Known limitations": mappen in de backup ontbreken, geeft voorbeelden van de paden en vraagt ze te hernoemen; de volgende backup neemt ze mee. Alleen als Nextcloud-melding: de app mailt niet zelf, of meldingen gemaild worden is de keuze van de beheerder (besluit 2026-11-06). - **Stand op branch `1.1.0` (2026-11-07):** klaar: betere melding, punten 4, 5, 6 en 8. Pin en notitie staan + **Stand op branch `1.1.0` (2026-11-07):** klaar: betere melding, punten 2, 4, 5, 6 en 8, en de maptijden + (`files:scan` na een volledige restore: 0 "Updated" in de simulatie; opslag gepakt in het metarecord). + Punt 2 reserveert de jobs in `oc_jobs` met één tijdstempel, alleen wat Nextcloud op dat moment zou kunnen + starten; 6 uur aan, dan 1 uur vrij; bij een crash geeft Nextcloud de reservering na 12 uur zelf vrij; + de logregels staan op niveau info, dus standaard niet zichtbaar. Pin en notitie staan in de catalogus, met een kopie in de database voor de lijsten. Een moment van vóór het instellen van een schema start niets. Geen schema: geen melding "geen recente backup", wel een melding aan alle beheerders bij het uitzetten en een herinnering, wekelijks zolang een andere locatie een schema heeft, diff --git a/README.md b/README.md index f13749e..3b6300a 100644 --- a/README.md +++ b/README.md @@ -336,6 +336,7 @@ On the backlog; know them before you rely on NG Backup: | `backup:browse`, `backup:restore` | browse and restore | | `backup:retention`, `backup:prune`, `backup:trash`, `backup:trash:restore` | retention, cleanup, trash | | `backup:pin`, `backup:note` | keep a restore point, a note you can change later | +| `backup:pause-jobs` | whether heavy background jobs of other apps pause during a backup | | `backup:verify` | checksum audit of a restore point (`--deep` downloads and decrypts everything) | | `backup:restore:full` | whole-server restore onto a fresh installation (after `backup:key:import`) | | `backup:key:import` | import a recovery kit on a fresh installation | diff --git a/appinfo/info.xml b/appinfo/info.xml index a20fdd1..de2dba1 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -83,6 +83,7 @@ you need help with restoring a backup after a crash.]]> OCA\NgBackup\Command\ListSnapshots OCA\NgBackup\Command\Browse OCA\NgBackup\Command\Restore + OCA\NgBackup\Command\PauseJobs OCA\NgBackup\Command\Pin OCA\NgBackup\Command\Note OCA\NgBackup\Command\Prune diff --git a/appinfo/routes.php b/appinfo/routes.php index 467974d..5714bda 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -25,6 +25,7 @@ return [ ['name' => 'AdminApi#setTargetLimit', 'url' => '/api/targets/{id}/limit', 'verb' => 'PUT'], ['name' => 'AdminApi#startBackup', 'url' => '/api/backups', 'verb' => 'POST'], ['name' => 'AdminApi#tick', 'url' => '/api/tick', 'verb' => 'POST'], + ['name' => 'AdminApi#setPauseHeavyJobs', 'url' => '/api/settings/pause-heavy-jobs', 'verb' => 'PUT'], ['name' => 'AdminApi#setTargetSchedule', 'url' => '/api/targets/{id}/schedule', 'verb' => 'PUT'], ['name' => 'AdminApi#snapshots', 'url' => '/api/targets/{targetId}/snapshots', 'verb' => 'GET'], ['name' => 'AdminApi#setSnapshotNote', 'url' => '/api/targets/{targetId}/snapshots/{snapshotId}/note', 'verb' => 'PUT'], diff --git a/lib/BackgroundJob/BackupJob.php b/lib/BackgroundJob/BackupJob.php index f91acf8..1e57bfc 100644 --- a/lib/BackgroundJob/BackupJob.php +++ b/lib/BackgroundJob/BackupJob.php @@ -33,6 +33,7 @@ final class BackupJob extends TimedJob { private IAppConfig $appConfig, private LoggerInterface $logger, private \OCA\NgBackup\Service\Schedule $schedule, + private \OCA\NgBackup\Service\HeavyJobPause $heavyJobs, ) { parent::__construct($time); // Well below the usual 5-minute cron interval: the last run is stamped a moment after cron @@ -47,6 +48,7 @@ final class BackupJob extends TimedJob { $budget = PHP_SAPI === 'cli' ? 240.0 : 20.0; $deadline = microtime(true) + $budget; $this->startScheduled(); + $this->heavyJobs->sync($this->backups->runsInTurn() !== []); // One after the other: runs that read the same disks at the same time are not faster together. foreach ($this->backups->runsInTurn() as $run) { while (microtime(true) < $deadline && $run->getStatus() === 'running') { diff --git a/lib/Command/PauseJobs.php b/lib/Command/PauseJobs.php new file mode 100644 index 0000000..232a1ea --- /dev/null +++ b/lib/Command/PauseJobs.php @@ -0,0 +1,45 @@ +setName('backup:pause-jobs') + ->setDescription('Show or set whether heavy background jobs (Memories, Recognize, Preview Generator, ...) pause while a backup runs') + ->addArgument('setting', InputArgument::OPTIONAL, '"on" or "off"; omit to show the current one'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $setting = $input->getArgument('setting'); + if ($setting !== null) { + if ($setting !== 'on' && $setting !== 'off') { + $output->writeln('Use "on" or "off".'); + return 1; + } + $this->pause->setEnabled($setting === 'on'); + } + $status = $this->pause->status(); + $output->writeln('Pausing heavy background jobs during a backup: ' . ($status['enabled'] ? 'on' : 'off')); + if ($status['jobs'] !== []) { + $output->writeln('Paused now: ' . implode(', ', $status['jobs'])); + } + return 0; + } +} diff --git a/lib/Controller/AdminApiController.php b/lib/Controller/AdminApiController.php index eddcd45..0466191 100644 --- a/lib/Controller/AdminApiController.php +++ b/lib/Controller/AdminApiController.php @@ -48,6 +48,7 @@ final class AdminApiController extends Controller { private \OCA\NgBackup\Service\KeySlotService $slots, private \OCA\NgBackup\Service\PruneService $prune, private \OCA\NgBackup\Service\Schedule $schedule, + private \OCA\NgBackup\Service\HeavyJobPause $heavyJobs, private \OCA\NgBackup\Service\CronHealth $cron, ) { parent::__construct(Application::APP_ID, $request); @@ -77,9 +78,15 @@ final class AdminApiController extends Controller { 'olderRuns' => $allRuns ? 0 : $this->runs->countAll() - count($shown), 'recentDays' => \OCA\NgBackup\Command\Status::RECENT_DAYS, 'cron' => $this->cron->check(), + 'heavyJobs' => $this->heavyJobs->status(), ]); } + public function setPauseHeavyJobs(bool $enabled): JSONResponse { + $this->heavyJobs->setEnabled($enabled); + return new JSONResponse(['enabled' => $enabled]); + } + /** Seconds of backup work one tick() does: well within a normal web request. */ public const TICK_SECONDS = 10.0; /** Seconds one request for the Bin reads, before the page shows how far it is and asks for more. */ @@ -97,6 +104,7 @@ final class AdminApiController extends Controller { } $deadline = microtime(true) + self::TICK_SECONDS; $stepped = 0; + $this->heavyJobs->sync($this->backups->runsInTurn() !== []); foreach ($this->backups->runsInTurn() as $run) { if (microtime(true) >= $deadline) { break; diff --git a/lib/Service/HeavyJobPause.php b/lib/Service/HeavyJobPause.php new file mode 100644 index 0000000..6a9ad3d --- /dev/null +++ b/lib/Service/HeavyJobPause.php @@ -0,0 +1,156 @@ + what to call it */ + private const JOBS = [ + 'OCA\\Memories\\' => 'Memories', + 'OCA\\Files_Antivirus\\BackgroundJob\\BackgroundScanner' => 'Antivirus background scan', + 'OCA\\Recognize\\' => 'Recognize', + 'OCA\\PreviewGenerator\\' => 'Preview Generator', + 'OCA\\FullTextSearch\\' => 'Full text search', + ]; + + public const NOTHING = 'nothing'; + public const PAUSE = 'pause'; + public const RELEASE = 'release'; + public const REST = 'rest'; + + public function __construct( + private IDBConnection $db, + private IAppConfig $appConfig, + private LoggerInterface $logger, + ) { + } + + /** @return self::NOTHING|self::PAUSE|self::RELEASE|self::REST */ + public static function decide(bool $enabled, bool $backupRunning, int $since, int $restUntil, int $now): string { + if ($since > 0 && $now - $since >= self::MAX_SECONDS) { + return self::REST; + } + if ($enabled && $backupRunning && $now >= $restUntil) { + return self::PAUSE; + } + return $since > 0 ? self::RELEASE : self::NOTHING; + } + + /** Call regularly: pauses or ends the pause as needed. A problem here never stops a backup. */ + public function sync(bool $backupRunning): void { + try { + $this->apply($backupRunning, time()); + } catch (\Throwable $e) { + $this->logger->warning('NG Backup: pausing heavy background jobs failed: ' . $e->getMessage(), ['exception' => $e]); + } + } + + private function apply(bool $backupRunning, int $now): void { + $since = $this->appConfig->getValueInt(Application::APP_ID, self::SINCE_KEY, 0); + $restUntil = $this->appConfig->getValueInt(Application::APP_ID, self::REST_KEY, 0); + switch (self::decide($this->enabled(), $backupRunning, $since, $restUntil, $now)) { + case self::PAUSE: + if ($since === 0) { + $since = $now; + $this->appConfig->setValueInt(Application::APP_ID, self::SINCE_KEY, $since); + $this->logger->info('NG Backup: heavy background jobs paused while a backup runs'); + } + $this->reserve($since, $now); // also the jobs that were added or running until now + break; + case self::REST: + $this->appConfig->setValueInt(Application::APP_ID, self::REST_KEY, $now + self::REST_SECONDS); + $this->release($since); + $this->logger->info('NG Backup: heavy background jobs get their turn for an hour during a long backup'); + break; + case self::RELEASE: + $this->release($since); + $this->logger->info('NG Backup: heavy background jobs run again'); + break; + } + } + + private function reserve(int $stamp, int $now): void { + $qb = $this->db->getQueryBuilder(); + $qb->update('jobs') + ->set('reserved_at', $qb->createNamedParameter($stamp, IQueryBuilder::PARAM_INT)) + ->where($qb->expr()->lte('reserved_at', $qb->createNamedParameter($now - self::RESERVATION_SECONDS, IQueryBuilder::PARAM_INT))) + ->andWhere($this->isHeavy($qb)); + $qb->executeStatement(); + } + + /** Only our own reservations: a job that Nextcloud reserved at another moment is running. */ + private function release(int $stamp): void { + $qb = $this->db->getQueryBuilder(); + $qb->update('jobs') + ->set('reserved_at', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ->where($qb->expr()->eq('reserved_at', $qb->createNamedParameter($stamp, IQueryBuilder::PARAM_INT))) + ->andWhere($this->isHeavy($qb)); + $qb->executeStatement(); + $this->appConfig->deleteKey(Application::APP_ID, self::SINCE_KEY); + } + + /** @return \OCP\DB\QueryBuilder\ICompositeExpression */ + private function isHeavy(IQueryBuilder $qb) { + $conditions = []; + foreach (array_keys(self::JOBS) as $class) { + $conditions[] = str_ends_with($class, '\\') + ? $qb->expr()->like('class', $qb->createNamedParameter($this->db->escapeLikeParameter($class) . '%')) + : $qb->expr()->eq('class', $qb->createNamedParameter($class)); + } + return $qb->expr()->orX(...$conditions); + } + + public function enabled(): bool { + return $this->appConfig->getValueString(Application::APP_ID, self::ENABLED_KEY, '1') !== '0'; + } + + public function setEnabled(bool $enabled): void { + $this->appConfig->setValueString(Application::APP_ID, self::ENABLED_KEY, $enabled ? '1' : '0'); + } + + /** @return array{enabled: bool, since: int, jobs: list} what is paused now */ + public function status(): array { + $since = $this->appConfig->getValueInt(Application::APP_ID, self::SINCE_KEY, 0); + $jobs = []; + if ($since > 0) { + $qb = $this->db->getQueryBuilder(); + $qb->selectDistinct('class')->from('jobs') + ->where($qb->expr()->eq('reserved_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT))) + ->andWhere($this->isHeavy($qb)); + $result = $qb->executeQuery(); + while (($class = $result->fetchOne()) !== false) { + foreach (self::JOBS as $start => $name) { + if (str_starts_with((string)$class, $start)) { + $jobs[$name] = true; + } + } + } + $result->closeCursor(); + } + return ['enabled' => $this->enabled(), 'since' => $since, 'jobs' => array_map('strval', array_keys($jobs))]; + } +} diff --git a/src/api.js b/src/api.js index 1000eca..f470947 100644 --- a/src/api.js +++ b/src/api.js @@ -34,6 +34,7 @@ export default { setTargetLimit: (id, maxGb) => call('put', `/targets/${id}/limit`, { maxGb }), startBackup: (targetId, label) => call('post', '/backups', { targetId, label }), tick: () => call('post', '/tick'), + setPauseHeavyJobs: (enabled) => call('put', '/settings/pause-heavy-jobs', { enabled }), setTargetSchedule: (id, schedule, timezone) => call('put', `/targets/${id}/schedule`, { schedule, timezone }), snapshots: (targetId) => call('get', `/targets/${targetId}/snapshots`), setSnapshotNote: (targetId, snapshotId, note) => call('put', `/targets/${targetId}/snapshots/${snapshotId}/note`, { note }), diff --git a/src/components/Backups.vue b/src/components/Backups.vue index e1a8310..9395aab 100644 --- a/src/components/Backups.vue +++ b/src/components/Backups.vue @@ -6,6 +6,10 @@ + + {{ t('ng_backup', 'Pause heavy background jobs (Memories, Recognize, Preview Generator, full text search, antivirus scan) while a backup runs, at most 6 hours in a row') }} + +

{{ t('ng_backup', 'Paused now: {jobs}', { jobs: status.heavyJobs.jobs.join(', ') }) }}

@@ -60,6 +64,7 @@ import { ref } from 'vue' import { n, t } from '@nextcloud/l10n' import NcButton from '@nextcloud/vue/components/NcButton' +import NcCheckboxRadioSwitch from '@nextcloud/vue/components/NcCheckboxRadioSwitch' import NcNoteCard from '@nextcloud/vue/components/NcNoteCard' import NcSettingsSection from '@nextcloud/vue/components/NcSettingsSection' import NcTextField from '@nextcloud/vue/components/NcTextField' @@ -101,6 +106,16 @@ const runStatus = (run) => { const isRunning = (targetId) => props.status.runs.some(r => r.targetId === targetId && r.kind === 'full' && r.status === 'running') const targetName = (id) => props.status.targets.find(tg => tg.id === id)?.name ?? '#' + id +async function savePause(enabled) { + error.value = '' + try { + await api.setPauseHeavyJobs(enabled) + emit('changed') + } catch (e) { + error.value = e.message + } +} + async function start(target) { error.value = '' try { diff --git a/tests/Unit/Service/HeavyJobPauseTest.php b/tests/Unit/Service/HeavyJobPauseTest.php new file mode 100644 index 0000000..76e2aee --- /dev/null +++ b/tests/Unit/Service/HeavyJobPauseTest.php @@ -0,0 +1,35 @@ +assertSame(P::PAUSE, P::decide(true, true, 0, 0, self::NOW)); + $this->assertSame(P::PAUSE, P::decide(true, true, self::NOW - 60, 0, self::NOW), 'jobs added since then too'); + } + + public function testWithoutABackupNothingHappensOrThePauseEnds(): void { + $this->assertSame(P::NOTHING, P::decide(true, false, 0, 0, self::NOW)); + $this->assertSame(P::RELEASE, P::decide(true, false, self::NOW - 60, 0, self::NOW)); + } + + public function testTurningItOffEndsThePause(): void { + $this->assertSame(P::NOTHING, P::decide(false, true, 0, 0, self::NOW)); + $this->assertSame(P::RELEASE, P::decide(false, true, self::NOW - 60, 0, self::NOW)); + } + + public function testAfterSixHoursTheyGetTheirTurnAndThenPauseAgain(): void { + $this->assertSame(P::PAUSE, P::decide(true, true, self::NOW - P::MAX_SECONDS + 1, 0, self::NOW)); + $this->assertSame(P::REST, P::decide(true, true, self::NOW - P::MAX_SECONDS, 0, self::NOW)); + $rest = self::NOW + P::REST_SECONDS; + $this->assertSame(P::NOTHING, P::decide(true, true, 0, $rest, $rest - 1)); + $this->assertSame(P::PAUSE, P::decide(true, true, 0, $rest, $rest)); + } +} diff --git a/tests/integration/roundtrip.sh b/tests/integration/roundtrip.sh index d1daecd..19b8c41 100755 --- a/tests/integration/roundtrip.sh +++ b/tests/integration/roundtrip.sh @@ -136,6 +136,33 @@ done [ "$RUNNING" = 0 ] || fail "the queued backups did not finish" occ backup:list ci3 | grep " turn " >/dev/null || fail "the queued backup made no snapshot" ok "locations in turn: the second backup queued, and done after the first" +# Heavy background jobs of other apps wait during a backup, and only then. Ticks keep this one moving. +HEAVY='OCA\Recognize\BackgroundJobs\NgbTestJob' +reserved() { $X php -r 'require "lib/base.php"; $q = \OC::$server->get(\OCP\IDBConnection::class)->getQueryBuilder(); echo (int)$q->select("reserved_at")->from("jobs")->where($q->expr()->eq("class", $q->createNamedParameter($argv[1])))->executeQuery()->fetchOne();' "$HEAVY"; } +tick_until_done() { + for i in $(seq 1 60); do + api -X POST "$API/tick" >/dev/null || fail "tick" + RUNNING=$(api "$API/status" | php -r 'echo count(array_filter(json_decode(stream_get_contents(STDIN), true)["runs"], fn ($r) => $r["status"] === "running"));') + [ "$RUNNING" != 0 ] || return 0 + done + fail "the backup did not finish" +} +$X php -r 'require "lib/base.php"; \OC::$server->get(\OCP\BackgroundJob\IJobList::class)->add($argv[1]);' "$HEAVY" || fail "add a job" +[ "$(reserved)" = 0 ] || fail "a new job is reserved" +L3=$(echo "$IDS" | awk '{print $2}') +api -X POST -d "{\"targetId\": $L3, \"label\": \"heavy\"}" "$API/backups" >/dev/null || fail "start a backup" +tick_until_done +[ "$(reserved)" != 0 ] || fail "the job was not paused during the backup" +api -X POST "$API/tick" >/dev/null +[ "$(reserved)" = 0 ] || fail "the job stays paused after the backup" +occ backup:pause-jobs off >/dev/null || fail "turn off" +sleep 8 # APCu app config cache of the web server +api -X POST -d "{\"targetId\": $L3, \"label\": \"heavy\"}" "$API/backups" >/dev/null || fail "start a backup" +tick_until_done +[ "$(reserved)" = 0 ] || fail "the job was paused with the setting off" +occ backup:pause-jobs on >/dev/null +$X php -r 'require "lib/base.php"; \OC::$server->get(\OCP\BackgroundJob\IJobList::class)->remove($argv[1]);' "$HEAVY" +ok "heavy background jobs: paused during a backup, back after it, left alone with the setting off" occ background:cron >/dev/null $X php -f /var/www/html/cron.php >/dev/null 2>&1 || true occ backup:status | grep "system cron" >/dev/null && fail "backup:status still warns after cron ran"