99f1655d04
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / psalm (push) Has been cancelled
CI / frontend (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
CI / integration-targets (push) Has been cancelled
CI / integration-user (32) (push) Has been cancelled
CI / integration-user (33) (push) Has been cancelled
CI / integration-user (34) (push) Has been cancelled
CI / integration-user (35) (push) Has been cancelled
CI / integration-full-restore (push) Has been cancelled
- A location that a passphrase change could not reach gets it at its next backup - A key from before 1.1.0 is never replaced as an unused one - An unreadable settings file on a location is told apart from a missing one - A schedule is shown and saved in the zone it was saved in - Saving an unchanged schedule keeps its moment; --timezone needs a schedule - Schedule coverage uses the time the backup began reading its data - Tests: catch-up scenario, upgrade test restores the new file, autumn clock change
142 lines
5.5 KiB
PHP
142 lines
5.5 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
namespace OCA\NgBackup\BackgroundJob;
|
|
|
|
use OCA\NgBackup\AppInfo\Application;
|
|
use OCA\NgBackup\Db\RunMapper;
|
|
use OCA\NgBackup\Service\BackupService;
|
|
use OCA\NgBackup\Service\TargetService;
|
|
use OCP\AppFramework\Utility\ITimeFactory;
|
|
use OCP\BackgroundJob\IJob;
|
|
use OCP\BackgroundJob\TimedJob;
|
|
use OCP\IAppConfig;
|
|
use Psr\Log\LoggerInterface;
|
|
|
|
/**
|
|
* Starts scheduled backups and moves running ones forward, a time-boxed step per cron run.
|
|
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
|
|
* in a normal web request (hosted Nextcloud).
|
|
*/
|
|
final class BackupJob extends TimedJob {
|
|
public function __construct(
|
|
ITimeFactory $time,
|
|
private BackupService $backups,
|
|
private \OCA\NgBackup\Service\PruneService $prune,
|
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
private TargetService $targets,
|
|
private RunMapper $runs,
|
|
private IAppConfig $appConfig,
|
|
private LoggerInterface $logger,
|
|
private \OCA\NgBackup\Service\Schedule $schedule,
|
|
private \OCA\NgBackup\Service\HeavyJobPause $heavyJobs,
|
|
) {
|
|
parent::__construct($time);
|
|
// Well below the usual 5-minute cron interval: the last run is stamped a moment after cron
|
|
// starts, so with 300 the next cron run came a few seconds too early and the job ran only
|
|
// every other time, idle for 6 of every 10 minutes during a backup. When nothing runs, the
|
|
// job only does a few cheap database checks.
|
|
$this->setInterval(60);
|
|
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
|
}
|
|
|
|
protected function run($argument): void {
|
|
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
|
$deadline = microtime(true) + $budget;
|
|
$this->startScheduled();
|
|
$this->heavyJobs->sync($this->backups->runsInTurn() !== []);
|
|
// One after the other: runs that read the same disks at the same time are not faster together.
|
|
foreach ($this->backups->runsInTurn() as $run) {
|
|
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
|
$next = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
|
if ($next === $run) {
|
|
break 2; // step() hands back the same object when another process holds the run's lease
|
|
}
|
|
$run = $next;
|
|
}
|
|
}
|
|
$this->failStaleRestores();
|
|
$this->weeklyPrune();
|
|
$targets = $this->targets->list();
|
|
foreach ($targets as $t) {
|
|
$this->alerts->checkStale($t);
|
|
$othersActive = array_filter($targets, static fn ($o) => $o->getId() !== $t->getId() && (string)$o->getSchedule() !== '') !== [];
|
|
$this->alerts->checkOff($t, $othersActive);
|
|
}
|
|
}
|
|
|
|
/** A restore run that has not moved for 6 hours is considered dead (it would block cleanup forever). */
|
|
private function failStaleRestores(): void {
|
|
foreach ($this->runs->findRunning(null, 'restore') as $run) {
|
|
if (time() - $run->getUpdatedAt() > 6 * 3600) {
|
|
$run->setStatus(\OCA\NgBackup\Db\Run::FAILED);
|
|
$run->setError('No progress for 6 hours; marked as failed');
|
|
$run->setFinishedAt(time());
|
|
$this->runs->update($run);
|
|
$this->alerts->runFailed($run, '#' . $run->getTargetId());
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Once a week, apply the retention policy to non-append-only locations: from Sunday on, at the
|
|
* first moment no backup is running, so a long Sunday backup only delays it instead of skipping
|
|
* the week.
|
|
*/
|
|
private function weeklyPrune(): void {
|
|
$lastWeek = $this->appConfig->getValueString(Application::APP_ID, 'last_prune_week', '');
|
|
$due = date('N') === '7' ? date('o-W') : date('o-W', (int)strtotime('last sunday'));
|
|
if (strcmp($lastWeek, $due) >= 0 || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
|
|
return;
|
|
}
|
|
foreach ($this->targets->list() as $t) {
|
|
if ($t->getAppendOnly()) {
|
|
continue;
|
|
}
|
|
try {
|
|
$this->prune->apply($t, false, true);
|
|
} catch (\Throwable $e) {
|
|
$this->logger->warning('NG Backup: cleanup of ' . $t->getName() . ' failed: ' . $e->getMessage());
|
|
}
|
|
}
|
|
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', $due);
|
|
}
|
|
|
|
/** Start the backups the schedules of the locations ask for; they run in turn (see LocationSchedule). */
|
|
private function startScheduled(): void {
|
|
$now = time();
|
|
foreach ($this->targets->list() as $t) {
|
|
$moment = \OCA\NgBackup\Service\LocationSchedule::lastMoment((string)$t->getSchedule(), $this->schedule->zoneOr($t->getScheduleZone()), $now);
|
|
if ($moment === null || $this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== []) {
|
|
continue;
|
|
}
|
|
$attempts = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $moment);
|
|
if (!\OCA\NgBackup\Service\LocationSchedule::isDue($moment, (int)$t->getScheduleSince(),
|
|
$this->dataStarted($this->runs->lastFinished($t->getId(), BackupService::KIND_FULL)),
|
|
count($attempts), ($attempts[0] ?? null)?->getFinishedAt(), $now)) {
|
|
continue;
|
|
}
|
|
try {
|
|
$this->backups->start($t, 'scheduled');
|
|
} catch (\Throwable $e) {
|
|
$this->logger->warning('NG Backup: scheduled backup to ' . $t->getName() . ' not started: ' . $e->getMessage());
|
|
}
|
|
}
|
|
}
|
|
|
|
/** When a backup began reading its data: a queued one can start well after it was asked for. */
|
|
private function dataStarted(?\OCA\NgBackup\Db\Run $run): ?int {
|
|
if ($run === null) {
|
|
return null;
|
|
}
|
|
$state = json_decode($run->getState(), true);
|
|
$started = is_array($state) ? ($state['files']['started'] ?? null) : null;
|
|
$at = is_string($started) ? strtotime($started) : false;
|
|
return $at !== false ? $at : $run->getStartedAt();
|
|
}
|
|
}
|