Files
nextcloud-ng_backup/lib/BackgroundJob/BackupJob.php
T
dillard 0f4a238414 Pre-beta review: data safety in storage, catalog and backup runs
Storage and repository:
- Listing skipped every name containing ".part-", directories too, so the
  exports of a user like "john.part-time" were invisible to prune on S3,
  SFTP, FTP and SMB and their data was deleted. Only real temporary uploads
  ("<name>.part-<8 hex>") are skipped now.
- prune refuses to run while a recorded per-user export's manifest is
  missing, instead of freeing that export's data.
- forget() and untrash() copy first, switch the catalog, and delete the
  originals last, so a crash can no longer leave the catalog pointing at a
  missing snapshot or database dump.
- Writes are checked against the expected size, so a short upload is an
  error instead of a truncated pack; raw restores check rename() and touch().
- A catalog write refuses to overwrite a generation another process just
  wrote, and the catalog lock lasts 300 s, so opening a large repository
  inside it no longer lets the lease run out.

Backup runs and scheduling:
- A file that keeps changing is read again at most three times; then the
  last read is kept and reported, instead of restarting forever and
  blocking cleanup.
- Unreadable files and folders are recorded, logged, shown by occ
  backup:run, stored in the snapshot and notified to admins, instead of
  being skipped silently.
- After a failed scheduled backup, at most three attempts a day, an hour
  apart, instead of a new run and notification every five minutes.
- The job no longer spins on a run whose lease another process holds, and
  the weekly cleanup catches up after a Sunday spent backing up.
- Notifications for the per-user export retention events.
2026-10-06 23:48:40 +02:00

129 lines
4.6 KiB
PHP

<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\BackgroundJob;
use OCA\NgBackup\AppInfo\Application;
use OCA\NgBackup\Db\RunMapper;
use OCA\NgBackup\Service\BackupService;
use OCA\NgBackup\Service\TargetService;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJob;
use OCP\BackgroundJob\TimedJob;
use OCP\IAppConfig;
use Psr\Log\LoggerInterface;
/**
* Starts scheduled backups and moves running ones forward, a time-boxed step per cron run.
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
* in a normal web request (hosted Nextcloud).
*/
final class BackupJob extends TimedJob {
public function __construct(
ITimeFactory $time,
private BackupService $backups,
private \OCA\NgBackup\Service\PruneService $prune,
private \OCA\NgBackup\Service\AlertService $alerts,
private TargetService $targets,
private RunMapper $runs,
private IAppConfig $appConfig,
private LoggerInterface $logger,
) {
parent::__construct($time);
$this->setInterval(300);
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
}
private const MAX_DAILY_ATTEMPTS = 3;
protected function run($argument): void {
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
$deadline = microtime(true) + $budget;
$this->startScheduled();
foreach ($this->runs->findRunning(null, BackupService::KIND_FULL) as $run) {
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
$before = $run->getUpdatedAt();
$run = $this->backups->step($run, min($deadline, microtime(true) + $budget));
if ($run->getUpdatedAt() === $before && $run->getStatus() === 'running') {
break; // another process holds this run's lease: leave it to that process
}
}
}
$this->failStaleRestores();
$this->weeklyPrune();
foreach ($this->targets->list() as $t) {
$this->alerts->checkStale($t);
}
}
/** A restore run that has not moved for 6 hours is considered dead (it would block cleanup forever). */
private function failStaleRestores(): void {
foreach ($this->runs->findRunning(null, 'restore') as $run) {
if (time() - $run->getUpdatedAt() > 6 * 3600) {
$run->setStatus(\OCA\NgBackup\Db\Run::FAILED);
$run->setError('No progress for 6 hours; marked as failed');
$run->setFinishedAt(time());
$this->runs->update($run);
$this->alerts->runFailed($run, '#' . $run->getTargetId());
}
}
}
/**
* Once a week, apply the retention policy to non-append-only locations: from Sunday on, at the
* first moment no backup is running, so a long Sunday backup only delays it instead of skipping
* the week.
*/
private function weeklyPrune(): void {
$lastWeek = $this->appConfig->getValueString(Application::APP_ID, 'last_prune_week', '');
$due = date('N') === '7' ? date('o-W') : date('o-W', (int)strtotime('last sunday'));
if (strcmp($lastWeek, $due) >= 0 || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
return;
}
foreach ($this->targets->list() as $t) {
if ($t->getAppendOnly()) {
continue;
}
try {
$this->prune->apply($t, false, true);
} catch (\Throwable $e) {
$this->logger->warning('NG Backup: cleanup of ' . $t->getName() . ' failed: ' . $e->getMessage());
}
}
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', $due);
}
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
private function startScheduled(): void {
$at = $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', '');
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
return;
}
$todayAt = (int)strtotime(date('Y-m-d') . ' ' . $at);
if (time() < $todayAt) {
return;
}
foreach ($this->targets->list() as $t) {
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
if ($this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
continue;
}
// After a failure: retry at most MAX_DAILY_ATTEMPTS times a day, an hour apart, instead
// of starting (and notifying about) a new full run every five minutes all day.
$today = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $todayAt);
if (count($today) >= self::MAX_DAILY_ATTEMPTS || ($today !== [] && time() - (int)$today[0]->getFinishedAt() < 3600)) {
continue;
}
try {
$this->backups->start($t, 'scheduled');
} catch (\Throwable $e) {
$this->logger->warning('NG Backup: scheduled backup to ' . $t->getName() . ' not started: ' . $e->getMessage());
}
}
}
}