0f4a238414
Storage and repository:
- Listing skipped every name containing ".part-", directories too, so the
exports of a user like "john.part-time" were invisible to prune on S3,
SFTP, FTP and SMB and their data was deleted. Only real temporary uploads
("<name>.part-<8 hex>") are skipped now.
- prune refuses to run while a recorded per-user export's manifest is
missing, instead of freeing that export's data.
- forget() and untrash() copy first, switch the catalog, and delete the
originals last, so a crash can no longer leave the catalog pointing at a
missing snapshot or database dump.
- Writes are checked against the expected size, so a short upload is an
error instead of a truncated pack; raw restores check rename() and touch().
- A catalog write refuses to overwrite a generation another process just
wrote, and the catalog lock lasts 300 s, so opening a large repository
inside it no longer lets the lease run out.
Backup runs and scheduling:
- A file that keeps changing is read again at most three times; then the
last read is kept and reported, instead of restarting forever and
blocking cleanup.
- Unreadable files and folders are recorded, logged, shown by occ
backup:run, stored in the snapshot and notified to admins, instead of
being skipped silently.
- After a failed scheduled backup, at most three attempts a day, an hour
apart, instead of a new run and notification every five minutes.
- The job no longer spins on a run whose lease another process holds, and
the weekly cleanup catches up after a Sunday spent backing up.
- Notifications for the per-user export retention events.
129 lines
4.6 KiB
PHP
129 lines
4.6 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
namespace OCA\NgBackup\BackgroundJob;
|
|
|
|
use OCA\NgBackup\AppInfo\Application;
|
|
use OCA\NgBackup\Db\RunMapper;
|
|
use OCA\NgBackup\Service\BackupService;
|
|
use OCA\NgBackup\Service\TargetService;
|
|
use OCP\AppFramework\Utility\ITimeFactory;
|
|
use OCP\BackgroundJob\IJob;
|
|
use OCP\BackgroundJob\TimedJob;
|
|
use OCP\IAppConfig;
|
|
use Psr\Log\LoggerInterface;
|
|
|
|
/**
|
|
* Starts scheduled backups and moves running ones forward, a time-boxed step per cron run.
|
|
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
|
|
* in a normal web request (hosted Nextcloud).
|
|
*/
|
|
final class BackupJob extends TimedJob {
|
|
public function __construct(
|
|
ITimeFactory $time,
|
|
private BackupService $backups,
|
|
private \OCA\NgBackup\Service\PruneService $prune,
|
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
private TargetService $targets,
|
|
private RunMapper $runs,
|
|
private IAppConfig $appConfig,
|
|
private LoggerInterface $logger,
|
|
) {
|
|
parent::__construct($time);
|
|
$this->setInterval(300);
|
|
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
|
}
|
|
|
|
private const MAX_DAILY_ATTEMPTS = 3;
|
|
|
|
protected function run($argument): void {
|
|
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
|
$deadline = microtime(true) + $budget;
|
|
$this->startScheduled();
|
|
foreach ($this->runs->findRunning(null, BackupService::KIND_FULL) as $run) {
|
|
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
|
$before = $run->getUpdatedAt();
|
|
$run = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
|
if ($run->getUpdatedAt() === $before && $run->getStatus() === 'running') {
|
|
break; // another process holds this run's lease: leave it to that process
|
|
}
|
|
}
|
|
}
|
|
$this->failStaleRestores();
|
|
$this->weeklyPrune();
|
|
foreach ($this->targets->list() as $t) {
|
|
$this->alerts->checkStale($t);
|
|
}
|
|
}
|
|
|
|
/** A restore run that has not moved for 6 hours is considered dead (it would block cleanup forever). */
|
|
private function failStaleRestores(): void {
|
|
foreach ($this->runs->findRunning(null, 'restore') as $run) {
|
|
if (time() - $run->getUpdatedAt() > 6 * 3600) {
|
|
$run->setStatus(\OCA\NgBackup\Db\Run::FAILED);
|
|
$run->setError('No progress for 6 hours; marked as failed');
|
|
$run->setFinishedAt(time());
|
|
$this->runs->update($run);
|
|
$this->alerts->runFailed($run, '#' . $run->getTargetId());
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Once a week, apply the retention policy to non-append-only locations: from Sunday on, at the
|
|
* first moment no backup is running, so a long Sunday backup only delays it instead of skipping
|
|
* the week.
|
|
*/
|
|
private function weeklyPrune(): void {
|
|
$lastWeek = $this->appConfig->getValueString(Application::APP_ID, 'last_prune_week', '');
|
|
$due = date('N') === '7' ? date('o-W') : date('o-W', (int)strtotime('last sunday'));
|
|
if (strcmp($lastWeek, $due) >= 0 || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
|
|
return;
|
|
}
|
|
foreach ($this->targets->list() as $t) {
|
|
if ($t->getAppendOnly()) {
|
|
continue;
|
|
}
|
|
try {
|
|
$this->prune->apply($t, false, true);
|
|
} catch (\Throwable $e) {
|
|
$this->logger->warning('NG Backup: cleanup of ' . $t->getName() . ' failed: ' . $e->getMessage());
|
|
}
|
|
}
|
|
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', $due);
|
|
}
|
|
|
|
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
|
private function startScheduled(): void {
|
|
$at = $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', '');
|
|
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
|
|
return;
|
|
}
|
|
$todayAt = (int)strtotime(date('Y-m-d') . ' ' . $at);
|
|
if (time() < $todayAt) {
|
|
return;
|
|
}
|
|
foreach ($this->targets->list() as $t) {
|
|
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
|
|
if ($this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
|
|
continue;
|
|
}
|
|
// After a failure: retry at most MAX_DAILY_ATTEMPTS times a day, an hour apart, instead
|
|
// of starting (and notifying about) a new full run every five minutes all day.
|
|
$today = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $todayAt);
|
|
if (count($today) >= self::MAX_DAILY_ATTEMPTS || ($today !== [] && time() - (int)$today[0]->getFinishedAt() < 3600)) {
|
|
continue;
|
|
}
|
|
try {
|
|
$this->backups->start($t, 'scheduled');
|
|
} catch (\Throwable $e) {
|
|
$this->logger->warning('NG Backup: scheduled backup to ' . $t->getName() . ' not started: ' . $e->getMessage());
|
|
}
|
|
}
|
|
}
|
|
}
|