dillard
d96229f370
A killed restore left its leases for up to two hours, so the first backup of the recovered server was refused. A backup from before 1.0.3 records no empty folders: the test now expects that.
472 lines
20 KiB
PHP
472 lines
20 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
namespace OCA\NgBackup\Service;
|
|
|
|
use OCA\NgBackup\Db\DbDumper;
|
|
use OCA\NgBackup\Db\Run;
|
|
use OCA\NgBackup\Db\RunMapper;
|
|
use OCA\NgBackup\Db\Snapshot;
|
|
use OCA\NgBackup\Db\SnapshotMapper;
|
|
use OCA\NgBackup\Db\Target;
|
|
use OCA\NgBackup\Job\BackupRun;
|
|
use OCA\NgBackup\Repository\Repository;
|
|
use OCA\NgBackup\Repository\RepositoryException;
|
|
use OCP\AppFramework\Db\DoesNotExistException;
|
|
use OCP\App\IAppManager;
|
|
use OCP\IConfig;
|
|
use OCP\DB\QueryBuilder\IQueryBuilder;
|
|
use OCP\IDBConnection;
|
|
use OCP\Lock\ILockingProvider;
|
|
use OCP\Lock\LockedException;
|
|
use Psr\Log\LoggerInterface;
|
|
|
|
/**
|
|
* Full backups of this installation: database (one consistent transaction), config and the
|
|
* data directory, as a resumable run whose state is stored in ngb_runs after every step.
|
|
*
|
|
* Excluded by default: previews, log files, the updater's backups and each user's cache and
|
|
* temporary uploads (all regenerable or temporary).
|
|
*/
|
|
final class BackupService {
|
|
public const KIND_FULL = 'full';
|
|
/** Characters of a snapshot label (the column holds 255) */
|
|
public const LABEL_MAX = 100;
|
|
|
|
/** @var \WeakMap<Repository, string> the location settings each kept repository was opened with */
|
|
private \WeakMap $openedWith;
|
|
|
|
public function __construct(
|
|
private TargetService $targets,
|
|
private KeyService $keys,
|
|
private RunMapper $runs,
|
|
private SnapshotMapper $snapshots,
|
|
private IConfig $config,
|
|
private IDBConnection $db,
|
|
private IAppManager $appManager,
|
|
private ILockingProvider $locking,
|
|
private LoggerInterface $logger,
|
|
private AlertService $alerts,
|
|
private LeaseService $leases,
|
|
) {
|
|
/** @var \WeakMap<Repository, string> $opened */
|
|
$opened = new \WeakMap();
|
|
$this->openedWith = $opened;
|
|
}
|
|
|
|
public function start(Target $target, string $label = ''): Run {
|
|
$label = trim($label);
|
|
if (mb_strlen($label) > self::LABEL_MAX) {
|
|
throw new \InvalidArgumentException('A label has at most ' . self::LABEL_MAX . ' characters');
|
|
}
|
|
if (!$this->keys->everConfirmed()) {
|
|
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
|
}
|
|
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
|
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
|
}
|
|
// Check and create the run under the location's exclusive lock, so a cleanup cannot start
|
|
// in between (it takes the same lock and refuses while a run exists).
|
|
// A run that is already in progress is simply continued; no lease needed for that.
|
|
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
|
if ($running !== []) {
|
|
return $running[0];
|
|
}
|
|
try {
|
|
return $this->leases->with(PruneService::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $this->createRun($target, $label));
|
|
} catch (LockedException) {
|
|
throw new \RuntimeException('A cleanup, a restore or another start for ' . $target->getName() . ' is running, or one was stopped by force (that clears by itself within two hours); try again later');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The expected size of the files phase, for the progress shown while it runs: the size of the
|
|
* previous snapshot, or for a first backup the users' files, versions and trash as Nextcloud's
|
|
* file cache knows them (approximate: other data in the data directory is not counted).
|
|
*
|
|
* @return array{bytes: int, approximate: bool}
|
|
*/
|
|
private function estimateBytes(Run $run, array $files): array {
|
|
if (($files['parent'] ?? null) !== null) {
|
|
try {
|
|
return ['bytes' => $this->snapshots->findOne($run->getTargetId(), (string)$files['parent'])->getBytes() ?? 0, 'approximate' => false];
|
|
} catch (\Throwable) {
|
|
}
|
|
}
|
|
try {
|
|
$qb = $this->db->getQueryBuilder();
|
|
$qb->select('numeric_id')->from('storages')
|
|
->where($qb->expr()->like('id', $qb->createNamedParameter('home::%')))
|
|
->orWhere($qb->expr()->like('id', $qb->createNamedParameter('object::user:%')));
|
|
$result = $qb->executeQuery();
|
|
$storages = [];
|
|
while (($id = $result->fetchOne()) !== false) {
|
|
$storages[] = (int)$id;
|
|
}
|
|
$result->closeCursor();
|
|
$bytes = 0;
|
|
foreach (array_chunk($storages, 500) as $chunk) {
|
|
$qb = $this->db->getQueryBuilder();
|
|
$qb->select($qb->func()->sum('size'))->from('filecache')
|
|
->where($qb->expr()->in('storage', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
|
|
->andWhere($qb->expr()->in('path_hash', $qb->createNamedParameter(array_map('md5', ['files', 'files_versions', 'files_trashbin']), IQueryBuilder::PARAM_STR_ARRAY)))
|
|
->andWhere($qb->expr()->gt('size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
|
|
$bytes += (int)$qb->executeQuery()->fetchOne();
|
|
}
|
|
return ['bytes' => $bytes, 'approximate' => true];
|
|
} catch (\Throwable) {
|
|
return ['bytes' => 0, 'approximate' => true];
|
|
}
|
|
}
|
|
|
|
private function createRun(Target $target, string $label): Run {
|
|
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
|
if ($running !== []) {
|
|
return $running[0];
|
|
}
|
|
$dataDir = rtrim($this->config->getSystemValueString('datadirectory', \OC::$SERVERROOT . '/data'), '/');
|
|
$parent = $this->snapshots->latest($target->getId(), self::KIND_FULL);
|
|
|
|
$files = BackupRun::start(
|
|
['config' => rtrim(\OC::$configDir, '/'), 'data' => $dataDir],
|
|
$parent?->getSnapshotId(),
|
|
$label,
|
|
$this->excludes($dataDir),
|
|
['kind' => self::KIND_FULL, 'nextcloud' => $this->config->getSystemValueString('version'),
|
|
'instanceid' => $this->config->getSystemValueString('instanceid'),
|
|
'dbtype' => $this->config->getSystemValueString('dbtype'),
|
|
'apps' => $this->installedApps()],
|
|
);
|
|
|
|
$run = new Run();
|
|
$run->setTargetId($target->getId());
|
|
$run->setKind(self::KIND_FULL);
|
|
$run->setStatus(Run::RUNNING);
|
|
$run->setPhase('db');
|
|
$run->setState(json_encode(['files' => $files], JSON_THROW_ON_ERROR));
|
|
$run->setSnapshotId($files['snapshot']);
|
|
$run->setStartedAt(time());
|
|
$run->setUpdatedAt(time());
|
|
return $this->runs->insert($run);
|
|
}
|
|
|
|
/**
|
|
* Do one step of a run until $deadline. Safe to call concurrently: a second caller finds the
|
|
* run locked and returns immediately.
|
|
*
|
|
* @param Repository|null $repo the location's repository, opened by an earlier step of this run
|
|
* in this process (see runToCompletion()); set when this step opens it
|
|
*/
|
|
public function step(Run $run, float $deadline, ?Repository &$repo = null): Run {
|
|
// Leases expire on their own: a step killed by a time limit or OOM blocks nothing for long.
|
|
// A step can run well past its deadline (the commit of a large snapshot writes its tree and
|
|
// index), so the lease holds at least 15 minutes; should it still expire and another process
|
|
// take the run over, the checks below keep the two from overwriting each other.
|
|
$ttl = (int)min(3600, max(900, $deadline - microtime(true) + 600));
|
|
// No new step once a cancel has begun: it waits for the step in progress, if any.
|
|
try {
|
|
$this->leases->release($this->leases->acquire(self::cancelKey($run->getId()), LeaseService::SHARED, 60));
|
|
} catch (LockedException) {
|
|
return $run;
|
|
}
|
|
try {
|
|
$runLease = $this->leases->acquire('run/' . $run->getId(), LeaseService::EXCLUSIVE, $ttl);
|
|
} catch (LockedException) {
|
|
return $run; // another process is doing this step
|
|
}
|
|
// Shared lease on the location while this step writes: a cleanup (exclusive) cannot run now.
|
|
try {
|
|
$targetLease = $this->leases->acquire('ng_backup/target/' . $run->getTargetId(), LeaseService::SHARED, $ttl);
|
|
} catch (LockedException) {
|
|
$this->leases->release($runLease);
|
|
return $run;
|
|
}
|
|
$loadedAt = null;
|
|
$savedBefore = $run->getUpdatedAt();
|
|
try {
|
|
$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
|
|
if ($run->getStatus() !== Run::RUNNING || $run->getKind() !== self::KIND_FULL) {
|
|
return $run;
|
|
}
|
|
$loadedAt = $run->getUpdatedAt();
|
|
$target = $this->targets->get((string)$run->getTargetId());
|
|
// Reopen when another process took a step since ours (its packs are not in our copy of
|
|
// the index) or the location was edited (password, folder, size limit).
|
|
$settings = json_encode([$target->getBackend(), $target->getAuth(), $target->getOptions(),
|
|
$target->getBasePath(), $target->getRepositoryId(), $target->getMaxBytes()], JSON_THROW_ON_ERROR);
|
|
if ($repo === null || $loadedAt !== $savedBefore || ($this->openedWith[$repo] ?? null) !== $settings) {
|
|
// Opening reads the index listing, which takes minutes on a slow location with many
|
|
// packs: that time does not count, or each step would store one blob and stop.
|
|
$opening = microtime(true);
|
|
$repo = null;
|
|
$repo = $this->targets->repository($target);
|
|
$this->openedWith[$repo] = $settings;
|
|
$deadline += microtime(true) - $opening;
|
|
}
|
|
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
|
|
|
if ($run->getPhase() === 'db') {
|
|
$dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_'));
|
|
$dbStats = $state['dbStats'] ?? [];
|
|
// With system cron the whole dump usually fits in one step (one transaction, fully
|
|
// consistent). Under web cron it continues per step; the snapshot then says so.
|
|
$dump = $dumper->dumpStep($repo, $state['dump'] ?? $dumper->startDump(), $deadline, $dbStats);
|
|
$state['dbStats'] = $dbStats;
|
|
if ($dump['done']) {
|
|
$manifest = $dumper->manifest($dump);
|
|
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
|
|
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
|
|
$state['files']['meta']['dbConsistent'] = $manifest['consistent'];
|
|
$state['dbStats'] += ['tables' => count($manifest['tables']), 'rows' => array_sum(array_column($manifest['tables'], 'rows')),
|
|
'steps' => $dump['steps'], 'consistent' => $manifest['consistent']];
|
|
unset($state['dump']);
|
|
$run->setPhase('files');
|
|
} else {
|
|
$state['dump'] = $dump;
|
|
}
|
|
} else {
|
|
$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
|
|
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
|
|
if (in_array($state['files']['phase'], ['finish', 'commit'], true)) {
|
|
$run->setPhase('commit'); // all files read: writing and recording the snapshot
|
|
}
|
|
if ($state['files']['phase'] === 'commit') {
|
|
// Catalog write under the catalog lock, from a repository opened inside it: a per-user
|
|
// export or restore may have written the catalog since this step opened $repo.
|
|
try {
|
|
$this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300,
|
|
fn () => $this->targets->repository($target)->recordSnapshot((string)$state['files']['snapshot']));
|
|
$state['files']['phase'] = 'done';
|
|
} catch (LockedException|RepositoryException) {
|
|
// Another catalog write is in progress (or just raced us to the next
|
|
// generation): the next step retries this commit on the fresh catalog.
|
|
}
|
|
}
|
|
if ($state['files']['phase'] === 'done') {
|
|
$this->recordSnapshot($run, $state['files']);
|
|
// The snapshot is committed: what follows only reports on it and must not turn
|
|
// the run into a failure.
|
|
try {
|
|
$this->alerts->storedBytes($target, $repo->storedBytes());
|
|
$stats = $state['files']['stats'];
|
|
if (($stats['unreadable'] ?? 0) > 0 || ($stats['changing'] ?? 0) > 0) {
|
|
$warnings = BackupRun::warnings($state['files']);
|
|
$this->logger->warning(sprintf('NG Backup run %d to %s: %d unreadable (missing from the snapshot): %s; %d changed while being read: %s',
|
|
$run->getId(), $target->getName(), $stats['unreadable'] ?? 0, implode(', ', $warnings['unreadable']),
|
|
$stats['changing'] ?? 0, implode(', ', $warnings['changing'])));
|
|
$this->alerts->runWarnings($run, $target->getName(), (int)($stats['unreadable'] ?? 0), (int)($stats['changing'] ?? 0), $warnings);
|
|
}
|
|
} catch (\Throwable $e) {
|
|
$this->logger->warning('NG Backup run ' . $run->getId() . ': reporting on the finished backup failed: ' . $e->getMessage(), ['exception' => $e]);
|
|
}
|
|
$run->setStatus(Run::DONE);
|
|
$run->setPhase('done');
|
|
$run->setFinishedAt(time());
|
|
}
|
|
}
|
|
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
|
|
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
|
|
// Another process may have taken the run over after this step's lease expired: then its
|
|
// writes stand and this step's are dropped.
|
|
$saved = $this->runs->updateIfUnchanged($run, $loadedAt);
|
|
if ($saved === null) {
|
|
$repo = null; // the run is another process's now: its packs are not in our index
|
|
return $this->runs->find($run->getId());
|
|
}
|
|
return $saved;
|
|
} catch (\Throwable $e) {
|
|
$repo = null; // may hold blobs of a half-written pack: the next step opens it afresh
|
|
if ($loadedAt === null) {
|
|
// The run could not even be loaded: there is nothing fresh to mark as failed, and
|
|
// the caller's copy may be stale.
|
|
$this->logger->error('NG Backup run ' . $run->getId() . ': step could not load the run: ' . $e->getMessage(), ['exception' => $e]);
|
|
return $run;
|
|
}
|
|
$run->setStatus(Run::FAILED);
|
|
$run->setError($e->getMessage());
|
|
$run->setFinishedAt(time());
|
|
$failed = $this->runs->updateIfUnchanged($run, $loadedAt);
|
|
if ($failed === null) {
|
|
// Another process that took the run over may have finished it: its result stands.
|
|
$this->logger->warning('NG Backup run ' . $run->getId() . ': a step that another process had taken over ended with: ' . $e->getMessage());
|
|
return $this->runs->find($run->getId());
|
|
}
|
|
$run = $failed;
|
|
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
|
try {
|
|
$this->alerts->runFailed($run, isset($target) ? $target->getName() : '#' . $run->getTargetId());
|
|
} catch (\Throwable) {
|
|
}
|
|
return $run;
|
|
} finally {
|
|
$this->leases->release($targetLease);
|
|
$this->leases->release($runLease);
|
|
}
|
|
}
|
|
|
|
/** Run to completion in this process (occ, system cron), reporting progress. */
|
|
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
|
|
$repo = null; // opened once, by the first step, and kept for the next ones
|
|
while ($run->getStatus() === Run::RUNNING) {
|
|
$before = $run->getUpdatedAt();
|
|
$run = $this->step($run, microtime(true) + $stepSeconds, $repo);
|
|
if ($run->getStatus() === Run::RUNNING && $run->getUpdatedAt() === $before) {
|
|
// Step was not possible (another process holds the run, or a crashed step's lease has
|
|
// not expired yet): wait instead of spinning.
|
|
sleep(5);
|
|
$run = $this->runs->find($run->getId());
|
|
$repo = null; // another process may have stored packs in the meantime
|
|
}
|
|
if ($progress !== null) {
|
|
$progress($run);
|
|
}
|
|
}
|
|
return $run;
|
|
}
|
|
|
|
/**
|
|
* Stop a running backup. First no new step may start, then this waits (up to $wait seconds)
|
|
* for the step in progress, so a step that is recording the snapshot finishes first. A process
|
|
* running the backup then finds it cancelled and stops. The packs it stored stay until the
|
|
* next cleanup.
|
|
*
|
|
* A process of an earlier version does not hold back its next step and may never leave a gap:
|
|
* after $wait the run is marked cancelled while it works, unless it is recording the snapshot.
|
|
* Its step then fails to save and the process stops.
|
|
*/
|
|
public function cancel(Run $run, int $wait = 120): Run {
|
|
$id = $run->getId();
|
|
$until = time() + $wait;
|
|
$hold = null;
|
|
for ($try = 0; $hold === null; $try++) {
|
|
try {
|
|
$hold = $this->leases->acquire(self::cancelKey($id), LeaseService::EXCLUSIVE, $wait + 300);
|
|
} catch (LockedException) {
|
|
if ($try >= 20) {
|
|
throw new \RuntimeException("Run $id is being cancelled already");
|
|
}
|
|
usleep(100000); // a step checking whether it may start
|
|
}
|
|
}
|
|
try {
|
|
$lease = null;
|
|
while ($lease === null && time() < $until) {
|
|
try {
|
|
$lease = $this->leases->acquire('run/' . $id, LeaseService::EXCLUSIVE, 60);
|
|
} catch (LockedException) {
|
|
sleep(2);
|
|
}
|
|
}
|
|
try {
|
|
for ($try = 0; $try < 10; $try++) {
|
|
$run = $this->runs->find($id);
|
|
if ($run->getStatus() !== Run::RUNNING) {
|
|
return $run;
|
|
}
|
|
$state = json_decode($run->getState(), true) ?: [];
|
|
if ($lease === null && ($state['files']['phase'] ?? '') === 'commit') {
|
|
throw new \RuntimeException("Run $id is recording its snapshot; try again in a minute");
|
|
}
|
|
$loadedAt = $run->getUpdatedAt();
|
|
$run->setStatus(Run::CANCELLED);
|
|
$run->setFinishedAt(time());
|
|
if (($cancelled = $this->runs->updateIfUnchanged($run, $loadedAt)) !== null) {
|
|
return $cancelled;
|
|
}
|
|
usleep(200000); // a step of another process saved just now
|
|
}
|
|
throw new \RuntimeException("Run $id keeps changing; try again");
|
|
} finally {
|
|
if ($lease !== null) {
|
|
$this->leases->release($lease);
|
|
}
|
|
}
|
|
} finally {
|
|
$this->leases->release($hold);
|
|
}
|
|
}
|
|
|
|
private static function cancelKey(int $runId): string {
|
|
return 'run/' . $runId . '/cancel';
|
|
}
|
|
|
|
private function recordSnapshot(Run $run, array $files): void {
|
|
try {
|
|
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
|
return; // already recorded, by an earlier attempt at this step
|
|
} catch (DoesNotExistException) {
|
|
}
|
|
$s = new Snapshot();
|
|
$s->setTargetId($run->getTargetId());
|
|
$s->setSnapshotId($files['snapshot']);
|
|
$s->setKind($run->getKind());
|
|
$s->setLabel($files['label'] !== '' ? $files['label'] : null);
|
|
$s->setCreatedAt(time());
|
|
$s->setFiles($files['stats']['files']);
|
|
$s->setBytes($files['stats']['bytes']);
|
|
try {
|
|
$this->snapshots->insert($s);
|
|
} catch (\OCP\DB\Exception $e) {
|
|
// Recorded by another process between the check above and this insert?
|
|
if ($e->getReason() !== \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
|
throw $e;
|
|
}
|
|
try {
|
|
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
|
} catch (DoesNotExistException) {
|
|
throw $e;
|
|
}
|
|
}
|
|
}
|
|
|
|
/** @return list<string> */
|
|
private function excludes(string $dataDir): array {
|
|
$instanceId = $this->config->getSystemValueString('instanceid');
|
|
$exclude = [
|
|
"data/appdata_$instanceId/preview",
|
|
"data/updater-$instanceId",
|
|
'data/nextcloud.log',
|
|
'data/nextcloud.log.*',
|
|
'data/audit.log',
|
|
'data/*/cache',
|
|
'data/*/uploads',
|
|
];
|
|
// SQLite: the database file is in the data directory; it is backed up by the logical dump,
|
|
// and copying the live file would give an inconsistent copy.
|
|
if ($this->config->getSystemValueString('dbtype') === 'sqlite3') {
|
|
$dbName = $this->config->getSystemValueString('dbname', 'owncloud');
|
|
$exclude[] = "data/$dbName.db";
|
|
$exclude[] = "data/$dbName.db-journal";
|
|
$exclude[] = "data/$dbName.db-wal";
|
|
$exclude[] = "data/$dbName.db-shm";
|
|
}
|
|
$logFile = $this->config->getSystemValueString('logfile', '');
|
|
if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
|
|
$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
|
|
}
|
|
// Never back up a local backup location into itself.
|
|
foreach ($this->targets->list() as $t) {
|
|
$opts = $this->targets->options($t);
|
|
$dir = rtrim((string)($opts['datadir'] ?? ''), '/');
|
|
if ($t->getBackend() === 'local' && $dir !== '' && str_starts_with($dir . '/', $dataDir . '/')) {
|
|
$exclude[] = trim('data/' . substr($dir, strlen($dataDir) + 1) . '/' . $t->getBasePath(), '/');
|
|
}
|
|
}
|
|
return array_values(array_unique($exclude));
|
|
}
|
|
|
|
/** @return array<string, string> app id => version */
|
|
private function installedApps(): array {
|
|
$apps = [];
|
|
foreach ($this->appManager->getEnabledApps() as $app) {
|
|
$apps[$app] = $this->appManager->getAppVersion($app);
|
|
}
|
|
ksort($apps);
|
|
return $apps;
|
|
}
|
|
}
|