dillard
dfd3f38d8c
Each 30-second step opened the location again and read its list of index files. Over SMB that cost a request per file, so on a large repository opening took longer than the step: each step stored one 4 MiB pack and the list grew further. A first backup of nc01 to a Storage Box over SMB managed 12 GiB in 35 hours. occ backup:run and system cron now keep the repository open for the whole run (reopened when another process took a step or the location was edited), opening no longer counts towards the step's time, and SMB folders are listed in one request. occ backup:cancel stops a running backup: no new step starts and the step in progress finishes first. A run of an earlier version, which does not hold back its next step, is marked cancelled after two minutes unless it is recording its snapshot.
463 lines
19 KiB
PHP
463 lines
19 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
namespace OCA\NgBackup\Service;
|
|
|
|
use OCA\NgBackup\Db\DbDumper;
|
|
use OCA\NgBackup\Db\Run;
|
|
use OCA\NgBackup\Db\RunMapper;
|
|
use OCA\NgBackup\Db\Snapshot;
|
|
use OCA\NgBackup\Db\SnapshotMapper;
|
|
use OCA\NgBackup\Db\Target;
|
|
use OCA\NgBackup\Job\BackupRun;
|
|
use OCA\NgBackup\Repository\Repository;
|
|
use OCA\NgBackup\Repository\RepositoryException;
|
|
use OCP\AppFramework\Db\DoesNotExistException;
|
|
use OCP\App\IAppManager;
|
|
use OCP\IConfig;
|
|
use OCP\DB\QueryBuilder\IQueryBuilder;
|
|
use OCP\IDBConnection;
|
|
use OCP\Lock\ILockingProvider;
|
|
use OCP\Lock\LockedException;
|
|
use Psr\Log\LoggerInterface;
|
|
|
|
/**
|
|
* Full backups of this installation: database (one consistent transaction), config and the
|
|
* data directory, as a resumable run whose state is stored in ngb_runs after every step.
|
|
*
|
|
* Excluded by default: previews, log files, the updater's backups and each user's cache and
|
|
* temporary uploads (all regenerable or temporary).
|
|
*/
|
|
final class BackupService {
|
|
public const KIND_FULL = 'full';
|
|
|
|
/** @var \WeakMap<Repository, string> the location settings each kept repository was opened with */
|
|
private \WeakMap $openedWith;
|
|
|
|
public function __construct(
|
|
private TargetService $targets,
|
|
private KeyService $keys,
|
|
private RunMapper $runs,
|
|
private SnapshotMapper $snapshots,
|
|
private IConfig $config,
|
|
private IDBConnection $db,
|
|
private IAppManager $appManager,
|
|
private ILockingProvider $locking,
|
|
private LoggerInterface $logger,
|
|
private AlertService $alerts,
|
|
private LeaseService $leases,
|
|
) {
|
|
/** @var \WeakMap<Repository, string> $opened */
|
|
$opened = new \WeakMap();
|
|
$this->openedWith = $opened;
|
|
}
|
|
|
|
public function start(Target $target, string $label = ''): Run {
|
|
if (!$this->keys->everConfirmed()) {
|
|
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
|
}
|
|
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
|
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
|
}
|
|
// Check and create the run under the location's exclusive lock, so a cleanup cannot start
|
|
// in between (it takes the same lock and refuses while a run exists).
|
|
// A run that is already in progress is simply continued; no lease needed for that.
|
|
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
|
if ($running !== []) {
|
|
return $running[0];
|
|
}
|
|
try {
|
|
return $this->leases->with(PruneService::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $this->createRun($target, $label));
|
|
} catch (LockedException) {
|
|
throw new \RuntimeException('A cleanup or another start for ' . $target->getName() . ' is running; try again later');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The expected size of the files phase, for the progress shown while it runs: the size of the
|
|
* previous snapshot, or for a first backup the users' files, versions and trash as Nextcloud's
|
|
* file cache knows them (approximate: other data in the data directory is not counted).
|
|
*
|
|
* @return array{bytes: int, approximate: bool}
|
|
*/
|
|
private function estimateBytes(Run $run, array $files): array {
|
|
if (($files['parent'] ?? null) !== null) {
|
|
try {
|
|
return ['bytes' => $this->snapshots->findOne($run->getTargetId(), (string)$files['parent'])->getBytes() ?? 0, 'approximate' => false];
|
|
} catch (\Throwable) {
|
|
}
|
|
}
|
|
try {
|
|
$qb = $this->db->getQueryBuilder();
|
|
$qb->select('numeric_id')->from('storages')
|
|
->where($qb->expr()->like('id', $qb->createNamedParameter('home::%')))
|
|
->orWhere($qb->expr()->like('id', $qb->createNamedParameter('object::user:%')));
|
|
$result = $qb->executeQuery();
|
|
$storages = [];
|
|
while (($id = $result->fetchOne()) !== false) {
|
|
$storages[] = (int)$id;
|
|
}
|
|
$result->closeCursor();
|
|
$bytes = 0;
|
|
foreach (array_chunk($storages, 500) as $chunk) {
|
|
$qb = $this->db->getQueryBuilder();
|
|
$qb->select($qb->func()->sum('size'))->from('filecache')
|
|
->where($qb->expr()->in('storage', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
|
|
->andWhere($qb->expr()->in('path_hash', $qb->createNamedParameter(array_map('md5', ['files', 'files_versions', 'files_trashbin']), IQueryBuilder::PARAM_STR_ARRAY)))
|
|
->andWhere($qb->expr()->gt('size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
|
|
$bytes += (int)$qb->executeQuery()->fetchOne();
|
|
}
|
|
return ['bytes' => $bytes, 'approximate' => true];
|
|
} catch (\Throwable) {
|
|
return ['bytes' => 0, 'approximate' => true];
|
|
}
|
|
}
|
|
|
|
private function createRun(Target $target, string $label): Run {
|
|
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
|
if ($running !== []) {
|
|
return $running[0];
|
|
}
|
|
$dataDir = rtrim($this->config->getSystemValueString('datadirectory', \OC::$SERVERROOT . '/data'), '/');
|
|
$parent = $this->snapshots->latest($target->getId(), self::KIND_FULL);
|
|
|
|
$files = BackupRun::start(
|
|
['config' => rtrim(\OC::$configDir, '/'), 'data' => $dataDir],
|
|
$parent?->getSnapshotId(),
|
|
$label,
|
|
$this->excludes($dataDir),
|
|
['kind' => self::KIND_FULL, 'nextcloud' => $this->config->getSystemValueString('version'),
|
|
'instanceid' => $this->config->getSystemValueString('instanceid'),
|
|
'dbtype' => $this->config->getSystemValueString('dbtype'),
|
|
'apps' => $this->installedApps()],
|
|
);
|
|
|
|
$run = new Run();
|
|
$run->setTargetId($target->getId());
|
|
$run->setKind(self::KIND_FULL);
|
|
$run->setStatus(Run::RUNNING);
|
|
$run->setPhase('db');
|
|
$run->setState(json_encode(['files' => $files], JSON_THROW_ON_ERROR));
|
|
$run->setSnapshotId($files['snapshot']);
|
|
$run->setStartedAt(time());
|
|
$run->setUpdatedAt(time());
|
|
return $this->runs->insert($run);
|
|
}
|
|
|
|
/**
|
|
* Do one step of a run until $deadline. Safe to call concurrently: a second caller finds the
|
|
* run locked and returns immediately.
|
|
*
|
|
* @param Repository|null $repo the location's repository, opened by an earlier step of this run
|
|
* in this process (see runToCompletion()); set when this step opens it
|
|
*/
|
|
public function step(Run $run, float $deadline, ?Repository &$repo = null): Run {
|
|
// Leases expire on their own: a step killed by a time limit or OOM blocks nothing for long.
|
|
// A step can run well past its deadline (the commit of a large snapshot writes its tree and
|
|
// index), so the lease holds at least 15 minutes; should it still expire and another process
|
|
// take the run over, the checks below keep the two from overwriting each other.
|
|
$ttl = (int)min(3600, max(900, $deadline - microtime(true) + 600));
|
|
// No new step once a cancel has begun: it waits for the step in progress, if any.
|
|
try {
|
|
$this->leases->release($this->leases->acquire(self::cancelKey($run->getId()), LeaseService::SHARED, 60));
|
|
} catch (LockedException) {
|
|
return $run;
|
|
}
|
|
try {
|
|
$runLease = $this->leases->acquire('run/' . $run->getId(), LeaseService::EXCLUSIVE, $ttl);
|
|
} catch (LockedException) {
|
|
return $run; // another process is doing this step
|
|
}
|
|
// Shared lease on the location while this step writes: a cleanup (exclusive) cannot run now.
|
|
try {
|
|
$targetLease = $this->leases->acquire('ng_backup/target/' . $run->getTargetId(), LeaseService::SHARED, $ttl);
|
|
} catch (LockedException) {
|
|
$this->leases->release($runLease);
|
|
return $run;
|
|
}
|
|
$loadedAt = null;
|
|
$savedBefore = $run->getUpdatedAt();
|
|
try {
|
|
$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
|
|
if ($run->getStatus() !== Run::RUNNING || $run->getKind() !== self::KIND_FULL) {
|
|
return $run;
|
|
}
|
|
$loadedAt = $run->getUpdatedAt();
|
|
$target = $this->targets->get((string)$run->getTargetId());
|
|
// Reopen when another process took a step since ours (its packs are not in our copy of
|
|
// the index) or the location was edited (password, folder, size limit).
|
|
$settings = json_encode([$target->getBackend(), $target->getAuth(), $target->getOptions(),
|
|
$target->getBasePath(), $target->getRepositoryId(), $target->getMaxBytes()], JSON_THROW_ON_ERROR);
|
|
if ($repo === null || $loadedAt !== $savedBefore || ($this->openedWith[$repo] ?? null) !== $settings) {
|
|
// Opening reads the index listing, which takes minutes on a slow location with many
|
|
// packs: that time does not count, or each step would store one blob and stop.
|
|
$opening = microtime(true);
|
|
$repo = null;
|
|
$repo = $this->targets->repository($target);
|
|
$this->openedWith[$repo] = $settings;
|
|
$deadline += microtime(true) - $opening;
|
|
}
|
|
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
|
|
|
if ($run->getPhase() === 'db') {
|
|
$dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_'));
|
|
$dbStats = $state['dbStats'] ?? [];
|
|
// With system cron the whole dump usually fits in one step (one transaction, fully
|
|
// consistent). Under web cron it continues per step; the snapshot then says so.
|
|
$dump = $dumper->dumpStep($repo, $state['dump'] ?? $dumper->startDump(), $deadline, $dbStats);
|
|
$state['dbStats'] = $dbStats;
|
|
if ($dump['done']) {
|
|
$manifest = $dumper->manifest($dump);
|
|
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
|
|
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
|
|
$state['files']['meta']['dbConsistent'] = $manifest['consistent'];
|
|
$state['dbStats'] += ['tables' => count($manifest['tables']), 'rows' => array_sum(array_column($manifest['tables'], 'rows')),
|
|
'steps' => $dump['steps'], 'consistent' => $manifest['consistent']];
|
|
unset($state['dump']);
|
|
$run->setPhase('files');
|
|
} else {
|
|
$state['dump'] = $dump;
|
|
}
|
|
} else {
|
|
$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
|
|
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
|
|
if ($state['files']['phase'] === 'commit') {
|
|
// Catalog write under the catalog lock, from a repository opened inside it: a per-user
|
|
// export or restore may have written the catalog since this step opened $repo.
|
|
try {
|
|
$this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300,
|
|
fn () => $this->targets->repository($target)->recordSnapshot((string)$state['files']['snapshot']));
|
|
$state['files']['phase'] = 'done';
|
|
} catch (LockedException|RepositoryException) {
|
|
// Another catalog write is in progress (or just raced us to the next
|
|
// generation): the next step retries this commit on the fresh catalog.
|
|
}
|
|
}
|
|
if ($state['files']['phase'] === 'done') {
|
|
$this->recordSnapshot($run, $state['files']);
|
|
// The snapshot is committed: what follows only reports on it and must not turn
|
|
// the run into a failure.
|
|
try {
|
|
$this->alerts->storedBytes($target, $repo->storedBytes());
|
|
$stats = $state['files']['stats'];
|
|
if (($stats['unreadable'] ?? 0) > 0 || ($stats['changing'] ?? 0) > 0) {
|
|
$warnings = BackupRun::warnings($state['files']);
|
|
$this->logger->warning(sprintf('NG Backup run %d to %s: %d unreadable (missing from the snapshot): %s; %d changed while being read: %s',
|
|
$run->getId(), $target->getName(), $stats['unreadable'] ?? 0, implode(', ', $warnings['unreadable']),
|
|
$stats['changing'] ?? 0, implode(', ', $warnings['changing'])));
|
|
$this->alerts->runWarnings($run, $target->getName(), (int)($stats['unreadable'] ?? 0), (int)($stats['changing'] ?? 0), $warnings);
|
|
}
|
|
} catch (\Throwable $e) {
|
|
$this->logger->warning('NG Backup run ' . $run->getId() . ': reporting on the finished backup failed: ' . $e->getMessage(), ['exception' => $e]);
|
|
}
|
|
$run->setStatus(Run::DONE);
|
|
$run->setPhase('done');
|
|
$run->setFinishedAt(time());
|
|
}
|
|
}
|
|
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
|
|
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
|
|
// Another process may have taken the run over after this step's lease expired: then its
|
|
// writes stand and this step's are dropped.
|
|
$saved = $this->runs->updateIfUnchanged($run, $loadedAt);
|
|
if ($saved === null) {
|
|
$repo = null; // the run is another process's now: its packs are not in our index
|
|
return $this->runs->find($run->getId());
|
|
}
|
|
return $saved;
|
|
} catch (\Throwable $e) {
|
|
$repo = null; // may hold blobs of a half-written pack: the next step opens it afresh
|
|
if ($loadedAt === null) {
|
|
// The run could not even be loaded: there is nothing fresh to mark as failed, and
|
|
// the caller's copy may be stale.
|
|
$this->logger->error('NG Backup run ' . $run->getId() . ': step could not load the run: ' . $e->getMessage(), ['exception' => $e]);
|
|
return $run;
|
|
}
|
|
$run->setStatus(Run::FAILED);
|
|
$run->setError($e->getMessage());
|
|
$run->setFinishedAt(time());
|
|
$failed = $this->runs->updateIfUnchanged($run, $loadedAt);
|
|
if ($failed === null) {
|
|
// Another process that took the run over may have finished it: its result stands.
|
|
$this->logger->warning('NG Backup run ' . $run->getId() . ': a step that another process had taken over ended with: ' . $e->getMessage());
|
|
return $this->runs->find($run->getId());
|
|
}
|
|
$run = $failed;
|
|
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
|
try {
|
|
$this->alerts->runFailed($run, isset($target) ? $target->getName() : '#' . $run->getTargetId());
|
|
} catch (\Throwable) {
|
|
}
|
|
return $run;
|
|
} finally {
|
|
$this->leases->release($targetLease);
|
|
$this->leases->release($runLease);
|
|
}
|
|
}
|
|
|
|
/** Run to completion in this process (occ, system cron), reporting progress. */
|
|
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
|
|
$repo = null; // opened once, by the first step, and kept for the next ones
|
|
while ($run->getStatus() === Run::RUNNING) {
|
|
$before = $run->getUpdatedAt();
|
|
$run = $this->step($run, microtime(true) + $stepSeconds, $repo);
|
|
if ($run->getStatus() === Run::RUNNING && $run->getUpdatedAt() === $before) {
|
|
// Step was not possible (another process holds the run, or a crashed step's lease has
|
|
// not expired yet): wait instead of spinning.
|
|
sleep(5);
|
|
$run = $this->runs->find($run->getId());
|
|
$repo = null; // another process may have stored packs in the meantime
|
|
}
|
|
if ($progress !== null) {
|
|
$progress($run);
|
|
}
|
|
}
|
|
return $run;
|
|
}
|
|
|
|
/**
|
|
* Stop a running backup. First no new step may start, then this waits (up to $wait seconds)
|
|
* for the step in progress, so a step that is recording the snapshot finishes first. A process
|
|
* running the backup then finds it cancelled and stops. The packs it stored stay until the
|
|
* next cleanup.
|
|
*
|
|
* A process of an earlier version does not hold back its next step and may never leave a gap:
|
|
* after $wait the run is marked cancelled while it works, unless it is recording the snapshot.
|
|
* Its step then fails to save and the process stops.
|
|
*/
|
|
public function cancel(Run $run, int $wait = 120): Run {
|
|
$id = $run->getId();
|
|
$until = time() + $wait;
|
|
$hold = null;
|
|
for ($try = 0; $hold === null; $try++) {
|
|
try {
|
|
$hold = $this->leases->acquire(self::cancelKey($id), LeaseService::EXCLUSIVE, $wait + 300);
|
|
} catch (LockedException) {
|
|
if ($try >= 20) {
|
|
throw new \RuntimeException("Run $id is being cancelled already");
|
|
}
|
|
usleep(100000); // a step checking whether it may start
|
|
}
|
|
}
|
|
try {
|
|
$lease = null;
|
|
while ($lease === null && time() < $until) {
|
|
try {
|
|
$lease = $this->leases->acquire('run/' . $id, LeaseService::EXCLUSIVE, 60);
|
|
} catch (LockedException) {
|
|
sleep(2);
|
|
}
|
|
}
|
|
try {
|
|
for ($try = 0; $try < 10; $try++) {
|
|
$run = $this->runs->find($id);
|
|
if ($run->getStatus() !== Run::RUNNING) {
|
|
return $run;
|
|
}
|
|
$state = json_decode($run->getState(), true) ?: [];
|
|
if ($lease === null && ($state['files']['phase'] ?? '') === 'commit') {
|
|
throw new \RuntimeException("Run $id is recording its snapshot; try again in a minute");
|
|
}
|
|
$loadedAt = $run->getUpdatedAt();
|
|
$run->setStatus(Run::CANCELLED);
|
|
$run->setFinishedAt(time());
|
|
if (($cancelled = $this->runs->updateIfUnchanged($run, $loadedAt)) !== null) {
|
|
return $cancelled;
|
|
}
|
|
usleep(200000); // a step of another process saved just now
|
|
}
|
|
throw new \RuntimeException("Run $id keeps changing; try again");
|
|
} finally {
|
|
if ($lease !== null) {
|
|
$this->leases->release($lease);
|
|
}
|
|
}
|
|
} finally {
|
|
$this->leases->release($hold);
|
|
}
|
|
}
|
|
|
|
private static function cancelKey(int $runId): string {
|
|
return 'run/' . $runId . '/cancel';
|
|
}
|
|
|
|
private function recordSnapshot(Run $run, array $files): void {
|
|
try {
|
|
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
|
return; // already recorded, by an earlier attempt at this step
|
|
} catch (DoesNotExistException) {
|
|
}
|
|
$s = new Snapshot();
|
|
$s->setTargetId($run->getTargetId());
|
|
$s->setSnapshotId($files['snapshot']);
|
|
$s->setKind($run->getKind());
|
|
$s->setLabel($files['label'] !== '' ? $files['label'] : null);
|
|
$s->setCreatedAt(time());
|
|
$s->setFiles($files['stats']['files']);
|
|
$s->setBytes($files['stats']['bytes']);
|
|
try {
|
|
$this->snapshots->insert($s);
|
|
} catch (\OCP\DB\Exception $e) {
|
|
// Recorded by another process between the check above and this insert?
|
|
if ($e->getReason() !== \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
|
throw $e;
|
|
}
|
|
try {
|
|
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
|
} catch (DoesNotExistException) {
|
|
throw $e;
|
|
}
|
|
}
|
|
}
|
|
|
|
/** @return list<string> */
|
|
private function excludes(string $dataDir): array {
|
|
$instanceId = $this->config->getSystemValueString('instanceid');
|
|
$exclude = [
|
|
"data/appdata_$instanceId/preview",
|
|
"data/updater-$instanceId",
|
|
'data/nextcloud.log',
|
|
'data/nextcloud.log.*',
|
|
'data/audit.log',
|
|
'data/*/cache',
|
|
'data/*/uploads',
|
|
];
|
|
// SQLite: the database file is in the data directory; it is backed up by the logical dump,
|
|
// and copying the live file would give an inconsistent copy.
|
|
if ($this->config->getSystemValueString('dbtype') === 'sqlite3') {
|
|
$dbName = $this->config->getSystemValueString('dbname', 'owncloud');
|
|
$exclude[] = "data/$dbName.db";
|
|
$exclude[] = "data/$dbName.db-journal";
|
|
$exclude[] = "data/$dbName.db-wal";
|
|
$exclude[] = "data/$dbName.db-shm";
|
|
}
|
|
$logFile = $this->config->getSystemValueString('logfile', '');
|
|
if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
|
|
$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
|
|
}
|
|
// Never back up a local backup location into itself.
|
|
foreach ($this->targets->list() as $t) {
|
|
$opts = $this->targets->options($t);
|
|
$dir = rtrim((string)($opts['datadir'] ?? ''), '/');
|
|
if ($t->getBackend() === 'local' && $dir !== '' && str_starts_with($dir . '/', $dataDir . '/')) {
|
|
$exclude[] = trim('data/' . substr($dir, strlen($dataDir) + 1) . '/' . $t->getBasePath(), '/');
|
|
}
|
|
}
|
|
return array_values(array_unique($exclude));
|
|
}
|
|
|
|
/** @return array<string, string> app id => version */
|
|
private function installedApps(): array {
|
|
$apps = [];
|
|
foreach ($this->appManager->getEnabledApps() as $app) {
|
|
$apps[$app] = $this->appManager->getAppVersion($app);
|
|
}
|
|
ksort($apps);
|
|
return $apps;
|
|
}
|
|
}
|