nextcloud-ng_backup/lib/Service/BackupService.php

<?php

declare(strict_types=1);

// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later

namespace OCA\NgBackup\Service;

use OCA\NgBackup\Db\DbDumper;
use OCA\NgBackup\Db\Run;
use OCA\NgBackup\Db\RunMapper;
use OCA\NgBackup\Db\Snapshot;
use OCA\NgBackup\Db\SnapshotMapper;
use OCA\NgBackup\Db\Target;
use OCA\NgBackup\Job\BackupRun;
use OCA\NgBackup\Repository\Repository;
use OCA\NgBackup\Repository\RepositoryException;
use OCP\AppFramework\Db\DoesNotExistException;
use OCP\App\IAppManager;
use OCP\IConfig;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use OCP\Lock\ILockingProvider;
use OCP\Lock\LockedException;
use Psr\Log\LoggerInterface;

/**
 * Full backups of this installation: database (one consistent transaction), config and the
 * data directory, as a resumable run whose state is stored in ngb_runs after every step.
 *
 * Excluded by default: previews, log files, the updater's backups and each user's cache and
 * temporary uploads (all regenerable or temporary).
 */
final class BackupService {
	public const KIND_FULL = 'full';
	/** Characters of a snapshot label (the column holds 255) */
	public const LABEL_MAX = 100;

	/** @var \WeakMap<Repository, string> the location settings each kept repository was opened with */
	private \WeakMap $openedWith;

	public function __construct(
		private TargetService $targets,
		private KeyService $keys,
		private RunMapper $runs,
		private SnapshotMapper $snapshots,
		private IConfig $config,
		private IDBConnection $db,
		private IAppManager $appManager,
		private ILockingProvider $locking,
		private LoggerInterface $logger,
		private AlertService $alerts,
		private LeaseService $leases,
	) {
		/** @var \WeakMap<Repository, string> $opened */
		$opened = new \WeakMap();
		$this->openedWith = $opened;
	}

	/**
	 * The running backups in the order they get their turn: first come, first served. The first one
	 * runs, the others are queued; only one at a time reads the disks.
	 *
	 * @return list<Run>
	 */
	public function runsInTurn(): array {
		$running = $this->runs->findRunning(null, self::KIND_FULL);
		usort($running, static fn (Run $a, Run $b): int => [$a->getStartedAt(), $a->getId()] <=> [$b->getStartedAt(), $b->getId()]);
		return $running;
	}

	public function start(Target $target, string $label = ''): Run {
		$label = trim($label);
		if (mb_strlen($label) > self::LABEL_MAX) {
			throw new \InvalidArgumentException('A label has at most ' . self::LABEL_MAX . ' characters');
		}
		if (!$this->keys->everConfirmed()) {
			throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
		}
		if ($this->config->getSystemValue('objectstore', null) !== null) {
			throw new \RuntimeException('Object storage as primary storage is not supported yet');
		}
		// Check and create the run under the location's exclusive lock, so a cleanup cannot start
		// in between (it takes the same lock and refuses while a run exists).
		// A run that is already in progress is simply continued; no lease needed for that.
		$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
		if ($running !== []) {
			return $running[0];
		}
		try {
			return $this->leases->with(PruneService::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $this->createRun($target, $label));
		} catch (LockedException) {
			throw new \RuntimeException('A cleanup, a restore or another start for ' . $target->getName() . ' is running, or one was stopped by force (that clears by itself within two hours); try again later');
		}
	}

	/**
	 * The expected size of the files phase, for the progress shown while it runs: the size of the
	 * previous snapshot, or for a first backup the users' files, versions and trash as Nextcloud's
	 * file cache knows them (approximate: other data in the data directory is not counted).
	 *
	 * @return array{bytes: int, approximate: bool}
	 */
	private function estimateBytes(Run $run, array $files): array {
		if (($files['parent'] ?? null) !== null) {
			try {
				return ['bytes' => $this->snapshots->findOne($run->getTargetId(), (string)$files['parent'])->getBytes() ?? 0, 'approximate' => false];
			} catch (\Throwable) {
			}
		}
		try {
			$qb = $this->db->getQueryBuilder();
			$qb->select('numeric_id')->from('storages')
				->where($qb->expr()->like('id', $qb->createNamedParameter('home::%')))
				->orWhere($qb->expr()->like('id', $qb->createNamedParameter('object::user:%')));
			$result = $qb->executeQuery();
			$storages = [];
			while (($id = $result->fetchOne()) !== false) {
				$storages[] = (int)$id;
			}
			$result->closeCursor();
			$bytes = 0;
			foreach (array_chunk($storages, 500) as $chunk) {
				$qb = $this->db->getQueryBuilder();
				$qb->select($qb->func()->sum('size'))->from('filecache')
					->where($qb->expr()->in('storage', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
					->andWhere($qb->expr()->in('path_hash', $qb->createNamedParameter(array_map('md5', ['files', 'files_versions', 'files_trashbin']), IQueryBuilder::PARAM_STR_ARRAY)))
					->andWhere($qb->expr()->gt('size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
				$bytes += (int)$qb->executeQuery()->fetchOne();
			}
			return ['bytes' => $bytes, 'approximate' => true];
		} catch (\Throwable) {
			return ['bytes' => 0, 'approximate' => true];
		}
	}

	private function createRun(Target $target, string $label): Run {
		$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
		if ($running !== []) {
			return $running[0];
		}
		$dataDir = rtrim($this->config->getSystemValueString('datadirectory', \OC::$SERVERROOT . '/data'), '/');
		$parent = $this->snapshots->latest($target->getId(), self::KIND_FULL);

		$files = BackupRun::start(
			['config' => rtrim(\OC::$configDir, '/'), 'data' => $dataDir],
			$parent?->getSnapshotId(),
			$label,
			$this->excludes($dataDir),
			['kind' => self::KIND_FULL, 'nextcloud' => $this->config->getSystemValueString('version'),
				'instanceid' => $this->config->getSystemValueString('instanceid'),
				'dbtype' => $this->config->getSystemValueString('dbtype'),
				'apps' => $this->installedApps()],
		);

		$run = new Run();
		$run->setTargetId($target->getId());
		$run->setKind(self::KIND_FULL);
		$run->setStatus(Run::RUNNING);
		$run->setPhase('db');
		$run->setState(json_encode(['files' => $files], JSON_THROW_ON_ERROR));
		$run->setSnapshotId($files['snapshot']);
		$run->setStartedAt(time());
		$run->setUpdatedAt(time());
		return $this->runs->insert($run);
	}

	/**
	 * Do one step of a run until $deadline. Safe to call concurrently: a second caller finds the
	 * run locked and returns immediately.
	 *
	 * @param Repository|null $repo the location's repository, opened by an earlier step of this run
	 *                              in this process (see runToCompletion()); set when this step opens it
	 */
	public function step(Run $run, float $deadline, ?Repository &$repo = null): Run {
		// Leases expire on their own: a step killed by a time limit or OOM blocks nothing for long.
		// A step can run well past its deadline (the commit of a large snapshot writes its tree and
		// index), so the lease holds at least 15 minutes; should it still expire and another process
		// take the run over, the checks below keep the two from overwriting each other.
		$ttl = (int)min(3600, max(900, $deadline - microtime(true) + 600));
		// No new step once a cancel has begun: it waits for the step in progress, if any.
		try {
			$this->leases->release($this->leases->acquire(self::cancelKey($run->getId()), LeaseService::SHARED, 60));
		} catch (LockedException) {
			return $run;
		}
		try {
			$runLease = $this->leases->acquire('run/' . $run->getId(), LeaseService::EXCLUSIVE, $ttl);
		} catch (LockedException) {
			return $run; // another process is doing this step
		}
		// Shared lease on the location while this step writes: a cleanup (exclusive) cannot run now.
		try {
			$targetLease = $this->leases->acquire('ng_backup/target/' . $run->getTargetId(), LeaseService::SHARED, $ttl);
		} catch (LockedException) {
			$this->leases->release($runLease);
			return $run;
		}
		$loadedAt = null;
		$savedBefore = $run->getUpdatedAt();
		try {
			$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
			if ($run->getStatus() !== Run::RUNNING || $run->getKind() !== self::KIND_FULL) {
				return $run;
			}
			$loadedAt = $run->getUpdatedAt();
			$target = $this->targets->get((string)$run->getTargetId());
			// Reopen when another process took a step since ours (its packs are not in our copy of
			// the index) or the location was edited (password, folder, size limit).
			$settings = json_encode([$target->getBackend(), $target->getAuth(), $target->getOptions(),
				$target->getBasePath(), $target->getRepositoryId(), $target->getMaxBytes()], JSON_THROW_ON_ERROR);
			if ($repo === null || $loadedAt !== $savedBefore || ($this->openedWith[$repo] ?? null) !== $settings) {
				// Opening reads the index listing, which takes minutes on a slow location with many
				// packs: that time does not count, or each step would store one blob and stop.
				$opening = microtime(true);
				$repo = null;
				$repo = $this->targets->repository($target);
				$this->openedWith[$repo] = $settings;
				$deadline += microtime(true) - $opening;
			}
			$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);

			if ($run->getPhase() === 'db') {
				if (!isset($state['dump'])) {
					// It may have been queued for hours: the restore point has the time of its data.
					$state['files']['started'] = gmdate('c');
					// Locations from before 1.1.0 get their sealed settings at their next backup.
					$this->targets->recordSettings($repo);
					$this->targets->catchUpWrappedKey($target, $repo);
				}
				$dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_'));
				$dbStats = $state['dbStats'] ?? [];
				// With system cron the whole dump usually fits in one step (one transaction, fully
				// consistent). Under web cron it continues per step; the snapshot then says so.
				$dump = $dumper->dumpStep($repo, $state['dump'] ?? $dumper->startDump(), $deadline, $dbStats);
				$state['dbStats'] = $dbStats;
				if ($dump['done']) {
					$manifest = $dumper->manifest($dump);
					$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
					$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
					$state['files']['meta']['dbConsistent'] = $manifest['consistent'];
					$state['dbStats'] += ['tables' => count($manifest['tables']), 'rows' => array_sum(array_column($manifest['tables'], 'rows')),
						'steps' => $dump['steps'], 'consistent' => $manifest['consistent']];
					unset($state['dump']);
					$run->setPhase('files');
				} else {
					$state['dump'] = $dump;
				}
			} else {
				$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
				$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
				if (in_array($state['files']['phase'], ['finish', 'commit'], true)) {
					$run->setPhase('commit'); // all files read: writing and recording the snapshot
				}
				if ($state['files']['phase'] === 'commit') {
					// Catalog write under the catalog lock, from a repository opened inside it: a per-user
					// export or restore may have written the catalog since this step opened $repo.
					try {
						$this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300,
							fn () => $this->targets->repository($target)->recordSnapshot((string)$state['files']['snapshot']));
						$state['files']['phase'] = 'done';
					} catch (LockedException|RepositoryException) {
						// Another catalog write is in progress (or just raced us to the next
						// generation): the next step retries this commit on the fresh catalog.
					}
				}
				if ($state['files']['phase'] === 'done') {
					$this->recordSnapshot($run, $state['files']);
					// The snapshot is committed: what follows only reports on it and must not turn
					// the run into a failure.
					try {
						$this->alerts->storedBytes($target, $repo->storedBytes());
						$stats = $state['files']['stats'];
						$warnings = BackupRun::warnings($state['files']);
						if (($stats['badNames'] ?? 0) > 0) {
							$this->logger->warning(sprintf('NG Backup run %d to %s: %d left out because their name is not supported (rename them): %s',
								$run->getId(), $target->getName(), $stats['badNames'], implode(', ', $warnings['badNames'])));
							$this->alerts->runBadNames($run, $target->getName(), (int)$stats['badNames'], $warnings['badNames']);
						}
						if (($stats['unreadable'] ?? 0) > 0 || ($stats['changing'] ?? 0) > 0) {
							$this->logger->warning(sprintf('NG Backup run %d to %s: %d unreadable (missing from the snapshot): %s; %d changed while being read: %s',
								$run->getId(), $target->getName(), $stats['unreadable'] ?? 0, implode(', ', $warnings['unreadable']),
								$stats['changing'] ?? 0, implode(', ', $warnings['changing'])));
							$this->alerts->runWarnings($run, $target->getName(), (int)($stats['unreadable'] ?? 0), (int)($stats['changing'] ?? 0), $warnings);
						}
					} catch (\Throwable $e) {
						$this->logger->warning('NG Backup run ' . $run->getId() . ': reporting on the finished backup failed: ' . $e->getMessage(), ['exception' => $e]);
					}
					$run->setStatus(Run::DONE);
					$run->setPhase('done');
					$run->setFinishedAt(time());
				}
			}
			$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
			$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
			// Another process may have taken the run over after this step's lease expired: then its
			// writes stand and this step's are dropped.
			$saved = $this->runs->updateIfUnchanged($run, $loadedAt);
			if ($saved === null) {
				$repo = null; // the run is another process's now: its packs are not in our index
				return $this->runs->find($run->getId());
			}
			return $saved;
		} catch (\Throwable $e) {
			$repo = null; // may hold blobs of a half-written pack: the next step opens it afresh
			if ($loadedAt === null) {
				// The run could not even be loaded: there is nothing fresh to mark as failed, and
				// the caller's copy may be stale.
				$this->logger->error('NG Backup run ' . $run->getId() . ': step could not load the run: ' . $e->getMessage(), ['exception' => $e]);
				return $run;
			}
			$run->setStatus(Run::FAILED);
			$run->setError($e->getMessage());
			$run->setFinishedAt(time());
			$failed = $this->runs->updateIfUnchanged($run, $loadedAt);
			if ($failed === null) {
				// Another process that took the run over may have finished it: its result stands.
				$this->logger->warning('NG Backup run ' . $run->getId() . ': a step that another process had taken over ended with: ' . $e->getMessage());
				return $this->runs->find($run->getId());
			}
			$run = $failed;
			$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
			try {
				$this->alerts->runFailed($run, isset($target) ? $target->getName() : '#' . $run->getTargetId());
			} catch (\Throwable) {
			}
			return $run;
		} finally {
			$this->leases->release($targetLease);
			$this->leases->release($runLease);
		}
	}

	/** Run to completion in this process (occ, system cron), reporting progress. */
	public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
		$repo = null; // opened once, by the first step, and kept for the next ones
		while ($run->getStatus() === Run::RUNNING) {
			$before = $run->getUpdatedAt();
			$run = $this->step($run, microtime(true) + $stepSeconds, $repo);
			if ($run->getStatus() === Run::RUNNING && $run->getUpdatedAt() === $before) {
				// Step was not possible (another process holds the run, or a crashed step's lease has
				// not expired yet): wait instead of spinning.
				sleep(5);
				$run = $this->runs->find($run->getId());
				$repo = null; // another process may have stored packs in the meantime
			}
			if ($progress !== null) {
				$progress($run);
			}
		}
		return $run;
	}

	/**
	 * Stop a running backup. First no new step may start, then this waits (up to $wait seconds)
	 * for the step in progress, so a step that is recording the snapshot finishes first. A process
	 * running the backup then finds it cancelled and stops. The packs it stored stay until the
	 * next cleanup.
	 *
	 * A process of an earlier version does not hold back its next step and may never leave a gap:
	 * after $wait the run is marked cancelled while it works, unless it is recording the snapshot.
	 * Its step then fails to save and the process stops.
	 */
	public function cancel(Run $run, int $wait = 120): Run {
		$id = $run->getId();
		$until = time() + $wait;
		$hold = null;
		for ($try = 0; $hold === null; $try++) {
			try {
				$hold = $this->leases->acquire(self::cancelKey($id), LeaseService::EXCLUSIVE, $wait + 300);
			} catch (LockedException) {
				if ($try >= 20) {
					throw new \RuntimeException("Run $id is being cancelled already");
				}
				usleep(100000); // a step checking whether it may start
			}
		}
		try {
			$lease = null;
			while ($lease === null && time() < $until) {
				try {
					$lease = $this->leases->acquire('run/' . $id, LeaseService::EXCLUSIVE, 60);
				} catch (LockedException) {
					sleep(2);
				}
			}
			try {
				for ($try = 0; $try < 10; $try++) {
					$run = $this->runs->find($id);
					if ($run->getStatus() !== Run::RUNNING) {
						return $run;
					}
					$state = json_decode($run->getState(), true) ?: [];
					if ($lease === null && ($state['files']['phase'] ?? '') === 'commit') {
						throw new \RuntimeException("Run $id is recording its snapshot; try again in a minute");
					}
					$loadedAt = $run->getUpdatedAt();
					$run->setStatus(Run::CANCELLED);
					$run->setFinishedAt(time());
					if (($cancelled = $this->runs->updateIfUnchanged($run, $loadedAt)) !== null) {
						return $cancelled;
					}
					usleep(200000); // a step of another process saved just now
				}
				throw new \RuntimeException("Run $id keeps changing; try again");
			} finally {
				if ($lease !== null) {
					$this->leases->release($lease);
				}
			}
		} finally {
			$this->leases->release($hold);
		}
	}

	private static function cancelKey(int $runId): string {
		return 'run/' . $runId . '/cancel';
	}

	private function recordSnapshot(Run $run, array $files): void {
		try {
			$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
			return; // already recorded, by an earlier attempt at this step
		} catch (DoesNotExistException) {
		}
		$s = new Snapshot();
		$s->setTargetId($run->getTargetId());
		$s->setSnapshotId($files['snapshot']);
		$s->setKind($run->getKind());
		$s->setLabel($files['label'] !== '' ? $files['label'] : null);
		$s->setCreatedAt(time());
		$s->setFiles($files['stats']['files']);
		$s->setBytes($files['stats']['bytes']);
		try {
			$this->snapshots->insert($s);
		} catch (\OCP\DB\Exception $e) {
			// Recorded by another process between the check above and this insert?
			if ($e->getReason() !== \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
				throw $e;
			}
			try {
				$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
			} catch (DoesNotExistException) {
				throw $e;
			}
		}
	}

	/** @return list<string> */
	private function excludes(string $dataDir): array {
		$instanceId = $this->config->getSystemValueString('instanceid');
		$exclude = [
			"data/appdata_$instanceId/preview",
			"data/updater-$instanceId",
			'data/nextcloud.log',
			'data/nextcloud.log.*',
			'data/audit.log',
			'data/*/cache',
			'data/*/uploads',
		];
		// SQLite: the database file is in the data directory; it is backed up by the logical dump,
		// and copying the live file would give an inconsistent copy.
		if ($this->config->getSystemValueString('dbtype') === 'sqlite3') {
			$dbName = $this->config->getSystemValueString('dbname', 'owncloud');
			$exclude[] = "data/$dbName.db";
			$exclude[] = "data/$dbName.db-journal";
			$exclude[] = "data/$dbName.db-wal";
			$exclude[] = "data/$dbName.db-shm";
		}
		$logFile = $this->config->getSystemValueString('logfile', '');
		if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
			$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
		}
		// Never back up a local backup location into itself.
		foreach ($this->targets->list() as $t) {
			$opts = $this->targets->options($t);
			$dir = rtrim((string)($opts['datadir'] ?? ''), '/');
			if ($t->getBackend() === 'local' && $dir !== '' && str_starts_with($dir . '/', $dataDir . '/')) {
				$exclude[] = trim('data/' . substr($dir, strlen($dataDir) + 1) . '/' . $t->getBasePath(), '/');
			}
		}
		return array_values(array_unique($exclude));
	}

	/** @return array<string, string> app id => version */
	private function installedApps(): array {
		$apps = [];
		foreach ($this->appManager->getEnabledApps() as $app) {
			$apps[$app] = $this->appManager->getAppVersion($app);
		}
		ksort($apps);
		return $apps;
	}
}