Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

M2: full backup runs (database, config, data) with occ and cron

Browse Source 
BackupService starts a run per location (database dump in one
transaction, then config + data directory as a resumable file run),
stores the state in ngb_runs after every step, locks a run while a step
executes and records the snapshot. Previews, logs, updater backups and
per-user cache/uploads are excluded (TreeWalker now supports patterns).
BackupJob (every 5 min) starts the daily schedule and steps running runs
with a short budget under web cron and a long one under system cron.
occ: backup:run, backup:status, backup:list.

Tested on NC34/PostgreSQL: first run 128 files / 1 GiB in 64 s (443 MiB
uploaded to S3), unchanged second run 2 s / 0 bytes; a background run to
SFTP completed through cron.php.
This commit is contained in:
Dillard Blom
2026-10-02 00:53:50 +02:00
parent 4a28bf8790
commit 3f6785bcc0
12 changed files with 693 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
appinfo/info.xml
+7 -1
View File
@@ -5,7 +5,7 @@
<name>NG Backup</name>
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
<description><![CDATA[Work in progress, not ready for use.]]></description>
<version>0.0.1</version>
<version>0.0.2</version>
<licence>agpl</licence>
<author>Dillard Blom</author>
<namespace>NgBackup</namespace>
@@ -16,6 +16,9 @@
<lib>sodium</lib>
<nextcloud min-version="32" max-version="35"/>
</dependencies>
<background-jobs>
<job>OCA\NgBackup\BackgroundJob\BackupJob</job>
</background-jobs>
<commands>
<command>OCA\NgBackup\Command\KeyInit</command>
<command>OCA\NgBackup\Command\KeyKit</command>
@@ -24,5 +27,8 @@
<command>OCA\NgBackup\Command\TargetAdd</command>
<command>OCA\NgBackup\Command\TargetList</command>
<command>OCA\NgBackup\Command\TargetRemove</command>
<command>OCA\NgBackup\Command\RunBackup</command>
<command>OCA\NgBackup\Command\Status</command>
<command>OCA\NgBackup\Command\ListSnapshots</command>
</commands>
</info>
lib/BackgroundJob/BackupJob.php
+72
View File
@@ -0,0 +1,72 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\BackgroundJob;
use OCA\NgBackup\AppInfo\Application;
use OCA\NgBackup\Db\RunMapper;
use OCA\NgBackup\Service\BackupService;
use OCA\NgBackup\Service\TargetService;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJob;
use OCP\BackgroundJob\TimedJob;
use OCP\IAppConfig;
use Psr\Log\LoggerInterface;
/**
* Starts scheduled backups and moves running ones forward, a time-boxed step per cron run.
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
* in a normal web request (hosted Nextcloud).
*/
class BackupJob extends TimedJob {
public function __construct(
ITimeFactory $time,
private BackupService $backups,
private TargetService $targets,
private RunMapper $runs,
private IAppConfig $appConfig,
private LoggerInterface $logger,
) {
parent::__construct($time);
$this->setInterval(300);
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
}
protected function run($argument): void {
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
$deadline = microtime(true) + $budget;
$this->startScheduled();
foreach ($this->runs->findRunning() as $run) {
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
$run = $this->backups->step($run, min($deadline, microtime(true) + $budget));
}
}
}
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
private function startScheduled(): void {
$at = $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', '');
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
return;
}
$todayAt = strtotime(date('Y-m-d') . ' ' . $at);
if (time() < $todayAt) {
return;
}
foreach ($this->targets->list() as $t) {
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
if ($this->runs->findRunning($t->getId()) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
continue;
}
try {
$this->backups->start($t, 'scheduled');
} catch (\Throwable $e) {
$this->logger->warning('NG Backup: scheduled backup to ' . $t->getName() . ' not started: ' . $e->getMessage());
}
}
}
}
lib/Command/ListSnapshots.php
+49
View File
@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Command;
use OCA\NgBackup\Db\SnapshotMapper;
use OCA\NgBackup\Service\TargetService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class ListSnapshots extends Command {
public function __construct(
private TargetService $targets,
private SnapshotMapper $snapshots,
) {
parent::__construct();
}
protected function configure(): void {
$this->setName('backup:list')
->setDescription('List the snapshots (restore points) on a location')
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all)');
}
protected function execute(InputInterface $input, OutputInterface $output): int {
$name = $input->getArgument('target');
$targets = $name === null ? $this->targets->list() : [$this->targets->get($name)];
$rows = [];
foreach ($targets as $t) {
foreach ($this->snapshots->findForTarget($t->getId()) as $s) {
$rows[] = [$t->getName(), $s->getSnapshotId(), date('Y-m-d H:i', $s->getCreatedAt()), $s->getKind(),
$s->getFiles(), sprintf('%.1f GiB', ($s->getBytes() ?? 0) / 1073741824), $s->getLabel() ?? ''];
}
}
if ($rows === []) {
$output->writeln('No snapshots yet (occ backup:run).');
return 0;
}
(new Table($output))->setHeaders(['location', 'snapshot', 'date', 'kind', 'files', 'size', 'label'])->setRows($rows)->render();
return 0;
}
}
lib/Command/RunBackup.php
+85
View File
@@ -0,0 +1,85 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Command;
use OCA\NgBackup\Db\Run;
use OCA\NgBackup\Service\BackupService;
use OCA\NgBackup\Service\TargetService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class RunBackup extends Command {
public function __construct(
private BackupService $backups,
private TargetService $targets,
) {
parent::__construct();
}
protected function configure(): void {
$this->setName('backup:run')
->setDescription('Back up this Nextcloud (database, config, data) to a location now')
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all locations)')
->addOption('label', 'l', InputOption::VALUE_REQUIRED, 'Label for the snapshot', '')
->addOption('background', 'b', InputOption::VALUE_NONE, 'Only start the run; background jobs finish it')
->setHelp(<<<'HELP'
Starts a backup (or continues one that is already running for that location) and runs it to
the end in this process. Unchanged files are not read again; only new data is uploaded.
Examples:
occ backup:run back up to every location
occ backup:run offsite -l "before upgrade"
occ backup:run offsite --background
HELP);
}
protected function execute(InputInterface $input, OutputInterface $output): int {
$name = $input->getArgument('target');
try {
$targets = $name === null ? $this->targets->list() : [$this->targets->get($name)];
} catch (\Throwable) {
$output->writeln('<error>No such location.</error>');
return 1;
}
if ($targets === []) {
$output->writeln('<error>No backup locations yet (occ backup:target:add).</error>');
return 1;
}
$failed = 0;
foreach ($targets as $t) {
try {
$run = $this->backups->start($t, (string)$input->getOption('label'));
} catch (\Throwable $e) {
$output->writeln('<error>' . $t->getName() . ': ' . $e->getMessage() . '</error>');
$failed++;
continue;
}
$output->writeln(sprintf('<info>%s</info>: run %d, snapshot %s', $t->getName(), $run->getId(), $run->getSnapshotId()));
if ($input->getOption('background')) {
continue;
}
$start = microtime(true);
$run = $this->backups->runToCompletion($run, function (Run $r) use ($output, $start) {
$s = json_decode($r->getStats() ?? '{}', true);
$f = $s['files'] ?? [];
$output->writeln(sprintf(' %5.0fs %-6s files %d (read %d, unchanged %d), uploaded %.1f MiB',
microtime(true) - $start, $r->getPhase(), $f['files'] ?? 0, $f['read'] ?? 0, $f['reused'] ?? 0, ($f['uploaded'] ?? 0) / 1048576));
});
if ($run->getStatus() === Run::DONE) {
$output->writeln(sprintf(' done in %.0fs', microtime(true) - $start));
} else {
$output->writeln('<error> ' . $run->getStatus() . ': ' . $run->getError() . '</error>');
$failed++;
}
}
return $failed === 0 ? 0 : 1;
}
}
lib/Command/Status.php
+56
View File
@@ -0,0 +1,56 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Command;
use OCA\NgBackup\Db\RunMapper;
use OCA\NgBackup\Service\KeyService;
use OCA\NgBackup\Service\TargetService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class Status extends Command {
public function __construct(
private KeyService $keys,
private TargetService $targets,
private RunMapper $runs,
) {
parent::__construct();
}
protected function configure(): void {
$this->setName('backup:status')->setDescription('Show the backup key, locations and recent runs');
}
protected function execute(InputInterface $input, OutputInterface $output): int {
if (!$this->keys->isInitialized()) {
$output->writeln('Key: <error>not created</error> (occ backup:key:init)');
} else {
$c = $this->keys->recoveryKitConfirmation();
$output->writeln('Key: ' . $this->keys->fingerprint() . ', recovery kit '
. ($c ? 'confirmed by ' . $c['uid'] . ' on ' . date('Y-m-d H:i', $c['time']) : '<error>not confirmed</error> (occ backup:key:confirm)'));
}
$names = [];
foreach ($this->targets->list() as $t) {
$names[$t->getId()] = $t->getName();
}
$output->writeln('Locations: ' . ($names === [] ? 'none' : implode(', ', $names)));
$rows = [];
foreach ($this->runs->findRecent(10) as $r) {
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . ')' : ''),
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
$s['files'] ?? '', isset($s['uploaded']) ? sprintf('%.1f MiB', $s['uploaded'] / 1048576) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
}
if ($rows !== []) {
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'uploaded', 'error'])->setRows($rows)->render();
}
return 0;
}
}
lib/Db/Run.php
+60
View File
@@ -0,0 +1,60 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Db;
use OCP\AppFramework\Db\Entity;
/**
* @method int getTargetId()
* @method void setTargetId(int $v)
* @method string getKind()
* @method void setKind(string $v)
* @method string getStatus()
* @method void setStatus(string $v)
* @method string getPhase()
* @method void setPhase(string $v)
* @method string getState()
* @method void setState(string $v)
* @method string|null getStats()
* @method void setStats(?string $v)
* @method string|null getError()
* @method void setError(?string $v)
* @method string|null getSnapshotId()
* @method void setSnapshotId(?string $v)
* @method int getStartedAt()
* @method void setStartedAt(int $v)
* @method int getUpdatedAt()
* @method void setUpdatedAt(int $v)
* @method int|null getFinishedAt()
* @method void setFinishedAt(?int $v)
*/
class Run extends Entity {
public const RUNNING = 'running';
public const DONE = 'done';
public const FAILED = 'failed';
public const CANCELLED = 'cancelled';
protected int $targetId = 0;
protected string $kind = '';
protected string $status = '';
protected string $phase = '';
protected string $state = '{}';
protected ?string $stats = null;
protected ?string $error = null;
protected ?string $snapshotId = null;
protected int $startedAt = 0;
protected int $updatedAt = 0;
protected ?int $finishedAt = null;
public function __construct() {
$this->addType('targetId', 'integer');
$this->addType('startedAt', 'integer');
$this->addType('updatedAt', 'integer');
$this->addType('finishedAt', 'integer');
}
}
lib/Db/RunMapper.php
+53
View File
@@ -0,0 +1,53 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Db;
use OCP\AppFramework\Db\QBMapper;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
/** @template-extends QBMapper<Run> */
class RunMapper extends QBMapper {
public function __construct(IDBConnection $db) {
parent::__construct($db, 'ngb_runs', Run::class);
}
public function find(int $id): Run {
$qb = $this->db->getQueryBuilder();
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
return $this->findEntity($qb);
}
/** @return list<Run> */
public function findRunning(?int $targetId = null): array {
$qb = $this->db->getQueryBuilder();
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('status', $qb->createNamedParameter(Run::RUNNING)));
if ($targetId !== null) {
$qb->andWhere($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)));
}
return $this->findEntities($qb->orderBy('id'));
}
/** @return list<Run> */
public function findRecent(int $limit = 20): array {
$qb = $this->db->getQueryBuilder();
$qb->select('*')->from($this->getTableName())->orderBy('id', 'DESC')->setMaxResults($limit);
return $this->findEntities($qb);
}
public function lastFinished(int $targetId, string $kind): ?Run {
$qb = $this->db->getQueryBuilder();
$qb->select('*')->from($this->getTableName())
->where($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->eq('kind', $qb->createNamedParameter($kind)))
->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Run::DONE)))
->orderBy('id', 'DESC')->setMaxResults(1);
$r = $this->findEntities($qb);
return $r[0] ?? null;
}
}
lib/Db/Snapshot.php
+43
View File
@@ -0,0 +1,43 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Db;
use OCP\AppFramework\Db\Entity;
/**
* @method int getTargetId()
* @method void setTargetId(int $v)
* @method string getSnapshotId()
* @method void setSnapshotId(string $v)
* @method string getKind()
* @method void setKind(string $v)
* @method string|null getLabel()
* @method void setLabel(?string $v)
* @method int getCreatedAt()
* @method void setCreatedAt(int $v)
* @method int|null getFiles()
* @method void setFiles(?int $v)
* @method int|null getBytes()
* @method void setBytes(?int $v)
*/
class Snapshot extends Entity {
protected int $targetId = 0;
protected string $snapshotId = '';
protected string $kind = '';
protected ?string $label = null;
protected int $createdAt = 0;
protected ?int $files = null;
protected ?int $bytes = null;
public function __construct() {
$this->addType('targetId', 'integer');
$this->addType('createdAt', 'integer');
$this->addType('files', 'integer');
$this->addType('bytes', 'integer');
}
}
lib/Db/SnapshotMapper.php
+45
View File
@@ -0,0 +1,45 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Db;
use OCP\AppFramework\Db\QBMapper;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
/** @template-extends QBMapper<Snapshot> */
class SnapshotMapper extends QBMapper {
public function __construct(IDBConnection $db) {
parent::__construct($db, 'ngb_snapshots', Snapshot::class);
}
/** @return list<Snapshot> newest first */
public function findForTarget(int $targetId): array {
$qb = $this->db->getQueryBuilder();
$qb->select('*')->from($this->getTableName())
->where($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)))
->orderBy('created_at', 'DESC');
return $this->findEntities($qb);
}
public function findOne(int $targetId, string $snapshotId): Snapshot {
$qb = $this->db->getQueryBuilder();
$qb->select('*')->from($this->getTableName())
->where($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->eq('snapshot_id', $qb->createNamedParameter($snapshotId)));
return $this->findEntity($qb);
}
public function latest(int $targetId, string $kind): ?Snapshot {
foreach ($this->findForTarget($targetId) as $s) {
if ($s->getKind() === $kind) {
return $s;
}
}
return null;
}
}
lib/Job/TreeWalker.php
+18 -13
View File
@@ -17,7 +17,8 @@ namespace OCA\NgBackup\Job;
final class TreeWalker {
/**
* @param array<string, string> $roots name => absolute directory, walked in this order
* @param list<string> $exclude path prefixes ("<root>/<relative>") that are skipped entirely
* @param list<string> $exclude path prefixes ("<root>/<relative>") that are skipped entirely;
* a pattern with "*" matches one path segment (fnmatch, FNM_PATHNAME)
*/
public function __construct(
private array $roots,
@@ -51,10 +52,8 @@ final class TreeWalker {
}
private function walkDir(string $abs, string $logical, ?string $after): \Generator {
foreach ($this->exclude as $prefix) {
if ($logical !== '' && ($logical === $prefix || str_starts_with($logical, $prefix . '/'))) {
return;
}
if ($logical !== '' && $this->isExcluded($logical, true)) {
return;
}
$names = @scandir($abs);
if ($names === false) {
@@ -78,20 +77,26 @@ final class TreeWalker {
if ($after !== null && self::compare($childLogical, $after) <= 0) {
continue;
}
$excluded = false;
foreach ($this->exclude as $prefix) {
if ($childLogical === $prefix) {
$excluded = true;
break;
}
}
if (!$excluded) {
if (!$this->isExcluded($childLogical, false)) {
yield $childLogical => $childAbs;
}
}
}
}
private function isExcluded(string $logical, bool $isDir): bool {
foreach ($this->exclude as $pattern) {
if (str_contains($pattern, '*')) {
if (fnmatch($pattern, $logical, FNM_PATHNAME)) {
return true;
}
} elseif ($logical === $pattern || ($isDir && str_starts_with($logical, $pattern . '/'))) {
return true;
}
}
return false;
}
private static function isPrefix(string $dir, string $path): bool {
return str_starts_with($path, $dir . '/');
}
lib/Service/BackupService.php
+204
View File
@@ -0,0 +1,204 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Service;
use OCA\NgBackup\Db\DbDumper;
use OCA\NgBackup\Db\Run;
use OCA\NgBackup\Db\RunMapper;
use OCA\NgBackup\Db\Snapshot;
use OCA\NgBackup\Db\SnapshotMapper;
use OCA\NgBackup\Db\Target;
use OCA\NgBackup\Job\BackupRun;
use OCP\App\IAppManager;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\Lock\ILockingProvider;
use OCP\Lock\LockedException;
use Psr\Log\LoggerInterface;
/**
* Full backups of this installation: database (one consistent transaction), config and the
* data directory, as a resumable run whose state is stored in ngb_runs after every step.
*
* Excluded by default: previews, log files, the updater's backups and each user's cache and
* temporary uploads (all regenerable or temporary).
*/
class BackupService {
public const KIND_FULL = 'full';
public function __construct(
private TargetService $targets,
private KeyService $keys,
private RunMapper $runs,
private SnapshotMapper $snapshots,
private IConfig $config,
private IDBConnection $db,
private IAppManager $appManager,
private ILockingProvider $locking,
private LoggerInterface $logger,
) {
}
public function start(Target $target, string $label = ''): Run {
if ($this->keys->recoveryKitConfirmation() === null) {
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
}
if ($this->config->getSystemValue('objectstore', null) !== null) {
throw new \RuntimeException('Object storage as primary storage is not supported yet');
}
$running = $this->runs->findRunning($target->getId());
if ($running !== []) {
return $running[0];
}
$dataDir = rtrim($this->config->getSystemValueString('datadirectory', \OC::$SERVERROOT . '/data'), '/');
$parent = $this->snapshots->latest($target->getId(), self::KIND_FULL);
$files = BackupRun::start(
['config' => rtrim(\OC::$configDir, '/'), 'data' => $dataDir],
$parent?->getSnapshotId(),
$label,
$this->excludes($dataDir),
['kind' => self::KIND_FULL, 'nextcloud' => $this->config->getSystemValueString('version'),
'instanceid' => $this->config->getSystemValueString('instanceid'),
'dbtype' => $this->config->getSystemValueString('dbtype'),
'apps' => $this->installedApps()],
);
$run = new Run();
$run->setTargetId($target->getId());
$run->setKind(self::KIND_FULL);
$run->setStatus(Run::RUNNING);
$run->setPhase('db');
$run->setState(json_encode(['files' => $files], JSON_THROW_ON_ERROR));
$run->setSnapshotId($files['snapshot']);
$run->setStartedAt(time());
$run->setUpdatedAt(time());
return $this->runs->insert($run);
}
/**
* Do one step of a run until $deadline. Safe to call concurrently: a second caller finds the
* run locked and returns immediately.
*/
public function step(Run $run, float $deadline): Run {
$lockKey = 'ng_backup/run/' . $run->getId();
try {
$this->locking->acquireLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
} catch (LockedException) {
return $run;
}
try {
$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
if ($run->getStatus() !== Run::RUNNING) {
return $run;
}
$target = $this->targets->get((string)$run->getTargetId());
$repo = $this->targets->repository($target);
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
if ($run->getPhase() === 'db') {
@set_time_limit(0); // the dump is one transaction; it cannot be split across requests yet
$dbStats = [];
$manifest = (new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_')))->dump($repo, $dbStats);
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
$state['dbStats'] = $dbStats + ['tables' => count($manifest['tables']),
'rows' => array_sum(array_column($manifest['tables'], 'rows'))];
$run->setPhase('files');
} else {
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
if ($state['files']['phase'] === 'done') {
$this->recordSnapshot($run, $state['files']);
$run->setStatus(Run::DONE);
$run->setPhase('done');
$run->setFinishedAt(time());
}
}
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
$run->setUpdatedAt(time());
return $this->runs->update($run);
} catch (\Throwable $e) {
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
$run->setStatus(Run::FAILED);
$run->setError($e->getMessage());
$run->setFinishedAt(time());
$run->setUpdatedAt(time());
return $this->runs->update($run);
} finally {
$this->locking->releaseLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
}
}
/** Run to completion in this process (occ, system cron), reporting progress. */
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
while ($run->getStatus() === Run::RUNNING) {
$run = $this->step($run, microtime(true) + $stepSeconds);
if ($progress !== null) {
$progress($run);
}
}
return $run;
}
public function cancel(Run $run): Run {
$run->setStatus(Run::CANCELLED);
$run->setFinishedAt(time());
$run->setUpdatedAt(time());
return $this->runs->update($run);
}
private function recordSnapshot(Run $run, array $files): void {
$s = new Snapshot();
$s->setTargetId($run->getTargetId());
$s->setSnapshotId($files['snapshot']);
$s->setKind($run->getKind());
$s->setLabel($files['label'] !== '' ? $files['label'] : null);
$s->setCreatedAt(time());
$s->setFiles($files['stats']['files']);
$s->setBytes($files['stats']['bytes']);
$this->snapshots->insert($s);
}
/** @return list<string> */
private function excludes(string $dataDir): array {
$instanceId = $this->config->getSystemValueString('instanceid');
$exclude = [
"data/appdata_$instanceId/preview",
"data/updater-$instanceId",
'data/nextcloud.log',
'data/nextcloud.log.*',
'data/audit.log',
'data/*/cache',
'data/*/uploads',
];
$logFile = $this->config->getSystemValueString('logfile', '');
if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
}
// Never back up a local backup location into itself.
foreach ($this->targets->list() as $t) {
$opts = $this->targets->options($t);
$dir = rtrim((string)($opts['datadir'] ?? ''), '/');
if ($t->getBackend() === 'local' && $dir !== '' && str_starts_with($dir . '/', $dataDir . '/')) {
$exclude[] = trim('data/' . substr($dir, strlen($dataDir) + 1) . '/' . $t->getBasePath(), '/');
}
}
return array_values(array_unique($exclude));
}
/** @return array<string, string> app id => version */
private function installedApps(): array {
$apps = [];
foreach ($this->appManager->getEnabledApps() as $app) {
$apps[$app] = $this->appManager->getAppVersion($app);
}
ksort($apps);
return $apps;
}
}
tests/.phpunit.result.cache
+1 -1
View File
@@ -1 +1 @@
{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.075,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.094,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.004,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.276,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.498,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.8,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":2.005,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.016,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.136,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.049,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.087,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.064,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.095,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.24,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.802,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.006,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.005}}
{"version":2,"defects":{"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":7},"times":{"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrapUnwrapKeepsKeys":1.068,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testWrongPassphrase":1.175,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testBlobIdsAreKeyed":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\KeyRingTest::testRejectsShortMasterKey":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#empty":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#small":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#one frame":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#frame+1":0.001,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testRoundTrip#several":0.003,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testStreamingKeepsMemoryBounded":0.234,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#flipped byte":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#truncated":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#trailing data":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#cut at frame boundary":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testCorruptionIsRejected#bad magic":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongAssociatedDataIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Crypto\\StreamCipherTest::testWrongKeyIsRejected":0,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testRunInManyTinyStepsRestoresIdentically":1.39,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testFileChangedBetweenStepsIsReadAgain":1.724,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBackupAndRestoreAreIdentical":1.985,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testUnchangedRunReadsAndUploadsNothing":1.045,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testChangeInLargeFileUploadsOnlyAffectedBlob":1.175,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testPartialRestore":1.101,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testWrongPassphrase":1.038,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testInitRefusesExistingRepository":1.024,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testTamperedPackIsDetected":1.06,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testBlobWriterAndReadLinesAcrossBlobBoundaries":1.378,"OCA\\NgBackup\\Tests\\Unit\\Repository\\RepositoryTest::testObjectsAreBoundToTheirPath":0.544,"OCA\\NgBackup\\Tests\\Unit\\Job\\BackupRunTest::testIncrementalRunInTinyStepsReadsNothing":2.984,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testOrderMatchesCompare":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testResumeAfterCursorYieldsTheRest":0.003,"OCA\\NgBackup\\Tests\\Unit\\Job\\TreeWalkerTest::testExcludeAndMultipleRoots":0.069}}