1.0.1: label for "Back up now", retention explained in the trash, occ times in the schedule's zone, single-file restore to a directory, leftovers of a stopped restore removed
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / psalm (push) Has been cancelled
CI / frontend (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
CI / integration-targets (push) Has been cancelled
CI / integration-user (32) (push) Has been cancelled
CI / integration-user (33) (push) Has been cancelled
CI / integration-user (34) (push) Has been cancelled
CI / integration-user (35) (push) Has been cancelled
CI / integration-full-restore (push) Has been cancelled
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / psalm (push) Has been cancelled
CI / frontend (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
CI / integration-targets (push) Has been cancelled
CI / integration-user (32) (push) Has been cancelled
CI / integration-user (33) (push) Has been cancelled
CI / integration-user (34) (push) Has been cancelled
CI / integration-user (35) (push) Has been cancelled
CI / integration-full-restore (push) Has been cancelled
- Admin page: an optional label (at most 100 characters) for a backup started with "Back up now"; an (i) by "removed by retention policy" in the trash that explains what it means and until when the restore point can be brought back. backup:trash says the same below its list. - backup:status, backup:list, backup:trash and backup:browse show times in the zone of the daily schedule and name it, instead of the server's UTC. - backup:restore --to-directory of a file restored nothing: the path got a "/" appended. - A restore removes "<name>.ngb-restore-<hex>" temporaries that a stopped restore left in the folders it writes to. - Roadmap: the cron warning (#13), reading ahead from S3/SMB and --salvage move to 1.0.2.
This commit is contained in:
@@ -7,7 +7,9 @@ declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\RestoreService;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\Table;
|
||||
@@ -19,6 +21,7 @@ final class Browse extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private RestoreService $restore,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -43,9 +46,10 @@ final class Browse extends Command {
|
||||
$output->writeln('Nothing here.');
|
||||
return 0;
|
||||
}
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
$rows = array_map(fn ($i) => [$i['type'] === 'dir' ? $i['name'] . '/' : $i['name'], $this->size($i['size']),
|
||||
$i['type'] === 'dir' ? $i['files'] . ' files' : date('Y-m-d H:i', (int)$i['mtime'])], $items);
|
||||
(new Table($output))->setHeaders(['name', 'size', 'files / modified'])->setRows($rows)->render();
|
||||
$i['type'] === 'dir' ? $i['files'] . ' files' : Format::time((int)$i['mtime'], $zone)], $items);
|
||||
(new Table($output))->setHeaders(['name', 'size', "files / modified ($zone)"])->setRows($rows)->render();
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ declare(strict_types=1);
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Db\SnapshotMapper;
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\Table;
|
||||
@@ -19,6 +21,7 @@ final class ListSnapshots extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private SnapshotMapper $snapshots,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -32,10 +35,11 @@ final class ListSnapshots extends Command {
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$name = $input->getArgument('target');
|
||||
$targets = $name === null ? $this->targets->list() : [$this->targets->get($name)];
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
$rows = [];
|
||||
foreach ($targets as $t) {
|
||||
foreach ($this->snapshots->findForTarget($t->getId()) as $s) {
|
||||
$rows[] = [$t->getName(), $s->getSnapshotId(), date('Y-m-d H:i', $s->getCreatedAt()), $s->getKind(),
|
||||
$rows[] = [$t->getName(), $s->getSnapshotId(), Format::time($s->getCreatedAt(), $zone), $s->getKind(),
|
||||
$s->getFiles(), sprintf('%.1f GiB', ($s->getBytes() ?? 0) / 1073741824), $s->getLabel() ?? ''];
|
||||
}
|
||||
}
|
||||
@@ -43,7 +47,7 @@ final class ListSnapshots extends Command {
|
||||
$output->writeln('No snapshots yet (occ backup:run).');
|
||||
return 0;
|
||||
}
|
||||
(new Table($output))->setHeaders(['location', 'snapshot', 'date', 'kind', 'files', 'size', 'label'])->setRows($rows)->render();
|
||||
(new Table($output))->setHeaders(['location', 'snapshot', "date ($zone)", 'kind', 'files', 'size', 'label'])->setRows($rows)->render();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,9 @@ declare(strict_types=1);
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Db\RunMapper;
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\KeyService;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use OCA\NgBackup\Service\VerifyService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
@@ -26,6 +28,7 @@ final class Status extends Command {
|
||||
private TargetService $targets,
|
||||
private RunMapper $runs,
|
||||
private VerifyService $verify,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -36,12 +39,13 @@ final class Status extends Command {
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
if (!$this->keys->isInitialized()) {
|
||||
$output->writeln('Key: <error>not created</error> (occ backup:key:init)');
|
||||
} else {
|
||||
$c = $this->keys->recoveryKitConfirmation();
|
||||
$output->writeln('Key: ' . $this->keys->fingerprint() . ', recovery kit '
|
||||
. ($c ? 'confirmed by ' . $c['uid'] . ' on ' . date('Y-m-d H:i', $c['time']) : '<error>not confirmed</error> (occ backup:key:confirm)'));
|
||||
. ($c ? 'confirmed by ' . $c['uid'] . ' on ' . Format::time($c['time'], $zone) . " ($zone)" : '<error>not confirmed</error> (occ backup:key:confirm)'));
|
||||
}
|
||||
$names = [];
|
||||
foreach ($this->targets->list() as $t) {
|
||||
@@ -54,12 +58,12 @@ final class Status extends Command {
|
||||
foreach ($shown as $r) {
|
||||
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
|
||||
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . self::percent($r) . ')' : ''),
|
||||
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? \OCA\NgBackup\Format::duration($r->getFinishedAt() - $r->getStartedAt()) : '',
|
||||
Format::time($r->getStartedAt(), $zone), $r->getFinishedAt() ? Format::duration($r->getFinishedAt() - $r->getStartedAt()) : '',
|
||||
$s['files'] ?? '', isset($s['bytes']) ? self::size((int)$s['bytes']) : '',
|
||||
isset($s['uploaded']) ? self::size((int)$s['uploaded']) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
|
||||
}
|
||||
if ($rows !== []) {
|
||||
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'size', 'uploaded', 'error'])->setRows($rows)->render();
|
||||
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', "started ($zone)", 'took', 'files', 'size', 'uploaded', 'error'])->setRows($rows)->render();
|
||||
}
|
||||
$older = $all ? 0 : $this->runs->countAll() - count($shown);
|
||||
if ($older > 0) {
|
||||
|
||||
@@ -7,7 +7,9 @@ declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\KeyService;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\Table;
|
||||
@@ -19,6 +21,7 @@ final class Trash extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private KeyService $keys,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -32,11 +35,14 @@ final class Trash extends Command {
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$name = $input->getArgument('target');
|
||||
$delay = $this->keys->deleteDelayDays() * 86400;
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
$rows = [];
|
||||
foreach ($name === null ? $this->targets->list() : [$this->targets->get($name)] as $t) {
|
||||
foreach ($this->targets->repository($t)->trash() as $id => $info) {
|
||||
$rows[] = [$t->getName(), $id, (string)($info['time'] ?? ''), $info['label'], date('Y-m-d H:i', $info['forgottenAt']), $info['by'],
|
||||
date('Y-m-d H:i', $info['forgottenAt'] + $delay)];
|
||||
$time = $info['time'] ?? null;
|
||||
$made = is_string($time) ? strtotime($time) : false;
|
||||
$rows[] = [$t->getName(), $id, $made !== false ? Format::time($made, $zone) : '', $info['label'], Format::time($info['forgottenAt'], $zone), $info['by'],
|
||||
Format::time($info['forgottenAt'] + $delay, $zone)];
|
||||
}
|
||||
}
|
||||
if ($rows === []) {
|
||||
@@ -44,6 +50,7 @@ final class Trash extends Command {
|
||||
return 0;
|
||||
}
|
||||
(new Table($output))->setHeaders(['location', 'snapshot', 'made', 'label', 'removed', 'by', 'deleted after'])->setRows($rows)->render();
|
||||
$output->writeln("Times in $zone. The retention policy keeps the newest snapshots and the last of each recent day, week and month; it moves the others here, and they are deleted permanently after " . $this->keys->deleteDelayDays() . ' days.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,4 +15,14 @@ final class Format {
|
||||
$time = gmdate('H:i:s', $seconds % 86400);
|
||||
return $days > 0 ? $days . 'd ' . $time : $time;
|
||||
}
|
||||
|
||||
/**
|
||||
* "2026-10-29 04:30" in $zone. Nextcloud runs PHP in UTC; occ shows times in the zone of the
|
||||
* schedule, the administrator's, as the web interface does.
|
||||
*
|
||||
* @param non-empty-string $zone
|
||||
*/
|
||||
public static function time(int $timestamp, string $zone): string {
|
||||
return (new \DateTimeImmutable('@' . $timestamp))->setTimezone(new \DateTimeZone($zone))->format('Y-m-d H:i');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,7 +214,8 @@ final class Repository {
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a snapshot (or only paths starting with $prefix) into a local directory.
|
||||
* Restore a snapshot (or only paths starting with $prefix) into a local directory. A $prefix
|
||||
* "dir/file/" also restores the file "dir/file" itself.
|
||||
*
|
||||
* @param bool $stripPrefix write "$target/<path without $prefix>" instead of
|
||||
* "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
|
||||
@@ -253,7 +254,7 @@ final class Repository {
|
||||
$plan = new ReadAheadPlan(rtrim($target, '/') . '/.ngb-restore-spool-' . bin2hex(random_bytes(4)));
|
||||
try {
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||
if (!self::underPrefix($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
foreach ($entry['b'] as $id) {
|
||||
@@ -286,11 +287,15 @@ final class Repository {
|
||||
|
||||
private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan): int {
|
||||
$count = 0;
|
||||
$cleaned = [];
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||
if (!self::underPrefix($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
$relPath = ($stripPrefix && $prefix !== '') ? substr($entry['p'], strlen(rtrim($prefix, '/')) + 1) : $entry['p'];
|
||||
$relPath = $entry['p'];
|
||||
if ($stripPrefix && $prefix !== '') {
|
||||
$relPath = $entry['p'] === rtrim($prefix, '/') ? basename($entry['p']) : substr($entry['p'], strlen($prefix));
|
||||
}
|
||||
$dest = rtrim($target, '/') . '/' . $relPath;
|
||||
if (!is_dir(dirname($dest))) {
|
||||
mkdir(dirname($dest), 0700, true);
|
||||
@@ -301,6 +306,10 @@ final class Repository {
|
||||
if ($base === false || $dir === false || ($dir !== $base && !str_starts_with($dir, $base . '/')) || is_link($dest)) {
|
||||
throw new RepositoryException('Refusing to write outside the restore directory: ' . $entry['p']);
|
||||
}
|
||||
if (!isset($cleaned[$dir])) {
|
||||
self::removeStaleTemporaries($dir);
|
||||
$cleaned[$dir] = true;
|
||||
}
|
||||
// Write next to the destination and rename after all blobs verified: at most one file
|
||||
// is ever present twice.
|
||||
$tmp = $dest . '.ngb-restore-' . bin2hex(random_bytes(6));
|
||||
@@ -331,6 +340,30 @@ final class Repository {
|
||||
return $count;
|
||||
}
|
||||
|
||||
/** $prefix is "" (everything) or ends with "/"; the path without that "/" is a file to restore too. */
|
||||
private static function underPrefix(string $path, string $prefix): bool {
|
||||
return $prefix === '' || str_starts_with($path, $prefix) || $path === rtrim($prefix, '/');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove what a restore that was stopped left in a directory this restore writes to:
|
||||
* "<name>.ngb-restore-<hex>" from here, ".<name>.ngb-restore-<hex>" from a restore through
|
||||
* Nextcloud. A files:scan would otherwise make them show up in Files.
|
||||
*/
|
||||
public static function removeStaleTemporaries(string $dir): void {
|
||||
$names = scandir($dir);
|
||||
foreach ($names === false ? [] : $names as $name) {
|
||||
$path = $dir . '/' . $name;
|
||||
if (self::isRestoreTemporary($name) && is_file($path) && !is_link($path)) {
|
||||
@unlink($path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static function isRestoreTemporary(string $name): bool {
|
||||
return preg_match('/.\.ngb-restore-[0-9a-f]{6}(?:[0-9a-f]{6})?$/', $name) === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start writing a generated stream (e.g. a table dump) as blobs. Call flushPacks() when the
|
||||
* run is done, so the last pack is uploaded before anything references it.
|
||||
|
||||
@@ -34,6 +34,8 @@ use Psr\Log\LoggerInterface;
|
||||
*/
|
||||
final class BackupService {
|
||||
public const KIND_FULL = 'full';
|
||||
/** Characters of a snapshot label (the column holds 255) */
|
||||
public const LABEL_MAX = 100;
|
||||
|
||||
/** @var \WeakMap<Repository, string> the location settings each kept repository was opened with */
|
||||
private \WeakMap $openedWith;
|
||||
@@ -57,6 +59,10 @@ final class BackupService {
|
||||
}
|
||||
|
||||
public function start(Target $target, string $label = ''): Run {
|
||||
$label = trim($label);
|
||||
if (mb_strlen($label) > self::LABEL_MAX) {
|
||||
throw new \InvalidArgumentException('A label has at most ' . self::LABEL_MAX . ' characters');
|
||||
}
|
||||
if (!$this->keys->everConfirmed()) {
|
||||
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
||||
}
|
||||
|
||||
@@ -33,6 +33,9 @@ final class RestoreService {
|
||||
public const MODE_MERGE = 'merge';
|
||||
public const MODE_REPLACE = 'replace';
|
||||
|
||||
/** @var array<string, true> folders of the current restore already cleared of stale temporaries */
|
||||
private array $cleaned = [];
|
||||
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private IRootFolder $root,
|
||||
@@ -133,6 +136,7 @@ final class RestoreService {
|
||||
$dest = $this->ensureFolder($userFolder, $parentRel);
|
||||
$names = $single ? [basename($rel) => $entries['']] : $entries;
|
||||
|
||||
$this->cleaned = [];
|
||||
$restored = 0;
|
||||
foreach ($names as $relPath => $entry) {
|
||||
$this->writeFile($repo, $dest, (string)$relPath, $entry);
|
||||
@@ -178,6 +182,10 @@ final class RestoreService {
|
||||
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {
|
||||
$dir = dirname($relPath);
|
||||
$folder = $dir === '.' ? $dest : $this->ensureFolder($dest, $dir);
|
||||
if (!isset($this->cleaned[$folder->getPath()])) {
|
||||
$this->removeStaleTemporaries($folder);
|
||||
$this->cleaned[$folder->getPath()] = true;
|
||||
}
|
||||
$name = basename($relPath);
|
||||
// Stage under a temporary name; only rename into place after every blob was verified.
|
||||
$tmpName = '.' . $name . '.ngb-restore-' . bin2hex(random_bytes(3));
|
||||
@@ -223,6 +231,19 @@ final class RestoreService {
|
||||
}
|
||||
}
|
||||
|
||||
/** See Repository::removeStaleTemporaries(): what a restore that was stopped left in this folder. */
|
||||
private function removeStaleTemporaries(Folder $folder): void {
|
||||
foreach ($folder->getDirectoryListing() as $node) {
|
||||
if ($node instanceof File && Repository::isRestoreTemporary($node->getName())) {
|
||||
try {
|
||||
$node->delete();
|
||||
} catch (\Throwable) {
|
||||
// in the way of nothing: the new temporary gets a name of its own
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function trashExtras(Folder $folder, string $prefix, array $keep): int {
|
||||
$count = 0;
|
||||
foreach ($folder->getDirectoryListing() as $node) {
|
||||
|
||||
Reference in New Issue
Block a user