dillard
b14b16c66c
- Record folders without files in the meta record of a snapshot and restore them (#27) - Errors writing to the restore directory stop the restore, also with --salvage - Resume: check the folder before skipping, clean temporaries in folders that are complete - Clean only read-ahead folders of our own; refresh the lease on skipped files - Drop tables a failed database restore created; keep --salvage from calling a restored database fresh - Retry a pack that fails to open on the storage (SMB) - Without cron, running backups take turns from the open page - Bin: read in parts with a counter and progress bar, show read errors, refresh when a run ends
1434 lines
56 KiB
PHP
1434 lines
56 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
namespace OCA\NgBackup\Repository;
|
|
|
|
use OCA\NgBackup\Backend\BackendException;
|
|
use OCA\NgBackup\Backend\IBackend;
|
|
use OCA\NgBackup\Backend\IReadsAhead;
|
|
use OCA\NgBackup\Crypto\KeyRing;
|
|
use OCA\NgBackup\Crypto\StreamCipher;
|
|
|
|
/**
|
|
* Content-addressed, encrypted, deduplicating repository.
|
|
*
|
|
* Layout on the backend:
|
|
* config plaintext JSON: format version, repository id, wrapped master key
|
|
* packs/xx/<packId> encrypted blobs + encrypted pack header
|
|
* index/<packId> encrypted list of the blobs in that pack
|
|
* snapshots/<snapId> encrypted JSON lines: one meta line, then one line per file
|
|
*
|
|
* Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in
|
|
* memory at a time. A blob that already exists (same keyed hash) is not stored again.
|
|
*/
|
|
final class Repository {
|
|
public const FORMAT = 1;
|
|
public const BLOB_SIZE = 4 * 1048576;
|
|
private const FLAG_DEFLATE = 1;
|
|
/** Blobs from this size on get a quick compression test before the full compression. */
|
|
private const DEFLATE_PROBE_FROM = 256 * 1024;
|
|
/** Bytes of each of the three slices (start, middle, end) in that test. */
|
|
private const DEFLATE_PROBE_SLICE = 32 * 1024;
|
|
/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
|
|
private const VERIFY_READ_ATTEMPTS = 3;
|
|
/** Largest read that loadBlobs() makes of blobs that follow each other in a pack. */
|
|
private const MERGED_READ_MAX = 16 * 1048576;
|
|
|
|
private StreamCipher $cipher;
|
|
private BlobIndex $index;
|
|
private ?PackWriter $runPacks = null;
|
|
private ?int $maxBytes = null;
|
|
private Catalog $catalog;
|
|
private PackReader $packReader;
|
|
/** @var array<string, array<string, mixed>> meta record of the snapshot read last */
|
|
private array $lastMeta = [];
|
|
/** verify() lists the pack folders from this many packs on (see presentPacks()); tests lower it. */
|
|
public int $listPacksAbove = 256;
|
|
|
|
private function __construct(
|
|
private IBackend $backend,
|
|
private KeyRing $keys,
|
|
private string $repositoryId,
|
|
) {
|
|
$this->cipher = new StreamCipher($keys);
|
|
$this->packReader = new PackReader($backend);
|
|
$this->backend = new PackReaderGuard($backend, $this->packReader);
|
|
$this->index = new BlobIndex($this->backend, $this->cipher);
|
|
}
|
|
|
|
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
|
$keys = KeyRing::generate();
|
|
return self::initWithKey($backend, $keys, $keys->wrap($passphrase));
|
|
}
|
|
|
|
/**
|
|
* Create a repository with an existing master key. $wrappedKey is the passphrase-wrapped
|
|
* copy (KeyRing::wrap) stored in the repository config, so the repository can be opened
|
|
* with the passphrase from the recovery kit even when the server is gone.
|
|
*/
|
|
public static function initWithKey(IBackend $backend, KeyRing $keys, array $wrappedKey, ?ICatalogAnchor $anchor = null): self {
|
|
if ($backend->exists('config')) {
|
|
throw new RepositoryException('A repository already exists at this location');
|
|
}
|
|
// No config, but backup data: the config is missing (or could not be seen just now, a
|
|
// connection that failed). A new repository here would write a config over the one that
|
|
// opens this data; refuse instead.
|
|
foreach (['index', 'snapshots', 'trash/snapshots', 'catalog'] as $dir) {
|
|
if ($backend->listFiles($dir) !== []) {
|
|
throw new RepositoryException("This location holds backup data ($dir/) but no repository config could be read: not creating a new repository over it. Check the connection and the folder; a missing config can be restored from the recovery kit.");
|
|
}
|
|
}
|
|
$id = bin2hex(random_bytes(16));
|
|
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
|
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
|
|
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
|
$repo = new self($backend, $keys, $id);
|
|
$repo->loadCatalog($anchor);
|
|
return $repo;
|
|
}
|
|
|
|
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
|
/** @param ?callable(int, int): void $progress index files downloaded so far, and in all (see BlobIndex::load()) */
|
|
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null, ?callable $progress = null): self {
|
|
$config = self::readConfig($backend);
|
|
try {
|
|
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
|
} catch (\OCA\NgBackup\Crypto\CryptoException) {
|
|
$ok = false;
|
|
}
|
|
if (!$ok) {
|
|
throw new RepositoryException('This repository was created with a different key');
|
|
}
|
|
$repo = new self($backend, $keys, $config['id']);
|
|
if ($cache !== null) {
|
|
$repo->index->setCache($cache, $config['id']);
|
|
}
|
|
$repo->index->load(false, $progress);
|
|
$repo->loadCatalog($anchor);
|
|
return $repo;
|
|
}
|
|
|
|
private function loadCatalog(?ICatalogAnchor $anchor): void {
|
|
$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
|
|
$this->catalog->load(
|
|
fn () => array_map('basename', $this->backend->listFiles('snapshots')),
|
|
fn () => array_map('basename', $this->backend->listFiles('trash/info')),
|
|
fn () => $this->backend->list('users'));
|
|
}
|
|
|
|
/** Refuse uploads that would make the stored data larger than this (null = no limit). */
|
|
public function setMaxBytes(?int $maxBytes): void {
|
|
$this->maxBytes = $maxBytes;
|
|
}
|
|
|
|
public function storedBytes(): int {
|
|
return $this->index->storedBytes();
|
|
}
|
|
|
|
/** Current catalog generation (for status and tests). */
|
|
public function catalogGeneration(): int {
|
|
return $this->catalog->gen;
|
|
}
|
|
|
|
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
|
public function indexDownloads(): int {
|
|
return $this->index->downloaded;
|
|
}
|
|
|
|
private static function readConfig(IBackend $backend): array {
|
|
try {
|
|
$fh = $backend->get('config');
|
|
} catch (BackendException) {
|
|
throw new RepositoryException('No repository at this location');
|
|
}
|
|
$config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR);
|
|
fclose($fh);
|
|
if (($config['format'] ?? 0) !== self::FORMAT) {
|
|
throw new RepositoryException('Unsupported repository format');
|
|
}
|
|
return $config;
|
|
}
|
|
|
|
/** Replace the passphrase-wrapped key slots in this repository's config (after a slot change). */
|
|
public function updateWrappedKey(array $wrappedKey): void {
|
|
$config = self::readConfig($this->backend);
|
|
$config['key'] = $wrappedKey;
|
|
self::putString($this->backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
|
}
|
|
|
|
public function id(): string {
|
|
return $this->repositoryId;
|
|
}
|
|
|
|
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
|
$config = self::readConfig($backend);
|
|
$repo = new self($backend, KeyRing::unwrapAny($config['key'], $passphrase), $config['id']);
|
|
$repo->index->load();
|
|
$repo->loadCatalog(null); // disaster recovery on a new server: no anchor yet, trust the newest valid chain
|
|
return $repo;
|
|
}
|
|
|
|
/**
|
|
* Back up a local directory tree. Files whose size and mtime match the parent snapshot are not
|
|
* read again; their blob list is reused.
|
|
*
|
|
* @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}
|
|
*/
|
|
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
|
$source = rtrim($source, '/');
|
|
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
|
$packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
|
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
|
|
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$snapshotId = bin2hex(random_bytes(16));
|
|
fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n");
|
|
|
|
foreach ($this->walk($source) as $relative => $info) {
|
|
$stats['files']++;
|
|
$prev = $previous[$relative] ?? null;
|
|
if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) {
|
|
$blobs = $prev['b'];
|
|
$stats['reused']++;
|
|
} else {
|
|
$blobs = $this->readStable($info->getPathname(), $packs, $stats);
|
|
$stats['read']++;
|
|
}
|
|
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
|
|
}
|
|
$packs->flush();
|
|
|
|
// The snapshot is written last: it only references blobs whose packs are already stored.
|
|
rewind($tree);
|
|
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
|
|
rewind($encrypted);
|
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
|
fclose($tree);
|
|
$this->catalog->snapshots[$snapshotId] = true;
|
|
$this->catalog->write();
|
|
|
|
return ['snapshot' => $snapshotId] + $stats;
|
|
}
|
|
|
|
/**
|
|
* Restore a snapshot (or only paths starting with $prefix) into a local directory. A $prefix
|
|
* "dir/file/" also restores the file "dir/file" itself.
|
|
*
|
|
* @param bool $stripPrefix write "$target/<path without $prefix>" instead of
|
|
* "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
|
|
* "config/" directly into the real data/config directory, not a subdirectory
|
|
* named after the prefix). Ignored when $prefix is '' (nothing to strip).
|
|
* @param array<string, string>|null $skipped salvage: an array to collect the files that could
|
|
* not be read or decrypted (path => reason), which are then left out instead of
|
|
* stopping the restore; null (default) stops at the first such file
|
|
* @param bool $keepUnchanged leave a file that is already in place with the size and
|
|
* modification time of the snapshot, as a backup takes such a file as unchanged:
|
|
* a restore that was stopped and runs again goes on where it was, without reading
|
|
* what it already restored from the location again
|
|
*/
|
|
public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null, bool $keepUnchanged = false): int {
|
|
if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
|
|
throw new RepositoryException('Invalid path prefix');
|
|
}
|
|
$plan = $this->readAhead($snapshotId, $prefix, $target, $stripPrefix, $keepUnchanged);
|
|
try {
|
|
return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan, $skipped, $keepUnchanged);
|
|
} finally {
|
|
$plan?->close();
|
|
if ($plan !== null) {
|
|
@rmdir($plan->spool);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start reading the packs of a restore ahead, when the location can (IReadsAhead): a first
|
|
* pass over the snapshot lists the ranges of packs it needs, in order. The spool is a
|
|
* folder in the target, which has room for the restore anyway.
|
|
*/
|
|
private function readAhead(string $snapshotId, string $prefix, string $target, bool $stripPrefix, bool $keepUnchanged): ?ReadAheadPlan {
|
|
if (!$this->backend instanceof IReadsAhead || !$this->backend->canReadAhead()) {
|
|
return null;
|
|
}
|
|
// what a restore that was stopped left behind: its files are unlinked once opened, but one
|
|
// that was being written when the process was killed stays (8 MiB, DR test 2026-11-05)
|
|
$stale = glob(rtrim($target, '/') . '/.ngb-restore-spool-*', GLOB_ONLYDIR);
|
|
foreach ($stale === false ? [] : $stale as $dir) {
|
|
if (is_link($dir) || preg_match('/^\.ngb-restore-spool-[0-9a-f]{8}$/', basename($dir)) !== 1) {
|
|
continue; // not a spool of ours, whatever it is called
|
|
}
|
|
$files = glob($dir . '/*');
|
|
foreach ($files === false ? [] : $files as $file) {
|
|
if (is_file($file) && !is_link($file)) {
|
|
@unlink($file);
|
|
}
|
|
}
|
|
@rmdir($dir);
|
|
}
|
|
$plan = new ReadAheadPlan(rtrim($target, '/') . '/.ngb-restore-spool-' . bin2hex(random_bytes(4)));
|
|
try {
|
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
|
if (!self::underPrefix($entry['p'], $prefix)) {
|
|
continue;
|
|
}
|
|
if ($keepUnchanged && self::inPlace(self::destination($target, $entry['p'], $prefix, $stripPrefix), $entry)) {
|
|
continue;
|
|
}
|
|
foreach ($entry['b'] as $id) {
|
|
try {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
} catch (RepositoryException) {
|
|
continue; // the restore reports it when it gets there
|
|
}
|
|
$plan->add('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
|
}
|
|
}
|
|
} catch (\Throwable) {
|
|
return null; // reading ahead is only faster: the restore itself reads the snapshot again and reports what is wrong
|
|
}
|
|
if (count($plan->ranges()) < 2 || (!is_dir($plan->spool) && !@mkdir($plan->spool, 0700, true))) {
|
|
return null;
|
|
}
|
|
try {
|
|
$streams = $this->backend->readAhead($plan->ranges(), $plan->spool);
|
|
} catch (\Throwable) {
|
|
$streams = null; // the usual reads report what is wrong
|
|
}
|
|
if ($streams === null) {
|
|
@rmdir($plan->spool);
|
|
return null;
|
|
}
|
|
$plan->readFrom($streams);
|
|
return $plan;
|
|
}
|
|
|
|
/** @param array<string, string>|null $skipped see restore() */
|
|
private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan, ?array &$skipped, bool $keepUnchanged): int {
|
|
$count = 0;
|
|
$cleaned = [];
|
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
|
if (!self::underPrefix($entry['p'], $prefix)) {
|
|
continue;
|
|
}
|
|
if ($heartbeat !== null) {
|
|
$heartbeat(); // also for a long row of files that are skipped or have no data
|
|
}
|
|
$dest = self::destination($target, $entry['p'], $prefix, $stripPrefix);
|
|
$dir = self::directoryInTarget($target, dirname($dest), $entry['p']);
|
|
if (is_link($dest)) {
|
|
throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $entry['p']);
|
|
}
|
|
if (!isset($cleaned[$dir])) {
|
|
self::removeStaleTemporaries($dir);
|
|
$cleaned[$dir] = true;
|
|
}
|
|
if ($keepUnchanged && self::inPlace($dest, $entry)) {
|
|
$count++;
|
|
continue;
|
|
}
|
|
// Write next to the destination and rename after all blobs verified: at most one file
|
|
// is ever present twice.
|
|
$tmp = $dest . '.ngb-restore-' . bin2hex(random_bytes(6));
|
|
$out = @fopen($tmp, 'xb'); // fails if anything (including a symlink) already exists there
|
|
if ($out === false) {
|
|
throw new RestoreTargetException('Cannot create a temporary file for ' . $entry['p'] . ' (disk full, or no permission?)');
|
|
}
|
|
try {
|
|
foreach ($entry['b'] as $id) {
|
|
self::writeAll($out, $this->restoreBlob($id, $plan), $entry['p']);
|
|
if ($heartbeat !== null) {
|
|
$heartbeat();
|
|
}
|
|
}
|
|
if (!@fclose($out)) {
|
|
throw new RestoreTargetException('Cannot write ' . $entry['p'] . ' to the restore directory (disk full?)');
|
|
}
|
|
clearstatcache(true, $tmp);
|
|
if (filesize($tmp) !== $entry['s']) {
|
|
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
|
}
|
|
} catch (RepositoryException|BackendException|\OCA\NgBackup\Crypto\CryptoException $e) {
|
|
// What the location gave back for this file is missing or damaged. Writing to the
|
|
// target (below) is not part of this: a full disk must still stop the restore.
|
|
if (is_resource($out)) {
|
|
fclose($out);
|
|
}
|
|
@unlink($tmp);
|
|
if ($skipped === null) {
|
|
throw $e;
|
|
}
|
|
$skipped[$entry['p']] = $e->getMessage();
|
|
continue;
|
|
} catch (\Throwable $e) {
|
|
if (is_resource($out)) {
|
|
@fclose($out);
|
|
}
|
|
@unlink($tmp);
|
|
throw $e;
|
|
}
|
|
if (!rename($tmp, $dest)) {
|
|
@unlink($tmp);
|
|
throw new RestoreTargetException('Cannot put the restored file in place: ' . $entry['p']);
|
|
}
|
|
if (!touch($dest, $entry['m'])) {
|
|
throw new RestoreTargetException('Cannot set the modification time of ' . $entry['p']);
|
|
}
|
|
$count++;
|
|
}
|
|
// Last: a file can only be restored into a folder that exists, and the meta record was
|
|
// read along with the files, so this asks nothing more of the location.
|
|
foreach ($this->emptyDirs($snapshotId) as [$path, $mtime]) {
|
|
if (self::underPrefix($path, $prefix)) {
|
|
self::restoreEmptyDir($target, self::destination($target, $path, $prefix, $stripPrefix), $path, $mtime);
|
|
}
|
|
}
|
|
return $count;
|
|
}
|
|
|
|
/**
|
|
* The folders without anything in them that a snapshot recorded (none before 1.0.3).
|
|
*
|
|
* @return list<array{0: string, 1: int}> path, modification time
|
|
*/
|
|
public function emptyDirs(string $snapshotId): array {
|
|
$meta = $this->lastMeta[$snapshotId] ?? $this->snapshotMeta($snapshotId);
|
|
$dirs = $meta['emptyDirs'] ?? [];
|
|
foreach (is_array($dirs) ? $dirs : [null] as $dir) {
|
|
if (!is_array($dir) || !is_string($dir[0] ?? null) || !self::isSafePath($dir[0]) || !is_int($dir[1] ?? null)) {
|
|
throw new RepositoryException('Invalid empty folder in snapshot ' . $snapshotId);
|
|
}
|
|
}
|
|
/** @var list<array{0: string, 1: int}> */
|
|
return $dirs;
|
|
}
|
|
|
|
private static function restoreEmptyDir(string $target, string $dest, string $path, int $mtime): void {
|
|
if (is_link($dest)) {
|
|
throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $path);
|
|
}
|
|
$existed = is_dir($dest);
|
|
self::directoryInTarget($target, $dest, $path);
|
|
if (!$existed && !@touch($dest, $mtime)) {
|
|
throw new RestoreTargetException('Cannot set the modification time of ' . $path);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Make sure a folder exists below the target, without following a link out of it: what exists
|
|
* of the path is checked before anything is created.
|
|
*
|
|
* @return string the folder's real path
|
|
*/
|
|
private static function directoryInTarget(string $target, string $dir, string $path): string {
|
|
if (!is_dir($target) && !@mkdir($target, 0700, true) && !is_dir($target)) {
|
|
throw new RestoreTargetException('Cannot create the restore directory ' . $target);
|
|
}
|
|
$base = realpath($target);
|
|
$existing = $dir;
|
|
while (!file_exists($existing) && !is_link($existing) && dirname($existing) !== $existing) {
|
|
$existing = dirname($existing);
|
|
}
|
|
$real = realpath($existing);
|
|
if ($base === false || $real === false || ($real !== $base && !str_starts_with($real, $base . '/'))) {
|
|
throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $path);
|
|
}
|
|
if ($existing === $dir) {
|
|
if (!is_dir($real)) {
|
|
throw new RestoreTargetException('Cannot create the folder for ' . $path . ': a file is in the way');
|
|
}
|
|
return $real;
|
|
}
|
|
if (!@mkdir($dir, 0700, true) && !is_dir($dir)) {
|
|
throw new RestoreTargetException('Cannot create the folder for ' . $path . ' (disk full, no permission, or a file in the way?)');
|
|
}
|
|
return (string)realpath($dir);
|
|
}
|
|
|
|
/**
|
|
* Write all of $data, or stop: a short write leaves a file that looks damaged only by its size.
|
|
*
|
|
* @param resource $out
|
|
*/
|
|
public static function writeAll($out, string $data, string $path): void {
|
|
if (@fwrite($out, $data) !== strlen($data)) {
|
|
throw new RestoreTargetException('Cannot write ' . $path . ' to the restore directory (disk full?)');
|
|
}
|
|
}
|
|
|
|
/** Where restore() writes the snapshot path $path. */
|
|
private static function destination(string $target, string $path, string $prefix, bool $stripPrefix): string {
|
|
$relPath = $path;
|
|
if ($stripPrefix && $prefix !== '') {
|
|
$relPath = $path === rtrim($prefix, '/') ? basename($path) : substr($path, strlen($prefix));
|
|
}
|
|
return rtrim($target, '/') . '/' . $relPath;
|
|
}
|
|
|
|
/**
|
|
* A regular file (no link) at $dest with the size and modification time of the snapshot
|
|
* entry: what a backup takes as unchanged, and what a finished restore of it leaves.
|
|
*
|
|
* @param array{s: int, m: int, ...} $entry
|
|
*/
|
|
private static function inPlace(string $dest, array $entry): bool {
|
|
clearstatcache(true, $dest);
|
|
return !is_link($dest) && is_file($dest) && filesize($dest) === $entry['s'] && filemtime($dest) === $entry['m'];
|
|
}
|
|
|
|
/** $prefix is "" (everything) or ends with "/"; the path without that "/" is a file to restore too. */
|
|
private static function underPrefix(string $path, string $prefix): bool {
|
|
return $prefix === '' || str_starts_with($path, $prefix) || $path === rtrim($prefix, '/');
|
|
}
|
|
|
|
/**
|
|
* Remove what a restore that was stopped left in a directory this restore writes to:
|
|
* "<name>.ngb-restore-<hex>" from here, ".<name>.ngb-restore-<hex>" from a restore through
|
|
* Nextcloud. A files:scan would otherwise make them show up in Files.
|
|
*/
|
|
public static function removeStaleTemporaries(string $dir): void {
|
|
$names = scandir($dir);
|
|
foreach ($names === false ? [] : $names as $name) {
|
|
$path = $dir . '/' . $name;
|
|
if (self::isRestoreTemporary($name) && is_file($path) && !is_link($path)) {
|
|
@unlink($path);
|
|
}
|
|
}
|
|
}
|
|
|
|
public static function isRestoreTemporary(string $name): bool {
|
|
return preg_match('/.\.ngb-restore-[0-9a-f]{6}(?:[0-9a-f]{6})?$/', $name) === 1;
|
|
}
|
|
|
|
/**
|
|
* Start writing a generated stream (e.g. a table dump) as blobs. Call flushPacks() when the
|
|
* run is done, so the last pack is uploaded before anything references it.
|
|
*/
|
|
public function blobWriter(array &$stats): BlobWriter {
|
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
|
$packs = $this->runPacks;
|
|
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
|
return $this->storeBlob($data, $packs, $stats);
|
|
});
|
|
}
|
|
|
|
/** See BlobIndex::holdCache(): no database writes until releaseIndexCache(). */
|
|
public function holdIndexCache(): void {
|
|
$this->index->holdCache();
|
|
}
|
|
|
|
public function releaseIndexCache(): void {
|
|
$this->index->releaseCache();
|
|
}
|
|
|
|
public function flushPacks(): void {
|
|
$this->runPacks?->flush();
|
|
}
|
|
|
|
/**
|
|
* Read a blob stream back as lines (lines may span blob boundaries).
|
|
*
|
|
* @param list<string> $blobIds
|
|
* @return \Generator<string>
|
|
*/
|
|
public function readLines(array $blobIds): \Generator {
|
|
$carry = '';
|
|
foreach ($this->loadBlobs($blobIds) as $blob) {
|
|
$data = $carry . $blob;
|
|
$lines = explode("\n", $data);
|
|
$carry = array_pop($lines);
|
|
yield from $lines;
|
|
}
|
|
if ($carry !== '') {
|
|
yield $carry;
|
|
}
|
|
}
|
|
|
|
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
|
public function storeData(string $data, array &$stats): string {
|
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
|
return $this->storeBlob($data, $this->runPacks, $stats);
|
|
}
|
|
|
|
/**
|
|
* Write a snapshot from a meta record and the tree lines (JSON, one per file). Must be called
|
|
* after flushPacks(), so the snapshot only references stored blobs.
|
|
*
|
|
* @param iterable<string> $lines
|
|
*/
|
|
public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
|
$this->putSnapshot($snapshotId, $meta, $lines);
|
|
$this->recordSnapshot($snapshotId);
|
|
}
|
|
|
|
/**
|
|
* Store a snapshot object without touching the catalog. Use recordSnapshot() afterwards, from a
|
|
* repository opened inside the catalog lock, so the catalog write builds on the latest generation.
|
|
*
|
|
* @param iterable<string> $lines
|
|
*/
|
|
public function putSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
|
|
foreach ($lines as $line) {
|
|
if ($line !== '') {
|
|
fwrite($tree, $line . "\n");
|
|
}
|
|
}
|
|
rewind($tree);
|
|
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
|
|
fclose($tree);
|
|
rewind($encrypted);
|
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
|
}
|
|
|
|
public function recordSnapshot(string $snapshotId): void {
|
|
$this->catalog->snapshots[$snapshotId] = true;
|
|
$this->catalog->write();
|
|
}
|
|
|
|
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
|
public function snapshotEntries(string $snapshotId): array {
|
|
return $this->loadEntries($snapshotId);
|
|
}
|
|
|
|
/**
|
|
* @param list<string> $blobIds
|
|
* @return \Generator<string> verified blob contents, in order
|
|
*/
|
|
public function readBlobs(array $blobIds): \Generator {
|
|
foreach ($blobIds as $id) {
|
|
yield $this->loadBlob($id);
|
|
}
|
|
}
|
|
|
|
/** Relative path of non-empty segments, without ".", "..", NUL, backslash or control characters. */
|
|
public static function isSafePath(string $path): bool {
|
|
if ($path === '' || str_starts_with($path, '/') || preg_match('/[\x00-\x1f\\\\]/', $path)) {
|
|
return false;
|
|
}
|
|
foreach (explode('/', $path) as $segment) {
|
|
if ($segment === '' || $segment === '.' || $segment === '..') {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/** Small encrypted objects (manifests). */
|
|
public function putObject(string $path, string $data): void {
|
|
self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
|
|
}
|
|
|
|
public function getObject(string $path): string {
|
|
$fh = $this->backend->get($path);
|
|
$data = $this->cipher->decryptString((string)stream_get_contents($fh), 'object:' . $path);
|
|
fclose($fh);
|
|
return $data;
|
|
}
|
|
|
|
/**
|
|
* Move a snapshot (and its database dump) to the trash on the location. Its data stays in
|
|
* use until purgeTrash() removes it after the deletion delay, so a malicious or mistaken
|
|
* cleanup can be undone with untrash() within that time.
|
|
*/
|
|
public function forget(string $snapshotId, string $by = ''): void {
|
|
$meta = $this->snapshotMeta($snapshotId);
|
|
$dbInTrash = null;
|
|
if (isset($meta['db']) && is_string($meta['db']) && $this->backend->exists($meta['db'])) {
|
|
// Its encryption is bound to its path (object:db/...), which changes on a move, so it
|
|
// must be decrypted and re-encrypted under the new path, not moved as raw bytes.
|
|
$dbInTrash = 'trash/' . $meta['db'];
|
|
$this->putObject($dbInTrash, $this->getObject($meta['db']));
|
|
}
|
|
// Copy first, switch the catalog, delete the originals last: a crash at any point leaves
|
|
// either the old state or the new one plus unreferenced leftovers, never a catalog entry
|
|
// pointing at a missing object. The snapshot object is bound to its id (not its path),
|
|
// so it can be copied as raw bytes.
|
|
$this->copyRaw('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
|
|
$this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null,
|
|
'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR));
|
|
unset($this->catalog->snapshots[$snapshotId]);
|
|
$this->catalog->trash[$snapshotId] = true;
|
|
$this->catalog->write();
|
|
if ($dbInTrash !== null) {
|
|
$this->backend->delete($meta['db']);
|
|
}
|
|
$this->backend->delete('snapshots/' . $snapshotId);
|
|
}
|
|
|
|
/** Undo forget(): move a snapshot back from the trash. */
|
|
public function untrash(string $snapshotId): void {
|
|
$info = $this->trashInfo($snapshotId);
|
|
$restoreDb = $info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash']);
|
|
if ($restoreDb) {
|
|
$this->putObject($info['db'], $this->getObject($info['dbInTrash']));
|
|
}
|
|
// Same order as forget(): copy, switch the catalog, then delete the trash copies.
|
|
$this->copyRaw('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
|
|
$this->catalog->snapshots[$snapshotId] = true;
|
|
unset($this->catalog->trash[$snapshotId]);
|
|
$this->catalog->write();
|
|
if ($restoreDb) {
|
|
$this->backend->delete($info['dbInTrash']);
|
|
}
|
|
$this->backend->delete('trash/snapshots/' . $snapshotId);
|
|
$this->backend->delete('trash/info/' . $snapshotId);
|
|
}
|
|
|
|
/** Copy an object as raw bytes, buffered locally (remote read streams cannot be uploaded directly everywhere). */
|
|
private function copyRaw(string $from, string $to): void {
|
|
$in = $this->backend->get($from);
|
|
$buffer = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
if ($buffer === false) {
|
|
fclose($in);
|
|
throw new RepositoryException("Cannot buffer $from");
|
|
}
|
|
try {
|
|
if (stream_copy_to_stream($in, $buffer) === false) {
|
|
throw new RepositoryException("Cannot read $from");
|
|
}
|
|
} finally {
|
|
fclose($in);
|
|
}
|
|
rewind($buffer);
|
|
$this->backend->put($to, $buffer);
|
|
}
|
|
|
|
/** Record a user_migration export's manifest in the catalog (rollback-protected like a snapshot). */
|
|
public function recordUserExport(string $manifestPath): void {
|
|
$this->catalog->userExports[$manifestPath] = true;
|
|
$this->catalog->write();
|
|
}
|
|
|
|
/**
|
|
* User export manifests according to the verified catalog, newest first. A manifest the
|
|
* catalog lists but the location no longer has means it was deleted or hidden behind NG
|
|
* Backup's back, same as a missing snapshot (see listSnapshots()).
|
|
*
|
|
* @return list<string>
|
|
*/
|
|
public function listUserExports(?string $uid = null): array {
|
|
$prefix = $uid !== null ? 'users/' . $uid . '/' : 'users/';
|
|
$out = [];
|
|
foreach (array_keys($this->catalog->userExports) as $path) {
|
|
if (!str_starts_with($path, $prefix)) {
|
|
continue;
|
|
}
|
|
if (!$this->backend->exists($path)) {
|
|
throw new RepositoryException("User export $path is missing on the location (deleted or hidden outside NG Backup)");
|
|
}
|
|
$out[] = $path;
|
|
}
|
|
sort($out, SORT_STRING);
|
|
return array_reverse($out);
|
|
}
|
|
|
|
/** Whether $manifestPath is a user export the verified catalog actually recorded. */
|
|
public function isRecordedUserExport(string $manifestPath): bool {
|
|
return isset($this->catalog->userExports[$manifestPath]);
|
|
}
|
|
|
|
/**
|
|
* The recorded export $ref names: a manifest path (users/<uid>/<exportId>) as is, or a bare
|
|
* user id (also "users/<uid>") when that user has exactly one export here. Anything else
|
|
* throws, with the exports to choose from in the message.
|
|
*/
|
|
public function resolveUserExport(string $ref): string {
|
|
if ($this->isRecordedUserExport($ref)) {
|
|
return $ref;
|
|
}
|
|
$rest = trim($ref, '/');
|
|
if (str_starts_with($rest, 'users/')) {
|
|
$rest = substr($rest, strlen('users/'));
|
|
}
|
|
$uid = explode('/', $rest)[0];
|
|
$exports = $uid !== '' ? $this->listUserExports($uid) : [];
|
|
if (!str_contains($rest, '/') && count($exports) === 1) {
|
|
return $exports[0];
|
|
}
|
|
if ($exports === []) {
|
|
throw new RepositoryException("$ref is not a recorded user export for this location, and $uid has no exports here. Expected users/<uid>/<exportId>; occ backup:user:list shows the exports");
|
|
}
|
|
$which = str_contains($rest, '/') ? "$ref is not a recorded user export for this location" : "$uid has " . count($exports) . ' exports here';
|
|
throw new RepositoryException("$which; pass one of these (newest first):\n " . implode("\n ", $exports));
|
|
}
|
|
|
|
/** Permanently delete a user export. Its blobs are freed by the next prune() like any other. */
|
|
public function forgetUserExport(string $manifestPath): void {
|
|
if (!isset($this->catalog->userExports[$manifestPath])) {
|
|
throw new RepositoryException("$manifestPath is not a recorded user export");
|
|
}
|
|
unset($this->catalog->userExports[$manifestPath]);
|
|
$this->catalog->write();
|
|
$this->backend->delete($manifestPath);
|
|
}
|
|
|
|
/** @return array<string, array{forgottenAt:int, by:string, db:?string, dbInTrash:?string, time:?string, label:string}> */
|
|
public function trash(): array {
|
|
$out = [];
|
|
foreach (array_keys($this->catalog->trash) as $id) {
|
|
try {
|
|
$out[$id] = $this->trashInfo($id);
|
|
} catch (\Throwable) {
|
|
throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
|
|
}
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
/** Permanently delete trash entries older than $delaySeconds. @return list<string> purged snapshot ids */
|
|
public function purgeTrash(int $delaySeconds): array {
|
|
$doomed = [];
|
|
foreach ($this->trash() as $id => $info) {
|
|
if (time() - $info['forgottenAt'] < $delaySeconds) {
|
|
continue;
|
|
}
|
|
$doomed[$id] = $info;
|
|
unset($this->catalog->trash[$id]);
|
|
}
|
|
if ($doomed === []) {
|
|
return [];
|
|
}
|
|
$this->catalog->write();
|
|
foreach ($doomed as $id => $info) {
|
|
if ($info['dbInTrash'] !== null) {
|
|
$this->backend->delete($info['dbInTrash']);
|
|
}
|
|
$this->backend->delete('trash/snapshots/' . $id);
|
|
$this->backend->delete('trash/info/' . $id);
|
|
}
|
|
return array_keys($doomed);
|
|
}
|
|
|
|
/**
|
|
* Part of trash(), for a page that shows how far it is: each entry is a read of its own on the
|
|
* location. Reads on from $offset until $deadline (microtime(true)), at least one entry.
|
|
*
|
|
* @return array{total: int, items: array<string, array<string, mixed>>}
|
|
*/
|
|
public function trashPage(int $offset, float $deadline): array {
|
|
$ids = array_map('strval', array_keys($this->catalog->trash));
|
|
sort($ids, SORT_STRING);
|
|
$items = [];
|
|
foreach (array_slice($ids, max(0, $offset)) as $id) {
|
|
try {
|
|
$items[$id] = $this->trashInfo($id);
|
|
} catch (\Throwable) {
|
|
throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
|
|
}
|
|
if (microtime(true) >= $deadline) {
|
|
break;
|
|
}
|
|
}
|
|
return ['total' => count($ids), 'items' => $items];
|
|
}
|
|
|
|
private function trashInfo(string $snapshotId): array {
|
|
return json_decode($this->getObject('trash/info/' . $snapshotId), true, 512, JSON_THROW_ON_ERROR);
|
|
}
|
|
|
|
/**
|
|
* Delete packs no snapshot or user export uses any more, and repack packs where more than
|
|
* $repackThreshold of the bytes are unused. Must not run while a backup to this repository
|
|
* is in progress (its blobs are not referenced by a snapshot yet); callers hold the lock.
|
|
*
|
|
* @return array{packs:int, deleted:int, repacked:int, freedBytes:int, keptBlobs:int, unusedBlobs:int}
|
|
*/
|
|
public function prune(bool $dryRun = false, float $repackThreshold = 0.5, array $ignoreSnapshots = []): array {
|
|
// Deleting is decided on the authenticated index files on the location, never on the local cache.
|
|
$this->index->load(true);
|
|
$used = [];
|
|
$ignore = array_flip($ignoreSnapshots); // e.g. snapshots a dry run would forget first
|
|
foreach ($this->listSnapshots() as $sid) {
|
|
if (isset($ignore[$sid])) {
|
|
continue;
|
|
}
|
|
foreach ($this->streamEntries($sid) as $e) {
|
|
foreach ($e['b'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
$meta = $this->snapshotMeta($sid);
|
|
if (isset($meta['db']) && $this->backend->exists($meta['db'])) {
|
|
foreach (json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
|
foreach ($t['blobs'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Snapshots in the trash keep their data until purgeTrash() removed them.
|
|
foreach ($this->trash() as $sid => $info) {
|
|
foreach ($this->streamEntries($sid, 'trash/snapshots/') as $e) {
|
|
foreach ($e['b'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
if ($info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash'])) {
|
|
foreach (json_decode($this->getObject($info['dbInTrash']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
|
foreach ($t['blobs'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// listUserExports() throws when a recorded export's manifest is missing: then stop, since
|
|
// freeing its blobs would turn a tampering or storage alarm into irreversible deletion.
|
|
$exports = array_unique(array_merge($this->listUserExports(), $this->backend->list('users')));
|
|
foreach ($exports as $path) {
|
|
foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
|
|
foreach ($e['b'] ?? [] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
$stats = ['packs' => 0, 'deleted' => 0, 'repacked' => 0, 'freedBytes' => 0, 'keptBlobs' => count($used), 'unusedBlobs' => 0];
|
|
$packs = $this->index->packs();
|
|
$stats['packs'] = count($packs);
|
|
$writer = null;
|
|
$toRemove = [];
|
|
foreach ($packs as $packId => $blobs) {
|
|
$total = 0;
|
|
$unused = 0;
|
|
foreach ($blobs as $b) {
|
|
$total += $b['length'];
|
|
if (!isset($used[$b['id']])) {
|
|
$unused += $b['length'];
|
|
$stats['unusedBlobs']++;
|
|
}
|
|
}
|
|
if ($unused === 0 || ($unused < $total && $unused / $total <= $repackThreshold)) {
|
|
continue;
|
|
}
|
|
$stats['freedBytes'] += $unused;
|
|
if ($unused === $total) {
|
|
$stats['deleted']++;
|
|
} else {
|
|
$stats['repacked']++;
|
|
if (!$dryRun) {
|
|
// No size limit here: repacking is how space is freed.
|
|
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
|
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
|
foreach ($blobs as $b) {
|
|
if (isset($used[$b['id']])) {
|
|
[, , , $raw, $flags] = $this->index->get($b['id']);
|
|
$writer->addRaw($b['id'], $this->backend->getRange($path, $b['offset'], $b['length']), $raw, $flags);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
$toRemove[] = $packId;
|
|
}
|
|
if ($dryRun) {
|
|
return $stats;
|
|
}
|
|
// New packs and their index entries are durable before any old pack is removed.
|
|
$writer?->flush();
|
|
foreach ($toRemove as $packId) {
|
|
$this->index->removePack($packId);
|
|
$this->backend->delete('packs/' . substr($packId, 0, 2) . '/' . $packId);
|
|
}
|
|
return $stats;
|
|
}
|
|
|
|
/**
|
|
* Snapshots according to the verified catalog. A snapshot the catalog lists but the location
|
|
* no longer has means it was deleted or hidden behind NG Backup's back.
|
|
*
|
|
* @return list<string>
|
|
*/
|
|
public function listSnapshots(): array {
|
|
$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
|
|
$listed = array_keys($this->catalog->snapshots);
|
|
foreach ($listed as $id) {
|
|
if (!isset($present[$id])) {
|
|
throw new RepositoryException("Snapshot $id is missing on the location (deleted or hidden outside NG Backup)");
|
|
}
|
|
}
|
|
sort($listed);
|
|
return $listed;
|
|
}
|
|
|
|
public function blobCount(): int {
|
|
return $this->index->count();
|
|
}
|
|
|
|
/**
|
|
* Read a file into blobs and make sure it did not change while reading (size, mtime, ctime
|
|
* before and after). A file that keeps changing is an error rather than a torn backup.
|
|
*
|
|
* @return list<string>
|
|
*/
|
|
private function readStable(string $path, PackWriter $packs, array &$stats): array {
|
|
for ($attempt = 1; $attempt <= 3; $attempt++) {
|
|
$before = self::fileVersion($path);
|
|
$blobs = [];
|
|
$fh = fopen($path, 'rb');
|
|
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
|
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
|
}
|
|
fclose($fh);
|
|
if (self::fileVersion($path) === $before) {
|
|
return $blobs;
|
|
}
|
|
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
|
}
|
|
throw new RepositoryException('File keeps changing while being read: ' . $path);
|
|
}
|
|
|
|
/** Cheap file identity: changes when the content may have changed. */
|
|
public static function fileVersion(string $path): string {
|
|
clearstatcache(true, $path);
|
|
$st = @stat($path);
|
|
return $st === false ? 'missing' : implode(':', [$st['size'], $st['mtime'], $st['ctime'], $st['ino']]);
|
|
}
|
|
|
|
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
|
|
$id = $this->keys->blobId($data);
|
|
if ($this->index->has($id)) {
|
|
$stats['dupBlobs'] = ($stats['dupBlobs'] ?? 0) + 1;
|
|
return $id;
|
|
}
|
|
$flags = 0;
|
|
$payload = $data;
|
|
$deflated = self::worthDeflating($data) ? gzdeflate($data, 6) : false;
|
|
if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
|
|
$payload = $deflated;
|
|
$flags |= self::FLAG_DEFLATE;
|
|
}
|
|
$stats['uploaded'] = ($stats['uploaded'] ?? 0) + $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
|
|
$stats['newBlobs'] = ($stats['newBlobs'] ?? 0) + 1;
|
|
return $id;
|
|
}
|
|
|
|
/**
|
|
* Photos, video and archives are compressed already: deflating them takes most of a backup's
|
|
* CPU time (about 24 MiB/s) only to be thrown away, as it saves less than the 5% that keeps
|
|
* it. A fast deflate of three small slices tells first; below 10% savings there, the full
|
|
* deflate is skipped.
|
|
*/
|
|
private static function worthDeflating(string $data): bool {
|
|
$n = strlen($data);
|
|
if ($n < self::DEFLATE_PROBE_FROM) {
|
|
return true;
|
|
}
|
|
$k = self::DEFLATE_PROBE_SLICE;
|
|
$probe = substr($data, 0, $k) . substr($data, intdiv($n - $k, 2), $k) . substr($data, $n - $k);
|
|
$deflated = gzdeflate($probe, 1);
|
|
return $deflated !== false && strlen($deflated) * 10 < strlen($probe) * 9;
|
|
}
|
|
|
|
/**
|
|
* Blobs spread over many packs (the tree segments of a run, one per step), through
|
|
* IBackend::getRanges(): on a remote location one at a time is a round trip each, which for
|
|
* a long run made writing its snapshot take a quarter of an hour. Blobs that follow each
|
|
* other in a pack are read as one range.
|
|
*
|
|
* @param list<string> $ids
|
|
* @return \Generator<int, string> verified blob contents, in order
|
|
*/
|
|
private function loadBlobs(array $ids): \Generator {
|
|
/** @var list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] of each read */
|
|
$ranges = [];
|
|
/** @var list<list<array{0:string, 1:int, 2:int}>> $parts per read: [blob id, offset in the read, length] */
|
|
$parts = [];
|
|
// the read being extended
|
|
$curPath = null;
|
|
$curOffset = 0;
|
|
$curLength = 0;
|
|
$curParts = [];
|
|
foreach ($ids as $id) {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
$path = 'packs/' . substr($pack, 0, 2) . '/' . $pack;
|
|
if ($curPath === $path && $curOffset + $curLength === $offset && $curLength + $length <= self::MERGED_READ_MAX) {
|
|
$curParts[] = [$id, $curLength, $length];
|
|
$curLength += $length;
|
|
continue;
|
|
}
|
|
if ($curPath !== null) {
|
|
$ranges[] = [$curPath, $curOffset, $curLength];
|
|
$parts[] = $curParts;
|
|
}
|
|
[$curPath, $curOffset, $curLength, $curParts] = [$path, $offset, $length, [[$id, 0, $length]]];
|
|
}
|
|
if ($curPath !== null) {
|
|
$ranges[] = [$curPath, $curOffset, $curLength];
|
|
$parts[] = $curParts;
|
|
}
|
|
$done = 0;
|
|
try {
|
|
foreach ($this->backend->getRanges($ranges) as $i => $data) {
|
|
foreach ($parts[$i] as [$id, $at, $length]) {
|
|
try {
|
|
$blob = $this->decodeBlob($id, substr($data, $at, $length));
|
|
} catch (\Exception) {
|
|
// a read that went wrong on the way (see loadBlob()): this blob on its own
|
|
$blob = $this->loadBlob($id);
|
|
// its pack stream is closed before the next range is read on the same connection
|
|
$this->packReader->close();
|
|
}
|
|
yield $blob;
|
|
$done++;
|
|
}
|
|
}
|
|
} catch (BackendException) {
|
|
// A pack moved, or the read failed for a while (see loadBlob()): the rest one blob at a
|
|
// time, each read once more after a failure.
|
|
foreach (array_slice($ids, $done) as $id) {
|
|
yield $this->loadBlob($id);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** A blob for restore(): read ahead when it could be, otherwise (or when that copy is bad) the usual way. */
|
|
private function restoreBlob(string $id, ?ReadAheadPlan $plan): string {
|
|
if ($plan !== null) {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
$encrypted = $plan->read('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
|
if ($encrypted !== null) {
|
|
try {
|
|
return $this->decodeBlob($id, $encrypted);
|
|
} catch (\Exception) {
|
|
// read again from the location below, which reports what is wrong
|
|
}
|
|
}
|
|
}
|
|
return $this->loadBlob($id);
|
|
}
|
|
|
|
private function loadBlob(string $id): string {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
try {
|
|
$encrypted = $this->packReader->read($pack, $offset, $length);
|
|
} catch (BackendException $e) {
|
|
// Another server's cleanup may have repacked this blob into a new pack since the index
|
|
// was loaded (a disaster recovery test while the original server still runs): add the
|
|
// new index files, and read the blob where it is now. Or the read failed for a while
|
|
// (a connection that died; the backend has replaced it by now): then read it once
|
|
// more where it was. A second failure is the real one.
|
|
try {
|
|
$this->index->refresh();
|
|
} catch (\Throwable) {
|
|
// the location could not be listed either: read again where the blob was
|
|
}
|
|
[$moved, $offset, $length] = $this->index->get($id);
|
|
try {
|
|
$encrypted = $this->packReader->read($moved, $offset, $length);
|
|
} catch (BackendException) {
|
|
throw $e;
|
|
}
|
|
$pack = $moved;
|
|
}
|
|
try {
|
|
return $this->decodeBlob($id, $encrypted);
|
|
} catch (\Exception $damaged) {
|
|
// Bytes that do not decrypt: damaged on the location, or a read that went wrong on the
|
|
// way (a stream that lost its place: an SMB restore stopped after 17 hours like this).
|
|
// Read exactly this blob once more, on a request of its own; only a second failure
|
|
// means the data itself is damaged.
|
|
$this->packReader->close();
|
|
try {
|
|
$again = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
|
} catch (\Exception) {
|
|
throw $damaged;
|
|
}
|
|
$data = $this->decodeBlob($id, $again);
|
|
if (function_exists('OCP\Log\logger')) {
|
|
\OCP\Log\logger('ng_backup')->warning('NG Backup: blob ' . $id . ' did not decrypt as first read from pack ' . $pack . ' at ' . $offset . '; read again, it is fine. First read: ' . self::howItDiffered($encrypted, $again));
|
|
}
|
|
return $data;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* What a read that went wrong looked like next to the right bytes, for the log: where it starts
|
|
* to differ, and whether the rest is zeros or the right bytes from another place (a stream that
|
|
* lost its place).
|
|
*/
|
|
private static function howItDiffered(string $wrong, string $right): string {
|
|
$at = strspn($wrong ^ $right, "\0");
|
|
$text = strlen($wrong) . ' bytes, differ from byte ' . $at . ' of ' . strlen($right);
|
|
$tail = substr($wrong, $at, 64);
|
|
if ($tail === '') {
|
|
return $text . ' (too short)';
|
|
}
|
|
if (trim($tail, "\0") === '') {
|
|
return $text . ' (zeros)';
|
|
}
|
|
$found = strpos($right, $tail);
|
|
return $text . ($found !== false ? ' (the right bytes from ' . ($found - $at) . ' bytes further on)' : ' (other bytes)');
|
|
}
|
|
|
|
/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
|
|
private function decodeBlob(string $id, string $encrypted): string {
|
|
[, , , $raw, $flags] = $this->index->get($id);
|
|
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
|
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
|
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
|
throw new RepositoryException('Blob verification failed: ' . $id);
|
|
}
|
|
return $data;
|
|
}
|
|
|
|
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
|
private function loadEntries(string $snapshotId): array {
|
|
$map = [];
|
|
foreach ($this->streamEntries($snapshotId) as $e) {
|
|
$map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']];
|
|
}
|
|
return $map;
|
|
}
|
|
|
|
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> entries of a snapshot, in stored order */
|
|
public function entries(string $snapshotId): \Generator {
|
|
return $this->streamEntries($snapshotId);
|
|
}
|
|
|
|
/**
|
|
* Checksum audit of a snapshot: every blob it references (files and the database dump, when
|
|
* the snapshot has one) must be in the index, freshly re-downloaded and re-authenticated from
|
|
* the location rather than trusted from the local cache (same reasoning as prune()), and its
|
|
* pack must still be present. With $deep, each blob is also downloaded and decrypted, which
|
|
* verifies its AEAD authentication tag and its content hash against the blob id (loadBlob()
|
|
* already does both on every real read -- this just does it for everything in the snapshot,
|
|
* deliberately, rather than relying on it to come up during a later restore). A blob
|
|
* referenced by more than one file is only checked once; a missing database manifest is a
|
|
* failure, not something to silently skip over.
|
|
*
|
|
* @param ?callable(array{phase:string, done:int, total:?int, blobs?:int, bytes?:int}):void $heartbeat
|
|
* called often during the check, with how far it is: phase "index" (index files
|
|
* downloaded), "packs" (pack folders listed), "files" and "database" (files of the
|
|
* snapshot checked); callers use it to keep a time-boxed lease alive and to report
|
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
*/
|
|
public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
|
|
$heartbeat ??= static function (array $progress): void {
|
|
};
|
|
$meta = $this->snapshotMeta($snapshotId);
|
|
$filesTotal = isset($meta['stats']['files']) ? (int)$meta['stats']['files'] : null;
|
|
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
|
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
|
});
|
|
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
|
$seen = [];
|
|
[$packSeen, $listedDirs] = $this->presentPacks($heartbeat);
|
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
|
$stats['filesChecked']++;
|
|
foreach ($entry['b'] as $id) {
|
|
if (isset($seen[$id])) {
|
|
continue;
|
|
}
|
|
$seen[$id] = true;
|
|
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
|
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
}
|
|
}
|
|
if (isset($meta['db']) && is_string($meta['db'])) {
|
|
if (!$this->backend->exists($meta['db'])) {
|
|
$stats['missing'][] = $meta['db'];
|
|
} else {
|
|
$dbManifest = json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
|
foreach ($dbManifest['tables'] as $info) {
|
|
foreach ($info['blobs'] as $id) {
|
|
if (isset($seen[$id])) {
|
|
continue;
|
|
}
|
|
$seen[$id] = true;
|
|
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
|
$heartbeat(['phase' => 'database', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return $stats;
|
|
}
|
|
|
|
/**
|
|
* Which packs are on the location, from one listing per pack folder instead of a request per
|
|
* pack. A folder that cannot be listed, or comes back empty (which a storage also reports when
|
|
* it could not look), is left out: its packs are then checked one by one. With no more packs
|
|
* than folders nothing is listed: a request per pack is fewer requests then, and on FTP every
|
|
* request takes a data connection, of which a server may have only a few.
|
|
*
|
|
* @return array{0: array<string, bool>, 1: array<string, true>} pack id => present, and the
|
|
* folders that were listed
|
|
*/
|
|
private function presentPacks(callable $heartbeat): array {
|
|
$present = [];
|
|
$listed = [];
|
|
if ($this->index->packCount() <= $this->listPacksAbove) {
|
|
return [$present, $listed];
|
|
}
|
|
for ($i = 0; $i < 256; $i++) {
|
|
$dir = sprintf('%02x', $i);
|
|
try {
|
|
$files = $this->backend->listFiles('packs/' . $dir);
|
|
} catch (BackendException) {
|
|
$files = [];
|
|
}
|
|
foreach ($files as $path) {
|
|
$present[basename($path)] = true;
|
|
}
|
|
if ($files !== []) {
|
|
$listed[$dir] = true;
|
|
}
|
|
$heartbeat(['phase' => 'packs', 'done' => $i + 1, 'total' => 256]);
|
|
}
|
|
return [$present, $listed];
|
|
}
|
|
|
|
/**
|
|
* Same checks as verify(), for a user export: every blob its manifest references must be in
|
|
* the freshly re-authenticated index with its pack present; $deep also decrypts each.
|
|
*
|
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
*/
|
|
public function verifyUserExport(string $manifestPath, bool $deep = false, ?callable $heartbeat = null): array {
|
|
if (!$this->isRecordedUserExport($manifestPath)) {
|
|
throw new RepositoryException("$manifestPath is not a recorded user export for this location");
|
|
}
|
|
$heartbeat ??= static function (array $progress): void {
|
|
};
|
|
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
|
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
|
});
|
|
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
|
$seen = [];
|
|
$packSeen = [];
|
|
$manifest = json_decode($this->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
|
foreach ($manifest['entries'] as $entry) {
|
|
if (($entry['t'] ?? '') !== 'f') {
|
|
continue;
|
|
}
|
|
$stats['filesChecked']++;
|
|
foreach ($entry['b'] ?? [] as $id) {
|
|
if (isset($seen[$id])) {
|
|
continue;
|
|
}
|
|
$seen[$id] = true;
|
|
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
|
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => null,
|
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
}
|
|
}
|
|
return $stats;
|
|
}
|
|
|
|
/**
|
|
* @param array<string, bool> $packSeen pack id => present; filled in as packs are checked
|
|
* @param array<string, true> $listedDirs pack folders whose listing is in $packSeen: a pack
|
|
* not in it is not there
|
|
*/
|
|
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen, array $listedDirs = []): void {
|
|
if (!$this->index->has($id)) {
|
|
$stats['missing'][] = $id;
|
|
return;
|
|
}
|
|
[$pack] = $this->index->get($id);
|
|
$packExists = $packSeen[$pack] ??= isset($listedDirs[substr($pack, 0, 2)])
|
|
? false
|
|
: $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
|
|
if (!$packExists) {
|
|
$stats['missing'][] = $id;
|
|
return;
|
|
}
|
|
$stats['blobsChecked']++;
|
|
if (!$deep) {
|
|
return;
|
|
}
|
|
// Reading and checking are kept apart: a read that fails is usually a network or
|
|
// storage error, so it is retried and reported as unreadable (a pack that stays
|
|
// unreadable may still be truncated). Only a blob that was read but does not
|
|
// decrypt or match its id counts as failed: that is proof of damage.
|
|
[, $offset, $length] = $this->index->get($id);
|
|
$encrypted = null;
|
|
for ($attempt = 1; $attempt <= self::VERIFY_READ_ATTEMPTS && $encrypted === null; $attempt++) {
|
|
try {
|
|
$encrypted = $this->packReader->read($pack, $offset, $length);
|
|
} catch (\Exception $e) { // storage adapters throw their own exceptions; \Error (a bug) is not retried
|
|
$stats['readError'] = $e->getMessage();
|
|
}
|
|
}
|
|
if ($encrypted === null) {
|
|
$stats['unreadable'][] = $id;
|
|
return;
|
|
}
|
|
try {
|
|
$stats['bytesChecked'] += strlen($this->decodeBlob($id, $encrypted));
|
|
} catch (\Exception) { // damage; an \Error is a bug and must not be reported as damage
|
|
$stats['failed'][] = $id;
|
|
}
|
|
}
|
|
|
|
/** The meta record (first line) of a snapshot. */
|
|
public function snapshotMeta(string $snapshotId): array {
|
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
|
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
|
fclose($fh);
|
|
rewind($plain);
|
|
$meta = json_decode((string)fgets($plain), true, 512, JSON_THROW_ON_ERROR);
|
|
fclose($plain);
|
|
return $meta;
|
|
}
|
|
|
|
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
|
private function streamEntries(string $snapshotId, string $dir = 'snapshots/'): \Generator {
|
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$fh = $this->backend->get($dir . $snapshotId);
|
|
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
|
fclose($fh);
|
|
rewind($plain);
|
|
$meta = json_decode((string)fgets($plain), true);
|
|
if (is_array($meta)) {
|
|
$this->lastMeta = [$snapshotId => $meta]; // a snapshot never changes
|
|
}
|
|
while (($line = fgets($plain)) !== false) {
|
|
$entry = json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
|
if (!is_array($entry) || !is_string($entry['p'] ?? null) || !self::isSafePath($entry['p'])
|
|
|| !is_int($entry['s'] ?? null) || !is_array($entry['b'] ?? null)) {
|
|
throw new RepositoryException('Invalid entry in snapshot ' . $snapshotId);
|
|
}
|
|
yield $entry;
|
|
}
|
|
fclose($plain);
|
|
}
|
|
|
|
/** @return \Generator<string, \SplFileInfo> relative path => file, in a stable order */
|
|
private function walk(string $root): \Generator {
|
|
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
|
|
$files = [];
|
|
foreach ($it as $file) {
|
|
if ($file->isFile()) {
|
|
$files[substr($file->getPathname(), strlen($root) + 1)] = $file;
|
|
}
|
|
}
|
|
ksort($files, SORT_STRING);
|
|
yield from $files;
|
|
}
|
|
|
|
private static function putString(IBackend $backend, string $path, string $data): void {
|
|
$s = fopen('php://memory', 'w+b');
|
|
fwrite($s, $data);
|
|
rewind($s);
|
|
$backend->put($path, $s);
|
|
}
|
|
|
|
private static function readFull($fh, int $length): string {
|
|
$buffer = '';
|
|
while (strlen($buffer) < $length && !feof($fh)) {
|
|
$buffer .= (string)fread($fh, $length - strlen($buffer));
|
|
}
|
|
return $buffer;
|
|
}
|
|
}
|